feat(backup): centralize backup eligibility

go
codex 2 weeks ago
parent 20a9984c02
commit 5aba7b149d

@ -0,0 +1,24 @@
<?php
namespace App\Services\Cloud;
use App\Models\CloudInstance;
/** Production adapter for the Agent backup-readiness port. */
final class CloudAgentBackupReadinessProbe implements CloudBackupReadinessProbe
{
private const TIMEOUT_SECONDS = 5.0;
public function inspect(CloudInstance $instance, array $actor): array
{
return CloudAgentSettingsService::clientForHost(
(int) $instance->host_id,
self::TIMEOUT_SECONDS
)->request(
'GET',
'/v1/vms/' . rawurlencode((string) $instance->uuid)
. '/backup-readiness',
$actor
);
}
}

@ -0,0 +1,100 @@
<?php
namespace App\Services\Cloud;
use App\Models\CloudInstance;
/**
* One deep module for every transient condition that may defer a backup.
*
* The Agent observation is advisory and never authorizes a mutation. The
* create path retains its locked panel checks and Agent-side final preflight.
*/
final class CloudBackupEligibility
{
private const AGENT_REASONS = [
CloudBackupEligibilityDecision::BACKUP_RESCUE_ACTIVE,
CloudBackupEligibilityDecision::BACKUP_DISK_TRANSITION_ACTIVE,
CloudBackupEligibilityDecision::DOMAIN_JOB_ACTIVE,
];
public function __construct(
private ?CloudBackupReadinessProbe $probe = null
) {
}
public function assess(
CloudInstance $instance,
array $actor
): CloudBackupEligibilityDecision {
try {
if (CloudInstanceMigrationService::hasOpenMigration(
(int) $instance->id
)) {
return self::blocked(
CloudBackupEligibilityDecision::MIGRATION_ACTIVE
);
}
if (CloudJobService::blockingQueryForInstance(
(int) $instance->id
)->exists()) {
return self::blocked(
CloudBackupEligibilityDecision::INSTANCE_MUTATION_ACTIVE
);
}
$payload = ($this->probe
?? new CloudAgentBackupReadinessProbe())->inspect(
$instance,
$actor
);
} catch (\Throwable $error) {
return self::blocked(
CloudBackupEligibilityDecision::READINESS_UNAVAILABLE
);
}
$eligible = $payload['eligible'] ?? null;
$retryable = $payload['retryable'] ?? null;
$reason = $payload['reason'] ?? '';
if (!is_bool($eligible) || !is_bool($retryable)
|| !is_string($reason)
|| ($eligible && ($retryable || $reason !== ''))
|| (!$eligible && (!$retryable
|| !in_array($reason, self::AGENT_REASONS, true)))) {
return self::blocked(
CloudBackupEligibilityDecision::READINESS_UNAVAILABLE
);
}
return $eligible
? new CloudBackupEligibilityDecision(
CloudBackupEligibilityDecision::READY,
''
)
: self::blocked($reason);
}
private static function blocked(
string $reason
): CloudBackupEligibilityDecision {
$messages = [
CloudBackupEligibilityDecision::MIGRATION_ACTIVE =>
'实例正在维护迁移,当前不能创建备份',
CloudBackupEligibilityDecision::INSTANCE_MUTATION_ACTIVE =>
'实例正在执行其他操作,当前不能创建备份',
CloudBackupEligibilityDecision::BACKUP_RESCUE_ACTIVE =>
'实例处于救援模式,当前不能创建备份',
CloudBackupEligibilityDecision::BACKUP_DISK_TRANSITION_ACTIVE =>
'实例正在切换系统盘或清理旧系统盘,当前不能创建备份',
CloudBackupEligibilityDecision::DOMAIN_JOB_ACTIVE =>
'实例磁盘任务仍在执行,当前不能创建备份',
CloudBackupEligibilityDecision::READINESS_UNAVAILABLE =>
'宿主机备份状态暂不可核对,请稍后重试',
];
return new CloudBackupEligibilityDecision(
$reason,
$messages[$reason] ?? $messages[
CloudBackupEligibilityDecision::READINESS_UNAVAILABLE
]
);
}
}

@ -0,0 +1,43 @@
<?php
namespace App\Services\Cloud;
/** Immutable admission decision shared by manual and automatic backups. */
final readonly class CloudBackupEligibilityDecision
{
public const READY = 'ready';
public const MIGRATION_ACTIVE = 'migration_active';
public const INSTANCE_MUTATION_ACTIVE = 'instance_mutation_active';
public const BACKUP_RESCUE_ACTIVE = 'backup_rescue_active';
public const BACKUP_DISK_TRANSITION_ACTIVE =
'backup_disk_transition_active';
public const DOMAIN_JOB_ACTIVE = 'domain_job_active';
public const READINESS_UNAVAILABLE = 'backup_readiness_unavailable';
private const REASONS = [
self::READY,
self::MIGRATION_ACTIVE,
self::INSTANCE_MUTATION_ACTIVE,
self::BACKUP_RESCUE_ACTIVE,
self::BACKUP_DISK_TRANSITION_ACTIVE,
self::DOMAIN_JOB_ACTIVE,
self::READINESS_UNAVAILABLE,
];
public function __construct(
public string $reasonCode,
public string $message
) {
if (!in_array($reasonCode, self::REASONS, true)
|| (($reasonCode === self::READY) !== ($message === ''))) {
throw new \InvalidArgumentException(
'Invalid backup eligibility decision'
);
}
}
public function allowed(): bool
{
return $this->reasonCode === self::READY;
}
}

@ -0,0 +1,11 @@
<?php
namespace App\Services\Cloud;
use App\Models\CloudInstance;
/** Port for the authenticated, read-only host backup-readiness observation. */
interface CloudBackupReadinessProbe
{
public function inspect(CloudInstance $instance, array $actor): array;
}

@ -650,6 +650,22 @@ final class CloudBackupService
return self::finishCreatePreparation($reserved);
}
// Do not allocate a backup row, storage artifact, multipart upload or
// Agent job while a known transient host state makes backup unsafe.
// Idempotent replays above still resolve their existing durable intent.
if ($automatic === null) {
$eligibility = (new CloudBackupEligibility())->assess(
$instance,
$actor
);
if (!$eligibility->allowed()) {
$status = $eligibility->reasonCode
=== CloudBackupEligibilityDecision::READINESS_UNAVAILABLE
? 503 : 409;
throw new CloudError($eligibility->message, $status);
}
}
try {
$backup = $instance->getConnection()->transaction(function () use (
$instance, $actor, $name, $mode, $fingerprint, $automatic,

@ -0,0 +1,205 @@
<?php
require dirname(__DIR__) . '/vendor/autoload.php';
use App\Models\CloudInstance;
use App\Services\Cloud\CloudBackupEligibility;
use App\Services\Cloud\CloudBackupEligibilityDecision;
use App\Services\Cloud\CloudBackupReadinessProbe;
use App\Services\Cloud\CloudInstanceMigrationService;
use App\Services\Cloud\CloudJobService;
use Illuminate\Database\Capsule\Manager as Capsule;
use Illuminate\Database\Schema\Blueprint;
final class BackupReadinessProbeFake implements CloudBackupReadinessProbe
{
public function __construct(
private array $payload = [],
private ?Throwable $failure = null
) {
}
public function inspect(CloudInstance $instance, array $actor): array
{
if ($this->failure !== null) {
throw $this->failure;
}
return $this->payload;
}
}
$assert = static function (bool $condition, string $message): void {
if (!$condition) {
throw new RuntimeException($message);
}
};
$db = new Capsule();
$db->addConnection([
'driver' => 'sqlite',
'database' => ':memory:',
], 'default');
$db->setAsGlobal();
$db->bootEloquent();
$schema = $db->schema('default');
$schema->create('cloud_instances', static function (Blueprint $t): void {
$t->increments('id');
$t->unsignedBigInteger('user_id');
$t->unsignedBigInteger('host_id');
$t->string('uuid');
});
$schema->create('cloud_instance_migrations',
static function (Blueprint $t): void {
$t->increments('id');
$t->unsignedBigInteger('instance_id');
$t->unsignedBigInteger('vpc_port_id')->nullable();
$t->string('state');
});
$schema->create('cloud_peer_migration_artifacts',
static function (Blueprint $t): void {
$t->increments('id');
});
$schema->create('cloud_jobs', static function (Blueprint $t): void {
$t->increments('id');
$t->string('operation_id')->unique();
$t->string('request_idempotency_key')->unique();
$t->string('request_fingerprint');
$t->unsignedBigInteger('parent_job_id')->nullable();
$t->unsignedBigInteger('order_id')->nullable();
$t->unsignedBigInteger('invoice_id')->nullable();
$t->unsignedBigInteger('instance_id')->nullable();
$t->unsignedBigInteger('snapshot_id')->nullable();
$t->unsignedBigInteger('backup_id')->nullable();
$t->unsignedBigInteger('backup_restore_id')->nullable();
$t->unsignedBigInteger('backup_recovery_id')->nullable();
$t->unsignedBigInteger('user_id');
$t->unsignedBigInteger('host_id')->nullable();
$t->string('kind');
$t->string('phase');
$t->string('outcome')->nullable();
$t->string('status');
$t->string('execution_mode')->nullable();
$t->text('sealed_payload');
$t->unsignedInteger('attempts')->default(1);
$t->unsignedInteger('dispatch_attempts')->nullable();
$t->unsignedBigInteger('dispatch_deadline_at')->nullable();
$t->unsignedBigInteger('dispatch_started_at')->nullable();
$t->unsignedBigInteger('agent_accepted_at')->nullable();
$t->unsignedBigInteger('receipt_deadline_at')->nullable();
$t->unsignedBigInteger('dispatched_at')->nullable();
$t->unsignedBigInteger('terminal_at')->nullable();
$t->boolean('effect_may_have_applied')->nullable();
$t->boolean('safe_to_resubmit')->nullable();
$t->string('effect_resolution');
$t->string('error_code')->nullable();
$t->boolean('finalization_incident')->default(false);
$t->string('finalization_error_code')->nullable();
$t->string('completion_event_id')->nullable();
$t->text('result_json')->nullable();
});
$instance = new CloudInstance();
$instance->id = 41;
$instance->user_id = 7;
$instance->host_id = 3;
$instance->uuid = '11111111-2222-4333-8444-555555555555';
$instance->save();
$actor = [
'user_id' => 7,
'is_admin' => true,
'is_super_admin' => true,
'can_manage' => true,
'session_id' => 'backup-test',
];
$assert(!CloudInstanceMigrationService::hasOpenMigration(41),
'Fresh instance unexpectedly has an open migration.');
$assert(!CloudJobService::blockingQueryForInstance(41)->exists(),
'Fresh instance unexpectedly has a blocking job.');
$assess = static function (array $payload) use ($instance, $actor) {
return (new CloudBackupEligibility(
new BackupReadinessProbeFake($payload)
))->assess($instance, $actor);
};
$ready = $assess([
'eligible' => true,
'retryable' => false,
]);
$assert($ready->allowed()
&& $ready->reasonCode === CloudBackupEligibilityDecision::READY,
'Ready Agent evidence was not admitted: ' . $ready->reasonCode);
foreach ([
CloudBackupEligibilityDecision::BACKUP_RESCUE_ACTIVE,
CloudBackupEligibilityDecision::BACKUP_DISK_TRANSITION_ACTIVE,
CloudBackupEligibilityDecision::DOMAIN_JOB_ACTIVE,
] as $reason) {
$decision = $assess([
'eligible' => false,
'retryable' => true,
'reason' => $reason,
]);
$assert(!$decision->allowed() && $decision->reasonCode === $reason
&& $decision->message !== '',
'Transient Agent blocker was not represented by a clear decision.');
}
$invalid = $assess([
'eligible' => false,
'retryable' => false,
'reason' => 'host-path:/secret',
]);
$assert($invalid->reasonCode
=== CloudBackupEligibilityDecision::READINESS_UNAVAILABLE,
'Malformed Agent readiness was accepted.');
$unavailable = (new CloudBackupEligibility(
new BackupReadinessProbeFake([], new RuntimeException('offline'))
))->assess($instance, $actor);
$assert($unavailable->reasonCode
=== CloudBackupEligibilityDecision::READINESS_UNAVAILABLE,
'Agent failure was not fail-closed.');
$db->table('cloud_instance_migrations')->insert([
'instance_id' => 41,
'state' => 'attention_required',
]);
$migration = $assess(['eligible' => true, 'retryable' => false]);
$assert($migration->reasonCode
=== CloudBackupEligibilityDecision::MIGRATION_ACTIVE,
'Open maintenance migration did not win admission ordering.');
$db->table('cloud_instance_migrations')->delete();
$db->table('cloud_jobs')->insert([
'operation_id' => str_repeat('a', 32),
'request_idempotency_key' => 'eligibility-running-job',
'request_fingerprint' => str_repeat('b', 64),
'instance_id' => 41,
'user_id' => 7,
'host_id' => 3,
'kind' => 'reinstall',
'phase' => 'running',
'status' => 'running',
'execution_mode' => 'agent',
'sealed_payload' => '{}',
'attempts' => 1,
'dispatch_attempts' => 1,
'dispatch_deadline_at' => time() + 300,
'effect_resolution' => 'not_applicable',
]);
$job = $assess(['eligible' => true, 'retryable' => false]);
$assert($job->reasonCode
=== CloudBackupEligibilityDecision::INSTANCE_MUTATION_ACTIVE,
'Blocking instance job did not prevent Agent admission.');
$source = (string) file_get_contents(
dirname(__DIR__) . '/app/Services/Cloud/CloudBackupService.php'
);
$assert(str_contains($source, '(new CloudBackupEligibility())->assess(')
&& strpos($source, '(new CloudBackupEligibility())->assess(')
< strpos($source, 'beginAgentPublication('),
'Manual backup does not run readiness before storage publication.');
echo "Cloud backup eligibility tests passed.\n";
Loading…
Cancel
Save