feat(backup): centralize backup eligibility
parent
20a9984c02
commit
5aba7b149d
@ -0,0 +1,24 @@
|
||||
<?php
|
||||
|
||||
namespace App\Services\Cloud;
|
||||
|
||||
use App\Models\CloudInstance;
|
||||
|
||||
/** Production adapter for the Agent backup-readiness port. */
|
||||
final class CloudAgentBackupReadinessProbe implements CloudBackupReadinessProbe
|
||||
{
|
||||
private const TIMEOUT_SECONDS = 5.0;
|
||||
|
||||
public function inspect(CloudInstance $instance, array $actor): array
|
||||
{
|
||||
return CloudAgentSettingsService::clientForHost(
|
||||
(int) $instance->host_id,
|
||||
self::TIMEOUT_SECONDS
|
||||
)->request(
|
||||
'GET',
|
||||
'/v1/vms/' . rawurlencode((string) $instance->uuid)
|
||||
. '/backup-readiness',
|
||||
$actor
|
||||
);
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,100 @@
|
||||
<?php
|
||||
|
||||
namespace App\Services\Cloud;
|
||||
|
||||
use App\Models\CloudInstance;
|
||||
|
||||
/**
|
||||
* One deep module for every transient condition that may defer a backup.
|
||||
*
|
||||
* The Agent observation is advisory and never authorizes a mutation. The
|
||||
* create path retains its locked panel checks and Agent-side final preflight.
|
||||
*/
|
||||
final class CloudBackupEligibility
|
||||
{
|
||||
private const AGENT_REASONS = [
|
||||
CloudBackupEligibilityDecision::BACKUP_RESCUE_ACTIVE,
|
||||
CloudBackupEligibilityDecision::BACKUP_DISK_TRANSITION_ACTIVE,
|
||||
CloudBackupEligibilityDecision::DOMAIN_JOB_ACTIVE,
|
||||
];
|
||||
|
||||
public function __construct(
|
||||
private ?CloudBackupReadinessProbe $probe = null
|
||||
) {
|
||||
}
|
||||
|
||||
public function assess(
|
||||
CloudInstance $instance,
|
||||
array $actor
|
||||
): CloudBackupEligibilityDecision {
|
||||
try {
|
||||
if (CloudInstanceMigrationService::hasOpenMigration(
|
||||
(int) $instance->id
|
||||
)) {
|
||||
return self::blocked(
|
||||
CloudBackupEligibilityDecision::MIGRATION_ACTIVE
|
||||
);
|
||||
}
|
||||
if (CloudJobService::blockingQueryForInstance(
|
||||
(int) $instance->id
|
||||
)->exists()) {
|
||||
return self::blocked(
|
||||
CloudBackupEligibilityDecision::INSTANCE_MUTATION_ACTIVE
|
||||
);
|
||||
}
|
||||
$payload = ($this->probe
|
||||
?? new CloudAgentBackupReadinessProbe())->inspect(
|
||||
$instance,
|
||||
$actor
|
||||
);
|
||||
} catch (\Throwable $error) {
|
||||
return self::blocked(
|
||||
CloudBackupEligibilityDecision::READINESS_UNAVAILABLE
|
||||
);
|
||||
}
|
||||
|
||||
$eligible = $payload['eligible'] ?? null;
|
||||
$retryable = $payload['retryable'] ?? null;
|
||||
$reason = $payload['reason'] ?? '';
|
||||
if (!is_bool($eligible) || !is_bool($retryable)
|
||||
|| !is_string($reason)
|
||||
|| ($eligible && ($retryable || $reason !== ''))
|
||||
|| (!$eligible && (!$retryable
|
||||
|| !in_array($reason, self::AGENT_REASONS, true)))) {
|
||||
return self::blocked(
|
||||
CloudBackupEligibilityDecision::READINESS_UNAVAILABLE
|
||||
);
|
||||
}
|
||||
return $eligible
|
||||
? new CloudBackupEligibilityDecision(
|
||||
CloudBackupEligibilityDecision::READY,
|
||||
''
|
||||
)
|
||||
: self::blocked($reason);
|
||||
}
|
||||
|
||||
private static function blocked(
|
||||
string $reason
|
||||
): CloudBackupEligibilityDecision {
|
||||
$messages = [
|
||||
CloudBackupEligibilityDecision::MIGRATION_ACTIVE =>
|
||||
'实例正在维护迁移,当前不能创建备份',
|
||||
CloudBackupEligibilityDecision::INSTANCE_MUTATION_ACTIVE =>
|
||||
'实例正在执行其他操作,当前不能创建备份',
|
||||
CloudBackupEligibilityDecision::BACKUP_RESCUE_ACTIVE =>
|
||||
'实例处于救援模式,当前不能创建备份',
|
||||
CloudBackupEligibilityDecision::BACKUP_DISK_TRANSITION_ACTIVE =>
|
||||
'实例正在切换系统盘或清理旧系统盘,当前不能创建备份',
|
||||
CloudBackupEligibilityDecision::DOMAIN_JOB_ACTIVE =>
|
||||
'实例磁盘任务仍在执行,当前不能创建备份',
|
||||
CloudBackupEligibilityDecision::READINESS_UNAVAILABLE =>
|
||||
'宿主机备份状态暂不可核对,请稍后重试',
|
||||
];
|
||||
return new CloudBackupEligibilityDecision(
|
||||
$reason,
|
||||
$messages[$reason] ?? $messages[
|
||||
CloudBackupEligibilityDecision::READINESS_UNAVAILABLE
|
||||
]
|
||||
);
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,43 @@
|
||||
<?php
|
||||
|
||||
namespace App\Services\Cloud;
|
||||
|
||||
/** Immutable admission decision shared by manual and automatic backups. */
|
||||
final readonly class CloudBackupEligibilityDecision
|
||||
{
|
||||
public const READY = 'ready';
|
||||
public const MIGRATION_ACTIVE = 'migration_active';
|
||||
public const INSTANCE_MUTATION_ACTIVE = 'instance_mutation_active';
|
||||
public const BACKUP_RESCUE_ACTIVE = 'backup_rescue_active';
|
||||
public const BACKUP_DISK_TRANSITION_ACTIVE =
|
||||
'backup_disk_transition_active';
|
||||
public const DOMAIN_JOB_ACTIVE = 'domain_job_active';
|
||||
public const READINESS_UNAVAILABLE = 'backup_readiness_unavailable';
|
||||
|
||||
private const REASONS = [
|
||||
self::READY,
|
||||
self::MIGRATION_ACTIVE,
|
||||
self::INSTANCE_MUTATION_ACTIVE,
|
||||
self::BACKUP_RESCUE_ACTIVE,
|
||||
self::BACKUP_DISK_TRANSITION_ACTIVE,
|
||||
self::DOMAIN_JOB_ACTIVE,
|
||||
self::READINESS_UNAVAILABLE,
|
||||
];
|
||||
|
||||
public function __construct(
|
||||
public string $reasonCode,
|
||||
public string $message
|
||||
) {
|
||||
if (!in_array($reasonCode, self::REASONS, true)
|
||||
|| (($reasonCode === self::READY) !== ($message === ''))) {
|
||||
throw new \InvalidArgumentException(
|
||||
'Invalid backup eligibility decision'
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
public function allowed(): bool
|
||||
{
|
||||
return $this->reasonCode === self::READY;
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,11 @@
|
||||
<?php
|
||||
|
||||
namespace App\Services\Cloud;
|
||||
|
||||
use App\Models\CloudInstance;
|
||||
|
||||
/** Port for the authenticated, read-only host backup-readiness observation. */
|
||||
interface CloudBackupReadinessProbe
|
||||
{
|
||||
public function inspect(CloudInstance $instance, array $actor): array;
|
||||
}
|
||||
@ -0,0 +1,205 @@
|
||||
<?php
|
||||
|
||||
require dirname(__DIR__) . '/vendor/autoload.php';
|
||||
|
||||
use App\Models\CloudInstance;
|
||||
use App\Services\Cloud\CloudBackupEligibility;
|
||||
use App\Services\Cloud\CloudBackupEligibilityDecision;
|
||||
use App\Services\Cloud\CloudBackupReadinessProbe;
|
||||
use App\Services\Cloud\CloudInstanceMigrationService;
|
||||
use App\Services\Cloud\CloudJobService;
|
||||
use Illuminate\Database\Capsule\Manager as Capsule;
|
||||
use Illuminate\Database\Schema\Blueprint;
|
||||
|
||||
final class BackupReadinessProbeFake implements CloudBackupReadinessProbe
|
||||
{
|
||||
public function __construct(
|
||||
private array $payload = [],
|
||||
private ?Throwable $failure = null
|
||||
) {
|
||||
}
|
||||
|
||||
public function inspect(CloudInstance $instance, array $actor): array
|
||||
{
|
||||
if ($this->failure !== null) {
|
||||
throw $this->failure;
|
||||
}
|
||||
return $this->payload;
|
||||
}
|
||||
}
|
||||
|
||||
$assert = static function (bool $condition, string $message): void {
|
||||
if (!$condition) {
|
||||
throw new RuntimeException($message);
|
||||
}
|
||||
};
|
||||
|
||||
$db = new Capsule();
|
||||
$db->addConnection([
|
||||
'driver' => 'sqlite',
|
||||
'database' => ':memory:',
|
||||
], 'default');
|
||||
$db->setAsGlobal();
|
||||
$db->bootEloquent();
|
||||
$schema = $db->schema('default');
|
||||
$schema->create('cloud_instances', static function (Blueprint $t): void {
|
||||
$t->increments('id');
|
||||
$t->unsignedBigInteger('user_id');
|
||||
$t->unsignedBigInteger('host_id');
|
||||
$t->string('uuid');
|
||||
});
|
||||
$schema->create('cloud_instance_migrations',
|
||||
static function (Blueprint $t): void {
|
||||
$t->increments('id');
|
||||
$t->unsignedBigInteger('instance_id');
|
||||
$t->unsignedBigInteger('vpc_port_id')->nullable();
|
||||
$t->string('state');
|
||||
});
|
||||
$schema->create('cloud_peer_migration_artifacts',
|
||||
static function (Blueprint $t): void {
|
||||
$t->increments('id');
|
||||
});
|
||||
$schema->create('cloud_jobs', static function (Blueprint $t): void {
|
||||
$t->increments('id');
|
||||
$t->string('operation_id')->unique();
|
||||
$t->string('request_idempotency_key')->unique();
|
||||
$t->string('request_fingerprint');
|
||||
$t->unsignedBigInteger('parent_job_id')->nullable();
|
||||
$t->unsignedBigInteger('order_id')->nullable();
|
||||
$t->unsignedBigInteger('invoice_id')->nullable();
|
||||
$t->unsignedBigInteger('instance_id')->nullable();
|
||||
$t->unsignedBigInteger('snapshot_id')->nullable();
|
||||
$t->unsignedBigInteger('backup_id')->nullable();
|
||||
$t->unsignedBigInteger('backup_restore_id')->nullable();
|
||||
$t->unsignedBigInteger('backup_recovery_id')->nullable();
|
||||
$t->unsignedBigInteger('user_id');
|
||||
$t->unsignedBigInteger('host_id')->nullable();
|
||||
$t->string('kind');
|
||||
$t->string('phase');
|
||||
$t->string('outcome')->nullable();
|
||||
$t->string('status');
|
||||
$t->string('execution_mode')->nullable();
|
||||
$t->text('sealed_payload');
|
||||
$t->unsignedInteger('attempts')->default(1);
|
||||
$t->unsignedInteger('dispatch_attempts')->nullable();
|
||||
$t->unsignedBigInteger('dispatch_deadline_at')->nullable();
|
||||
$t->unsignedBigInteger('dispatch_started_at')->nullable();
|
||||
$t->unsignedBigInteger('agent_accepted_at')->nullable();
|
||||
$t->unsignedBigInteger('receipt_deadline_at')->nullable();
|
||||
$t->unsignedBigInteger('dispatched_at')->nullable();
|
||||
$t->unsignedBigInteger('terminal_at')->nullable();
|
||||
$t->boolean('effect_may_have_applied')->nullable();
|
||||
$t->boolean('safe_to_resubmit')->nullable();
|
||||
$t->string('effect_resolution');
|
||||
$t->string('error_code')->nullable();
|
||||
$t->boolean('finalization_incident')->default(false);
|
||||
$t->string('finalization_error_code')->nullable();
|
||||
$t->string('completion_event_id')->nullable();
|
||||
$t->text('result_json')->nullable();
|
||||
});
|
||||
|
||||
$instance = new CloudInstance();
|
||||
$instance->id = 41;
|
||||
$instance->user_id = 7;
|
||||
$instance->host_id = 3;
|
||||
$instance->uuid = '11111111-2222-4333-8444-555555555555';
|
||||
$instance->save();
|
||||
$actor = [
|
||||
'user_id' => 7,
|
||||
'is_admin' => true,
|
||||
'is_super_admin' => true,
|
||||
'can_manage' => true,
|
||||
'session_id' => 'backup-test',
|
||||
];
|
||||
|
||||
$assert(!CloudInstanceMigrationService::hasOpenMigration(41),
|
||||
'Fresh instance unexpectedly has an open migration.');
|
||||
$assert(!CloudJobService::blockingQueryForInstance(41)->exists(),
|
||||
'Fresh instance unexpectedly has a blocking job.');
|
||||
|
||||
$assess = static function (array $payload) use ($instance, $actor) {
|
||||
return (new CloudBackupEligibility(
|
||||
new BackupReadinessProbeFake($payload)
|
||||
))->assess($instance, $actor);
|
||||
};
|
||||
|
||||
$ready = $assess([
|
||||
'eligible' => true,
|
||||
'retryable' => false,
|
||||
]);
|
||||
$assert($ready->allowed()
|
||||
&& $ready->reasonCode === CloudBackupEligibilityDecision::READY,
|
||||
'Ready Agent evidence was not admitted: ' . $ready->reasonCode);
|
||||
|
||||
foreach ([
|
||||
CloudBackupEligibilityDecision::BACKUP_RESCUE_ACTIVE,
|
||||
CloudBackupEligibilityDecision::BACKUP_DISK_TRANSITION_ACTIVE,
|
||||
CloudBackupEligibilityDecision::DOMAIN_JOB_ACTIVE,
|
||||
] as $reason) {
|
||||
$decision = $assess([
|
||||
'eligible' => false,
|
||||
'retryable' => true,
|
||||
'reason' => $reason,
|
||||
]);
|
||||
$assert(!$decision->allowed() && $decision->reasonCode === $reason
|
||||
&& $decision->message !== '',
|
||||
'Transient Agent blocker was not represented by a clear decision.');
|
||||
}
|
||||
|
||||
$invalid = $assess([
|
||||
'eligible' => false,
|
||||
'retryable' => false,
|
||||
'reason' => 'host-path:/secret',
|
||||
]);
|
||||
$assert($invalid->reasonCode
|
||||
=== CloudBackupEligibilityDecision::READINESS_UNAVAILABLE,
|
||||
'Malformed Agent readiness was accepted.');
|
||||
|
||||
$unavailable = (new CloudBackupEligibility(
|
||||
new BackupReadinessProbeFake([], new RuntimeException('offline'))
|
||||
))->assess($instance, $actor);
|
||||
$assert($unavailable->reasonCode
|
||||
=== CloudBackupEligibilityDecision::READINESS_UNAVAILABLE,
|
||||
'Agent failure was not fail-closed.');
|
||||
|
||||
$db->table('cloud_instance_migrations')->insert([
|
||||
'instance_id' => 41,
|
||||
'state' => 'attention_required',
|
||||
]);
|
||||
$migration = $assess(['eligible' => true, 'retryable' => false]);
|
||||
$assert($migration->reasonCode
|
||||
=== CloudBackupEligibilityDecision::MIGRATION_ACTIVE,
|
||||
'Open maintenance migration did not win admission ordering.');
|
||||
$db->table('cloud_instance_migrations')->delete();
|
||||
|
||||
$db->table('cloud_jobs')->insert([
|
||||
'operation_id' => str_repeat('a', 32),
|
||||
'request_idempotency_key' => 'eligibility-running-job',
|
||||
'request_fingerprint' => str_repeat('b', 64),
|
||||
'instance_id' => 41,
|
||||
'user_id' => 7,
|
||||
'host_id' => 3,
|
||||
'kind' => 'reinstall',
|
||||
'phase' => 'running',
|
||||
'status' => 'running',
|
||||
'execution_mode' => 'agent',
|
||||
'sealed_payload' => '{}',
|
||||
'attempts' => 1,
|
||||
'dispatch_attempts' => 1,
|
||||
'dispatch_deadline_at' => time() + 300,
|
||||
'effect_resolution' => 'not_applicable',
|
||||
]);
|
||||
$job = $assess(['eligible' => true, 'retryable' => false]);
|
||||
$assert($job->reasonCode
|
||||
=== CloudBackupEligibilityDecision::INSTANCE_MUTATION_ACTIVE,
|
||||
'Blocking instance job did not prevent Agent admission.');
|
||||
|
||||
$source = (string) file_get_contents(
|
||||
dirname(__DIR__) . '/app/Services/Cloud/CloudBackupService.php'
|
||||
);
|
||||
$assert(str_contains($source, '(new CloudBackupEligibility())->assess(')
|
||||
&& strpos($source, '(new CloudBackupEligibility())->assess(')
|
||||
< strpos($source, 'beginAgentPublication('),
|
||||
'Manual backup does not run readiness before storage publication.');
|
||||
|
||||
echo "Cloud backup eligibility tests passed.\n";
|
||||
Loading…
Reference in New Issue