diff --git a/web/app/Services/Cloud/CloudAgentBackupReadinessProbe.php b/web/app/Services/Cloud/CloudAgentBackupReadinessProbe.php new file mode 100644 index 00000000..1b2d99cb --- /dev/null +++ b/web/app/Services/Cloud/CloudAgentBackupReadinessProbe.php @@ -0,0 +1,24 @@ +host_id, + self::TIMEOUT_SECONDS + )->request( + 'GET', + '/v1/vms/' . rawurlencode((string) $instance->uuid) + . '/backup-readiness', + $actor + ); + } +} diff --git a/web/app/Services/Cloud/CloudBackupEligibility.php b/web/app/Services/Cloud/CloudBackupEligibility.php new file mode 100644 index 00000000..f6da5bea --- /dev/null +++ b/web/app/Services/Cloud/CloudBackupEligibility.php @@ -0,0 +1,100 @@ +id + )) { + return self::blocked( + CloudBackupEligibilityDecision::MIGRATION_ACTIVE + ); + } + if (CloudJobService::blockingQueryForInstance( + (int) $instance->id + )->exists()) { + return self::blocked( + CloudBackupEligibilityDecision::INSTANCE_MUTATION_ACTIVE + ); + } + $payload = ($this->probe + ?? new CloudAgentBackupReadinessProbe())->inspect( + $instance, + $actor + ); + } catch (\Throwable $error) { + return self::blocked( + CloudBackupEligibilityDecision::READINESS_UNAVAILABLE + ); + } + + $eligible = $payload['eligible'] ?? null; + $retryable = $payload['retryable'] ?? null; + $reason = $payload['reason'] ?? ''; + if (!is_bool($eligible) || !is_bool($retryable) + || !is_string($reason) + || ($eligible && ($retryable || $reason !== '')) + || (!$eligible && (!$retryable + || !in_array($reason, self::AGENT_REASONS, true)))) { + return self::blocked( + CloudBackupEligibilityDecision::READINESS_UNAVAILABLE + ); + } + return $eligible + ? new CloudBackupEligibilityDecision( + CloudBackupEligibilityDecision::READY, + '' + ) + : self::blocked($reason); + } + + private static function blocked( + string $reason + ): CloudBackupEligibilityDecision { + $messages = [ + CloudBackupEligibilityDecision::MIGRATION_ACTIVE => + '实例正在维护迁移,当前不能创建备份', + CloudBackupEligibilityDecision::INSTANCE_MUTATION_ACTIVE => + '实例正在执行其他操作,当前不能创建备份', + CloudBackupEligibilityDecision::BACKUP_RESCUE_ACTIVE => + '实例处于救援模式,当前不能创建备份', + CloudBackupEligibilityDecision::BACKUP_DISK_TRANSITION_ACTIVE => + '实例正在切换系统盘或清理旧系统盘,当前不能创建备份', + CloudBackupEligibilityDecision::DOMAIN_JOB_ACTIVE => + '实例磁盘任务仍在执行,当前不能创建备份', + CloudBackupEligibilityDecision::READINESS_UNAVAILABLE => + '宿主机备份状态暂不可核对,请稍后重试', + ]; + return new CloudBackupEligibilityDecision( + $reason, + $messages[$reason] ?? $messages[ + CloudBackupEligibilityDecision::READINESS_UNAVAILABLE + ] + ); + } +} diff --git a/web/app/Services/Cloud/CloudBackupEligibilityDecision.php b/web/app/Services/Cloud/CloudBackupEligibilityDecision.php new file mode 100644 index 00000000..303066fb --- /dev/null +++ b/web/app/Services/Cloud/CloudBackupEligibilityDecision.php @@ -0,0 +1,43 @@ +reasonCode === self::READY; + } +} diff --git a/web/app/Services/Cloud/CloudBackupReadinessProbe.php b/web/app/Services/Cloud/CloudBackupReadinessProbe.php new file mode 100644 index 00000000..56e782b2 --- /dev/null +++ b/web/app/Services/Cloud/CloudBackupReadinessProbe.php @@ -0,0 +1,11 @@ +assess( + $instance, + $actor + ); + if (!$eligibility->allowed()) { + $status = $eligibility->reasonCode + === CloudBackupEligibilityDecision::READINESS_UNAVAILABLE + ? 503 : 409; + throw new CloudError($eligibility->message, $status); + } + } + try { $backup = $instance->getConnection()->transaction(function () use ( $instance, $actor, $name, $mode, $fingerprint, $automatic, diff --git a/web/tests/CloudBackupEligibilityTest.php b/web/tests/CloudBackupEligibilityTest.php new file mode 100644 index 00000000..22827047 --- /dev/null +++ b/web/tests/CloudBackupEligibilityTest.php @@ -0,0 +1,205 @@ +failure !== null) { + throw $this->failure; + } + return $this->payload; + } +} + +$assert = static function (bool $condition, string $message): void { + if (!$condition) { + throw new RuntimeException($message); + } +}; + +$db = new Capsule(); +$db->addConnection([ + 'driver' => 'sqlite', + 'database' => ':memory:', +], 'default'); +$db->setAsGlobal(); +$db->bootEloquent(); +$schema = $db->schema('default'); +$schema->create('cloud_instances', static function (Blueprint $t): void { + $t->increments('id'); + $t->unsignedBigInteger('user_id'); + $t->unsignedBigInteger('host_id'); + $t->string('uuid'); +}); +$schema->create('cloud_instance_migrations', + static function (Blueprint $t): void { + $t->increments('id'); + $t->unsignedBigInteger('instance_id'); + $t->unsignedBigInteger('vpc_port_id')->nullable(); + $t->string('state'); + }); +$schema->create('cloud_peer_migration_artifacts', + static function (Blueprint $t): void { + $t->increments('id'); + }); +$schema->create('cloud_jobs', static function (Blueprint $t): void { + $t->increments('id'); + $t->string('operation_id')->unique(); + $t->string('request_idempotency_key')->unique(); + $t->string('request_fingerprint'); + $t->unsignedBigInteger('parent_job_id')->nullable(); + $t->unsignedBigInteger('order_id')->nullable(); + $t->unsignedBigInteger('invoice_id')->nullable(); + $t->unsignedBigInteger('instance_id')->nullable(); + $t->unsignedBigInteger('snapshot_id')->nullable(); + $t->unsignedBigInteger('backup_id')->nullable(); + $t->unsignedBigInteger('backup_restore_id')->nullable(); + $t->unsignedBigInteger('backup_recovery_id')->nullable(); + $t->unsignedBigInteger('user_id'); + $t->unsignedBigInteger('host_id')->nullable(); + $t->string('kind'); + $t->string('phase'); + $t->string('outcome')->nullable(); + $t->string('status'); + $t->string('execution_mode')->nullable(); + $t->text('sealed_payload'); + $t->unsignedInteger('attempts')->default(1); + $t->unsignedInteger('dispatch_attempts')->nullable(); + $t->unsignedBigInteger('dispatch_deadline_at')->nullable(); + $t->unsignedBigInteger('dispatch_started_at')->nullable(); + $t->unsignedBigInteger('agent_accepted_at')->nullable(); + $t->unsignedBigInteger('receipt_deadline_at')->nullable(); + $t->unsignedBigInteger('dispatched_at')->nullable(); + $t->unsignedBigInteger('terminal_at')->nullable(); + $t->boolean('effect_may_have_applied')->nullable(); + $t->boolean('safe_to_resubmit')->nullable(); + $t->string('effect_resolution'); + $t->string('error_code')->nullable(); + $t->boolean('finalization_incident')->default(false); + $t->string('finalization_error_code')->nullable(); + $t->string('completion_event_id')->nullable(); + $t->text('result_json')->nullable(); +}); + +$instance = new CloudInstance(); +$instance->id = 41; +$instance->user_id = 7; +$instance->host_id = 3; +$instance->uuid = '11111111-2222-4333-8444-555555555555'; +$instance->save(); +$actor = [ + 'user_id' => 7, + 'is_admin' => true, + 'is_super_admin' => true, + 'can_manage' => true, + 'session_id' => 'backup-test', +]; + +$assert(!CloudInstanceMigrationService::hasOpenMigration(41), + 'Fresh instance unexpectedly has an open migration.'); +$assert(!CloudJobService::blockingQueryForInstance(41)->exists(), + 'Fresh instance unexpectedly has a blocking job.'); + +$assess = static function (array $payload) use ($instance, $actor) { + return (new CloudBackupEligibility( + new BackupReadinessProbeFake($payload) + ))->assess($instance, $actor); +}; + +$ready = $assess([ + 'eligible' => true, + 'retryable' => false, +]); +$assert($ready->allowed() + && $ready->reasonCode === CloudBackupEligibilityDecision::READY, + 'Ready Agent evidence was not admitted: ' . $ready->reasonCode); + +foreach ([ + CloudBackupEligibilityDecision::BACKUP_RESCUE_ACTIVE, + CloudBackupEligibilityDecision::BACKUP_DISK_TRANSITION_ACTIVE, + CloudBackupEligibilityDecision::DOMAIN_JOB_ACTIVE, +] as $reason) { + $decision = $assess([ + 'eligible' => false, + 'retryable' => true, + 'reason' => $reason, + ]); + $assert(!$decision->allowed() && $decision->reasonCode === $reason + && $decision->message !== '', + 'Transient Agent blocker was not represented by a clear decision.'); +} + +$invalid = $assess([ + 'eligible' => false, + 'retryable' => false, + 'reason' => 'host-path:/secret', +]); +$assert($invalid->reasonCode + === CloudBackupEligibilityDecision::READINESS_UNAVAILABLE, + 'Malformed Agent readiness was accepted.'); + +$unavailable = (new CloudBackupEligibility( + new BackupReadinessProbeFake([], new RuntimeException('offline')) +))->assess($instance, $actor); +$assert($unavailable->reasonCode + === CloudBackupEligibilityDecision::READINESS_UNAVAILABLE, + 'Agent failure was not fail-closed.'); + +$db->table('cloud_instance_migrations')->insert([ + 'instance_id' => 41, + 'state' => 'attention_required', +]); +$migration = $assess(['eligible' => true, 'retryable' => false]); +$assert($migration->reasonCode + === CloudBackupEligibilityDecision::MIGRATION_ACTIVE, + 'Open maintenance migration did not win admission ordering.'); +$db->table('cloud_instance_migrations')->delete(); + +$db->table('cloud_jobs')->insert([ + 'operation_id' => str_repeat('a', 32), + 'request_idempotency_key' => 'eligibility-running-job', + 'request_fingerprint' => str_repeat('b', 64), + 'instance_id' => 41, + 'user_id' => 7, + 'host_id' => 3, + 'kind' => 'reinstall', + 'phase' => 'running', + 'status' => 'running', + 'execution_mode' => 'agent', + 'sealed_payload' => '{}', + 'attempts' => 1, + 'dispatch_attempts' => 1, + 'dispatch_deadline_at' => time() + 300, + 'effect_resolution' => 'not_applicable', +]); +$job = $assess(['eligible' => true, 'retryable' => false]); +$assert($job->reasonCode + === CloudBackupEligibilityDecision::INSTANCE_MUTATION_ACTIVE, + 'Blocking instance job did not prevent Agent admission.'); + +$source = (string) file_get_contents( + dirname(__DIR__) . '/app/Services/Cloud/CloudBackupService.php' +); +$assert(str_contains($source, '(new CloudBackupEligibility())->assess(') + && strpos($source, '(new CloudBackupEligibility())->assess(') + < strpos($source, 'beginAgentPublication('), + 'Manual backup does not run readiness before storage publication.'); + +echo "Cloud backup eligibility tests passed.\n";