> */ private static array $values = []; public static function encrypt(array $value, string $purpose): string { $id = bin2hex(random_bytes(16)); self::$values[$id] = $value; return 'backup-preparation-test:' . $id; } public static function decrypt(string $value, string $purpose): array { $id = substr($value, strlen('backup-preparation-test:')); if (!isset(self::$values[$id])) { throw new \RuntimeException('Missing test envelope.'); } return self::$values[$id]; } } } namespace { require dirname(__DIR__) . '/vendor/autoload.php'; use App\Models\CloudBackupSubscription; use App\Models\CloudInstance; use App\Models\CloudInstanceBackup; use App\Models\CloudJob; use App\Models\CloudStorageBackend; use App\Models\CloudStorageOperation; use App\Services\Cloud\CloudBackupPublicationPreparation; use App\Services\Cloud\CloudError; use App\Services\Cloud\CloudInput; use App\Services\Cloud\CloudJobService; use App\Services\Cloud\CloudObjectStorageS3Client; use App\Services\Cloud\CloudSecrets; use App\Services\Cloud\CloudStorageArtifactService; use GuzzleHttp\Client; use GuzzleHttp\Handler\MockHandler; use GuzzleHttp\HandlerStack; use GuzzleHttp\Middleware; use GuzzleHttp\Psr7\Response; use Illuminate\Database\Capsule\Manager as Capsule; use Illuminate\Database\Schema\Blueprint; $assert = static function (bool $condition, string $message): void { if (!$condition) { throw new RuntimeException($message); } }; $expect = static function (callable $callback, int $status) use ( $assert ): void { try { $callback(); } catch (CloudError $error) { $assert($error->httpStatus === $status, 'Unexpected CloudError status: ' . $error->httpStatus); return; } throw new RuntimeException('Expected CloudError was not raised.'); }; $db = new Capsule(); $db->addConnection([ 'driver' => 'sqlite', 'database' => ':memory:', ], 'default'); $db->setAsGlobal(); $db->bootEloquent(); $schema = $db->schema('default'); $schema->create('cloud_storage_backends', static function (Blueprint $t): void { $t->increments('id'); $t->string('backend_key'); $t->string('identity_sha256'); $t->string('name'); $t->string('driver'); $t->string('endpoint'); $t->string('region'); $t->string('bucket'); $t->string('prefix'); $t->boolean('path_style'); $t->string('failure_domain'); $t->text('sealed_credentials'); $t->unsignedBigInteger('config_revision'); $t->unsignedBigInteger('tested_revision'); $t->string('admin_state'); $t->string('health_state'); $t->unsignedInteger('write_priority'); $t->unsignedInteger('consecutive_failures')->default(0); $t->unsignedInteger('consecutive_successes')->default(0); $t->string('last_error_code')->default(''); $t->unsignedBigInteger('last_tested_at')->default(0); $t->unsignedBigInteger('last_healthy_at')->default(0); $t->unsignedBigInteger('next_probe_at')->default(0); $t->unsignedBigInteger('created_at'); $t->unsignedBigInteger('updated_at'); }); $schema->create('cloud_storage_artifacts', static function (Blueprint $t): void { $t->increments('id'); $t->string('artifact_key')->unique(); $t->string('kind'); $t->unsignedBigInteger('maximum_size_bytes'); $t->unsignedBigInteger('size_bytes'); $t->string('sha256'); $t->string('durability_policy'); $t->unsignedTinyInteger('desired_copies'); $t->unsignedTinyInteger('required_copies'); $t->string('state'); $t->string('last_operation_id')->nullable(); $t->unsignedBigInteger('verified_at'); $t->unsignedBigInteger('created_at'); $t->unsignedBigInteger('updated_at'); }); $schema->create('cloud_storage_replicas', static function (Blueprint $t): void { $t->increments('id'); $t->string('replica_key')->unique(); $t->unsignedBigInteger('artifact_id'); $t->unsignedBigInteger('backend_id'); $t->string('role'); $t->string('object_key'); $t->string('object_version_id')->default(''); $t->string('object_etag')->default(''); $t->unsignedBigInteger('size_bytes')->default(0); $t->string('sha256')->default(''); $t->string('state'); $t->string('publish_operation_id')->nullable(); $t->string('delete_operation_id')->nullable(); $t->unsignedInteger('delete_generation')->default(0); $t->unsignedInteger('delete_cycle_attempts')->default(0); $t->unsignedInteger('delete_presence_confirmations')->default(0); $t->unsignedBigInteger('delete_last_checked_at')->default(0); $t->unsignedBigInteger('delete_next_check_at')->default(0); $t->unsignedBigInteger('delete_attention_at')->nullable(); $t->string('lease_token')->nullable(); $t->unsignedBigInteger('lease_until')->default(0); $t->string('error_code')->default(''); $t->unsignedBigInteger('verified_at')->default(0); $t->unsignedBigInteger('created_at'); $t->unsignedBigInteger('updated_at'); }); $schema->create('cloud_storage_operations', static function (Blueprint $t): void { $t->increments('id'); $t->string('operation_id')->unique(); $t->string('request_idempotency_key')->unique(); $t->string('request_fingerprint'); $t->unsignedBigInteger('artifact_id'); $t->unsignedBigInteger('replica_id')->nullable(); $t->unsignedBigInteger('source_backend_id')->nullable(); $t->unsignedBigInteger('target_backend_id')->nullable(); $t->string('kind'); $t->string('phase'); $t->string('outcome')->nullable(); $t->string('effect_resolution'); $t->boolean('effect_may_have_applied')->nullable(); $t->boolean('safe_to_resubmit')->nullable(); $t->unsignedInteger('attempts'); $t->string('lease_token')->nullable(); $t->unsignedBigInteger('lease_until'); $t->unsignedBigInteger('available_at'); $t->unsignedBigInteger('next_check_at'); $t->string('error_code'); $t->text('result_json')->nullable(); $t->unsignedBigInteger('terminal_at')->nullable(); $t->unsignedBigInteger('created_at'); $t->unsignedBigInteger('updated_at'); }); $schema->create('cloud_storage_publications', static function (Blueprint $t): void { $t->increments('id'); $t->string('publication_key')->unique(); $t->unsignedBigInteger('artifact_id')->unique(); $t->unsignedBigInteger('backup_id')->nullable()->unique(); $t->string('template_import_intent_id')->nullable()->unique(); $t->string('rescue_template_id')->nullable()->unique(); $t->string('state'); $t->unsignedInteger('attempts')->default(0); $t->unsignedBigInteger('available_at')->default(0); $t->string('lease_token')->nullable(); $t->unsignedBigInteger('lease_until')->default(0); $t->string('last_error_code')->default(''); $t->unsignedBigInteger('coverage_ready_at')->nullable(); $t->unsignedBigInteger('finalized_at')->nullable(); $t->unsignedBigInteger('created_at'); $t->unsignedBigInteger('updated_at'); }); $schema->create('cloud_instance_backups', static function (Blueprint $t): void { $t->increments('id'); $t->string('backup_key')->unique(); $t->unsignedBigInteger('storage_artifact_id')->nullable()->unique(); $t->unsignedBigInteger('create_request_actor_id')->nullable(); $t->string('create_request_idempotency_key')->nullable(); $t->string('create_request_fingerprint')->nullable(); $t->unsignedBigInteger('source_instance_id'); $t->unsignedBigInteger('source_host_id'); $t->unsignedBigInteger('user_id'); $t->string('source_uuid'); $t->string('name'); $t->string('purpose'); $t->string('source'); $t->unsignedBigInteger('subscription_id')->nullable(); $t->unsignedBigInteger('scheduled_for')->nullable(); $t->string('state'); $t->string('consistency'); $t->string('object_key'); $t->text('source_portable_restore_contract_json'); $t->string('source_portable_restore_contract_sha256'); $t->text('spec_json'); $t->text('sealed_credential')->nullable(); $t->unsignedBigInteger('storage_revision'); $t->unsignedBigInteger('active_job_id')->nullable(); $t->string('last_operation_id')->nullable(); $t->string('error_code')->default(''); $t->unsignedBigInteger('created_at'); $t->unsignedBigInteger('updated_at'); $t->unique(['create_request_actor_id', 'create_request_idempotency_key']); }); $schema->create('cloud_instances', static function (Blueprint $t): void { $t->increments('id'); $t->unsignedBigInteger('user_id'); $t->unsignedBigInteger('host_id'); $t->string('uuid'); $t->string('status'); $t->string('service_status'); $t->unsignedBigInteger('expires_at')->nullable(); $t->text('spec_json'); }); $schema->create('cloud_instance_migrations', static function (Blueprint $t): void { $t->increments('id'); $t->unsignedBigInteger('instance_id'); $t->string('state'); }); $schema->create('cloud_peer_migration_artifacts', static function (Blueprint $t): void { $t->increments('id'); }); $schema->create('cloud_jobs', static function (Blueprint $t): void { $t->increments('id'); $t->string('operation_id')->unique(); $t->string('request_idempotency_key'); $t->string('request_fingerprint'); $t->unsignedBigInteger('instance_id')->nullable(); $t->unsignedBigInteger('backup_id')->nullable(); $t->unsignedBigInteger('user_id'); $t->unsignedBigInteger('host_id')->nullable(); $t->string('kind'); $t->string('execution_mode'); $t->string('phase'); $t->string('outcome')->nullable(); $t->string('status'); $t->string('error_code')->default(''); $t->text('sealed_payload'); $t->integer('attempts')->default(0); $t->integer('dispatch_attempts')->default(0); $t->unsignedBigInteger('dispatch_deadline_at'); $t->unsignedBigInteger('dispatch_started_at')->nullable(); $t->unsignedBigInteger('agent_accepted_at')->nullable(); $t->unsignedBigInteger('dispatched_at')->nullable(); $t->unsignedBigInteger('receipt_deadline_at')->nullable(); $t->unsignedBigInteger('terminal_at')->nullable(); $t->boolean('effect_may_have_applied')->nullable(); $t->boolean('safe_to_resubmit')->nullable(); $t->string('effect_resolution'); $t->boolean('finalization_incident')->default(false); $t->string('finalization_error_code')->nullable(); $t->text('result_json')->nullable(); $t->string('completion_event_id')->nullable(); $t->unsignedBigInteger('read_at')->nullable(); $t->unsignedBigInteger('available_at'); $t->unsignedBigInteger('created_at'); $t->unsignedBigInteger('updated_at'); $t->unique(['user_id', 'request_idempotency_key']); }); $schema->create('cloud_backup_subscriptions', static function (Blueprint $t): void { $t->increments('id'); $t->unsignedBigInteger('instance_id')->unique(); $t->unsignedBigInteger('user_id'); $t->unsignedBigInteger('offer_id'); $t->boolean('enabled'); $t->boolean('renew_next_period'); $t->string('frequency'); $t->unsignedTinyInteger('weekday'); $t->unsignedTinyInteger('retention_count'); $t->string('timezone'); $t->string('price'); $t->string('currency'); $t->unsignedInteger('price_version'); $t->unsignedInteger('state_revision')->default(1); $t->unsignedBigInteger('next_run_at'); $t->unsignedBigInteger('last_scheduled_at')->nullable(); $t->unsignedBigInteger('last_backup_id')->nullable(); $t->unsignedBigInteger('created_at'); $t->unsignedBigInteger('updated_at'); }); $schema->create('cloud_backup_period_entitlements', static function (Blueprint $t): void { $t->increments('id'); $t->unsignedBigInteger('subscription_id'); $t->unsignedBigInteger('instance_id'); $t->unsignedBigInteger('user_id'); $t->unsignedBigInteger('starts_at'); $t->unsignedBigInteger('ends_at'); }); $now = time(); $backend = new CloudStorageBackend(); foreach ([ 'id' => 1, 'backend_key' => str_repeat('1', 32), 'identity_sha256' => str_repeat('1', 64), 'name' => 'primary', 'driver' => 's3', 'endpoint' => 'https://objects.example.test', 'region' => 'us-east-1', 'bucket' => 'virt-backups', 'prefix' => 'primary', 'path_style' => true, 'failure_domain' => 'a', 'sealed_credentials' => 'sealed', 'config_revision' => 1, 'tested_revision' => 1, 'admin_state' => 'active', 'health_state' => 'healthy', 'write_priority' => 10, 'consecutive_failures' => 0, 'consecutive_successes' => 1, 'last_error_code' => '', 'last_tested_at' => $now, 'last_healthy_at' => $now, 'next_probe_at' => $now + 300, 'created_at' => $now, 'updated_at' => $now, ] as $field => $value) { $backend->{$field} = $value; } $backend->save(); $runtime = [ 'config' => [ 'endpoint' => 'https://objects.example.test', 'region' => 'us-east-1', 'bucket' => 'virt-backups', 'prefix' => 'primary', 'path_style' => true, ], 'credentials' => [ 'access_key_id' => 'TESTACCESSKEY', 'secret_access_key' => 'test-secret-never-persist', ], 'revision' => 1, ]; $runtimeResolver = static fn (int $backendId, bool $healthy): array => $runtime; $clock = $now; $clockFn = static function () use (&$clock): int { return $clock; }; $newBackup = static function ( int $id, string $source, ?int $subscriptionId = null, ?int $scheduledFor = null ) use ($now): array { $requestKey = 'backup-prepare-request-' . str_pad((string) $id, 4, '0', STR_PAD_LEFT); $uuid = sprintf('00000000-0000-4000-8000-%012d', $id); $canonical = [ 'instance_id' => 100 + $id, 'name' => 'backup-' . $id, 'consistency_mode' => 'filesystem', ]; if ($source === 'automatic') { $canonical['source'] = 'automatic'; $canonical['subscription_id'] = $subscriptionId; $canonical['scheduled_for'] = $scheduledFor; } $target = '/v1/vms/' . rawurlencode($uuid) . '/backups'; $fingerprint = CloudJobService::standaloneRequestFingerprint( 7, 'backup_create', 'POST', $target, $canonical ); $instance = CloudInstance::find(100 + $id); if (!$instance instanceof CloudInstance) { $instance = new CloudInstance(); foreach ([ 'id' => 100 + $id, 'user_id' => 33, 'host_id' => 7, 'uuid' => $uuid, 'status' => 'active', 'service_status' => 'active', 'expires_at' => $now + 86400, 'spec_json' => json_encode([ 'managed' => true, 'os_source' => 'cloud_template', ], JSON_THROW_ON_ERROR), ] as $field => $value) { $instance->{$field} = $value; } $instance->save(); } $backup = new CloudInstanceBackup(); foreach ([ 'id' => $id, 'backup_key' => str_pad(dechex($id), 32, 'a'), 'storage_artifact_id' => null, 'create_request_actor_id' => 91, 'create_request_idempotency_key' => $requestKey, 'create_request_fingerprint' => $fingerprint, 'source_instance_id' => 100 + $id, 'source_host_id' => 7, 'user_id' => 33, 'source_uuid' => $uuid, 'name' => 'backup-' . $id, 'purpose' => 'portable', 'source' => $source, 'subscription_id' => $subscriptionId, 'scheduled_for' => $scheduledFor, 'state' => 'creating', 'consistency' => 'filesystem', 'object_key' => 'pending-' . $id, 'source_portable_restore_contract_json' => '{}', 'source_portable_restore_contract_sha256' => str_repeat('c', 64), 'spec_json' => '{}', 'sealed_credential' => 'sealed-guest', 'storage_revision' => 0, 'active_job_id' => null, 'last_operation_id' => null, 'error_code' => '', 'created_at' => $now, 'updated_at' => $now, ] as $field => $value) { $backup->{$field} = $value; } $backup->save(); $publication = CloudStorageArtifactService::beginAgentPublication( 'backup', 'vm-backups/' . $backup->backup_key . '/system.qcow2', 128 * 1024 * 1024, 'backup-publication-test-' . $id, hash('sha256', 'backup-publication-test-' . $id), ['backup_id' => $id] ); $backup->storage_artifact_id = (int) $publication['artifact_id']; $backup->object_key = (string) $publication['object_key']; $backup->save(); return [$backup, $publication]; }; // Known CreateMultipart result: one durable job is bound atomically and // repeated ensure calls never contact S3 or create a replacement job. [$manual, $manualPublication] = $newBackup(1, 'manual'); $manualHistory = []; $manualStack = HandlerStack::create(new MockHandler([ static function () use ($assert): Response { $assert(Capsule::connection('default')->transactionLevel() === 0, 'CreateMultipartUpload was called inside a database transaction.'); return new Response(200, ['Content-Type' => 'application/xml'], 'manual-upload' . ''); }, ])); $manualStack->push(Middleware::history($manualHistory)); $manualService = new CloudBackupPublicationPreparation( new CloudObjectStorageS3Client(new Client(['handler' => $manualStack])), $runtimeResolver, $clockFn ); $manualJob = $manualService->ensureAgentJob((int) $manual->id); $assert($manualJob instanceof CloudJob, 'Known multipart result did not create an Agent job.'); $manual->refresh(); $assert((int) $manual->active_job_id === (int) $manualJob->id && (string) $manual->last_operation_id === (string) $manualJob->operation_id && (int) $manualJob->backup_id === (int) $manual->id && (int) $manualJob->instance_id === (int) $manual->source_instance_id, 'Agent job and backup were not bound in both directions.'); $sameJob = $manualService->ensureAgentJob((int) $manual->id); $assert($sameJob instanceof CloudJob && (int) $sameJob->id === (int) $manualJob->id && CloudJob::count() === 1 && count($manualHistory) === 1, 'Repeated preparation created another job or multipart upload.'); $manualOperation = CloudStorageOperation::where( 'operation_id', (string) $manualPublication['operation_id'] )->firstOrFail(); $stored = (string) $manualOperation->result_json; $assert(str_contains($stored, 'manual-upload') && !str_contains($stored, 'http') && !str_contains($stored, 'TESTACCESSKEY') && !str_contains($stored, 'test-secret-never-persist'), 'Storage operation persisted a capability URL or credential.'); $payload = CloudSecrets::decrypt( (string) $manualJob->sealed_payload, 'job-payload' ); $assert(isset($payload['body']['object_upload']['part_urls'][0]), 'Short-lived upload capabilities were not confined to the sealed job.'); $terminalPayload = json_encode( CloudInput::safeResult($payload), JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES ); $assert(!str_contains($terminalPayload, 'https://') && !str_contains($terminalPayload, 'part_urls') && !str_contains($terminalPayload, 'complete_url') && !str_contains($terminalPayload, 'abort_url') && !str_contains($terminalPayload, 'verify_url'), 'Resolved terminal payload retained an upload capability URL.'); // A 5xx CreateMultipart response is unknown, not a safe retry. Exact-key // reconciliation adopts the one original session and creates one job. [$unknown, $unknownPublication] = $newBackup(2, 'manual'); $unknownService = new CloudBackupPublicationPreparation( new CloudObjectStorageS3Client(new Client([ 'handler' => HandlerStack::create(new MockHandler([ new Response(503), ])), ])), $runtimeResolver, $clockFn ); $assert($unknownService->ensureAgentJob((int) $unknown->id) === null, 'Unknown multipart result incorrectly created an Agent job.'); $unknownOperation = CloudStorageOperation::where( 'operation_id', (string) $unknownPublication['operation_id'] )->firstOrFail(); $assert((string) $unknownOperation->phase === 'reconciling' && (string) $unknownOperation->effect_resolution === 'unknown' && CloudJob::where('backup_id', (int) $unknown->id)->count() === 0, 'Unknown CreateMultipart effect was not retained on its original operation.'); $clock += 60; $initiated = gmdate('Y-m-d\TH:i:s\Z', $now); $key = (string) $unknownPublication['object_key']; $lookupXml = '' . 'false' . htmlspecialchars($key, ENT_XML1) . 'recovered-upload' . $initiated . ''; $recoveryHistory = []; $recoveryStack = HandlerStack::create(new MockHandler([ new Response(200, ['Content-Type' => 'application/xml'], $lookupXml), ])); $recoveryStack->push(Middleware::history($recoveryHistory)); $recoveryService = new CloudBackupPublicationPreparation( new CloudObjectStorageS3Client(new Client(['handler' => $recoveryStack])), $runtimeResolver, $clockFn ); $recoveredJob = $recoveryService->ensureAgentJob((int) $unknown->id); $assert($recoveredJob instanceof CloudJob && CloudJob::where('backup_id', (int) $unknown->id)->count() === 1 && count($recoveryHistory) === 1, 'Exact-key reconciliation did not adopt one original multipart session.'); $unknownOperation->refresh(); $assert(str_contains((string) $unknownOperation->result_json, 'recovered-upload') && (int) $unknownOperation->attempts === 1, 'Recovered upload identity was replaced or the provider effect was replayed.'); $assert((int) $recoveryService->ensureAgentJob((int) $unknown->id)->id === (int) $recoveredJob->id && CloudJob::where('backup_id', (int) $unknown->id)->count() === 1, 'A second worker/call created a duplicate recovered job.'); // Automatic intent without a pre-existing job uses the same preparation // path and advances its subscription exactly when the job binding commits. $scheduledFor = $now - 60; $subscription = new CloudBackupSubscription(); foreach ([ 'id' => 5, 'instance_id' => 103, 'user_id' => 33, 'offer_id' => 8, 'enabled' => true, 'renew_next_period' => true, 'frequency' => 'daily', 'weekday' => 0, 'retention_count' => 2, 'timezone' => 'UTC', 'price' => '2.00', 'currency' => 'USD', 'price_version' => 1, 'state_revision' => 1, 'next_run_at' => $scheduledFor, 'last_scheduled_at' => null, 'last_backup_id' => null, 'created_at' => $now, 'updated_at' => $now, ] as $field => $value) { $subscription->{$field} = $value; } $subscription->save(); Capsule::table('cloud_backup_period_entitlements')->insert([ 'subscription_id' => (int) $subscription->id, 'instance_id' => (int) $subscription->instance_id, 'user_id' => (int) $subscription->user_id, 'starts_at' => $now - 3600, 'ends_at' => $now + 3600, ]); [$automatic] = $newBackup(3, 'automatic', 5, $scheduledFor); $automaticService = new CloudBackupPublicationPreparation( new CloudObjectStorageS3Client(new Client([ 'handler' => HandlerStack::create(new MockHandler([ new Response(200, ['Content-Type' => 'application/xml'], '' . 'automatic-upload' . ''), ])), ])), $runtimeResolver, $clockFn ); $automaticJob = $automaticService->ensureAgentJob((int) $automatic->id); $subscription->refresh(); $assert($automaticJob instanceof CloudJob && (int) $subscription->last_scheduled_at === $scheduledFor && (int) $subscription->last_backup_id === (int) $automatic->id && (int) $subscription->next_run_at > $scheduledFor, 'Automatic backup intent did not atomically bind its job and advance schedule.'); $advancedNextRun = (int) $subscription->next_run_at; $automaticService->ensureAgentJob((int) $automatic->id); $subscription->refresh(); $assert((int) $subscription->next_run_at === $advancedNextRun && CloudJob::where('backup_id', (int) $automatic->id)->count() === 1, 'Repeated automatic preparation advanced the schedule or duplicated the job.'); // A migration may start after the durable storage intent commits but // before its Agent job is bound. The post-S3 instance fence must reject // the bind without replacing the original multipart identity. Once the // blocker is removed, the same upload is signed and bound without another // CreateMultipartUpload request. [$blocked, $blockedPublication] = $newBackup(4, 'manual'); $blockedHistory = []; $blockedStack = HandlerStack::create(new MockHandler([ static function () use ($assert): Response { $assert(Capsule::connection('default')->transactionLevel() === 0, 'Multipart preparation crossed the business transaction.'); Capsule::table('cloud_instance_migrations')->insert([ 'instance_id' => 104, 'state' => 'preparing_source', ]); return new Response(200, ['Content-Type' => 'application/xml'], '' . 'blocked-upload' . ''); }, ])); $blockedStack->push(Middleware::history($blockedHistory)); $blockedService = new CloudBackupPublicationPreparation( new CloudObjectStorageS3Client(new Client([ 'handler' => $blockedStack, ])), $runtimeResolver, $clockFn ); $expect(static fn () => $blockedService->ensureAgentJob( (int) $blocked->id ), 409); $blockedOperation = CloudStorageOperation::where( 'operation_id', (string) $blockedPublication['operation_id'] )->firstOrFail(); $assert((string) $blockedOperation->phase === 'awaiting_receipt' && str_contains( (string) $blockedOperation->result_json, 'blocked-upload' ) && CloudJob::where('backup_id', (int) $blocked->id)->count() === 0, 'A concurrent migration did not fence the pre-Agent backup bind.'); Capsule::table('cloud_instance_migrations') ->where('instance_id', 104)->delete(); $clock += 301; $resumedService = new CloudBackupPublicationPreparation( new CloudObjectStorageS3Client(new Client([ 'handler' => HandlerStack::create(new MockHandler([])), ])), $runtimeResolver, $clockFn ); $resumedJob = $resumedService->ensureAgentJob((int) $blocked->id); $blockedOperation->refresh(); $assert($resumedJob instanceof CloudJob && count($blockedHistory) === 1 && str_contains( (string) $blockedOperation->result_json, 'blocked-upload' ), 'Clearing the blocker replayed or replaced the durable upload.'); // The same late fence applies to an ordinary in-flight instance job, not // only to the migration ledger. [$jobBlocked, $jobBlockedPublication] = $newBackup(7, 'manual'); $jobBlockedService = new CloudBackupPublicationPreparation( new CloudObjectStorageS3Client(new Client([ 'handler' => HandlerStack::create(new MockHandler([ static function () use ($clock): Response { Capsule::table('cloud_jobs')->insert([ 'operation_id' => str_repeat('d', 32), 'request_idempotency_key' => 'concurrent-instance-job-0007', 'request_fingerprint' => str_repeat('e', 64), 'instance_id' => 107, 'backup_id' => null, 'user_id' => 33, 'host_id' => 7, 'kind' => 'action', 'execution_mode' => 'agent', 'phase' => 'accepted', 'outcome' => null, 'status' => 'queued', 'error_code' => '', 'sealed_payload' => 'sealed', 'attempts' => 0, 'dispatch_attempts' => 0, 'dispatch_deadline_at' => $clock + 600, 'dispatch_started_at' => null, 'agent_accepted_at' => null, 'dispatched_at' => null, 'receipt_deadline_at' => null, 'terminal_at' => null, 'effect_may_have_applied' => null, 'safe_to_resubmit' => null, 'effect_resolution' => 'not_applicable', 'finalization_incident' => 0, 'finalization_error_code' => null, 'result_json' => null, 'completion_event_id' => null, 'read_at' => null, 'available_at' => $clock, 'created_at' => $clock, 'updated_at' => $clock, ]); return new Response(200, [ 'Content-Type' => 'application/xml', ], '' . 'job-blocked-upload' . ''); }, ])), ])), $runtimeResolver, $clockFn ); $expect(static fn () => $jobBlockedService->ensureAgentJob( (int) $jobBlocked->id ), 409); $jobBlockedOperation = CloudStorageOperation::where( 'operation_id', (string) $jobBlockedPublication['operation_id'] )->firstOrFail(); $assert(CloudJob::where( 'backup_id', (int) $jobBlocked->id )->count() === 0 && str_contains( (string) $jobBlockedOperation->result_json, 'job-blocked-upload' ), 'A concurrent instance job did not fence the backup bind.'); CloudJob::where('operation_id', str_repeat('d', 32))->delete(); // The user changes the schedule while CreateMultipartUpload is in // progress. The old slot may still bind, but it must not overwrite the // newly selected frequency or next execution time. $staleSlot = $clock - 60; $newNextRun = $clock + 7 * 86400; $changedSubscription = new CloudBackupSubscription(); foreach ([ 'id' => 6, 'instance_id' => 105, 'user_id' => 33, 'offer_id' => 8, 'enabled' => true, 'renew_next_period' => true, 'frequency' => 'daily', 'weekday' => 0, 'retention_count' => 2, 'timezone' => 'UTC', 'price' => '2.00', 'currency' => 'USD', 'price_version' => 1, 'state_revision' => 1, 'next_run_at' => $staleSlot, 'last_scheduled_at' => null, 'last_backup_id' => null, 'created_at' => $now, 'updated_at' => $now, ] as $field => $value) { $changedSubscription->{$field} = $value; } $changedSubscription->save(); Capsule::table('cloud_backup_period_entitlements')->insert([ 'subscription_id' => 6, 'instance_id' => 105, 'user_id' => 33, 'starts_at' => $now - 3600, 'ends_at' => $clock + 3600, ]); [$changedBackup] = $newBackup(5, 'automatic', 6, $staleSlot); $changedService = new CloudBackupPublicationPreparation( new CloudObjectStorageS3Client(new Client([ 'handler' => HandlerStack::create(new MockHandler([ static function () use ($newNextRun): Response { Capsule::table('cloud_backup_subscriptions') ->where('id', 6)->update([ 'frequency' => 'weekly', 'weekday' => 3, 'state_revision' => 2, 'next_run_at' => $newNextRun, ]); return new Response(200, [ 'Content-Type' => 'application/xml', ], '' . 'changed-plan-upload' . ''); }, ])), ])), $runtimeResolver, $clockFn ); $changedJob = $changedService->ensureAgentJob((int) $changedBackup->id); $changedSubscription->refresh(); $assert($changedJob instanceof CloudJob && (string) $changedSubscription->frequency === 'weekly' && (int) $changedSubscription->weekday === 3 && (int) $changedSubscription->next_run_at === $newNextRun && (int) $changedSubscription->last_scheduled_at === $staleSlot && (int) $changedSubscription->last_backup_id === (int) $changedBackup->id, 'An old backup intent overwrote the user\'s newer schedule.'); // Disabling the subscription during provider preparation is also a real // fence. The remote multipart identity is retained for reconciliation, // but no Agent job may escape after the paid entitlement is withdrawn. $disabledSlot = $clock - 30; $disabledSubscription = new CloudBackupSubscription(); foreach ([ 'id' => 7, 'instance_id' => 106, 'user_id' => 33, 'offer_id' => 8, 'enabled' => true, 'renew_next_period' => true, 'frequency' => 'daily', 'weekday' => 0, 'retention_count' => 2, 'timezone' => 'UTC', 'price' => '2.00', 'currency' => 'USD', 'price_version' => 1, 'state_revision' => 1, 'next_run_at' => $disabledSlot, 'last_scheduled_at' => null, 'last_backup_id' => null, 'created_at' => $now, 'updated_at' => $now, ] as $field => $value) { $disabledSubscription->{$field} = $value; } $disabledSubscription->save(); Capsule::table('cloud_backup_period_entitlements')->insert([ 'subscription_id' => 7, 'instance_id' => 106, 'user_id' => 33, 'starts_at' => $now - 3600, 'ends_at' => $clock + 3600, ]); [$disabledBackup, $disabledPublication] = $newBackup( 6, 'automatic', 7, $disabledSlot ); $disabledService = new CloudBackupPublicationPreparation( new CloudObjectStorageS3Client(new Client([ 'handler' => HandlerStack::create(new MockHandler([ static function (): Response { Capsule::table('cloud_backup_subscriptions') ->where('id', 7)->update([ 'enabled' => 0, 'state_revision' => 2, ]); return new Response(200, [ 'Content-Type' => 'application/xml', ], '' . 'disabled-upload' . ''); }, ])), ])), $runtimeResolver, $clockFn ); $expect(static fn () => $disabledService->ensureAgentJob( (int) $disabledBackup->id ), 409); $disabledOperation = CloudStorageOperation::where( 'operation_id', (string) $disabledPublication['operation_id'] )->firstOrFail(); $assert(CloudJob::where( 'backup_id', (int) $disabledBackup->id )->count() === 0 && (string) $disabledOperation->phase === 'awaiting_receipt' && str_contains( (string) $disabledOperation->result_json, 'disabled-upload' ), 'A disabled subscription still escaped as an Agent backup job.'); echo "Cloud backup publication preparation behavior tests passed.\n"; }