httpStatus === 409, 'Unexpected conflict status: ' . $error->httpStatus); return; } throw new RuntimeException('Expected storage identity conflict.'); }; $db = new Capsule(); $db->addConnection(['driver' => 'sqlite', 'database' => ':memory:'], 'default'); $db->setAsGlobal(); $db->bootEloquent(); $schema = $db->schema('default'); $schema->create('cloud_storage_backends', static function (Blueprint $t): void { $t->increments('id'); $t->string('backend_key'); $t->string('identity_sha256'); $t->string('name'); $t->string('driver'); $t->string('endpoint'); $t->string('region'); $t->string('bucket'); $t->string('prefix'); $t->boolean('path_style'); $t->string('failure_domain'); $t->text('sealed_credentials'); $t->unsignedBigInteger('config_revision'); $t->unsignedBigInteger('tested_revision'); $t->string('admin_state'); $t->string('health_state'); $t->unsignedInteger('write_priority'); $t->unsignedInteger('consecutive_failures')->default(0); $t->unsignedInteger('consecutive_successes')->default(0); $t->string('last_error_code')->default(''); $t->unsignedBigInteger('last_tested_at')->default(0); $t->unsignedBigInteger('last_healthy_at')->default(0); $t->unsignedBigInteger('next_probe_at')->default(0); $t->unsignedBigInteger('created_at'); $t->unsignedBigInteger('updated_at'); }); $schema->create('cloud_storage_artifacts', static function (Blueprint $t): void { $t->increments('id'); $t->string('artifact_key')->unique(); $t->string('kind'); $t->unsignedBigInteger('maximum_size_bytes'); $t->unsignedBigInteger('size_bytes'); $t->string('sha256'); $t->string('durability_policy'); $t->unsignedTinyInteger('desired_copies'); $t->unsignedTinyInteger('required_copies'); $t->string('state'); $t->string('last_operation_id')->nullable(); $t->unsignedBigInteger('verified_at'); $t->unsignedBigInteger('created_at'); $t->unsignedBigInteger('updated_at'); }); $schema->create('cloud_storage_replicas', static function (Blueprint $t): void { $t->increments('id'); $t->string('replica_key')->unique(); $t->unsignedBigInteger('artifact_id'); $t->unsignedBigInteger('backend_id'); $t->string('role'); $t->string('object_key'); $t->string('object_version_id')->default(''); $t->string('object_etag')->default(''); $t->unsignedBigInteger('size_bytes')->default(0); $t->string('sha256')->default(''); $t->string('state'); $t->string('publish_operation_id')->nullable(); $t->string('delete_operation_id')->nullable(); $t->unsignedInteger('delete_generation')->default(0); $t->unsignedInteger('delete_cycle_attempts')->default(0); $t->unsignedInteger('delete_presence_confirmations')->default(0); $t->unsignedBigInteger('delete_last_checked_at')->default(0); $t->unsignedBigInteger('delete_next_check_at')->default(0); $t->unsignedBigInteger('delete_attention_at')->nullable(); $t->string('lease_token')->nullable(); $t->unsignedBigInteger('lease_until')->default(0); $t->string('error_code')->default(''); $t->unsignedBigInteger('verified_at')->default(0); $t->unsignedBigInteger('created_at'); $t->unsignedBigInteger('updated_at'); }); $schema->create('cloud_storage_operations', static function (Blueprint $t): void { $t->increments('id'); $t->string('operation_id')->unique(); $t->string('request_idempotency_key')->unique(); $t->string('request_fingerprint'); $t->unsignedBigInteger('artifact_id'); $t->unsignedBigInteger('replica_id')->nullable(); $t->unsignedBigInteger('source_backend_id')->nullable(); $t->unsignedBigInteger('target_backend_id')->nullable(); $t->string('kind'); $t->string('phase'); $t->string('outcome')->nullable(); $t->string('effect_resolution'); $t->boolean('effect_may_have_applied')->nullable(); $t->boolean('safe_to_resubmit')->nullable(); $t->unsignedInteger('attempts'); $t->string('lease_token')->nullable(); $t->unsignedBigInteger('lease_until'); $t->unsignedBigInteger('available_at'); $t->unsignedBigInteger('next_check_at'); $t->string('error_code'); $t->text('result_json')->nullable(); $t->unsignedBigInteger('terminal_at')->nullable(); $t->unsignedBigInteger('created_at'); $t->unsignedBigInteger('updated_at'); }); $schema->create('cloud_storage_publications', static function (Blueprint $t): void { $t->increments('id'); $t->string('publication_key')->unique(); $t->unsignedBigInteger('artifact_id')->unique(); $t->unsignedBigInteger('backup_id')->nullable()->unique(); $t->string('template_import_intent_id')->nullable()->unique(); $t->string('rescue_template_id')->nullable()->unique(); $t->string('state'); $t->unsignedInteger('attempts')->default(0); $t->unsignedBigInteger('available_at')->default(0); $t->string('lease_token')->nullable(); $t->unsignedBigInteger('lease_until')->default(0); $t->string('last_error_code')->default(''); $t->unsignedBigInteger('coverage_ready_at')->nullable(); $t->unsignedBigInteger('finalized_at')->nullable(); $t->unsignedBigInteger('created_at'); $t->unsignedBigInteger('updated_at'); }); $schema->create('cloud_instance_backups', static function (Blueprint $t): void { $t->increments('id'); $t->string('backup_key')->unique(); $t->unsignedBigInteger('storage_artifact_id')->nullable(); $t->unsignedBigInteger('source_instance_id'); $t->unsignedBigInteger('source_host_id'); $t->unsignedBigInteger('user_id'); $t->string('source_uuid'); $t->string('name'); $t->string('purpose')->default('portable'); $t->string('source')->default('manual'); $t->unsignedBigInteger('subscription_id')->nullable(); $t->unsignedBigInteger('scheduled_for')->nullable(); $t->unsignedInteger('schedule_attempt')->default(0); $t->string('state'); $t->string('consistency'); $t->string('object_key'); $t->unsignedBigInteger('object_size_bytes')->default(0); $t->unsignedBigInteger('virtual_size_bytes')->default(0); $t->string('sha256')->default(''); $t->string('object_etag')->default(''); $t->string('object_version_id')->default(''); $t->text('source_portable_restore_contract_json')->default('{}'); $t->string('source_portable_restore_contract_sha256')->default(''); $t->text('spec_json'); $t->text('sealed_credential')->nullable(); $t->unsignedBigInteger('storage_revision')->default(0); $t->unsignedBigInteger('active_job_id')->nullable(); $t->string('last_operation_id')->nullable(); $t->string('error_code')->default(''); $t->unsignedBigInteger('ready_at')->nullable(); $t->unsignedBigInteger('created_at'); $t->unsignedBigInteger('updated_at'); }); $schema->create('cloud_jobs', static function (Blueprint $t): void { $t->increments('id'); $t->string('kind'); $t->unsignedBigInteger('host_id'); $t->unsignedBigInteger('instance_id')->nullable(); $t->unsignedBigInteger('backup_id')->nullable(); $t->unsignedBigInteger('user_id')->nullable(); $t->string('operation_id')->unique(); $t->string('effect_resolution')->default('not_applicable'); $t->boolean('finalization_incident')->default(false); }); $schema->create('cloud_backup_subscriptions', static function (Blueprint $t): void { $t->increments('id'); $t->unsignedBigInteger('instance_id')->unique(); $t->unsignedBigInteger('user_id'); $t->unsignedBigInteger('offer_id'); $t->boolean('enabled'); $t->boolean('renew_next_period'); $t->string('frequency'); $t->unsignedTinyInteger('weekday'); $t->unsignedTinyInteger('retention_count'); $t->string('timezone'); $t->string('price'); $t->string('currency'); $t->unsignedInteger('price_version'); $t->unsignedInteger('state_revision')->default(1); $t->unsignedBigInteger('next_run_at'); $t->unsignedBigInteger('deferred_until')->default(0); $t->string('defer_reason')->default(''); $t->unsignedBigInteger('deferred_at')->nullable(); $t->unsignedInteger('defer_count')->default(0); $t->unsignedBigInteger('last_scheduled_at')->nullable(); $t->unsignedBigInteger('last_backup_id')->nullable(); $t->unsignedBigInteger('created_at'); $t->unsignedBigInteger('updated_at'); }); $now = time(); foreach ([[1, 'provider-a', 10], [2, 'provider-b', 20]] as [$id, $name, $priority]) { Capsule::table('cloud_storage_backends')->insert([ 'id' => $id, 'backend_key' => str_repeat((string) $id, 32), 'identity_sha256' => str_repeat((string) $id, 64), 'name' => $name, 'driver' => 's3', 'endpoint' => 'https://' . $name . '.example.test', 'region' => 'us-east-1', 'bucket' => $name . '-bucket', 'prefix' => $name, 'path_style' => 1, 'failure_domain' => $name, 'sealed_credentials' => 'sealed', 'config_revision' => 1, 'tested_revision' => 1, 'admin_state' => 'active', 'health_state' => 'healthy', 'write_priority' => $priority, 'consecutive_failures' => 0, 'consecutive_successes' => 1, 'last_error_code' => '', 'last_tested_at' => $now, 'last_healthy_at' => $now, 'next_probe_at' => $now + 300, 'created_at' => $now, 'updated_at' => $now, ]); } $portableRestore = [ 'contract' => 1, 'architecture' => 'x86_64', 'machine' => 'q35', 'firmware' => 'uefi', 'secure_boot' => false, 'disk_format' => 'qcow2', 'system_disk_only' => true, 'public_network_contract' => 1, 'vpc_inherited' => false, ]; $compatibility = CloudPortableRestoreCompatibility::fromPayload( $portableRestore ); $sourceSpec = json_encode([ 'managed' => true, 'vcpu' => 2, 'memory_mib' => 2048, 'disk_gib' => 20, 'bandwidth_mbps' => 1000, ], JSON_THROW_ON_ERROR); $fixture = static function (int $id, bool $automatic = false) use ( $now, $sourceSpec, $compatibility ): array { $backupKey = substr(hash('sha256', 'storage-finalizer-' . $id), 0, 32); $agentOperationId = substr(hash('sha256', 'agent-' . $id), 0, 32); $scheduledFor = $automatic ? $now - 60 : null; $subscriptionId = $automatic ? 100 + $id : null; if ($automatic) { Capsule::table('cloud_backup_subscriptions')->insert([ 'id' => $subscriptionId, 'instance_id' => 100 + $id, 'user_id' => 9, 'offer_id' => 1, 'enabled' => 1, 'renew_next_period' => 1, 'frequency' => 'daily', 'weekday' => 0, 'retention_count' => 2, 'timezone' => 'UTC', 'price' => '2.00', 'currency' => 'USD', 'price_version' => 1, 'state_revision' => 1, 'next_run_at' => $scheduledFor, 'deferred_until' => 0, 'defer_reason' => '', 'deferred_at' => null, 'defer_count' => 0, 'last_scheduled_at' => null, 'last_backup_id' => null, 'created_at' => $now, 'updated_at' => $now, ]); } Capsule::table('cloud_instance_backups')->insert([ 'id' => $id, 'backup_key' => $backupKey, 'storage_artifact_id' => null, 'source_instance_id' => 100 + $id, 'source_host_id' => 3, 'user_id' => 9, 'source_uuid' => sprintf('00000000-0000-4000-8000-%012d', $id), 'name' => 'storage-finalizer-' . $id, 'state' => 'creating', 'source' => $automatic ? 'automatic' : 'manual', 'subscription_id' => $subscriptionId, 'scheduled_for' => $scheduledFor, 'schedule_attempt' => $automatic ? 1 : 0, 'consistency' => 'filesystem', 'object_key' => 'vm-backups/' . $backupKey . '/system.qcow2', 'source_portable_restore_contract_json' => $compatibility->json(), 'source_portable_restore_contract_sha256' => $compatibility->sha256(), 'spec_json' => $sourceSpec, 'sealed_credential' => 'sealed-guest', 'storage_revision' => 0, 'active_job_id' => $id, 'last_operation_id' => $agentOperationId, 'created_at' => $now, 'updated_at' => $now, ]); $publication = CloudStorageArtifactService::beginAgentPublication( 'backup', 'vm-backups/' . $backupKey . '/system.qcow2', 21 * 1073741824, 'storage-finalizer-' . str_pad((string) $id, 4, '0', STR_PAD_LEFT), hash('sha256', 'storage-finalizer-fingerprint-' . $id), ['backup_id' => $id] ); Capsule::table('cloud_instance_backups')->where('id', $id)->update([ 'storage_artifact_id' => (int) $publication['artifact_id'], 'object_key' => (string) $publication['object_key'], ]); $work = CloudStorageArtifactService::claimPrimaryPreparation( (string) $publication['operation_id'], $now + $id, 300 ); CloudStorageArtifactService::markPrimaryAwaitingReceipt( (string) $publication['operation_id'], (string) $work['lease_token'], [ 'upload_id' => 'upload-' . $id, 'part_size_bytes' => 67108864, 'part_count' => 336, 'maximum_size_bytes' => 21 * 1073741824, ], $now + $id ); Capsule::table('cloud_jobs')->insert([ 'id' => $id, 'kind' => 'backup_create', 'host_id' => 3, 'instance_id' => 100 + $id, 'backup_id' => $id, 'user_id' => 9, 'operation_id' => $agentOperationId, 'effect_resolution' => 'not_applicable', ]); return [ 'backup' => CloudInstanceBackup::findOrFail($id), 'job' => CloudJob::findOrFail($id), 'storage_operation_id' => (string) $publication['operation_id'], 'artifact_id' => (int) $publication['artifact_id'], 'subscription_id' => $subscriptionId, 'scheduled_for' => $scheduledFor, ]; }; $payload = static fn (array $item): array => [ 'owner_id' => 9, 'backup_key' => (string) $item['backup']->backup_key, 'maximum_size_bytes' => 21 * 1073741824, 'storage_artifact_id' => (int) $item['artifact_id'], 'storage_publication_operation_id' => (string) $item['storage_operation_id'], 'source_portable_restore_contract_sha256' => $compatibility->sha256(), ]; $receipt = static fn (array $item): array => [ 'backup_key' => (string) $item['backup']->backup_key, 'size_bytes' => 4096, 'virtual_size_bytes' => 20 * 1073741824, 'sha256' => str_repeat(dechex(((int) $item['backup']->id % 14) + 1), 64), 'object_etag' => '"etag-' . (int) $item['backup']->id . '"', 'object_version_id' => 'version-' . (int) $item['backup']->id, 'consistency' => 'filesystem', 'cleanup_pending' => false, 'source_contract' => $portableRestore, ]; $success = $fixture(1); $successPayload = $payload($success); $successReceipt = $receipt($success); CloudBackupService::finishJobLocked( $success['job'], 'succeeded', $successReceipt, $successPayload ); $ready = CloudInstanceBackup::findOrFail(1); $artifact = CloudStorageArtifact::findOrFail($success['artifact_id']); $primary = CloudStorageReplica::where( 'publish_operation_id', $success['storage_operation_id'] )->firstOrFail(); $operation = CloudStorageOperation::where( 'operation_id', $success['storage_operation_id'] )->firstOrFail(); $successPublication = CloudStoragePublication::where( 'artifact_id', $success['artifact_id'] )->firstOrFail(); $successFinalizedAt = (int) $successPublication->finalized_at; $assert((string) $ready->state === 'ready' && $ready->active_job_id === null && (string) $operation->outcome === 'succeeded' && (string) $artifact->sha256 === (string) $ready->sha256 && (int) $artifact->size_bytes === (int) $ready->object_size_bytes && (string) $primary->state === 'ready' && (string) $primary->object_version_id === (string) $ready->object_version_id && (string) $successPublication->state === 'completed' && (int) $successPublication->attempts === 1 && $successFinalizedAt > 0, 'Agent success did not settle primary storage before exposing the backup.'); CloudBackupService::finishJobLocked( $success['job'], 'succeeded', $successReceipt, $successPayload ); $assert(CloudStorageArtifact::count() === 1 && CloudStorageOperation::where('kind', 'publish')->count() === 1 && (int) CloudStoragePublication::where( 'artifact_id', $success['artifact_id'] )->value('attempts') === 1 && (int) CloudStoragePublication::where( 'artifact_id', $success['artifact_id'] )->value('finalized_at') === $successFinalizedAt, 'Identical finalizer replay allocated or replaced storage identity.'); $mismatch = $fixture(2); $badPayload = $payload($mismatch); $badPayload['storage_artifact_id'] = $success['artifact_id']; $expectConflict(static fn () => CloudBackupService::finishJobLocked( $mismatch['job'], 'succeeded', $receipt($mismatch), $badPayload )); $assert((string) CloudInstanceBackup::findOrFail(2)->state === 'creating' && (string) CloudStorageOperation::where( 'operation_id', $mismatch['storage_operation_id'] )->value('phase') === 'awaiting_receipt', 'Mismatched artifact payload mutated backup or storage state.'); $uncertain = $fixture(3); CloudBackupService::finishJobLocked( $uncertain['job'], 'uncertain', null, $payload($uncertain) ); $uncertainOperation = CloudStorageOperation::where( 'operation_id', $uncertain['storage_operation_id'] )->firstOrFail(); $assert((string) CloudInstanceBackup::findOrFail(3)->state === 'uncertain' && (string) $uncertainOperation->phase === 'reconciling' && (string) $uncertainOperation->effect_resolution === 'unknown' && (string) $uncertainOperation->error_code === 'backup_operation_effect_unknown', 'Unknown Agent effect did not preserve original primary reconciliation.'); $failed = $fixture(4); CloudBackupService::finishJobLocked( $failed['job'], 'failed', null, $payload($failed) ); $failedOperation = CloudStorageOperation::where( 'operation_id', $failed['storage_operation_id'] )->firstOrFail(); $failedPublication = CloudStoragePublication::where( 'artifact_id', $failed['artifact_id'] )->firstOrFail(); $assert((string) CloudInstanceBackup::findOrFail(4)->state === 'failed' && (string) $failedOperation->phase === 'terminal' && (string) $failedOperation->effect_resolution === 'not_applied' && (string) $failedPublication->state === 'failed', 'A failed Agent result did not close the primary publication as definitely not applied.'); $bound = $fixture(5); $foreign = $fixture(6); $wrongOperationPayload = $payload($bound); $wrongOperationPayload['storage_publication_operation_id'] = $foreign['storage_operation_id']; $expectConflict(static fn () => CloudBackupService::finishJobLocked( $bound['job'], 'succeeded', $receipt($bound), $wrongOperationPayload )); $assert((string) CloudInstanceBackup::findOrFail(5)->state === 'creating' && (string) CloudStorageOperation::where( 'operation_id', $bound['storage_operation_id'] )->value('phase') === 'awaiting_receipt' && (string) CloudStorageOperation::where( 'operation_id', $foreign['storage_operation_id'] )->value('phase') === 'awaiting_receipt', 'A foreign successful receipt mutated either artifact ledger.'); $expectConflict(static fn () => CloudBackupService::finishJobLocked( $bound['job'], 'failed', null, $wrongOperationPayload )); $assert((string) CloudInstanceBackup::findOrFail(5)->state === 'creating' && (string) CloudStorageOperation::where( 'operation_id', $bound['storage_operation_id'] )->value('phase') === 'awaiting_receipt' && (string) CloudStorageOperation::where( 'operation_id', $foreign['storage_operation_id'] )->value('phase') === 'awaiting_receipt', 'A foreign storage operation mutated either artifact ledger.'); $safeTransient = [ 'status' => 'failed', 'effect_may_have_applied' => false, 'safe_to_resubmit' => true, 'cause_code' => 'backup_disk_transition_active', ]; $automaticRetry = $fixture(7, true); $retryStartedAt = time(); CloudBackupService::finishJobLocked( $automaticRetry['job'], 'failed', null, $payload($automaticRetry), false, $safeTransient ); $retiredBackup = CloudInstanceBackup::findOrFail(7); $retiredArtifact = CloudStorageArtifact::findOrFail( $automaticRetry['artifact_id'] ); $retiredPublication = CloudStoragePublication::where( 'artifact_id', $automaticRetry['artifact_id'] )->firstOrFail(); $retiredSubscription = CloudBackupSubscription::findOrFail( $automaticRetry['subscription_id'] ); $retiredOperations = CloudStorageOperation::where( 'artifact_id', $automaticRetry['artifact_id'] )->get(); $assert((string) $retiredBackup->state === 'deleted' && (string) $retiredBackup->error_code === 'automatic_transient_retried' && $retiredBackup->sealed_credential === null && $retiredBackup->active_job_id === null && (string) $retiredArtifact->state === 'deleted' && (string) $retiredPublication->state === 'retired' && $retiredPublication->coverage_ready_at === null && (int) $retiredPublication->finalized_at >= $retryStartedAt && CloudStorageReplica::where( 'artifact_id', $automaticRetry['artifact_id'] )->where('state', '<>', 'deleted')->count() === 0 && $retiredOperations->count() === 2 && $retiredOperations->every( static fn (CloudStorageOperation $operation): bool => (string) $operation->phase === 'terminal' && (string) $operation->outcome === 'failed' && (string) $operation->effect_resolution === 'not_applied' ) && (int) $retiredSubscription->next_run_at === (int) $automaticRetry['scheduled_for'] && (string) $retiredSubscription->defer_reason === 'backup_disk_transition_active' && (int) $retiredSubscription->defer_count === 1 && (int) $retiredSubscription->deferred_until >= $retryStartedAt + 300 && $retiredSubscription->last_scheduled_at === null && $retiredSubscription->last_backup_id === null, 'A proven not-applied automatic race was not retired and re-armed on the same slot.'); $retryDeferredUntil = (int) $retiredSubscription->deferred_until; CloudBackupService::finishJobLocked( $automaticRetry['job'], 'failed', null, $payload($automaticRetry), false, $safeTransient ); $retiredSubscription->refresh(); $assert((int) $retiredSubscription->defer_count === 1 && (int) $retiredSubscription->deferred_until === $retryDeferredUntil && CloudStorageOperation::where( 'artifact_id', $automaticRetry['artifact_id'] )->count() === 2, 'A repeated transient failure receipt re-armed or replaced durable evidence.'); $abortFailure = $fixture(8, true); CloudBackupService::finishJobLocked( $abortFailure['job'], 'failed', null, $payload($abortFailure), false, [ 'status' => 'failed', 'effect_may_have_applied' => false, 'safe_to_resubmit' => true, 'cause_code' => 'backup_upload_abort_failed', ] ); $abortSubscription = CloudBackupSubscription::findOrFail( $abortFailure['subscription_id'] ); $assert((string) CloudInstanceBackup::findOrFail(8)->state === 'failed' && (string) CloudStorageArtifact::findOrFail( $abortFailure['artifact_id'] )->state === 'publishing' && (string) CloudStoragePublication::where( 'artifact_id', $abortFailure['artifact_id'] )->value('state') === 'failed' && (int) $abortSubscription->last_scheduled_at === (int) $abortFailure['scheduled_for'] && (int) $abortSubscription->last_backup_id === 8 && (int) $abortSubscription->next_run_at > (int) $abortFailure['scheduled_for'] && (int) $abortSubscription->deferred_until === 0, 'Multipart abort failure was incorrectly treated as a safe transient retry.'); $automaticSuccess = $fixture(9, true); CloudBackupService::finishJobLocked( $automaticSuccess['job'], 'succeeded', $receipt($automaticSuccess), $payload($automaticSuccess) ); $successSubscription = CloudBackupSubscription::findOrFail( $automaticSuccess['subscription_id'] ); $successNextRun = (int) $successSubscription->next_run_at; CloudBackupService::finishJobLocked( $automaticSuccess['job'], 'succeeded', $receipt($automaticSuccess), $payload($automaticSuccess) ); $successSubscription->refresh(); $assert((string) CloudInstanceBackup::findOrFail(9)->state === 'ready' && (int) $successSubscription->last_scheduled_at === (int) $automaticSuccess['scheduled_for'] && (int) $successSubscription->last_backup_id === 9 && (int) $successSubscription->next_run_at === $successNextRun, 'Ordinary automatic terminal success did not advance exactly once.'); $unknownAutomatic = $fixture(10, true); CloudBackupService::finishJobLocked( $unknownAutomatic['job'], 'uncertain', null, $payload($unknownAutomatic), false, [ 'status' => 'failed', 'effect_may_have_applied' => true, 'safe_to_resubmit' => false, 'cause_code' => 'backup_disk_transition_active', ] ); $unknownSubscription = CloudBackupSubscription::findOrFail( $unknownAutomatic['subscription_id'] ); $assert((string) CloudInstanceBackup::findOrFail(10)->state === 'uncertain' && (string) CloudStorageArtifact::findOrFail( $unknownAutomatic['artifact_id'] )->state !== 'deleted' && (string) CloudStoragePublication::where( 'artifact_id', $unknownAutomatic['artifact_id'] )->value('state') !== 'retired' && (int) $unknownSubscription->next_run_at > (int) $unknownAutomatic['scheduled_for'] && (int) $unknownSubscription->deferred_until === 0, 'Unknown automatic backup effect was replayed as a safe transient failure.'); $manualTransient = $fixture(11); CloudBackupService::finishJobLocked( $manualTransient['job'], 'failed', null, $payload($manualTransient), false, $safeTransient ); $assert((string) CloudInstanceBackup::findOrFail(11)->state === 'failed' && (string) CloudStorageArtifact::findOrFail( $manualTransient['artifact_id'] )->state === 'publishing' && (string) CloudStoragePublication::where( 'artifact_id', $manualTransient['artifact_id'] )->value('state') === 'failed', 'A manual backup failure was incorrectly armed for automatic retry.'); $rollback = $fixture(12, true); $connection = Capsule::connection('default'); $connection->beginTransaction(); try { $poisonedReplicaId = CloudStorageReplica::where( 'artifact_id', $rollback['artifact_id'] )->orderBy('id')->value('id'); Capsule::table('cloud_storage_replicas')->where( 'id', $poisonedReplicaId )->update([ 'object_etag' => 'unexpected-provider-identity', ]); CloudBackupService::finishJobLocked( $rollback['job'], 'failed', null, $payload($rollback), false, $safeTransient ); throw new RuntimeException('Expected blank retirement conflict.'); } catch (CloudError $error) { $assert($error->httpStatus === 409, 'Unexpected blank retirement conflict status.'); $connection->rollBack(); } $assert((string) CloudInstanceBackup::findOrFail(12)->state === 'creating' && (string) CloudStorageArtifact::findOrFail( $rollback['artifact_id'] )->state === 'publishing' && (string) CloudStorageOperation::where( 'operation_id', $rollback['storage_operation_id'] )->value('phase') === 'awaiting_receipt' && (string) CloudStoragePublication::where( 'artifact_id', $rollback['artifact_id'] )->value('state') === 'waiting_primary', 'A failed transient finalization did not roll back the entire ledger.'); echo "Cloud backup storage finalizer tests passed.\n";