httpStatus === 409, 'Unexpected conflict status: ' . $error->httpStatus); return; } throw new RuntimeException('Expected storage identity conflict.'); }; $db = new Capsule(); $db->addConnection(['driver' => 'sqlite', 'database' => ':memory:'], 'default'); $db->setAsGlobal(); $db->bootEloquent(); $schema = $db->schema('default'); $schema->create('cloud_storage_backends', static function (Blueprint $t): void { $t->increments('id'); $t->string('backend_key'); $t->string('identity_sha256'); $t->string('name'); $t->string('driver'); $t->string('endpoint'); $t->string('region'); $t->string('bucket'); $t->string('prefix'); $t->boolean('path_style'); $t->string('failure_domain'); $t->text('sealed_credentials'); $t->unsignedBigInteger('config_revision'); $t->unsignedBigInteger('tested_revision'); $t->string('admin_state'); $t->string('health_state'); $t->unsignedInteger('write_priority'); $t->unsignedInteger('consecutive_failures')->default(0); $t->unsignedInteger('consecutive_successes')->default(0); $t->string('last_error_code')->default(''); $t->unsignedBigInteger('last_tested_at')->default(0); $t->unsignedBigInteger('last_healthy_at')->default(0); $t->unsignedBigInteger('next_probe_at')->default(0); $t->unsignedBigInteger('created_at'); $t->unsignedBigInteger('updated_at'); }); $schema->create('cloud_storage_artifacts', static function (Blueprint $t): void { $t->increments('id'); $t->string('artifact_key')->unique(); $t->string('kind'); $t->unsignedBigInteger('maximum_size_bytes'); $t->unsignedBigInteger('size_bytes'); $t->string('sha256'); $t->string('durability_policy'); $t->unsignedTinyInteger('desired_copies'); $t->unsignedTinyInteger('required_copies'); $t->string('state'); $t->string('last_operation_id')->nullable(); $t->unsignedBigInteger('verified_at'); $t->unsignedBigInteger('created_at'); $t->unsignedBigInteger('updated_at'); }); $schema->create('cloud_storage_replicas', static function (Blueprint $t): void { $t->increments('id'); $t->string('replica_key')->unique(); $t->unsignedBigInteger('artifact_id'); $t->unsignedBigInteger('backend_id'); $t->string('role'); $t->string('object_key'); $t->string('object_version_id')->default(''); $t->string('object_etag')->default(''); $t->unsignedBigInteger('size_bytes')->default(0); $t->string('sha256')->default(''); $t->string('state'); $t->string('publish_operation_id')->nullable(); $t->string('delete_operation_id')->nullable(); $t->unsignedInteger('delete_generation')->default(0); $t->unsignedInteger('delete_cycle_attempts')->default(0); $t->unsignedInteger('delete_presence_confirmations')->default(0); $t->unsignedBigInteger('delete_last_checked_at')->default(0); $t->unsignedBigInteger('delete_next_check_at')->default(0); $t->unsignedBigInteger('delete_attention_at')->nullable(); $t->string('lease_token')->nullable(); $t->unsignedBigInteger('lease_until')->default(0); $t->string('error_code')->default(''); $t->unsignedBigInteger('verified_at')->default(0); $t->unsignedBigInteger('created_at'); $t->unsignedBigInteger('updated_at'); }); $schema->create('cloud_storage_operations', static function (Blueprint $t): void { $t->increments('id'); $t->string('operation_id')->unique(); $t->string('request_idempotency_key')->unique(); $t->string('request_fingerprint'); $t->unsignedBigInteger('artifact_id'); $t->unsignedBigInteger('replica_id')->nullable(); $t->unsignedBigInteger('source_backend_id')->nullable(); $t->unsignedBigInteger('target_backend_id')->nullable(); $t->string('kind'); $t->string('phase'); $t->string('outcome')->nullable(); $t->string('effect_resolution'); $t->boolean('effect_may_have_applied')->nullable(); $t->boolean('safe_to_resubmit')->nullable(); $t->unsignedInteger('attempts'); $t->string('lease_token')->nullable(); $t->unsignedBigInteger('lease_until'); $t->unsignedBigInteger('available_at'); $t->unsignedBigInteger('next_check_at'); $t->string('error_code'); $t->text('result_json')->nullable(); $t->unsignedBigInteger('terminal_at')->nullable(); $t->unsignedBigInteger('created_at'); $t->unsignedBigInteger('updated_at'); }); $schema->create('cloud_storage_publications', static function (Blueprint $t): void { $t->increments('id'); $t->string('publication_key')->unique(); $t->unsignedBigInteger('artifact_id')->unique(); $t->unsignedBigInteger('backup_id')->nullable()->unique(); $t->string('template_import_intent_id')->nullable()->unique(); $t->string('rescue_template_id')->nullable()->unique(); $t->string('state'); $t->unsignedInteger('attempts')->default(0); $t->unsignedBigInteger('available_at')->default(0); $t->string('lease_token')->nullable(); $t->unsignedBigInteger('lease_until')->default(0); $t->string('last_error_code')->default(''); $t->unsignedBigInteger('coverage_ready_at')->nullable(); $t->unsignedBigInteger('finalized_at')->nullable(); $t->unsignedBigInteger('created_at'); $t->unsignedBigInteger('updated_at'); }); $schema->create('cloud_instance_backups', static function (Blueprint $t): void { $t->increments('id'); $t->string('backup_key')->unique(); $t->unsignedBigInteger('storage_artifact_id')->nullable(); $t->unsignedBigInteger('source_instance_id'); $t->unsignedBigInteger('source_host_id'); $t->unsignedBigInteger('user_id'); $t->string('source_uuid'); $t->string('name'); $t->string('purpose')->default('portable'); $t->string('source')->default('manual'); $t->unsignedBigInteger('subscription_id')->nullable(); $t->unsignedBigInteger('scheduled_for')->nullable(); $t->string('state'); $t->string('consistency'); $t->string('object_key'); $t->unsignedBigInteger('object_size_bytes')->default(0); $t->unsignedBigInteger('virtual_size_bytes')->default(0); $t->string('sha256')->default(''); $t->string('object_etag')->default(''); $t->string('object_version_id')->default(''); $t->text('source_portable_restore_contract_json')->default('{}'); $t->string('source_portable_restore_contract_sha256')->default(''); $t->text('spec_json'); $t->text('sealed_credential')->nullable(); $t->unsignedBigInteger('storage_revision')->default(0); $t->unsignedBigInteger('active_job_id')->nullable(); $t->string('last_operation_id')->nullable(); $t->string('error_code')->default(''); $t->unsignedBigInteger('ready_at')->nullable(); $t->unsignedBigInteger('created_at'); $t->unsignedBigInteger('updated_at'); }); $schema->create('cloud_jobs', static function (Blueprint $t): void { $t->increments('id'); $t->string('kind'); $t->unsignedBigInteger('host_id'); $t->unsignedBigInteger('instance_id')->nullable(); $t->unsignedBigInteger('backup_id')->nullable(); $t->unsignedBigInteger('user_id')->nullable(); $t->string('operation_id')->unique(); $t->string('effect_resolution')->default('not_applicable'); }); $now = time(); foreach ([[1, 'provider-a', 10], [2, 'provider-b', 20]] as [$id, $name, $priority]) { Capsule::table('cloud_storage_backends')->insert([ 'id' => $id, 'backend_key' => str_repeat((string) $id, 32), 'identity_sha256' => str_repeat((string) $id, 64), 'name' => $name, 'driver' => 's3', 'endpoint' => 'https://' . $name . '.example.test', 'region' => 'us-east-1', 'bucket' => $name . '-bucket', 'prefix' => $name, 'path_style' => 1, 'failure_domain' => $name, 'sealed_credentials' => 'sealed', 'config_revision' => 1, 'tested_revision' => 1, 'admin_state' => 'active', 'health_state' => 'healthy', 'write_priority' => $priority, 'consecutive_failures' => 0, 'consecutive_successes' => 1, 'last_error_code' => '', 'last_tested_at' => $now, 'last_healthy_at' => $now, 'next_probe_at' => $now + 300, 'created_at' => $now, 'updated_at' => $now, ]); } $portableRestore = [ 'contract' => 1, 'architecture' => 'x86_64', 'machine' => 'q35', 'firmware' => 'uefi', 'secure_boot' => false, 'disk_format' => 'qcow2', 'system_disk_only' => true, 'public_network_contract' => 1, 'vpc_inherited' => false, ]; $compatibility = CloudPortableRestoreCompatibility::fromPayload( $portableRestore ); $sourceSpec = json_encode([ 'managed' => true, 'vcpu' => 2, 'memory_mib' => 2048, 'disk_gib' => 20, 'bandwidth_mbps' => 1000, ], JSON_THROW_ON_ERROR); $fixture = static function (int $id) use ( $now, $sourceSpec, $compatibility ): array { $backupKey = substr(hash('sha256', 'storage-finalizer-' . $id), 0, 32); $agentOperationId = substr(hash('sha256', 'agent-' . $id), 0, 32); Capsule::table('cloud_instance_backups')->insert([ 'id' => $id, 'backup_key' => $backupKey, 'storage_artifact_id' => null, 'source_instance_id' => 100 + $id, 'source_host_id' => 3, 'user_id' => 9, 'source_uuid' => sprintf('00000000-0000-4000-8000-%012d', $id), 'name' => 'storage-finalizer-' . $id, 'state' => 'creating', 'consistency' => 'filesystem', 'object_key' => 'vm-backups/' . $backupKey . '/system.qcow2', 'source_portable_restore_contract_json' => $compatibility->json(), 'source_portable_restore_contract_sha256' => $compatibility->sha256(), 'spec_json' => $sourceSpec, 'sealed_credential' => 'sealed-guest', 'storage_revision' => 0, 'active_job_id' => $id, 'last_operation_id' => $agentOperationId, 'created_at' => $now, 'updated_at' => $now, ]); $publication = CloudStorageArtifactService::beginAgentPublication( 'backup', 'vm-backups/' . $backupKey . '/system.qcow2', 21 * 1073741824, 'storage-finalizer-' . str_pad((string) $id, 4, '0', STR_PAD_LEFT), hash('sha256', 'storage-finalizer-fingerprint-' . $id), ['backup_id' => $id] ); Capsule::table('cloud_instance_backups')->where('id', $id)->update([ 'storage_artifact_id' => (int) $publication['artifact_id'], 'object_key' => (string) $publication['object_key'], ]); $work = CloudStorageArtifactService::claimPrimaryPreparation( (string) $publication['operation_id'], $now + $id, 300 ); CloudStorageArtifactService::markPrimaryAwaitingReceipt( (string) $publication['operation_id'], (string) $work['lease_token'], [ 'upload_id' => 'upload-' . $id, 'part_size_bytes' => 67108864, 'part_count' => 336, 'maximum_size_bytes' => 21 * 1073741824, ], $now + $id ); Capsule::table('cloud_jobs')->insert([ 'id' => $id, 'kind' => 'backup_create', 'host_id' => 3, 'instance_id' => 100 + $id, 'backup_id' => $id, 'user_id' => 9, 'operation_id' => $agentOperationId, 'effect_resolution' => 'not_applicable', ]); return [ 'backup' => CloudInstanceBackup::findOrFail($id), 'job' => CloudJob::findOrFail($id), 'storage_operation_id' => (string) $publication['operation_id'], 'artifact_id' => (int) $publication['artifact_id'], ]; }; $payload = static fn (array $item): array => [ 'owner_id' => 9, 'backup_key' => (string) $item['backup']->backup_key, 'maximum_size_bytes' => 21 * 1073741824, 'storage_artifact_id' => (int) $item['artifact_id'], 'storage_publication_operation_id' => (string) $item['storage_operation_id'], 'source_portable_restore_contract_sha256' => $compatibility->sha256(), ]; $receipt = static fn (array $item): array => [ 'backup_key' => (string) $item['backup']->backup_key, 'size_bytes' => 4096, 'virtual_size_bytes' => 20 * 1073741824, 'sha256' => str_repeat(dechex(((int) $item['backup']->id % 14) + 1), 64), 'object_etag' => '"etag-' . (int) $item['backup']->id . '"', 'object_version_id' => 'version-' . (int) $item['backup']->id, 'consistency' => 'filesystem', 'cleanup_pending' => false, 'source_contract' => $portableRestore, ]; $success = $fixture(1); $successPayload = $payload($success); $successReceipt = $receipt($success); CloudBackupService::finishJobLocked( $success['job'], 'succeeded', $successReceipt, $successPayload ); $ready = CloudInstanceBackup::findOrFail(1); $artifact = CloudStorageArtifact::findOrFail($success['artifact_id']); $primary = CloudStorageReplica::where( 'publish_operation_id', $success['storage_operation_id'] )->firstOrFail(); $operation = CloudStorageOperation::where( 'operation_id', $success['storage_operation_id'] )->firstOrFail(); $successPublication = CloudStoragePublication::where( 'artifact_id', $success['artifact_id'] )->firstOrFail(); $successFinalizedAt = (int) $successPublication->finalized_at; $assert((string) $ready->state === 'ready' && $ready->active_job_id === null && (string) $operation->outcome === 'succeeded' && (string) $artifact->sha256 === (string) $ready->sha256 && (int) $artifact->size_bytes === (int) $ready->object_size_bytes && (string) $primary->state === 'ready' && (string) $primary->object_version_id === (string) $ready->object_version_id && (string) $successPublication->state === 'completed' && (int) $successPublication->attempts === 1 && $successFinalizedAt > 0, 'Agent success did not settle primary storage before exposing the backup.'); CloudBackupService::finishJobLocked( $success['job'], 'succeeded', $successReceipt, $successPayload ); $assert(CloudStorageArtifact::count() === 1 && CloudStorageOperation::where('kind', 'publish')->count() === 1 && (int) CloudStoragePublication::where( 'artifact_id', $success['artifact_id'] )->value('attempts') === 1 && (int) CloudStoragePublication::where( 'artifact_id', $success['artifact_id'] )->value('finalized_at') === $successFinalizedAt, 'Identical finalizer replay allocated or replaced storage identity.'); $mismatch = $fixture(2); $badPayload = $payload($mismatch); $badPayload['storage_artifact_id'] = $success['artifact_id']; $expectConflict(static fn () => CloudBackupService::finishJobLocked( $mismatch['job'], 'succeeded', $receipt($mismatch), $badPayload )); $assert((string) CloudInstanceBackup::findOrFail(2)->state === 'creating' && (string) CloudStorageOperation::where( 'operation_id', $mismatch['storage_operation_id'] )->value('phase') === 'awaiting_receipt', 'Mismatched artifact payload mutated backup or storage state.'); $uncertain = $fixture(3); CloudBackupService::finishJobLocked( $uncertain['job'], 'uncertain', null, $payload($uncertain) ); $uncertainOperation = CloudStorageOperation::where( 'operation_id', $uncertain['storage_operation_id'] )->firstOrFail(); $assert((string) CloudInstanceBackup::findOrFail(3)->state === 'uncertain' && (string) $uncertainOperation->phase === 'reconciling' && (string) $uncertainOperation->effect_resolution === 'unknown' && (string) $uncertainOperation->error_code === 'backup_operation_effect_unknown', 'Unknown Agent effect did not preserve original primary reconciliation.'); $failed = $fixture(4); CloudBackupService::finishJobLocked( $failed['job'], 'failed', null, $payload($failed) ); $failedOperation = CloudStorageOperation::where( 'operation_id', $failed['storage_operation_id'] )->firstOrFail(); $failedPublication = CloudStoragePublication::where( 'artifact_id', $failed['artifact_id'] )->firstOrFail(); $assert((string) CloudInstanceBackup::findOrFail(4)->state === 'failed' && (string) $failedOperation->phase === 'terminal' && (string) $failedOperation->effect_resolution === 'not_applied' && (string) $failedPublication->state === 'failed', 'A failed Agent result did not close the primary publication as definitely not applied.'); $bound = $fixture(5); $foreign = $fixture(6); $wrongOperationPayload = $payload($bound); $wrongOperationPayload['storage_publication_operation_id'] = $foreign['storage_operation_id']; $expectConflict(static fn () => CloudBackupService::finishJobLocked( $bound['job'], 'succeeded', $receipt($bound), $wrongOperationPayload )); $assert((string) CloudInstanceBackup::findOrFail(5)->state === 'creating' && (string) CloudStorageOperation::where( 'operation_id', $bound['storage_operation_id'] )->value('phase') === 'awaiting_receipt' && (string) CloudStorageOperation::where( 'operation_id', $foreign['storage_operation_id'] )->value('phase') === 'awaiting_receipt', 'A foreign successful receipt mutated either artifact ledger.'); $expectConflict(static fn () => CloudBackupService::finishJobLocked( $bound['job'], 'failed', null, $wrongOperationPayload )); $assert((string) CloudInstanceBackup::findOrFail(5)->state === 'creating' && (string) CloudStorageOperation::where( 'operation_id', $bound['storage_operation_id'] )->value('phase') === 'awaiting_receipt' && (string) CloudStorageOperation::where( 'operation_id', $foreign['storage_operation_id'] )->value('phase') === 'awaiting_receipt', 'A foreign storage operation mutated either artifact ledger.'); echo "Cloud backup storage finalizer tests passed.\n";