httpStatus === 409, 'Unexpected CloudError status: ' . $error->httpStatus); return; } throw new RuntimeException('Expected CloudError was not raised.'); }; $db = new Capsule(); $db->addConnection(['driver' => 'sqlite', 'database' => ':memory:']); $db->setAsGlobal(); $db->bootEloquent(); $db->schema()->create('cloud_storage_backends', static function (Blueprint $t): void { $t->increments('id'); $t->string('backend_key'); $t->string('identity_sha256'); $t->string('name'); $t->string('driver'); $t->string('endpoint'); $t->string('region'); $t->string('bucket'); $t->string('prefix'); $t->boolean('path_style'); $t->string('failure_domain'); $t->text('sealed_credentials'); $t->unsignedBigInteger('config_revision'); $t->unsignedBigInteger('tested_revision'); $t->string('admin_state'); $t->string('health_state'); $t->unsignedInteger('write_priority'); $t->unsignedInteger('consecutive_failures')->default(0); $t->unsignedInteger('consecutive_successes')->default(0); $t->string('last_error_code')->default(''); $t->unsignedBigInteger('last_tested_at')->default(0); $t->unsignedBigInteger('last_healthy_at')->default(0); $t->unsignedBigInteger('next_probe_at')->default(0); $t->unsignedBigInteger('created_at'); $t->unsignedBigInteger('updated_at'); }); $db->schema()->create('cloud_storage_artifacts', static function (Blueprint $t): void { $t->increments('id'); $t->string('artifact_key')->unique(); $t->string('kind'); $t->unsignedBigInteger('maximum_size_bytes'); $t->unsignedBigInteger('size_bytes'); $t->string('sha256'); $t->string('durability_policy'); $t->unsignedTinyInteger('desired_copies'); $t->unsignedTinyInteger('required_copies'); $t->string('state'); $t->string('last_operation_id')->nullable(); $t->unsignedBigInteger('verified_at'); $t->unsignedBigInteger('created_at'); $t->unsignedBigInteger('updated_at'); }); $db->schema()->create('cloud_storage_replicas', static function (Blueprint $t): void { $t->increments('id'); $t->string('replica_key')->unique(); $t->unsignedBigInteger('artifact_id'); $t->unsignedBigInteger('backend_id'); $t->string('role'); $t->string('object_key'); $t->string('object_version_id')->default(''); $t->string('object_etag')->default(''); $t->unsignedBigInteger('size_bytes')->default(0); $t->string('sha256')->default(''); $t->string('state'); $t->string('publish_operation_id')->nullable(); $t->string('delete_operation_id')->nullable(); $t->unsignedInteger('delete_generation')->default(0); $t->unsignedInteger('delete_cycle_attempts')->default(0); $t->unsignedInteger('delete_presence_confirmations')->default(0); $t->unsignedBigInteger('delete_last_checked_at')->default(0); $t->unsignedBigInteger('delete_next_check_at')->default(0); $t->unsignedBigInteger('delete_attention_at')->nullable(); $t->string('lease_token')->nullable(); $t->unsignedBigInteger('lease_until')->default(0); $t->string('error_code')->default(''); $t->unsignedBigInteger('verified_at')->default(0); $t->unsignedBigInteger('created_at'); $t->unsignedBigInteger('updated_at'); }); $db->schema()->create('cloud_storage_operations', static function (Blueprint $t): void { $t->increments('id'); $t->string('operation_id')->unique(); $t->string('request_idempotency_key')->unique(); $t->string('request_fingerprint'); $t->unsignedBigInteger('artifact_id'); $t->unsignedBigInteger('replica_id')->nullable(); $t->unsignedBigInteger('source_backend_id')->nullable(); $t->unsignedBigInteger('target_backend_id')->nullable(); $t->string('kind'); $t->string('phase'); $t->string('outcome')->nullable(); $t->string('effect_resolution'); $t->boolean('effect_may_have_applied')->nullable(); $t->boolean('safe_to_resubmit')->nullable(); $t->unsignedInteger('attempts'); $t->string('lease_token')->nullable(); $t->unsignedBigInteger('lease_until'); $t->unsignedBigInteger('available_at'); $t->unsignedBigInteger('next_check_at'); $t->string('error_code'); $t->text('result_json')->nullable(); $t->unsignedBigInteger('terminal_at')->nullable(); $t->unsignedBigInteger('created_at'); $t->unsignedBigInteger('updated_at'); }); $db->schema()->create('cloud_storage_publications', static function (Blueprint $t): void { $t->increments('id'); $t->string('publication_key')->unique(); $t->unsignedBigInteger('artifact_id')->unique(); $t->unsignedBigInteger('backup_id')->nullable()->unique(); $t->string('template_import_intent_id')->nullable()->unique(); $t->string('rescue_template_id')->nullable()->unique(); $t->string('state'); $t->unsignedInteger('attempts')->default(0); $t->unsignedBigInteger('available_at')->default(0); $t->string('lease_token')->nullable(); $t->unsignedBigInteger('lease_until')->default(0); $t->string('last_error_code')->default(''); $t->unsignedBigInteger('coverage_ready_at')->nullable(); $t->unsignedBigInteger('finalized_at')->nullable(); $t->unsignedBigInteger('created_at'); $t->unsignedBigInteger('updated_at'); }); $db->schema()->create('cloud_instance_backups', static function (Blueprint $t): void { $t->increments('id'); $t->unsignedBigInteger('storage_artifact_id')->nullable(); $t->unsignedBigInteger('active_job_id')->nullable(); $t->string('state'); $t->string('error_code')->default(''); $t->unsignedBigInteger('updated_at')->default(0); }); $now = 2000000000; foreach ([[1, 'a', 10], [2, 'b', 20]] as [$id, $domain, $priority]) { $backend = new CloudStorageBackend(); $backend->id = $id; $backend->backend_key = str_repeat((string) $id, 32); $backend->identity_sha256 = str_repeat((string) $id, 64); $backend->name = 'provider-' . $domain; $backend->driver = 's3'; $backend->endpoint = 'https://' . $domain . '.example.test'; $backend->region = 'us-east-1'; $backend->bucket = 'bucket-' . $domain; $backend->prefix = $domain; $backend->path_style = true; $backend->failure_domain = $domain; $backend->sealed_credentials = 'sealed'; $backend->config_revision = $backend->tested_revision = 1; $backend->admin_state = 'active'; $backend->health_state = 'healthy'; $backend->write_priority = $priority; $backend->consecutive_failures = 0; $backend->consecutive_successes = 1; $backend->last_error_code = ''; $backend->last_tested_at = $backend->last_healthy_at = $now; $backend->next_probe_at = $now + 300; $backend->created_at = $backend->updated_at = $now; $backend->save(); } $newPublication = static function (int $backupId, string $suffix) use ($now): array { return CloudStorageArtifactService::beginAgentPublication( 'backup', $suffix, 20 * 1073741824, 'backup-prepare-' . $backupId, hash('sha256', 'backup-' . $backupId), ['backup_id' => $backupId] ); }; $view = $newPublication(41, 'vm-backups/a/system.qcow2'); $operationId = (string) $view['operation_id']; $assert(CloudStorageArtifactService::primaryOperationForBackup(41) === $operationId, 'Backup owner did not resolve to its original primary operation.'); $work = CloudStorageArtifactService::claimPrimaryPreparation( $operationId, $now, 120 ); $assert(is_array($work) && $work['mode'] === 'dispatch' && $work['backup_id'] === 41, 'Accepted primary preparation was not leased for dispatch.'); $assert(CloudStorageArtifactService::claimPrimaryPreparation( $operationId, $now + 1, 120 ) === null, 'A second worker crossed the active preparation lease.'); CloudStorageArtifactService::markPrimaryPreparationUnknown( $operationId, (string) $work['lease_token'], 'multipart_create_unknown', $now + 2, 5, [ 'absence_confirmations' => 1, 'multipart_absence_confirmations' => 1, ] ); $reconcile = CloudStorageArtifactService::claimPrimaryPreparation( $operationId, $now + 7, 120 ); $assert(is_array($reconcile) && $reconcile['mode'] === 'reconcile', 'Unknown CreateMultipart effect did not enter read-only reconciliation.'); $expectConflict(static fn () => CloudStorageArtifactService::reschedulePrimaryAfterStableAbsence( $operationId, (string) $reconcile['lease_token'], [ 'absence_confirmations' => 1, 'multipart_absence_confirmations' => 1, ], $now + 8, 5 ) ); CloudStorageArtifactService::reschedulePrimaryAfterStableAbsence( $operationId, (string) $reconcile['lease_token'], [ 'absence_confirmations' => 2, 'multipart_absence_confirmations' => 2, ], $now + 8, 5 ); $rescheduled = CloudStorageOperation::where( 'operation_id', $operationId )->firstOrFail(); $assert((string) $rescheduled->phase === 'accepted' && (int) $rescheduled->attempts === 1, 'Two stable absence proofs did not reopen the same operation.'); $dispatch = CloudStorageArtifactService::claimPrimaryPreparation( $operationId, $now + 13, 120 ); $metadata = [ 'upload_id' => 'upload-a', 'part_size_bytes' => 67108864, 'part_count' => 320, 'maximum_size_bytes' => 20 * 1073741824, 'multipart_presence_confirmations' => 0, ]; CloudStorageArtifactService::markPrimaryAwaitingReceipt( $operationId, (string) $dispatch['lease_token'], $metadata, $now + 14 ); $stored = (string) CloudStorageOperation::where( 'operation_id', $operationId )->value('result_json'); $assert(str_contains($stored, 'upload-a') && !str_contains($stored, 'http'), 'Primary preparation persisted a capability URL or lost upload identity.'); $expectConflict(static fn () => CloudStorageArtifactService::markPrimaryAwaitingReceipt( $operationId, (string) $dispatch['lease_token'], array_merge($metadata, ['upload_id' => 'replacement']), $now + 15 ) ); $expectConflict(static fn () => CloudStorageArtifactService::markPrimaryAwaitingReceipt( $operationId, (string) $dispatch['lease_token'], array_merge($metadata, ['part_count' => 319]), $now + 15 ) ); $expectConflict(static fn () => CloudStorageArtifactService::markPrimaryPreparationUnknown( $operationId, (string) $dispatch['lease_token'], 'multipart_create_unknown', $now + 15, 5, array_merge($metadata, ['part_count' => 319]) ) ); CloudStorageArtifactService::releasePrimaryPreparationLease( $operationId, (string) $dispatch['lease_token'], $now + 15 ); $sign = CloudStorageArtifactService::claimPrimaryPreparation( $operationId, $now + 16, 120 ); $assert(is_array($sign) && $sign['mode'] === 'sign' && ($sign['metadata']['upload_id'] ?? '') === 'upload-a', 'Persisted multipart identity was not recoverable for local re-signing.'); CloudStorageArtifactService::releasePrimaryPreparationLease( $operationId, (string) $sign['lease_token'], $now + 17 ); $expiredView = $newPublication(43, 'vm-backups/expired/system.qcow2'); $expiredId = (string) $expiredView['operation_id']; $expiredWork = CloudStorageArtifactService::claimPrimaryPreparation( $expiredId, $now + 18, 120 ); $expiredMetadata = [ 'upload_id' => 'expired-upload', 'part_size_bytes' => 67108864, 'part_count' => 320, 'maximum_size_bytes' => 20 * 1073741824, ]; CloudStorageArtifactService::markPrimaryAwaitingReceipt( $expiredId, (string) $expiredWork['lease_token'], $expiredMetadata, $now + 19 ); CloudStorageArtifactService::markPrimaryPreparationUnknown( $expiredId, (string) $expiredWork['lease_token'], 'primary_multipart_absence_unconfirmed', $now + 20, 5, $expiredMetadata + [ 'absence_confirmations' => 1, 'multipart_absence_confirmations' => 1, ] ); $expiredReconcile = CloudStorageArtifactService::claimPrimaryPreparation( $expiredId, $now + 25, 120 ); CloudStorageArtifactService::reschedulePrimaryAfterStableAbsence( $expiredId, (string) $expiredReconcile['lease_token'], $expiredMetadata + [ 'absence_confirmations' => 2, 'multipart_absence_confirmations' => 2, ], $now + 26, 5 ); $expiredOperation = CloudStorageOperation::where( 'operation_id', $expiredId )->firstOrFail(); $assert((string) $expiredOperation->phase === 'accepted' && !str_contains((string) $expiredOperation->result_json, 'expired-upload'), 'Stable object and multipart absence did not retire the old upload identity.'); $attentionView = $newPublication(42, 'vm-backups/b/system.qcow2'); $attentionId = (string) $attentionView['operation_id']; Capsule::table('cloud_instance_backups')->insert([ 'id' => 42, 'storage_artifact_id' => (int) $attentionView['artifact_id'], 'active_job_id' => null, 'state' => 'creating', ]); $attentionWork = CloudStorageArtifactService::claimPrimaryPreparation( $attentionId, $now + 20, 120 ); CloudStorageArtifactService::markPrimaryPreparationUnknown( $attentionId, (string) $attentionWork['lease_token'], 'multipart_create_unknown', $now + 21, 5, [ 'multipart_attention_confirmations' => 1, 'multipart_attention_error_code' => 'primary_multipart_identity_ambiguous', ] ); $attentionReconcile = CloudStorageArtifactService::claimPrimaryPreparation( $attentionId, $now + 26, 120 ); $expectConflict(static fn () => CloudStorageArtifactService::markPrimaryPreparationAttention( $attentionId, (string) $attentionReconcile['lease_token'], 'primary_multipart_identity_ambiguous', $now + 27, [ 'multipart_attention_confirmations' => 1, 'multipart_attention_error_code' => 'primary_multipart_identity_ambiguous', ] ) ); CloudStorageArtifactService::markPrimaryPreparationAttention( $attentionId, (string) $attentionReconcile['lease_token'], 'primary_multipart_identity_ambiguous', $now + 27, [ 'multipart_attention_confirmations' => 2, 'multipart_attention_error_code' => 'primary_multipart_identity_ambiguous', ] ); $attention = CloudStorageOperation::where( 'operation_id', $attentionId )->firstOrFail(); $attentionReplica = CloudStorageReplica::where( 'publish_operation_id', $attentionId )->firstOrFail(); $attentionBackup = CloudInstanceBackup::findOrFail(42); $assert((string) $attention->phase === 'terminal' && (string) $attention->effect_resolution === 'unknown' && !(bool) $attention->safe_to_resubmit && (string) $attentionReplica->state === 'degraded' && (string) $attentionBackup->state === 'uncertain', 'Stable multipart conflict did not stop automatic replay in attention.'); $attentionList = CloudStorageReplicaAttentionService::listView(); $attentionItem = $attentionList['items'][0] ?? []; $encodedAttention = json_encode($attentionList, JSON_THROW_ON_ERROR); $assert(($attentionList['summary']['count'] ?? 0) === 1 && ($attentionItem['operation_kind'] ?? '') === 'publish' && ($attentionItem['owner_type'] ?? '') === 'backup' && ($attentionItem['owner_id'] ?? 0) === 42 && ($attentionItem['can_retry'] ?? false) === true && !str_contains($encodedAttention, 'object_key') && !str_contains($encodedAttention, 'upload_id') && !str_contains($encodedAttention, 'part_urls'), 'Primary preparation attention was not safely exposed to administrators.'); CloudStorageReplicaAttentionService::retry($attentionId); $attention->refresh(); $attentionReplica->refresh(); $attentionBackup->refresh(); $assert((string) $attention->phase === 'reconciling' && (string) $attention->error_code === 'admin_reconcile_requested' && $attention->terminal_at === null && (string) $attentionReplica->state === 'publishing' && (string) $attentionBackup->state === 'creating', 'Administrator reconciliation did not reopen the original operation.'); $attentionBackup->state = 'uncertain'; $attentionBackup->error_code = 'stale_projection'; $attentionBackup->save(); CloudStorageArtifactService::reopenPrimaryPreparationAttention( $attentionId, $now + 28 ); $attentionBackup->refresh(); $assert((string) $attentionBackup->state === 'creating' && (string) $attentionBackup->error_code === 'admin_reconcile_requested', 'Idempotent administrator retry did not normalize the backup projection.'); $reopened = CloudStorageArtifactService::claimPrimaryPreparation( $attentionId, $now + 28, 120 ); $assert(is_array($reopened) && $reopened['mode'] === 'reconcile', 'Reopened primary attention did not return to read-only reconciliation.'); echo "Storage primary preparation tests passed.\n";