httpStatus === 409, $message . ' (wrong status)'); return; } throw new RuntimeException($message); }; // The coordinator fixture deliberately carries only fields it needs. Extend // the same in-memory job table with the worker envelope columns exercised by // this stage; production schema already owns these columns. $schema->table('cloud_jobs', static function (Blueprint $t): void { $t->string('request_idempotency_key')->default(''); $t->string('request_fingerprint', 64)->default(''); $t->integer('instance_id')->nullable(); $t->integer('parent_job_id')->nullable(); $t->integer('vpc_repair_root_job_id')->nullable(); $t->string('execution_mode')->default('agent'); $t->string('status')->default('queued'); $t->text('sealed_payload')->default(''); $t->integer('attempts')->default(0); $t->integer('dispatch_attempts')->default(0); $t->integer('dispatch_deadline_at')->default(0); $t->integer('dispatch_started_at')->nullable(); $t->integer('agent_accepted_at')->nullable(); $t->integer('receipt_deadline_at')->nullable(); $t->integer('effect_may_have_applied')->nullable(); $t->integer('safe_to_resubmit')->nullable(); $t->integer('effect_resolved_at')->nullable(); $t->integer('effect_resolved_by')->nullable(); $t->string('effect_resolution_reason')->default(''); $t->string('finalization_error_code')->nullable(); $t->string('completion_event_id')->nullable(); $t->integer('read_at')->nullable(); $t->integer('dispatched_at')->nullable(); $t->string('lease_token')->nullable(); $t->integer('lease_until')->default(0); $t->integer('available_at')->default(0); $t->string('error_code')->nullable(); $t->text('result_json')->nullable(); $t->integer('created_at')->default(0); $t->integer('updated_at')->default(0); }); $schema->create('cloud_agent_completion_events', static function ( Blueprint $t ): void { $t->increments('id'); $t->integer('host_id'); $t->string('operation_id'); }); $schema->create('cloud_storage_backends', static function ( Blueprint $t ): void { $t->increments('id'); $t->string('backend_key'); $t->string('identity_sha256'); $t->string('name'); $t->string('driver'); $t->string('endpoint'); $t->string('region'); $t->string('bucket'); $t->string('prefix'); $t->boolean('path_style'); $t->string('failure_domain'); $t->text('sealed_credentials'); $t->integer('config_revision'); $t->integer('tested_revision'); $t->string('admin_state'); $t->string('health_state'); $t->integer('write_priority'); $t->integer('consecutive_failures')->default(0); $t->integer('consecutive_successes')->default(0); $t->string('last_error_code')->default(''); $t->integer('last_tested_at')->default(0); $t->integer('last_healthy_at')->default(0); $t->integer('next_probe_at')->default(0); $t->integer('created_at'); $t->integer('updated_at'); }); $schema->create('cloud_storage_replicas', static function ( Blueprint $t ): void { $t->increments('id'); $t->string('replica_key'); $t->integer('artifact_id'); $t->integer('backend_id'); $t->string('role'); $t->string('object_key'); $t->string('object_version_id')->default(''); $t->string('object_etag')->default(''); $t->integer('size_bytes'); $t->string('sha256', 64); $t->string('state'); $t->string('publish_operation_id')->nullable(); $t->string('delete_operation_id')->nullable(); $t->integer('delete_generation')->default(0); $t->integer('delete_cycle_attempts')->default(0); $t->integer('delete_presence_confirmations')->default(0); $t->integer('delete_last_checked_at')->default(0); $t->integer('delete_next_check_at')->default(0); $t->integer('delete_attention_at')->nullable(); $t->string('lease_token')->nullable(); $t->integer('lease_until')->default(0); $t->string('error_code')->default(''); $t->integer('verified_at'); $t->integer('created_at'); $t->integer('updated_at'); }); Capsule::table('sqlite_sequence')->where('name', 'cloud_jobs')->update([ 'seq' => 600, ]); global $System_Config; $System_Config['cloud_data_key'] = base64_encode(str_repeat('r', 32)); $runtime = [ 'config' => [ 'endpoint' => 'https://objects.example.com', 'region' => 'us-east-1', 'bucket' => 'portable-backups', 'prefix' => 'virt', 'path_style' => true, ], 'credentials' => [ 'access_key_id' => 'RESTOREACCESS', 'secret_access_key' => 'restore-secret-value', ], 'revision' => 7, ]; Capsule::table('cloud_storage_backends')->insert([ 'id' => 71, 'backend_key' => str_repeat('1', 32), 'identity_sha256' => str_repeat('2', 64), 'name' => 'portable-primary', 'driver' => 's3', 'endpoint' => 'https://objects.example.com', 'region' => 'us-east-1', 'bucket' => 'portable-backups', 'prefix' => 'virt', 'path_style' => 1, 'failure_domain' => 'portable-primary', 'sealed_credentials' => CloudSecrets::encrypt( $runtime['credentials'], 'object-storage-backend-credentials' ), 'config_revision' => 1, 'tested_revision' => 1, 'admin_state' => 'active', 'health_state' => 'healthy', 'write_priority' => 10, 'consecutive_failures' => 0, 'consecutive_successes' => 1, 'last_error_code' => '', 'last_tested_at' => time(), 'last_healthy_at' => time(), 'next_probe_at' => time() + 300, 'created_at' => time(), 'updated_at' => time(), ]); $sourceBackupRow = Capsule::table('cloud_instance_backups') ->where('id', 1)->first(); $assert(is_object($sourceBackupRow), 'Portable source backup is missing.'); Capsule::table('cloud_storage_artifacts')->insert([ 'id' => 83, 'artifact_key' => str_repeat('3', 32), 'kind' => 'backup', 'maximum_size_bytes' => 2048, 'size_bytes' => (int) $sourceBackupRow->object_size_bytes, 'sha256' => (string) $sourceBackupRow->sha256, 'durability_policy' => 'mirror_async_2', 'desired_copies' => 2, 'required_copies' => 1, 'state' => 'available', 'last_operation_id' => str_repeat('4', 32), 'verified_at' => time(), 'created_at' => time(), 'updated_at' => time(), ]); Capsule::table('cloud_storage_publications')->insert([ 'artifact_id' => 83, 'backup_id' => 1, 'state' => 'ready_to_finalize', 'created_at' => time(), 'updated_at' => time(), ]); Capsule::table('cloud_storage_replicas')->insert([ 'id' => 73, 'replica_key' => str_repeat('5', 32), 'artifact_id' => 83, 'backend_id' => 71, 'role' => 'primary', 'object_key' => (string) $sourceBackupRow->object_key, 'object_version_id' => '', 'object_etag' => '', 'size_bytes' => (int) $sourceBackupRow->object_size_bytes, 'sha256' => (string) $sourceBackupRow->sha256, 'state' => 'ready', 'publish_operation_id' => str_repeat('6', 32), 'delete_operation_id' => null, 'delete_generation' => 0, 'delete_cycle_attempts' => 0, 'delete_presence_confirmations' => 0, 'delete_last_checked_at' => 0, 'delete_next_check_at' => 0, 'delete_attention_at' => null, 'lease_token' => null, 'lease_until' => 0, 'error_code' => '', 'verified_at' => time(), 'created_at' => time(), 'updated_at' => time(), ]); Capsule::table('cloud_instance_backups')->where('id', 1)->update([ 'storage_artifact_id' => 83, 'storage_revision' => 0, ]); $backup = CloudInstanceBackup::findOrFail(1); $plans = CloudPortableRestoreTargetService::candidates($backup); $plan = array_values(array_filter( $plans, static fn (CloudPortableRestoreTargetPlan $candidate): bool => $candidate->toArray()['target']['host_id'] === 3 && $candidate->toArray()['target']['resource_pool_id'] === 300 ))[0] ?? null; $assert($plan instanceof CloudPortableRestoreTargetPlan, 'Fresh portable restore dispatch plan is unavailable.'); $requestKey = 'dispatch-request-0000000000000001'; $reserved = CloudPortableRestoreAttemptCoordinator::reserve( $actor, $backup, $plan, $plan->sha256(), 'dispatch-vm', $requestKey )->toArray(); $attempt = CloudBackupRestoreAttempt::where( 'attempt_key', $reserved['attempt_key'] )->firstOrFail(); $reservation = CloudResourceReservation::findOrFail( $reserved['reservation_id'] ); $jobCount = CloudJob::count(); $instanceCount = Capsule::table('cloud_instances')->count(); $claimSnapshot = Capsule::table('cloud_resource_reservation_ips') ->where('reservation_id', (int) $reservation->id) ->orderBy('address_family')->get()->map( static fn ($row): array => (array) $row )->all(); // Any torn input must roll back without an orphan job or changing the attempt. $reservation->operation_id = str_repeat('7', 32); $reservation->save(); $rejects409( static fn () => CloudPortableRestoreJobDispatcher::queue( $actor, (string) $attempt->attempt_key ), 'Dispatcher accepted a torn reservation operation.' ); $assert(CloudJob::count() === $jobCount && (string) CloudBackupRestoreAttempt::findOrFail($attempt->id)->state === 'reserved', 'Rejected dispatch left a job or changed attempt state.'); $reservation->operation_id = (string) $attempt->operation_id; $reservation->save(); $restoreBeforeQueue = CloudBackupRestore::findOrFail( (int) $attempt->backup_restore_id ); $restoreBeforeQueue->target_instance_id = 777; $restoreBeforeQueue->save(); $rejects409( static fn () => CloudPortableRestoreJobDispatcher::queue( $actor, (string) $attempt->attempt_key ), 'Dispatcher accepted a pre-created target instance.' ); $restoreBeforeQueue->target_instance_id = null; $restoreBeforeQueue->save(); foreach (['backup_restore_id' => 777, 'order_id' => 777] as $field => $value) { $reservation->{$field} = $value; $reservation->save(); $rejects409( static fn () => CloudPortableRestoreJobDispatcher::queue( $actor, (string) $attempt->attempt_key ), 'Dispatcher accepted a doubly-owned reservation: ' . $field . '.' ); $reservation->{$field} = null; $reservation->save(); } $assert(CloudJob::count() === $jobCount && (string) CloudBackupRestoreAttempt::findOrFail($attempt->id)->state === 'reserved', 'Rejected ownership tear left a job or changed attempt state.'); $ipv6Claim = Capsule::table('cloud_resource_reservation_ips') ->where('reservation_id', (int) $reservation->id) ->where('address_family', 6)->first(); $assert(is_object($ipv6Claim), 'IPv6 claim fixture is missing.'); Capsule::table('cloud_resource_reservation_ips') ->where('id', (int) $ipv6Claim->id) ->update(['operation_id' => str_repeat('6', 32)]); $rejects409( static fn () => CloudPortableRestoreJobDispatcher::queue( $actor, (string) $attempt->attempt_key ), 'Dispatcher accepted a torn IPv6 claim.' ); $assert(CloudJob::count() === $jobCount && Capsule::table('cloud_instances')->count() === $instanceCount, 'Rejected claim dispatch produced a job or instance.'); Capsule::table('cloud_resource_reservation_ips') ->where('id', (int) $ipv6Claim->id) ->update(['operation_id' => (string) $attempt->operation_id]); $originalPlanSha = (string) $attempt->target_plan_sha256; $attempt->target_plan_sha256 = str_repeat('5', 64); $attempt->save(); $rejects409( static fn () => CloudPortableRestoreJobDispatcher::queue( $actor, (string) $attempt->attempt_key ), 'Dispatcher accepted a torn target plan.' ); $attempt->target_plan_sha256 = $originalPlanSha; $attempt->save(); $job = CloudPortableRestoreJobDispatcher::queue( $actor, (string) $attempt->attempt_key ); $attempt = CloudBackupRestoreAttempt::findOrFail($attempt->id); $restore = CloudBackupRestore::findOrFail($attempt->backup_restore_id); $reservation = CloudResourceReservation::findOrFail($reservation->id); $assert((int) $job->id > 0 && (int) $job->user_id === (int) $backup->user_id && (int) $job->backup_id === (int) $backup->id && (int) $job->backup_restore_id === (int) $restore->id && (int) $job->host_id === (int) $attempt->target_host_id && (string) $job->kind === 'backup_restore' && (string) $job->execution_mode === 'agent' && (string) $job->phase === 'accepted' && (string) $job->status === 'queued' && hash_equals((string) $job->operation_id, (string) $attempt->operation_id) && hash_equals((string) $job->request_idempotency_key, (string) $attempt->attempt_key) && (string) $attempt->state === 'queued' && (int) $attempt->job_id === (int) $job->id && (string) $attempt->effect_resolution === 'not_applicable' && (string) $reservation->state === 'held' && Capsule::table('cloud_instances')->count() === $instanceCount, 'Reserved restore attempt was not atomically converted into one queued job.'); $assert((int) $restore->job_id === (int) $job->id, 'Generation-one job projection was not written to the parent restore.'); $payload = CloudSecrets::decrypt((string) $job->sealed_payload, 'job-payload'); $body = is_array($payload['body'] ?? null) ? $payload['body'] : []; $public = is_array($body['public_network'] ?? null) ? $body['public_network'] : []; $assert(($payload['method'] ?? null) === 'POST' && ($payload['request_target'] ?? null) === '/v1/vms/' . rawurlencode((string) $attempt->target_uuid) . '/restore-backup' && ($payload['attempt_key'] ?? null) === (string) $attempt->attempt_key && (int) ($payload['generation'] ?? 0) === (int) $attempt->generation && ($payload['target_plan_sha256'] ?? null) === (string) $attempt->target_plan_sha256 && ($payload['target_claim_set_sha256'] ?? null) === (string) $attempt->target_claim_set_sha256 && ($payload['target_public_network_sha256'] ?? null) === (string) $attempt->target_public_network_sha256 && ($payload['owner_id'] ?? null) === (int) $backup->user_id && ($body['uuid'] ?? null) === strtolower((string) $attempt->target_uuid) && ($body['name'] ?? null) === (string) $attempt->target_name && ($body['backup_key'] ?? null) === (string) $backup->backup_key && ($body['source_size_bytes'] ?? null) === (int) $backup->object_size_bytes && ($body['source_sha256'] ?? null) === (string) $backup->sha256 && !array_key_exists('source_url', $body) && ($body['source'] ?? null) === '' && ($body['template_id'] ?? null) === '' && ($body['template_revision'] ?? null) === 0 && ($body['template_generation_token'] ?? null) === '' && ($body['startup_script'] ?? null) === '' && ($body['hostname'] ?? null) === '' && ($body['guest_ssh_public_key'] ?? null) === '' && ($body['root_password'] ?? null) === '' && ($body['debian_12_network_fix'] ?? null) === false && is_array($body['platform_network_policy_snapshot'] ?? null) && ($public['contract'] ?? null) === 1 && is_array($public['ipv4'] ?? null) && is_array($public['ipv6'] ?? null), 'Queued restore payload lost immutable backup or dual-stack request identity.'); foreach (['network', 'public_pool_key', 'public_pool_cidr', 'public_ip_cidr', 'gateway', 'dns_servers', 'vpc_port'] as $legacy) { $assert(!array_key_exists($legacy, $body), 'Queued restore body retained legacy or VPC field: ' . $legacy); } $encodedPayload = json_encode( $payload, JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES ); foreach (['restore-secret-value', 'RESTOREACCESS', 'sealed_storage_runtime', 'object_key', 'source_url', 'X-Amz-', 'vpc_port_id'] as $secret) { $assert(!str_contains($encodedPayload, $secret), 'Queued restore payload leaked long-lived or unrelated data: ' . $secret); } $secretCheck = new ReflectionMethod( CloudPortableRestoreJobEnvelope::class, 'assertSecretFree' ); $legalNamePayload = $payload; $legalNamePayload['body']['name'] = 'secret_access_key'; $secretCheck->invoke(null, $legalNamePayload); // A bounded capability belongs only to the one Agent wire request. The worker // persists only the exact Journal fingerprint before crossing the dispatch // CAS, never the URL itself. $prepareWire = new ReflectionMethod( CloudWorkerService::class, 'portableRestoreDispatchPayload' ); [$stableDispatchPayload, $wirePayload] = $prepareWire->invoke( null, $job, $payload ); $assert(str_starts_with( (string) $wirePayload['body']['source_url'], 'https://' ) && str_contains( (string) $wirePayload['body']['source_url'], 'X-Amz-Signature=' ), 'Dispatch-time wire fixture did not contain a bounded download capability.'); $stableEncoded = json_encode( $stableDispatchPayload, JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES ); $persistedIdentity = $stableDispatchPayload[ CloudOperationRecoveryIdentity::payloadKey() ] ?? null; $assert(is_array($persistedIdentity) && !str_contains($stableEncoded, 'source_url') && !str_contains($stableEncoded, 'X-Amz-') && CloudOperationRecoveryIdentity::same( CloudOperationRecoveryIdentity::expectedFromPayload( $job, $wirePayload ), $persistedIdentity ), 'Worker did not separate the stable Journal identity from the wire capability.'); // Reconciliation must recover the exact wire fingerprint from stable durable // state after dispatch, without retaining or recreating the expired URL. $originalPhase = (string) $job->phase; $originalDispatchAttempts = (int) $job->dispatch_attempts; $originalSealed = (string) $job->sealed_payload; $job->phase = 'dispatching'; $job->dispatch_attempts = 1; $job->sealed_payload = CloudSecrets::encrypt( $stableDispatchPayload, 'job-payload' ); $assert(CloudOperationRecoveryIdentity::same( CloudOperationRecoveryIdentity::expected($job), $persistedIdentity ), 'Dispatched restore identity could not be rebuilt without its source URL.'); // Recovery authority follows the attempt ledger after dispatch and never // requires the discarded URL. Each non-terminal state keeps the same stable // Agent identity while enforcing the matching resource lifecycle. foreach ([ ['awaiting_receipt', 'not_applicable', 'held', 'reserved'], ['effect_unknown', 'unknown', 'uncertain', 'uncertain'], ['finalization_pending', 'applied', 'uncertain', 'uncertain'], ] as [$attemptState, $effect, $reservationState, $claimState]) { $evidenceJson = match ($attemptState) { 'effect_unknown' => json_encode( ['effect_may_have_applied' => true], JSON_THROW_ON_ERROR ), 'finalization_pending' => json_encode( ['proof' => 'validated'], JSON_THROW_ON_ERROR ), default => null, }; Capsule::table('cloud_backup_restore_attempts') ->where('id', (int) $attempt->id)->update([ 'state' => $attemptState, 'effect_resolution' => $effect, 'evidence_resolution' => $evidenceJson === null ? 'not_applicable' : $effect, 'evidence_json' => $evidenceJson, 'evidence_sha256' => $evidenceJson === null ? null : hash('sha256', $evidenceJson), 'evidence_recorded_at' => $evidenceJson === null ? null : time(), 'effect_resolved_at' => $attemptState === 'finalization_pending' ? time() : null, ]); Capsule::table('cloud_resource_reservations') ->where('id', (int) $reservation->id)->update([ 'state' => $reservationState, ]); Capsule::table('cloud_resource_reservation_ips') ->where('reservation_id', (int) $reservation->id)->update([ 'state' => $claimState, ]); Capsule::table('cloud_public_ips') ->where('reservation_id', (int) $reservation->id)->update([ 'state' => $claimState, ]); $assert(CloudOperationRecoveryIdentity::same( CloudOperationRecoveryIdentity::expected($job), $persistedIdentity ), 'Recovery identity rejected valid attempt state: ' . $attemptState); } $tamperedRecoveryPayload = $stableDispatchPayload; $tamperedRecoveryPayload['generation']++; $job->sealed_payload = CloudSecrets::encrypt( $tamperedRecoveryPayload, 'job-payload' ); $rejects409( static fn () => CloudOperationRecoveryIdentity::expected($job), 'Recovery identity accepted a changed attempt generation.' ); $originalAttemptOperation = (string) $attempt->operation_id; Capsule::table('cloud_backup_restore_attempts') ->where('id', (int) $attempt->id)->update([ 'operation_id' => str_repeat('f', 32), ]); $rejects409( static fn () => CloudOperationRecoveryIdentity::expected($job), 'Portable ownership downgraded to legacy after an operation mismatch.' ); Capsule::table('cloud_backup_restore_attempts') ->where('id', (int) $attempt->id)->update([ 'operation_id' => $originalAttemptOperation, ]); Capsule::table('cloud_backup_restore_attempts') ->where('id', (int) $attempt->id)->update([ 'state' => 'queued', 'effect_resolution' => 'not_applicable', 'evidence_resolution' => 'not_applicable', 'evidence_json' => null, 'evidence_sha256' => null, 'evidence_recorded_at' => null, 'effect_resolved_at' => null, ]); Capsule::table('cloud_resource_reservations') ->where('id', (int) $reservation->id)->update(['state' => 'held']); Capsule::table('cloud_resource_reservation_ips') ->where('reservation_id', (int) $reservation->id)->update([ 'state' => 'reserved', ]); Capsule::table('cloud_public_ips') ->where('reservation_id', (int) $reservation->id)->update([ 'state' => 'reserved', ]); $job->sealed_payload = CloudSecrets::encrypt( $stableDispatchPayload, 'job-payload' ); // Admin inspection uses the typed dual-stack projection for attempt-owned // restores while preserving the scalar contract only for legacy jobs. $inspectProjection = new ReflectionMethod( CloudAdminOperationRecoveryService::class, 'inspection' ); $inspectionBase = [ 'operation_id' => (string) $job->operation_id, 'action' => $persistedIdentity['action'], 'resource_id' => $persistedIdentity['resource_id'], 'request_fingerprint' => $persistedIdentity['request_fingerprint'], 'operation_status' => 'succeeded', 'checked_at' => time(), 'outcome' => 'succeeded', 'summary_code' => 'operation_succeeded', 'effect_resolution' => 'applied', 'fence_state' => 'not_required', 'fence_resolution_required' => false, 'abort_job' => ['available' => false], 'cleanup' => ['available' => false], ]; $rejectsInspection = static function (callable $callback, string $message) use ( $assert ): void { try { $callback(); } catch (CloudAgentError $error) { $assert($error->httpStatus === 502, $message . ' (wrong status)'); return; } throw new RuntimeException($message); }; $portableProjection = [ 'uuid' => strtolower((string) $attempt->target_uuid), 'name' => (string) $attempt->target_name, 'vcpu' => (int) $body['vcpu'], 'memory_mib' => (int) $body['memory_mib'], 'disk_gib' => (int) $body['disk_gib'], 'bandwidth_mbps' => (int) $body['bandwidth_mbps'], 'public_network' => $public, 'machine' => 'q35', 'firmware' => 'uefi', 'secure_boot' => false, 'backup_key' => (string) $backup->backup_key, 'sha256' => (string) $backup->sha256, ]; $portableInspection = $inspectProjection->invoke( null, $inspectionBase + ['effect_projection' => $portableProjection], (string) $job->operation_id, 'backup_restore', true, false, true ); $assert(($portableInspection['effect_projection']['public_network'] ?? null) === $public, 'Portable restore inspection lost its typed dual-stack projection.'); $legacyScalarProjection = [ 'uuid' => strtolower((string) $attempt->target_uuid), 'public_ip_cidr' => (string) $public['ipv4']['address_cidr'], 'public_pool_key' => str_repeat('a', 32), 'backup_key' => (string) $backup->backup_key, 'sha256' => (string) $backup->sha256, ]; $rejectsInspection( static fn () => $inspectProjection->invoke( null, $inspectionBase + ['effect_projection' => $legacyScalarProjection], (string) $job->operation_id, 'backup_restore', true, false, true ), 'Portable restore inspection accepted an IPv4 scalar-only projection.' ); $rejectsInspection( static fn () => $inspectProjection->invoke( null, $inspectionBase + ['effect_projection' => $legacyScalarProjection], (string) $job->operation_id, 'backup_restore', true, false, false ), 'Legacy restore inspection retained an IPv4 scalar-only projection.' ); $notAppliedInspection = $inspectionBase; $notAppliedInspection['operation_status'] = 'failed'; $notAppliedInspection['outcome'] = 'failed'; $notAppliedInspection['summary_code'] = 'operation_failed'; $notAppliedInspection['effect_resolution'] = 'not_applied'; $notAppliedInspection['effect_projection'] = ['creation_absent' => true]; $rejectsInspection( static fn () => $inspectProjection->invoke( null, $notAppliedInspection, (string) $job->operation_id, 'backup_restore', true, false, true ), 'Portable not-applied inspection accepted incomplete absence evidence.' ); $finishIdentity = new ReflectionMethod( CloudWorkerService::class, 'operationIdentityForFinalization' ); $assert(CloudOperationRecoveryIdentity::same( $finishIdentity->invoke( null, $job, $stableDispatchPayload, false, false ), $persistedIdentity ), 'Normal restore finalization required the discarded source URL.'); $job->phase = $originalPhase; $job->dispatch_attempts = $originalDispatchAttempts; $job->sealed_payload = $originalSealed; // Prepared recovery must compact the transient wire fixture while retaining // the exact restore-attempt identity needed for convergence. $compact = new ReflectionMethod( CloudWorkerService::class, 'compactPreparedRecoveryPayload' ); $prepared = $compact->invoke(null, 'backup_restore', $wirePayload); $preparedEncoded = json_encode( $prepared, JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES ); $assert(($prepared['attempt_key'] ?? null) === (string) $attempt->attempt_key && (int) ($prepared['generation'] ?? 0) === (int) $attempt->generation && ($prepared['target_plan_sha256'] ?? null) === (string) $attempt->target_plan_sha256 && ($prepared['target_claim_set_sha256'] ?? null) === (string) $attempt->target_claim_set_sha256 && ($prepared['target_public_network_sha256'] ?? null) === (string) $attempt->target_public_network_sha256 && !str_contains($preparedEncoded, 'source_url') && !str_contains($preparedEncoded, 'X-Amz-') && !str_contains($preparedEncoded, 'RESTOREACCESS') && !str_contains($preparedEncoded, 'restore-secret-value'), 'Prepared recovery lost restore identity or retained a download capability.'); $replay = CloudPortableRestoreJobDispatcher::queue( $actor, (string) $attempt->attempt_key ); $assert((int) $replay->id === (int) $job->id && CloudJob::count() === $jobCount + 1 && Capsule::table('cloud_instances')->count() === $instanceCount && Capsule::table('cloud_resource_reservation_ips') ->where('reservation_id', (int) $reservation->id) ->orderBy('address_family')->get()->map( static fn ($row): array => (array) $row )->all() === $claimSnapshot, 'Exact queue replay created another job, instance, or changed claims.'); // Once queued, every durable identity must still be checked before returning // the existing job; replay cannot hide a torn cross-table association. $jobMutations = [ ['operation_id', str_repeat('4', 32), (string) $job->operation_id], ['host_id', 1, (int) $job->host_id], ['user_id', 99, (int) $job->user_id], ['backup_id', 999, (int) $job->backup_id], ['backup_restore_id', 999, (int) $job->backup_restore_id], ]; foreach ($jobMutations as [$field, $invalid, $valid]) { Capsule::table('cloud_jobs')->where('id', (int) $job->id) ->update([$field => $invalid]); $rejects409( static fn () => CloudPortableRestoreJobDispatcher::queue( $actor, (string) $attempt->attempt_key ), 'Queue replay accepted torn job identity: ' . $field . '.' ); $assert(CloudJob::count() === $jobCount + 1, 'Rejected queue replay created another job.'); Capsule::table('cloud_jobs')->where('id', (int) $job->id) ->update([$field => $valid]); } // The remaining fault-injection cases run in an outer rollback so this file // remains a reusable queued-attempt fixture for the finalizer test suite. Capsule::connection()->beginTransaction(); try { // Close the valid envelope fixture with exact local absence evidence so its // addresses can be reused by the independent pre-dispatch failure cases. $finish = new ReflectionMethod(CloudWorkerService::class, 'finish'); $finish->invoke( new CloudWorkerService(), $job, [ 'operation_id' => (string) $job->operation_id, 'status' => 'failed', 'result' => [ 'creation_absent' => true, 'managed_artifacts_absent' => true, ], 'effect_may_have_applied' => false, 'safe_to_resubmit' => true, ], $payload ); $assert((string) CloudJob::findOrFail((int) $job->id)->effect_resolution === 'not_applied' && (string) CloudResourceReservation::findOrFail( (int) $reservation->id )->state === 'released', 'Valid never-dispatched restore fixture did not release its dual stack.'); // Exercise the public Worker seam rather than its private admission helper: // a retired restore payload without an attempt identity must fail locally, // before Agent contact, without changing any canonical resource ledger. $missingAttemptPayload = $payload; unset($missingAttemptPayload['attempt_key']); $retiredRestoreJob = new CloudJob(); $retiredRestoreJob->operation_id = bin2hex(random_bytes(16)); $retiredRestoreJob->request_idempotency_key = 'retired-restore-worker-gate-0001'; $retiredRestoreJob->request_fingerprint = hash( 'sha256', 'retired-restore-worker-gate-0001' ); $retiredRestoreJob->user_id = (int) $job->user_id; $retiredRestoreJob->host_id = (int) $job->host_id; $retiredRestoreJob->instance_id = null; $retiredRestoreJob->backup_id = (int) $job->backup_id; $retiredRestoreJob->backup_restore_id = (int) $job->backup_restore_id; $retiredRestoreJob->kind = 'backup_restore'; $retiredRestoreJob->execution_mode = 'agent'; $retiredRestoreJob->sealed_payload = CloudSecrets::encrypt( $missingAttemptPayload, 'job-payload' ); $retiredRestoreJob->available_at = $retiredRestoreJob->created_at = $retiredRestoreJob->updated_at = time(); $retiredRestoreJob->save(); $retiredRestoreIpLedger = Capsule::table('cloud_public_ips') ->orderBy('id')->get([ 'id', 'state', 'reservation_id', 'instance_id', ])->map(static fn ($row): array => (array) $row)->all(); $retiredRestoreClaimLedger = Capsule::table( 'cloud_resource_reservation_ips' )->orderBy('id')->get([ 'id', 'state', 'reservation_id', 'public_ip_id', 'instance_id', ])->map(static fn ($row): array => (array) $row)->all(); $retiredRestoreAgentCalls = 0; $retiredRestoreTransport = new \GuzzleHttp\Client([ 'handler' => static function () use (&$retiredRestoreAgentCalls): void { $retiredRestoreAgentCalls++; throw new RuntimeException('Retired restore reached Agent.'); }, ]); $retiredRestoreAgent = new CloudAgentClient( '3', str_repeat('a', 32), $retiredRestoreTransport, 20.0, 'http://127.0.0.1:8001/internal/agent/request', str_repeat('b', 32) ); $assert((new CloudWorkerService($retiredRestoreAgent))->tick() === true, 'Worker did not consume the retired restore job.'); $retiredRestoreJob->refresh(); $assert($retiredRestoreAgentCalls === 0 && (int) $retiredRestoreJob->dispatch_attempts === 0 && (string) $retiredRestoreJob->phase === 'terminal' && (string) $retiredRestoreJob->outcome === 'failed' && Capsule::table('cloud_public_ips')->orderBy('id')->get([ 'id', 'state', 'reservation_id', 'instance_id', ])->map(static fn ($row): array => (array) $row)->all() === $retiredRestoreIpLedger && Capsule::table('cloud_resource_reservation_ips') ->orderBy('id')->get([ 'id', 'state', 'reservation_id', 'public_ip_id', 'instance_id', ])->map(static fn ($row): array => (array) $row)->all() === $retiredRestoreClaimLedger, 'Retired restore reached Agent, remained claimable, or changed the ' . 'canonical address ledgers.'); // The restore remains artifact-backed. Corrupting the selected backend's // credential envelope after reservation makes capability minting fail at // the local pre-dispatch boundary without reviving any backup-owned secret. $failurePlan = $plan; $failureReserved = CloudPortableRestoreAttemptCoordinator::reserve( $actor, $backup, $failurePlan, $failurePlan->sha256(), 'predispatch-vm', 'predispatch-request-0000000001' )->toArray(); $failureJob = CloudPortableRestoreJobDispatcher::queue( $actor, (string) $failureReserved['attempt_key'] ); $failureAttempt = CloudBackupRestoreAttempt::where( 'attempt_key', (string) $failureReserved['attempt_key'] )->firstOrFail(); $failureReservation = CloudResourceReservation::findOrFail( (int) $failureReserved['reservation_id'] ); $failureRestore = CloudBackupRestore::findOrFail( (int) $failureAttempt->backup_restore_id ); // A local capability-minting failure happens before the immutable dispatch // CAS. It must never contact Agent; its exact not-dispatched evidence must // atomically release both public-address claims and reserved host capacity. $agentCalls = 0; $tripwireTransport = new \GuzzleHttp\Client([ 'handler' => static function () use (&$agentCalls): void { $agentCalls++; throw new RuntimeException( 'Local portable-restore failure reached Agent.' ); }, ]); $tripwireAgent = new CloudAgentClient( '3', str_repeat('a', 32), $tripwireTransport, 20.0, 'http://127.0.0.1:8001/internal/agent/request', str_repeat('b', 32) ); $worker = new CloudWorkerService($tripwireAgent); $tickJob = new ReflectionMethod(CloudWorkerService::class, 'tickJob'); $validBackendCredentials = (string) Capsule::table( 'cloud_storage_backends' )->where('id', 71)->value('sealed_credentials'); Capsule::table('cloud_storage_backends')->where('id', 71)->update([ 'sealed_credentials' => 'invalid-backend-credential-envelope', ]); try { $assert($tickJob->invoke($worker) === true, 'Worker did not consume the local pre-dispatch restore failure.'); } finally { Capsule::table('cloud_storage_backends')->where('id', 71)->update([ 'sealed_credentials' => $validBackendCredentials, ]); } $failedJob = CloudJob::findOrFail((int) $failureJob->id); $failedAttempt = CloudBackupRestoreAttempt::findOrFail( (int) $failureAttempt->id ); $failedReservation = CloudResourceReservation::findOrFail( (int) $failureReservation->id ); $failedRestore = CloudBackupRestore::findOrFail((int) $failureRestore->id); $releasedClaims = Capsule::table('cloud_resource_reservation_ips') ->where('reservation_id', (int) $failureReservation->id) ->orderBy('address_family')->get(); $releasedAddresses = Capsule::table('cloud_public_ips') ->whereIn('id', $releasedClaims->pluck('public_ip_id')->all()) ->orderBy('address_family')->get(); $durablePayload = (string) $failedJob->sealed_payload === '' ? [] : CloudSecrets::decrypt( (string) $failedJob->sealed_payload, 'job-payload' ); $assert($agentCalls === 0 && (string) $failedJob->phase === 'terminal' && (string) $failedJob->outcome === 'failed' && (int) $failedJob->dispatch_attempts === 0 && (string) $failedJob->effect_resolution === 'not_applied' && (int) $failedJob->finalization_incident === 0 && (string) $failedAttempt->state === 'failed' && $failedAttempt->active_attempt === null && (string) $failedReservation->state === 'released' && (string) $failedRestore->state === 'failed' && $releasedClaims->count() === 2 && $releasedClaims->every(static fn ($claim): bool => (string) $claim->state === 'released') && $releasedAddresses->count() === 2 && $releasedAddresses->every(static fn ($address): bool => (string) $address->state === 'available' && $address->reservation_id === null && $address->instance_id === null) && !str_contains(json_encode( $durablePayload, JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES ), 'source_url'), 'Local pre-dispatch failure contacted Agent, retained a dual-stack claim, ' . 'or persisted its transient capability.'); // Corrupting one immutable envelope field is not an ordinary local failure. // It must stop before Agent, retain every resource fence and publish an // operator-visible finalization incident instead of releasing addresses. $tornPlan = $plan; $tornReserved = CloudPortableRestoreAttemptCoordinator::reserve( $actor, $backup, $tornPlan, $tornPlan->sha256(), 'torn-envelope-vm', 'torn-envelope-request-000000001' )->toArray(); $tornJob = CloudPortableRestoreJobDispatcher::queue( $actor, (string) $tornReserved['attempt_key'] ); $tornPayload = CloudSecrets::decrypt( (string) $tornJob->sealed_payload, 'job-payload' ); $tornPayload['generation'] = (int) $tornPayload['generation'] + 1; $tornJob->sealed_payload = CloudSecrets::encrypt( $tornPayload, 'job-payload' ); $tornJob->save(); $assert($tickJob->invoke($worker) === true, 'Worker did not consume the torn portable-restore envelope.'); $tornJob->refresh(); $tornAttempt = CloudBackupRestoreAttempt::where( 'attempt_key', (string) $tornReserved['attempt_key'] )->firstOrFail(); $tornReservation = CloudResourceReservation::findOrFail( (int) $tornReserved['reservation_id'] ); $tornClaims = Capsule::table('cloud_resource_reservation_ips') ->where('reservation_id', (int) $tornReservation->id)->get(); $assert($agentCalls === 0 && (string) $tornJob->phase === 'terminal' && (string) $tornJob->outcome === 'failed' && (int) $tornJob->dispatch_attempts === 0 && (string) $tornJob->effect_resolution === 'not_applied' && (int) $tornJob->finalization_incident === 1 && (string) $tornAttempt->state === 'queued' && (int) $tornAttempt->active_attempt === 1 && (string) $tornReservation->state === 'held' && $tornClaims->count() === 2 && $tornClaims->every(static fn ($claim): bool => (string) $claim->state === 'reserved') && Capsule::table('cloud_instances')->count() === $instanceCount, 'Torn restore envelope reached Agent or released fenced dual-stack state.'); } finally { Capsule::connection()->rollBack(); } echo "Portable restore job dispatcher behavior tests passed.\n";