table('cloud_instances', static function (Blueprint $t): void { $t->string('uuid')->nullable(); $t->integer('order_id')->nullable(); $t->integer('plan_id')->nullable(); $t->integer('offer_id')->nullable(); $t->integer('current_period_id')->nullable(); $t->string('service_status')->default('exempt'); $t->string('network_status')->default('active'); $t->text('sealed_credential')->nullable(); $t->integer('expires_at')->nullable(); $t->decimal('renewal_price', 12, 2)->nullable(); $t->integer('renewal_days')->default(0); $t->integer('renewal_traffic_bytes')->default(0); $t->integer('traffic_quota_bytes')->default(0); $t->integer('traffic_download_bytes')->default(0); $t->integer('traffic_upload_bytes')->default(0); $t->integer('lifetime_download_bytes')->default(0); $t->integer('lifetime_upload_bytes')->default(0); $t->integer('traffic_synced_at')->default(0); $t->integer('suspended_at')->nullable(); $t->integer('network_suspended_at')->nullable(); $t->integer('created_at')->default(0); $t->integer('updated_at')->default(0); }); $attempt = CloudBackupRestoreAttempt::whereNotNull('job_id') ->orderByDesc('id')->firstOrFail(); $job = CloudJob::findOrFail((int) $attempt->job_id); $restore = CloudBackupRestore::findOrFail((int) $attempt->backup_restore_id); $reservation = CloudResourceReservation::findOrFail( (int) $attempt->reservation_id ); $queuedPayload = CloudSecrets::decrypt( (string) $job->sealed_payload, 'job-payload' ); $prepareWire = new ReflectionMethod( \App\Services\Cloud\CloudWorkerService::class, 'portableRestoreDispatchPayload' ); [$stablePayload] = $prepareWire->invoke(null, $job, $queuedPayload); $now = time(); Capsule::table('cloud_jobs')->where('id', (int) $job->id)->update([ 'phase' => 'dispatching', 'status' => 'running', 'attempts' => 1, 'dispatch_attempts' => 1, 'dispatch_started_at' => $now, 'dispatched_at' => $now, 'receipt_deadline_at' => $now + 600, 'sealed_payload' => CloudSecrets::encrypt( $stablePayload, 'job-payload' ), 'updated_at' => $now, ]); $job = CloudJob::findOrFail((int) $job->id); $body = $stablePayload['body']; $result = [ 'uuid' => $body['uuid'], 'name' => $body['name'], 'status' => 'running', 'vcpu' => $body['vcpu'], 'memory_mib' => $body['memory_mib'], 'disk_gib' => $body['disk_gib'], 'bandwidth_mbps' => $body['bandwidth_mbps'], 'public_network' => $body['public_network'], 'machine' => 'q35', 'firmware' => 'uefi', 'secure_boot' => false, 'disks' => [['device' => 'vda', 'format' => 'qcow2']], 'interfaces' => [], 'backup_key' => $body['backup_key'], 'sha256' => $body['source_sha256'], ]; $baselineInstances = CloudInstance::count(); $connection = $job->getConnection(); $assertHeld = static function () use ($assert, $reservation): void { $locked = CloudResourceReservation::findOrFail((int) $reservation->id); $claims = Capsule::table('cloud_resource_reservation_ips') ->where('reservation_id', (int) $reservation->id) ->orderBy('address_family')->get(); $assert((string) $locked->state === 'held' && $locked->instance_id === null && $claims->count() === 2 && $claims->every(static fn ($claim): bool => (string) $claim->state === 'reserved' && (int) $claim->active_claim === 1), 'Restore finalizer changed the held bundle before a valid conclusion.'); }; // Success creates the panel row only after the Agent effect is proven, then // commits both address families and every authority row in the same outer // transaction. $connection->beginTransaction(); try { CloudPortableRestoreFinalizer::finishJobLocked( $job, 'succeeded', $result, $stablePayload ); CloudPortableRestoreFinalizer::finishJobLocked( $job, 'succeeded', $result, $stablePayload ); $finishedAttempt = CloudBackupRestoreAttempt::findOrFail( (int) $attempt->id ); $finishedRestore = CloudBackupRestore::findOrFail((int) $restore->id); $finishedReservation = CloudResourceReservation::findOrFail( (int) $reservation->id ); $instance = CloudInstance::where('uuid', (string) $body['uuid'])->first(); $claims = Capsule::table('cloud_resource_reservation_ips') ->where('reservation_id', (int) $reservation->id) ->orderBy('address_family')->get(); $addresses = Capsule::table('cloud_public_ips')->whereIn( 'id', $claims->pluck('public_ip_id')->map( static fn ($value): int => (int) $value )->all() )->get(); $spec = $instance instanceof CloudInstance ? json_decode((string) $instance->spec_json, true) : null; $assert($instance instanceof CloudInstance && CloudInstance::count() === $baselineInstances + 1 && (int) $instance->host_id === (int) $attempt->target_host_id && (int) $instance->user_id === (int) $job->user_id && (int) $instance->reservation_id === (int) $reservation->id && (int) $instance->resource_pool_id === (int) $reservation->resource_pool_id && (string) $instance->status === 'active' && (string) $instance->service_status === 'exempt' && (string) $instance->network_status === 'active' && $instance->sealed_credential === null && is_array($spec) && ($spec['public_network'] ?? null) == $body['public_network'] && ($spec['restored_from_backup'] ?? null) === $body['backup_key'] && !array_key_exists('root_password', $spec) && !array_key_exists('guest_hostname', $spec), 'Successful restore did not create the canonical disk-owned instance projection.'); $assert((string) $finishedReservation->state === 'committed' && (int) $finishedReservation->instance_id === (int) $instance->id && $claims->count() === 2 && $claims->every(static fn ($claim): bool => (string) $claim->state === 'committed' && (int) $claim->active_claim === 1) && $addresses->count() === 2 && $addresses->every(static fn ($address): bool => (string) $address->state === 'assigned' && (int) $address->instance_id === (int) $instance->id), 'Successful restore did not atomically commit IPv4 and IPv6.'); $assert((string) $finishedAttempt->state === 'succeeded' && (string) $finishedAttempt->effect_resolution === 'applied' && $finishedAttempt->active_attempt === null && $finishedAttempt->effect_resolved_at !== null && $finishedAttempt->terminal_at !== null && (string) $finishedRestore->state === 'ready' && (int) $finishedRestore->target_instance_id === (int) $instance->id && (int) $finishedRestore->job_id === (int) $job->id && (int) $finishedRestore->reservation_id === (int) $reservation->id, 'Successful restore did not settle attempt and generation-one projections.'); // All three delete entry points share this gate and final settlement. // Admission must recognize the attempt-owned source before any host // mutation, then confirmed absence releases both families exactly once. $instance->name = 'renamed-restored-vm'; $instance->save(); CloudPublicNetworkTransitionGate::assertInstanceDeleteReady($instance); $instance->status = 'deleted'; $instance->save(); CloudPlacementService::releaseDeletedInstance( $instance, 'instance_deleted' ); $released = CloudResourceReservation::findOrFail( (int) $reservation->id ); $releasedClaims = Capsule::table('cloud_resource_reservation_ips') ->where('reservation_id', (int) $reservation->id) ->orderBy('address_family')->get(); $releasedAddresses = Capsule::table('cloud_public_ips')->whereIn( 'id', $releasedClaims->pluck('public_ip_id')->map( static fn ($value): int => (int) $value )->all() )->get(); $releasedAt = (int) $released->released_at; $assert((string) $released->state === 'released' && (int) $released->instance_id === (int) $instance->id && $releasedAt > 0 && (string) $released->release_reason === 'instance_deleted' && $releasedClaims->count() === 2 && $releasedClaims->every(static fn ($claim): bool => (string) $claim->state === 'released' && $claim->active_claim === null && $claim->released_at !== null) && $releasedAddresses->count() === 2 && $releasedAddresses->every(static fn ($address): bool => (string) $address->state === 'available' && $address->reservation_id === null && $address->reservation_ip_claim_id === null && $address->instance_id === null && $address->operation_id === null), 'Confirmed restored-instance deletion did not release dual-stack ownership.'); $releasedAt = 123456789; Capsule::table('cloud_resource_reservations')->where( 'id', (int) $reservation->id )->update([ 'released_at' => $releasedAt, ]); CloudPlacementService::releaseDeletedInstance( $instance->fresh(), 'instance_deleted' ); $assert((int) CloudResourceReservation::findOrFail( (int) $reservation->id )->released_at === $releasedAt && (string) CloudBackupRestoreAttempt::findOrFail( (int) $attempt->id )->state === 'succeeded' && (string) CloudBackupRestore::findOrFail( (int) $restore->id )->state === 'ready', 'Restored-instance deletion replay mutated terminal restore history.'); $assert((string) CloudResourceReservation::findOrFail( (int) $reservation->id )->release_reason === 'instance_deleted', 'Restored-instance deletion replay rewrote its release reason.'); } finally { $connection->rollBack(); } // An unknown or torn restore attempt must be rejected before a delete reaches // the host, preserving the complete committed pair for reconciliation. $connection->beginTransaction(); try { CloudPortableRestoreFinalizer::finishJobLocked( $job, 'succeeded', $result, $stablePayload ); $guardedInstance = CloudInstance::where( 'uuid', (string) $body['uuid'] )->firstOrFail(); Capsule::table('cloud_resource_reservations')->where( 'id', (int) $reservation->id )->update(['backup_restore_attempt_key' => null]); $missingSourceRejected = false; try { CloudPublicNetworkTransitionGate::assertInstanceDeleteReady( $guardedInstance ); } catch (CloudError $error) { $missingSourceRejected = $error->httpStatus === 409; } $assert($missingSourceRejected, 'A restore reservation without its attempt key reached deletion.'); Capsule::table('cloud_resource_reservations')->where( 'id', (int) $reservation->id )->update([ 'backup_restore_attempt_key' => (string) $attempt->attempt_key, ]); Capsule::table('cloud_backup_restore_attempts')->where( 'id', (int) $attempt->id )->update([ 'state' => 'effect_unknown', 'effect_resolution' => 'unknown', 'active_attempt' => 1, 'effect_resolved_at' => null, 'terminal_at' => null, ]); $deleteRejected = false; try { CloudPublicNetworkTransitionGate::assertInstanceDeleteReady( $guardedInstance ); } catch (CloudError $error) { $deleteRejected = $error->httpStatus === 409; } $guardedReservation = CloudResourceReservation::findOrFail( (int) $reservation->id ); $guardedClaims = Capsule::table('cloud_resource_reservation_ips') ->where('reservation_id', (int) $reservation->id)->get(); $assert($deleteRejected && (string) $guardedReservation->state === 'committed' && $guardedClaims->count() === 2 && $guardedClaims->every(static fn ($claim): bool => (string) $claim->state === 'committed' && (int) $claim->active_claim === 1), 'Unknown restore attempt reached deletion or released its bundle.'); } finally { $connection->rollBack(); } // The restored domain must match the one fixed portable platform contract; // UUID, disk and network equality cannot authorize a BIOS/i440fx variant. $connection->beginTransaction(); try { foreach ([ ['machine', 'pc-i440fx'], ['firmware', 'bios'], ['secure_boot', true], ] as [$field, $wrong]) { $incompatible = $result; $incompatible[$field] = $wrong; $rejected = false; try { CloudPortableRestoreFinalizer::finishJobLocked( $job, 'succeeded', $incompatible, $stablePayload ); } catch (CloudError $error) { $rejected = $error->httpStatus === 502; } $assert($rejected, 'Restore accepted incompatible platform field: ' . $field); $assertHeld(); } } finally { $connection->rollBack(); } // A deterministic not-applied conclusion releases both claims and concrete // addresses. Replaying the same conclusion is an exact no-op, not a second // release or a new instance. $connection->beginTransaction(); try { CloudPortableRestoreFinalizer::finishJobLocked( $job, 'failed', [ 'creation_absent' => true, 'managed_artifacts_absent' => true, ], $stablePayload ); CloudPortableRestoreFinalizer::finishJobLocked( $job, 'failed', [ 'creation_absent' => true, 'managed_artifacts_absent' => true, ], $stablePayload ); $failedAttempt = CloudBackupRestoreAttempt::findOrFail((int) $attempt->id); $failedRestore = CloudBackupRestore::findOrFail((int) $restore->id); $failedReservation = CloudResourceReservation::findOrFail( (int) $reservation->id ); $claims = Capsule::table('cloud_resource_reservation_ips') ->where('reservation_id', (int) $reservation->id) ->orderBy('address_family')->get(); $addresses = Capsule::table('cloud_public_ips')->whereIn( 'id', $claims->pluck('public_ip_id')->map( static fn ($value): int => (int) $value )->all() )->get(); $assert(CloudInstance::count() === $baselineInstances && (string) $failedReservation->state === 'released' && $failedReservation->instance_id === null && $failedReservation->released_at !== null && $claims->count() === 2 && $claims->every(static fn ($claim): bool => (string) $claim->state === 'released' && $claim->active_claim === null && $claim->released_at !== null) && $addresses->count() === 2 && $addresses->every(static fn ($address): bool => (string) $address->state === 'available' && $address->reservation_id === null && $address->reservation_ip_claim_id === null && $address->instance_id === null && $address->operation_id === null), 'Confirmed-not-applied restore retained a resource or address claim.'); $assert((string) $failedAttempt->state === 'failed' && (string) $failedAttempt->effect_resolution === 'not_applied' && $failedAttempt->active_attempt === null && $failedAttempt->effect_resolved_at !== null && $failedAttempt->terminal_at !== null && (string) $failedRestore->state === 'failed' && $failedRestore->target_instance_id === null && (int) $failedRestore->job_id === (int) $job->id && (int) $failedRestore->reservation_id === (int) $reservation->id, 'Confirmed-not-applied restore did not expose an explicit retry boundary.'); } finally { $connection->rollBack(); } // A generic failure is not proof that the host and managed artifacts are // absent. The deep finalizer itself must reject incomplete absence evidence. $connection->beginTransaction(); try { $absenceRejected = false; try { CloudPortableRestoreFinalizer::finishJobLocked( $job, 'failed', ['creation_absent' => true], $stablePayload ); } catch (\App\Services\Cloud\CloudRecoveryEvidenceConflict $error) { $absenceRejected = $error->httpStatus === 409; } $assert($absenceRejected, 'Incomplete absence evidence released a restore.'); $assertHeld(); } finally { $connection->rollBack(); } // Unknown effect never releases or reassigns anything. It fences both legs, // keeps the generation active, and therefore cannot be retried automatically. $connection->beginTransaction(); try { CloudPortableRestoreFinalizer::finishJobLocked( $job, 'uncertain', null, $stablePayload ); CloudPortableRestoreFinalizer::finishJobLocked( $job, 'uncertain', null, $stablePayload ); $unknownAttempt = CloudBackupRestoreAttempt::findOrFail((int) $attempt->id); $unknownRestore = CloudBackupRestore::findOrFail((int) $restore->id); $unknownReservation = CloudResourceReservation::findOrFail( (int) $reservation->id ); $claims = Capsule::table('cloud_resource_reservation_ips') ->where('reservation_id', (int) $reservation->id) ->orderBy('address_family')->get(); $addresses = Capsule::table('cloud_public_ips')->whereIn( 'id', $claims->pluck('public_ip_id')->map( static fn ($value): int => (int) $value )->all() )->get(); $assert(CloudInstance::count() === $baselineInstances && (string) $unknownReservation->state === 'uncertain' && $unknownReservation->instance_id === null && $claims->count() === 2 && $claims->every(static fn ($claim): bool => (string) $claim->state === 'uncertain' && (int) $claim->active_claim === 1) && $addresses->count() === 2 && $addresses->every(static fn ($address): bool => (string) $address->state === 'uncertain' && (int) $address->reservation_id === (int) $reservation->id), 'Unknown restore effect released or reassigned a fenced resource.'); $assert((string) $unknownAttempt->state === 'effect_unknown' && (string) $unknownAttempt->effect_resolution === 'unknown' && (int) $unknownAttempt->active_attempt === 1 && $unknownAttempt->effect_resolved_at === null && $unknownAttempt->terminal_at === null && (string) $unknownRestore->state === 'uncertain' && $unknownRestore->target_instance_id === null && (int) $unknownRestore->job_id === (int) $job->id && (int) $unknownRestore->reservation_id === (int) $reservation->id, 'Unknown restore effect did not retain its active recovery fence.'); } finally { $connection->rollBack(); } // Recovery prepare validates and persists exact evidence but must not expose // an instance or commit either address family before the Agent fence commit. $connection->beginTransaction(); try { $preparedJob = CloudJob::findOrFail((int) $job->id); CloudPortableRestoreFinalizer::finishJobLocked( $preparedJob, 'succeeded', $result, $stablePayload, true ); $pendingAttempt = CloudBackupRestoreAttempt::findOrFail((int) $attempt->id); $assert((string) $pendingAttempt->state === 'finalization_pending' && (string) $pendingAttempt->effect_resolution === 'applied' && (string) $pendingAttempt->finalization_error_code === 'recovery_prepare_pending' && CloudInstance::count() === $baselineInstances, 'Prepared restore published the target before fence resolution.'); $assertHeld(); $compactMethod = new ReflectionMethod( \App\Services\Cloud\CloudWorkerService::class, 'compactPreparedRecoveryPayload' ); $compactPayload = $compactMethod->invoke( null, 'backup_restore', $stablePayload ); $compactPayload['_operation_recovery'] = [ 'version' => 1, 'kind' => 'backup_restore', 'prepared_marker' => 'prepared:test', 'operation' => ['status' => 'succeeded'], ]; Capsule::table('cloud_jobs')->where('id', (int) $job->id)->update([ 'sealed_payload' => CloudSecrets::encrypt( $compactPayload, 'job-payload' ), ]); $preparedJob = CloudJob::findOrFail((int) $job->id); $assert(CloudOperationRecoveryIdentity::same( CloudOperationRecoveryIdentity::expected($preparedJob), $compactPayload[CloudOperationRecoveryIdentity::payloadKey()] ), 'Compact prepared restore payload lost its recovery identity.'); $tamperedCompact = $compactPayload; $tamperedCompact['generation'] = (int) $tamperedCompact['generation'] + 1; $preparedJob->sealed_payload = CloudSecrets::encrypt( $tamperedCompact, 'job-payload' ); $preparedTamperRejected = false; try { CloudOperationRecoveryIdentity::expected($preparedJob); } catch (CloudError $error) { $preparedTamperRejected = $error->httpStatus === 409; } $assert($preparedTamperRejected, 'Prepared escrow bypassed portable restore recovery authority.'); $preparedJob->sealed_payload = CloudSecrets::encrypt( $compactPayload, 'job-payload' ); CloudPortableRestoreFinalizer::publishPreparedLocked( $preparedJob, 'applied' ); CloudPortableRestoreFinalizer::publishPreparedLocked( $preparedJob, 'applied' ); $published = CloudBackupRestoreAttempt::findOrFail((int) $attempt->id); $assert((string) $published->state === 'succeeded' && CloudInstance::count() === $baselineInstances + 1, 'Prepared restore did not publish exactly once after fence resolution.'); } finally { $connection->rollBack(); } // If exact evidence survived but the ordinary Worker transaction failed, the // local retry consumes only that evidence and never dispatches the mutation. $connection->beginTransaction(); try { $retryJob = CloudJob::findOrFail((int) $job->id); CloudPortableRestoreFinalizer::finishJobLocked( $retryJob, 'succeeded', $result, $stablePayload, true ); Capsule::table('cloud_backup_restore_attempts') ->where('id', (int) $attempt->id)->update([ 'last_error_code' => 'local_finalization_failed', 'finalization_error_code' => 'local_finalization_failed', 'updated_at' => time(), ]); $leaseToken = str_repeat('a', 32); Capsule::table('cloud_jobs')->where('id', (int) $job->id)->update([ 'phase' => 'terminal', 'outcome' => 'failed', 'status' => 'failed', 'terminal_at' => time(), 'effect_may_have_applied' => 1, 'safe_to_resubmit' => 0, 'effect_resolution' => 'unknown', 'error_code' => 'local_finalization_failed', 'finalization_incident' => 1, 'finalization_error_code' => 'local_finalization_failed', 'lease_token' => $leaseToken, 'lease_until' => time() + 60, 'updated_at' => time(), ]); CloudPortableRestoreFinalizer::retryPendingFinalization( (int) $job->id, $leaseToken ); $retriedJob = CloudJob::findOrFail((int) $job->id); $retriedAttempt = CloudBackupRestoreAttempt::findOrFail((int) $attempt->id); $assert((string) $retriedAttempt->state === 'succeeded' && (string) $retriedJob->effect_resolution === 'applied' && !(bool) $retriedJob->finalization_incident && $retriedJob->lease_token === null && CloudInstance::count() === $baselineInstances + 1, 'Portable restore PHP-only retry did not converge from durable evidence.'); } finally { $connection->rollBack(); } // Any frozen-identity tear must abort the whole outer transaction before an // instance row or one-sided address mutation can escape. $connection->beginTransaction(); try { $torn = CloudBackupRestoreAttempt::findOrFail((int) $attempt->id); $torn->target_public_network_sha256 = str_repeat('0', 64); $torn->save(); $rejected = false; try { CloudPortableRestoreFinalizer::finishJobLocked( $job, 'succeeded', $result, $stablePayload ); } catch (CloudError $error) { $rejected = $error->httpStatus === 409; } $assert($rejected && CloudInstance::count() === $baselineInstances, 'Torn restore identity produced a partial instance.'); $assertHeld(); } finally { $connection->rollBack(); } echo "Portable restore finalizer behavior tests passed.\n";