httpStatus === $status, $message . ' Unexpected status: ' . $error->httpStatus); return; } throw new RuntimeException($message); }; $terminalize = static function (CloudJob $job): void { $now = time(); $job->phase = 'terminal'; $job->outcome = $job->status = 'succeeded'; $job->attempts = $job->dispatch_attempts = 1; $job->dispatch_started_at = $job->dispatched_at = $now - 1; $job->agent_accepted_at = $now - 1; $job->receipt_deadline_at = $now + 900; $job->terminal_at = $now; $job->effect_may_have_applied = 0; $job->safe_to_resubmit = 0; $job->effect_resolution = 'applied'; $job->updated_at = $now; $job->save(); }; $terminalizeFailed = static function (CloudJob $job): void { $finishedAt = time() - 600; $job->phase = 'terminal'; $job->outcome = $job->status = 'failed'; $job->attempts = $job->dispatch_attempts = 1; $job->dispatch_started_at = $job->dispatched_at = $finishedAt - 1; $job->agent_accepted_at = $finishedAt - 1; $job->receipt_deadline_at = $finishedAt + 900; $job->terminal_at = $finishedAt; $job->effect_may_have_applied = 0; $job->safe_to_resubmit = 1; $job->effect_resolution = 'not_applied'; $job->error_code = 'vpc_host_remove_failed'; $job->updated_at = $finishedAt; $job->save(); }; $convergeRoutingJobs = static function ( array $jobs ) use ($terminalize): void { foreach ($jobs as $job) { $state = CloudVpcHostState::where( 'vpc_id', (int) $job->vpc_id )->where('host_id', (int) $job->host_id)->first(); if ((string) $job->kind === 'vpc_host_remove') { if ($state === null) { throw new RuntimeException( 'Route-remove test job lost its host state.' ); } $state->delete(); } else { if ($state === null) { throw new RuntimeException( 'Route-apply test job lost its host state.' ); } $state->applied_revision = (int) $state->desired_revision; $state->applied_digest = (string) $state->desired_digest; $state->sync_state = 'ready'; $state->last_error_code = ''; $state->save(); } $terminalize($job); } }; $capsule = new Capsule(); $capsule->addConnection([ 'driver' => 'sqlite', 'database' => ':memory:', ], 'default'); $capsule->setAsGlobal(); $capsule->bootEloquent(); $schema = $capsule->schema('default'); $schema->create('user', static function (Blueprint $table): void { $table->increments('id'); }); Capsule::table('user')->insert(['id' => 7]); $schema->create('cloud_network_lines', static function ( Blueprint $table ): void { $table->increments('id'); $table->integer('region_id'); }); $schema->create('cloud_resource_pools', static function ( Blueprint $table ): void { $table->increments('id'); $table->integer('line_id'); }); $schema->create('cloud_hosts', static function (Blueprint $table): void { $table->increments('id'); $table->boolean('enabled'); }); $schema->create('cloud_host_vpc_transports', static function ( Blueprint $table ): void { $table->integer('host_id')->primary(); $table->string('transport_address'); $table->integer('transport_prefix_length'); $table->string('transport_interface'); $table->integer('mtu'); $table->integer('desired_revision'); $table->integer('observed_revision'); $table->string('sync_state'); }); $schema->create('cloud_instances', static function ( Blueprint $table ): void { $table->increments('id'); $table->string('uuid'); $table->integer('host_id'); $table->integer('user_id'); $table->integer('resource_pool_id')->nullable(); $table->string('name'); $table->string('status'); $table->string('service_status'); $table->string('network_status'); $table->integer('expires_at')->nullable(); $table->text('spec_json'); $table->integer('created_at'); $table->integer('updated_at'); }); $schema->create('cloud_vpcs', static function (Blueprint $table): void { $table->increments('id'); $table->string('vpc_key'); $table->integer('user_id'); $table->integer('region_id'); $table->integer('cidr_pool_id'); $table->string('name'); $table->string('cidr'); $table->integer('prefix_length'); $table->string('gateway'); $table->integer('active_claim')->nullable(); $table->string('state'); $table->integer('desired_revision'); $table->integer('deleted_at')->nullable(); $table->integer('created_at'); $table->integer('updated_at'); }); $schema->create('cloud_vpc_ports', static function ( Blueprint $table ): void { $table->increments('id'); $table->string('port_key')->unique(); $table->integer('vpc_id'); $table->integer('instance_id'); $table->integer('migration_host_id')->nullable(); $table->integer('user_id'); $table->string('private_ip'); $table->string('mac_address')->unique(); $table->integer('active_claim')->nullable(); $table->string('state'); $table->integer('desired_revision'); $table->integer('applied_revision'); $table->string('last_operation_id')->nullable(); $table->string('last_error_code'); $table->integer('attached_at')->nullable(); $table->integer('released_at')->nullable(); $table->integer('created_at'); $table->integer('updated_at'); $table->unique(['vpc_id', 'private_ip', 'active_claim']); $table->unique(['instance_id', 'active_claim']); }); $schema->create('cloud_vpc_host_states', static function ( Blueprint $table ): void { $table->increments('id'); $table->integer('vpc_id'); $table->integer('host_id'); $table->integer('desired_revision'); $table->integer('applied_revision'); $table->string('desired_digest'); $table->string('applied_digest')->nullable(); $table->string('sync_state'); $table->string('last_operation_id')->nullable(); $table->string('last_error_code'); $table->integer('last_success_at'); $table->integer('last_failure_at'); $table->integer('created_at'); $table->integer('updated_at'); $table->unique(['vpc_id', 'host_id']); }); $schema->create('cloud_jobs', static function (Blueprint $table): void { $table->increments('id'); $table->string('operation_id')->unique(); $table->string('request_idempotency_key'); $table->string('request_fingerprint'); $table->integer('parent_job_id')->nullable(); $table->integer('supersedes_job_id')->nullable(); $table->integer('vpc_repair_root_job_id')->nullable(); $table->integer('order_id')->nullable(); $table->integer('invoice_id')->nullable(); $table->integer('instance_id')->nullable(); $table->integer('snapshot_id')->nullable(); $table->integer('backup_id')->nullable(); $table->integer('backup_restore_id')->nullable(); $table->integer('backup_recovery_id')->nullable(); $table->integer('vpc_id')->nullable(); $table->integer('vpc_port_id')->nullable(); $table->integer('vpc_batch_id')->nullable(); $table->integer('user_id'); $table->integer('host_id')->nullable(); $table->string('kind'); $table->string('execution_mode'); $table->string('phase'); $table->string('outcome')->nullable(); $table->string('status'); $table->text('sealed_payload'); $table->integer('attempts'); $table->integer('dispatch_attempts'); $table->integer('dispatch_deadline_at'); $table->integer('dispatch_started_at')->nullable(); $table->integer('agent_accepted_at')->nullable(); $table->integer('receipt_deadline_at')->nullable(); $table->integer('terminal_at')->nullable(); $table->boolean('effect_may_have_applied')->nullable(); $table->boolean('safe_to_resubmit')->nullable(); $table->string('effect_resolution'); $table->integer('effect_resolved_at')->nullable(); $table->integer('effect_resolved_by')->nullable(); $table->text('effect_resolution_reason')->default(''); $table->boolean('finalization_incident')->default(0); $table->string('finalization_error_code')->nullable(); $table->string('completion_event_id')->nullable(); $table->integer('read_at')->nullable(); $table->integer('dispatched_at')->nullable(); $table->string('lease_token')->nullable(); $table->integer('lease_until')->default(0); $table->integer('available_at'); $table->string('error_code')->nullable(); $table->text('result_json')->nullable(); $table->integer('created_at'); $table->integer('updated_at'); $table->unique(['user_id', 'request_idempotency_key']); }); $schema->create('cloud_vpc_membership_batches', static function ( Blueprint $table ): void { $table->increments('id'); $table->string('batch_key')->unique(); $table->integer('vpc_id'); $table->integer('user_id'); $table->integer('parent_job_id')->unique(); $table->string('kind'); $table->string('state'); $table->integer('desired_revision'); $table->integer('active_claim')->nullable(); $table->string('last_error_code')->default(''); $table->integer('terminal_at')->nullable(); $table->integer('created_at'); $table->integer('updated_at'); $table->unique(['vpc_id', 'active_claim']); }); $schema->create('cloud_vpc_membership_batch_items', static function ( Blueprint $table ): void { $table->increments('id'); $table->integer('batch_id'); $table->integer('user_id'); $table->integer('instance_id'); $table->integer('vpc_port_id'); $table->integer('host_id'); $table->integer('child_job_id')->nullable(); $table->string('state'); $table->string('last_error_code')->default(''); $table->integer('created_at'); $table->integer('updated_at'); $table->unique(['batch_id', 'instance_id']); $table->unique(['batch_id', 'vpc_port_id']); $table->unique('child_job_id'); }); Capsule::table('cloud_network_lines')->insert([ 'id' => 1, 'region_id' => 1, ]); Capsule::table('cloud_resource_pools')->insert([ 'id' => 1, 'line_id' => 1, ]); foreach ([1, 2] as $hostId) { Capsule::table('cloud_hosts')->insert([ 'id' => $hostId, 'enabled' => 1, ]); Capsule::table('cloud_host_vpc_transports')->insert([ 'host_id' => $hostId, 'transport_address' => '172.16.10.' . (10 + $hostId), 'transport_prefix_length' => 24, 'transport_interface' => 'br-private', 'mtu' => 1500, 'desired_revision' => 1, 'observed_revision' => 1, 'sync_state' => 'ready', ]); } $newInstance = static function (int $id, int $hostId): CloudInstance { $instance = new CloudInstance(); $instance->id = $id; $instance->uuid = sprintf('11111111-1111-4111-8111-%012d', $id); $instance->host_id = $hostId; $instance->user_id = 7; $instance->resource_pool_id = 1; $instance->name = 'vm-' . $id; $instance->status = 'active'; $instance->service_status = 'exempt'; $instance->network_status = 'active'; $instance->expires_at = null; $instance->spec_json = json_encode([ 'os_source' => 'cloud_template', 'public_network' => [ 'contract' => 1, 'network_selector' => 'bridge:br0', 'ipv4' => [ 'pool_key' => str_repeat('1', 32), 'pool_cidr' => '8.8.8.0/24', 'address_cidr' => '8.8.8.' . (10 + $id) . '/24', 'gateway' => '8.8.8.1', 'gateway_on_link' => false, ], 'ipv6' => [ 'pool_key' => str_repeat('2', 32), 'pool_cidr' => '2404:c140:156:3a::/64', 'address_cidr' => '2404:c140:156:3a::' . (10 + $id) . '/64', 'gateway' => '2404:c140:156::1', 'gateway_on_link' => true, ], 'dns_servers' => ['1.1.1.1', '2606:4700:4700::1111'], ], ], JSON_THROW_ON_ERROR); $instance->created_at = $instance->updated_at = time(); $instance->save(); return $instance; }; $target = $newInstance(1, 1); $peer = $newInstance(2, 2); $invalidNetwork = $newInstance(3, 1); $invalidNetwork->spec_json = json_encode([ 'os_source' => 'cloud_template', // Flat aliases are intentionally untrusted and must not authorize a // new VPC port when the canonical snapshot is absent. 'public_ip' => '8.8.8.13', 'public_ip_cidr' => '8.8.8.13/24', 'gateway' => '8.8.8.1', ], JSON_THROW_ON_ERROR); $invalidNetwork->save(); $missingSource = $newInstance(4, 1); $missingSourceSpec = json_decode( (string) $missingSource->spec_json, true, 512, JSON_THROW_ON_ERROR ); unset($missingSourceSpec['os_source']); $missingSource->spec_json = json_encode( $missingSourceSpec, JSON_THROW_ON_ERROR ); $missingSource->save(); $vpc = new CloudVpc(); $vpc->id = 1; $vpc->vpc_key = str_repeat('a', 32); $vpc->user_id = 7; $vpc->region_id = 1; $vpc->cidr_pool_id = 1; $vpc->name = 'private'; $vpc->cidr = '10.60.0.0/24'; $vpc->prefix_length = 24; $vpc->gateway = '10.60.0.1'; $vpc->active_claim = 1; $vpc->state = 'active'; $vpc->desired_revision = 1; $vpc->deleted_at = null; $vpc->created_at = $vpc->updated_at = time(); $vpc->save(); $peerPort = new CloudVpcPort(); $peerPort->id = 1; $peerPort->port_key = str_repeat('b', 32); $peerPort->vpc_id = 1; $peerPort->instance_id = 2; $peerPort->user_id = 7; $peerPort->private_ip = '10.60.0.3'; $peerPort->mac_address = '02:00:00:00:00:03'; $peerPort->active_claim = 1; $peerPort->state = 'attached'; $peerPort->desired_revision = $peerPort->applied_revision = 1; $peerPort->last_operation_id = null; $peerPort->last_error_code = ''; $peerPort->attached_at = time(); $peerPort->released_at = null; $peerPort->created_at = $peerPort->updated_at = time(); $peerPort->save(); $initial = CloudVpcProjection::build([ 'vpc_key' => (string) $vpc->vpc_key, 'cidr' => (string) $vpc->cidr, 'gateway' => (string) $vpc->gateway, ], [['private_ip' => '10.60.0.3', 'mac_address' => '02:00:00:00:00:03', 'host_id' => 2]], [ 2 => ['transport_address' => '172.16.10.12', 'transport_prefix_length' => 24, 'transport_interface' => 'br-private', 'mtu' => 1500], ], 1); $state = new CloudVpcHostState(); $state->vpc_id = 1; $state->host_id = 2; $state->desired_revision = $state->applied_revision = 1; $state->desired_digest = $state->applied_digest = $initial[2]['digest']; $state->sync_state = 'ready'; $state->last_operation_id = null; $state->last_error_code = ''; $state->last_success_at = time(); $state->last_failure_at = 0; $state->created_at = $state->updated_at = time(); $state->save(); $management = CloudVpcPortService::managementListing(['user_id' => 7]); $targetManagement = collect($management)->firstWhere('id', '1'); $peerManagement = collect($management)->firstWhere('id', '2'); $assert(($targetManagement['region_id'] ?? null) === 1 && ($targetManagement['public_ipv4'] ?? null) === '8.8.8.11' && ($targetManagement['public_ipv6'] ?? null) === '2404:c140:156:3a::11' && array_key_exists('port', $targetManagement) && $targetManagement['port'] === null && ($targetManagement['can_attach'] ?? false) === true, 'The VPC page projection did not expose the eligible same-region instance.'); $assert(($peerManagement['port']['vpc_key'] ?? null) === (string) $vpc->vpc_key && ($peerManagement['public_ipv4'] ?? null) === '8.8.8.12' && ($peerManagement['public_ipv6'] ?? null) === '2404:c140:156:3a::12' && ($peerManagement['can_detach'] ?? false) === true, 'The VPC page projection did not expose its attached instance action.'); $assert(collect($management)->firstWhere('id', '3') === null, 'An unattached instance without a canonical public snapshot was offered for VPC attach.'); $invalidListing = CloudVpcPortService::listing( $invalidNetwork, ['user_id' => 7] ); $assert(($invalidListing['public_network_valid'] ?? null) === false && ($invalidListing['can_attach'] ?? true) === false, 'The per-instance VPC DTO enabled attach without a canonical public snapshot.'); $jobsBeforeInvalidAttach = CloudJob::count(); $portsBeforeInvalidAttach = CloudVpcPort::count(); $expectError(static fn () => CloudVpcPortService::attach( $invalidNetwork, [ 'user_id' => 7, 'request_idempotency_key' => 'vpc-invalid-direct-attach-0001', ], [ 'vpc_key' => (string) $vpc->vpc_key, 'confirm_restart' => true, ] ), 409, 'Direct VPC attach accepted an instance without a canonical public snapshot.'); $assert(CloudJob::count() === $jobsBeforeInvalidAttach && CloudVpcPort::count() === $portsBeforeInvalidAttach, 'Rejected direct VPC attach left a job or private-IP claim behind.'); $expectError(static fn () => CloudVpcMembershipBatchService::start([ 'user_id' => 7, 'request_idempotency_key' => 'vpc-invalid-batch-attach-0001', ], (string) $vpc->vpc_key, [ 'action' => 'attach', 'instance_ids' => [(int) $invalidNetwork->id], 'confirm_restart' => true, ]), 409, 'Batch VPC attach accepted an instance without a canonical public snapshot.'); $assert(CloudJob::count() === $jobsBeforeInvalidAttach && CloudVpcPort::count() === $portsBeforeInvalidAttach && CloudVpcMembershipBatch::count() === 0, 'Rejected batch VPC attach left a parent, batch or private-IP claim behind.'); $expectError(static fn () => CloudVpcPortService::attach( $missingSource, [ 'user_id' => 7, 'request_idempotency_key' => 'vpc-missing-source-attach-0001', ], [ 'vpc_key' => (string) $vpc->vpc_key, 'confirm_restart' => true, ] ), 409, 'VPC attach accepted a canonical snapshot without cloud_template source authority.'); $actor = ['user_id' => 7, 'request_idempotency_key' => 'vpc-attach-request-0001']; $queued = CloudVpcPortService::attach($target, $actor, [ 'vpc_key' => (string) $vpc->vpc_key, 'confirm_restart' => true, ]); $assert($queued['job'] !== null && CloudJob::where('kind', 'vpc_host_apply')->count() === 2, 'Attach did not queue a complete two-host projection.'); $replayedAttach = CloudVpcPortService::attach($target, $actor, [ 'vpc_key' => (string) $vpc->vpc_key, 'confirm_restart' => true, ]); $activeHostJobIds = CloudJob::where('kind', 'vpc_host_apply') ->pluck('id')->map(static fn ($id): int => (int) $id)->all(); $assert($replayedAttach['job'] !== null && in_array((int) $replayedAttach['job']->id, $activeHostJobIds, true) && CloudJob::where('kind', 'vpc_host_apply')->count() === 2, 'An accepted attach retry did not return its existing workflow.'); $reconnectedAttach = CloudVpcPortService::attach($target, [ 'user_id' => 7, 'request_idempotency_key' => 'vpc-attach-new-browser-0002', ], [ 'vpc_key' => (string) $vpc->vpc_key, 'confirm_restart' => true, ]); $assert($reconnectedAttach['job'] !== null && in_array((int) $reconnectedAttach['job']->id, $activeHostJobIds, true) && CloudJob::where('kind', 'vpc_host_apply')->count() === 2, 'A reconnect with a new browser request key duplicated the accepted workflow.'); $port = CloudVpcPort::where('instance_id', 1) ->where('active_claim', 1)->first(); $assert($port !== null && (string) $port->state === 'preparing' && (string) $port->private_ip === '10.60.0.2', 'Attach did not reserve the first usable free address.'); $listing = CloudVpcPortService::listing($target, ['user_id' => 7]); $assert(($listing['active_task']['task_id'] ?? null) === (string) $replayedAttach['job']->id && ($listing['can_attach'] ?? true) === false, 'The read model did not expose the tenant VPC task after reconnect.'); $management = CloudVpcPortService::managementListing(['user_id' => 7]); $targetManagement = collect($management)->firstWhere('id', '1'); $assert(($targetManagement['port']['state'] ?? null) === 'preparing' && ($targetManagement['active_task']['task_id'] ?? null) === (string) $replayedAttach['job']->id && ($targetManagement['can_retry_attach'] ?? true) === false, 'The VPC page projection did not retain the active attach task fence.'); foreach (CloudJob::where('kind', 'vpc_host_apply')->orderBy('id')->get() as $hostJob) { $payload = CloudSecrets::decrypt( (string) $hostJob->sealed_payload, 'job-payload' ); $hostState = CloudVpcHostState::where('vpc_id', 1) ->where('host_id', (int) $hostJob->host_id)->first(); CloudVpcPortService::finishJobLocked($hostJob, 'succeeded', [ 'vpc_key' => (string) $vpc->vpc_key, 'desired_revision' => (int) $hostState->desired_revision, 'desired_digest' => (string) $hostState->desired_digest, 'bridge' => CloudVpcProjection::bridgeName((string) $vpc->vpc_key), 'applied' => true, ], $payload, ['operation_id' => (string) $hostJob->operation_id]); $terminalize($hostJob); } $attachJob = CloudJob::where('kind', 'vpc_port_attach')->first(); $assert($attachJob !== null, 'Converged host projections did not queue the VM port attach.'); $attachPayload = CloudSecrets::decrypt( (string) $attachJob->sealed_payload, 'job-payload' ); $body = $attachPayload['body']; CloudVpcPortService::finishJobLocked($attachJob, 'succeeded', [ 'uuid' => (string) $target->uuid, 'port_key' => (string) $port->port_key, 'vpc_key' => (string) $vpc->vpc_key, 'vpc_port' => [ 'port_key' => (string) $port->port_key, 'vpc_key' => (string) $vpc->vpc_key, 'private_ip' => (string) $port->private_ip, 'cidr' => (string) $vpc->cidr, 'gateway' => (string) $vpc->gateway, 'mac' => (string) $port->mac_address, 'bridge' => CloudVpcProjection::bridgeName((string) $vpc->vpc_key), 'host_revision' => (int) $body['host_revision'], 'host_digest' => (string) $body['host_digest'], ], 'status' => 'running', 'attached' => true, ], $attachPayload, ['operation_id' => (string) $attachJob->operation_id]); $terminalize($attachJob); $assert((string) $port->fresh()->state === 'attached', 'Successful VM port result did not attach the port ledger.'); $migration = new CloudInstanceMigration(); $migration->id = 99; $migration->instance_id = 1; $migration->vpc_port_id = CloudVpcPortService::migrationPortIdLocked( $target, 2 ); $migration->source_host_id = 1; $migration->target_host_id = 2; $migration->source_uuid = (string) $target->uuid; $targetJobs = $target->getConnection()->transaction( static fn (): array => CloudVpcPortService::beginMigrationToTargetLocked( $migration, $target ) ); $migrationPayload = CloudSecrets::decrypt( (string) $targetJobs[0]->sealed_payload, 'job-payload' ); $assert(count($targetJobs) === 2 && ($migrationPayload['maintenance_migration_id'] ?? null) === 99 && ($migrationPayload['vpc_migration_direction'] ?? null) === 'target' && (string) $port->fresh()->state === 'migrating' && (int) $port->fresh()->migration_host_id === 2, 'Maintenance migration did not durably project the port to target.'); $convergeRoutingJobs($targetJobs); $assert(CloudVpcPortService::finishMigrationProjectionLocked( $migration, $target, 'target' ) === true, 'Converged target VPC projection did not complete.'); $targetMetadata = CloudVpcPortService::migrationTargetMetadataLocked( $migration, $target ); $assert(($targetMetadata['private_ip'] ?? null) === '10.60.0.2' && ($targetMetadata['mac'] ?? null) === (string) $port->mac_address && ($targetMetadata['host_revision'] ?? 0) > 1, 'Migration target metadata did not preserve the logical port identity.'); CloudVpcPortService::finalizeMigrationCutoverLocked($migration, $target); $cutoverPort = $port->fresh(); $assert((string) $cutoverPort->state === 'attached' && $cutoverPort->migration_host_id === null && (string) $cutoverPort->private_ip === '10.60.0.2' && (string) $cutoverPort->mac_address === (string) $port->mac_address, 'VPC cutover changed or retained migration state on the logical port.'); $target->host_id = 2; $target->save(); $reverse = new CloudInstanceMigration(); $reverse->id = 100; $reverse->instance_id = 1; $reverse->vpc_port_id = CloudVpcPortService::migrationPortIdLocked( $target, 1 ); $reverse->source_host_id = 2; $reverse->target_host_id = 1; $reverse->source_uuid = (string) $target->uuid; $reverseJobs = $target->getConnection()->transaction( static fn (): array => CloudVpcPortService::beginMigrationToTargetLocked( $reverse, $target ) ); $convergeRoutingJobs($reverseJobs); $assert(CloudVpcPortService::finishMigrationProjectionLocked( $reverse, $target, 'target' ), 'Reverse target projection did not converge.'); CloudVpcPortService::finalizeMigrationCutoverLocked($reverse, $target); $target->host_id = 1; $target->save(); $abort = new CloudInstanceMigration(); $abort->id = 101; $abort->instance_id = 1; $abort->vpc_port_id = CloudVpcPortService::migrationPortIdLocked( $target, 2 ); $abort->source_host_id = 1; $abort->target_host_id = 2; $abort->source_uuid = (string) $target->uuid; $abortTargetJobs = $target->getConnection()->transaction( static fn (): array => CloudVpcPortService::beginMigrationToTargetLocked( $abort, $target ) ); $unknownJob = $abortTargetJobs[0]; foreach ($abortTargetJobs as $routeJob) { $state = CloudVpcHostState::where('vpc_id', 1) ->where('host_id', (int) $routeJob->host_id)->first(); if ((int) $routeJob->id === (int) $unknownJob->id) { $state->sync_state = 'unknown'; $state->last_operation_id = (string) $routeJob->operation_id; $state->last_error_code = 'vpc_result_uncertain'; $state->save(); $routeJob->phase = 'terminal'; $routeJob->outcome = $routeJob->status = 'failed'; $routeJob->effect_may_have_applied = 1; $routeJob->safe_to_resubmit = 0; $routeJob->effect_resolution = 'unknown'; $routeJob->terminal_at = $routeJob->updated_at = time(); $routeJob->save(); continue; } if ((string) $routeJob->kind === 'vpc_host_remove') { $state->delete(); } else { $state->applied_revision = (int) $state->desired_revision; $state->applied_digest = (string) $state->desired_digest; $state->sync_state = 'ready'; $state->save(); } $terminalize($routeJob); } $jobsBeforeUnknownAbort = CloudJob::count(); $unknownReplayBlocked = false; try { $target->getConnection()->transaction( static fn (): array => CloudVpcPortService::beginMigrationToSourceLocked( $abort, $target ) ); } catch (Throwable $error) { $unknownReplayBlocked = true; } $assert($unknownReplayBlocked && CloudJob::count() === $jobsBeforeUnknownAbort, 'Unknown VPC routing effect was overwritten by source rollback.'); $unknownState = CloudVpcHostState::where('vpc_id', 1) ->where('host_id', (int) $unknownJob->host_id)->first(); $unknownState->sync_state = 'failed'; $unknownState->save(); $terminalizeFailed($unknownJob); $sourceJobs = $target->getConnection()->transaction( static fn (): array => CloudVpcPortService::beginMigrationToSourceLocked( $abort, $target ) ); $convergeRoutingJobs($sourceJobs); $assert(CloudVpcPortService::finishMigrationProjectionLocked( $abort, $target, 'source' ) && (string) $port->fresh()->state === 'attached' && $port->fresh()->migration_host_id === null, 'Confirmed-not-applied abort did not restore the source VPC route.'); $detachActor = ['user_id' => 7, 'request_idempotency_key' => 'vpc-detach-request-0001']; $target->spec_json = '{malformed-public-network'; $target->save(); $damagedAttached = collect( CloudVpcPortService::managementListing(['user_id' => 7]) )->firstWhere('id', '1'); $assert(($damagedAttached['public_network_valid'] ?? null) === false && ($damagedAttached['can_detach'] ?? false) === true, 'Damaged public metadata removed the cleanup exit for an attached VPC port.'); $detaching = CloudVpcPortService::detach($target, $detachActor, [ 'confirm_restart' => true, ]); $detachJob = $detaching['job']; $replayedDetach = CloudVpcPortService::detach($target, $detachActor, [ 'confirm_restart' => true, ]); $assert((int) ($replayedDetach['job']->id ?? 0) === (int) $detachJob->id && CloudJob::where('kind', 'vpc_port_detach')->count() === 1, 'An accepted detach retry did not return its existing workflow.'); $detachPayload = CloudSecrets::decrypt( (string) $detachJob->sealed_payload, 'job-payload' ); CloudVpcPortService::finishJobLocked($detachJob, 'succeeded', [ 'uuid' => (string) $target->uuid, 'port_key' => (string) $port->port_key, 'vpc_key' => (string) $vpc->vpc_key, 'status' => 'running', 'removed' => true, ], $detachPayload, ['operation_id' => (string) $detachJob->operation_id]); $terminalize($detachJob); $assert(CloudJob::whereIn('kind', ['vpc_host_apply', 'vpc_host_remove'])->where('phase', 'accepted') ->count() === 2, 'Detach did not queue route cleanup after VM port removal.'); $cleanupJobs = CloudJob::whereIn('kind', ['vpc_host_apply', 'vpc_host_remove'])->where('phase', 'accepted') ->orderBy('id')->get(); foreach ($cleanupJobs as $cleanupJob) { $payload = CloudSecrets::decrypt( (string) $cleanupJob->sealed_payload, 'job-payload' ); $hostState = CloudVpcHostState::where('vpc_id', 1) ->where('host_id', (int) $cleanupJob->host_id)->first(); $result = [ 'vpc_key' => (string) $vpc->vpc_key, 'desired_revision' => (int) ($cleanupJob->kind === 'vpc_host_apply' ? $hostState->desired_revision : $hostState->applied_revision), 'desired_digest' => (string) ($cleanupJob->kind === 'vpc_host_apply' ? $hostState->desired_digest : $hostState->applied_digest), ]; if ((string) $cleanupJob->kind === 'vpc_host_apply') { $result['bridge'] = CloudVpcProjection::bridgeName( (string) $vpc->vpc_key ); $result['applied'] = true; } else { $result['removed'] = true; } CloudVpcPortService::finishJobLocked($cleanupJob, 'succeeded', $result, $payload, ['operation_id' => (string) $cleanupJob->operation_id]); $terminalize($cleanupJob); } $released = $port->fresh(); $assert((string) $released->state === 'released' && $released->active_claim === null && CloudVpcHostState::where('vpc_id', 1)->count() === 1 && (int) CloudVpcHostState::where('vpc_id', 1)->value('host_id') === 2, 'Detach did not release the IP after every host cleanup converged.'); $knownFailedState = CloudVpcHostState::where('vpc_id', 1) ->where('host_id', 2)->first(); $knownFailedState->desired_revision = (int) $knownFailedState->desired_revision + 1; $knownFailedState->desired_digest = str_repeat('f', 64); $knownFailedState->sync_state = 'failed'; $knownFailedState->last_error_code = 'vpc_host_apply_failed'; $knownFailedState->save(); $knownFailedJob = CloudJob::where( 'operation_id', (string) $knownFailedState->last_operation_id )->first(); $assert($knownFailedJob !== null, 'Failed host-state fixture lost its exact operation generation.'); $terminalizeFailed($knownFailedJob); $vpc->desired_revision = (int) $knownFailedState->desired_revision; $vpc->state = 'degraded'; $vpc->save(); $deleteCleanupJobs = $peer->getConnection()->transaction( static function (): array { $locked = CloudInstance::where('id', 2) ->lockForUpdate()->first(); return CloudVpcPortService::releaseAfterInstanceDeletedLocked( $locked ); } ); $peerPort = $peerPort->fresh(); $assert(count($deleteCleanupJobs) === 1 && (string) $deleteCleanupJobs[0]->kind === 'vpc_host_remove' && (string) $peerPort->state === 'release_pending' && $peerPort->active_claim !== null, 'Confirmed instance deletion released its private IP before route cleanup.'); $deleteCleanupJob = $deleteCleanupJobs[0]; $deleteCleanupPayload = CloudSecrets::decrypt( (string) $deleteCleanupJob->sealed_payload, 'job-payload' ); $deleteHostState = CloudVpcHostState::where('vpc_id', 1) ->where('host_id', 2)->first(); CloudVpcPortService::finishJobLocked($deleteCleanupJob, 'failed', null, $deleteCleanupPayload, [ 'operation_id' => (string) $deleteCleanupJob->operation_id, 'error_code' => 'vpc_host_remove_failed', ]); $terminalizeFailed($deleteCleanupJob); $peer->status = 'deleted'; $peer->save(); $deleteCleanupJob->effect_resolution = 'unknown'; $deleteCleanupJob->effect_may_have_applied = 1; $deleteCleanupJob->safe_to_resubmit = 0; $deleteCleanupJob->save(); $jobCountBeforeUnknownRetry = CloudJob::count(); $assert(CloudVpcPortService::retryDeletedCleanupDue(10) === 0 && CloudJob::count() === $jobCountBeforeUnknownRetry, 'Unknown-effect deleted-instance cleanup was replayed.'); $terminalizeFailed($deleteCleanupJob); $assert(CloudVpcPortService::retryDeletedCleanupDue(10) === 1, 'Known-not-applied deleted-instance cleanup was not retried.'); $peerPort = $peerPort->fresh(); $assert((string) $peerPort->state === 'release_pending' && $peerPort->active_claim !== null, 'A failed deleted-instance cleanup released the address early.'); $deleteCleanupJob = CloudJob::where('kind', 'vpc_host_remove') ->where('phase', 'accepted')->orderByDesc('id')->first(); $deleteCleanupPayload = CloudSecrets::decrypt( (string) $deleteCleanupJob->sealed_payload, 'job-payload' ); $deleteHostState = CloudVpcHostState::where('vpc_id', 1) ->where('host_id', 2)->first(); CloudVpcPortService::finishJobLocked($deleteCleanupJob, 'succeeded', [ 'vpc_key' => (string) $vpc->vpc_key, 'desired_revision' => (int) $deleteHostState->applied_revision, 'desired_digest' => (string) $deleteHostState->applied_digest, 'removed' => true, ], $deleteCleanupPayload, [ 'operation_id' => (string) $deleteCleanupJob->operation_id, ]); $terminalize($deleteCleanupJob); $peerPort = $peerPort->fresh(); $assert((string) $peerPort->state === 'released' && $peerPort->active_claim === null && CloudVpcHostState::where('vpc_id', 1)->count() === 0, 'Deleted instance VPC cleanup did not release the address after convergence.'); // A membership batch has no four-instance ceiling. All attach mutations // remain fenced behind one route generation, then become claimable in the // same finalizer transaction. Detach mutations are claimable immediately // and route cleanup is deferred until every VM result is known. $batchInstances = []; foreach (range(10, 15) as $id) { $batchInstances[] = $newInstance($id, $id % 2 === 0 ? 1 : 2); } $attachBatch = CloudVpcMembershipBatchService::start([ 'user_id' => 7, 'request_idempotency_key' => 'vpc-batch-attach-more-than-four-0001', ], (string) $vpc->vpc_key, [ 'action' => 'attach', 'instance_ids' => array_map( static fn (CloudInstance $instance): int => (int) $instance->id, $batchInstances ), 'confirm_restart' => true, ]); $attachReplay = CloudVpcMembershipBatchService::start([ 'user_id' => 7, 'request_idempotency_key' => 'vpc-batch-attach-more-than-four-0001', ], (string) $vpc->vpc_key, [ 'action' => 'attach', 'instance_ids' => [15, 14, 13, 12, 11, 10], 'confirm_restart' => true, ]); $assert((int) $attachReplay['job']->id === (int) $attachBatch['job']->id && ($attachReplay['unchanged'] ?? false) === true, 'Batch retry did not return the same durable parent task.'); $attachBatchId = (int) CloudVpcMembershipBatch::where( 'parent_job_id', (int) $attachBatch['job']->id )->value('id'); $attachRouteJobs = CloudJob::where('vpc_batch_id', $attachBatchId) ->where('kind', 'vpc_host_apply')->orderBy('id')->get(); $assert($attachBatchId > 0 && $attachRouteJobs->count() === 2 && CloudJob::where('vpc_batch_id', $attachBatchId) ->where('kind', 'vpc_port_attach')->count() === 0, 'Batch attach did not fence more than four instances behind one route generation.'); $attachFences = CloudJob::where('vpc_batch_id', $attachBatchId) ->where('kind', 'vpc_batch_instance_fence')->orderBy('instance_id') ->get(); $assert($attachFences->count() === count($batchInstances) && $attachFences->every(static fn (CloudJob $job): bool => (string) $job->phase === 'accepted' && $job->host_id === null && (int) $job->parent_job_id === (int) $attachBatch['job']->id) && collect($batchInstances)->every(static fn ( CloudInstance $instance ): bool => CloudJobService::blockingQueryForInstance( (int) $instance->id )->where('kind', 'vpc_batch_instance_fence')->exists()), 'Batch route stage did not create one durable instance fence per VM.'); foreach ($attachRouteJobs as $index => $routeJob) { $payload = CloudSecrets::decrypt( (string) $routeJob->sealed_payload, 'job-payload' ); $hostState = CloudVpcHostState::where('vpc_id', 1) ->where('host_id', (int) $routeJob->host_id)->first(); CloudVpcPortService::finishJobLocked($routeJob, 'succeeded', [ 'vpc_key' => (string) $vpc->vpc_key, 'desired_revision' => (int) $hostState->desired_revision, 'desired_digest' => (string) $hostState->desired_digest, 'bridge' => CloudVpcProjection::bridgeName((string) $vpc->vpc_key), 'applied' => true, ], $payload, ['operation_id' => (string) $routeJob->operation_id]); $terminalize($routeJob); $releasedCount = CloudJob::where('vpc_batch_id', $attachBatchId) ->where('kind', 'vpc_port_attach')->count(); $assert($releasedCount === ($index + 1 === $attachRouteJobs->count() ? count($batchInstances) : 0), 'Batch attach released VM jobs before or after the exact route barrier.'); } $attachPortJobs = CloudJob::where('vpc_batch_id', $attachBatchId) ->where('kind', 'vpc_port_attach')->get(); $assert($attachPortJobs->count() === 6 && $attachPortJobs->every(static fn (CloudJob $job): bool => (string) $job->phase === 'accepted' && (int) $job->dispatch_attempts === 0), 'Batch attach did not release every selected instance together.'); $assert($attachFences->every(static fn (CloudJob $job): bool => (string) $job->fresh()->phase === 'terminal') && collect($batchInstances)->every(static fn ( CloudInstance $instance ): bool => CloudJobService::blockingQueryForInstance( (int) $instance->id )->where('kind', 'vpc_port_attach')->exists()), 'Batch route fence was not atomically handed to each Agent port job.'); foreach ($attachPortJobs as $index => $attachPortJob) { $payload = CloudSecrets::decrypt( (string) $attachPortJob->sealed_payload, 'job-payload' ); $body = $payload['body']; $instance = CloudInstance::find((int) $attachPortJob->instance_id); $batchPort = CloudVpcPort::find((int) $attachPortJob->vpc_port_id); $result = [ 'uuid' => (string) $instance->uuid, 'port_key' => (string) $batchPort->port_key, 'vpc_key' => (string) $vpc->vpc_key, 'vpc_port' => [ 'port_key' => (string) $batchPort->port_key, 'vpc_key' => (string) $vpc->vpc_key, 'private_ip' => (string) $batchPort->private_ip, 'cidr' => (string) $vpc->cidr, 'gateway' => (string) $vpc->gateway, 'mac' => (string) $batchPort->mac_address, 'bridge' => CloudVpcProjection::bridgeName( (string) $vpc->vpc_key ), 'host_revision' => (int) $body['host_revision'], 'host_digest' => (string) $body['host_digest'], ], 'status' => 'running', 'attached' => true, ]; if ($index === 0) { // Recovery of one uncertain child must use the same VPC finalizer // as an ordinary completion. In particular, it must advance the // membership batch without completing the parent early or // leaving the selected instance behind a permanent fence. $identity = CloudOperationRecoveryIdentity::expected( $attachPortJob, true ); $operation = [ 'operation_id' => (string) $attachPortJob->operation_id, 'status' => 'succeeded', 'effect_may_have_applied' => false, 'safe_to_resubmit' => false, 'retryable' => false, 'error_code' => '', 'action' => $identity['action'], 'resource_id' => $identity['resource_id'], 'request_fingerprint' => $identity['request_fingerprint'], 'result' => $result, ]; $finishedAt = time(); $attachPortJob->phase = 'terminal'; $attachPortJob->outcome = $attachPortJob->status = 'failed'; $attachPortJob->attempts = $attachPortJob->dispatch_attempts = 1; $attachPortJob->dispatch_started_at = $attachPortJob->dispatched_at = $finishedAt - 2; $attachPortJob->agent_accepted_at = $finishedAt - 1; $attachPortJob->receipt_deadline_at = $finishedAt + 900; $attachPortJob->terminal_at = $finishedAt; $attachPortJob->effect_may_have_applied = 1; $attachPortJob->safe_to_resubmit = 0; $attachPortJob->effect_resolution = 'unknown'; $attachPortJob->error_code = 'operation_effect_unknown'; $attachPortJob->updated_at = $finishedAt; $attachPortJob->save(); CloudOperationRecoveryService::prepareApplied( (int) $attachPortJob->id, $operation, 'test_probe:vpc_batch_child' ); $preparedChild = $attachPortJob->fresh(); $assert((string) $preparedChild->effect_resolution === 'unknown' && str_starts_with( (string) $preparedChild->effect_resolution_reason, 'prepared:test_probe:vpc_batch_child:' ), 'Recovered VPC child released its fence before commit.'); CloudOperationRecoveryService::commitApplied( (int) $attachPortJob->id, $operation, 'test_probe:vpc_batch_child' ); $assert((string) $attachPortJob->fresh()->effect_resolution === 'applied', 'Recovered VPC child did not publish its exact resolution.'); } else { CloudVpcPortService::finishJobLocked( $attachPortJob, 'succeeded', $result, $payload, ['operation_id' => (string) $attachPortJob->operation_id] ); $terminalize($attachPortJob); } if ($index + 1 < $attachPortJobs->count()) { $assert((string) $attachBatch['job']->fresh()->phase !== 'terminal', 'Batch attach parent completed before every VM result.'); } } $assert((string) $attachBatch['job']->fresh()->outcome === 'succeeded' && CloudVpcPort::where('vpc_id', 1) ->where('active_claim', 1)->where('state', 'attached')->count() === 6, 'Batch attach did not converge its parent and all selected ports.'); $assert(collect($batchInstances)->every(static fn ( CloudInstance $instance ): bool => !CloudJobService::blockingQueryForInstance( (int) $instance->id )->exists()), 'Completed batch left an instance operation fence behind.'); $batchDetachSpec = json_decode( (string) $batchInstances[0]->spec_json, true, 512, JSON_THROW_ON_ERROR ); unset($batchDetachSpec['os_source']); $batchInstances[0]->spec_json = json_encode( $batchDetachSpec, JSON_THROW_ON_ERROR ); $batchInstances[0]->save(); $detachBatch = CloudVpcMembershipBatchService::start([ 'user_id' => 7, 'request_idempotency_key' => 'vpc-batch-detach-more-than-four-0001', ], (string) $vpc->vpc_key, [ 'action' => 'detach', 'instance_ids' => array_map( static fn (CloudInstance $instance): int => (int) $instance->id, $batchInstances ), 'confirm_restart' => true, ]); $detachBatchId = (int) CloudVpcMembershipBatch::where( 'parent_job_id', (int) $detachBatch['job']->id )->value('id'); $detachPortJobs = CloudJob::where('vpc_batch_id', $detachBatchId) ->where('kind', 'vpc_port_detach')->get(); $assert($detachBatchId > 0 && $detachPortJobs->count() === 6 && $detachPortJobs->every(static fn (CloudJob $job): bool => (string) $job->phase === 'accepted' && (int) $job->dispatch_attempts === 0) && CloudJob::where('vpc_batch_id', $detachBatchId) ->whereIn('kind', ['vpc_host_apply', 'vpc_host_remove']) ->count() === 0, 'Batch detach did not release every selected instance before route cleanup.'); foreach ($detachPortJobs as $index => $detachPortJob) { $payload = CloudSecrets::decrypt( (string) $detachPortJob->sealed_payload, 'job-payload' ); $instance = CloudInstance::find((int) $detachPortJob->instance_id); $batchPort = CloudVpcPort::find((int) $detachPortJob->vpc_port_id); CloudVpcPortService::finishJobLocked($detachPortJob, 'succeeded', [ 'uuid' => (string) $instance->uuid, 'port_key' => (string) $batchPort->port_key, 'vpc_key' => (string) $vpc->vpc_key, 'status' => 'running', 'removed' => true, ], $payload, [ 'operation_id' => (string) $detachPortJob->operation_id, ]); $terminalize($detachPortJob); $cleanupCount = CloudJob::where('vpc_batch_id', $detachBatchId) ->whereIn('kind', ['vpc_host_apply', 'vpc_host_remove'])->count(); $assert($cleanupCount === ($index + 1 === $detachPortJobs->count() ? 2 : 0), 'Batch detach started route cleanup before or after the exact VM barrier.'); } $detachRouteJobs = CloudJob::where('vpc_batch_id', $detachBatchId) ->whereIn('kind', ['vpc_host_apply', 'vpc_host_remove']) ->orderBy('id')->get(); foreach ($detachRouteJobs as $routeJob) { $payload = CloudSecrets::decrypt( (string) $routeJob->sealed_payload, 'job-payload' ); $hostState = CloudVpcHostState::where('vpc_id', 1) ->where('host_id', (int) $routeJob->host_id)->first(); $result = [ 'vpc_key' => (string) $vpc->vpc_key, 'desired_revision' => (int) $hostState->applied_revision, 'desired_digest' => (string) $hostState->applied_digest, 'removed' => true, ]; CloudVpcPortService::finishJobLocked($routeJob, 'succeeded', $result, $payload, ['operation_id' => (string) $routeJob->operation_id]); $terminalize($routeJob); } $assert((string) $detachBatch['job']->fresh()->outcome === 'succeeded' && CloudVpcPort::where('vpc_id', 1) ->where('active_claim', 1)->count() === 0 && CloudVpcHostState::where('vpc_id', 1)->count() === 0, 'Batch detach did not converge route cleanup before releasing ports.'); // A known route-stage failure automatically creates a new projection that // excludes every reserved port. Instance fences remain active until that // rollback converges, then the IPs and fences are released together. $rollbackInstances = [$newInstance(20, 1), $newInstance(21, 2)]; $rollbackBatch = CloudVpcMembershipBatchService::start([ 'user_id' => 7, 'request_idempotency_key' => 'vpc-batch-attach-rollback-safe-0001', ], (string) $vpc->vpc_key, [ 'action' => 'attach', 'instance_ids' => [20, 21], 'confirm_restart' => true, ]); $rollbackBatchId = (int) CloudVpcMembershipBatch::where( 'parent_job_id', (int) $rollbackBatch['job']->id )->value('id'); $rollbackRouteJobs = CloudJob::where('vpc_batch_id', $rollbackBatchId) ->where('kind', 'vpc_host_apply')->orderBy('id')->get(); foreach ($rollbackRouteJobs as $index => $routeJob) { $payload = CloudSecrets::decrypt( (string) $routeJob->sealed_payload, 'job-payload' ); if ($index === 0) { $hostState = CloudVpcHostState::where('vpc_id', 1) ->where('host_id', (int) $routeJob->host_id)->first(); CloudVpcPortService::finishJobLocked($routeJob, 'succeeded', [ 'vpc_key' => (string) $vpc->vpc_key, 'desired_revision' => (int) $hostState->desired_revision, 'desired_digest' => (string) $hostState->desired_digest, 'bridge' => CloudVpcProjection::bridgeName( (string) $vpc->vpc_key ), 'applied' => true, ], $payload, [ 'operation_id' => (string) $routeJob->operation_id, ]); $terminalize($routeJob); } else { CloudVpcPortService::finishJobLocked( $routeJob, 'failed', null, $payload, [ 'operation_id' => (string) $routeJob->operation_id, 'error_code' => 'vpc_host_apply_failed', ] ); $terminalizeFailed($routeJob); } } $rollbackBatchRow = CloudVpcMembershipBatch::find($rollbackBatchId); $rollbackCleanupJobs = CloudJob::where( 'vpc_batch_id', $rollbackBatchId )->whereIn('kind', ['vpc_host_apply', 'vpc_host_remove']) ->where('phase', 'accepted')->orderBy('id')->get(); $assert((string) $rollbackBatchRow->state === 'cleanup' && $rollbackCleanupJobs->count() === 1 && CloudVpcPort::whereIn('instance_id', [20, 21]) ->where('state', 'release_pending')->count() === 2 && CloudJob::where('vpc_batch_id', $rollbackBatchId) ->where('kind', 'vpc_batch_instance_fence') ->where('phase', '<>', 'terminal')->count() === 2, 'Known route failure did not enter a fenced rollback generation.'); foreach ($rollbackCleanupJobs as $cleanupJob) { $payload = CloudSecrets::decrypt( (string) $cleanupJob->sealed_payload, 'job-payload' ); $hostState = CloudVpcHostState::where('vpc_id', 1) ->where('host_id', (int) $cleanupJob->host_id)->first(); CloudVpcPortService::finishJobLocked($cleanupJob, 'succeeded', [ 'vpc_key' => (string) $vpc->vpc_key, 'desired_revision' => (int) $hostState->applied_revision, 'desired_digest' => (string) $hostState->applied_digest, 'removed' => true, ], $payload, [ 'operation_id' => (string) $cleanupJob->operation_id, ]); $terminalize($cleanupJob); } $assert((string) $rollbackBatch['job']->fresh()->outcome === 'failed' && (int) $rollbackBatch['job']->fresh()->safe_to_resubmit === 1 && CloudVpcPort::whereIn('instance_id', [20, 21]) ->whereNotNull('active_claim')->count() === 0 && CloudJob::where('vpc_batch_id', $rollbackBatchId) ->where('kind', 'vpc_batch_instance_fence') ->where('phase', '<>', 'terminal')->count() === 0, 'Successful attach rollback did not release ports and instance fences.'); // A cleanup failure remains fenced. Once its exact Agent job is terminal // and proven not applied, the user can continue the same rollback parent; // an unknown result would keep can_cancel false and be rejected. $newInstance(30, 1); $newInstance(31, 2); $retryBatch = CloudVpcMembershipBatchService::start([ 'user_id' => 7, 'request_idempotency_key' => 'vpc-batch-attach-rollback-retry-0001', ], (string) $vpc->vpc_key, [ 'action' => 'attach', 'instance_ids' => [30, 31], 'confirm_restart' => true, ]); $retryBatchId = (int) CloudVpcMembershipBatch::where( 'parent_job_id', (int) $retryBatch['job']->id )->value('id'); $initialJobs = CloudJob::where('vpc_batch_id', $retryBatchId) ->where('kind', 'vpc_host_apply')->orderBy('id')->get(); foreach ($initialJobs as $index => $routeJob) { $payload = CloudSecrets::decrypt( (string) $routeJob->sealed_payload, 'job-payload' ); if ($index === 0) { $hostState = CloudVpcHostState::where('vpc_id', 1) ->where('host_id', (int) $routeJob->host_id)->first(); CloudVpcPortService::finishJobLocked($routeJob, 'succeeded', [ 'vpc_key' => (string) $vpc->vpc_key, 'desired_revision' => (int) $hostState->desired_revision, 'desired_digest' => (string) $hostState->desired_digest, 'bridge' => CloudVpcProjection::bridgeName( (string) $vpc->vpc_key ), 'applied' => true, ], $payload, [ 'operation_id' => (string) $routeJob->operation_id, ]); $terminalize($routeJob); } else { CloudVpcPortService::finishJobLocked($routeJob, 'failed', null, $payload, [ 'operation_id' => (string) $routeJob->operation_id, 'error_code' => 'vpc_host_apply_failed', ]); $terminalizeFailed($routeJob); } } $failedCleanup = CloudJob::where('vpc_batch_id', $retryBatchId) ->where('kind', 'vpc_host_remove')->where('phase', 'accepted') ->first(); $failedCleanupPayload = CloudSecrets::decrypt( (string) $failedCleanup->sealed_payload, 'job-payload' ); CloudVpcPortService::finishJobLocked($failedCleanup, 'failed', null, $failedCleanupPayload, [ 'operation_id' => (string) $failedCleanup->operation_id, 'error_code' => 'vpc_host_remove_failed', ]); $terminalizeFailed($failedCleanup); $retryBatchRow = CloudVpcMembershipBatch::find($retryBatchId); $retryDto = collect( CloudVpcMembershipBatchService::activeForOwner(7) )->firstWhere('batch_key', (string) $retryBatchRow->batch_key); $expectedItemStates = CloudVpcMembershipBatchItem::query() ->where('batch_id', $retryBatchId)->orderBy('instance_id')->get() ->mapWithKeys(static fn ($item): array => [ (string) $item->instance_id => (string) $item->state, ])->all(); $projectedItemStates = collect($retryDto['items'] ?? []) ->mapWithKeys(static fn (array $item): array => [ (string) ($item['instance_id'] ?? '') => (string) ($item['state'] ?? ''), ])->all(); $assert((string) $retryBatchRow->state === 'blocked' && ($retryDto['can_cancel'] ?? false) === true && $projectedItemStates === $expectedItemStates && collect([30, 31])->every(static fn (int $id): bool => CloudJobService::blockingQueryForInstance($id)->exists()), 'Known-not-applied rollback failure did not expose item progress and a safe cancel exit.'); $retryResult = CloudVpcMembershipBatchService::cancelAttach( ['user_id' => 7], (string) $vpc->vpc_key, (string) $retryBatchRow->batch_key ); $assert((int) $retryResult['job']->id === (int) $retryBatch['job']->id && (string) CloudVpcMembershipBatch::find($retryBatchId)->state === 'cleanup', 'Safe rollback retry did not continue the original parent task.'); $retriedCleanup = CloudJob::where('vpc_batch_id', $retryBatchId) ->where('kind', 'vpc_host_remove')->where('phase', 'accepted') ->first(); $retriedPayload = CloudSecrets::decrypt( (string) $retriedCleanup->sealed_payload, 'job-payload' ); $retryHostState = CloudVpcHostState::where('vpc_id', 1) ->where('host_id', (int) $retriedCleanup->host_id)->first(); CloudVpcPortService::finishJobLocked($retriedCleanup, 'succeeded', [ 'vpc_key' => (string) $vpc->vpc_key, 'desired_revision' => (int) $retryHostState->applied_revision, 'desired_digest' => (string) $retryHostState->applied_digest, 'removed' => true, ], $retriedPayload, [ 'operation_id' => (string) $retriedCleanup->operation_id, ]); $terminalize($retriedCleanup); $assert((string) $retryBatch['job']->fresh()->outcome === 'failed', 'Retried rollback did not terminalize its original parent.'); $assert(CloudVpcPort::whereIn('instance_id', [30, 31]) ->whereNotNull('active_claim')->count() === 0, 'Retried rollback did not release every reserved private IP.'); $assert(collect([30, 31])->every(static fn (int $id): bool => !CloudJobService::blockingQueryForInstance($id)->exists()), 'Retried rollback left an instance operation fence behind.'); // A reachable but inconclusive VPC generation is repaired by a new // operation. If the first repair also loses its result, a second exact // repair may supersede the complete chain without rewriting history. $repairInstance = $newInstance(40, 1); $repairAttach = CloudVpcPortService::attach($repairInstance, [ 'user_id' => 7, 'request_idempotency_key' => 'vpc-repair-fixture-attach-0001', ], [ 'vpc_key' => (string) $vpc->vpc_key, 'confirm_restart' => true, ]); $repairOriginal = CloudJob::where('kind', 'vpc_host_apply') ->where('instance_id', 40)->where('phase', 'accepted')->first(); $assert($repairAttach['job'] !== null && $repairOriginal !== null, 'VPC repair fixture did not create its host generation.'); $repairFinishedAt = time(); $repairOriginal->phase = 'terminal'; $repairOriginal->outcome = $repairOriginal->status = 'failed'; $repairOriginal->attempts = $repairOriginal->dispatch_attempts = 1; $repairOriginal->dispatch_started_at = $repairOriginal->dispatched_at = $repairFinishedAt - 2; $repairOriginal->agent_accepted_at = $repairFinishedAt - 1; $repairOriginal->receipt_deadline_at = $repairFinishedAt + 900; $repairOriginal->terminal_at = $repairFinishedAt; $repairOriginal->effect_may_have_applied = 1; $repairOriginal->safe_to_resubmit = 0; $repairOriginal->effect_resolution = 'unknown'; $repairOriginal->error_code = 'operation_effect_unknown'; $repairOriginal->updated_at = $repairFinishedAt; $repairOriginal->save(); CloudVpcPortService::markJobUncertain($repairOriginal); $repairInspection = static function (CloudJob $job): array { return [ 'operation_id' => (string) $job->operation_id, 'journal_status' => 'failed', 'checked_at' => time(), 'outcome' => 'inconclusive', 'effect_resolution' => 'inconclusive', 'fence_state' => 'active', 'fence_resolution_required' => true, ]; }; $repairActor = [ 'user_id' => 1, 'is_admin' => true, 'is_super_admin' => true, 'can_manage' => true, ]; $repairOne = CloudVpcPortService::repairUnknownEffect( $repairActor, (int) $repairOriginal->id, $repairInspection($repairOriginal) ); $repairOnePayload = CloudSecrets::decrypt( (string) $repairOne->sealed_payload, 'job-payload' ); $repairState = CloudVpcHostState::where('vpc_id', 1) ->where('host_id', 1)->first(); $assert((int) $repairOne->supersedes_job_id === (int) $repairOriginal->id && (int) $repairOne->vpc_repair_root_job_id === (int) $repairOriginal->id && ($repairOnePayload['body']['vpc_repair'] ?? false) === true && ($repairOnePayload['body']['supersedes_operation_id'] ?? null) === (string) $repairOriginal->operation_id && str_ends_with( (string) $repairOnePayload['request_target'], '/repair' ) && (string) $repairState->last_operation_id === (string) $repairOne->operation_id, 'First VPC repair was not bound to the exact predecessor generation.'); CloudVpcPortService::finishJobLocked( $repairOne, 'uncertain', null, $repairOnePayload, [ 'operation_id' => (string) $repairOne->operation_id, 'error_code' => 'vpc_reconcile_unconfirmed', ] ); $repairOne->phase = 'terminal'; $repairOne->outcome = $repairOne->status = 'failed'; $repairOne->attempts = $repairOne->dispatch_attempts = 1; $repairOne->dispatch_started_at = $repairOne->dispatched_at = time() - 2; $repairOne->agent_accepted_at = time() - 1; $repairOne->receipt_deadline_at = time() + 900; $repairOne->terminal_at = time(); $repairOne->effect_may_have_applied = 1; $repairOne->safe_to_resubmit = 0; $repairOne->effect_resolution = 'unknown'; $repairOne->error_code = 'operation_effect_unknown'; $repairOne->updated_at = time(); $repairOne->save(); $repairTwo = CloudVpcPortService::repairUnknownEffect( $repairActor, (int) $repairOne->id, $repairInspection($repairOne) ); $repairTwoPayload = CloudSecrets::decrypt( (string) $repairTwo->sealed_payload, 'job-payload' ); $repairState = CloudVpcHostState::where('vpc_id', 1) ->where('host_id', 1)->first(); CloudVpcPortService::finishJobLocked($repairTwo, 'succeeded', [ 'vpc_key' => (string) $vpc->vpc_key, 'desired_revision' => (int) $repairState->desired_revision, 'desired_digest' => (string) $repairState->desired_digest, 'bridge' => CloudVpcProjection::bridgeName((string) $vpc->vpc_key), 'applied' => true, ], $repairTwoPayload, [ 'operation_id' => (string) $repairTwo->operation_id, ]); $terminalize($repairTwo); $assert((int) $repairTwo->supersedes_job_id === (int) $repairOne->id && (int) $repairTwo->vpc_repair_root_job_id === (int) $repairOriginal->id && (string) $repairOriginal->fresh()->effect_resolution === 'applied' && str_starts_with( (string) $repairOriginal->fresh()->effect_resolution_reason, 'superseded_by_vpc_repair:' ) && (string) $repairOne->fresh()->effect_resolution === 'applied' && (string) CloudVpcHostState::where('vpc_id', 1) ->where('host_id', 1)->value('sync_state') === 'ready' && CloudJob::where('kind', 'vpc_port_attach') ->where('instance_id', 40)->where('phase', 'accepted')->exists(), 'A successful chained VPC repair did not converge every fence and ledger.'); $worker = file_get_contents( dirname(__DIR__) . '/app/Services/Cloud/CloudWorkerService.php' ); $cleanupPosition = strpos($worker, 'CloudVpcPortService::releaseAfterInstanceDeletedLocked'); $deletedPosition = strpos($worker, "\$instance->status = 'deleted';"); $placementPosition = strpos($worker, 'CloudPlacementService::releaseDeletedInstance'); $assert($cleanupPosition !== false && $deletedPosition !== false && $placementPosition !== false && $cleanupPosition < $deletedPosition && $deletedPosition < $placementPosition, 'Delete finalization does not fence VPC cleanup before resource release.'); echo "VPC host projection and VM port workflow passed.\n"; }