phase = 'terminal'; $job->outcome = $job->status = 'succeeded'; $job->attempts = $job->dispatch_attempts = 1; $job->dispatch_started_at = $job->dispatched_at = $now - 1; $job->agent_accepted_at = $now - 1; $job->receipt_deadline_at = $now + 900; $job->terminal_at = $now; $job->effect_may_have_applied = 0; $job->safe_to_resubmit = 0; $job->effect_resolution = 'applied'; $job->updated_at = $now; $job->save(); }; $terminalizeFailed = static function (CloudJob $job): void { $finishedAt = time() - 600; $job->phase = 'terminal'; $job->outcome = $job->status = 'failed'; $job->attempts = $job->dispatch_attempts = 1; $job->dispatch_started_at = $job->dispatched_at = $finishedAt - 1; $job->agent_accepted_at = $finishedAt - 1; $job->receipt_deadline_at = $finishedAt + 900; $job->terminal_at = $finishedAt; $job->effect_may_have_applied = 0; $job->safe_to_resubmit = 1; $job->effect_resolution = 'not_applied'; $job->error_code = 'vpc_host_remove_failed'; $job->updated_at = $finishedAt; $job->save(); }; $convergeRoutingJobs = static function ( array $jobs ) use ($terminalize): void { foreach ($jobs as $job) { $state = CloudVpcHostState::where( 'vpc_id', (int) $job->vpc_id )->where('host_id', (int) $job->host_id)->first(); if ((string) $job->kind === 'vpc_host_remove') { if ($state === null) { throw new RuntimeException( 'Route-remove test job lost its host state.' ); } $state->delete(); } else { if ($state === null) { throw new RuntimeException( 'Route-apply test job lost its host state.' ); } $state->applied_revision = (int) $state->desired_revision; $state->applied_digest = (string) $state->desired_digest; $state->sync_state = 'ready'; $state->last_error_code = ''; $state->save(); } $terminalize($job); } }; $capsule = new Capsule(); $capsule->addConnection([ 'driver' => 'sqlite', 'database' => ':memory:', ], 'default'); $capsule->setAsGlobal(); $capsule->bootEloquent(); $schema = $capsule->schema('default'); $schema->create('cloud_network_lines', static function ( Blueprint $table ): void { $table->increments('id'); $table->integer('region_id'); }); $schema->create('cloud_resource_pools', static function ( Blueprint $table ): void { $table->increments('id'); $table->integer('line_id'); }); $schema->create('cloud_hosts', static function (Blueprint $table): void { $table->increments('id'); $table->boolean('enabled'); }); $schema->create('cloud_host_vpc_transports', static function ( Blueprint $table ): void { $table->integer('host_id')->primary(); $table->string('transport_address'); $table->integer('transport_prefix_length'); $table->string('transport_interface'); $table->integer('mtu'); $table->integer('desired_revision'); $table->integer('observed_revision'); $table->string('sync_state'); }); $schema->create('cloud_instances', static function ( Blueprint $table ): void { $table->increments('id'); $table->string('uuid'); $table->integer('host_id'); $table->integer('user_id'); $table->integer('resource_pool_id')->nullable(); $table->string('name'); $table->string('status'); $table->string('service_status'); $table->string('network_status'); $table->integer('expires_at')->nullable(); $table->text('spec_json'); $table->integer('created_at'); $table->integer('updated_at'); }); $schema->create('cloud_vpcs', static function (Blueprint $table): void { $table->increments('id'); $table->string('vpc_key'); $table->integer('user_id'); $table->integer('region_id'); $table->integer('cidr_pool_id'); $table->string('name'); $table->string('cidr'); $table->integer('prefix_length'); $table->string('gateway'); $table->integer('active_claim')->nullable(); $table->string('state'); $table->integer('desired_revision'); $table->integer('deleted_at')->nullable(); $table->integer('created_at'); $table->integer('updated_at'); }); $schema->create('cloud_vpc_ports', static function ( Blueprint $table ): void { $table->increments('id'); $table->string('port_key')->unique(); $table->integer('vpc_id'); $table->integer('instance_id'); $table->integer('migration_host_id')->nullable(); $table->integer('user_id'); $table->string('private_ip'); $table->string('mac_address')->unique(); $table->integer('active_claim')->nullable(); $table->string('state'); $table->integer('desired_revision'); $table->integer('applied_revision'); $table->string('last_operation_id')->nullable(); $table->string('last_error_code'); $table->integer('attached_at')->nullable(); $table->integer('released_at')->nullable(); $table->integer('created_at'); $table->integer('updated_at'); $table->unique(['vpc_id', 'private_ip', 'active_claim']); $table->unique(['instance_id', 'active_claim']); }); $schema->create('cloud_vpc_host_states', static function ( Blueprint $table ): void { $table->increments('id'); $table->integer('vpc_id'); $table->integer('host_id'); $table->integer('desired_revision'); $table->integer('applied_revision'); $table->string('desired_digest'); $table->string('applied_digest')->nullable(); $table->string('sync_state'); $table->string('last_operation_id')->nullable(); $table->string('last_error_code'); $table->integer('last_success_at'); $table->integer('last_failure_at'); $table->integer('created_at'); $table->integer('updated_at'); $table->unique(['vpc_id', 'host_id']); }); $schema->create('cloud_jobs', static function (Blueprint $table): void { $table->increments('id'); $table->string('operation_id')->unique(); $table->string('request_idempotency_key'); $table->string('request_fingerprint'); $table->integer('parent_job_id')->nullable(); $table->integer('order_id')->nullable(); $table->integer('invoice_id')->nullable(); $table->integer('instance_id')->nullable(); $table->integer('snapshot_id')->nullable(); $table->integer('backup_id')->nullable(); $table->integer('backup_restore_id')->nullable(); $table->integer('backup_recovery_id')->nullable(); $table->integer('vpc_id')->nullable(); $table->integer('vpc_port_id')->nullable(); $table->integer('user_id'); $table->integer('host_id')->nullable(); $table->string('kind'); $table->string('execution_mode'); $table->string('phase'); $table->string('outcome')->nullable(); $table->string('status'); $table->text('sealed_payload'); $table->integer('attempts'); $table->integer('dispatch_attempts'); $table->integer('dispatch_deadline_at'); $table->integer('dispatch_started_at')->nullable(); $table->integer('agent_accepted_at')->nullable(); $table->integer('receipt_deadline_at')->nullable(); $table->integer('terminal_at')->nullable(); $table->boolean('effect_may_have_applied')->nullable(); $table->boolean('safe_to_resubmit')->nullable(); $table->string('effect_resolution'); $table->integer('effect_resolved_at')->nullable(); $table->integer('effect_resolved_by')->nullable(); $table->text('effect_resolution_reason')->default(''); $table->boolean('finalization_incident')->default(0); $table->string('finalization_error_code')->nullable(); $table->string('completion_event_id')->nullable(); $table->integer('read_at')->nullable(); $table->integer('dispatched_at')->nullable(); $table->string('lease_token')->nullable(); $table->integer('lease_until')->default(0); $table->integer('available_at'); $table->string('error_code')->nullable(); $table->text('result_json')->nullable(); $table->integer('created_at'); $table->integer('updated_at'); $table->unique(['user_id', 'request_idempotency_key']); }); Capsule::table('cloud_network_lines')->insert([ 'id' => 1, 'region_id' => 1, ]); Capsule::table('cloud_resource_pools')->insert([ 'id' => 1, 'line_id' => 1, ]); foreach ([1, 2] as $hostId) { Capsule::table('cloud_hosts')->insert([ 'id' => $hostId, 'enabled' => 1, ]); Capsule::table('cloud_host_vpc_transports')->insert([ 'host_id' => $hostId, 'transport_address' => '172.16.10.' . (10 + $hostId), 'transport_prefix_length' => 24, 'transport_interface' => 'br-private', 'mtu' => 1500, 'desired_revision' => 1, 'observed_revision' => 1, 'sync_state' => 'ready', ]); } $newInstance = static function (int $id, int $hostId): CloudInstance { $instance = new CloudInstance(); $instance->id = $id; $instance->uuid = sprintf('11111111-1111-4111-8111-%012d', $id); $instance->host_id = $hostId; $instance->user_id = 7; $instance->resource_pool_id = 1; $instance->name = 'vm-' . $id; $instance->status = 'active'; $instance->service_status = 'exempt'; $instance->network_status = 'active'; $instance->expires_at = null; $instance->spec_json = json_encode(['os_source' => 'cloud_template'], JSON_THROW_ON_ERROR); $instance->created_at = $instance->updated_at = time(); $instance->save(); return $instance; }; $target = $newInstance(1, 1); $peer = $newInstance(2, 2); $vpc = new CloudVpc(); $vpc->id = 1; $vpc->vpc_key = str_repeat('a', 32); $vpc->user_id = 7; $vpc->region_id = 1; $vpc->cidr_pool_id = 1; $vpc->name = 'private'; $vpc->cidr = '10.60.0.0/24'; $vpc->prefix_length = 24; $vpc->gateway = '10.60.0.1'; $vpc->active_claim = 1; $vpc->state = 'active'; $vpc->desired_revision = 1; $vpc->deleted_at = null; $vpc->created_at = $vpc->updated_at = time(); $vpc->save(); $peerPort = new CloudVpcPort(); $peerPort->id = 1; $peerPort->port_key = str_repeat('b', 32); $peerPort->vpc_id = 1; $peerPort->instance_id = 2; $peerPort->user_id = 7; $peerPort->private_ip = '10.60.0.3'; $peerPort->mac_address = '02:00:00:00:00:03'; $peerPort->active_claim = 1; $peerPort->state = 'attached'; $peerPort->desired_revision = $peerPort->applied_revision = 1; $peerPort->last_operation_id = null; $peerPort->last_error_code = ''; $peerPort->attached_at = time(); $peerPort->released_at = null; $peerPort->created_at = $peerPort->updated_at = time(); $peerPort->save(); $initial = CloudVpcProjection::build([ 'vpc_key' => (string) $vpc->vpc_key, 'cidr' => (string) $vpc->cidr, 'gateway' => (string) $vpc->gateway, ], [['private_ip' => '10.60.0.3', 'mac_address' => '02:00:00:00:00:03', 'host_id' => 2]], [ 2 => ['transport_address' => '172.16.10.12', 'transport_prefix_length' => 24, 'transport_interface' => 'br-private', 'mtu' => 1500], ], 1); $state = new CloudVpcHostState(); $state->vpc_id = 1; $state->host_id = 2; $state->desired_revision = $state->applied_revision = 1; $state->desired_digest = $state->applied_digest = $initial[2]['digest']; $state->sync_state = 'ready'; $state->last_operation_id = null; $state->last_error_code = ''; $state->last_success_at = time(); $state->last_failure_at = 0; $state->created_at = $state->updated_at = time(); $state->save(); $actor = ['user_id' => 7, 'request_idempotency_key' => 'vpc-attach-request-0001']; $queued = CloudVpcPortService::attach($target, $actor, [ 'vpc_key' => (string) $vpc->vpc_key, 'confirm_restart' => true, ]); $assert($queued['job'] !== null && CloudJob::where('kind', 'vpc_host_apply')->count() === 2, 'Attach did not queue a complete two-host projection.'); $replayedAttach = CloudVpcPortService::attach($target, $actor, [ 'vpc_key' => (string) $vpc->vpc_key, 'confirm_restart' => true, ]); $activeHostJobIds = CloudJob::where('kind', 'vpc_host_apply') ->pluck('id')->map(static fn ($id): int => (int) $id)->all(); $assert($replayedAttach['job'] !== null && in_array((int) $replayedAttach['job']->id, $activeHostJobIds, true) && CloudJob::where('kind', 'vpc_host_apply')->count() === 2, 'An accepted attach retry did not return its existing workflow.'); $reconnectedAttach = CloudVpcPortService::attach($target, [ 'user_id' => 7, 'request_idempotency_key' => 'vpc-attach-new-browser-0002', ], [ 'vpc_key' => (string) $vpc->vpc_key, 'confirm_restart' => true, ]); $assert($reconnectedAttach['job'] !== null && in_array((int) $reconnectedAttach['job']->id, $activeHostJobIds, true) && CloudJob::where('kind', 'vpc_host_apply')->count() === 2, 'A reconnect with a new browser request key duplicated the accepted workflow.'); $port = CloudVpcPort::where('instance_id', 1) ->where('active_claim', 1)->first(); $assert($port !== null && (string) $port->state === 'preparing' && (string) $port->private_ip === '10.60.0.2', 'Attach did not reserve the first usable free address.'); $listing = CloudVpcPortService::listing($target, ['user_id' => 7]); $assert(($listing['active_task']['task_id'] ?? null) === (string) $replayedAttach['job']->id && ($listing['can_attach'] ?? true) === false, 'The read model did not expose the tenant VPC task after reconnect.'); foreach (CloudJob::where('kind', 'vpc_host_apply')->orderBy('id')->get() as $hostJob) { $payload = CloudSecrets::decrypt( (string) $hostJob->sealed_payload, 'job-payload' ); $hostState = CloudVpcHostState::where('vpc_id', 1) ->where('host_id', (int) $hostJob->host_id)->first(); CloudVpcPortService::finishJobLocked($hostJob, 'succeeded', [ 'vpc_key' => (string) $vpc->vpc_key, 'desired_revision' => (int) $hostState->desired_revision, 'desired_digest' => (string) $hostState->desired_digest, 'bridge' => CloudVpcProjection::bridgeName((string) $vpc->vpc_key), 'applied' => true, ], $payload, ['operation_id' => (string) $hostJob->operation_id]); $terminalize($hostJob); } $attachJob = CloudJob::where('kind', 'vpc_port_attach')->first(); $assert($attachJob !== null, 'Converged host projections did not queue the VM port attach.'); $attachPayload = CloudSecrets::decrypt( (string) $attachJob->sealed_payload, 'job-payload' ); $body = $attachPayload['body']; CloudVpcPortService::finishJobLocked($attachJob, 'succeeded', [ 'uuid' => (string) $target->uuid, 'port_key' => (string) $port->port_key, 'vpc_key' => (string) $vpc->vpc_key, 'vpc_port' => [ 'port_key' => (string) $port->port_key, 'vpc_key' => (string) $vpc->vpc_key, 'private_ip' => (string) $port->private_ip, 'cidr' => (string) $vpc->cidr, 'gateway' => (string) $vpc->gateway, 'mac' => (string) $port->mac_address, 'bridge' => CloudVpcProjection::bridgeName((string) $vpc->vpc_key), 'host_revision' => (int) $body['host_revision'], 'host_digest' => (string) $body['host_digest'], ], 'status' => 'running', 'attached' => true, ], $attachPayload, ['operation_id' => (string) $attachJob->operation_id]); $terminalize($attachJob); $assert((string) $port->fresh()->state === 'attached', 'Successful VM port result did not attach the port ledger.'); $migration = new CloudInstanceMigration(); $migration->id = 99; $migration->instance_id = 1; $migration->vpc_port_id = CloudVpcPortService::migrationPortIdLocked( $target, 2 ); $migration->source_host_id = 1; $migration->target_host_id = 2; $migration->source_uuid = (string) $target->uuid; $targetJobs = $target->getConnection()->transaction( static fn (): array => CloudVpcPortService::beginMigrationToTargetLocked( $migration, $target ) ); $migrationPayload = CloudSecrets::decrypt( (string) $targetJobs[0]->sealed_payload, 'job-payload' ); $assert(count($targetJobs) === 2 && ($migrationPayload['maintenance_migration_id'] ?? null) === 99 && ($migrationPayload['vpc_migration_direction'] ?? null) === 'target' && (string) $port->fresh()->state === 'migrating' && (int) $port->fresh()->migration_host_id === 2, 'Maintenance migration did not durably project the port to target.'); $convergeRoutingJobs($targetJobs); $assert(CloudVpcPortService::finishMigrationProjectionLocked( $migration, $target, 'target' ) === true, 'Converged target VPC projection did not complete.'); $targetMetadata = CloudVpcPortService::migrationTargetMetadataLocked( $migration, $target ); $assert(($targetMetadata['private_ip'] ?? null) === '10.60.0.2' && ($targetMetadata['mac'] ?? null) === (string) $port->mac_address && ($targetMetadata['host_revision'] ?? 0) > 1, 'Migration target metadata did not preserve the logical port identity.'); CloudVpcPortService::finalizeMigrationCutoverLocked($migration, $target); $cutoverPort = $port->fresh(); $assert((string) $cutoverPort->state === 'attached' && $cutoverPort->migration_host_id === null && (string) $cutoverPort->private_ip === '10.60.0.2' && (string) $cutoverPort->mac_address === (string) $port->mac_address, 'VPC cutover changed or retained migration state on the logical port.'); $target->host_id = 2; $target->save(); $reverse = new CloudInstanceMigration(); $reverse->id = 100; $reverse->instance_id = 1; $reverse->vpc_port_id = CloudVpcPortService::migrationPortIdLocked( $target, 1 ); $reverse->source_host_id = 2; $reverse->target_host_id = 1; $reverse->source_uuid = (string) $target->uuid; $reverseJobs = $target->getConnection()->transaction( static fn (): array => CloudVpcPortService::beginMigrationToTargetLocked( $reverse, $target ) ); $convergeRoutingJobs($reverseJobs); $assert(CloudVpcPortService::finishMigrationProjectionLocked( $reverse, $target, 'target' ), 'Reverse target projection did not converge.'); CloudVpcPortService::finalizeMigrationCutoverLocked($reverse, $target); $target->host_id = 1; $target->save(); $abort = new CloudInstanceMigration(); $abort->id = 101; $abort->instance_id = 1; $abort->vpc_port_id = CloudVpcPortService::migrationPortIdLocked( $target, 2 ); $abort->source_host_id = 1; $abort->target_host_id = 2; $abort->source_uuid = (string) $target->uuid; $abortTargetJobs = $target->getConnection()->transaction( static fn (): array => CloudVpcPortService::beginMigrationToTargetLocked( $abort, $target ) ); $unknownJob = $abortTargetJobs[0]; foreach ($abortTargetJobs as $routeJob) { $state = CloudVpcHostState::where('vpc_id', 1) ->where('host_id', (int) $routeJob->host_id)->first(); if ((int) $routeJob->id === (int) $unknownJob->id) { $state->sync_state = 'unknown'; $state->last_operation_id = (string) $routeJob->operation_id; $state->last_error_code = 'vpc_result_uncertain'; $state->save(); $routeJob->phase = 'terminal'; $routeJob->outcome = $routeJob->status = 'failed'; $routeJob->effect_may_have_applied = 1; $routeJob->safe_to_resubmit = 0; $routeJob->effect_resolution = 'unknown'; $routeJob->terminal_at = $routeJob->updated_at = time(); $routeJob->save(); continue; } if ((string) $routeJob->kind === 'vpc_host_remove') { $state->delete(); } else { $state->applied_revision = (int) $state->desired_revision; $state->applied_digest = (string) $state->desired_digest; $state->sync_state = 'ready'; $state->save(); } $terminalize($routeJob); } $jobsBeforeUnknownAbort = CloudJob::count(); $unknownReplayBlocked = false; try { $target->getConnection()->transaction( static fn (): array => CloudVpcPortService::beginMigrationToSourceLocked( $abort, $target ) ); } catch (Throwable $error) { $unknownReplayBlocked = true; } $assert($unknownReplayBlocked && CloudJob::count() === $jobsBeforeUnknownAbort, 'Unknown VPC routing effect was overwritten by source rollback.'); $unknownState = CloudVpcHostState::where('vpc_id', 1) ->where('host_id', (int) $unknownJob->host_id)->first(); $unknownState->sync_state = 'failed'; $unknownState->save(); $terminalizeFailed($unknownJob); $sourceJobs = $target->getConnection()->transaction( static fn (): array => CloudVpcPortService::beginMigrationToSourceLocked( $abort, $target ) ); $convergeRoutingJobs($sourceJobs); $assert(CloudVpcPortService::finishMigrationProjectionLocked( $abort, $target, 'source' ) && (string) $port->fresh()->state === 'attached' && $port->fresh()->migration_host_id === null, 'Confirmed-not-applied abort did not restore the source VPC route.'); $detachActor = ['user_id' => 7, 'request_idempotency_key' => 'vpc-detach-request-0001']; $detaching = CloudVpcPortService::detach($target, $detachActor, [ 'confirm_restart' => true, ]); $detachJob = $detaching['job']; $replayedDetach = CloudVpcPortService::detach($target, $detachActor, [ 'confirm_restart' => true, ]); $assert((int) ($replayedDetach['job']->id ?? 0) === (int) $detachJob->id && CloudJob::where('kind', 'vpc_port_detach')->count() === 1, 'An accepted detach retry did not return its existing workflow.'); $detachPayload = CloudSecrets::decrypt( (string) $detachJob->sealed_payload, 'job-payload' ); CloudVpcPortService::finishJobLocked($detachJob, 'succeeded', [ 'uuid' => (string) $target->uuid, 'port_key' => (string) $port->port_key, 'vpc_key' => (string) $vpc->vpc_key, 'status' => 'running', 'removed' => true, ], $detachPayload, ['operation_id' => (string) $detachJob->operation_id]); $terminalize($detachJob); $assert(CloudJob::whereIn('kind', ['vpc_host_apply', 'vpc_host_remove'])->where('phase', 'accepted') ->count() === 2, 'Detach did not queue route cleanup after VM port removal.'); $cleanupJobs = CloudJob::whereIn('kind', ['vpc_host_apply', 'vpc_host_remove'])->where('phase', 'accepted') ->orderBy('id')->get(); foreach ($cleanupJobs as $cleanupJob) { $payload = CloudSecrets::decrypt( (string) $cleanupJob->sealed_payload, 'job-payload' ); $hostState = CloudVpcHostState::where('vpc_id', 1) ->where('host_id', (int) $cleanupJob->host_id)->first(); $result = [ 'vpc_key' => (string) $vpc->vpc_key, 'desired_revision' => (int) ($cleanupJob->kind === 'vpc_host_apply' ? $hostState->desired_revision : $hostState->applied_revision), 'desired_digest' => (string) ($cleanupJob->kind === 'vpc_host_apply' ? $hostState->desired_digest : $hostState->applied_digest), ]; if ((string) $cleanupJob->kind === 'vpc_host_apply') { $result['bridge'] = CloudVpcProjection::bridgeName( (string) $vpc->vpc_key ); $result['applied'] = true; } else { $result['removed'] = true; } CloudVpcPortService::finishJobLocked($cleanupJob, 'succeeded', $result, $payload, ['operation_id' => (string) $cleanupJob->operation_id]); $terminalize($cleanupJob); } $released = $port->fresh(); $assert((string) $released->state === 'released' && $released->active_claim === null && CloudVpcHostState::where('vpc_id', 1)->count() === 1 && (int) CloudVpcHostState::where('vpc_id', 1)->value('host_id') === 2, 'Detach did not release the IP after every host cleanup converged.'); $knownFailedState = CloudVpcHostState::where('vpc_id', 1) ->where('host_id', 2)->first(); $knownFailedState->desired_revision = (int) $knownFailedState->desired_revision + 1; $knownFailedState->desired_digest = str_repeat('f', 64); $knownFailedState->sync_state = 'failed'; $knownFailedState->last_error_code = 'vpc_host_apply_failed'; $knownFailedState->save(); $knownFailedJob = CloudJob::where( 'operation_id', (string) $knownFailedState->last_operation_id )->first(); $assert($knownFailedJob !== null, 'Failed host-state fixture lost its exact operation generation.'); $terminalizeFailed($knownFailedJob); $vpc->desired_revision = (int) $knownFailedState->desired_revision; $vpc->state = 'degraded'; $vpc->save(); $deleteCleanupJobs = $peer->getConnection()->transaction( static function (): array { $locked = CloudInstance::where('id', 2) ->lockForUpdate()->first(); return CloudVpcPortService::releaseAfterInstanceDeletedLocked( $locked ); } ); $peerPort = $peerPort->fresh(); $assert(count($deleteCleanupJobs) === 1 && (string) $deleteCleanupJobs[0]->kind === 'vpc_host_remove' && (string) $peerPort->state === 'release_pending' && $peerPort->active_claim !== null, 'Confirmed instance deletion released its private IP before route cleanup.'); $deleteCleanupJob = $deleteCleanupJobs[0]; $deleteCleanupPayload = CloudSecrets::decrypt( (string) $deleteCleanupJob->sealed_payload, 'job-payload' ); $deleteHostState = CloudVpcHostState::where('vpc_id', 1) ->where('host_id', 2)->first(); CloudVpcPortService::finishJobLocked($deleteCleanupJob, 'failed', null, $deleteCleanupPayload, [ 'operation_id' => (string) $deleteCleanupJob->operation_id, 'error_code' => 'vpc_host_remove_failed', ]); $terminalizeFailed($deleteCleanupJob); $peer->status = 'deleted'; $peer->save(); $deleteCleanupJob->effect_resolution = 'unknown'; $deleteCleanupJob->effect_may_have_applied = 1; $deleteCleanupJob->safe_to_resubmit = 0; $deleteCleanupJob->save(); $jobCountBeforeUnknownRetry = CloudJob::count(); $assert(CloudVpcPortService::retryDeletedCleanupDue(10) === 0 && CloudJob::count() === $jobCountBeforeUnknownRetry, 'Unknown-effect deleted-instance cleanup was replayed.'); $terminalizeFailed($deleteCleanupJob); $assert(CloudVpcPortService::retryDeletedCleanupDue(10) === 1, 'Known-not-applied deleted-instance cleanup was not retried.'); $peerPort = $peerPort->fresh(); $assert((string) $peerPort->state === 'release_pending' && $peerPort->active_claim !== null, 'A failed deleted-instance cleanup released the address early.'); $deleteCleanupJob = CloudJob::where('kind', 'vpc_host_remove') ->where('phase', 'accepted')->orderByDesc('id')->first(); $deleteCleanupPayload = CloudSecrets::decrypt( (string) $deleteCleanupJob->sealed_payload, 'job-payload' ); $deleteHostState = CloudVpcHostState::where('vpc_id', 1) ->where('host_id', 2)->first(); CloudVpcPortService::finishJobLocked($deleteCleanupJob, 'succeeded', [ 'vpc_key' => (string) $vpc->vpc_key, 'desired_revision' => (int) $deleteHostState->applied_revision, 'desired_digest' => (string) $deleteHostState->applied_digest, 'removed' => true, ], $deleteCleanupPayload, [ 'operation_id' => (string) $deleteCleanupJob->operation_id, ]); $terminalize($deleteCleanupJob); $peerPort = $peerPort->fresh(); $assert((string) $peerPort->state === 'released' && $peerPort->active_claim === null && CloudVpcHostState::where('vpc_id', 1)->count() === 0, 'Deleted instance VPC cleanup did not release the address after convergence.'); $worker = file_get_contents( dirname(__DIR__) . '/app/Services/Cloud/CloudWorkerService.php' ); $cleanupPosition = strpos($worker, 'CloudVpcPortService::releaseAfterInstanceDeletedLocked'); $deletedPosition = strpos($worker, "\$instance->status = 'deleted';"); $placementPosition = strpos($worker, 'CloudPlacementService::releaseDeletedInstance'); $assert($cleanupPosition !== false && $deletedPosition !== false && $placementPosition !== false && $cleanupPosition < $deletedPosition && $deletedPosition < $placementPosition, 'Delete finalization does not fence VPC cleanup before resource release.'); echo "VPC host projection and VM port workflow passed.\n"; }