diff --git a/web/tests/CloudVpcPortWorkflowTest.php b/web/tests/CloudVpcPortWorkflowTest.php index 788655a4..b5ba6281 100644 --- a/web/tests/CloudVpcPortWorkflowTest.php +++ b/web/tests/CloudVpcPortWorkflowTest.php @@ -29,6 +29,8 @@ namespace { use App\Models\CloudVpcPort; use App\Services\Cloud\CloudSecrets; use App\Services\Cloud\CloudJobService; + use App\Services\Cloud\CloudOperationRecoveryIdentity; + use App\Services\Cloud\CloudOperationRecoveryService; use App\Services\Cloud\CloudVpcMembershipBatchService; use App\Services\Cloud\CloudVpcPortService; use App\Services\Cloud\CloudVpcProjection; @@ -108,6 +110,10 @@ namespace { $capsule->setAsGlobal(); $capsule->bootEloquent(); $schema = $capsule->schema('default'); + $schema->create('user', static function (Blueprint $table): void { + $table->increments('id'); + }); + Capsule::table('user')->insert(['id' => 7]); $schema->create('cloud_network_lines', static function ( Blueprint $table ): void { @@ -843,7 +849,7 @@ namespace { $body = $payload['body']; $instance = CloudInstance::find((int) $attachPortJob->instance_id); $batchPort = CloudVpcPort::find((int) $attachPortJob->vpc_port_id); - CloudVpcPortService::finishJobLocked($attachPortJob, 'succeeded', [ + $result = [ 'uuid' => (string) $instance->uuid, 'port_key' => (string) $batchPort->port_key, 'vpc_key' => (string) $vpc->vpc_key, @@ -861,10 +867,75 @@ namespace { 'host_digest' => (string) $body['host_digest'], ], 'status' => 'running', 'attached' => true, - ], $payload, [ - 'operation_id' => (string) $attachPortJob->operation_id, - ]); - $terminalize($attachPortJob); + ]; + if ($index === 0) { + // Recovery of one uncertain child must use the same VPC finalizer + // as an ordinary completion. In particular, it must advance the + // membership batch without completing the parent early or + // leaving the selected instance behind a permanent fence. + $identity = CloudOperationRecoveryIdentity::expected( + $attachPortJob, + true + ); + $operation = [ + 'operation_id' => (string) $attachPortJob->operation_id, + 'status' => 'succeeded', + 'effect_may_have_applied' => false, + 'safe_to_resubmit' => false, + 'retryable' => false, + 'error_code' => '', + 'action' => $identity['action'], + 'resource_id' => $identity['resource_id'], + 'request_fingerprint' => $identity['request_fingerprint'], + 'result' => $result, + ]; + $finishedAt = time(); + $attachPortJob->phase = 'terminal'; + $attachPortJob->outcome = $attachPortJob->status = 'failed'; + $attachPortJob->attempts = + $attachPortJob->dispatch_attempts = 1; + $attachPortJob->dispatch_started_at = + $attachPortJob->dispatched_at = $finishedAt - 2; + $attachPortJob->agent_accepted_at = $finishedAt - 1; + $attachPortJob->receipt_deadline_at = $finishedAt + 900; + $attachPortJob->terminal_at = $finishedAt; + $attachPortJob->effect_may_have_applied = 1; + $attachPortJob->safe_to_resubmit = 0; + $attachPortJob->effect_resolution = 'unknown'; + $attachPortJob->error_code = 'operation_effect_unknown'; + $attachPortJob->updated_at = $finishedAt; + $attachPortJob->save(); + + CloudOperationRecoveryService::prepareApplied( + (int) $attachPortJob->id, + $operation, + 'test_probe:vpc_batch_child' + ); + $preparedChild = $attachPortJob->fresh(); + $assert((string) $preparedChild->effect_resolution === 'unknown' + && str_starts_with( + (string) $preparedChild->effect_resolution_reason, + 'prepared:test_probe:vpc_batch_child:' + ), + 'Recovered VPC child released its fence before commit.'); + CloudOperationRecoveryService::commitApplied( + (int) $attachPortJob->id, + $operation, + 'test_probe:vpc_batch_child' + ); + $assert((string) $attachPortJob->fresh()->effect_resolution + === 'applied', + 'Recovered VPC child did not publish its exact resolution.'); + } else { + CloudVpcPortService::finishJobLocked( + $attachPortJob, + 'succeeded', + $result, + $payload, + ['operation_id' => (string) $attachPortJob->operation_id] + ); + $terminalize($attachPortJob); + } if ($index + 1 < $attachPortJobs->count()) { $assert((string) $attachBatch['job']->fresh()->phase !== 'terminal',