requests[] = $request; if ($this->responses === []) { throw new \RuntimeException('Unexpected Agent request.'); } $response = array_shift($this->responses); if (is_callable($response)) { $response = $response($request); } if ($response instanceof \Throwable) { throw $response; } if (!is_array($response)) { throw new \RuntimeException('Invalid test Agent response.'); } return $response; } } final class CloudAgentSettingsService { public static ?AutomaticRecoveryTestAgentClient $client = null; public static function clientForConfiguration( int $hostId, float $timeoutSeconds = 120.0 ): AutomaticRecoveryTestAgentClient { if ($hostId !== 7 || $timeoutSeconds !== 30.0 || self::$client === null) { throw new \RuntimeException( 'Unexpected automatic recovery client request.' ); } return self::$client; } } } namespace { require dirname(__DIR__) . '/vendor/autoload.php'; use App\Models\CloudJob; use App\Services\Cloud\AutomaticRecoveryTestAgentClient; use App\Services\Cloud\CloudAgentError; use App\Services\Cloud\CloudAgentSettingsService; use App\Services\Cloud\CloudAutomaticOperationRecoveryService; use App\Services\Cloud\CloudError; use App\Services\Cloud\CloudJobService; use App\Services\Cloud\CloudOperationCompletionService; use App\Services\Cloud\CloudOperationRecoveryIdentity; use App\Services\Cloud\CloudOperationRecoveryService; use App\Services\Cloud\CloudRecoveryInstanceStateConflict; use App\Services\Cloud\CloudRecoveryOperationIdentityConflict; use App\Services\Cloud\CloudSecrets; use App\Services\Cloud\CloudWorkerService; use Illuminate\Database\Capsule\Manager as Capsule; use Illuminate\Database\Schema\Blueprint; $assert = static function (bool $condition, string $message): void { if (!$condition) { throw new RuntimeException($message); } }; $capsule = new Capsule(); $capsule->addConnection([ 'driver' => 'sqlite', 'database' => ':memory:', ], 'default'); $capsule->setAsGlobal(); $capsule->bootEloquent(); $schema = $capsule->schema('default'); $schema->create('user', static function (Blueprint $table): void { $table->increments('id'); $table->integer('enable')->default(1); $table->integer('is_admin')->default(0); }); $schema->create('cloud_jobs', static function (Blueprint $table): void { $table->increments('id'); $table->string('operation_id')->unique(); $table->string('request_idempotency_key')->unique(); $table->string('request_fingerprint'); $table->integer('parent_job_id')->nullable(); $table->integer('order_id')->nullable(); $table->integer('invoice_id')->nullable(); $table->integer('instance_id')->nullable(); $table->integer('snapshot_id')->nullable(); $table->integer('vpc_repair_root_job_id')->nullable(); $table->integer('user_id'); $table->integer('host_id')->nullable(); $table->string('kind'); $table->string('execution_mode'); $table->string('phase'); $table->string('outcome')->nullable(); $table->string('status'); $table->text('sealed_payload'); $table->integer('attempts')->default(0); $table->integer('dispatch_attempts')->default(0); $table->integer('dispatch_deadline_at'); $table->integer('dispatch_started_at')->nullable(); $table->integer('agent_accepted_at')->nullable(); $table->integer('receipt_deadline_at')->nullable(); $table->integer('terminal_at')->nullable(); $table->boolean('effect_may_have_applied')->nullable(); $table->boolean('safe_to_resubmit')->nullable(); $table->string('effect_resolution'); $table->integer('effect_resolved_at')->nullable(); $table->integer('effect_resolved_by')->nullable(); $table->string('effect_resolution_reason')->default(''); $table->string('completion_event_id')->nullable(); $table->integer('read_at')->nullable(); $table->integer('dispatched_at')->nullable(); $table->string('lease_token')->nullable(); $table->integer('lease_until')->default(0); $table->integer('available_at')->default(0); $table->string('error_code')->nullable(); $table->boolean('finalization_incident')->default(0); $table->string('finalization_error_code')->nullable(); $table->text('result_json')->nullable(); $table->integer('created_at'); $table->integer('updated_at'); }); $schema->create('cloud_instances', static function (Blueprint $table): void { $table->increments('id'); $table->string('uuid'); $table->integer('host_id'); $table->integer('user_id'); $table->integer('order_id')->nullable(); $table->integer('plan_id')->nullable(); $table->integer('offer_id')->nullable(); $table->integer('resource_pool_id')->nullable(); $table->integer('reservation_id')->nullable(); $table->integer('current_period_id')->nullable(); $table->string('name'); $table->string('status'); $table->string('service_status'); $table->string('network_status'); $table->integer('expires_at')->nullable(); $table->decimal('renewal_price', 12, 2)->nullable(); $table->integer('renewal_days')->default(0); $table->integer('renewal_traffic_bytes')->default(0); $table->integer('traffic_quota_bytes')->default(0); $table->integer('traffic_download_bytes')->default(0); $table->integer('traffic_upload_bytes')->default(0); $table->integer('lifetime_download_bytes')->default(0); $table->integer('lifetime_upload_bytes')->default(0); $table->integer('traffic_synced_at')->default(0); $table->integer('suspended_at')->nullable(); $table->integer('network_suspended_at')->nullable(); $table->text('spec_json'); $table->text('sealed_credential')->nullable(); $table->integer('created_at'); $table->integer('updated_at'); }); $schema->create('cloud_agent_completion_events', static function (Blueprint $table): void { $table->string('event_id')->primary(); $table->integer('host_id'); $table->string('operation_id'); $table->string('payload_sha256'); $table->string('outcome'); $table->boolean('effect_may_have_applied'); $table->boolean('safe_to_resubmit'); $table->string('error_code')->default(''); $table->text('result_json')->nullable(); $table->string('disposition')->default('received'); $table->integer('received_at'); $table->integer('applied_at')->nullable(); $table->unique(['host_id', 'operation_id']); }); $schema->create('cloud_instance_snapshots', static function (Blueprint $table): void { $table->increments('id'); $table->integer('instance_id'); $table->string('state'); $table->string('last_operation_id')->default(''); $table->integer('deleted_at')->nullable(); $table->integer('updated_at'); }); $schema->create('cloud_templates', static function (Blueprint $table): void { $table->string('id')->primary(); $table->string('catalog_key')->default('test-catalog'); $table->string('version_label')->default('test-version'); $table->boolean('is_default')->default(true); $table->integer('active_job_id')->nullable(); $table->string('state')->default('active'); $table->integer('revision')->default(1); $table->string('generation_token')->default(''); $table->integer('image_bytes')->default(0); $table->string('sha256')->default(''); $table->integer('updated_at')->default(0); }); $schema->create('cloud_user_isos', static function (Blueprint $table): void { $table->string('id')->primary(); $table->integer('active_job_id')->nullable(); $table->string('state')->default('active'); }); $schema->create('cloud_template_replicas', static function (Blueprint $table): void { $table->integer('host_id'); $table->string('template_id'); $table->integer('revision')->default(0); $table->string('generation_token')->default(''); $table->string('state')->default('pending'); $table->integer('image_bytes')->default(0); $table->string('sha256')->default(''); $table->string('operation_id')->default(''); $table->integer('job_id')->nullable(); $table->string('error_code')->default(''); $table->integer('created_at')->default(0); $table->integer('updated_at')->default(0); $table->primary(['host_id', 'template_id']); }); $schema->create('cloud_user_iso_replicas', static function (Blueprint $table): void { $table->integer('host_id'); $table->string('user_iso_id'); $table->integer('user_id'); $table->string('state')->default('pending'); $table->integer('size_bytes')->default(0); $table->string('sha256')->default(''); $table->string('operation_id')->default(''); $table->integer('job_id')->nullable(); $table->string('error_code')->default(''); $table->integer('created_at')->default(0); $table->integer('updated_at')->default(0); $table->primary(['host_id', 'user_iso_id']); }); Capsule::table('user')->insert([ 'id' => 1, 'enable' => 1, 'is_admin' => 0, ]); $now = time(); $nextId = 100; $insertSettingsJob = static function ( string $operationId, array $body, ?array $credential = null ) use (&$nextId, $now): array { $id = $nextId++; $uuid = sprintf( '00000000-0000-4000-8000-%012d', $id ); Capsule::table('cloud_instances')->insert([ 'id' => $id, 'uuid' => $uuid, 'host_id' => 7, 'user_id' => 1, 'order_id' => null, 'plan_id' => null, 'offer_id' => null, 'resource_pool_id' => null, 'reservation_id' => null, 'current_period_id' => null, 'name' => 'settings-' . $id, 'status' => 'active', 'service_status' => 'exempt', 'network_status' => 'active', 'expires_at' => null, 'renewal_price' => null, 'renewal_days' => 0, 'renewal_traffic_bytes' => 0, 'traffic_quota_bytes' => 0, 'traffic_download_bytes' => 0, 'traffic_upload_bytes' => 0, 'lifetime_download_bytes' => 0, 'lifetime_upload_bytes' => 0, 'traffic_synced_at' => 0, 'suspended_at' => null, 'network_suspended_at' => null, 'spec_json' => json_encode( ['managed' => true], JSON_THROW_ON_ERROR ), 'sealed_credential' => null, 'created_at' => $now - 3600, 'updated_at' => $now - 3600, ]); $fields = []; foreach ([ 'hostname' => 'hostname', 'root_password' => 'password', 'ssh_public_key' => 'ssh_key', ] as $field => $code) { if (array_key_exists($field, $body)) { $fields[] = $code; } } $payload = [ 'method' => 'POST', 'request_target' => '/v1/vms/' . $uuid . '/settings', 'body' => $body + ['confirm_restart' => true], 'owner_id' => 1, 'can_manage' => false, 'credential' => $credential, 'settings_fields' => $fields, ]; Capsule::table('cloud_jobs')->insert([ 'id' => $id, 'operation_id' => $operationId, 'request_idempotency_key' => $operationId, 'request_fingerprint' => hash('sha256', $operationId), 'parent_job_id' => null, 'order_id' => null, 'invoice_id' => null, 'instance_id' => $id, 'snapshot_id' => null, 'user_id' => 1, 'host_id' => 7, 'kind' => 'instance_settings', 'execution_mode' => 'agent', 'phase' => 'terminal', 'outcome' => 'failed', 'status' => 'failed', 'sealed_payload' => CloudSecrets::encrypt( $payload, 'job-payload' ), 'attempts' => 1, 'dispatch_attempts' => 1, 'dispatch_deadline_at' => $now - 1000, 'dispatch_started_at' => $now - 990, 'agent_accepted_at' => $now - 980, 'receipt_deadline_at' => $now - 20, 'terminal_at' => $now - 10, 'effect_may_have_applied' => 1, 'safe_to_resubmit' => 0, 'effect_resolution' => 'unknown', 'effect_resolved_at' => null, 'effect_resolved_by' => null, 'effect_resolution_reason' => '', 'completion_event_id' => null, 'read_at' => null, 'dispatched_at' => $now - 980, 'lease_token' => null, 'lease_until' => 0, 'available_at' => 0, 'error_code' => 'operation_deadline_exceeded', 'finalization_incident' => 0, 'finalization_error_code' => null, 'result_json' => json_encode( ['cause_code' => 'operation_deadline_exceeded'], JSON_THROW_ON_ERROR ), 'created_at' => $now - 1000, 'updated_at' => $now - 10, ]); return [$id, $uuid]; }; $insertDispatchableJob = static function ( string $operationId ) use (&$nextId, $now): int { $id = $nextId++; Capsule::table('cloud_jobs')->insert([ 'id' => $id, 'operation_id' => $operationId, 'request_idempotency_key' => $operationId, 'request_fingerprint' => hash('sha256', $operationId), 'parent_job_id' => null, 'order_id' => null, 'invoice_id' => null, 'instance_id' => null, 'snapshot_id' => null, // A missing actor closes safely before dispatch. It keeps this // fixture focused on the public worker scheduling seam rather // than sharing an HTTP transport with system recovery. 'user_id' => 999, 'host_id' => 7, 'kind' => 'power_start', 'execution_mode' => 'agent', 'phase' => 'accepted', 'outcome' => null, 'status' => 'queued', 'sealed_payload' => CloudSecrets::encrypt([ 'method' => 'POST', 'request_target' => '/v1/vms/' . '00000000-0000-4000-8000-000000000999/start', 'body' => [], 'owner_id' => 999, 'can_manage' => false, ], 'job-payload'), 'attempts' => 0, 'dispatch_attempts' => 0, 'dispatch_deadline_at' => $now + 300, 'dispatch_started_at' => null, 'agent_accepted_at' => null, 'receipt_deadline_at' => null, 'terminal_at' => null, 'effect_may_have_applied' => null, 'safe_to_resubmit' => null, 'effect_resolution' => 'not_applicable', 'effect_resolved_at' => null, 'effect_resolved_by' => null, 'effect_resolution_reason' => '', 'completion_event_id' => null, 'read_at' => null, 'dispatched_at' => null, 'lease_token' => null, 'lease_until' => 0, 'available_at' => 0, 'error_code' => null, 'finalization_incident' => 0, 'finalization_error_code' => null, 'result_json' => null, 'created_at' => $now, 'updated_at' => $now, ]); return $id; }; $audit = static function (int $id): array { $job = CloudJob::find($id); return [ 'operation_id' => (string) $job->operation_id, 'kind' => (string) $job->kind, 'phase' => (string) $job->phase, 'outcome' => (string) $job->outcome, 'status' => (string) $job->status, 'dispatch_attempts' => (int) $job->dispatch_attempts, 'effect_may_have_applied' => (bool) $job->effect_may_have_applied, 'safe_to_resubmit' => (bool) $job->safe_to_resubmit, 'error_code' => (string) $job->error_code, 'result_json' => $job->result_json, 'terminal_at' => (int) $job->terminal_at, 'completion_event_id' => $job->completion_event_id, 'read_at' => $job->read_at, ]; }; $inspection = static function ( string $operationId, string $journalStatus, string $resolution, ?array $projection = null, ?bool $fenceResolutionRequired = null, ?string $errorCode = null, ?string $fenceState = null ): array { $applied = $resolution === 'applied'; $uncertain = in_array( $resolution, ['partial', 'inconclusive'], true ); $value = [ 'operation_id' => $operationId, 'operation_status' => $journalStatus, 'checked_at' => time(), 'outcome' => $uncertain ? 'uncertain' : ($applied ? 'succeeded' : 'failed'), 'summary_code' => $uncertain ? 'evidence_incomplete' : ($applied ? 'operation_succeeded' : 'operation_failed'), 'effect_resolution' => $resolution, 'fence_resolution_required' => $fenceResolutionRequired ?? $journalStatus === 'failed', 'fence_state' => $fenceState ?? (($fenceResolutionRequired ?? $journalStatus === 'failed') ? 'active' : 'not_required'), 'abort_job' => ['available' => false], 'cleanup' => ['available' => false], ] + CloudOperationRecoveryIdentity::expected( CloudJob::where('operation_id', $operationId)->firstOrFail(), $applied ); if ($projection !== null) { $value['effect_projection'] = $projection; } if ($resolution === 'not_applied') { $value['error_code'] = $errorCode ?? 'settings_not_committed'; } return $value; }; $notFound = static fn (): CloudAgentError => new CloudAgentError( 'not found', 404, 'fence_resolution_not_found', false, 404 ); $fenceResponse = static function ( array $request, int $resolvedAt ): array { $payload = $request['payload']; return [ 'operation_id' => $request['operationId'], 'resolution_id' => $payload['resolution_id'], 'resolution' => $payload['resolution'], 'inspection_checked_at' => $payload['inspection_checked_at'], 'reason_sha256' => $payload['reason_sha256'], 'resolved_at' => $resolvedAt, 'resolved_by' => 0, 'replayed' => false, ]; }; $client = new AutomaticRecoveryTestAgentClient(); CloudAgentSettingsService::$client = $client; // An immutable received receipt must outrank a legacy aggregate whose // child can remain active indefinitely. This prevents the aggregate sweep // from starving both PHP convergence and the Agent outbox ACK. [$receiptPriorityId, $receiptPriorityUuid] = $insertSettingsJob( str_repeat('0a', 16), ['hostname' => 'receipt-priority.example'] ); $receiptPriorityJob = CloudJob::findOrFail($receiptPriorityId); $receiptPriorityIdentity = CloudOperationRecoveryIdentity::expected( $receiptPriorityJob, true ); $receiptPriorityResult = [ 'uuid' => $receiptPriorityUuid, 'hostname' => 'receipt-priority.example', 'settings_changed' => true, ]; $receiptPriorityWire = [ 'event_id' => 'automatic-priority-receipt', 'host_id' => 7, 'operation_id' => (string) $receiptPriorityJob->operation_id, 'action' => $receiptPriorityIdentity['action'], 'resource_id' => $receiptPriorityIdentity['resource_id'], 'request_fingerprint' => $receiptPriorityIdentity['request_fingerprint'], 'status' => 'succeeded', 'effect_may_have_applied' => false, 'safe_to_resubmit' => false, 'error_code' => '', 'result' => $receiptPriorityResult, ]; Capsule::table('cloud_agent_completion_events')->insert([ 'event_id' => $receiptPriorityWire['event_id'], 'host_id' => 7, 'operation_id' => $receiptPriorityWire['operation_id'], 'payload_sha256' => hash('sha256', json_encode( $receiptPriorityWire, JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE )), 'outcome' => 'succeeded', 'effect_may_have_applied' => 0, 'safe_to_resubmit' => 0, 'error_code' => '', 'result_json' => json_encode([ 'schema' => 'virt_cloud_completion_event', 'version' => 1, 'identity' => $receiptPriorityIdentity, 'result' => $receiptPriorityResult, ], JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE), 'disposition' => 'received', 'received_at' => $now, 'applied_at' => null, ]); [$legacyParentId] = $insertSettingsJob( str_repeat('0b', 16), ['hostname' => 'legacy-parent.example'] ); Capsule::table('cloud_jobs')->where('id', $legacyParentId)->update([ 'kind' => 'template_import', 'execution_mode' => 'aggregate', 'host_id' => null, 'instance_id' => null, 'dispatch_attempts' => 0, 'dispatched_at' => null, 'available_at' => 0, 'lease_until' => 0, ]); [$legacyChildId] = $insertSettingsJob( str_repeat('0c', 16), ['hostname' => 'legacy-child.example'] ); Capsule::table('cloud_jobs')->where('id', $legacyChildId)->update([ 'parent_job_id' => $legacyParentId, 'kind' => 'template_import_replica', 'phase' => 'awaiting_receipt', 'outcome' => null, 'status' => 'running', 'effect_may_have_applied' => null, 'safe_to_resubmit' => null, 'effect_resolution' => 'not_applicable', 'terminal_at' => null, ]); $beforePriorityRequests = count($client->requests); $assert((new CloudAutomaticOperationRecoveryService())->tick(), 'An exact completion receipt was not claimed ahead of legacy sweep.'); $priorityEvent = Capsule::table('cloud_agent_completion_events') ->where('event_id', 'automatic-priority-receipt')->first(); $assert((string) CloudJob::findOrFail($receiptPriorityId) ->effect_resolution === 'applied' && (string) $priorityEvent->disposition === 'applied' && (int) CloudJob::findOrFail($legacyParentId)->available_at === 0 && count($client->requests) === $beforePriorityRequests, 'A waiting aggregate starved or contaminated an exact receipt.'); Capsule::table('cloud_jobs')->whereIn( 'id', [$legacyParentId, $legacyChildId] )->delete(); Capsule::table('cloud_instances')->whereIn( 'id', [$legacyParentId, $legacyChildId] )->delete(); // Sustained completion traffic must not monopolize the recovery worker. // The long-lived worker has three independently useful lanes: immutable // receipts, ordinary Agent leaves, and PHP-only aggregate projection. // With all three continuously ready, every lane must make progress within // one three-tick rotation; replenishing receipts may not starve either of // the other two lanes. [$fairLeafId] = $insertSettingsJob( md5('automatic-fair-leaf'), ['hostname' => 'fair-leaf.example'] ); Capsule::table('cloud_jobs')->where('id', $fairLeafId)->update([ 'host_id' => null, 'available_at' => 0, 'lease_token' => null, 'lease_until' => 0, ]); [$fairParentId] = $insertSettingsJob( md5('automatic-fair-aggregate'), ['hostname' => 'fair-aggregate.example'] ); Capsule::table('cloud_jobs')->where('id', $fairParentId)->update([ 'parent_job_id' => null, 'order_id' => null, 'invoice_id' => null, 'instance_id' => null, 'snapshot_id' => null, 'host_id' => null, 'kind' => 'template_import', 'execution_mode' => 'aggregate', 'attempts' => 0, 'dispatch_attempts' => 0, 'dispatch_started_at' => null, 'agent_accepted_at' => null, 'dispatched_at' => null, 'completion_event_id' => null, 'effect_may_have_applied' => 1, 'safe_to_resubmit' => 0, 'effect_resolution' => 'unknown', 'finalization_incident' => 1, 'finalization_error_code' => 'recovery_child_convergence', 'available_at' => 0, 'lease_token' => null, 'lease_until' => 0, ]); [$fairChildId] = $insertSettingsJob( md5('automatic-fair-aggregate-child'), ['hostname' => 'fair-aggregate-child.example'] ); Capsule::table('cloud_jobs')->where('id', $fairChildId)->update([ 'parent_job_id' => $fairParentId, 'kind' => 'template_import_replica', 'effect_may_have_applied' => 0, 'safe_to_resubmit' => 1, 'effect_resolution' => 'not_applied', 'effect_resolved_at' => $now, 'effect_resolution_reason' => 'normal_completion', 'available_at' => $now + 3600, 'lease_token' => null, 'lease_until' => $now + 3600, ]); $insertFairReceipt = static function (int $sequence) use ( $insertSettingsJob, $now ): string { $operationId = md5('automatic-fair-receipt:' . $sequence); [$jobId, $uuid] = $insertSettingsJob( $operationId, ['hostname' => 'fair-receipt-' . $sequence . '.example'] ); $job = CloudJob::findOrFail($jobId); $identity = CloudOperationRecoveryIdentity::expected($job, true); $result = [ 'uuid' => $uuid, 'hostname' => 'fair-receipt-' . $sequence . '.example', 'settings_changed' => true, ]; $wire = [ 'event_id' => 'automatic-fair-receipt-' . $sequence, 'host_id' => 7, 'operation_id' => $operationId, 'action' => $identity['action'], 'resource_id' => $identity['resource_id'], 'request_fingerprint' => $identity['request_fingerprint'], 'status' => 'succeeded', 'effect_may_have_applied' => false, 'safe_to_resubmit' => false, 'error_code' => '', 'result' => $result, ]; Capsule::table('cloud_agent_completion_events')->insert([ 'event_id' => $wire['event_id'], 'host_id' => 7, 'operation_id' => $operationId, 'payload_sha256' => hash('sha256', json_encode( $wire, JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE )), 'outcome' => 'succeeded', 'effect_may_have_applied' => 0, 'safe_to_resubmit' => 0, 'error_code' => '', 'result_json' => json_encode([ 'schema' => 'virt_cloud_completion_event', 'version' => 1, 'identity' => $identity, 'result' => $result, ], JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE), 'disposition' => 'received', 'received_at' => $now + $sequence, 'applied_at' => null, ]); return $wire['event_id']; }; $fairService = new CloudAutomaticOperationRecoveryService(); $fairReceiptIds = []; for ($sequence = 1; $sequence <= 6; $sequence++) { $fairReceiptIds[] = $insertFairReceipt($sequence); $assert($fairService->tick(), 'A fair recovery rotation unexpectedly found no work.'); } $fairAppliedReceipts = Capsule::table( 'cloud_agent_completion_events' )->whereIn('event_id', $fairReceiptIds) ->where('disposition', 'applied')->count(); $fairLeaf = CloudJob::findOrFail($fairLeafId); $fairParent = CloudJob::findOrFail($fairParentId); $assert($fairAppliedReceipts >= 2 && (bool) $fairLeaf->finalization_incident && (string) $fairLeaf->finalization_error_code === 'recovery_instance_state_conflict' && (bool) $fairParent->finalization_incident && (int) $fairParent->available_at >= time() + 290, 'Continuous receipt traffic starved a leaf or aggregate lane, ' . 'or fairness starved immutable receipts.'); $fairReceiptOperations = Capsule::table( 'cloud_agent_completion_events' )->whereIn('event_id', $fairReceiptIds)->pluck('operation_id')->all(); $fairReceiptJobs = Capsule::table('cloud_jobs') ->whereIn('operation_id', $fairReceiptOperations); $fairReceiptJobIds = $fairReceiptJobs->pluck('id')->all(); $fairReceiptInstanceIds = Capsule::table('cloud_jobs') ->whereIn('id', $fairReceiptJobIds)->pluck('instance_id')->all(); Capsule::table('cloud_agent_completion_events') ->whereIn('event_id', $fairReceiptIds)->delete(); Capsule::table('cloud_jobs')->whereIn('id', $fairReceiptJobIds)->delete(); Capsule::table('cloud_instances') ->whereIn('id', $fairReceiptInstanceIds)->delete(); Capsule::table('cloud_jobs')->whereIn('id', [ $fairLeafId, $fairParentId, $fairChildId, ])->delete(); Capsule::table('cloud_instances')->whereIn('id', [ $fairLeafId, $fairParentId, $fairChildId, ])->delete(); // Every explicitly registered Agent leaf remains eligible. Inconclusive // evidence keeps the exact operation fenced and schedules another check. [$nonSettingsId, $nonSettingsUuid] = $insertSettingsJob( str_repeat('0f', 16), ['hostname' => 'not-a-settings-recovery.example'] ); Capsule::table('cloud_jobs')->where('id', $nonSettingsId)->update([ 'kind' => 'network_change', 'sealed_payload' => CloudSecrets::encrypt([ 'method' => 'POST', 'request_target' => '/v1/vms/' . $nonSettingsUuid . '/network', 'body' => [ 'operation_id' => str_repeat('0f', 16), 'network' => 'bridge:br0', 'public_pool_key' => str_repeat('a', 32), 'public_pool_cidr' => '203.0.113.0/24', 'public_ip_cidr' => '203.0.113.10/24', 'gateway' => '203.0.113.1', 'dns_servers' => ['1.1.1.1'], 'confirm_restart' => true, ], 'owner_id' => 1, 'can_manage' => false, ], 'job-payload'), ]); $beforeNonSettingsRequests = count($client->requests); $client->responses[] = $inspection( str_repeat('0f', 16), 'failed', 'inconclusive' ); $assert((new CloudAutomaticOperationRecoveryService())->tick(), 'The recovery worker did not claim a registered non-settings kind.'); $nonSettings = CloudJob::find($nonSettingsId); $assert((string) $nonSettings->effect_resolution === 'unknown' && (string) $nonSettings->finalization_error_code === 'recovery_evidence_inconclusive' && (int) $nonSettings->available_at >= time() + 290 && count($client->requests) === $beforeNonSettingsRequests + 1, 'Inconclusive non-settings evidence did not stay recoverable.'); // A successful read-only inspection must adapt back to the same terminal // operation contract as an immediate Agent response. This is the public // recovery seam for a template deletion whose original success receipt // reached PHP but failed local catalog finalization. $templateDeleteId = 'automatic-delete-template'; [$templateDeleteParentId] = $insertSettingsJob( md5('automatic-template-delete-parent'), ['hostname' => 'unused-parent.example'] ); Capsule::table('cloud_jobs')->where( 'id', $templateDeleteParentId )->update([ 'parent_job_id' => null, 'instance_id' => null, 'host_id' => null, 'kind' => 'template_delete', 'execution_mode' => 'aggregate', 'attempts' => 0, 'dispatch_attempts' => 0, 'dispatch_started_at' => null, 'agent_accepted_at' => null, 'dispatched_at' => null, 'effect_resolution' => 'unknown', 'finalization_incident' => 1, 'finalization_error_code' => 'recovery_child_convergence', 'available_at' => 0, 'lease_token' => null, 'lease_until' => 0, ]); [$templateDeleteChildId] = $insertSettingsJob( md5('automatic-template-delete-child'), ['hostname' => 'unused-child.example'] ); $templateDeleteOperationId = (string) CloudJob::findOrFail( $templateDeleteChildId )->operation_id; Capsule::table('cloud_jobs')->where( 'id', $templateDeleteChildId )->update([ 'parent_job_id' => $templateDeleteParentId, 'instance_id' => null, 'kind' => 'template_delete_replica', 'sealed_payload' => CloudSecrets::encrypt([ 'method' => 'DELETE', 'request_target' => '/v1/templates/' . $templateDeleteId, 'body' => ['operation_id' => $templateDeleteOperationId], 'owner_id' => 1, 'can_manage' => true, 'template_id' => $templateDeleteId, ], 'job-payload'), 'finalization_incident' => 1, 'finalization_error_code' => 'recovery_receipt_invalid', 'available_at' => 0, 'lease_token' => null, 'lease_until' => 0, ]); Capsule::table('cloud_templates')->insert([ 'id' => $templateDeleteId, 'active_job_id' => $templateDeleteParentId, 'state' => 'deleting', 'revision' => 1, 'generation_token' => str_repeat('d', 32), 'image_bytes' => 4096, 'sha256' => str_repeat('e', 64), 'updated_at' => $now, ]); Capsule::table('cloud_template_replicas')->insert([ 'host_id' => 7, 'template_id' => $templateDeleteId, 'revision' => 1, 'generation_token' => str_repeat('d', 32), 'state' => 'delete_failed', 'image_bytes' => 4096, 'sha256' => str_repeat('e', 64), 'operation_id' => $templateDeleteOperationId, 'job_id' => $templateDeleteChildId, 'error_code' => 'completion_finalization_failed', 'created_at' => $now - 120, 'updated_at' => $now, ]); $client->responses[] = $inspection( $templateDeleteOperationId, 'succeeded', 'applied', [ 'resource_type' => 'template', 'id' => $templateDeleteId, 'deleted' => true, 'managed_image_absent' => true, 'managed_metadata_absent' => true, 'managed_artifacts_absent' => true, ], false ); $beforeTemplateDeleteRequests = count($client->requests); $assert((new CloudAutomaticOperationRecoveryService())->tick(), 'The recovery worker did not inspect the template deletion.'); $templateDeleteChild = CloudJob::findOrFail($templateDeleteChildId); $templateDeleteParent = CloudJob::findOrFail($templateDeleteParentId); $assert((string) $templateDeleteChild->effect_resolution === 'applied' && !(bool) $templateDeleteChild->finalization_incident && $templateDeleteChild->finalization_error_code === null && (string) $templateDeleteParent->effect_resolution === 'applied' && !(bool) $templateDeleteParent->finalization_incident && (string) Capsule::table('cloud_templates') ->where('id', $templateDeleteId)->value('state') === 'retired' && !Capsule::table('cloud_template_replicas') ->where('template_id', $templateDeleteId)->exists() && count($client->requests) === $beforeTemplateDeleteRequests + 1 && str_ends_with( $client->requests[$beforeTemplateDeleteRequests]['target'], '/inspect' ), 'A successful template deletion inspection did not converge its ' . 'existing operation and catalog ledger.'); // A poisoned Agent completion outbox is never evidence for either effect // verdict. Even a syntactically complete not-applied projection remains // fenced and the reviewed cause is visible without exposing raw data. [$quarantinedId] = $insertSettingsJob( str_repeat('0e', 16), ['hostname' => 'quarantined-receipt.example'] ); $quarantinedInspection = $inspection( str_repeat('0e', 16), 'failed', 'not_applied', null, true, 'settings_not_committed' ); $quarantinedInspection['completion_delivery_state'] = 'quarantined'; $quarantinedInspection['completion_delivery_error_code'] = 'completion_identity_mismatch'; $client->responses[] = $quarantinedInspection; $beforeQuarantinedRequests = count($client->requests); $assert((new CloudAutomaticOperationRecoveryService())->tick(), 'The recovery worker did not inspect a quarantined completion.'); $quarantined = CloudJob::findOrFail($quarantinedId); $quarantinedUser = CloudJobService::recoveryPresentation( $quarantined, false ); $quarantinedAdmin = CloudJobService::recoveryPresentation( $quarantined, true ); $assert((string) $quarantined->effect_resolution === 'unknown' && (string) $quarantined->finalization_error_code === 'recovery_completion_identity_mismatch' && (int) $quarantined->available_at >= time() + 290 && count($client->requests) === $beforeQuarantinedRequests + 1 && ($quarantinedAdmin['code'] ?? null) === 'recovery_completion_identity_mismatch' && str_contains( (string) ($quarantinedAdmin['message'] ?? ''), '身份与原操作记录不匹配' ) && str_contains( (string) ($quarantinedUser['message'] ?? ''), '身份与原操作记录不匹配' ) && !array_key_exists('code', $quarantinedUser), 'A quarantined completion was trusted or its safe cause was hidden.'); // A journal success is immutable exact evidence and has no active fence. [$succeededId, $succeededUuid] = $insertSettingsJob( str_repeat('1', 32), ['root_password' => 'New-automatic9!Pass'], [ 'password' => 'New-automatic9!Pass', 'login_name' => 'root', 'auth_mode' => 'password', ] ); $succeededAudit = $audit($succeededId); $client->responses[] = $inspection( str_repeat('1', 32), 'succeeded', 'applied', [ 'uuid' => $succeededUuid, 'status' => 'running', 'settings_changed' => true, ] ); $dispatchId = $insertDispatchableJob(str_repeat('8', 32)); $beforeDispatchRequests = count($client->requests); $assert((new CloudWorkerService())->tick(), 'The worker did not claim the ordinary dispatchable task first.'); $dispatch = CloudJob::find($dispatchId); $unrecovered = CloudJob::find($succeededId); $assert((string) $dispatch->phase === 'terminal' && (string) $dispatch->outcome === 'failed' && (int) $dispatch->dispatch_attempts === 0 && (string) $unrecovered->effect_resolution === 'unknown' && count($client->requests) === $beforeDispatchRequests, 'Automatic recovery ran before an ordinary dispatchable task.'); $queuedBehindRecovery = $insertDispatchableJob(str_repeat('b', 32)); $beforeSucceededRecovery = count($client->requests); $assert((new CloudAutomaticOperationRecoveryService())->tick(), 'The dedicated recovery service did not claim a recoverable task.'); $assert((string) CloudJob::find($queuedBehindRecovery)->phase === 'accepted', 'The dedicated recovery service consumed an ordinary task.'); $assert((new CloudWorkerService())->tick() && (string) CloudJob::find($queuedBehindRecovery)->phase === 'terminal', 'The ordinary worker did not keep processing its independent queue.'); $succeeded = CloudJob::find($succeededId); $credential = CloudSecrets::decrypt( (string) Capsule::table('cloud_instances') ->where('id', $succeededId)->value('sealed_credential'), 'instance-credential' ); $redacted = CloudSecrets::decrypt( (string) $succeeded->sealed_payload, 'job-payload' ); $assert((string) $succeeded->effect_resolution === 'applied' && (int) $succeeded->effect_resolved_by === 0 && $audit($succeededId) === $succeededAudit && ($credential['password'] ?? null) === 'New-automatic9!Pass' && !array_key_exists('credential', $redacted) && !CloudJobService::blockingQueryForInstance( $succeededId )->exists() && count($client->requests) === $beforeSucceededRecovery + 1 && str_ends_with( $client->requests[array_key_last($client->requests)]['target'], '/inspect' ), 'Journal-success recovery did not converge without a fence write.'); // A malformed or contradictory receipt is isolated from an existing // prepared intent. The job itself remains scheduled for a fresh Agent // inspection instead of treating the incident as a permanent stop. $invalidProofRequests = count($client->requests); $invalidProofCases = [ [ 'operation_id' => str_repeat('c1', 16), 'body' => ['root_password' => 'Invalid-proof9!Pass'], 'credential' => [ 'password' => 'Invalid-proof9!Pass', 'login_name' => 'root', 'auth_mode' => 'password', ], 'receipt_result' => [], 'message' => 'missing settings_changed proof', ], [ 'operation_id' => str_repeat('d2', 16), 'body' => ['hostname' => 'prepared-proof.example'], 'credential' => null, 'receipt_result' => [ 'settings_changed' => true, 'hostname' => 'different-proof.example', ], 'message' => 'mismatched hostname proof', ], ]; foreach ($invalidProofCases as $index => $case) { [$invalidProofId, $invalidProofUuid] = $insertSettingsJob( $case['operation_id'], $case['body'], $case['credential'] ); Capsule::table('cloud_jobs')->where( 'id', $invalidProofId )->update([ 'effect_resolution_reason' => 'automatic_recovery:v1:' . time() . ':1:pending', ]); $preparedResult = [ 'uuid' => $invalidProofUuid, 'settings_changed' => true, ]; if (array_key_exists('hostname', $case['body'])) { $preparedResult['hostname'] = $case['body']['hostname']; } $operationIdentity = CloudOperationRecoveryIdentity::expected( CloudJob::findOrFail($invalidProofId), true ); CloudOperationRecoveryService::prepareApplied( $invalidProofId, $operationIdentity + [ 'operation_id' => $case['operation_id'], 'status' => 'succeeded', 'effect_may_have_applied' => false, 'safe_to_resubmit' => false, 'error_code' => '', 'result' => $preparedResult, ], 'automatic_applied_succeeded_nofence:' . str_repeat((string) ($index + 3), 64) ); $preparedInvalidProof = CloudJob::find($invalidProofId); $preparedMarker = (string) $preparedInvalidProof ->effect_resolution_reason; $receiptResult = ['uuid' => $invalidProofUuid] + $case['receipt_result']; $eventId = 'automatic-invalid-proof-' . ($index + 1); $accepted = CloudOperationCompletionService::accept([ 'type' => 'completion', 'id' => $eventId, 'agent_id' => '7', 'operation_id' => $case['operation_id'], 'action' => $operationIdentity['action'], 'resource_id' => $operationIdentity['resource_id'], 'request_fingerprint' => $operationIdentity['request_fingerprint'], 'status' => 'succeeded', 'effect_may_have_applied' => false, 'safe_to_resubmit' => false, 'error_code' => '', 'result' => $receiptResult, ]); $invalidProofJob = CloudJob::find($invalidProofId); $invalidProofEvent = Capsule::table( 'cloud_agent_completion_events' )->where('event_id', $eventId)->first(); $assert($accepted === [ 'accepted' => true, 'disposition' => 'late', ] && (string) $invalidProofJob->effect_resolution === 'unknown' && (string) $invalidProofJob->effect_resolution_reason === $preparedMarker && (bool) $invalidProofJob->finalization_incident && in_array( (string) $invalidProofJob->finalization_error_code, [ 'recovery_receipt_invalid', 'recovery_evidence_conflict', ], true ) && (int) $invalidProofJob->available_at >= time() + 290 && (string) $invalidProofEvent->disposition === 'late' && CloudJobService::blockingQueryForInstance( $invalidProofId )->exists(), 'A success receipt with ' . $case['message'] . ' was not isolated while leaving inspection retryable.'); } $assert(count($client->requests) === $invalidProofRequests, 'Invalid late receipt proof unexpectedly contacted the Agent.'); // A well-formed completion for a different immutable Agent request is a // bad event, not a reason to strand the already-prepared PHP convergence. // Quarantine it, then let the normal prepared crash-resume path finish. [$wrongIdentityId, $wrongIdentityUuid] = $insertSettingsJob( str_repeat('e3', 16), ['hostname' => 'prepared-identity.example'] ); $expectedIdentity = CloudOperationRecoveryIdentity::expected( CloudJob::findOrFail($wrongIdentityId), true ); CloudOperationRecoveryService::prepareApplied( $wrongIdentityId, $expectedIdentity + [ 'operation_id' => str_repeat('e3', 16), 'status' => 'succeeded', 'effect_may_have_applied' => false, 'safe_to_resubmit' => false, 'error_code' => '', 'result' => [ 'uuid' => $wrongIdentityUuid, 'hostname' => 'prepared-identity.example', 'settings_changed' => true, ], ], 'automatic_applied_succeeded_nofence:' . str_repeat('9', 64) ); $preparedWrongIdentity = CloudJob::findOrFail($wrongIdentityId); $preparedWrongIdentityMarker = (string) $preparedWrongIdentity ->effect_resolution_reason; $wrongIdentityEventId = 'automatic-wrong-identity'; $acceptedWrongIdentity = CloudOperationCompletionService::accept([ 'type' => 'completion', 'id' => $wrongIdentityEventId, 'agent_id' => '7', 'operation_id' => str_repeat('e3', 16), 'action' => $expectedIdentity['action'], 'resource_id' => '00000000-0000-4000-8000-000000009999', 'request_fingerprint' => $expectedIdentity['request_fingerprint'], 'status' => 'succeeded', 'effect_may_have_applied' => false, 'safe_to_resubmit' => false, 'error_code' => '', 'result' => [ 'uuid' => $wrongIdentityUuid, 'hostname' => 'prepared-identity.example', 'settings_changed' => true, ], ]); $preparedWrongIdentity = CloudJob::findOrFail($wrongIdentityId); $wrongIdentityEvent = Capsule::table( 'cloud_agent_completion_events' )->where('event_id', $wrongIdentityEventId)->first(); $assert($acceptedWrongIdentity === [ 'accepted' => true, 'disposition' => 'rejected', ] && (string) $wrongIdentityEvent->disposition === 'rejected' && (string) $preparedWrongIdentity->effect_resolution === 'unknown' && (string) $preparedWrongIdentity->effect_resolution_reason === $preparedWrongIdentityMarker && (bool) $preparedWrongIdentity->finalization_incident && (string) $preparedWrongIdentity->finalization_error_code === 'recovery_operation_identity_conflict', 'A definitive wrong-operation receipt was not isolated from prepared recovery.'); Capsule::table('cloud_jobs')->where('id', $wrongIdentityId)->update([ 'available_at' => 0, 'lease_token' => null, 'lease_until' => 0, ]); $beforeWrongIdentityResume = count($client->requests); $assert((new CloudAutomaticOperationRecoveryService())->tick(), 'Prepared recovery did not resume after quarantining a wrong receipt.'); $preparedWrongIdentity = CloudJob::findOrFail($wrongIdentityId); $wrongIdentityEvent = Capsule::table( 'cloud_agent_completion_events' )->where('event_id', $wrongIdentityEventId)->first(); $assert((string) $preparedWrongIdentity->effect_resolution === 'applied' && !(bool) $preparedWrongIdentity->finalization_incident && $preparedWrongIdentity->finalization_error_code === null && (string) $wrongIdentityEvent->disposition === 'rejected' && count($client->requests) === $beforeWrongIdentityResume, 'The quarantined wrong receipt starved or contaminated prepared convergence.'); // Failed journal + dual settings markers: prepare first, then resolve the // fence. Simulate a lost POST response and prove one GET recovers it. [$lostId, $lostUuid] = $insertSettingsJob( str_repeat('2', 32), ['hostname' => 'recovered.example'] ); $lostAudit = $audit($lostId); $lostFenceRequest = null; $client->responses[] = $inspection( str_repeat('2', 32), 'failed', 'applied', [ 'uuid' => $lostUuid, 'hostname' => 'recovered.example', 'status' => 'shutoff', 'settings_changed' => true, ] ); $client->responses[] = $notFound(); $client->responses[] = static function (array $request) use ( &$lostFenceRequest ): Throwable { $lostFenceRequest = $request; return new RuntimeException('response lost after durable commit'); }; $client->responses[] = static function (array $request) use ( &$lostFenceRequest, $fenceResponse ): array { $read = $lostFenceRequest; $read['method'] = $request['method']; $read['target'] = $request['target']; return $fenceResponse($read, time()); }; $beforeLostRequests = count($client->requests); $assert((new CloudAutomaticOperationRecoveryService())->tick(), 'The worker did not recover a failed-journal applied task.'); $lost = CloudJob::find($lostId); $lostSpec = json_decode( (string) Capsule::table('cloud_instances') ->where('id', $lostId)->value('spec_json'), true, flags: JSON_THROW_ON_ERROR ); $lostRequests = array_slice( $client->requests, $beforeLostRequests ); $assert((string) $lost->effect_resolution === 'applied' && $audit($lostId) === $lostAudit && ($lostSpec['guest_hostname'] ?? null) === 'recovered.example' && count(array_filter( $lostRequests, static fn (array $request): bool => $request['method'] === 'POST' )) === 1 && count($lostRequests) === 4 && !CloudJobService::blockingQueryForInstance($lostId)->exists(), 'Lost fence response recovery was not single-write and convergent.'); // Exact absence markers prove a settings request did not commit. PHP has // no instance ledger delta, but still prepares before releasing the fence. [$absentId] = $insertSettingsJob( str_repeat('3', 32), ['hostname' => 'not-applied.example'] ); $absentAudit = $audit($absentId); $absentSpec = (string) Capsule::table('cloud_instances') ->where('id', $absentId)->value('spec_json'); $client->responses[] = $inspection( str_repeat('3', 32), 'failed', 'not_applied' ); $client->responses[] = $notFound(); $client->responses[] = static fn (array $request): array => $fenceResponse($request, time()); $assert((new CloudAutomaticOperationRecoveryService())->tick(), 'The worker did not converge exact not-applied evidence.'); $absent = CloudJob::find($absentId); $assert((string) $absent->effect_resolution === 'not_applied' && (int) $absent->effect_resolved_by === 0 && $audit($absentId) === $absentAudit && (string) Capsule::table('cloud_instances') ->where('id', $absentId)->value('spec_json') === $absentSpec && !CloudJobService::blockingQueryForInstance($absentId)->exists(), 'Not-applied recovery changed the ledger or retained its blocker.'); // An immutable failed Journal row which explicitly records no possible // effect has no Agent fence. Its bounded original error code is evidence, // not a reason to invent a settings_not_committed fence marker. [$exactAbsentId] = $insertSettingsJob( str_repeat('9', 32), ['hostname' => 'exact-not-applied.example'] ); $exactAbsentAudit = $audit($exactAbsentId); $client->responses[] = $inspection( str_repeat('9', 32), 'failed', 'not_applied', null, false, 'cloudinit_validation_failed' ); $beforeExactAbsent = count($client->requests); $assert((new CloudAutomaticOperationRecoveryService())->tick(), 'The worker did not converge exact no-effect Journal evidence.'); $exactAbsentRequests = array_slice( $client->requests, $beforeExactAbsent ); $exactAbsent = CloudJob::find($exactAbsentId); $assert((string) $exactAbsent->effect_resolution === 'not_applied' && $audit($exactAbsentId) === $exactAbsentAudit && !CloudJobService::blockingQueryForInstance( $exactAbsentId )->exists() && count($exactAbsentRequests) === 1 && $exactAbsentRequests[0]['method'] === 'GET' && str_ends_with($exactAbsentRequests[0]['target'], '/inspect'), 'Exact no-effect Journal recovery contacted a fence endpoint.'); // A Journal fence already resolved outside this coordinator is not the // same thing as an operation which never needed a fence. Without PHP's // matching durable prepare marker, automatic recovery must not accept the // possibly conflicting Agent verdict. [$externallyResolvedId] = $insertSettingsJob( str_repeat('a', 32), ['hostname' => 'externally-resolved.example'] ); $client->responses[] = $inspection( str_repeat('a', 32), 'failed', 'not_applied', null, false, 'settings_not_committed', 'resolved' ); $beforeExternallyResolved = count($client->requests); $assert((new CloudAutomaticOperationRecoveryService())->tick(), 'The externally resolved fence was not inspected.'); $externallyResolved = CloudJob::find($externallyResolvedId); $externallyResolvedRequests = array_slice( $client->requests, $beforeExternallyResolved ); $assert((string) $externallyResolved->effect_resolution === 'unknown' && (int) $externallyResolved->available_at >= time() + 290 && CloudJobService::blockingQueryForInstance( $externallyResolvedId )->exists() && count($externallyResolvedRequests) === 1 && str_ends_with( $externallyResolvedRequests[0]['target'], '/inspect' ), 'An unprepared externally resolved fence bypassed PHP recovery intent.'); // Partial and inconclusive evidence stay blocked and receive a five-minute // retry delay. Two tasks also prove each worker tick handles only one. [$partialId] = $insertSettingsJob( str_repeat('4', 32), ['hostname' => 'partial.example'] ); [$inconclusiveId] = $insertSettingsJob( str_repeat('5', 32), ['hostname' => 'inconclusive.example'] ); $client->responses[] = $inspection( str_repeat('4', 32), 'failed', 'partial' ); $client->responses[] = $inspection( str_repeat('5', 32), 'failed', 'inconclusive' ); $assert((new CloudAutomaticOperationRecoveryService())->tick(), 'The partial-evidence task was not inspected.'); $partial = CloudJob::find($partialId); $notYetClaimed = CloudJob::find($inconclusiveId); $assert((string) $partial->effect_resolution === 'unknown' && (int) $partial->available_at >= time() + 290 && CloudJobService::blockingQueryForInstance($partialId)->exists() && (int) $notYetClaimed->available_at === 0, 'One worker tick did not stop after one inconclusive recovery.'); $assert((new CloudAutomaticOperationRecoveryService())->tick(), 'The inconclusive-evidence task was not inspected.'); $inconclusive = CloudJob::find($inconclusiveId); $assert((string) $inconclusive->effect_resolution === 'unknown' && (int) $inconclusive->available_at >= time() + 290 && CloudJobService::blockingQueryForInstance( $inconclusiveId )->exists(), 'Inconclusive evidence released its blocker or hot-looped.'); // Agent downtime backs off without changing the immutable task outcome. [$offlineId] = $insertSettingsJob( str_repeat('6', 32), ['hostname' => 'offline.example'] ); $offlineAudit = $audit($offlineId); $client->responses[] = new CloudAgentError( 'offline', 503, 'agent_unavailable', true, 503 ); $assert((new CloudAutomaticOperationRecoveryService())->tick(), 'The offline recovery task was not claimed.'); $offline = CloudJob::find($offlineId); $assert((string) $offline->effect_resolution === 'unknown' && $audit($offlineId) === $offlineAudit && $offline->lease_token === null && (int) $offline->available_at >= time() + 290 && CloudJobService::blockingQueryForInstance($offlineId)->exists(), 'Agent downtime changed audit state or did not back off.'); // A retried low-ID offline host must not repeatedly jump ahead of a // never-attempted higher-ID task. Persisted available_at age, then ID, // defines the recovery queue order. Capsule::table('cloud_jobs')->where('id', $offlineId)->update([ 'available_at' => time() - 1, 'lease_until' => 0, 'lease_token' => null, ]); [$fairId, $fairUuid] = $insertSettingsJob( str_repeat('d', 32), ['hostname' => 'fair-queue.example'] ); $client->responses[] = $inspection( str_repeat('d', 32), 'succeeded', 'applied', [ 'uuid' => $fairUuid, 'hostname' => 'fair-queue.example', 'settings_changed' => true, ] ); $assert((new CloudAutomaticOperationRecoveryService())->tick(), 'The recovery queue did not claim a fair successor.'); $assert((string) CloudJob::find($fairId)->effect_resolution === 'applied' && (string) CloudJob::find($offlineId)->effect_resolution === 'unknown', 'A retried low-ID offline task starved a never-attempted task.'); Capsule::table('cloud_jobs')->where('id', $offlineId)->update([ 'available_at' => time() + 300, ]); // Attempt count is audit metadata only. The same operation remains in the // recovery queue beyond the former 12-attempt boundary. [$limitedId] = $insertSettingsJob( str_repeat('c', 32), ['hostname' => 'retry-limit.example'] ); $beforeLimitedRequests = count($client->requests); for ($attempt = 1; $attempt <= 13; $attempt++) { Capsule::table('cloud_jobs')->where('id', $limitedId)->update([ 'available_at' => 0, 'lease_until' => 0, 'lease_token' => null, ]); $client->responses[] = new CloudAgentError( 'offline', 503, 'agent_unavailable', true, 503 ); $assert((new CloudAutomaticOperationRecoveryService())->tick(), 'A persistent recovery attempt was not claimed.'); } $limited = CloudJob::find($limitedId); $limitedPayloadBefore = (string) $limited->sealed_payload; $limitedRequestCount = count($client->requests); $assert((bool) $limited->finalization_incident && (string) $limited->finalization_error_code === 'recovery_waiting_connection' && preg_match( '/^automatic_recovery:v1:[1-9][0-9]{0,10}:13:pending$/D', (string) $limited->effect_resolution_reason ) === 1 && (string) $limited->effect_resolution === 'unknown' && (string) $limited->sealed_payload === $limitedPayloadBefore && CloudJobService::blockingQueryForInstance($limitedId)->exists() && $limitedRequestCount - $beforeLimitedRequests === 13 && (int) $limited->available_at >= time() + 290, 'Recovery did not remain queued beyond the former attempt limit.'); // First pass prepares PHP, commits the Agent fence, then loses both the // POST response and its verification GET. A later pass must read that // existing fence and commit PHP without depending on a new host inspect. [$crashId, $crashUuid] = $insertSettingsJob( str_repeat('7', 32), ['hostname' => 'crash-recovered.example'] ); Capsule::table('cloud_jobs')->where('id', $crashId)->update([ // The pending inspection budget is about to expire. A successful // prepare must start a separate crash-resume budget instead of // stranding the durable intent at attempt 12. 'effect_resolution_reason' => 'automatic_recovery:v1:' . (time() - 300) . ':11:pending', ]); $crashInspection = $inspection( str_repeat('7', 32), 'failed', 'applied', [ 'uuid' => $crashUuid, 'hostname' => 'crash-recovered.example', 'settings_changed' => true, ] ); $client->responses[] = $crashInspection; $client->responses[] = $notFound(); $crashFenceRequest = null; $client->responses[] = static function (array $request) use ( &$crashFenceRequest ): Throwable { $crashFenceRequest = $request; return new RuntimeException('fence committed before response loss'); }; $client->responses[] = new CloudAgentError( 'verification read offline', 503, 'agent_unavailable', true, 503 ); $assert((new CloudAutomaticOperationRecoveryService())->tick(), 'The crash-window fixture was not prepared.'); $prepared = CloudJob::find($crashId); $assert(str_starts_with( (string) $prepared->effect_resolution_reason, 'automatic_recovery:v1:' ) && preg_match( '/^automatic_recovery:v1:[1-9][0-9]{0,10}:1:prepared:/D', (string) $prepared->effect_resolution_reason ) === 1 && str_contains( (string) $prepared->effect_resolution_reason, ':prepared:automatic_applied_failed_fence:' ) && (string) $prepared->effect_resolution === 'unknown' && CloudJobService::blockingQueryForInstance($crashId)->exists(), 'Prepare failure window did not retain its blocker and intent.'); Capsule::table('cloud_jobs')->where('id', $crashId)->update([ 'available_at' => 0, 'lease_until' => 0, 'lease_token' => null, ]); $client->responses[] = static function (array $request) use ( &$crashFenceRequest, $fenceResponse ): array { $read = $crashFenceRequest; $read['method'] = $request['method']; $read['target'] = $request['target']; return $fenceResponse($read, time()); }; $beforeCrashRetry = count($client->requests); $assert((new CloudAutomaticOperationRecoveryService())->tick(), 'Prepared recovery did not resume after the crash window.'); $crashRetryRequests = array_slice( $client->requests, $beforeCrashRetry ); $assert((string) CloudJob::find($crashId)->effect_resolution === 'applied' && count(array_filter( $crashRetryRequests, static fn (array $request): bool => $request['method'] === 'POST' )) === 0 && !CloudJobService::blockingQueryForInstance($crashId)->exists() && count($crashRetryRequests) === 1 && str_ends_with( $crashRetryRequests[0]['target'], '/fence-resolution' ), 'Prepared recovery re-inspected/replayed or failed to commit.'); // Prepared crash-resume also remains eligible beyond the former limit. [$preparedLimitId] = $insertSettingsJob( str_repeat('e', 32), ['hostname' => 'prepared-limit.example'] ); $client->responses[] = $inspection( str_repeat('e', 32), 'failed', 'not_applied' ); $client->responses[] = $notFound(); $client->responses[] = new RuntimeException( 'fence response lost before verification' ); $client->responses[] = new CloudAgentError( 'verification read offline', 503, 'agent_unavailable', true, 503 ); $beforePreparedLimit = count($client->requests); $assert((new CloudAutomaticOperationRecoveryService())->tick(), 'The prepared-limit fixture did not enter its resume stage.'); $preparedLimitFirst = CloudJob::find($preparedLimitId); $assert(preg_match( '/^automatic_recovery:v1:[1-9][0-9]{0,10}:1:prepared:automatic_not_applied_failed_fence:/D', (string) $preparedLimitFirst->effect_resolution_reason ) === 1, 'Preparing did not reset the independent resume budget.'); for ($attempt = 2; $attempt <= 13; $attempt++) { Capsule::table('cloud_jobs')->where('id', $preparedLimitId)->update([ 'available_at' => 0, 'lease_until' => 0, 'lease_token' => null, ]); $client->responses[] = new CloudAgentError( 'prepared fence read offline', 503, 'agent_unavailable', true, 503 ); $assert((new CloudAutomaticOperationRecoveryService())->tick(), 'A persistent prepared-resume attempt was not claimed.'); } $preparedLimited = CloudJob::find($preparedLimitId); $preparedLimitRequestCount = count($client->requests); $preparedLimitPayload = (string) $preparedLimited->sealed_payload; $preparedLimitEscrow = CloudSecrets::decrypt( $preparedLimitPayload, 'job-payload' ); $preparedLimitRequests = array_slice( $client->requests, $beforePreparedLimit ); $preparedLimitFenceWrite = null; foreach ($preparedLimitRequests as $request) { if ($request['method'] === 'POST' && str_ends_with($request['target'], '/resolve-fence')) { $preparedLimitFenceWrite = $request; break; } } $assert((bool) $preparedLimited->finalization_incident && (string) $preparedLimited->finalization_error_code === 'recovery_waiting_connection' && preg_match( '/^automatic_recovery:v1:[1-9][0-9]{0,10}:13:prepared:/D', (string) $preparedLimited->effect_resolution_reason ) === 1 && (string) $preparedLimited->effect_resolution === 'unknown' && CloudJobService::blockingQueryForInstance( $preparedLimitId )->exists() && count($client->requests) - $beforePreparedLimit === 16 && is_array($preparedLimitEscrow['_operation_recovery'] ?? null) && ($preparedLimitEscrow['_operation_recovery']['operation'] ['operation_id'] ?? null) === str_repeat('e', 32) && ($preparedLimitEscrow['_operation_recovery']['operation'] ['status'] ?? null) === 'failed' && is_array($preparedLimitFenceWrite) && (int) $preparedLimited->available_at >= time() + 290, 'Prepared recovery did not remain queued beyond the former limit.'); // A fourteenth pass can still finish the original crash-resume protocol. // The retry marker contains only hashes, so this succeeds only if the // worker reconstructs the exact failed operation from the encrypted // _operation_recovery envelope retained across attempts 1 through 13. Capsule::table('cloud_jobs')->where('id', $preparedLimitId)->update([ 'available_at' => 0, 'lease_until' => 0, 'lease_token' => null, ]); $client->responses[] = static fn (): array => $fenceResponse($preparedLimitFenceWrite, time()); $beforePreparedLimitResume = count($client->requests); $assert((new CloudAutomaticOperationRecoveryService())->tick(), 'Prepared recovery was not claimable after attempt thirteen.'); $preparedLimitResumeRequests = array_slice( $client->requests, $beforePreparedLimitResume ); $preparedLimitResolved = CloudJob::find($preparedLimitId); $preparedLimitFinalPayload = CloudSecrets::decrypt( (string) $preparedLimitResolved->sealed_payload, 'job-payload' ); $assert((string) $preparedLimitResolved->effect_resolution === 'not_applied' && !(bool) $preparedLimitResolved->finalization_incident && $preparedLimitResolved->finalization_error_code === null && !array_key_exists( '_operation_recovery', $preparedLimitFinalPayload ) && !CloudJobService::blockingQueryForInstance( $preparedLimitId )->exists() && count($preparedLimitResumeRequests) === 1 && $preparedLimitResumeRequests[0]['method'] === 'GET' && str_ends_with( $preparedLimitResumeRequests[0]['target'], '/fence-resolution' ), 'Attempt thirteen stranded encrypted prepared recovery state.'); // A timed-out graceful delete may later be proven not applied because the // Agent requires an explicitly authorized force generation. Recovery must // preserve that exact business error: it is the immutable authorization // anchor for enqueueForceDelete(), not an instance-state contradiction. [$forceRequiredId, $forceRequiredUuid] = $insertSettingsJob( str_repeat('ab', 16), ['hostname' => 'delete-force-required.example'] ); Capsule::table('cloud_jobs')->where('id', $forceRequiredId)->update([ 'kind' => 'delete', 'sealed_payload' => CloudSecrets::encrypt([ 'method' => 'DELETE', 'request_target' => '/v1/vms/' . $forceRequiredUuid, 'body' => ['remove_managed_disks' => true], 'owner_id' => 1, 'can_manage' => false, ], 'job-payload'), ]); $beforeForceRequiredRecovery = count($client->requests); $client->responses[] = $inspection( str_repeat('ab', 16), 'failed', 'not_applied', null, false, 'force_required', 'not_required' ); $assert((new CloudAutomaticOperationRecoveryService())->tick(), 'The force-required delete recovery was not claimed.'); $forceRequired = CloudJob::find($forceRequiredId); $forceRequiredInstance = \App\Models\CloudInstance::find( $forceRequiredId ); $assert((string) $forceRequired->effect_resolution === 'not_applied' && (string) $forceRequired->error_code === 'force_required' && (string) $forceRequired->phase === 'terminal' && (string) $forceRequired->outcome === 'failed' && $forceRequired->dispatched_at !== null, 'Force-required inspection did not converge into its authorization anchor: ' . json_encode($forceRequired->toArray(), JSON_THROW_ON_ERROR)); $beforeForceJobCount = CloudJob::count(); Capsule::table('cloud_jobs')->where('id', $forceRequiredId)->update([ 'finalization_incident' => 1, 'finalization_error_code' => 'recovery_ledger_conflict', ]); try { CloudJobService::enqueueForceDelete( $forceRequired->fresh(), [ 'user_id' => 99, 'is_admin' => true, 'is_super_admin' => true, 'can_manage' => true, ] ); throw new RuntimeException( 'A sticky recovery incident created a force-delete generation.' ); } catch (CloudError $error) { $assert($error->httpStatus === 409 && CloudJob::count() === $beforeForceJobCount, 'Sticky force-delete rejection changed the immutable job ledger.'); } Capsule::table('cloud_jobs')->where('id', $forceRequiredId)->update([ 'finalization_incident' => 0, 'finalization_error_code' => null, 'receipt_deadline_at' => null, ]); try { CloudJobService::enqueueForceDelete( $forceRequired->fresh(), [ 'user_id' => 99, 'is_admin' => true, 'is_super_admin' => true, 'can_manage' => true, ] ); throw new RuntimeException( 'A damaged Agent envelope created a force-delete generation.' ); } catch (CloudError $error) { $assert($error->httpStatus === 409 && CloudJob::count() === $beforeForceJobCount, 'Envelope force-delete rejection changed the immutable job ledger.'); } Capsule::table('cloud_jobs')->where('id', $forceRequiredId)->update([ 'receipt_deadline_at' => $now + 120, ]); $forceRequired->refresh(); $forceDelete = CloudJobService::enqueueForceDelete( $forceRequired, [ 'user_id' => 99, 'is_admin' => true, 'is_super_admin' => true, 'can_manage' => true, ] ); $nextId = max($nextId, (int) $forceDelete->id + 1); $forceDeletePayload = CloudSecrets::decrypt( (string) $forceDelete->sealed_payload, 'job-payload' ); $assert((string) $forceRequired->effect_resolution === 'not_applied' && (string) $forceRequired->error_code === 'force_required' && (string) $forceRequiredInstance->status === 'active' && !(bool) $forceRequired->finalization_incident && (string) $forceDelete->kind === 'delete_force' && ($forceDeletePayload['prior_job_id'] ?? null) === $forceRequiredId && ($forceDeletePayload['prior_operation_id'] ?? null) === str_repeat('ab', 16) && count($client->requests) === $beforeForceRequiredRecovery + 1 && $client->requests[array_key_last($client->requests)]['method'] === 'GET', 'Recovered force-required evidence did not preserve the separately authorized force-delete path.'); // Age is also audit metadata only; a six-hour-old operation is still // inspected with the original operation ID. [$agedId] = $insertSettingsJob( str_repeat('f', 32), ['hostname' => 'age-limit.example'] ); Capsule::table('cloud_jobs')->where('id', $agedId)->update([ 'effect_resolution_reason' => 'automatic_recovery:v1:' . (time() - 21601) . ':1:pending', ]); $beforeAged = count($client->requests); $client->responses[] = new CloudAgentError( 'offline', 503, 'agent_unavailable', true, 503 ); $assert((new CloudAutomaticOperationRecoveryService())->tick(), 'A six-hour-old recovery was not claimed.'); $aged = CloudJob::find($agedId); $assert((bool) $aged->finalization_incident && (string) $aged->finalization_error_code === 'recovery_waiting_connection' && preg_match( '/^automatic_recovery:v1:[1-9][0-9]{0,10}:2:pending$/D', (string) $aged->effect_resolution_reason ) === 1 && (string) $aged->effect_resolution === 'unknown' && CloudJobService::blockingQueryForInstance($agedId)->exists() && count($client->requests) === $beforeAged + 1, 'An aged recovery stopped retrying or released safety.'); // A corrupt dispatch ledger remains a recovery candidate forever, but it // cannot authorize either an Agent read or consumption of even an exact // received completion. The only safe automatic action is to record the // concrete incident and back off until an operator repairs the 0/1 fence. [$invalidDispatchId, $invalidDispatchUuid] = $insertSettingsJob( str_repeat('dc', 16), ['hostname' => 'invalid-dispatch-ledger.example'] ); $invalidDispatchEventId = 'automatic-invalid-dispatch-ledger'; Capsule::table('cloud_jobs')->where('id', $invalidDispatchId)->update([ 'dispatch_attempts' => 2, // Simulate a historical fail-open release which mistook the corrupt // non-zero counter for a fully settled successful Agent mutation. 'outcome' => 'succeeded', 'status' => 'succeeded', 'error_code' => null, 'effect_may_have_applied' => 0, 'safe_to_resubmit' => 0, 'effect_resolution' => 'applied', 'completion_event_id' => $invalidDispatchEventId, 'available_at' => 0, 'lease_token' => null, 'lease_until' => 0, 'finalization_incident' => 0, 'finalization_error_code' => null, ]); $invalidDispatchJob = CloudJob::findOrFail($invalidDispatchId); $invalidDispatchEnvelope = (string) $invalidDispatchJob->sealed_payload; $invalidDispatchTerminalAt = (int) $invalidDispatchJob->terminal_at; $invalidDispatchResultJson = (string) $invalidDispatchJob->result_json; $invalidDispatchIdentity = CloudOperationRecoveryIdentity::expected( $invalidDispatchJob, true ); $invalidDispatchResult = [ 'uuid' => $invalidDispatchUuid, 'hostname' => 'invalid-dispatch-ledger.example', 'settings_changed' => true, ]; $invalidDispatchWire = [ 'event_id' => $invalidDispatchEventId, 'host_id' => 7, 'operation_id' => (string) $invalidDispatchJob->operation_id, 'action' => $invalidDispatchIdentity['action'], 'resource_id' => $invalidDispatchIdentity['resource_id'], 'request_fingerprint' => $invalidDispatchIdentity['request_fingerprint'], 'status' => 'succeeded', 'effect_may_have_applied' => false, 'safe_to_resubmit' => false, 'error_code' => '', 'result' => $invalidDispatchResult, ]; Capsule::table('cloud_agent_completion_events')->insert([ 'event_id' => $invalidDispatchWire['event_id'], 'host_id' => 7, 'operation_id' => $invalidDispatchWire['operation_id'], 'payload_sha256' => hash('sha256', json_encode( $invalidDispatchWire, JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE )), 'outcome' => 'succeeded', 'effect_may_have_applied' => 0, 'safe_to_resubmit' => 0, 'error_code' => '', 'result_json' => json_encode([ 'schema' => 'virt_cloud_completion_event', 'version' => 1, 'identity' => $invalidDispatchIdentity, 'result' => $invalidDispatchResult, ], JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE), 'disposition' => 'received', 'received_at' => time(), 'applied_at' => null, ]); $beforeInvalidDispatchRequests = count($client->requests); $assert((new CloudAutomaticOperationRecoveryService())->tick(), 'An invalid dispatch ledger permanently left the recovery queue.'); $invalidDispatchJob = CloudJob::findOrFail($invalidDispatchId); $invalidDispatchEvent = Capsule::table( 'cloud_agent_completion_events' )->where( 'event_id', $invalidDispatchWire['event_id'] )->first(); $assert((string) $invalidDispatchJob->phase === 'terminal' && (string) $invalidDispatchJob->outcome === 'succeeded' && (string) $invalidDispatchJob->status === 'succeeded' && $invalidDispatchJob->error_code === null && !(bool) $invalidDispatchJob->effect_may_have_applied && (string) $invalidDispatchJob->effect_resolution === 'applied' && !(bool) $invalidDispatchJob->safe_to_resubmit && (int) $invalidDispatchJob->terminal_at === $invalidDispatchTerminalAt && (string) $invalidDispatchJob->result_json === $invalidDispatchResultJson && (string) $invalidDispatchJob->completion_event_id === $invalidDispatchEventId && (bool) $invalidDispatchJob->finalization_incident && (string) $invalidDispatchJob->finalization_error_code === 'recovery_dispatch_ledger_conflict' && (string) $invalidDispatchJob->sealed_payload === $invalidDispatchEnvelope && $invalidDispatchJob->lease_token === null && (int) $invalidDispatchJob->lease_until === 0 && (int) $invalidDispatchJob->available_at >= time() + 290 && (string) $invalidDispatchEvent->disposition === 'received' && $invalidDispatchEvent->applied_at === null && count($client->requests) === $beforeInvalidDispatchRequests, 'Invalid dispatch recovery contacted Agent, consumed evidence, or released its fence.'); // Once the durable ledger is repaired to the only valid dispatched state, // the same unconsumed receipt must resume convergence without waiting for // an Agent retransmission or issuing any Agent request. Capsule::table('cloud_jobs')->where('id', $invalidDispatchId)->update([ 'dispatch_attempts' => 1, 'available_at' => 0, 'lease_token' => null, 'lease_until' => 0, ]); $assert((new CloudAutomaticOperationRecoveryService())->tick(), 'A repaired dispatch ledger could not resume its queued receipt.'); $repairedDispatchJob = CloudJob::findOrFail($invalidDispatchId); $repairedDispatchEvent = Capsule::table( 'cloud_agent_completion_events' )->where( 'event_id', $invalidDispatchWire['event_id'] )->first(); $assert((string) $repairedDispatchJob->phase === 'terminal' && (string) $repairedDispatchJob->outcome === 'succeeded' && (string) $repairedDispatchJob->status === 'succeeded' && (int) $repairedDispatchJob->terminal_at === $invalidDispatchTerminalAt && $repairedDispatchJob->error_code === null && (string) $repairedDispatchJob->result_json === $invalidDispatchResultJson && (string) $repairedDispatchJob->effect_resolution === 'applied' && !(bool) $repairedDispatchJob->finalization_incident && $repairedDispatchJob->finalization_error_code === null && (string) $repairedDispatchEvent->disposition === 'applied' && (int) $repairedDispatchEvent->applied_at > 0 && count($client->requests) === $beforeInvalidDispatchRequests, 'Repairing the dispatch ledger did not resume the exact stored completion.'); // An exact immutable receipt cannot authorize convergence when any other // field in the dispatched Agent envelope is damaged. Claiming must keep // the receipt untouched and record a local-only state conflict without an // Agent request. [$corruptReceiptEnvelopeId, $corruptReceiptEnvelopeUuid] = $insertSettingsJob( str_repeat('b1', 16), ['hostname' => 'corrupt-receipt-envelope.example'] ); $corruptReceiptEnvelopeJob = CloudJob::findOrFail( $corruptReceiptEnvelopeId ); $corruptReceiptEnvelopeIdentity = CloudOperationRecoveryIdentity::expected( $corruptReceiptEnvelopeJob, true ); $corruptReceiptEnvelopeResult = [ 'uuid' => $corruptReceiptEnvelopeUuid, 'hostname' => 'corrupt-receipt-envelope.example', 'settings_changed' => true, ]; $corruptReceiptEnvelopeWire = [ 'event_id' => 'automatic-corrupt-receipt-envelope', 'host_id' => 7, 'operation_id' => (string) $corruptReceiptEnvelopeJob->operation_id, 'action' => $corruptReceiptEnvelopeIdentity['action'], 'resource_id' => $corruptReceiptEnvelopeIdentity['resource_id'], 'request_fingerprint' => $corruptReceiptEnvelopeIdentity['request_fingerprint'], 'status' => 'succeeded', 'effect_may_have_applied' => false, 'safe_to_resubmit' => false, 'error_code' => '', 'result' => $corruptReceiptEnvelopeResult, ]; Capsule::table('cloud_agent_completion_events')->insert([ 'event_id' => $corruptReceiptEnvelopeWire['event_id'], 'host_id' => 7, 'operation_id' => $corruptReceiptEnvelopeWire['operation_id'], 'payload_sha256' => hash('sha256', json_encode( $corruptReceiptEnvelopeWire, JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE )), 'outcome' => 'succeeded', 'effect_may_have_applied' => 0, 'safe_to_resubmit' => 0, 'error_code' => '', 'result_json' => json_encode([ 'schema' => 'virt_cloud_completion_event', 'version' => 1, 'identity' => $corruptReceiptEnvelopeIdentity, 'result' => $corruptReceiptEnvelopeResult, ], JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE), 'disposition' => 'received', 'received_at' => time(), 'applied_at' => null, ]); Capsule::table('cloud_jobs')->where( 'id', $corruptReceiptEnvelopeId )->update([ 'request_idempotency_key' => 'bad-receipt-key', 'available_at' => 0, 'lease_token' => null, 'lease_until' => 0, ]); Capsule::table('cloud_templates')->insert([ 'id' => 'identity-priority-template', 'active_job_id' => $corruptReceiptEnvelopeId, 'state' => 'active', ]); $beforeCorruptReceiptEnvelopeRequests = count($client->requests); $assert((new CloudAutomaticOperationRecoveryService())->tick(), 'A damaged envelope with an exact receipt left recovery.'); $corruptReceiptEnvelopeJob = CloudJob::findOrFail( $corruptReceiptEnvelopeId ); $corruptReceiptEnvelopeEvent = Capsule::table( 'cloud_agent_completion_events' )->where( 'event_id', $corruptReceiptEnvelopeWire['event_id'] )->first(); $assert((string) $corruptReceiptEnvelopeJob->effect_resolution === 'unknown' && (bool) $corruptReceiptEnvelopeJob->finalization_incident && (string) $corruptReceiptEnvelopeJob->finalization_error_code === 'recovery_operation_identity_conflict' && (int) $corruptReceiptEnvelopeJob->available_at >= time() + 290 && (int) $corruptReceiptEnvelopeJob->lease_until >= time() + 290 && (string) $corruptReceiptEnvelopeEvent->disposition === 'received' && $corruptReceiptEnvelopeEvent->applied_at === null && (int) Capsule::table('cloud_templates')->where( 'id', 'identity-priority-template' )->value('active_job_id') === $corruptReceiptEnvelopeId && CloudJobService::blockingQueryForInstance( $corruptReceiptEnvelopeId )->exists() && count($client->requests) === $beforeCorruptReceiptEnvelopeRequests, 'A damaged envelope consumed its receipt, contacted Agent, or falsely unlocked.'); Capsule::table('cloud_templates')->where( 'id', 'identity-priority-template' )->delete(); // A prepared no-fence conclusion must revalidate the complete envelope at // the locked publication boundary. This protects direct callers and the // crash-resume path even if claim-time validation is bypassed or races. [$preparedStateConflictId, $preparedStateConflictUuid] = $insertSettingsJob( str_repeat('b2', 16), ['hostname' => 'prepared-state-conflict.example'] ); $preparedStateConflictIdentity = CloudOperationRecoveryIdentity::expected( CloudJob::findOrFail($preparedStateConflictId), true ); $preparedStateConflictOperation = $preparedStateConflictIdentity + [ 'operation_id' => str_repeat('b2', 16), 'status' => 'succeeded', 'effect_may_have_applied' => false, 'safe_to_resubmit' => false, 'error_code' => '', 'result' => [ 'uuid' => $preparedStateConflictUuid, 'hostname' => 'prepared-state-conflict.example', 'settings_changed' => true, ], ]; $preparedStateConflictSource = 'automatic_applied_succeeded_nofence:' . str_repeat('2', 64); CloudOperationRecoveryService::prepareApplied( $preparedStateConflictId, $preparedStateConflictOperation, $preparedStateConflictSource ); $preparedStateConflictMarker = (string) CloudJob::findOrFail( $preparedStateConflictId )->effect_resolution_reason; Capsule::table('cloud_jobs')->where( 'id', $preparedStateConflictId )->update([ 'receipt_deadline_at' => 0, 'available_at' => 0, 'lease_token' => null, 'lease_until' => 0, ]); try { CloudOperationRecoveryService::commitApplied( $preparedStateConflictId, $preparedStateConflictOperation, $preparedStateConflictSource, 0 ); throw new RuntimeException( 'A direct applied commit published a damaged Agent envelope.' ); } catch (CloudRecoveryInstanceStateConflict $error) { $assert($error->httpStatus === 409, 'The applied envelope guard returned the wrong status.'); } $preparedStateConflict = CloudJob::findOrFail( $preparedStateConflictId ); $assert((string) $preparedStateConflict->effect_resolution === 'unknown' && (string) $preparedStateConflict->effect_resolution_reason === $preparedStateConflictMarker, 'A rejected applied commit changed its prepared recovery intent.'); $beforePreparedStateConflictRequests = count($client->requests); $assert((new CloudAutomaticOperationRecoveryService())->tick(), 'A prepared no-fence envelope conflict left recovery.'); $preparedStateConflict = CloudJob::findOrFail( $preparedStateConflictId ); $assert((string) $preparedStateConflict->effect_resolution === 'unknown' && (string) $preparedStateConflict->effect_resolution_reason === $preparedStateConflictMarker && (bool) $preparedStateConflict->finalization_incident && (string) $preparedStateConflict->finalization_error_code === 'recovery_instance_state_conflict' && CloudJobService::blockingQueryForInstance( $preparedStateConflictId )->exists() && count($client->requests) === $beforePreparedStateConflictRequests, 'A prepared no-fence conflict contacted Agent, published, or unlocked.'); // The not-applied publication boundary applies the same immutable identity // guard. A prepared proof cannot rely only on its encrypted escrow after // the durable request fingerprint column has been damaged. [$preparedIdentityConflictId] = $insertSettingsJob( str_repeat('b3', 16), ['hostname' => 'prepared-identity-conflict.example'] ); $preparedIdentityConflictIdentity = CloudOperationRecoveryIdentity::expected( CloudJob::findOrFail($preparedIdentityConflictId), false ); $preparedIdentityConflictOperation = $preparedIdentityConflictIdentity + [ 'operation_id' => str_repeat('b3', 16), 'status' => 'failed', 'effect_may_have_applied' => false, 'safe_to_resubmit' => true, 'error_code' => 'settings_not_committed', 'result' => ['agent_detail' => 'bounded'], ]; $preparedIdentityConflictSource = 'automatic_not_applied_failed_nofence:' . str_repeat('3', 64); CloudOperationRecoveryService::prepareNotApplied( $preparedIdentityConflictId, $preparedIdentityConflictOperation, $preparedIdentityConflictSource ); $preparedIdentityConflictMarker = (string) CloudJob::findOrFail( $preparedIdentityConflictId )->effect_resolution_reason; Capsule::table('cloud_jobs')->where( 'id', $preparedIdentityConflictId )->update([ 'request_fingerprint' => 'bad-fingerprint', 'available_at' => 0, 'lease_token' => null, 'lease_until' => 0, ]); try { CloudOperationRecoveryService::commitNotApplied( $preparedIdentityConflictId, $preparedIdentityConflictOperation, $preparedIdentityConflictSource, 0 ); throw new RuntimeException( 'A direct not-applied commit published damaged request identity.' ); } catch (CloudRecoveryOperationIdentityConflict $error) { $assert($error->httpStatus === 409, 'The not-applied identity guard returned the wrong status.'); } $preparedIdentityConflict = CloudJob::findOrFail( $preparedIdentityConflictId ); $assert((string) $preparedIdentityConflict->effect_resolution === 'unknown' && (string) $preparedIdentityConflict->effect_resolution_reason === $preparedIdentityConflictMarker, 'A rejected not-applied commit changed its prepared recovery intent.'); $beforePreparedIdentityConflictRequests = count($client->requests); $assert((new CloudAutomaticOperationRecoveryService())->tick(), 'A prepared request-identity conflict left recovery.'); $preparedIdentityConflict = CloudJob::findOrFail( $preparedIdentityConflictId ); $assert((string) $preparedIdentityConflict->effect_resolution === 'unknown' && (string) $preparedIdentityConflict->effect_resolution_reason === $preparedIdentityConflictMarker && (bool) $preparedIdentityConflict->finalization_incident && (string) $preparedIdentityConflict->finalization_error_code === 'recovery_operation_identity_conflict' && CloudJobService::blockingQueryForInstance( $preparedIdentityConflictId )->exists() && count($client->requests) === $beforePreparedIdentityConflictRequests, 'A prepared identity conflict contacted Agent, published, or unlocked.'); // A previously published effect with an older incident is not exempt from // envelope validation. The immutable terminal audit remains resolved, but // the concrete envelope conflict becomes visible and the resource remains // blocked without consulting Agent state. [$resolvedEnvelopeConflictId] = $insertSettingsJob( str_repeat('b4', 16), ['hostname' => 'resolved-envelope-conflict.example'] ); Capsule::table('cloud_jobs')->where( 'id', $resolvedEnvelopeConflictId )->update([ 'effect_resolution' => 'applied', 'effect_resolved_at' => time() - 5, 'effect_resolved_by' => 0, 'effect_resolution_reason' => 'prior_exact_evidence', 'receipt_deadline_at' => 0, 'finalization_incident' => 1, 'finalization_error_code' => 'recovery_waiting_connection', 'available_at' => 0, 'lease_token' => null, 'lease_until' => 0, ]); $beforeResolvedEnvelopeConflictRequests = count($client->requests); $assert((new CloudAutomaticOperationRecoveryService())->tick(), 'A resolved task with a damaged envelope left local recovery.'); $resolvedEnvelopeConflict = CloudJob::findOrFail( $resolvedEnvelopeConflictId ); $assert((string) $resolvedEnvelopeConflict->effect_resolution === 'applied' && (string) $resolvedEnvelopeConflict->effect_resolution_reason === 'prior_exact_evidence' && (bool) $resolvedEnvelopeConflict->finalization_incident && (string) $resolvedEnvelopeConflict->finalization_error_code === 'recovery_instance_state_conflict' && (int) $resolvedEnvelopeConflict->available_at >= time() + 290 && CloudJobService::blockingQueryForInstance( $resolvedEnvelopeConflictId )->exists() && count($client->requests) === $beforeResolvedEnvelopeConflictRequests, 'A resolved envelope conflict contacted Agent, hid its cause, or falsely unlocked.'); // Repairing a failed row to the dispatched state is not enough when the // immutable host/operation/time envelope is still damaged. Keep it in the // local envelope-conflict queue instead of publishing a recovery that can // never satisfy the normal Agent claim. [$incompleteRepairId] = $insertSettingsJob( str_repeat('bd', 16), ['hostname' => 'incomplete-dispatch-repair.example'] ); Capsule::table('cloud_jobs')->where('id', $incompleteRepairId)->update([ 'dispatch_attempts' => 2, 'available_at' => 0, 'lease_token' => null, 'lease_until' => 0, ]); CloudOperationCompletionService::deferDispatchLedgerConflict( CloudJob::findOrFail($incompleteRepairId) ); Capsule::table('cloud_jobs')->where('id', $incompleteRepairId)->update([ 'dispatch_attempts' => 1, 'dispatched_at' => null, 'available_at' => 0, 'lease_token' => null, 'lease_until' => 0, ]); $beforeIncompleteRepairRequests = count($client->requests); $assert((new CloudAutomaticOperationRecoveryService())->tick(), 'An incomplete repaired dispatch envelope was not checked.'); $incompleteRepair = CloudJob::findOrFail($incompleteRepairId); $assert((string) $incompleteRepair->effect_resolution === 'unknown' && (bool) $incompleteRepair->finalization_incident && (string) $incompleteRepair->finalization_error_code === 'recovery_dispatch_repair_instance_state_conflict' && (int) $incompleteRepair->lease_until >= time() + 290 && (int) $incompleteRepair->available_at >= time() + 290 && count($client->requests) === $beforeIncompleteRepairRequests, 'An incomplete repaired dispatch envelope left its sticky queue.'); // A pre-existing dispatched/unknown row with a damaged immutable envelope // must still be claimed forever, but without attempting an Agent request. [$malformedEnvelopeId] = $insertSettingsJob( str_repeat('bc', 16), ['hostname' => 'malformed-envelope.example'] ); Capsule::table('cloud_jobs')->where('id', $malformedEnvelopeId)->update([ 'dispatched_at' => null, 'finalization_incident' => 0, 'finalization_error_code' => null, 'available_at' => 0, 'lease_token' => null, 'lease_until' => 0, ]); $beforeMalformedEnvelopeRequests = count($client->requests); $assert((new CloudAutomaticOperationRecoveryService())->tick(), 'A damaged dispatched recovery envelope fell out of the queue.'); $malformedEnvelope = CloudJob::findOrFail($malformedEnvelopeId); $assert((bool) $malformedEnvelope->finalization_incident && (string) $malformedEnvelope->finalization_error_code === 'recovery_instance_state_conflict' && (string) $malformedEnvelope->effect_resolution === 'unknown' && (int) $malformedEnvelope->lease_until >= time() + 290 && (int) $malformedEnvelope->available_at >= time() + 290 && count($client->requests) === $beforeMalformedEnvelopeRequests, 'A damaged dispatched recovery envelope contacted Agent or lost its concrete incident.'); [$invalidOperationIdJob] = $insertSettingsJob( str_repeat('G', 32), ['hostname' => 'invalid-operation-id.example'] ); Capsule::table('cloud_jobs')->where('id', $invalidOperationIdJob)->update([ 'available_at' => 0, 'lease_token' => null, 'lease_until' => 0, ]); $beforeInvalidOperationIdRequests = count($client->requests); $assert((new CloudAutomaticOperationRecoveryService())->tick(), 'A malformed Agent operation ID fell out of recovery.'); $invalidOperationIdentity = CloudJob::findOrFail( $invalidOperationIdJob ); $assert((bool) $invalidOperationIdentity->finalization_incident && (string) $invalidOperationIdentity->finalization_error_code === 'recovery_operation_identity_conflict' && (string) $invalidOperationIdentity->effect_resolution === 'unknown' && (int) $invalidOperationIdentity->available_at >= time() + 290 && count($client->requests) === $beforeInvalidOperationIdRequests, 'A malformed operation ID lost its concrete identity-conflict status or contacted Agent.'); // A historical succeeded row without a bound completion event must still // prove the immutable dispatched envelope before its sticky repair // incident can be cleared. Otherwise repairing only the numeric counter // would release the instance while host/time identity remained corrupt. [$incompleteSucceededRepairId] = $insertSettingsJob( str_repeat('be', 16), ['hostname' => 'incomplete-succeeded-repair.example'] ); $incompleteSucceededResult = json_encode([ 'hostname' => 'incomplete-succeeded-repair.example', 'settings_changed' => true, ], JSON_THROW_ON_ERROR); Capsule::table('cloud_jobs')->where( 'id', $incompleteSucceededRepairId )->update([ 'dispatch_attempts' => 2, 'outcome' => 'succeeded', 'status' => 'succeeded', 'error_code' => null, 'effect_may_have_applied' => 0, 'safe_to_resubmit' => 0, 'effect_resolution' => 'applied', 'completion_event_id' => null, 'result_json' => $incompleteSucceededResult, 'available_at' => 0, 'lease_token' => null, 'lease_until' => 0, ]); $incompleteSucceededBefore = $audit($incompleteSucceededRepairId); $incompleteSucceededBefore['dispatch_attempts'] = 1; $incompleteSucceededEnvelope = (string) CloudJob::findOrFail( $incompleteSucceededRepairId )->sealed_payload; CloudOperationCompletionService::deferDispatchLedgerConflict( CloudJob::findOrFail($incompleteSucceededRepairId) ); Capsule::table('cloud_jobs')->where( 'id', $incompleteSucceededRepairId )->update([ 'dispatch_attempts' => 1, 'dispatched_at' => null, 'available_at' => 0, 'lease_token' => null, 'lease_until' => 0, ]); $beforeIncompleteSucceededRequests = count($client->requests); $assert((new CloudAutomaticOperationRecoveryService())->tick(), 'An incomplete succeeded dispatch envelope was not checked.'); $incompleteSucceededRepair = CloudJob::findOrFail( $incompleteSucceededRepairId ); $assert((bool) $incompleteSucceededRepair->finalization_incident && (string) $incompleteSucceededRepair->finalization_error_code === 'recovery_dispatch_repair_instance_state_conflict' && (int) $incompleteSucceededRepair->lease_until >= time() + 290 && (int) $incompleteSucceededRepair->available_at >= time() + 290 && count($client->requests) === $beforeIncompleteSucceededRequests, 'A succeeded task cleared its repair incident without a complete dispatched envelope.'); $assert($audit($incompleteSucceededRepairId) === $incompleteSucceededBefore, 'Checking an incomplete succeeded envelope rewrote terminal audit data: ' . json_encode([ 'expected' => $incompleteSucceededBefore, 'actual' => $audit($incompleteSucceededRepairId), ], JSON_THROW_ON_ERROR)); Capsule::table('cloud_jobs')->where( 'id', $incompleteSucceededRepairId )->update([ 'dispatched_at' => time() - 980, 'sealed_payload' => 'automatic-recovery-test:missing', 'available_at' => 0, 'lease_token' => null, 'lease_until' => 0, ]); $assert((new CloudAutomaticOperationRecoveryService())->tick(), 'A succeeded repair with a damaged identity escrow was not checked.'); $damagedSucceededRepair = CloudJob::findOrFail( $incompleteSucceededRepairId ); $assert((bool) $damagedSucceededRepair->finalization_incident && (string) $damagedSucceededRepair->finalization_error_code === 'recovery_dispatch_repair_payload_unavailable' && (int) $damagedSucceededRepair->lease_until >= time() + 290 && (int) $damagedSucceededRepair->available_at >= time() + 290, 'A succeeded repair cleared its sticky incident without validating the immutable identity.'); Capsule::table('cloud_jobs')->where( 'id', $incompleteSucceededRepairId )->update([ 'sealed_payload' => $incompleteSucceededEnvelope, 'available_at' => 0, 'lease_token' => null, 'lease_until' => 0, ]); $assert((new CloudAutomaticOperationRecoveryService())->tick(), 'A completed succeeded dispatch envelope did not resume repair.'); $completedSucceededRepair = CloudJob::findOrFail( $incompleteSucceededRepairId ); $assert(!(bool) $completedSucceededRepair->finalization_incident && $completedSucceededRepair->finalization_error_code === null && $audit($incompleteSucceededRepairId) === $incompleteSucceededBefore && count($client->requests) === $beforeIncompleteSucceededRequests, 'A complete succeeded dispatch envelope was not unlocked without rewriting its terminal audit.'); // Repairing a corrupt counter to zero cannot erase already durable Agent // evidence. The sticky repair queue must outrank the PHP-only finalizer; // after the operator chooses one, the same receipt resumes convergence. [$zeroCounterConflictId, $zeroCounterConflictUuid] = $insertSettingsJob( str_repeat('bf', 16), ['hostname' => 'zero-counter-conflict.example'] ); $zeroCounterConflictJob = CloudJob::findOrFail($zeroCounterConflictId); $zeroCounterIdentity = CloudOperationRecoveryIdentity::expected( $zeroCounterConflictJob, true ); $zeroCounterEventId = 'zero-counter-proof-01'; $zeroCounterWire = [ 'type' => 'completion', 'id' => $zeroCounterEventId, 'agent_id' => '7', 'operation_id' => (string) $zeroCounterConflictJob->operation_id, 'action' => $zeroCounterIdentity['action'], 'resource_id' => $zeroCounterIdentity['resource_id'], 'request_fingerprint' => $zeroCounterIdentity['request_fingerprint'], 'status' => 'succeeded', 'effect_may_have_applied' => false, 'safe_to_resubmit' => false, 'error_code' => '', 'result' => [ 'uuid' => $zeroCounterConflictUuid, 'hostname' => 'zero-counter-conflict.example', 'settings_changed' => true, ], ]; Capsule::table('cloud_jobs')->where( 'id', $zeroCounterConflictId )->update([ 'dispatch_attempts' => 2, 'available_at' => 0, 'lease_token' => null, 'lease_until' => 0, ]); $beforeZeroCounterRequests = count($client->requests); $acceptedZeroCounter = CloudOperationCompletionService::accept( $zeroCounterWire ); $zeroCounterEnvelope = (string) CloudJob::findOrFail( $zeroCounterConflictId )->sealed_payload; $zeroCounterTerminalAt = (int) CloudJob::findOrFail( $zeroCounterConflictId )->terminal_at; Capsule::table('cloud_jobs')->where( 'id', $zeroCounterConflictId )->update([ 'dispatch_attempts' => 0, 'available_at' => 0, 'lease_token' => null, 'lease_until' => 0, ]); $assert((new CloudAutomaticOperationRecoveryService())->tick(), 'A repaired-zero ledger with Agent evidence was not checked.'); $zeroCounterConflict = CloudJob::findOrFail($zeroCounterConflictId); $zeroCounterEvent = Capsule::table( 'cloud_agent_completion_events' )->where('event_id', $zeroCounterEventId)->first(); $assert($acceptedZeroCounter === [ 'accepted' => true, 'disposition' => 'received', ] && (bool) $zeroCounterConflict->finalization_incident && (string) $zeroCounterConflict->finalization_error_code === 'recovery_dispatch_repair_instance_state_conflict' && (string) $zeroCounterConflict->effect_resolution === 'unknown' && (string) $zeroCounterConflict->sealed_payload === $zeroCounterEnvelope && (string) $zeroCounterEvent->disposition === 'received' && $zeroCounterEvent->applied_at === null && (int) $zeroCounterConflict->lease_until >= time() + 290 && (int) $zeroCounterConflict->available_at >= time() + 290 && count($client->requests) === $beforeZeroCounterRequests, 'Repairing a dispatched operation to zero ran the PHP-only finalizer or consumed its receipt.'); Capsule::table('cloud_jobs')->where( 'id', $zeroCounterConflictId )->update([ 'dispatch_attempts' => 1, 'available_at' => 0, 'lease_token' => null, 'lease_until' => 0, ]); $assert(CloudOperationCompletionService::accept($zeroCounterWire) === [ 'accepted' => true, 'disposition' => 'received', ] && (string) Capsule::table('cloud_agent_completion_events') ->where('event_id', $zeroCounterEventId)->value('disposition') === 'received', 'A retransmitted receipt bypassed the sticky dispatch-repair coordinator.'); $assert((new CloudAutomaticOperationRecoveryService())->tick(), 'The repaired dispatched ledger did not re-enter Agent recovery.'); $assert((new CloudAutomaticOperationRecoveryService())->tick(), 'The repaired dispatched ledger did not consume its retained receipt.'); $zeroCounterResolved = CloudJob::findOrFail($zeroCounterConflictId); $zeroCounterResolvedEvent = Capsule::table( 'cloud_agent_completion_events' )->where('event_id', $zeroCounterEventId)->first(); $assert((string) $zeroCounterResolved->phase === 'terminal' && (string) $zeroCounterResolved->outcome === 'failed' && (string) $zeroCounterResolved->status === 'failed' && (int) $zeroCounterResolved->terminal_at === $zeroCounterTerminalAt && (string) $zeroCounterResolved->effect_resolution === 'applied' && !(bool) $zeroCounterResolved->finalization_incident && $zeroCounterResolved->finalization_error_code === null && (string) $zeroCounterResolvedEvent->disposition === 'applied' && (int) $zeroCounterResolvedEvent->applied_at > 0 && count($client->requests) === $beforeZeroCounterRequests, 'Repairing the counter to one did not safely resume the retained Agent receipt.'); // The same contradiction on an active generation-zero task must move the // row to a sticky terminal fence before the ordinary worker can dispatch. [$activeZeroConflictId] = $insertSettingsJob( str_repeat('c0', 16), ['hostname' => 'active-zero-conflict.example'] ); Capsule::table('cloud_jobs')->where('id', $activeZeroConflictId)->update([ 'phase' => 'accepted', 'outcome' => null, 'status' => 'queued', 'attempts' => 0, 'dispatch_attempts' => 0, 'dispatch_started_at' => null, 'agent_accepted_at' => null, 'receipt_deadline_at' => null, 'terminal_at' => null, 'effect_may_have_applied' => null, 'safe_to_resubmit' => null, 'effect_resolution' => 'not_applicable', 'dispatched_at' => null, 'error_code' => null, 'finalization_incident' => 0, 'finalization_error_code' => null, 'result_json' => null, 'available_at' => 0, 'lease_token' => null, 'lease_until' => 0, ]); $activeZeroJob = CloudJob::findOrFail($activeZeroConflictId); $activeZeroIdentity = CloudOperationRecoveryIdentity::expected( $activeZeroJob, true ); $activeZeroEventId = 'active-zero-proof-01'; $activeZeroAccepted = CloudOperationCompletionService::accept([ 'type' => 'completion', 'id' => $activeZeroEventId, 'agent_id' => '7', 'operation_id' => (string) $activeZeroJob->operation_id, 'action' => $activeZeroIdentity['action'], 'resource_id' => $activeZeroIdentity['resource_id'], 'request_fingerprint' => $activeZeroIdentity['request_fingerprint'], 'status' => 'succeeded', 'effect_may_have_applied' => false, 'safe_to_resubmit' => false, 'error_code' => '', 'result' => [ 'hostname' => 'active-zero-conflict.example', 'settings_changed' => true, ], ]); $activeZeroConflict = CloudJob::findOrFail($activeZeroConflictId); $activeZeroEvent = Capsule::table( 'cloud_agent_completion_events' )->where('event_id', $activeZeroEventId)->first(); $assert($activeZeroAccepted === [ 'accepted' => true, 'disposition' => 'received', ] && (string) $activeZeroConflict->phase === 'terminal' && (string) $activeZeroConflict->outcome === 'failed' && (string) $activeZeroConflict->status === 'failed' && (string) $activeZeroConflict->effect_resolution === 'unknown' && (bool) $activeZeroConflict->finalization_incident && (string) $activeZeroConflict->finalization_error_code === 'recovery_dispatch_repair_instance_state_conflict' && (string) $activeZeroEvent->disposition === 'received' && !CloudJob::where('id', $activeZeroConflictId) ->where('phase', 'accepted') ->where('dispatch_attempts', 0)->exists() && count($client->requests) === $beforeZeroCounterRequests, 'An active zero-counter task retained a replayable mutation after Agent completion evidence arrived.'); // Defence in depth for a pre-upgrade/crash snapshot where only the event // became durable: the ordinary worker query itself excludes any Agent job // with same-operation completion evidence. Capsule::table('cloud_jobs')->where('id', $activeZeroConflictId)->update([ 'phase' => 'accepted', 'outcome' => null, 'status' => 'queued', 'attempts' => 0, 'dispatch_attempts' => 0, 'dispatch_deadline_at' => time() + 300, 'dispatch_started_at' => null, 'agent_accepted_at' => null, 'receipt_deadline_at' => null, 'terminal_at' => null, 'effect_may_have_applied' => null, 'safe_to_resubmit' => null, 'effect_resolution' => 'not_applicable', 'dispatched_at' => null, 'error_code' => null, 'finalization_incident' => 0, 'finalization_error_code' => null, 'available_at' => 0, 'lease_token' => null, 'lease_until' => 0, ]); $beforeCrashWindowRequests = count($client->requests); (new CloudWorkerService())->tick(); $crashWindowJob = CloudJob::findOrFail($activeZeroConflictId); $assert((string) $crashWindowJob->phase === 'accepted' && (int) $crashWindowJob->dispatch_attempts === 0 && count($client->requests) === $beforeCrashWindowRequests, 'The ordinary worker replayed a mutation after durable same-operation completion evidence.'); // A repaired ledger with a corrupt bound receipt stays visible but obeys // the same retry gate on both ordinary and receipt-priority passes. Without // the lease clock this exact event would be reclaimed on every tick. [$repairBackoffId, $repairBackoffUuid] = $insertSettingsJob( str_repeat('ce', 16), ['hostname' => 'dispatch-repair-backoff.example'] ); $repairBackoffEventId = 'dispatch-repair-invalid-event'; Capsule::table('cloud_jobs')->where('id', $repairBackoffId)->update([ 'dispatch_attempts' => 2, 'outcome' => 'succeeded', 'status' => 'succeeded', 'error_code' => null, 'effect_may_have_applied' => 0, 'safe_to_resubmit' => 0, 'effect_resolution' => 'applied', 'completion_event_id' => $repairBackoffEventId, 'available_at' => 0, 'lease_token' => null, 'lease_until' => 0, ]); $repairBackoffJob = CloudJob::findOrFail($repairBackoffId); $repairBackoffIdentity = CloudOperationRecoveryIdentity::expected( $repairBackoffJob, true ); Capsule::table('cloud_agent_completion_events')->insert([ 'event_id' => $repairBackoffEventId, 'host_id' => 7, 'operation_id' => (string) $repairBackoffJob->operation_id, // Valid shape but deliberately not the hash of the immutable receipt. 'payload_sha256' => str_repeat('0', 64), 'outcome' => 'succeeded', 'effect_may_have_applied' => 0, 'safe_to_resubmit' => 0, 'error_code' => '', 'result_json' => json_encode([ 'schema' => 'virt_cloud_completion_event', 'version' => 1, 'identity' => $repairBackoffIdentity, 'result' => [ 'uuid' => $repairBackoffUuid, 'hostname' => 'dispatch-repair-backoff.example', 'settings_changed' => true, ], ], JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE), 'disposition' => 'received', 'received_at' => time(), 'applied_at' => null, ]); CloudOperationCompletionService::deferDispatchLedgerConflict( $repairBackoffJob ); Capsule::table('cloud_jobs')->where('id', $repairBackoffId)->update([ 'dispatch_attempts' => 1, 'available_at' => 0, 'lease_token' => null, 'lease_until' => 0, ]); $beforeRepairBackoffRequests = count($client->requests); $repairBackoffService = new CloudAutomaticOperationRecoveryService(); $assert($repairBackoffService->tick(), 'A repaired ledger with an invalid bound receipt was not checked.'); $repairBackoffJob = CloudJob::findOrFail($repairBackoffId); $repairBackoffUpdatedAt = (int) $repairBackoffJob->updated_at; $assert((bool) $repairBackoffJob->finalization_incident && (string) $repairBackoffJob->finalization_error_code === 'recovery_dispatch_repair_receipt_invalid' && (int) $repairBackoffJob->lease_until >= time() + 290 && (int) $repairBackoffJob->available_at >= time() + 290 && count($client->requests) === $beforeRepairBackoffRequests, 'An invalid repaired-ledger receipt did not enter bounded backoff.'); Capsule::table('cloud_jobs')->where('id', '<>', $repairBackoffId) ->update([ 'lease_until' => time() + 3600, 'available_at' => time() + 3600, ]); $assert(!$repairBackoffService->tick() && (int) CloudJob::findOrFail($repairBackoffId)->updated_at === $repairBackoffUpdatedAt && count($client->requests) === $beforeRepairBackoffRequests, 'Receipt-priority recovery ignored the repaired-ledger retry gate.'); // Contradictory terminal/unknown safety tuples are PHP-only incidents. // They must remain scheduled forever without inspection, fence writes, or // any replay of the original mutation. Cover both legal dispatch counters // and each independent part of the canonical terminal tuple. $stateCases = [ [str_repeat('f1', 16), [ 'effect_may_have_applied' => 1, 'safe_to_resubmit' => 1, ]], [str_repeat('f2', 16), [ 'effect_may_have_applied' => 0, 'safe_to_resubmit' => 0, ]], [str_repeat('f3', 16), [ 'outcome' => 'succeeded', 'status' => 'succeeded', ]], [str_repeat('f4', 16), [ 'dispatch_attempts' => 0, 'dispatched_at' => null, 'dispatch_started_at' => null, 'agent_accepted_at' => null, 'receipt_deadline_at' => null, 'effect_may_have_applied' => 0, 'safe_to_resubmit' => 0, ]], // The safety booleans look canonical for an unknown dispatched // operation, but generation zero proves no Agent mutation crossed the // boundary. Without an undispatched-finalizer incident this is a // contradictory historical row, not an Agent recovery candidate. [str_repeat('f8', 16), [ 'dispatch_attempts' => 0, 'dispatched_at' => null, 'dispatch_started_at' => null, 'agent_accepted_at' => null, 'receipt_deadline_at' => null, 'effect_may_have_applied' => 1, 'safe_to_resubmit' => 0, ]], ]; foreach ($stateCases as [$operationId, $overrides]) { [$stateId] = $insertSettingsJob( $operationId, ['hostname' => 'damaged-state.example'] ); Capsule::table('cloud_jobs')->where('id', $stateId)->update( $overrides + [ 'finalization_incident' => 0, 'finalization_error_code' => null, 'available_at' => 0, 'lease_token' => null, 'lease_until' => 0, ] ); $beforeStateRequests = count($client->requests); $stateService = new CloudAutomaticOperationRecoveryService(); $assert($stateService->tick(), 'A damaged leaf terminal tuple left the recovery queue.'); $stateJob = CloudJob::findOrFail($stateId); $assert((bool) $stateJob->finalization_incident && (string) $stateJob->finalization_error_code === 'recovery_instance_state_conflict' && (int) $stateJob->available_at >= time() + 290 && (int) $stateJob->lease_until >= time() + 290 && count($client->requests) === $beforeStateRequests, 'A damaged leaf terminal tuple contacted Agent or lost backoff.'); Capsule::table('cloud_jobs')->where('id', $stateId)->update([ 'available_at' => time() + 3600, 'lease_until' => time() + 3600, ]); } // A completed Agent effect can still carry a sticky PHP projection // incident. Both resolved outcomes remain in the local retry queue, retain // their concrete code, and must never trigger another Agent request. $resolvedIncidentCases = [ [ str_repeat('f9', 16), 'applied', 'succeeded', 'recovery_payload_unavailable', ], [ str_repeat('fa', 16), 'not_applied', 'failed', 'recovery_operation_identity_conflict', ], ]; foreach ($resolvedIncidentCases as [ $operationId, $resolution, $outcome, $incidentCode, ]) { [$resolvedIncidentId] = $insertSettingsJob( $operationId, ['hostname' => 'resolved-local-incident.example'] ); Capsule::table('cloud_jobs') ->where('id', $resolvedIncidentId)->update([ 'outcome' => $outcome, 'status' => $outcome, 'dispatch_attempts' => 1, 'effect_may_have_applied' => 0, 'safe_to_resubmit' => 0, 'effect_resolution' => $resolution, 'effect_resolved_at' => time() - 1, 'finalization_incident' => 1, 'finalization_error_code' => $incidentCode, 'available_at' => 0, 'lease_token' => null, 'lease_until' => 0, ]); $beforeResolvedIncidentRequests = count($client->requests); $assert((new CloudAutomaticOperationRecoveryService())->tick(), 'A resolved sticky incident left the local recovery queue.'); $resolvedIncident = CloudJob::findOrFail($resolvedIncidentId); $assert((bool) $resolvedIncident->finalization_incident && (string) $resolvedIncident->finalization_error_code === $incidentCode && (string) $resolvedIncident->effect_resolution === $resolution && (int) $resolvedIncident->available_at >= time() + 290 && (int) $resolvedIncident->lease_until >= time() + 290 && count($client->requests) === $beforeResolvedIncidentRequests, 'A resolved sticky incident contacted Agent, lost its code, ' . 'or failed to enter local backoff.'); } // The same rule applies to a replica child below an already-resolved // aggregate. Child ancestry must not turn a local finalization incident // back into an Agent recovery or remove its concrete cause. [$resolvedIncidentParentId] = $insertSettingsJob( str_repeat('fb', 16), ['hostname' => 'resolved-incident-parent.example'] ); Capsule::table('cloud_jobs') ->where('id', $resolvedIncidentParentId)->update([ 'instance_id' => null, 'host_id' => null, 'kind' => 'template_import', 'execution_mode' => 'aggregate', 'outcome' => 'succeeded', 'status' => 'succeeded', 'attempts' => 0, 'dispatch_attempts' => 0, 'dispatch_started_at' => null, 'agent_accepted_at' => null, 'dispatched_at' => null, 'completion_event_id' => null, 'effect_may_have_applied' => 0, 'safe_to_resubmit' => 0, 'effect_resolution' => 'applied', 'effect_resolved_at' => time() - 1, 'finalization_incident' => 0, 'finalization_error_code' => null, 'available_at' => time() + 3600, 'lease_token' => null, 'lease_until' => time() + 3600, ]); [$resolvedIncidentChildId] = $insertSettingsJob( str_repeat('fc', 16), ['hostname' => 'resolved-incident-child.example'] ); Capsule::table('cloud_jobs') ->where('id', $resolvedIncidentChildId)->update([ 'parent_job_id' => $resolvedIncidentParentId, 'kind' => 'template_import_replica', 'outcome' => 'failed', 'status' => 'failed', 'dispatch_attempts' => 1, 'effect_may_have_applied' => 0, 'safe_to_resubmit' => 0, 'effect_resolution' => 'not_applied', 'effect_resolved_at' => time() - 1, 'finalization_incident' => 1, 'finalization_error_code' => 'recovery_payload_unavailable', 'available_at' => 0, 'lease_token' => null, 'lease_until' => 0, ]); $beforeResolvedIncidentChildRequests = count($client->requests); $assert((new CloudAutomaticOperationRecoveryService())->tick(), 'A resolved sticky aggregate child left the local recovery queue.'); $resolvedIncidentChild = CloudJob::findOrFail( $resolvedIncidentChildId ); $assert((bool) $resolvedIncidentChild->finalization_incident && (string) $resolvedIncidentChild->finalization_error_code === 'recovery_payload_unavailable' && (string) $resolvedIncidentChild->effect_resolution === 'not_applied' && (int) $resolvedIncidentChild->available_at >= time() + 290 && (int) $resolvedIncidentChild->lease_until >= time() + 290 && count($client->requests) === $beforeResolvedIncidentChildRequests, 'A resolved sticky aggregate child contacted Agent, lost its code, ' . 'or failed to enter local backoff.'); [$aggregateStateId] = $insertSettingsJob( str_repeat('f5', 16), ['hostname' => 'aggregate-state.example'] ); Capsule::table('cloud_jobs')->where('id', $aggregateStateId)->update([ 'instance_id' => null, 'host_id' => null, 'kind' => 'template_import', 'execution_mode' => 'aggregate', 'dispatch_attempts' => 0, 'dispatched_at' => null, 'dispatch_started_at' => null, 'agent_accepted_at' => null, 'receipt_deadline_at' => null, 'safe_to_resubmit' => 1, 'finalization_incident' => 0, 'finalization_error_code' => null, 'available_at' => 0, 'lease_token' => null, 'lease_until' => 0, ]); $beforeAggregateStateRequests = count($client->requests); $aggregateStateService = new CloudAutomaticOperationRecoveryService(); $assert($aggregateStateService->tick(), 'A damaged aggregate-parent tuple left the recovery queue.'); $aggregateStateJob = CloudJob::findOrFail($aggregateStateId); $assert((bool) $aggregateStateJob->finalization_incident && (string) $aggregateStateJob->finalization_error_code === 'recovery_aggregate_state_conflict' && (int) $aggregateStateJob->available_at >= time() + 290 && (int) $aggregateStateJob->lease_until >= time() + 290 && count($client->requests) === $beforeAggregateStateRequests, 'A damaged aggregate-parent tuple contacted Agent or lost backoff.'); Capsule::table('cloud_jobs')->where('id', $aggregateStateId)->update([ 'available_at' => time() + 3600, 'lease_until' => time() + 3600, ]); // A resolved aggregate parent must still surface and retain an anomalous // Agent child; only the child enters the PHP-only incident loop. [$resolvedParentId] = $insertSettingsJob( str_repeat('f6', 16), ['hostname' => 'resolved-parent.example'] ); Capsule::table('cloud_jobs')->where('id', $resolvedParentId)->update([ 'instance_id' => null, 'host_id' => null, 'kind' => 'template_import', 'execution_mode' => 'aggregate', 'dispatch_attempts' => 0, 'dispatched_at' => null, 'outcome' => 'succeeded', 'status' => 'succeeded', 'effect_may_have_applied' => 0, 'safe_to_resubmit' => 0, 'effect_resolution' => 'applied', 'available_at' => time() + 3600, 'lease_until' => time() + 3600, ]); [$stateChildId] = $insertSettingsJob( str_repeat('f7', 16), ['hostname' => 'damaged-child.example'] ); Capsule::table('cloud_jobs')->where('id', $stateChildId)->update([ 'parent_job_id' => $resolvedParentId, 'kind' => 'template_import_replica', 'safe_to_resubmit' => 1, 'finalization_incident' => 0, 'finalization_error_code' => null, 'available_at' => 0, 'lease_token' => null, 'lease_until' => 0, ]); $beforeStateChildRequests = count($client->requests); $stateChildService = new CloudAutomaticOperationRecoveryService(); $assert($stateChildService->tick(), 'An anomalous aggregate child left the recovery queue.'); $stateChild = CloudJob::findOrFail($stateChildId); $assert((bool) $stateChild->finalization_incident && (string) $stateChild->finalization_error_code === 'recovery_instance_state_conflict' && count($client->requests) === $beforeStateChildRequests, 'An anomalous aggregate child contacted Agent or lost its incident.'); foreach ($client->requests as $request) { $allowed = preg_match( '#^/v1/operations/[a-f0-9]{32}/(?:inspect|fence-resolution|resolve-fence)$#D', $request['target'] ) === 1; $assert($allowed && ($request['method'] === 'GET' || ($request['method'] === 'POST' && str_ends_with( $request['target'], '/resolve-fence' ))), 'Automatic recovery called an Agent mutation endpoint.'); } echo "Cloud automatic operation recovery tests passed.\n"; }