From dfd752696cf5765f262f604d2185b1e14fb3fe73 Mon Sep 17 00:00:00 2001 From: codex Date: Wed, 21 Oct 2026 22:40:12 +0800 Subject: [PATCH] fix(cloud): converge catalog deletion recovery --- ...CloudAutomaticOperationRecoveryService.php | 14 +- .../Cloud/CloudOperationCompletionService.php | 5 + .../Cloud/CloudOperationRecoveryService.php | 90 ++++++++- web/app/Services/Cloud/CloudWorkerService.php | 38 +++- .../CloudAggregateRecoveryConvergenceTest.php | 172 ++++++++++++++++++ ...dAutomaticOperationRecoveryServiceTest.php | 123 +++++++++++++ .../CloudOperationCompletionContractTest.php | 51 ++++++ .../CloudRescueTemplateRetryBoundaryTest.php | 63 +++++++ web/tests/CloudTaskDeadlineFallbackTest.php | 13 +- 9 files changed, 553 insertions(+), 16 deletions(-) diff --git a/web/app/Services/Cloud/CloudAutomaticOperationRecoveryService.php b/web/app/Services/Cloud/CloudAutomaticOperationRecoveryService.php index cfae4f59..8bb8627d 100644 --- a/web/app/Services/Cloud/CloudAutomaticOperationRecoveryService.php +++ b/web/app/Services/Cloud/CloudAutomaticOperationRecoveryService.php @@ -1191,6 +1191,7 @@ final class CloudAutomaticOperationRecoveryService 'status' => 'failed', 'effect_may_have_applied' => false, 'safe_to_resubmit' => false, + 'retryable' => false, 'error_code' => $errorCode, 'result' => [], ]; @@ -1225,6 +1226,7 @@ final class CloudAutomaticOperationRecoveryService 'status' => 'succeeded', 'effect_may_have_applied' => false, 'safe_to_resubmit' => false, + 'retryable' => false, 'error_code' => '', // CloudAdminOperationRecoveryService already applied the strict // per-kind projection schema. The existing worker finalizer now @@ -1356,14 +1358,10 @@ final class CloudAutomaticOperationRecoveryService 'result' => $result, ]; } - $operationHash = hash( - 'sha256', - json_encode( - self::canonicalize($operation), - JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES - | JSON_UNESCAPED_UNICODE - ) - ); + $operationHash = CloudOperationRecoveryService:: + preparedOperationHash( + $operation + ); $validOperation = $resolution === 'applied' ? (($operation['status'] ?? null) === 'succeeded' && ($operation['effect_may_have_applied'] ?? null) === false diff --git a/web/app/Services/Cloud/CloudOperationCompletionService.php b/web/app/Services/Cloud/CloudOperationCompletionService.php index 181d25a7..179040d1 100644 --- a/web/app/Services/Cloud/CloudOperationCompletionService.php +++ b/web/app/Services/Cloud/CloudOperationCompletionService.php @@ -280,6 +280,11 @@ final class CloudOperationCompletionService 'status' => (string) $event->outcome, 'effect_may_have_applied' => (bool) $event->effect_may_have_applied, 'safe_to_resubmit' => (bool) $event->safe_to_resubmit, + // Completion delivery contains only terminal Journal records. + // Agent operation responses define retryable as false, but that + // display-only field is intentionally absent from the durable + // completion wire format, so restore its canonical value here. + 'retryable' => false, 'error_code' => (string) $event->error_code, 'action' => $envelope['identity']['action'], 'resource_id' => $envelope['identity']['resource_id'], diff --git a/web/app/Services/Cloud/CloudOperationRecoveryService.php b/web/app/Services/Cloud/CloudOperationRecoveryService.php index 5db981fd..ea12e863 100644 --- a/web/app/Services/Cloud/CloudOperationRecoveryService.php +++ b/web/app/Services/Cloud/CloudOperationRecoveryService.php @@ -118,11 +118,16 @@ final class CloudOperationRecoveryService $evidenceSource, $resolution ); + $prepared = self::preparedMarker($operation, $evidenceSource); $marker = self::transitionPreparedMarker( (string) $job->effect_resolution_reason, - self::preparedMarker($operation, $evidenceSource), + $prepared, $evidenceSource, - $resolution + $resolution, + self::legacyCompletionPreparedMarker( + $operation, + $evidenceSource + ) ); return (new CloudWorkerService())->prepareRecoveredEffect( $jobId, @@ -698,7 +703,11 @@ final class CloudOperationRecoveryService (string) $job->effect_resolution_reason, $prepared, $source, - $resolution + $resolution, + self::legacyCompletionPreparedMarker( + $operation, + $source + ) ); // The conflict disappeared before both rows were // locked. Leave the receipt retryable rather than @@ -1063,10 +1072,17 @@ final class CloudOperationRecoveryService string $source ): string { self::assertEvidenceSource($source); + return 'prepared:' . $source . ':' + . self::preparedOperationHash($operation); + } + + /** Hash the exact normalized operation bound into a prepared marker. */ + public static function preparedOperationHash(array $operation): string + { $canonical = self::canonicalize( self::recoveryCriticalOperation($operation) ); - return 'prepared:' . $source . ':' . hash( + return hash( 'sha256', json_encode( $canonical, @@ -1077,6 +1093,22 @@ final class CloudOperationRecoveryService ); } + /** Hash the sole legacy omission accepted from terminal completion wire. */ + public static function legacyCompletionOperationHash( + array $operation + ): ?string { + if (($operation['retryable'] ?? null) !== false + || !in_array( + $operation['status'] ?? null, + ['succeeded', 'failed'], + true + )) { + return null; + } + unset($operation['retryable']); + return self::preparedOperationHash($operation); + } + /** * Preserve the durable automatic-attempt envelope while replacing its * pending payload with the exact prepared evidence. An immutable late @@ -1087,11 +1119,16 @@ final class CloudOperationRecoveryService string $current, string $prepared, string $source, - string $resolution + string $resolution, + ?string $legacyCompatiblePrepared = null ): string { if ($current === '' || hash_equals($prepared, $current)) { return $prepared; } + if ($legacyCompatiblePrepared !== null + && hash_equals($legacyCompatiblePrepared, $current)) { + return $prepared; + } $state = self::automaticRecoveryState($current); if ($state === null) { self::throwPreparedEvidenceConflict($source); @@ -1099,11 +1136,29 @@ final class CloudOperationRecoveryService if (hash_equals($prepared, $state['payload'])) { return $current; } + if ($legacyCompatiblePrepared !== null + && hash_equals( + $legacyCompatiblePrepared, + $state['payload'] + )) { + return $prepared; + } if ($state['payload'] !== 'pending') { $existing = []; $late = []; + $legacy = []; $journalStatus = $resolution === 'applied' ? 'succeeded' : 'failed'; + $compatibleOperationHash = false; + if ($legacyCompatiblePrepared !== null + && preg_match( + '/^prepared:late_completion_receipt:' + . '[A-Za-z0-9_.-]{1,160}:([a-f0-9]{64})$/D', + $legacyCompatiblePrepared, + $legacy + ) === 1) { + $compatibleOperationHash = $legacy[1]; + } if (str_starts_with($source, 'late_completion_receipt:') && preg_match( '/^prepared:automatic_' @@ -1118,7 +1173,12 @@ final class CloudOperationRecoveryService $prepared, $late ) === 1 - && hash_equals($existing[1], $late[1])) { + && (hash_equals($existing[1], $late[1]) + || (is_string($compatibleOperationHash) + && hash_equals( + $existing[1], + $compatibleOperationHash + )))) { return $prepared; } self::throwPreparedEvidenceConflict($source); @@ -1209,6 +1269,24 @@ final class CloudOperationRecoveryService return $operation; } + /** + * Completion receipts before retryable was restored at the PHP adapter + * produced prepared hashes without that canonical terminal field. Accept + * only that one historical omission while upgrading the durable marker to + * the fully bound operation; every other field remains hash-identical. + */ + private static function legacyCompletionPreparedMarker( + array $operation, + string $source + ): ?string { + if (!str_starts_with($source, 'late_completion_receipt:')) { + return null; + } + $legacyHash = self::legacyCompletionOperationHash($operation); + return $legacyHash === null + ? null : 'prepared:' . $source . ':' . $legacyHash; + } + private static function canonicalize($value) { if (!is_array($value)) { diff --git a/web/app/Services/Cloud/CloudWorkerService.php b/web/app/Services/Cloud/CloudWorkerService.php index f7f12156..c2c20bf6 100644 --- a/web/app/Services/Cloud/CloudWorkerService.php +++ b/web/app/Services/Cloud/CloudWorkerService.php @@ -1486,6 +1486,35 @@ final class CloudWorkerService if ($current === '' || hash_equals($prepared, $current)) { return true; } + $legacyOperationHash = CloudOperationRecoveryService:: + legacyCompletionOperationHash($operation); + $currentLate = []; + $preparedLate = []; + if ($legacyOperationHash !== null + && preg_match( + '/^(?:automatic_recovery:v1:[1-9][0-9]{0,10}:' + . '[1-9][0-9]{0,9}:)?' + . 'prepared:late_completion_receipt:' + . '([A-Za-z0-9_.-]{1,160}):([a-f0-9]{64})$/D', + $current, + $currentLate + ) === 1 + && preg_match( + '/^prepared:late_completion_receipt:' + . '([A-Za-z0-9_.-]{1,160}):([a-f0-9]{64})$/D', + $prepared, + $preparedLate + ) === 1 + && hash_equals($currentLate[1], $preparedLate[1]) + && hash_equals($currentLate[2], $legacyOperationHash) + && hash_equals( + $preparedLate[2], + CloudOperationRecoveryService::preparedOperationHash( + $operation + ) + )) { + return true; + } if (preg_match( '/^automatic_recovery:v1:[1-9][0-9]{0,10}:[1-9][0-9]{0,9}:pending$/D', $current @@ -1522,8 +1551,15 @@ final class CloudWorkerService ) !== 1) { return false; } + $operationHashMatches = hash_equals($existing[4], $late[1]); + if (!$operationHashMatches && $legacyOperationHash !== null) { + $operationHashMatches = hash_equals( + $existing[4], + $legacyOperationHash + ); + } return $existing[3] === 'nofence' - && hash_equals($existing[4], $late[1]) + && $operationHashMatches && (($existing[1] === 'applied' && $existing[2] === 'succeeded' && ($operation['status'] ?? null) === 'succeeded') diff --git a/web/tests/CloudAggregateRecoveryConvergenceTest.php b/web/tests/CloudAggregateRecoveryConvergenceTest.php index c1658a8b..b2ced264 100644 --- a/web/tests/CloudAggregateRecoveryConvergenceTest.php +++ b/web/tests/CloudAggregateRecoveryConvergenceTest.php @@ -48,6 +48,7 @@ namespace { use App\Services\Cloud\CloudAutomaticOperationRecoveryService; use App\Services\Cloud\CloudError; use App\Services\Cloud\CloudJobService; + use App\Services\Cloud\CloudOperationCompletionService; use App\Services\Cloud\CloudOperationRecoveryIdentity; use App\Services\Cloud\CloudRecoveryAggregateStateConflict; use App\Services\Cloud\CloudRecoveryInstanceStateConflict; @@ -3611,5 +3612,176 @@ namespace { && (int) $healthyAfterAudit->available_at >= time() + 290, 'A healthy published aggregate audit hot-looped or created an incident.'); + // A success receipt arriving after the immutable deadline failure must + // converge the original delete generation in place. It may update the + // effect/catalog projection, but it must not flip either failed audit to + // succeeded or contact the destructive Agent endpoint again. + $lateDeleteParentId = $insertParent('template_delete'); + $lateDeleteChildId = $insertResolvedChild( + $lateDeleteParentId, + 'not_applied', + 'template_delete_replica' + ); + $lateDeleteTemplateId = $insertProjectionForParent( + $lateDeleteParentId + ); + $lateDeleteChild = CloudJob::findOrFail($lateDeleteChildId); + Capsule::table('cloud_jobs')->where('id', $lateDeleteChildId)->update([ + 'sealed_payload' => CloudSecrets::encrypt([ + 'method' => 'DELETE', + 'request_target' => '/v1/templates/' + . rawurlencode($lateDeleteTemplateId), + 'body' => [ + 'operation_id' => (string) $lateDeleteChild->operation_id, + ], + 'owner_id' => 1, + 'can_manage' => true, + 'template_id' => $lateDeleteTemplateId, + ], 'job-payload'), + 'effect_may_have_applied' => 1, + 'safe_to_resubmit' => 0, + 'effect_resolution' => 'unknown', + 'effect_resolved_at' => null, + 'effect_resolution_reason' => '', + 'completion_event_id' => null, + 'finalization_incident' => 1, + 'finalization_error_code' => 'recovery_receipt_invalid', + 'available_at' => 0, + 'lease_token' => null, + 'lease_until' => 0, + ]); + $lateDeleteChild = CloudJob::findOrFail($lateDeleteChildId); + $lateDeleteIdentity = CloudOperationRecoveryIdentity::expected( + $lateDeleteChild, + true + ); + $beforeLateDeleteRequests = CloudAgentSettingsService::$requests; + $lateDeleteAccepted = CloudOperationCompletionService::accept([ + 'type' => 'completion', + 'id' => 'aggregate-late-template-delete', + 'agent_id' => (string) $lateDeleteChild->host_id, + 'operation_id' => (string) $lateDeleteChild->operation_id, + 'action' => $lateDeleteIdentity['action'], + 'resource_id' => $lateDeleteIdentity['resource_id'], + 'request_fingerprint' => + $lateDeleteIdentity['request_fingerprint'], + 'status' => 'succeeded', + 'effect_may_have_applied' => false, + 'safe_to_resubmit' => false, + 'error_code' => '', + 'result' => [ + 'resource_type' => 'template', + 'id' => $lateDeleteTemplateId, + 'deleted' => true, + 'managed_image_absent' => true, + 'managed_metadata_absent' => true, + 'managed_artifacts_absent' => true, + ], + ]); + $lateDeleteChild = CloudJob::findOrFail($lateDeleteChildId); + $lateDeleteParent = CloudJob::findOrFail($lateDeleteParentId); + $assert($lateDeleteAccepted === [ + 'accepted' => true, + 'disposition' => 'applied', + ] + && (string) $lateDeleteChild->outcome === 'failed' + && (string) $lateDeleteChild->effect_resolution === 'applied' + && !(bool) $lateDeleteChild->finalization_incident + && (string) $lateDeleteParent->outcome === 'failed' + && (string) $lateDeleteParent->error_code + === 'aggregate_deadline_exceeded' + && (string) $lateDeleteParent->effect_resolution === 'applied' + && !(bool) $lateDeleteParent->finalization_incident + && (string) Capsule::table('cloud_templates') + ->where('id', $lateDeleteTemplateId)->value('state') === 'retired' + && !Capsule::table('cloud_template_replicas') + ->where('template_id', $lateDeleteTemplateId)->exists() + && CloudAgentSettingsService::$requests === $beforeLateDeleteRequests, + 'A late template deletion receipt did not converge the original ' + . 'operation without replaying or rewriting its failure audit.'); + + $lateIsoParentId = $insertParent('user_iso_delete'); + $lateIsoChildId = $insertResolvedChild( + $lateIsoParentId, + 'not_applied', + 'user_iso_delete_replica' + ); + $lateIsoId = $insertProjectionForParent($lateIsoParentId); + $lateIsoChild = CloudJob::findOrFail($lateIsoChildId); + Capsule::table('cloud_jobs')->where('id', $lateIsoChildId)->update([ + 'sealed_payload' => CloudSecrets::encrypt([ + 'method' => 'DELETE', + 'request_target' => '/v1/user-isos/' + . rawurlencode($lateIsoId) . '?owner_id=1', + 'body' => [ + 'operation_id' => (string) $lateIsoChild->operation_id, + ], + 'owner_id' => 1, + 'can_manage' => false, + 'user_iso_id' => $lateIsoId, + ], 'job-payload'), + 'effect_may_have_applied' => 1, + 'safe_to_resubmit' => 0, + 'effect_resolution' => 'unknown', + 'effect_resolved_at' => null, + 'effect_resolution_reason' => '', + 'completion_event_id' => null, + 'finalization_incident' => 1, + 'finalization_error_code' => 'recovery_receipt_invalid', + 'available_at' => 0, + 'lease_token' => null, + 'lease_until' => 0, + ]); + $lateIsoChild = CloudJob::findOrFail($lateIsoChildId); + $lateIsoIdentity = CloudOperationRecoveryIdentity::expected( + $lateIsoChild, + true + ); + $lateIsoAccepted = CloudOperationCompletionService::accept([ + 'type' => 'completion', + 'id' => 'aggregate-late-user-iso-delete', + 'agent_id' => (string) $lateIsoChild->host_id, + 'operation_id' => (string) $lateIsoChild->operation_id, + 'action' => $lateIsoIdentity['action'], + 'resource_id' => $lateIsoIdentity['resource_id'], + 'request_fingerprint' => $lateIsoIdentity['request_fingerprint'], + 'status' => 'succeeded', + 'effect_may_have_applied' => false, + 'safe_to_resubmit' => false, + 'error_code' => '', + 'result' => [ + 'resource_type' => 'user_iso', + 'id' => $lateIsoId, + 'owner_id' => 1, + 'deleted' => true, + 'managed_image_absent' => true, + 'managed_metadata_absent' => true, + 'managed_artifacts_absent' => true, + ], + ]); + $lateIsoChild = CloudJob::findOrFail($lateIsoChildId); + $lateIsoParent = CloudJob::findOrFail($lateIsoParentId); + $lateIso = Capsule::table('cloud_user_isos') + ->where('id', $lateIsoId)->first(); + $assert($lateIsoAccepted === [ + 'accepted' => true, + 'disposition' => 'applied', + ] + && (string) $lateIsoChild->outcome === 'failed' + && (string) $lateIsoChild->effect_resolution === 'applied' + && !(bool) $lateIsoChild->finalization_incident + && (string) $lateIsoParent->outcome === 'failed' + && (string) $lateIsoParent->effect_resolution === 'applied' + && !(bool) $lateIsoParent->finalization_incident + && $lateIso !== null + && (string) $lateIso->state === 'retired' + && (string) $lateIso->name === '' + && (int) $lateIso->size_bytes === 0 + && !Capsule::table('cloud_user_iso_replicas') + ->where('user_iso_id', $lateIsoId)->exists() + && CloudAgentSettingsService::$requests === $beforeLateDeleteRequests, + 'A late user-ISO deletion receipt did not converge its owner-bound ' + . 'catalog ledgers without replay.'); + echo "Cloud aggregate recovery convergence tests passed.\n"; } diff --git a/web/tests/CloudAutomaticOperationRecoveryServiceTest.php b/web/tests/CloudAutomaticOperationRecoveryServiceTest.php index 62cc4577..d57a08b3 100644 --- a/web/tests/CloudAutomaticOperationRecoveryServiceTest.php +++ b/web/tests/CloudAutomaticOperationRecoveryServiceTest.php @@ -219,6 +219,11 @@ namespace { $table->string('id')->primary(); $table->integer('active_job_id')->nullable(); $table->string('state')->default('active'); + $table->integer('revision')->default(1); + $table->string('generation_token')->default(''); + $table->integer('image_bytes')->default(0); + $table->string('sha256')->default(''); + $table->integer('updated_at')->default(0); }); $schema->create('cloud_user_isos', static function (Blueprint $table): void { @@ -833,6 +838,124 @@ namespace { && count($client->requests) === $beforeNonSettingsRequests + 1, 'Inconclusive non-settings evidence did not stay recoverable.'); + // A successful read-only inspection must adapt back to the same terminal + // operation contract as an immediate Agent response. This is the public + // recovery seam for a template deletion whose original success receipt + // reached PHP but failed local catalog finalization. + $templateDeleteId = 'automatic-delete-template'; + [$templateDeleteParentId] = $insertSettingsJob( + md5('automatic-template-delete-parent'), + ['hostname' => 'unused-parent.example'] + ); + Capsule::table('cloud_jobs')->where( + 'id', + $templateDeleteParentId + )->update([ + 'parent_job_id' => null, + 'instance_id' => null, + 'host_id' => null, + 'kind' => 'template_delete', + 'execution_mode' => 'aggregate', + 'attempts' => 0, + 'dispatch_attempts' => 0, + 'dispatch_started_at' => null, + 'agent_accepted_at' => null, + 'dispatched_at' => null, + 'effect_resolution' => 'unknown', + 'finalization_incident' => 1, + 'finalization_error_code' => 'recovery_child_convergence', + 'available_at' => 0, + 'lease_token' => null, + 'lease_until' => 0, + ]); + [$templateDeleteChildId] = $insertSettingsJob( + md5('automatic-template-delete-child'), + ['hostname' => 'unused-child.example'] + ); + $templateDeleteOperationId = (string) CloudJob::findOrFail( + $templateDeleteChildId + )->operation_id; + Capsule::table('cloud_jobs')->where( + 'id', + $templateDeleteChildId + )->update([ + 'parent_job_id' => $templateDeleteParentId, + 'instance_id' => null, + 'kind' => 'template_delete_replica', + 'sealed_payload' => CloudSecrets::encrypt([ + 'method' => 'DELETE', + 'request_target' => '/v1/templates/' . $templateDeleteId, + 'body' => ['operation_id' => $templateDeleteOperationId], + 'owner_id' => 1, + 'can_manage' => true, + 'template_id' => $templateDeleteId, + ], 'job-payload'), + 'finalization_incident' => 1, + 'finalization_error_code' => 'recovery_receipt_invalid', + 'available_at' => 0, + 'lease_token' => null, + 'lease_until' => 0, + ]); + Capsule::table('cloud_templates')->insert([ + 'id' => $templateDeleteId, + 'active_job_id' => $templateDeleteParentId, + 'state' => 'deleting', + 'revision' => 1, + 'generation_token' => str_repeat('d', 32), + 'image_bytes' => 4096, + 'sha256' => str_repeat('e', 64), + 'updated_at' => $now, + ]); + Capsule::table('cloud_template_replicas')->insert([ + 'host_id' => 7, + 'template_id' => $templateDeleteId, + 'revision' => 1, + 'generation_token' => str_repeat('d', 32), + 'state' => 'delete_failed', + 'image_bytes' => 4096, + 'sha256' => str_repeat('e', 64), + 'operation_id' => $templateDeleteOperationId, + 'job_id' => $templateDeleteChildId, + 'error_code' => 'completion_finalization_failed', + 'created_at' => $now - 120, + 'updated_at' => $now, + ]); + $client->responses[] = $inspection( + $templateDeleteOperationId, + 'succeeded', + 'applied', + [ + 'resource_type' => 'template', + 'id' => $templateDeleteId, + 'deleted' => true, + 'managed_image_absent' => true, + 'managed_metadata_absent' => true, + 'managed_artifacts_absent' => true, + ], + false + ); + $beforeTemplateDeleteRequests = count($client->requests); + $assert((new CloudAutomaticOperationRecoveryService())->tick(), + 'The recovery worker did not inspect the template deletion.'); + $templateDeleteChild = CloudJob::findOrFail($templateDeleteChildId); + $templateDeleteParent = CloudJob::findOrFail($templateDeleteParentId); + $assert((string) $templateDeleteChild->effect_resolution === 'applied' + && !(bool) $templateDeleteChild->finalization_incident + && $templateDeleteChild->finalization_error_code === null + && (string) $templateDeleteParent->effect_resolution === 'applied' + && !(bool) $templateDeleteParent->finalization_incident + && (string) Capsule::table('cloud_templates') + ->where('id', $templateDeleteId)->value('state') === 'retired' + && !Capsule::table('cloud_template_replicas') + ->where('template_id', $templateDeleteId)->exists() + && count($client->requests) === $beforeTemplateDeleteRequests + 1 + && str_ends_with( + $client->requests[$beforeTemplateDeleteRequests]['target'], + '/inspect' + ), + 'A successful template deletion inspection did not converge its ' + . 'existing operation and catalog ledger.'); + // A poisoned Agent completion outbox is never evidence for either effect // verdict. Even a syntactically complete not-applied projection remains // fenced and the reviewed cause is visible without exposing raw data. diff --git a/web/tests/CloudOperationCompletionContractTest.php b/web/tests/CloudOperationCompletionContractTest.php index 1b9f52ba..dafe305b 100644 --- a/web/tests/CloudOperationCompletionContractTest.php +++ b/web/tests/CloudOperationCompletionContractTest.php @@ -3,6 +3,7 @@ require dirname(__DIR__) . '/vendor/autoload.php'; use App\Services\Cloud\CloudError; +use App\Services\Cloud\CloudAgentCatalogDeletion; use App\Services\Cloud\CloudOperationCompletionService; use App\Models\CloudAgentCompletionEvent; use App\Models\CloudJob; @@ -75,6 +76,56 @@ $assert(($validFailure['status'] ?? null) === 'failed' && ($validFailure['safe_to_resubmit'] ?? null) === false, 'A conservative terminal failure receipt was rejected.'); +// Completion delivery intentionally omits the display-only retryable field. +// Reconstructing its terminal Agent operation must restore the canonical +// false value so every catalog deletion finalizer sees the same contract as +// an immediate Agent response. +foreach ([ + ['template', 'debian-13-contract', 0], + ['rescue_template', str_repeat('b', 32), 0], + ['user_iso', str_repeat('c', 32), 42], +] as [$resourceType, $resourceId, $ownerId]) { + $catalogResult = [ + 'resource_type' => $resourceType, + 'id' => $resourceId, + 'deleted' => true, + 'managed_image_absent' => true, + 'managed_metadata_absent' => true, + 'managed_artifacts_absent' => true, + ]; + if ($resourceType === 'user_iso') { + $catalogResult['owner_id'] = $ownerId; + } + $catalogEvent = new CloudAgentCompletionEvent(); + $catalogEvent->operation_id = str_repeat('9', 32); + $catalogEvent->outcome = 'succeeded'; + $catalogEvent->effect_may_have_applied = 0; + $catalogEvent->safe_to_resubmit = 0; + $catalogEvent->error_code = ''; + $catalogEvent->result_json = json_encode([ + 'schema' => 'virt_cloud_completion_event', + 'version' => 1, + 'identity' => [ + 'action' => 'delete-' . str_replace('_', '-', $resourceType), + 'resource_id' => $resourceType . ':' . $resourceId, + 'request_fingerprint' => str_repeat('8', 64), + ], + 'result' => $catalogResult, + ], JSON_THROW_ON_ERROR); + $catalogOperation = CloudOperationCompletionService::operation( + $catalogEvent + ); + $assert(($catalogOperation['retryable'] ?? null) === false + && CloudAgentCatalogDeletion::confirms( + $catalogOperation, + $resourceType, + $resourceId, + $ownerId + ), + 'A durable successful ' . $resourceType + . ' deletion receipt lost its terminal Agent contract.'); +} + // Crash window: finish() committed the terminal task and completion_event_id, // but PHP died before marking the durable receipt applied. The same immutable // receipt must repair that marker; it is not a late or rejected completion. diff --git a/web/tests/CloudRescueTemplateRetryBoundaryTest.php b/web/tests/CloudRescueTemplateRetryBoundaryTest.php index 5bb7d173..5aebc91e 100644 --- a/web/tests/CloudRescueTemplateRetryBoundaryTest.php +++ b/web/tests/CloudRescueTemplateRetryBoundaryTest.php @@ -29,9 +29,11 @@ namespace { require dirname(__DIR__) . '/vendor/autoload.php'; use App\Models\CloudJob; + use App\Models\CloudAgentCompletionEvent; use App\Models\CloudRescueTemplateDeletion; use App\Models\CloudRescueTemplateImport; use App\Services\Cloud\CloudError; + use App\Services\Cloud\CloudOperationCompletionService; use App\Services\Cloud\CloudRescueTemplateDeletionService; use App\Services\Cloud\CloudRescueTemplateImportService; use App\Services\Cloud\CloudSecrets; @@ -546,5 +548,66 @@ namespace { $assert(CloudJob::count() === $beforeIdentityConflict, 'A rescue import identity conflict persisted a new job.'); + // Rescue-template deletion consumes the same reconstructed completion + // operation contract as template and user-ISO deletion. Exercise its + // actual ledger finalizer so a future adapter change cannot strand this + // standalone catalog operation in a failed state. + $completionTemplateId = str_repeat('3', 32); + $insertImport( + 'completion-rescue', + $completionTemplateId, + 'active', + 'applied' + ); + $completionDelete = CloudRescueTemplateDeletionService::delete( + 1, + $completionTemplateId, + $actor + ); + $completionDeleteJob = CloudJob::findOrFail( + (int) $completionDelete['task_id'] + ); + $completionResult = [ + 'resource_type' => 'rescue_template', + 'id' => $completionTemplateId, + 'deleted' => true, + 'managed_image_absent' => true, + 'managed_metadata_absent' => true, + 'managed_artifacts_absent' => true, + ]; + $completionEvent = new CloudAgentCompletionEvent(); + $completionEvent->operation_id = + (string) $completionDeleteJob->operation_id; + $completionEvent->outcome = 'succeeded'; + $completionEvent->effect_may_have_applied = 0; + $completionEvent->safe_to_resubmit = 0; + $completionEvent->error_code = ''; + $completionEvent->result_json = json_encode([ + 'schema' => 'virt_cloud_completion_event', + 'version' => 1, + 'identity' => [ + 'action' => 'delete-rescue-template', + 'resource_id' => 'rescue-template:' . $completionTemplateId, + 'request_fingerprint' => str_repeat('4', 64), + ], + 'result' => $completionResult, + ], JSON_THROW_ON_ERROR); + CloudRescueTemplateDeletionService::finishJobLocked( + $completionDeleteJob, + 'succeeded', + $completionResult, + CloudOperationCompletionService::operation($completionEvent) + ); + $assert((string) CloudRescueTemplateDeletion::query() + ->where('host_id', 1) + ->where('template_id', $completionTemplateId) + ->value('state') === 'succeeded' + && (string) CloudRescueTemplateImport::query() + ->where('host_id', 1) + ->where('template_id', $completionTemplateId) + ->value('state') === 'retired', + 'A durable rescue-template deletion receipt did not finalize its ' + . 'catalog ledgers.'); + echo "Cloud rescue-template retry boundary checks passed.\n"; } diff --git a/web/tests/CloudTaskDeadlineFallbackTest.php b/web/tests/CloudTaskDeadlineFallbackTest.php index 00441b4e..9daab70d 100644 --- a/web/tests/CloudTaskDeadlineFallbackTest.php +++ b/web/tests/CloudTaskDeadlineFallbackTest.php @@ -1766,14 +1766,25 @@ namespace { 'late_completion_receipt:' . $lateCrashEventId ); $lateCrashPrepared = CloudJob::find($lateCrashJobId); + $legacyLateCrashPreparedMarker = (string) + $lateCrashPrepared->effect_resolution_reason; $assert(str_starts_with( - (string) $lateCrashPrepared->effect_resolution_reason, + $legacyLateCrashPreparedMarker, 'prepared:late_completion_receipt:' . $lateCrashEventId . ':' ) && !(bool) $lateCrashPrepared->finalization_incident && $lateCrashPrepared->finalization_error_code === null && (string) $lateCrashPrepared->effect_resolution === 'unknown', 'Exact late evidence did not replace exhausted matching prepare state.'); + // Older crash windows could retain the stable late-receipt marker inside + // the mutable automatic-attempt envelope. The exact same receipt must + // upgrade that legacy wrapper without weakening operation-hash matching. + Capsule::table('cloud_jobs')->where('id', $lateCrashJobId)->update([ + 'effect_resolution_reason' => 'automatic_recovery:v1:' + . (time() - 21600) . ':12:' + . $legacyLateCrashPreparedMarker, + ]); + $lateCrashPrepared = CloudJob::find($lateCrashJobId); $lateCrashAudit = [ 'phase' => (string) $lateCrashPrepared->phase, 'outcome' => (string) $lateCrashPrepared->outcome,