fix(cloud): converge catalog deletion recovery

go
codex 4 weeks ago
parent 1df2dfa4ef
commit dfd752696c

@ -1191,6 +1191,7 @@ final class CloudAutomaticOperationRecoveryService
'status' => 'failed',
'effect_may_have_applied' => false,
'safe_to_resubmit' => false,
'retryable' => false,
'error_code' => $errorCode,
'result' => [],
];
@ -1225,6 +1226,7 @@ final class CloudAutomaticOperationRecoveryService
'status' => 'succeeded',
'effect_may_have_applied' => false,
'safe_to_resubmit' => false,
'retryable' => false,
'error_code' => '',
// CloudAdminOperationRecoveryService already applied the strict
// per-kind projection schema. The existing worker finalizer now
@ -1356,14 +1358,10 @@ final class CloudAutomaticOperationRecoveryService
'result' => $result,
];
}
$operationHash = hash(
'sha256',
json_encode(
self::canonicalize($operation),
JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES
| JSON_UNESCAPED_UNICODE
)
);
$operationHash = CloudOperationRecoveryService::
preparedOperationHash(
$operation
);
$validOperation = $resolution === 'applied'
? (($operation['status'] ?? null) === 'succeeded'
&& ($operation['effect_may_have_applied'] ?? null) === false

@ -280,6 +280,11 @@ final class CloudOperationCompletionService
'status' => (string) $event->outcome,
'effect_may_have_applied' => (bool) $event->effect_may_have_applied,
'safe_to_resubmit' => (bool) $event->safe_to_resubmit,
// Completion delivery contains only terminal Journal records.
// Agent operation responses define retryable as false, but that
// display-only field is intentionally absent from the durable
// completion wire format, so restore its canonical value here.
'retryable' => false,
'error_code' => (string) $event->error_code,
'action' => $envelope['identity']['action'],
'resource_id' => $envelope['identity']['resource_id'],

@ -118,11 +118,16 @@ final class CloudOperationRecoveryService
$evidenceSource,
$resolution
);
$prepared = self::preparedMarker($operation, $evidenceSource);
$marker = self::transitionPreparedMarker(
(string) $job->effect_resolution_reason,
self::preparedMarker($operation, $evidenceSource),
$prepared,
$evidenceSource,
$resolution
$resolution,
self::legacyCompletionPreparedMarker(
$operation,
$evidenceSource
)
);
return (new CloudWorkerService())->prepareRecoveredEffect(
$jobId,
@ -698,7 +703,11 @@ final class CloudOperationRecoveryService
(string) $job->effect_resolution_reason,
$prepared,
$source,
$resolution
$resolution,
self::legacyCompletionPreparedMarker(
$operation,
$source
)
);
// The conflict disappeared before both rows were
// locked. Leave the receipt retryable rather than
@ -1063,10 +1072,17 @@ final class CloudOperationRecoveryService
string $source
): string {
self::assertEvidenceSource($source);
return 'prepared:' . $source . ':'
. self::preparedOperationHash($operation);
}
/** Hash the exact normalized operation bound into a prepared marker. */
public static function preparedOperationHash(array $operation): string
{
$canonical = self::canonicalize(
self::recoveryCriticalOperation($operation)
);
return 'prepared:' . $source . ':' . hash(
return hash(
'sha256',
json_encode(
$canonical,
@ -1077,6 +1093,22 @@ final class CloudOperationRecoveryService
);
}
/** Hash the sole legacy omission accepted from terminal completion wire. */
public static function legacyCompletionOperationHash(
array $operation
): ?string {
if (($operation['retryable'] ?? null) !== false
|| !in_array(
$operation['status'] ?? null,
['succeeded', 'failed'],
true
)) {
return null;
}
unset($operation['retryable']);
return self::preparedOperationHash($operation);
}
/**
* Preserve the durable automatic-attempt envelope while replacing its
* pending payload with the exact prepared evidence. An immutable late
@ -1087,11 +1119,16 @@ final class CloudOperationRecoveryService
string $current,
string $prepared,
string $source,
string $resolution
string $resolution,
?string $legacyCompatiblePrepared = null
): string {
if ($current === '' || hash_equals($prepared, $current)) {
return $prepared;
}
if ($legacyCompatiblePrepared !== null
&& hash_equals($legacyCompatiblePrepared, $current)) {
return $prepared;
}
$state = self::automaticRecoveryState($current);
if ($state === null) {
self::throwPreparedEvidenceConflict($source);
@ -1099,11 +1136,29 @@ final class CloudOperationRecoveryService
if (hash_equals($prepared, $state['payload'])) {
return $current;
}
if ($legacyCompatiblePrepared !== null
&& hash_equals(
$legacyCompatiblePrepared,
$state['payload']
)) {
return $prepared;
}
if ($state['payload'] !== 'pending') {
$existing = [];
$late = [];
$legacy = [];
$journalStatus = $resolution === 'applied'
? 'succeeded' : 'failed';
$compatibleOperationHash = false;
if ($legacyCompatiblePrepared !== null
&& preg_match(
'/^prepared:late_completion_receipt:'
. '[A-Za-z0-9_.-]{1,160}:([a-f0-9]{64})$/D',
$legacyCompatiblePrepared,
$legacy
) === 1) {
$compatibleOperationHash = $legacy[1];
}
if (str_starts_with($source, 'late_completion_receipt:')
&& preg_match(
'/^prepared:automatic_'
@ -1118,7 +1173,12 @@ final class CloudOperationRecoveryService
$prepared,
$late
) === 1
&& hash_equals($existing[1], $late[1])) {
&& (hash_equals($existing[1], $late[1])
|| (is_string($compatibleOperationHash)
&& hash_equals(
$existing[1],
$compatibleOperationHash
)))) {
return $prepared;
}
self::throwPreparedEvidenceConflict($source);
@ -1209,6 +1269,24 @@ final class CloudOperationRecoveryService
return $operation;
}
/**
* Completion receipts before retryable was restored at the PHP adapter
* produced prepared hashes without that canonical terminal field. Accept
* only that one historical omission while upgrading the durable marker to
* the fully bound operation; every other field remains hash-identical.
*/
private static function legacyCompletionPreparedMarker(
array $operation,
string $source
): ?string {
if (!str_starts_with($source, 'late_completion_receipt:')) {
return null;
}
$legacyHash = self::legacyCompletionOperationHash($operation);
return $legacyHash === null
? null : 'prepared:' . $source . ':' . $legacyHash;
}
private static function canonicalize($value)
{
if (!is_array($value)) {

@ -1486,6 +1486,35 @@ final class CloudWorkerService
if ($current === '' || hash_equals($prepared, $current)) {
return true;
}
$legacyOperationHash = CloudOperationRecoveryService::
legacyCompletionOperationHash($operation);
$currentLate = [];
$preparedLate = [];
if ($legacyOperationHash !== null
&& preg_match(
'/^(?:automatic_recovery:v1:[1-9][0-9]{0,10}:'
. '[1-9][0-9]{0,9}:)?'
. 'prepared:late_completion_receipt:'
. '([A-Za-z0-9_.-]{1,160}):([a-f0-9]{64})$/D',
$current,
$currentLate
) === 1
&& preg_match(
'/^prepared:late_completion_receipt:'
. '([A-Za-z0-9_.-]{1,160}):([a-f0-9]{64})$/D',
$prepared,
$preparedLate
) === 1
&& hash_equals($currentLate[1], $preparedLate[1])
&& hash_equals($currentLate[2], $legacyOperationHash)
&& hash_equals(
$preparedLate[2],
CloudOperationRecoveryService::preparedOperationHash(
$operation
)
)) {
return true;
}
if (preg_match(
'/^automatic_recovery:v1:[1-9][0-9]{0,10}:[1-9][0-9]{0,9}:pending$/D',
$current
@ -1522,8 +1551,15 @@ final class CloudWorkerService
) !== 1) {
return false;
}
$operationHashMatches = hash_equals($existing[4], $late[1]);
if (!$operationHashMatches && $legacyOperationHash !== null) {
$operationHashMatches = hash_equals(
$existing[4],
$legacyOperationHash
);
}
return $existing[3] === 'nofence'
&& hash_equals($existing[4], $late[1])
&& $operationHashMatches
&& (($existing[1] === 'applied'
&& $existing[2] === 'succeeded'
&& ($operation['status'] ?? null) === 'succeeded')

@ -48,6 +48,7 @@ namespace {
use App\Services\Cloud\CloudAutomaticOperationRecoveryService;
use App\Services\Cloud\CloudError;
use App\Services\Cloud\CloudJobService;
use App\Services\Cloud\CloudOperationCompletionService;
use App\Services\Cloud\CloudOperationRecoveryIdentity;
use App\Services\Cloud\CloudRecoveryAggregateStateConflict;
use App\Services\Cloud\CloudRecoveryInstanceStateConflict;
@ -3611,5 +3612,176 @@ namespace {
&& (int) $healthyAfterAudit->available_at >= time() + 290,
'A healthy published aggregate audit hot-looped or created an incident.');
// A success receipt arriving after the immutable deadline failure must
// converge the original delete generation in place. It may update the
// effect/catalog projection, but it must not flip either failed audit to
// succeeded or contact the destructive Agent endpoint again.
$lateDeleteParentId = $insertParent('template_delete');
$lateDeleteChildId = $insertResolvedChild(
$lateDeleteParentId,
'not_applied',
'template_delete_replica'
);
$lateDeleteTemplateId = $insertProjectionForParent(
$lateDeleteParentId
);
$lateDeleteChild = CloudJob::findOrFail($lateDeleteChildId);
Capsule::table('cloud_jobs')->where('id', $lateDeleteChildId)->update([
'sealed_payload' => CloudSecrets::encrypt([
'method' => 'DELETE',
'request_target' => '/v1/templates/'
. rawurlencode($lateDeleteTemplateId),
'body' => [
'operation_id' => (string) $lateDeleteChild->operation_id,
],
'owner_id' => 1,
'can_manage' => true,
'template_id' => $lateDeleteTemplateId,
], 'job-payload'),
'effect_may_have_applied' => 1,
'safe_to_resubmit' => 0,
'effect_resolution' => 'unknown',
'effect_resolved_at' => null,
'effect_resolution_reason' => '',
'completion_event_id' => null,
'finalization_incident' => 1,
'finalization_error_code' => 'recovery_receipt_invalid',
'available_at' => 0,
'lease_token' => null,
'lease_until' => 0,
]);
$lateDeleteChild = CloudJob::findOrFail($lateDeleteChildId);
$lateDeleteIdentity = CloudOperationRecoveryIdentity::expected(
$lateDeleteChild,
true
);
$beforeLateDeleteRequests = CloudAgentSettingsService::$requests;
$lateDeleteAccepted = CloudOperationCompletionService::accept([
'type' => 'completion',
'id' => 'aggregate-late-template-delete',
'agent_id' => (string) $lateDeleteChild->host_id,
'operation_id' => (string) $lateDeleteChild->operation_id,
'action' => $lateDeleteIdentity['action'],
'resource_id' => $lateDeleteIdentity['resource_id'],
'request_fingerprint' =>
$lateDeleteIdentity['request_fingerprint'],
'status' => 'succeeded',
'effect_may_have_applied' => false,
'safe_to_resubmit' => false,
'error_code' => '',
'result' => [
'resource_type' => 'template',
'id' => $lateDeleteTemplateId,
'deleted' => true,
'managed_image_absent' => true,
'managed_metadata_absent' => true,
'managed_artifacts_absent' => true,
],
]);
$lateDeleteChild = CloudJob::findOrFail($lateDeleteChildId);
$lateDeleteParent = CloudJob::findOrFail($lateDeleteParentId);
$assert($lateDeleteAccepted === [
'accepted' => true,
'disposition' => 'applied',
]
&& (string) $lateDeleteChild->outcome === 'failed'
&& (string) $lateDeleteChild->effect_resolution === 'applied'
&& !(bool) $lateDeleteChild->finalization_incident
&& (string) $lateDeleteParent->outcome === 'failed'
&& (string) $lateDeleteParent->error_code
=== 'aggregate_deadline_exceeded'
&& (string) $lateDeleteParent->effect_resolution === 'applied'
&& !(bool) $lateDeleteParent->finalization_incident
&& (string) Capsule::table('cloud_templates')
->where('id', $lateDeleteTemplateId)->value('state') === 'retired'
&& !Capsule::table('cloud_template_replicas')
->where('template_id', $lateDeleteTemplateId)->exists()
&& CloudAgentSettingsService::$requests === $beforeLateDeleteRequests,
'A late template deletion receipt did not converge the original '
. 'operation without replaying or rewriting its failure audit.');
$lateIsoParentId = $insertParent('user_iso_delete');
$lateIsoChildId = $insertResolvedChild(
$lateIsoParentId,
'not_applied',
'user_iso_delete_replica'
);
$lateIsoId = $insertProjectionForParent($lateIsoParentId);
$lateIsoChild = CloudJob::findOrFail($lateIsoChildId);
Capsule::table('cloud_jobs')->where('id', $lateIsoChildId)->update([
'sealed_payload' => CloudSecrets::encrypt([
'method' => 'DELETE',
'request_target' => '/v1/user-isos/'
. rawurlencode($lateIsoId) . '?owner_id=1',
'body' => [
'operation_id' => (string) $lateIsoChild->operation_id,
],
'owner_id' => 1,
'can_manage' => false,
'user_iso_id' => $lateIsoId,
], 'job-payload'),
'effect_may_have_applied' => 1,
'safe_to_resubmit' => 0,
'effect_resolution' => 'unknown',
'effect_resolved_at' => null,
'effect_resolution_reason' => '',
'completion_event_id' => null,
'finalization_incident' => 1,
'finalization_error_code' => 'recovery_receipt_invalid',
'available_at' => 0,
'lease_token' => null,
'lease_until' => 0,
]);
$lateIsoChild = CloudJob::findOrFail($lateIsoChildId);
$lateIsoIdentity = CloudOperationRecoveryIdentity::expected(
$lateIsoChild,
true
);
$lateIsoAccepted = CloudOperationCompletionService::accept([
'type' => 'completion',
'id' => 'aggregate-late-user-iso-delete',
'agent_id' => (string) $lateIsoChild->host_id,
'operation_id' => (string) $lateIsoChild->operation_id,
'action' => $lateIsoIdentity['action'],
'resource_id' => $lateIsoIdentity['resource_id'],
'request_fingerprint' => $lateIsoIdentity['request_fingerprint'],
'status' => 'succeeded',
'effect_may_have_applied' => false,
'safe_to_resubmit' => false,
'error_code' => '',
'result' => [
'resource_type' => 'user_iso',
'id' => $lateIsoId,
'owner_id' => 1,
'deleted' => true,
'managed_image_absent' => true,
'managed_metadata_absent' => true,
'managed_artifacts_absent' => true,
],
]);
$lateIsoChild = CloudJob::findOrFail($lateIsoChildId);
$lateIsoParent = CloudJob::findOrFail($lateIsoParentId);
$lateIso = Capsule::table('cloud_user_isos')
->where('id', $lateIsoId)->first();
$assert($lateIsoAccepted === [
'accepted' => true,
'disposition' => 'applied',
]
&& (string) $lateIsoChild->outcome === 'failed'
&& (string) $lateIsoChild->effect_resolution === 'applied'
&& !(bool) $lateIsoChild->finalization_incident
&& (string) $lateIsoParent->outcome === 'failed'
&& (string) $lateIsoParent->effect_resolution === 'applied'
&& !(bool) $lateIsoParent->finalization_incident
&& $lateIso !== null
&& (string) $lateIso->state === 'retired'
&& (string) $lateIso->name === ''
&& (int) $lateIso->size_bytes === 0
&& !Capsule::table('cloud_user_iso_replicas')
->where('user_iso_id', $lateIsoId)->exists()
&& CloudAgentSettingsService::$requests === $beforeLateDeleteRequests,
'A late user-ISO deletion receipt did not converge its owner-bound '
. 'catalog ledgers without replay.');
echo "Cloud aggregate recovery convergence tests passed.\n";
}

@ -219,6 +219,11 @@ namespace {
$table->string('id')->primary();
$table->integer('active_job_id')->nullable();
$table->string('state')->default('active');
$table->integer('revision')->default(1);
$table->string('generation_token')->default('');
$table->integer('image_bytes')->default(0);
$table->string('sha256')->default('');
$table->integer('updated_at')->default(0);
});
$schema->create('cloud_user_isos',
static function (Blueprint $table): void {
@ -833,6 +838,124 @@ namespace {
&& count($client->requests) === $beforeNonSettingsRequests + 1,
'Inconclusive non-settings evidence did not stay recoverable.');
// A successful read-only inspection must adapt back to the same terminal
// operation contract as an immediate Agent response. This is the public
// recovery seam for a template deletion whose original success receipt
// reached PHP but failed local catalog finalization.
$templateDeleteId = 'automatic-delete-template';
[$templateDeleteParentId] = $insertSettingsJob(
md5('automatic-template-delete-parent'),
['hostname' => 'unused-parent.example']
);
Capsule::table('cloud_jobs')->where(
'id',
$templateDeleteParentId
)->update([
'parent_job_id' => null,
'instance_id' => null,
'host_id' => null,
'kind' => 'template_delete',
'execution_mode' => 'aggregate',
'attempts' => 0,
'dispatch_attempts' => 0,
'dispatch_started_at' => null,
'agent_accepted_at' => null,
'dispatched_at' => null,
'effect_resolution' => 'unknown',
'finalization_incident' => 1,
'finalization_error_code' => 'recovery_child_convergence',
'available_at' => 0,
'lease_token' => null,
'lease_until' => 0,
]);
[$templateDeleteChildId] = $insertSettingsJob(
md5('automatic-template-delete-child'),
['hostname' => 'unused-child.example']
);
$templateDeleteOperationId = (string) CloudJob::findOrFail(
$templateDeleteChildId
)->operation_id;
Capsule::table('cloud_jobs')->where(
'id',
$templateDeleteChildId
)->update([
'parent_job_id' => $templateDeleteParentId,
'instance_id' => null,
'kind' => 'template_delete_replica',
'sealed_payload' => CloudSecrets::encrypt([
'method' => 'DELETE',
'request_target' => '/v1/templates/' . $templateDeleteId,
'body' => ['operation_id' => $templateDeleteOperationId],
'owner_id' => 1,
'can_manage' => true,
'template_id' => $templateDeleteId,
], 'job-payload'),
'finalization_incident' => 1,
'finalization_error_code' => 'recovery_receipt_invalid',
'available_at' => 0,
'lease_token' => null,
'lease_until' => 0,
]);
Capsule::table('cloud_templates')->insert([
'id' => $templateDeleteId,
'active_job_id' => $templateDeleteParentId,
'state' => 'deleting',
'revision' => 1,
'generation_token' => str_repeat('d', 32),
'image_bytes' => 4096,
'sha256' => str_repeat('e', 64),
'updated_at' => $now,
]);
Capsule::table('cloud_template_replicas')->insert([
'host_id' => 7,
'template_id' => $templateDeleteId,
'revision' => 1,
'generation_token' => str_repeat('d', 32),
'state' => 'delete_failed',
'image_bytes' => 4096,
'sha256' => str_repeat('e', 64),
'operation_id' => $templateDeleteOperationId,
'job_id' => $templateDeleteChildId,
'error_code' => 'completion_finalization_failed',
'created_at' => $now - 120,
'updated_at' => $now,
]);
$client->responses[] = $inspection(
$templateDeleteOperationId,
'succeeded',
'applied',
[
'resource_type' => 'template',
'id' => $templateDeleteId,
'deleted' => true,
'managed_image_absent' => true,
'managed_metadata_absent' => true,
'managed_artifacts_absent' => true,
],
false
);
$beforeTemplateDeleteRequests = count($client->requests);
$assert((new CloudAutomaticOperationRecoveryService())->tick(),
'The recovery worker did not inspect the template deletion.');
$templateDeleteChild = CloudJob::findOrFail($templateDeleteChildId);
$templateDeleteParent = CloudJob::findOrFail($templateDeleteParentId);
$assert((string) $templateDeleteChild->effect_resolution === 'applied'
&& !(bool) $templateDeleteChild->finalization_incident
&& $templateDeleteChild->finalization_error_code === null
&& (string) $templateDeleteParent->effect_resolution === 'applied'
&& !(bool) $templateDeleteParent->finalization_incident
&& (string) Capsule::table('cloud_templates')
->where('id', $templateDeleteId)->value('state') === 'retired'
&& !Capsule::table('cloud_template_replicas')
->where('template_id', $templateDeleteId)->exists()
&& count($client->requests) === $beforeTemplateDeleteRequests + 1
&& str_ends_with(
$client->requests[$beforeTemplateDeleteRequests]['target'],
'/inspect'
),
'A successful template deletion inspection did not converge its '
. 'existing operation and catalog ledger.');
// A poisoned Agent completion outbox is never evidence for either effect
// verdict. Even a syntactically complete not-applied projection remains
// fenced and the reviewed cause is visible without exposing raw data.

@ -3,6 +3,7 @@
require dirname(__DIR__) . '/vendor/autoload.php';
use App\Services\Cloud\CloudError;
use App\Services\Cloud\CloudAgentCatalogDeletion;
use App\Services\Cloud\CloudOperationCompletionService;
use App\Models\CloudAgentCompletionEvent;
use App\Models\CloudJob;
@ -75,6 +76,56 @@ $assert(($validFailure['status'] ?? null) === 'failed'
&& ($validFailure['safe_to_resubmit'] ?? null) === false,
'A conservative terminal failure receipt was rejected.');
// Completion delivery intentionally omits the display-only retryable field.
// Reconstructing its terminal Agent operation must restore the canonical
// false value so every catalog deletion finalizer sees the same contract as
// an immediate Agent response.
foreach ([
['template', 'debian-13-contract', 0],
['rescue_template', str_repeat('b', 32), 0],
['user_iso', str_repeat('c', 32), 42],
] as [$resourceType, $resourceId, $ownerId]) {
$catalogResult = [
'resource_type' => $resourceType,
'id' => $resourceId,
'deleted' => true,
'managed_image_absent' => true,
'managed_metadata_absent' => true,
'managed_artifacts_absent' => true,
];
if ($resourceType === 'user_iso') {
$catalogResult['owner_id'] = $ownerId;
}
$catalogEvent = new CloudAgentCompletionEvent();
$catalogEvent->operation_id = str_repeat('9', 32);
$catalogEvent->outcome = 'succeeded';
$catalogEvent->effect_may_have_applied = 0;
$catalogEvent->safe_to_resubmit = 0;
$catalogEvent->error_code = '';
$catalogEvent->result_json = json_encode([
'schema' => 'virt_cloud_completion_event',
'version' => 1,
'identity' => [
'action' => 'delete-' . str_replace('_', '-', $resourceType),
'resource_id' => $resourceType . ':' . $resourceId,
'request_fingerprint' => str_repeat('8', 64),
],
'result' => $catalogResult,
], JSON_THROW_ON_ERROR);
$catalogOperation = CloudOperationCompletionService::operation(
$catalogEvent
);
$assert(($catalogOperation['retryable'] ?? null) === false
&& CloudAgentCatalogDeletion::confirms(
$catalogOperation,
$resourceType,
$resourceId,
$ownerId
),
'A durable successful ' . $resourceType
. ' deletion receipt lost its terminal Agent contract.');
}
// Crash window: finish() committed the terminal task and completion_event_id,
// but PHP died before marking the durable receipt applied. The same immutable
// receipt must repair that marker; it is not a late or rejected completion.

@ -29,9 +29,11 @@ namespace {
require dirname(__DIR__) . '/vendor/autoload.php';
use App\Models\CloudJob;
use App\Models\CloudAgentCompletionEvent;
use App\Models\CloudRescueTemplateDeletion;
use App\Models\CloudRescueTemplateImport;
use App\Services\Cloud\CloudError;
use App\Services\Cloud\CloudOperationCompletionService;
use App\Services\Cloud\CloudRescueTemplateDeletionService;
use App\Services\Cloud\CloudRescueTemplateImportService;
use App\Services\Cloud\CloudSecrets;
@ -546,5 +548,66 @@ namespace {
$assert(CloudJob::count() === $beforeIdentityConflict,
'A rescue import identity conflict persisted a new job.');
// Rescue-template deletion consumes the same reconstructed completion
// operation contract as template and user-ISO deletion. Exercise its
// actual ledger finalizer so a future adapter change cannot strand this
// standalone catalog operation in a failed state.
$completionTemplateId = str_repeat('3', 32);
$insertImport(
'completion-rescue',
$completionTemplateId,
'active',
'applied'
);
$completionDelete = CloudRescueTemplateDeletionService::delete(
1,
$completionTemplateId,
$actor
);
$completionDeleteJob = CloudJob::findOrFail(
(int) $completionDelete['task_id']
);
$completionResult = [
'resource_type' => 'rescue_template',
'id' => $completionTemplateId,
'deleted' => true,
'managed_image_absent' => true,
'managed_metadata_absent' => true,
'managed_artifacts_absent' => true,
];
$completionEvent = new CloudAgentCompletionEvent();
$completionEvent->operation_id =
(string) $completionDeleteJob->operation_id;
$completionEvent->outcome = 'succeeded';
$completionEvent->effect_may_have_applied = 0;
$completionEvent->safe_to_resubmit = 0;
$completionEvent->error_code = '';
$completionEvent->result_json = json_encode([
'schema' => 'virt_cloud_completion_event',
'version' => 1,
'identity' => [
'action' => 'delete-rescue-template',
'resource_id' => 'rescue-template:' . $completionTemplateId,
'request_fingerprint' => str_repeat('4', 64),
],
'result' => $completionResult,
], JSON_THROW_ON_ERROR);
CloudRescueTemplateDeletionService::finishJobLocked(
$completionDeleteJob,
'succeeded',
$completionResult,
CloudOperationCompletionService::operation($completionEvent)
);
$assert((string) CloudRescueTemplateDeletion::query()
->where('host_id', 1)
->where('template_id', $completionTemplateId)
->value('state') === 'succeeded'
&& (string) CloudRescueTemplateImport::query()
->where('host_id', 1)
->where('template_id', $completionTemplateId)
->value('state') === 'retired',
'A durable rescue-template deletion receipt did not finalize its '
. 'catalog ledgers.');
echo "Cloud rescue-template retry boundary checks passed.\n";
}

@ -1766,14 +1766,25 @@ namespace {
'late_completion_receipt:' . $lateCrashEventId
);
$lateCrashPrepared = CloudJob::find($lateCrashJobId);
$legacyLateCrashPreparedMarker = (string)
$lateCrashPrepared->effect_resolution_reason;
$assert(str_starts_with(
(string) $lateCrashPrepared->effect_resolution_reason,
$legacyLateCrashPreparedMarker,
'prepared:late_completion_receipt:' . $lateCrashEventId . ':'
)
&& !(bool) $lateCrashPrepared->finalization_incident
&& $lateCrashPrepared->finalization_error_code === null
&& (string) $lateCrashPrepared->effect_resolution === 'unknown',
'Exact late evidence did not replace exhausted matching prepare state.');
// Older crash windows could retain the stable late-receipt marker inside
// the mutable automatic-attempt envelope. The exact same receipt must
// upgrade that legacy wrapper without weakening operation-hash matching.
Capsule::table('cloud_jobs')->where('id', $lateCrashJobId)->update([
'effect_resolution_reason' => 'automatic_recovery:v1:'
. (time() - 21600) . ':12:'
. $legacyLateCrashPreparedMarker,
]);
$lateCrashPrepared = CloudJob::find($lateCrashJobId);
$lateCrashAudit = [
'phase' => (string) $lateCrashPrepared->phase,
'outcome' => (string) $lateCrashPrepared->outcome,

Loading…
Cancel
Save