@ -25,6 +25,61 @@ namespace App\Services\Cloud {
public static function encrypt(array $payload, string $purpose): string { $id = bin2hex(random_bytes(16)); self::$memory[$id] = $payload; return 'test-only:' . $id; }
public static function decrypt(string $cipher, string $purpose): array { return self::$memory[substr($cipher, 10)]; }
}
/** Isolated read-only Agent transport for create recovery checks. */
final class FinancialCreateRecoveryAgentClient
{
public array $requests = [];
public array $responses = [];
public function systemRecoveryRequest(
string $method,
string $target,
string $operationId,
?array $payload = null
): array {
$this->requests[] = compact(
'method', 'target', 'operationId', 'payload'
);
if ($this->responses === []) {
throw new \RuntimeException(
'Unexpected create recovery Agent request.'
);
}
$response = array_shift($this->responses);
if ($response instanceof \Throwable) {
throw $response;
}
if (is_callable($response)) {
$response = $response(end($this->requests));
}
if (!is_array($response)) {
throw new \RuntimeException(
'Invalid create recovery Agent response.'
);
}
return $response;
}
}
final class CloudAgentSettingsService
{
public static ?FinancialCreateRecoveryAgentClient $client = null;
public static int $hostId = 0;
public static function clientForConfiguration(
int $hostId,
float $timeoutSeconds = 120.0
): FinancialCreateRecoveryAgentClient {
if ($hostId !== self::$hostId || $timeoutSeconds !== 30.0
|| self::$client === null) {
throw new \RuntimeException(
'Unexpected create recovery client request.'
);
}
return self::$client;
}
}
}
namespace {
require dirname(__DIR__) . '/vendor/autoload.php';
@ -69,6 +124,10 @@ namespace {
use App\Services\Cloud\CloudPlatformTestNetworkPolicy;
use App\Services\Cloud\CloudDeploymentPageService;
use App\Services\Cloud\CloudTemplateCatalogService;
use App\Services\Cloud\CloudAgentSettingsService;
use App\Services\Cloud\CloudAgentError;
use App\Services\Cloud\CloudCreateFailureRecoveryService;
use App\Services\Cloud\FinancialCreateRecoveryAgentClient;
use GuzzleHttp\Client;
use GuzzleHttp\Handler\MockHandler;
use GuzzleHttp\HandlerStack;
@ -2915,5 +2974,892 @@ namespace {
'Unknown, applied, or sticky lifecycle evidence created a new mutation generation.');
}
// A create which crossed the Agent boundary may time out locally before
// the authenticated terminal failure reaches PHP. Exact Agent proof that
// both the guest and every managed artifact are absent must finish the
// original paid order locally; it must never dispatch a second create.
foreach (['203.0.113.20', '203.0.113.21', '203.0.113.22'] as $address) {
if (!CloudPublicIp::where('address', $address)->exists()) {
$ip = new CloudPublicIp();
$ip->host_id = $host->id;
$ip->public_ip_pool_id = $sourcePublicPool->id;
$ip->address = $address;
$ip->prefix_length = 28;
$ip->state = 'available';
$ip->created_at = $ip->updated_at = time();
$ip->save();
}
}
$lateFailureOrder = CloudOrderService::purchase(array_replace($input, [
'name' => 'vm-late-failure',
'idempotency_key' => 'test-request-key-late-failure',
]), $actor);
$lateFailureInvoice = CloudInvoice::where(
'order_id',
$lateFailureOrder->id
)->firstOrFail();
CloudInvoiceService::payBalance(
$lateFailureInvoice,
$actor,
'provision-balance-late-failure'
);
$lateFailureJob = CloudJob::where(
'order_id',
$lateFailureOrder->id
)->firstOrFail();
// Earlier lifecycle tests intentionally leave separately audited retries
// queued. Park those unrelated rows so this tracer observes exactly one
// create dispatch rather than depending on suite-wide queue order.
Capsule::table('cloud_jobs')
->where('id', '< >', (int) $lateFailureJob->id)
->where('phase', '< >', 'terminal')
->update([
'available_at' => time() + 3600,
'dispatch_deadline_at' => time() + 3600,
'receipt_deadline_at' => time() + 3600,
]);
Capsule::table('cloud_jobs')->where(
'id',
(int) $lateFailureJob->id
)->update(['available_at' => 0]);
$lateFailureJob->refresh();
$lateFailureIdentity = CloudOperationRecoveryIdentity::expected(
$lateFailureJob,
false
);
$mock->append(new Response(202, [], json_encode([
'operation_id' => $lateFailureJob->operation_id,
'status' => 'reserved',
...$lateFailureIdentity,
'replayed' => false,
'retryable' => false,
], JSON_THROW_ON_ERROR)));
$beforeLateFailureDispatch = count($history);
$assert($worker->tick(),
'Late-failure create did not cross the Agent dispatch boundary.');
$lateFailureJob->refresh();
$assert((string) $lateFailureJob->phase === 'awaiting_receipt'
& & (int) $lateFailureJob->dispatch_attempts === 1
& & count($history) === $beforeLateFailureDispatch + 1,
'Late-failure create did not persist its single dispatch generation.');
Capsule::table('cloud_jobs')->where(
'id',
(int) $lateFailureJob->id
)->update([
'receipt_deadline_at' => time() - 1,
'lease_token' => null,
'lease_until' => 0,
]);
$assert(CloudWorkerService::expireDueTaskForAdmin(
(int) $lateFailureJob->id
) === 1, 'Late-failure create did not reach its local deadline.');
$lateFailureJob->refresh();
$lateFailureOrder->refresh();
$lateFailureInvoice->refresh();
$lateFailureReservation = CloudResourceReservation::where(
'order_id',
$lateFailureOrder->id
)->firstOrFail();
$lateFailureIp = CloudPublicIp::findOrFail(
(int) $lateFailureReservation->public_ip_id
);
$walletWhileFenced = (string) User::findOrFail(1)->money;
$stockWhileFenced = (int) CloudOffer::findOrFail(1)->stock;
$assert((string) $lateFailureJob->phase === 'terminal'
& & (string) $lateFailureJob->outcome === 'failed'
& & (string) $lateFailureJob->effect_resolution === 'unknown'
& & (string) $lateFailureOrder->status === 'failed'
& & $lateFailureOrder->refunded_at === null
& & (string) $lateFailureInvoice->status === 'paid'
& & (string) $lateFailureReservation->state === 'uncertain'
& & (string) $lateFailureIp->state === 'uncertain',
'Timed-out dispatched create did not retain its financial fence.');
$lateFailureEventId = 'create-late-failure-event-0001';
$lateFailureReceipt = [
'type' => 'completion',
'id' => $lateFailureEventId,
'agent_id' => (string) $lateFailureJob->host_id,
'operation_id' => (string) $lateFailureJob->operation_id,
...$lateFailureIdentity,
'status' => 'failed',
'effect_may_have_applied' => false,
'safe_to_resubmit' => false,
'error_code' => 'creation_absent',
'result' => [
'uuid' => $lateFailureIdentity['resource_id'],
'creation_absent' => true,
'managed_artifacts_absent' => true,
],
];
$lateFailureAccepted = CloudOperationCompletionService::accept(
$lateFailureReceipt
);
$lateFailureJob->refresh();
$lateFailureOrder->refresh();
$lateFailureInvoice->refresh();
$lateFailureReservation->refresh();
$lateFailureIp->refresh();
$lateFailureEvent = \App\Models\CloudAgentCompletionEvent::where(
'event_id',
$lateFailureEventId
)->firstOrFail();
$assert($lateFailureAccepted === [
'accepted' => true,
'disposition' => 'applied',
]
& & (string) $lateFailureEvent->disposition === 'applied'
& & $lateFailureEvent->applied_at !== null
& & (string) $lateFailureJob->effect_resolution === 'not_applied'
& & !(bool) $lateFailureJob->finalization_incident
& & (string) $lateFailureOrder->status === 'failed'
& & $lateFailureOrder->refunded_at !== null
& & (string) $lateFailureInvoice->status === 'refunded'
& & (string) $lateFailureReservation->state === 'released'
& & (string) $lateFailureIp->state === 'available'
& & (string) User::findOrFail(1)->money
=== CloudMoney::decimal(
CloudMoney::cents($walletWhileFenced) + 999
)
& & (int) CloudOffer::findOrFail(1)->stock
=== $stockWhileFenced + 1
& & count($history) === $beforeLateFailureDispatch + 1,
'Exact late create failure did not atomically refund, release, ACK, '
. 'or replayed the original mutation.');
$lateFailureWalletAfter = (string) User::findOrFail(1)->money;
$lateFailureStockAfter = (int) CloudOffer::findOrFail(1)->stock;
$lateFailureRefundedAt = (int) $lateFailureOrder->refunded_at;
$lateFailureJobUpdatedAt = (int) $lateFailureJob->updated_at;
$lateFailureAppliedAt = (int) $lateFailureEvent->applied_at;
$lateFailureDuplicate = CloudOperationCompletionService::accept(
$lateFailureReceipt
);
$assert($lateFailureDuplicate === [
'accepted' => true,
'disposition' => 'applied',
]
& & (string) User::findOrFail(1)->money
=== $lateFailureWalletAfter
& & (int) CloudOffer::findOrFail(1)->stock
=== $lateFailureStockAfter
& & (int) CloudOrder::findOrFail(
(int) $lateFailureOrder->id
)->refunded_at === $lateFailureRefundedAt
& & (int) CloudJob::findOrFail(
(int) $lateFailureJob->id
)->updated_at === $lateFailureJobUpdatedAt
& & (int) \App\Models\CloudAgentCompletionEvent::findOrFail(
$lateFailureEventId
)->applied_at === $lateFailureAppliedAt
& & count($history) === $beforeLateFailureDispatch + 1,
'Duplicate late create failure refunded, released, ACKed, or dispatched '
. 'more than once.');
/**
* Create one paid order, cross the Agent boundary exactly once, and force
* the panel-side receipt deadline to the terminal unknown-effect state.
* Later recovery assertions can then focus on finance and evidence rather
* than reproducing dispatch bookkeeping by hand.
*
* @return array< string , mixed >
*/
$makeTimedOutCreate = static function (
string $suffix
) use (
$input,
$actor,
$mock,
$worker,
& $history,
$assert
): array {
$order = CloudOrderService::purchase(array_replace($input, [
'name' => 'vm-create-recovery-' . $suffix,
'idempotency_key' => 'test-create-recovery-' . $suffix,
]), $actor);
$invoice = CloudInvoice::where('order_id', $order->id)
->firstOrFail();
CloudInvoiceService::payBalance(
$invoice,
$actor,
'provision-create-recovery-' . $suffix
);
$job = CloudJob::where('order_id', $order->id)->firstOrFail();
$future = time() + 3600;
Capsule::table('cloud_jobs')
->where('id', '< >', (int) $job->id)
->update([
'available_at' => $future,
'lease_until' => $future,
]);
Capsule::table('cloud_jobs')->where('id', (int) $job->id)
->update(['available_at' => 0, 'lease_until' => 0]);
$job->refresh();
$identity = CloudOperationRecoveryIdentity::expected($job, false);
$mock->append(new Response(202, [], json_encode([
'operation_id' => $job->operation_id,
'status' => 'reserved',
...$identity,
'replayed' => false,
'retryable' => false,
], JSON_THROW_ON_ERROR)));
$historyBefore = count($history);
$assert($worker->tick(),
'Create recovery fixture did not dispatch: ' . $suffix);
$job->refresh();
$assert((string) $job->phase === 'awaiting_receipt'
& & (int) $job->dispatch_attempts === 1
& & count($history) === $historyBefore + 1,
'Create recovery fixture did not persist one dispatch: '
. $suffix);
Capsule::table('cloud_jobs')->where('id', (int) $job->id)
->update([
'receipt_deadline_at' => time() - 1,
'lease_token' => null,
'lease_until' => 0,
]);
$assert(CloudWorkerService::expireDueTaskForAdmin((int) $job->id)
=== 1, 'Create recovery fixture did not expire: ' . $suffix);
$job->refresh();
$order->refresh();
$invoice->refresh();
$reservation = CloudResourceReservation::where(
'order_id',
$order->id
)->firstOrFail();
$ip = CloudPublicIp::findOrFail((int) $reservation->public_ip_id);
$assert((string) $job->phase === 'terminal'
& & (string) $job->outcome === 'failed'
& & (string) $job->effect_resolution === 'unknown'
& & (string) $order->status === 'failed'
& & $order->refunded_at === null
& & (string) $invoice->status === 'paid'
& & (string) $reservation->state === 'uncertain'
& & (string) $ip->state === 'uncertain',
'Create recovery fixture did not retain its safety fence: '
. $suffix);
return compact(
'order', 'invoice', 'job', 'identity', 'reservation', 'ip',
'historyBefore'
) + [
'walletBefore' => (string) User::findOrFail(1)->money,
'stockBefore' => (int) CloudOffer::findOrFail(1)->stock,
];
};
$makeRecoveryReady = static function (CloudJob $job): void {
$future = time() + 3600;
Capsule::table('cloud_jobs')
->where('id', '< >', (int) $job->id)
->update([
'available_at' => $future,
'lease_until' => $future,
]);
Capsule::table('cloud_jobs')->where('id', (int) $job->id)
->update([
'available_at' => 0,
'lease_token' => null,
'lease_until' => 0,
]);
$job->refresh();
};
$inspectionFor = static function (
CloudJob $job,
array $identity,
bool $fenced = false
): array {
return [
'operation_id' => (string) $job->operation_id,
...$identity,
'operation_status' => 'failed',
'checked_at' => time(),
'outcome' => 'failed',
'summary_code' => 'operation_failed',
'error_code' => 'creation_absent',
'effect_resolution' => 'not_applied',
'fence_state' => $fenced ? 'active' : 'not_required',
'fence_resolution_required' => $fenced,
'effect_projection' => [
'creation_absent' => true,
'managed_artifacts_absent' => true,
],
'abort_job' => ['available' => false],
'cleanup' => ['available' => false],
];
};
$assertCreateSettled = static function (
array $case,
string $message
) use ($assert, & $history): void {
$case['job']->refresh();
$case['order']->refresh();
$case['invoice']->refresh();
$case['reservation']->refresh();
$case['ip']->refresh();
$assert((string) $case['job']->effect_resolution === 'not_applied'
& & !(bool) $case['job']->finalization_incident
& & $case['job']->lease_token === null
& & (string) $case['order']->status === 'failed'
& & $case['order']->refunded_at !== null
& & (string) $case['invoice']->status === 'refunded'
& & (string) $case['reservation']->state === 'released'
& & (string) $case['ip']->state === 'available'
& & (string) User::findOrFail(1)->money
=== CloudMoney::decimal(
CloudMoney::cents($case['walletBefore']) + 999
)
& & (int) CloudOffer::findOrFail(1)->stock
=== $case['stockBefore'] + 1
& & count($history) === $case['historyBefore'] + 1,
$message);
};
$failedCreateReceipt = static function (
array $case,
string $eventId
): array {
return [
'type' => 'completion',
'id' => $eventId,
'agent_id' => (string) $case['job']->host_id,
'operation_id' => (string) $case['job']->operation_id,
...$case['identity'],
'status' => 'failed',
'effect_may_have_applied' => false,
'safe_to_resubmit' => false,
'error_code' => 'creation_absent',
'result' => [
'uuid' => $case['identity']['resource_id'],
'creation_absent' => true,
'managed_artifacts_absent' => true,
],
];
};
// Immutable host absence evidence must wait for a temporarily inconsistent
// local invoice. Keeping the event received allows the exact same receipt
// to finish atomically after the PHP ledger is repaired.
$invoiceGuardCase = $makeTimedOutCreate('receipt-invoice-guard');
$invoiceGuardCase['invoice']->purpose = 'renewal';
$invoiceGuardCase['invoice']->save();
$invoiceGuardReceipt = $failedCreateReceipt(
$invoiceGuardCase,
'create-invoice-guard-failure-event-0001'
);
$invoiceGuardPending = CloudOperationCompletionService::accept(
$invoiceGuardReceipt
);
$invoiceGuardCase['job']->refresh();
$invoiceGuardCase['order']->refresh();
$invoiceGuardCase['reservation']->refresh();
$invoiceGuardCase['ip']->refresh();
$invoiceGuardEvent = \App\Models\CloudAgentCompletionEvent::findOrFail(
$invoiceGuardReceipt['id']
);
$assert($invoiceGuardPending === [
'accepted' => true,
'disposition' => 'received',
]
& & (string) $invoiceGuardEvent->disposition === 'received'
& & $invoiceGuardEvent->applied_at === null
& & (string) $invoiceGuardCase['job']->effect_resolution === 'unknown'
& & $invoiceGuardCase['order']->refunded_at === null
& & (string) $invoiceGuardCase['reservation']->state === 'uncertain'
& & (string) $invoiceGuardCase['ip']->state === 'uncertain'
& & (string) User::findOrFail(1)->money
=== $invoiceGuardCase['walletBefore']
& & (int) CloudOffer::findOrFail(1)->stock
=== $invoiceGuardCase['stockBefore'],
'Exact create failure bypassed an inconsistent local invoice: '
. json_encode([
'accepted' => $invoiceGuardPending,
'event_disposition' => $invoiceGuardEvent->disposition,
'event_applied_at' => $invoiceGuardEvent->applied_at,
'job_resolution' =>
$invoiceGuardCase['job']->effect_resolution,
'refunded_at' => $invoiceGuardCase['order']->refunded_at,
'reservation' => $invoiceGuardCase['reservation']->state,
'ip' => $invoiceGuardCase['ip']->state,
'wallet_before' => $invoiceGuardCase['walletBefore'],
'wallet_after' => User::findOrFail(1)->money,
'stock_before' => $invoiceGuardCase['stockBefore'],
'stock_after' => CloudOffer::findOrFail(1)->stock,
], JSON_THROW_ON_ERROR));
$invoiceGuardCase['invoice']->purpose = 'provision';
$invoiceGuardCase['invoice']->save();
$assert(CloudOperationCompletionService::accept($invoiceGuardReceipt)
=== ['accepted' => true, 'disposition' => 'applied'],
'Repaired local invoice did not resume the same create receipt.');
$assertCreateSettled(
$invoiceGuardCase,
'Repaired local invoice did not atomically settle create finance.'
);
// The specialised create finalizer must validate the complete dispatched
// envelope itself. Generic recovery policy deliberately does not own
// create, so neither a damaged attempt count nor a missing receipt
// deadline may reach the refund path.
foreach ([
'attempts' => 2,
'receipt_deadline_at' => null,
] as $damagedField => $damagedValue) {
$suffix = 'receipt-envelope-' . str_replace('_', '-', $damagedField);
$envelopeCase = $makeTimedOutCreate($suffix);
$eventId = 'create-envelope-' . str_replace('_', '-', $damagedField)
. '-event-0001';
$envelopeReceipt = $failedCreateReceipt($envelopeCase, $eventId);
$originalValue = $envelopeCase['job']->{$damagedField};
Capsule::table('cloud_jobs')->where(
'id',
(int) $envelopeCase['job']->id
)->update([$damagedField => $damagedValue]);
$envelopePending = CloudOperationCompletionService::accept(
$envelopeReceipt
);
$envelopeCase['job']->refresh();
$envelopeCase['order']->refresh();
$envelopeCase['invoice']->refresh();
$envelopeCase['reservation']->refresh();
$envelopeCase['ip']->refresh();
$envelopeEvent = \App\Models\CloudAgentCompletionEvent::findOrFail(
$eventId
);
$assert($envelopePending === [
'accepted' => true,
'disposition' => 'received',
]
& & (string) $envelopeEvent->disposition === 'received'
& & $envelopeEvent->applied_at === null
& & (string) $envelopeCase['job']->effect_resolution === 'unknown'
& & $envelopeCase['order']->refunded_at === null
& & (string) $envelopeCase['invoice']->status === 'paid'
& & (string) $envelopeCase['reservation']->state === 'uncertain'
& & (string) $envelopeCase['ip']->state === 'uncertain'
& & (string) User::findOrFail(1)->money
=== $envelopeCase['walletBefore']
& & (int) CloudOffer::findOrFail(1)->stock
=== $envelopeCase['stockBefore'],
'Damaged create dispatch envelope released finance: '
. $damagedField);
Capsule::table('cloud_jobs')->where(
'id',
(int) $envelopeCase['job']->id
)->update([$damagedField => $originalValue]);
$assert(CloudOperationCompletionService::accept($envelopeReceipt)
=== ['accepted' => true, 'disposition' => 'applied'],
'Repaired create dispatch envelope did not resume receipt: '
. $damagedField);
$assertCreateSettled(
$envelopeCase,
'Repaired dispatch envelope did not settle create finance: '
. $damagedField
);
}
// A fresh read-only Journal failure with complete absence projection can
// finish a timed-out create even when no completion event ever arrived.
$inspectionCase = $makeTimedOutCreate('inspection-no-receipt');
$makeRecoveryReady($inspectionCase['job']);
$inspectionClient = new FinancialCreateRecoveryAgentClient();
$inspectionClient->responses[] = $inspectionFor(
$inspectionCase['job'],
$inspectionCase['identity']
);
CloudAgentSettingsService::$hostId =
(int) $inspectionCase['job']->host_id;
CloudAgentSettingsService::$client = $inspectionClient;
$inspectionRecovery = new CloudAutomaticOperationRecoveryService();
$assert($inspectionRecovery->tick(),
'No-receipt create was not claimed for read-only recovery.');
$assert(count($inspectionClient->requests) === 1
& & $inspectionClient->requests[0]['method'] === 'GET'
& & str_ends_with(
$inspectionClient->requests[0]['target'],
'/inspect'
)
& & $inspectionClient->requests[0]['payload'] === null,
'No-receipt create recovery used something other than one read-only '
. 'inspection.');
$assertCreateSettled(
$inspectionCase,
'Read-only create absence evidence did not atomically settle finance.'
);
// Once inspection has settled the ledger, the first later failure receipt
// is an idempotent ACK of the late delivery. It must not re-enter the received queue,
// rewrite the job incident, or credit/release anything twice.
$inspectionCase['job']->refresh();
$inspectionJobUpdatedAt = (int) $inspectionCase['job']->updated_at;
$inspectionWalletAfter = (string) User::findOrFail(1)->money;
$inspectionStockAfter = (int) CloudOffer::findOrFail(1)->stock;
$inspectionLateReceipt = [
'type' => 'completion',
'id' => 'create-inspection-late-failure-event-0001',
'agent_id' => (string) $inspectionCase['job']->host_id,
'operation_id' => (string) $inspectionCase['job']->operation_id,
...$inspectionCase['identity'],
'status' => 'failed',
'effect_may_have_applied' => false,
'safe_to_resubmit' => false,
'error_code' => 'creation_absent',
'result' => [
'uuid' => $inspectionCase['identity']['resource_id'],
'creation_absent' => true,
'managed_artifacts_absent' => true,
],
];
$inspectionLateAccepted = CloudOperationCompletionService::accept(
$inspectionLateReceipt
);
$inspectionLateDuplicate = CloudOperationCompletionService::accept(
$inspectionLateReceipt
);
$inspectionCase['job']->refresh();
$inspectionLateEvent = \App\Models\CloudAgentCompletionEvent::findOrFail(
$inspectionLateReceipt['id']
);
$assert($inspectionLateAccepted === [
'accepted' => true,
'disposition' => 'applied',
]
& & $inspectionLateDuplicate === $inspectionLateAccepted
& & (string) $inspectionLateEvent->disposition === 'applied'
& & $inspectionLateEvent->applied_at !== null
& & (int) $inspectionCase['job']->updated_at
=== $inspectionJobUpdatedAt
& & !(bool) $inspectionCase['job']->finalization_incident
& & (string) User::findOrFail(1)->money === $inspectionWalletAfter
& & (int) CloudOffer::findOrFail(1)->stock === $inspectionStockAfter,
'A failure receipt arriving after inspection settlement was requeued '
. 'or changed the financial ledger: ' . json_encode([
'first' => $inspectionLateAccepted,
'duplicate' => $inspectionLateDuplicate,
'event_disposition' => $inspectionLateEvent->disposition,
'event_applied_at' => $inspectionLateEvent->applied_at,
'job_updated_at_before' => $inspectionJobUpdatedAt,
'job_updated_at_after' => $inspectionCase['job']->updated_at,
'job_incident' => $inspectionCase['job']->finalization_incident,
'wallet_before' => $inspectionWalletAfter,
'wallet_after' => User::findOrFail(1)->money,
'stock_before' => $inspectionStockAfter,
'stock_after' => CloudOffer::findOrFail(1)->stock,
], JSON_THROW_ON_ERROR));
// An incomplete failed receipt is retained as a late audit event, then a
// fresh read-only inspection supplies the missing managed-artifact proof.
$incompleteCase = $makeTimedOutCreate('incomplete-receipt');
$incompleteReceipt = [
'type' => 'completion',
'id' => 'create-incomplete-failure-event-0001',
'agent_id' => (string) $incompleteCase['job']->host_id,
'operation_id' => (string) $incompleteCase['job']->operation_id,
...$incompleteCase['identity'],
'status' => 'failed',
'effect_may_have_applied' => false,
'safe_to_resubmit' => false,
'error_code' => 'creation_absent',
'result' => [
'uuid' => $incompleteCase['identity']['resource_id'],
'creation_absent' => true,
],
];
$assert(CloudOperationCompletionService::accept($incompleteReceipt)
=== ['accepted' => true, 'disposition' => 'late'],
'Incomplete create absence receipt was not ACKed as late.');
$makeRecoveryReady($incompleteCase['job']);
$incompleteClient = new FinancialCreateRecoveryAgentClient();
$incompleteClient->responses[] = $inspectionFor(
$incompleteCase['job'],
$incompleteCase['identity']
);
CloudAgentSettingsService::$client = $incompleteClient;
$assert((new CloudAutomaticOperationRecoveryService())->tick(),
'Incomplete receipt create did not enter fresh inspection recovery.');
$assertCreateSettled(
$incompleteCase,
'Fresh inspection did not finish an incomplete failed receipt create.'
);
// If PHP fails after durably preparing a no-fence conclusion, a later
// tick must finish locally even while the Agent is offline.
$noFenceRetry = $makeTimedOutCreate('nofence-db-retry');
$noFenceRetry['invoice']->purpose = 'renewal';
$noFenceRetry['invoice']->save();
$makeRecoveryReady($noFenceRetry['job']);
$noFenceClient = new FinancialCreateRecoveryAgentClient();
$noFenceClient->responses[] = $inspectionFor(
$noFenceRetry['job'],
$noFenceRetry['identity']
);
CloudAgentSettingsService::$client = $noFenceClient;
$assert((new CloudAutomaticOperationRecoveryService())->tick(),
'No-fence DB-failure create was not claimed.');
$noFenceRetry['job']->refresh();
$noFenceRetry['order']->refresh();
$noFenceRetry['invoice']->refresh();
$assert(preg_match(
'/^prepared:create_not_applied:v1:[a-f0-9]{64}$/D',
(string) $noFenceRetry['job']->effect_resolution_reason
) === 1
& & (string) $noFenceRetry['job']->effect_resolution === 'unknown'
& & $noFenceRetry['order']->refunded_at === null
& & (string) $noFenceRetry['invoice']->status === 'paid'
& & count($noFenceClient->requests) === 1,
'No-fence financial failure did not retain its durable prepare.');
$noFenceRetry['invoice']->purpose = 'provision';
$noFenceRetry['invoice']->save();
$makeRecoveryReady($noFenceRetry['job']);
// Deliberately leave the response queue empty: any Agent call now fails.
$assert((new CloudAutomaticOperationRecoveryService())->tick(),
'Prepared no-fence create was not retried locally.');
$assert(count($noFenceClient->requests) === 1,
'Prepared no-fence retry contacted an offline Agent.');
$assertCreateSettled(
$noFenceRetry,
'Prepared no-fence create did not settle after the DB fault cleared.'
);
// If the Agent fence write committed but the same PHP transaction later
// failed, recovery reads that immutable verdict once and never POSTs it a
// second time.
$fenceRetry = $makeTimedOutCreate('fence-db-retry');
$fenceRetry['invoice']->purpose = 'renewal';
$fenceRetry['invoice']->save();
$makeRecoveryReady($fenceRetry['job']);
$fenceClient = new FinancialCreateRecoveryAgentClient();
$fenceClient->responses[] = $inspectionFor(
$fenceRetry['job'],
$fenceRetry['identity'],
true
);
$fenceClient->responses[] = new CloudAgentError(
'no committed fence',
404,
'not_found',
false,
404
);
$committedFence = null;
$fenceClient->responses[] = static function (
array $request
) use (& $committedFence, $fenceRetry): array {
$payload = $request['payload'];
$committedFence = [
'operation_id' => (string) $fenceRetry['job']->operation_id,
'resolution_id' => (string) $payload['resolution_id'],
'resolution' => (string) $payload['resolution'],
'inspection_checked_at' =>
(int) $payload['inspection_checked_at'],
'reason_sha256' => (string) $payload['reason_sha256'],
'resolved_at' => time(),
'resolved_by' => 0,
'replayed' => false,
];
return $committedFence;
};
CloudAgentSettingsService::$client = $fenceClient;
$assert((new CloudAutomaticOperationRecoveryService())->tick(),
'Active-fence DB-failure create was not claimed.');
$fenceRetry['job']->refresh();
$assert(is_array($committedFence)
& & count($fenceClient->requests) === 3
& & $fenceClient->requests[0]['method'] === 'GET'
& & str_ends_with($fenceClient->requests[0]['target'], '/inspect')
& & $fenceClient->requests[1]['method'] === 'GET'
& & str_ends_with(
$fenceClient->requests[1]['target'],
'/fence-resolution'
)
& & $fenceClient->requests[2]['method'] === 'POST'
& & str_ends_with(
$fenceClient->requests[2]['target'],
'/resolve-fence'
)
& & preg_match(
'/^prepared:create_not_applied:v1:[a-f0-9]{64}$/D',
(string) $fenceRetry['job']->effect_resolution_reason
) === 1
& & (string) $fenceRetry['job']->effect_resolution === 'unknown',
'Fence commit plus PHP failure did not preserve resumable evidence.');
$fenceRetry['invoice']->purpose = 'provision';
$fenceRetry['invoice']->save();
$makeRecoveryReady($fenceRetry['job']);
$fenceClient->responses[] = $committedFence;
$assert((new CloudAutomaticOperationRecoveryService())->tick(),
'Prepared active-fence create was not retried.');
$assert(count($fenceClient->requests) === 4
& & $fenceClient->requests[3]['method'] === 'GET'
& & str_ends_with(
$fenceClient->requests[3]['target'],
'/fence-resolution'
),
'Prepared active-fence retry wrote or re-inspected Agent state.');
$assertCreateSettled(
$fenceRetry,
'Prepared active-fence create did not settle after DB recovery.'
);
// A fence write can fail before the Agent commits it. The next tick first
// proves that no prior verdict exists, then obtains a fresh inspection.
// checked_at is freshness metadata, not part of the durable conclusion:
// the same facts must produce the same marker and fence reason.
$stableCase = $makeTimedOutCreate('stable-inspection-marker');
$makeRecoveryReady($stableCase['job']);
$stableClient = new FinancialCreateRecoveryAgentClient();
$stableFirstInspection = $inspectionFor(
$stableCase['job'],
$stableCase['identity'],
true
);
$stableClient->responses = [
$stableFirstInspection,
new CloudAgentError('no committed fence', 404, 'not_found', false, 404),
new CloudAgentError('fence write unavailable', 503, 'unavailable', true, 503),
new CloudAgentError('no committed fence', 404, 'not_found', false, 404),
];
CloudAgentSettingsService::$client = $stableClient;
$assert((new CloudAutomaticOperationRecoveryService())->tick(),
'Stable-marker first recovery attempt was not claimed.');
$stableCase['job']->refresh();
$stableMarker = (string) $stableCase['job']->effect_resolution_reason;
$assert(preg_match(
'/^prepared:create_not_applied:v1:[a-f0-9]{64}$/D',
$stableMarker
) === 1
& & (string) $stableCase['job']->effect_resolution === 'unknown'
& & count($stableClient->requests) === 4,
'Failed fence write did not retain a stable prepared create marker.');
$makeRecoveryReady($stableCase['job']);
$stableSecondInspection = $stableFirstInspection;
$stableSecondInspection['checked_at'] =
(int) $stableFirstInspection['checked_at'] + 1;
$stableCommittedFence = null;
$stableClient->responses = [
new CloudAgentError('no committed fence', 404, 'not_found', false, 404),
$stableSecondInspection,
new CloudAgentError('no committed fence', 404, 'not_found', false, 404),
static function (array $request) use (
& $stableCommittedFence,
$stableCase
): array {
$payload = $request['payload'];
$stableCommittedFence = [
'operation_id' => (string) $stableCase['job']->operation_id,
'resolution_id' => (string) $payload['resolution_id'],
'resolution' => (string) $payload['resolution'],
'inspection_checked_at' =>
(int) $payload['inspection_checked_at'],
'reason_sha256' => (string) $payload['reason_sha256'],
'resolved_at' => time(),
'resolved_by' => 0,
'replayed' => false,
];
return $stableCommittedFence;
},
];
$assert((new CloudAutomaticOperationRecoveryService())->tick(),
'Stable-marker create did not resume from fresh inspection.');
$stableExpectedReason =
'自动恢复已确认原开通操作未创建虚拟机或面板管理资源; operation_id='
. (string) $stableCase['job']->operation_id . '; evidence='
. substr($stableMarker, -64);
$assert(is_array($stableCommittedFence)
& & count($stableClient->requests) === 8
& & $stableClient->requests[4]['method'] === 'GET'
& & str_ends_with(
$stableClient->requests[4]['target'],
'/fence-resolution'
)
& & $stableClient->requests[5]['method'] === 'GET'
& & str_ends_with($stableClient->requests[5]['target'], '/inspect')
& & $stableClient->requests[7]['method'] === 'POST'
& & (string) $stableClient->requests[7]['payload']['reason_sha256']
=== hash('sha256', $stableExpectedReason),
'Fresh equivalent inspection changed the prepared fence reason or '
. 'replayed the original create.');
$assertCreateSettled(
$stableCase,
'Stable-marker retry did not finish the create financial ledger.'
);
// Partial/residual host state and an identity mismatch are both
// contradictions, not refund evidence. They remain fenced and retryable.
$conflictCase = $makeTimedOutCreate('inspection-conflicts');
$makeRecoveryReady($conflictCase['job']);
$conflictClient = new FinancialCreateRecoveryAgentClient();
$partialInspection = $inspectionFor(
$conflictCase['job'],
$conflictCase['identity'],
true
);
$partialInspection['outcome'] = 'uncertain';
$partialInspection['summary_code'] = 'evidence_incomplete';
$partialInspection['effect_resolution'] = 'partial';
$partialInspection['error_code'] = 'cleanup_required';
$partialInspection['cleanup'] = [
'available' => true,
'plan_id' => str_repeat('a', 64),
'steps' => ['delete_managed_volumes'],
];
unset($partialInspection['effect_projection']);
$conflictClient->responses[] = $partialInspection;
CloudAgentSettingsService::$client = $conflictClient;
$assert((new CloudAutomaticOperationRecoveryService())->tick(),
'Partial create evidence was not safely deferred.');
$conflictCase['job']->refresh();
$conflictCase['order']->refresh();
$conflictCase['invoice']->refresh();
$assert((string) $conflictCase['job']->effect_resolution === 'unknown'
& & (bool) $conflictCase['job']->finalization_incident
& & $conflictCase['order']->refunded_at === null
& & (string) $conflictCase['invoice']->status === 'paid'
& & (string) User::findOrFail(1)->money
=== $conflictCase['walletBefore']
& & (int) CloudOffer::findOrFail(1)->stock
=== $conflictCase['stockBefore'],
'Partial create evidence released a financial/resource fence.');
$makeRecoveryReady($conflictCase['job']);
$wrongIdentity = $inspectionFor(
$conflictCase['job'],
$conflictCase['identity']
);
$wrongIdentity['resource_id'] = 'wrong-create-resource';
$conflictClient->responses[] = $wrongIdentity;
$assert((new CloudAutomaticOperationRecoveryService())->tick(),
'Create identity contradiction was not safely deferred.');
$conflictCase['job']->refresh();
$assert((string) $conflictCase['job']->effect_resolution === 'unknown'
& & (bool) $conflictCase['job']->finalization_incident
& & $conflictCase['order']->fresh()->refunded_at === null
& & (string) User::findOrFail(1)->money
=== $conflictCase['walletBefore']
& & (int) CloudOffer::findOrFail(1)->stock
=== $conflictCase['stockBefore'],
'Create identity contradiction released a financial/resource fence.');
// A caller cannot reuse an automatic claim after its lease has expired.
$leaseToken = bin2hex(random_bytes(16));
Capsule::table('cloud_jobs')->where('id', (int) $conflictCase['job']->id)
->update([
'available_at' => 0,
'lease_token' => $leaseToken,
'lease_until' => time() + 180,
]);
$staleClaim = CloudJob::findOrFail((int) $conflictCase['job']->id);
Capsule::table('cloud_jobs')->where('id', (int) $conflictCase['job']->id)
->update(['lease_until' => time() - 1]);
$staleRequestCount = count($conflictClient->requests);
$staleRejected = false;
try {
CloudCreateFailureRecoveryService::recover($staleClaim);
} catch (CloudError $error) {
$staleRejected = true;
}
$assert($staleRejected
& & count($conflictClient->requests) === $staleRequestCount
& & (string) CloudJob::findOrFail(
(int) $conflictCase['job']->id
)->effect_resolution === 'unknown'
& & $conflictCase['order']->fresh()->refunded_at === null,
'Expired create recovery lease contacted Agent or released finance.');
echo "Cloud SQLite purchase, provisioning, refund, ambiguity, quota and tenant checks passed (crypto double only).\n";
}