You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
virt/web/tests/CloudUndispatchedFinalizerR...

963 lines
38 KiB
PHP

<?php
declare(strict_types=1);
namespace App\Services\Cloud {
/** Isolated encrypted envelope for PHP-only finalizer recovery tests. */
final class CloudSecrets
{
private static array $values = [];
private static bool $available = true;
public static function setAvailable(bool $available): void
{
self::$available = $available;
}
public static function encrypt(array $value, string $purpose): string
{
$key = bin2hex(random_bytes(16));
self::$values[$key] = $value;
return 'undispatched-finalizer-test:' . $key;
}
public static function decrypt(string $value, string $purpose): array
{
if (!self::$available) {
throw new \RuntimeException(
'Finalizer test key is temporarily unavailable.'
);
}
$key = substr(
$value,
strlen('undispatched-finalizer-test:')
);
if (!array_key_exists($key, self::$values)) {
throw new \RuntimeException('Missing finalizer test payload.');
}
return self::$values[$key];
}
}
/** Any Agent lookup is a test failure for generation-zero recovery. */
final class CloudAgentSettingsService
{
public static int $requests = 0;
public static function clientForHost(
int $hostId,
float $timeoutSeconds = 120.0
): object {
self::$requests++;
throw new \RuntimeException(
'Undispatched finalizer recovery contacted the Agent.'
);
}
public static function clientForConfiguration(
int $hostId,
float $timeoutSeconds = 120.0
): object {
return self::clientForHost($hostId, $timeoutSeconds);
}
}
}
namespace {
require dirname(__DIR__) . '/vendor/autoload.php';
use App\Models\CloudJob;
use App\Models\CloudRescueTemplateImport;
use App\Services\Cloud\CloudAutomaticOperationRecoveryService;
use App\Services\Cloud\CloudAgentSettingsService;
use App\Services\Cloud\CloudDispatchLedger;
use App\Services\Cloud\CloudJobService;
use App\Services\Cloud\CloudRescueTemplateImportService;
use App\Services\Cloud\CloudSecrets;
use App\Services\Cloud\CloudWorkerService;
use Illuminate\Database\Capsule\Manager as Capsule;
use Illuminate\Database\Schema\Blueprint;
$assert = static function (bool $condition, string $message): void {
if (!$condition) {
throw new RuntimeException($message);
}
};
$capsule = new Capsule();
$capsule->addConnection([
'driver' => 'sqlite', 'database' => ':memory:',
], 'default');
$capsule->setAsGlobal();
$capsule->bootEloquent();
$schema = $capsule->schema('default');
$schema->create('user', static function (Blueprint $table): void {
$table->increments('id');
$table->string('money')->default('0.00');
$table->integer('enable')->default(1);
$table->integer('is_admin')->default(0);
});
$schema->create('cloud_jobs', static function (Blueprint $table): void {
$table->increments('id');
$table->string('operation_id')->unique();
$table->string('request_idempotency_key')->unique();
$table->string('request_fingerprint');
$table->integer('parent_job_id')->nullable();
$table->integer('order_id')->nullable();
$table->integer('invoice_id')->nullable();
$table->integer('instance_id')->nullable();
$table->integer('snapshot_id')->nullable();
$table->integer('vpc_repair_root_job_id')->nullable();
$table->integer('user_id');
$table->integer('host_id')->nullable();
$table->string('kind');
$table->string('execution_mode');
$table->string('phase');
$table->string('outcome')->nullable();
$table->string('status');
$table->text('sealed_payload');
$table->integer('attempts')->default(0);
$table->integer('dispatch_attempts')->default(0);
$table->integer('dispatch_deadline_at')->default(0);
$table->integer('dispatch_started_at')->nullable();
$table->integer('agent_accepted_at')->nullable();
$table->integer('receipt_deadline_at')->nullable();
$table->integer('terminal_at')->nullable();
$table->boolean('effect_may_have_applied')->nullable();
$table->boolean('safe_to_resubmit')->nullable();
$table->string('effect_resolution');
$table->integer('effect_resolved_at')->nullable();
$table->integer('effect_resolved_by')->nullable();
$table->string('effect_resolution_reason')->default('');
$table->string('completion_event_id')->nullable();
$table->integer('read_at')->nullable();
$table->integer('dispatched_at')->nullable();
$table->string('lease_token')->nullable();
$table->integer('lease_until')->default(0);
$table->integer('available_at')->default(0);
$table->string('error_code')->nullable();
$table->boolean('finalization_incident')->default(0);
$table->string('finalization_error_code')->nullable();
$table->text('result_json')->nullable();
$table->integer('created_at');
$table->integer('updated_at');
});
$schema->create('cloud_instances', static function (Blueprint $table): void {
$table->increments('id');
$table->string('uuid');
$table->integer('host_id');
$table->integer('user_id');
$table->integer('order_id')->nullable();
$table->string('name');
$table->string('status');
$table->string('service_status');
$table->string('network_status');
$table->text('spec_json');
$table->text('sealed_credential')->nullable();
$table->integer('created_at');
$table->integer('updated_at');
});
$schema->create('cloud_agent_completion_events',
static function (Blueprint $table): void {
$table->string('event_id')->primary();
$table->integer('host_id');
$table->string('operation_id');
$table->string('payload_sha256');
$table->string('outcome');
$table->boolean('effect_may_have_applied');
$table->boolean('safe_to_resubmit');
$table->string('error_code')->default('');
$table->text('result_json')->nullable();
$table->string('disposition')->default('received');
$table->integer('received_at');
$table->integer('applied_at')->nullable();
$table->unique(['host_id', 'operation_id']);
});
$schema->create('cloud_templates',
static function (Blueprint $table): void {
$table->string('id')->primary();
$table->string('catalog_key')->default('test-catalog');
$table->string('version_label')->default('test-version');
$table->boolean('is_default')->default(true);
$table->integer('active_job_id')->nullable();
$table->string('state')->default('active');
});
$schema->create('cloud_user_isos',
static function (Blueprint $table): void {
$table->string('id')->primary();
$table->integer('active_job_id')->nullable();
$table->string('state')->default('active');
});
$schema->create('cloud_rescue_template_imports',
static function (Blueprint $table): void {
$table->string('operation_id')->primary();
$table->integer('host_id');
$table->string('template_id', 32);
$table->integer('job_id');
$table->string('import_type');
$table->string('name');
$table->integer('size_bytes')->default(0);
$table->string('sha256', 64)->default('');
$table->string('state');
$table->string('error_code')->default('');
$table->integer('created_at');
$table->integer('updated_at');
});
$schema->create('cloud_template_replicas',
static function (Blueprint $table): void {
$table->integer('host_id');
$table->string('template_id');
$table->integer('revision')->default(0);
$table->string('generation_token')->default('');
$table->string('state')->default('pending');
$table->integer('image_bytes')->default(0);
$table->string('sha256')->default('');
$table->string('operation_id')->default('');
$table->integer('job_id')->nullable();
$table->string('error_code')->default('');
$table->integer('created_at')->default(0);
$table->integer('updated_at')->default(0);
$table->primary(['host_id', 'template_id']);
});
$schema->create('cloud_user_iso_replicas',
static function (Blueprint $table): void {
$table->integer('host_id');
$table->string('user_iso_id');
$table->integer('user_id');
$table->string('state')->default('pending');
$table->integer('size_bytes')->default(0);
$table->string('sha256')->default('');
$table->string('operation_id')->default('');
$table->integer('job_id')->nullable();
$table->string('error_code')->default('');
$table->integer('created_at')->default(0);
$table->integer('updated_at')->default(0);
$table->primary(['host_id', 'user_iso_id']);
});
$schema->create('cloud_hosts', static function (Blueprint $table): void {
$table->increments('id');
});
$schema->create('cloud_offers', static function (Blueprint $table): void {
$table->increments('id');
$table->integer('stock')->default(-1);
$table->integer('updated_at')->default(0);
});
$schema->create('cloud_orders', static function (Blueprint $table): void {
$table->increments('id');
$table->integer('user_id');
$table->integer('plan_id');
$table->integer('offer_id');
$table->integer('resource_pool_id');
$table->integer('host_id')->nullable();
$table->integer('reservation_id')->nullable();
$table->string('idempotency_key');
$table->string('request_hash');
$table->string('amount');
$table->string('currency')->default('USD');
$table->string('status');
$table->text('spec_json');
$table->text('sealed_payload')->nullable();
$table->integer('instance_id')->nullable();
$table->integer('refunded_at')->nullable();
$table->integer('stock_reserved_at')->nullable();
$table->integer('stock_released_at')->nullable();
$table->integer('created_at');
$table->integer('updated_at');
});
$schema->create('cloud_resource_reservations',
static function (Blueprint $table): void {
$table->increments('id');
$table->string('reservation_key');
$table->string('operation_id');
$table->integer('order_id');
$table->integer('offer_id');
$table->integer('resource_pool_id');
$table->integer('host_id');
$table->integer('user_id');
$table->integer('instance_id')->nullable();
$table->integer('network_attachment_id')->nullable();
$table->integer('public_ip_pool_id')->nullable();
$table->integer('public_ip_id')->nullable();
$table->integer('cpu_cores');
$table->integer('memory_mib');
$table->integer('disk_gib');
$table->string('state');
$table->integer('expires_at')->nullable();
$table->integer('released_at')->nullable();
$table->string('release_reason')->default('');
$table->integer('created_at');
$table->integer('updated_at');
});
$schema->create('cloud_public_ips',
static function (Blueprint $table): void {
$table->increments('id');
$table->integer('host_id');
$table->integer('public_ip_pool_id')->nullable();
$table->string('address');
$table->integer('prefix_length');
$table->string('state');
$table->integer('reservation_id')->nullable();
$table->integer('instance_id')->nullable();
$table->string('operation_id')->nullable();
$table->integer('created_at');
$table->integer('updated_at');
});
$schema->create('cloud_invoices',
static function (Blueprint $table): void {
$table->increments('id');
$table->string('invoice_no');
$table->string('purpose');
$table->integer('instance_id')->nullable();
$table->integer('order_id')->nullable();
$table->integer('user_id');
$table->integer('plan_id')->nullable();
$table->integer('offer_id')->nullable();
$table->integer('renewal_id')->nullable();
$table->integer('renews_expires_at')->nullable();
$table->integer('due_at');
$table->string('amount');
$table->string('currency')->default('USD');
$table->integer('validity_days');
$table->integer('traffic_quota_bytes');
$table->string('status');
$table->string('paid_via')->nullable();
$table->string('payment_reference')->nullable();
$table->integer('paid_at')->nullable();
$table->integer('created_at');
$table->integer('updated_at');
});
$now = time();
$operationId = str_repeat('ad', 16);
$uuid = '00000000-0000-4000-8000-000000000501';
$payload = [
'method' => 'POST',
'request_target' => '/v1/vms/' . $uuid . '/settings',
'body' => [
'hostname' => 'local-finalizer.example',
'root_password' => 'Secret9!Password',
'confirm_restart' => true,
],
'owner_id' => 1,
'can_manage' => false,
'credential' => [
'password' => 'Secret9!Password',
'login_name' => 'root',
'auth_mode' => 'password',
],
'settings_fields' => ['hostname', 'password'],
];
Capsule::table('user')->insert([
'id' => 1, 'enable' => 1, 'is_admin' => 0,
]);
Capsule::table('cloud_instances')->insert([
'id' => 501,
'uuid' => $uuid,
'host_id' => 7,
'user_id' => 1,
'name' => 'local-finalizer',
'status' => 'active',
'service_status' => 'exempt',
'network_status' => 'active',
'spec_json' => json_encode(
['managed' => true],
JSON_THROW_ON_ERROR
),
'sealed_credential' => null,
'created_at' => $now - 3600,
'updated_at' => $now - 3600,
]);
$originalEnvelope = CloudSecrets::encrypt($payload, 'job-payload');
$originalResult = json_encode(
['cause_code' => 'local_finalization_failed'],
JSON_THROW_ON_ERROR
);
Capsule::table('cloud_jobs')->insert([
'id' => 501,
'operation_id' => $operationId,
'request_idempotency_key' => $operationId,
'request_fingerprint' => hash('sha256', $operationId),
'parent_job_id' => null,
'order_id' => null,
'invoice_id' => null,
'instance_id' => 501,
'snapshot_id' => null,
'user_id' => 1,
'host_id' => 7,
'kind' => 'instance_settings',
'execution_mode' => 'agent',
'phase' => 'terminal',
'outcome' => 'failed',
'status' => 'failed',
'sealed_payload' => $originalEnvelope,
'attempts' => 0,
'dispatch_attempts' => 0,
'dispatch_deadline_at' => $now - 30,
'terminal_at' => $now - 20,
'effect_may_have_applied' => 0,
'safe_to_resubmit' => 0,
'effect_resolution' => 'not_applied',
'effect_resolution_reason' => '',
'lease_until' => 0,
'available_at' => 0,
'error_code' => 'local_finalization_failed',
'finalization_incident' => 1,
'finalization_error_code' => 'recovery_ledger_conflict',
'result_json' => $originalResult,
'created_at' => $now - 30,
'updated_at' => $now - 20,
]);
$before = CloudJob::findOrFail(501);
$assert(CloudJobService::blockingQueryForInstance(501)->exists(),
'An undispatched local finalizer incident did not block mutations.');
$service = new CloudAutomaticOperationRecoveryService();
$assert($service->tick(),
'The undispatched PHP-only finalizer was not claimed.');
$after = CloudJob::findOrFail(501);
$scrubbed = CloudSecrets::decrypt(
(string) $after->sealed_payload,
'job-payload'
);
$assert((string) $after->phase === (string) $before->phase
&& (string) $after->outcome === (string) $before->outcome
&& (string) $after->status === (string) $before->status
&& (int) $after->terminal_at === (int) $before->terminal_at
&& (string) $after->error_code === (string) $before->error_code
&& (string) $after->result_json === (string) $before->result_json
&& $after->completion_event_id === $before->completion_event_id
&& !(bool) $after->effect_may_have_applied
&& (bool) $after->safe_to_resubmit
&& (string) $after->effect_resolution === 'not_applied'
&& (string) $after->effect_resolution_reason
=== 'automatic_local_not_dispatched'
&& (int) $after->effect_resolved_at > 0
&& !(bool) $after->finalization_incident
&& $after->finalization_error_code === null
&& $after->lease_token === null
&& (int) $after->lease_until === 0
&& (int) $after->available_at === 0
&& (string) $after->sealed_payload !== $originalEnvelope
&& !isset($scrubbed['credential'])
&& !isset($scrubbed['body']['root_password'])
&& !CloudJobService::blockingQueryForInstance(501)->exists(),
'The PHP-only finalizer changed terminal audit data or failed to converge safely.');
$updatedAt = (int) $after->updated_at;
$assert(!$service->tick(),
'A converged PHP-only finalizer was claimed a second time.');
$afterSecondTick = CloudJob::findOrFail(501);
$assert((int) $afterSecondTick->updated_at === $updatedAt,
'A converged PHP-only finalizer repeated its side effects.');
// A missing/corrupt encrypted escrow remains a concrete payload incident;
// it must not degrade into a generic ledger-conflict message.
Capsule::table('cloud_instances')->insert([
'id' => 502,
'uuid' => '00000000-0000-4000-8000-000000000502',
'host_id' => 7,
'user_id' => 1,
'name' => 'local-finalizer-corrupt-payload',
'status' => 'active',
'service_status' => 'exempt',
'network_status' => 'active',
'spec_json' => json_encode(['managed' => true], JSON_THROW_ON_ERROR),
'sealed_credential' => null,
'created_at' => $now - 3600,
'updated_at' => $now - 3600,
]);
$corruptOperationId = str_repeat('ae', 16);
Capsule::table('cloud_jobs')->insert([
'id' => 502,
'operation_id' => $corruptOperationId,
'request_idempotency_key' => $corruptOperationId,
'request_fingerprint' => hash('sha256', $corruptOperationId),
'parent_job_id' => null,
'order_id' => null,
'invoice_id' => null,
'instance_id' => 502,
'snapshot_id' => null,
'user_id' => 1,
'host_id' => 7,
'kind' => 'instance_settings',
'execution_mode' => 'agent',
'phase' => 'terminal',
'outcome' => 'failed',
'status' => 'failed',
'sealed_payload' => 'undispatched-finalizer-test:missing',
'attempts' => 0,
'dispatch_attempts' => 0,
'dispatch_deadline_at' => $now - 30,
'terminal_at' => $now - 20,
'effect_may_have_applied' => 0,
'safe_to_resubmit' => 0,
'effect_resolution' => 'not_applied',
'effect_resolution_reason' => '',
'lease_until' => 0,
'available_at' => 0,
'error_code' => 'local_finalization_failed',
'finalization_incident' => 1,
'finalization_error_code' => 'recovery_payload_unavailable',
'result_json' => $originalResult,
'created_at' => $now - 30,
'updated_at' => $now - 20,
]);
$assert($service->tick(),
'The corrupt undispatched recovery payload was not checked.');
$corruptPayloadJob = CloudJob::findOrFail(502);
$assert((bool) $corruptPayloadJob->finalization_incident
&& (string) $corruptPayloadJob->finalization_error_code
=== 'recovery_payload_unavailable'
&& (string) $corruptPayloadJob->sealed_payload
=== 'undispatched-finalizer-test:missing'
&& (int) $corruptPayloadJob->lease_until >= time() + 290
&& (int) $corruptPayloadJob->available_at >= time() + 290
&& CloudJobService::blockingQueryForInstance(502)->exists(),
'A corrupt PHP-only recovery payload lost its concrete incident or resource fence.');
// A create that never crossed the dispatch boundary has definitive
// no-effect proof, but its paid ledgers still require the create-specific
// atomic finalizer. Temporary key or database damage must retain escrow
// and every fence; once repaired, the worker refunds/releases exactly
// once and never sends the original create request to an Agent.
$createOperationId = str_repeat('af', 16);
$createPayload = [
'method' => 'POST',
'request_target' => '/v1/vms',
'body' => [
'operation_id' => $createOperationId,
'source' => 'cloud',
'name' => 'undispatched-create',
'root_password' => 'Secret9!Password',
],
'credential' => [
'password' => 'Secret9!Password',
'login_name' => 'root',
'auth_mode' => 'password',
],
];
$createEnvelope = CloudSecrets::encrypt(
$createPayload,
'job-payload'
);
Capsule::table('user')->where('id', 1)->update(['money' => '0.00']);
Capsule::table('cloud_hosts')->insert(['id' => 7]);
Capsule::table('cloud_offers')->insert([
'id' => 601,
'stock' => 0,
'updated_at' => $now - 30,
]);
Capsule::table('cloud_orders')->insert([
'id' => 601,
'user_id' => 1,
'plan_id' => 601,
'offer_id' => 601,
'resource_pool_id' => 601,
'host_id' => 7,
'reservation_id' => 601,
'idempotency_key' => 'undispatched-create-order',
'request_hash' => hash('sha256', 'undispatched-create-order'),
'amount' => '12.34',
'currency' => 'USD',
'status' => 'queued',
'spec_json' => '{}',
'sealed_payload' => null,
'instance_id' => null,
'refunded_at' => null,
'stock_reserved_at' => $now - 60,
'stock_released_at' => null,
'created_at' => $now - 120,
'updated_at' => $now - 30,
]);
Capsule::table('cloud_resource_reservations')->insert([
'id' => 601,
'reservation_key' => 'undispatched-create-reservation',
'operation_id' => $createOperationId,
'order_id' => 601,
'offer_id' => 601,
'resource_pool_id' => 601,
'host_id' => 7,
'user_id' => 1,
'instance_id' => null,
'network_attachment_id' => 601,
'public_ip_pool_id' => 601,
'public_ip_id' => 601,
'cpu_cores' => 1,
'memory_mib' => 512,
'disk_gib' => 10,
'state' => 'held',
'expires_at' => null,
'released_at' => null,
'release_reason' => '',
'created_at' => $now - 120,
'updated_at' => $now - 30,
]);
Capsule::table('cloud_public_ips')->insert([
'id' => 601,
'host_id' => 7,
'public_ip_pool_id' => 601,
'address' => '203.0.113.61',
'prefix_length' => 24,
'state' => 'reserved',
'reservation_id' => 601,
'instance_id' => null,
'operation_id' => $createOperationId,
'created_at' => $now - 120,
'updated_at' => $now - 30,
]);
Capsule::table('cloud_jobs')->insert([
'id' => 601,
'operation_id' => $createOperationId,
'request_idempotency_key' => $createOperationId,
'request_fingerprint' => hash('sha256', $createOperationId),
'parent_job_id' => null,
'order_id' => 601,
'invoice_id' => null,
'instance_id' => null,
'snapshot_id' => null,
'user_id' => 1,
'host_id' => 7,
'kind' => 'create',
'execution_mode' => 'agent',
'phase' => 'accepted',
'outcome' => null,
'status' => 'queued',
'sealed_payload' => $createEnvelope,
'attempts' => 0,
'dispatch_attempts' => 0,
'dispatch_deadline_at' => $now - 30,
'dispatch_started_at' => null,
'agent_accepted_at' => null,
'receipt_deadline_at' => null,
'terminal_at' => null,
'effect_may_have_applied' => null,
'safe_to_resubmit' => null,
'effect_resolution' => 'not_applicable',
'effect_resolved_at' => null,
'effect_resolved_by' => null,
'effect_resolution_reason' => '',
'completion_event_id' => null,
'read_at' => null,
'dispatched_at' => null,
'lease_token' => str_repeat('6', 32),
'lease_until' => $now + 180,
'available_at' => 0,
'error_code' => null,
'finalization_incident' => 0,
'finalization_error_code' => null,
'result_json' => $originalResult,
'created_at' => $now - 30,
'updated_at' => $now - 20,
]);
CloudSecrets::setAvailable(false);
$forceTerminalFailure = new ReflectionMethod(
CloudWorkerService::class,
'forceTerminalFailure'
);
$forceTerminalFailure->invoke(
null,
601,
str_repeat('6', 32),
false,
true,
'dispatch_not_sent'
);
$createAfterEmergency = CloudJob::findOrFail(601);
$assert((string) $createAfterEmergency->phase === 'terminal'
&& (string) $createAfterEmergency->outcome === 'failed'
&& (int) $createAfterEmergency->dispatch_attempts === 0
&& !(bool) $createAfterEmergency->effect_may_have_applied
&& !(bool) $createAfterEmergency->safe_to_resubmit
&& (string) $createAfterEmergency->effect_resolution
=== 'not_applied'
&& (bool) $createAfterEmergency->finalization_incident
&& (string) $createAfterEmergency->finalization_error_code
=== 'recovery_payload_unavailable'
&& (string) $createAfterEmergency->sealed_payload === $createEnvelope
&& (int) $createAfterEmergency->available_at >= time() + 290,
'Create emergency terminalization lost its generation-zero proof or escrow.');
Capsule::table('cloud_jobs')->where('id', 601)->update([
'available_at' => 0,
'lease_token' => null,
'lease_until' => 0,
]);
$createService = new CloudAutomaticOperationRecoveryService();
$assert($createService->tick(),
'An undispatched create with unavailable escrow was not claimed.');
$createAfterKeyFailure = CloudJob::findOrFail(601);
$assert((bool) $createAfterKeyFailure->finalization_incident
&& (string) $createAfterKeyFailure->finalization_error_code
=== 'recovery_payload_unavailable'
&& (string) $createAfterKeyFailure->sealed_payload
=== $createEnvelope
&& (string) Capsule::table('user')->where('id', 1)
->value('money') === '0.00'
&& (string) Capsule::table('cloud_resource_reservations')
->where('id', 601)->value('state') === 'held'
&& CloudAgentSettingsService::$requests === 0,
'Unavailable create escrow released money/resources or contacted Agent.');
CloudSecrets::setAvailable(true);
Capsule::table('cloud_jobs')->where('id', 601)->update([
'available_at' => 0,
'lease_token' => null,
'lease_until' => 0,
]);
$assert($createService->tick(),
'An undispatched create with a missing invoice was not retried.');
$createAfterLedgerFailure = CloudJob::findOrFail(601);
$assert((bool) $createAfterLedgerFailure->finalization_incident
&& (string) Capsule::table('user')->where('id', 1)
->value('money') === '0.00'
&& (string) Capsule::table('cloud_resource_reservations')
->where('id', 601)->value('state') === 'held'
&& (string) Capsule::table('cloud_public_ips')
->where('id', 601)->value('state') === 'reserved'
&& CloudAgentSettingsService::$requests === 0,
'Missing create ledger data caused a partial refund/release or Agent call.');
Capsule::table('cloud_invoices')->insert([
'id' => 601,
'invoice_no' => 'VC-UNDISPATCHED-601',
'purpose' => 'provision',
'instance_id' => null,
'order_id' => 601,
'user_id' => 1,
'plan_id' => 601,
'offer_id' => 601,
'renewal_id' => null,
'renews_expires_at' => null,
'due_at' => $now - 120,
'amount' => '12.34',
'currency' => 'USD',
'validity_days' => 30,
'traffic_quota_bytes' => 0,
'status' => 'paid',
'paid_via' => 'balance',
'payment_reference' => 'undispatched-create-payment',
'paid_at' => $now - 60,
'created_at' => $now - 120,
'updated_at' => $now - 60,
]);
Capsule::table('cloud_jobs')->where('id', 601)->update([
'available_at' => 0,
'lease_token' => null,
'lease_until' => 0,
]);
$beforeCreateAudit = CloudJob::findOrFail(601);
$assert($createService->tick(),
'A repaired undispatched create was not finalized locally.');
$createAfterRecovery = CloudJob::findOrFail(601);
$createSafePayload = CloudSecrets::decrypt(
(string) $createAfterRecovery->sealed_payload,
'job-payload'
);
$assert((string) $createAfterRecovery->phase
=== (string) $beforeCreateAudit->phase
&& (string) $createAfterRecovery->outcome
=== (string) $beforeCreateAudit->outcome
&& (int) $createAfterRecovery->terminal_at
=== (int) $beforeCreateAudit->terminal_at
&& (string) $createAfterRecovery->error_code
=== (string) $beforeCreateAudit->error_code
&& !(bool) $createAfterRecovery->effect_may_have_applied
&& !(bool) $createAfterRecovery->safe_to_resubmit
&& (string) $createAfterRecovery->effect_resolution === 'not_applied'
&& (string) $createAfterRecovery->effect_resolution_reason
=== 'automatic_local_not_dispatched'
&& !(bool) $createAfterRecovery->finalization_incident
&& $createAfterRecovery->finalization_error_code === null
&& !isset($createSafePayload['credential'])
&& !isset($createSafePayload['body']['root_password'])
&& (string) Capsule::table('user')->where('id', 1)
->value('money') === '12.34'
&& (string) Capsule::table('cloud_invoices')->where('id', 601)
->value('status') === 'refunded'
&& Capsule::table('cloud_orders')->where('id', 601)
->value('refunded_at') !== null
&& Capsule::table('cloud_orders')->where('id', 601)
->value('stock_released_at') !== null
&& (int) Capsule::table('cloud_offers')->where('id', 601)
->value('stock') === 1
&& (string) Capsule::table('cloud_resource_reservations')
->where('id', 601)->value('state') === 'released'
&& (string) Capsule::table('cloud_public_ips')
->where('id', 601)->value('state') === 'available'
&& Capsule::table('cloud_public_ips')->where('id', 601)
->value('reservation_id') === null
&& CloudAgentSettingsService::$requests === 0,
'Recovered create did not atomically refund, release, scrub, and close.');
$createUpdatedAt = (int) $createAfterRecovery->updated_at;
$assert(!$createService->tick(),
'A converged undispatched create was claimed a second time.');
$assert((string) Capsule::table('user')->where('id', 1)
->value('money') === '12.34'
&& (int) Capsule::table('cloud_offers')->where('id', 601)
->value('stock') === 1
&& (int) CloudJob::findOrFail(601)->updated_at === $createUpdatedAt
&& CloudAgentSettingsService::$requests === 0,
'Undispatched create recovery repeated refund/release or contacted Agent.');
// The dispatch CAS may already be generation one when an exact Agent
// validation response proves that its operation journal was never entered.
// If PHP's catalog finalizer then fails, preserve that proof without
// rewriting the immutable counter, recover only the local ledger, and
// allow a new same-name import generation afterward.
$transportOperationId = str_repeat('ba', 16);
$transportTemplateId = str_repeat('c', 32);
$transportName = 'transport-not-dispatched-rescue';
$transportBytes = 2 * 1024 * 1024;
$transportPayload = [
'host_id' => 7,
'import_type' => 'upload',
'name' => $transportName,
'source' => '',
'sha256' => '',
'source_size_bytes' => $transportBytes,
'can_manage' => true,
'rescue_template_id' => $transportTemplateId,
'method' => 'POST',
'request_target' => '/v1/rescue-templates',
'body' => [
'operation_id' => $transportOperationId,
'id' => $transportTemplateId,
'name' => $transportName,
'source_url' => 'https://panel.example.test/staging/rescue.iso',
'source_size_bytes' => $transportBytes,
'sha256' => '',
],
'owner_id' => 1,
];
$transportEnvelope = CloudSecrets::encrypt(
$transportPayload,
'job-payload'
);
Capsule::table('cloud_jobs')->insert([
'id' => 701,
'operation_id' => $transportOperationId,
'request_idempotency_key' => $transportOperationId,
'request_fingerprint' => hash('sha256', $transportOperationId),
'parent_job_id' => null,
'order_id' => null,
'invoice_id' => null,
'instance_id' => null,
'snapshot_id' => null,
'user_id' => 1,
'host_id' => 7,
'kind' => 'rescue_template_import',
'execution_mode' => 'agent',
'phase' => 'dispatching',
'outcome' => null,
'status' => 'running',
'sealed_payload' => $transportEnvelope,
'attempts' => 1,
'dispatch_attempts' => 1,
'dispatch_deadline_at' => $now + 60,
'dispatch_started_at' => $now,
'agent_accepted_at' => null,
'receipt_deadline_at' => $now + 300,
'terminal_at' => null,
'effect_may_have_applied' => null,
'safe_to_resubmit' => null,
'effect_resolution' => 'unknown',
'effect_resolved_at' => null,
'effect_resolved_by' => null,
'effect_resolution_reason' => '',
'completion_event_id' => null,
'read_at' => null,
'dispatched_at' => $now,
'lease_token' => str_repeat('7', 32),
'lease_until' => $now + 180,
'available_at' => 0,
'error_code' => null,
'finalization_incident' => 0,
'finalization_error_code' => null,
'result_json' => null,
'created_at' => $now - 1,
'updated_at' => $now,
]);
$forceTerminalFailure->invoke(
null,
701,
str_repeat('7', 32),
false,
true,
'local_finalization_failed',
true
);
$transportIncident = CloudJob::findOrFail(701);
$assert((int) $transportIncident->dispatch_attempts === 1
&& CloudDispatchLedger::hasTransportNotDispatchedProof(
$transportIncident
)
&& (string) $transportIncident->sealed_payload === $transportEnvelope
&& (int) $transportIncident->available_at >= time() + 290
&& CloudAgentSettingsService::$requests === 0,
'A strict transport not-dispatched response lost its proof or immutable generation.');
Capsule::table('cloud_rescue_template_imports')->insert([
'operation_id' => $transportOperationId,
'host_id' => 7,
'template_id' => $transportTemplateId,
'job_id' => 701,
'import_type' => 'upload',
'name' => $transportName,
'size_bytes' => 0,
'sha256' => '',
'state' => 'importing',
'error_code' => '',
'created_at' => $now,
'updated_at' => $now,
]);
Capsule::table('cloud_jobs')->where('id', 701)->update([
'available_at' => 0,
'lease_token' => null,
'lease_until' => 0,
]);
$transportService = new CloudAutomaticOperationRecoveryService();
$assert($transportService->tick(),
'The strict transport not-dispatched finalizer was not claimed.');
$transportRecovered = CloudJob::findOrFail(701);
$assert((int) $transportRecovered->dispatch_attempts === 1
&& !(bool) $transportRecovered->effect_may_have_applied
&& (bool) $transportRecovered->safe_to_resubmit
&& (string) $transportRecovered->effect_resolution === 'not_applied'
&& (string) $transportRecovered->effect_resolution_reason
=== 'automatic_local_not_dispatched'
&& !(bool) $transportRecovered->finalization_incident
&& $transportRecovered->finalization_error_code === null
&& (string) CloudRescueTemplateImport::query()
->where('operation_id', $transportOperationId)
->value('state') === 'failed'
&& CloudAgentSettingsService::$requests === 0,
'Transport not-dispatched recovery did not converge the rescue ledger locally.');
$retryActor = [
'user_id' => 1,
'is_admin' => true,
'is_super_admin' => true,
'can_manage' => true,
];
$transportRetry = CloudRescueTemplateImportService::start(
7,
'upload',
$transportName,
'https://panel.example.test/staging/rescue-retry.iso',
'',
'',
$transportBytes,
$retryActor
);
$assert((int) $transportRetry['task_id'] !== 701
&& CloudRescueTemplateImport::query()
->where('host_id', 7)
->where('name', $transportName)
->count() === 2
&& CloudAgentSettingsService::$requests === 0,
'A locally converged rescue import still blocked its same-name retry.');
echo "Cloud undispatched finalizer recovery tests passed.\n";
}