You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
virt/web/tests/CloudUndispatchedFinalizerR...

965 lines
39 KiB
PHP

<?php
declare(strict_types=1);
namespace App\Services\Cloud {
/** Isolated encrypted envelope for PHP-only finalizer recovery tests. */
final class CloudSecrets
{
private static array $values = [];
private static bool $available = true;
public static function setAvailable(bool $available): void
{
self::$available = $available;
}
public static function encrypt(array $value, string $purpose): string
{
$key = bin2hex(random_bytes(16));
self::$values[$key] = $value;
return 'undispatched-finalizer-test:' . $key;
}
public static function decrypt(string $value, string $purpose): array
{
if (!self::$available) {
throw new \RuntimeException(
'Finalizer test key is temporarily unavailable.'
);
}
$key = substr(
$value,
strlen('undispatched-finalizer-test:')
);
if (!array_key_exists($key, self::$values)) {
throw new \RuntimeException('Missing finalizer test payload.');
}
return self::$values[$key];
}
}
/** Any Agent lookup is a test failure for generation-zero recovery. */
final class CloudAgentSettingsService
{
public static int $requests = 0;
public static function clientForHost(
int $hostId,
float $timeoutSeconds = 120.0
): object {
self::$requests++;
throw new \RuntimeException(
'Undispatched finalizer recovery contacted the Agent.'
);
}
public static function clientForConfiguration(
int $hostId,
float $timeoutSeconds = 120.0
): object {
return self::clientForHost($hostId, $timeoutSeconds);
}
}
}
namespace {
require dirname(__DIR__) . '/vendor/autoload.php';
use App\Models\CloudJob;
use App\Services\Cloud\CloudAutomaticOperationRecoveryService;
use App\Services\Cloud\CloudAgentSettingsService;
use App\Services\Cloud\CloudDispatchLedger;
use App\Services\Cloud\CloudJobService;
use App\Services\Cloud\CloudSecrets;
use App\Services\Cloud\CloudWorkerService;
use Illuminate\Database\Capsule\Manager as Capsule;
use Illuminate\Database\Schema\Blueprint;
$assert = static function (bool $condition, string $message): void {
if (!$condition) {
throw new RuntimeException($message);
}
};
$capsule = new Capsule();
$capsule->addConnection([
'driver' => 'sqlite', 'database' => ':memory:',
], 'default');
$capsule->setAsGlobal();
$capsule->bootEloquent();
$schema = $capsule->schema('default');
$schema->create('user', static function (Blueprint $table): void {
$table->increments('id');
$table->string('money')->default('0.00');
$table->integer('enable')->default(1);
$table->integer('is_admin')->default(0);
});
$schema->create('cloud_jobs', static function (Blueprint $table): void {
$table->increments('id');
$table->string('operation_id')->unique();
$table->string('request_idempotency_key')->unique();
$table->string('request_fingerprint');
$table->integer('parent_job_id')->nullable();
$table->integer('order_id')->nullable();
$table->integer('invoice_id')->nullable();
$table->integer('instance_id')->nullable();
$table->integer('snapshot_id')->nullable();
$table->integer('public_network_transition_id')->nullable();
$table->integer('vpc_repair_root_job_id')->nullable();
$table->integer('user_id');
$table->integer('host_id')->nullable();
$table->string('kind');
$table->string('execution_mode');
$table->string('phase');
$table->string('outcome')->nullable();
$table->string('status');
$table->text('sealed_payload');
$table->integer('attempts')->default(0);
$table->integer('dispatch_attempts')->default(0);
$table->integer('dispatch_deadline_at')->default(0);
$table->integer('dispatch_started_at')->nullable();
$table->integer('agent_accepted_at')->nullable();
$table->integer('receipt_deadline_at')->nullable();
$table->integer('terminal_at')->nullable();
$table->boolean('effect_may_have_applied')->nullable();
$table->boolean('safe_to_resubmit')->nullable();
$table->string('effect_resolution');
$table->integer('effect_resolved_at')->nullable();
$table->integer('effect_resolved_by')->nullable();
$table->string('effect_resolution_reason')->default('');
$table->string('completion_event_id')->nullable();
$table->integer('read_at')->nullable();
$table->integer('dispatched_at')->nullable();
$table->string('lease_token')->nullable();
$table->integer('lease_until')->default(0);
$table->integer('available_at')->default(0);
$table->string('error_code')->nullable();
$table->boolean('finalization_incident')->default(0);
$table->string('finalization_error_code')->nullable();
$table->text('result_json')->nullable();
$table->integer('created_at');
$table->integer('updated_at');
});
$schema->create('cloud_backup_restore_attempts',
static function (Blueprint $table): void {
$table->increments('id');
$table->integer('job_id');
$table->string('state');
$table->string('effect_resolution');
$table->integer('active_attempt')->nullable();
$table->string('last_error_code')->default('');
});
$schema->create('cloud_public_network_transitions',
static function (Blueprint $table): void {
$table->increments('id');
$table->string('state');
$table->string('effect_resolution');
$table->string('last_error_code')->default('');
});
$schema->create('cloud_instances', static function (Blueprint $table): void {
$table->increments('id');
$table->string('uuid');
$table->integer('host_id');
$table->integer('user_id');
$table->integer('order_id')->nullable();
$table->string('name');
$table->string('status');
$table->string('service_status');
$table->string('network_status');
$table->text('spec_json');
$table->text('sealed_credential')->nullable();
$table->integer('created_at');
$table->integer('updated_at');
});
$schema->create('cloud_agent_completion_events',
static function (Blueprint $table): void {
$table->string('event_id')->primary();
$table->integer('host_id');
$table->string('operation_id');
$table->string('payload_sha256');
$table->string('outcome');
$table->boolean('effect_may_have_applied');
$table->boolean('safe_to_resubmit');
$table->string('error_code')->default('');
$table->text('result_json')->nullable();
$table->string('disposition')->default('received');
$table->integer('received_at');
$table->integer('applied_at')->nullable();
$table->unique(['host_id', 'operation_id']);
});
$schema->create('cloud_templates',
static function (Blueprint $table): void {
$table->string('id')->primary();
$table->string('catalog_key')->default('test-catalog');
$table->string('version_label')->default('test-version');
$table->boolean('is_default')->default(true);
$table->integer('active_job_id')->nullable();
$table->string('state')->default('active');
});
$schema->create('cloud_user_isos',
static function (Blueprint $table): void {
$table->string('id')->primary();
$table->integer('active_job_id')->nullable();
$table->string('state')->default('active');
});
$schema->create('cloud_template_replicas',
static function (Blueprint $table): void {
$table->integer('host_id');
$table->string('template_id');
$table->integer('revision')->default(0);
$table->string('generation_token')->default('');
$table->string('state')->default('pending');
$table->integer('image_bytes')->default(0);
$table->string('sha256')->default('');
$table->string('operation_id')->default('');
$table->integer('job_id')->nullable();
$table->string('error_code')->default('');
$table->integer('created_at')->default(0);
$table->integer('updated_at')->default(0);
$table->primary(['host_id', 'template_id']);
});
$schema->create('cloud_user_iso_replicas',
static function (Blueprint $table): void {
$table->integer('host_id');
$table->string('user_iso_id');
$table->integer('user_id');
$table->string('state')->default('pending');
$table->integer('size_bytes')->default(0);
$table->string('sha256')->default('');
$table->string('operation_id')->default('');
$table->integer('job_id')->nullable();
$table->string('error_code')->default('');
$table->integer('created_at')->default(0);
$table->integer('updated_at')->default(0);
$table->primary(['host_id', 'user_iso_id']);
});
$schema->create('cloud_hosts', static function (Blueprint $table): void {
$table->increments('id');
});
$schema->create('cloud_host_network_attachments',
static function (Blueprint $table): void {
$table->increments('id');
$table->integer('host_id');
$table->integer('line_id');
$table->string('network_selector');
$table->string('state');
$table->integer('priority')->default(0);
});
$schema->create('cloud_public_ip_pools',
static function (Blueprint $table): void {
$table->increments('id');
$table->string('pool_key');
$table->integer('host_id');
$table->integer('network_attachment_id');
$table->integer('address_family');
$table->string('cidr');
$table->string('gateway');
$table->boolean('gateway_on_link');
$table->integer('guest_prefix_length');
$table->text('dns_servers_json');
$table->text('reserved_ips_json');
$table->string('allocation_start')->nullable();
$table->string('allocation_end')->nullable();
$table->string('allocation_cursor')->nullable();
$table->integer('max_allocations');
$table->string('state');
$table->integer('priority')->default(0);
$table->integer('desired_revision');
$table->integer('observed_revision');
$table->string('sync_state');
});
$schema->create('cloud_offers', static function (Blueprint $table): void {
$table->increments('id');
$table->integer('stock')->default(-1);
$table->integer('updated_at')->default(0);
});
$schema->create('cloud_orders', static function (Blueprint $table): void {
$table->increments('id');
$table->integer('user_id');
$table->integer('plan_id');
$table->integer('offer_id');
$table->integer('resource_pool_id');
$table->integer('host_id')->nullable();
$table->integer('reservation_id')->nullable();
$table->string('idempotency_key');
$table->string('request_hash');
$table->string('amount');
$table->string('currency')->default('USD');
$table->string('status');
$table->text('spec_json');
$table->text('sealed_payload')->nullable();
$table->integer('instance_id')->nullable();
$table->integer('refunded_at')->nullable();
$table->integer('stock_reserved_at')->nullable();
$table->integer('stock_released_at')->nullable();
$table->integer('created_at');
$table->integer('updated_at');
});
$schema->create('cloud_resource_reservations',
static function (Blueprint $table): void {
$table->increments('id');
$table->string('reservation_key');
$table->string('operation_id');
$table->integer('order_id');
$table->integer('backup_restore_id')->nullable();
$table->integer('offer_id');
$table->integer('resource_pool_id');
$table->integer('host_id');
$table->integer('user_id');
$table->integer('instance_id')->nullable();
$table->integer('network_attachment_id')->nullable();
$table->integer('public_ip_pool_id')->nullable();
$table->integer('public_ip_id')->nullable();
$table->integer('cpu_cores');
$table->integer('memory_mib');
$table->integer('disk_gib');
$table->string('state');
$table->integer('expires_at')->nullable();
$table->integer('released_at')->nullable();
$table->string('release_reason')->default('');
$table->integer('created_at');
$table->integer('updated_at');
});
$schema->create('cloud_public_ips',
static function (Blueprint $table): void {
$table->increments('id');
$table->integer('host_id');
$table->integer('public_ip_pool_id')->nullable();
$table->integer('address_family');
$table->string('address');
$table->binary('address_binary');
$table->integer('prefix_length');
$table->string('state');
$table->integer('reservation_id')->nullable();
$table->integer('reservation_ip_claim_id')->nullable();
$table->integer('instance_id')->nullable();
$table->string('operation_id')->nullable();
$table->integer('created_at');
$table->integer('updated_at');
});
$schema->create('cloud_resource_reservation_ips',
static function (Blueprint $table): void {
$table->increments('id');
$table->integer('reservation_id');
$table->integer('address_family');
$table->integer('claim_generation')->default(1);
$table->integer('active_claim')->nullable()->default(1);
$table->integer('network_attachment_id');
$table->integer('public_ip_pool_id');
$table->integer('public_ip_id');
$table->string('operation_id');
$table->string('state');
$table->integer('created_at');
$table->integer('updated_at');
$table->integer('released_at')->nullable();
$table->string('release_reason')->default('');
});
$schema->create('cloud_invoices',
static function (Blueprint $table): void {
$table->increments('id');
$table->string('invoice_no');
$table->string('purpose');
$table->integer('instance_id')->nullable();
$table->integer('order_id')->nullable();
$table->integer('user_id');
$table->integer('plan_id')->nullable();
$table->integer('offer_id')->nullable();
$table->integer('renewal_id')->nullable();
$table->integer('renews_expires_at')->nullable();
$table->integer('due_at');
$table->string('amount');
$table->string('currency')->default('USD');
$table->integer('validity_days');
$table->integer('traffic_quota_bytes');
$table->string('status');
$table->string('paid_via')->nullable();
$table->string('payment_reference')->nullable();
$table->integer('paid_at')->nullable();
$table->integer('created_at');
$table->integer('updated_at');
});
$now = time();
$operationId = str_repeat('ad', 16);
$uuid = '00000000-0000-4000-8000-000000000501';
$payload = [
'method' => 'POST',
'request_target' => '/v1/vms/' . $uuid . '/settings',
'body' => [
'hostname' => 'local-finalizer.example',
'root_password' => 'Secret9!Password',
'confirm_restart' => true,
],
'owner_id' => 1,
'can_manage' => false,
'credential' => [
'password' => 'Secret9!Password',
'login_name' => 'root',
'auth_mode' => 'password',
],
'settings_fields' => ['hostname', 'password'],
];
Capsule::table('user')->insert([
'id' => 1, 'enable' => 1, 'is_admin' => 0,
]);
Capsule::table('cloud_instances')->insert([
'id' => 501,
'uuid' => $uuid,
'host_id' => 7,
'user_id' => 1,
'name' => 'local-finalizer',
'status' => 'active',
'service_status' => 'exempt',
'network_status' => 'active',
'spec_json' => json_encode(
['managed' => true],
JSON_THROW_ON_ERROR
),
'sealed_credential' => null,
'created_at' => $now - 3600,
'updated_at' => $now - 3600,
]);
$originalEnvelope = CloudSecrets::encrypt($payload, 'job-payload');
$originalResult = json_encode(
['cause_code' => 'local_finalization_failed'],
JSON_THROW_ON_ERROR
);
Capsule::table('cloud_jobs')->insert([
'id' => 501,
'operation_id' => $operationId,
'request_idempotency_key' => $operationId,
'request_fingerprint' => hash('sha256', $operationId),
'parent_job_id' => null,
'order_id' => null,
'invoice_id' => null,
'instance_id' => 501,
'snapshot_id' => null,
'user_id' => 1,
'host_id' => 7,
'kind' => 'instance_settings',
'execution_mode' => 'agent',
'phase' => 'terminal',
'outcome' => 'failed',
'status' => 'failed',
'sealed_payload' => $originalEnvelope,
'attempts' => 0,
'dispatch_attempts' => 0,
'dispatch_deadline_at' => $now - 30,
'terminal_at' => $now - 20,
'effect_may_have_applied' => 0,
'safe_to_resubmit' => 0,
'effect_resolution' => 'not_applied',
'effect_resolution_reason' => '',
'lease_until' => 0,
'available_at' => 0,
'error_code' => 'local_finalization_failed',
'finalization_incident' => 1,
'finalization_error_code' => 'recovery_ledger_conflict',
'result_json' => $originalResult,
'created_at' => $now - 30,
'updated_at' => $now - 20,
]);
$before = CloudJob::findOrFail(501);
$assert(CloudJobService::blockingQueryForInstance(501)->exists(),
'An undispatched local finalizer incident did not block mutations.');
$service = new CloudAutomaticOperationRecoveryService();
$assert($service->tick(),
'The undispatched PHP-only finalizer was not claimed.');
$after = CloudJob::findOrFail(501);
$scrubbed = CloudSecrets::decrypt(
(string) $after->sealed_payload,
'job-payload'
);
$assert((string) $after->phase === (string) $before->phase
&& (string) $after->outcome === (string) $before->outcome
&& (string) $after->status === (string) $before->status
&& (int) $after->terminal_at === (int) $before->terminal_at
&& (string) $after->error_code === (string) $before->error_code
&& (string) $after->result_json === (string) $before->result_json
&& $after->completion_event_id === $before->completion_event_id
&& !(bool) $after->effect_may_have_applied
&& (bool) $after->safe_to_resubmit
&& (string) $after->effect_resolution === 'not_applied'
&& (string) $after->effect_resolution_reason
=== 'automatic_local_not_dispatched'
&& (int) $after->effect_resolved_at > 0
&& !(bool) $after->finalization_incident
&& $after->finalization_error_code === null
&& $after->lease_token === null
&& (int) $after->lease_until === 0
&& (int) $after->available_at === 0
&& (string) $after->sealed_payload !== $originalEnvelope
&& !isset($scrubbed['credential'])
&& !isset($scrubbed['body']['root_password'])
&& !CloudJobService::blockingQueryForInstance(501)->exists(),
'The PHP-only finalizer changed terminal audit data or failed to converge safely.');
$updatedAt = (int) $after->updated_at;
$assert(!$service->tick(),
'A converged PHP-only finalizer was claimed a second time.');
$afterSecondTick = CloudJob::findOrFail(501);
$assert((int) $afterSecondTick->updated_at === $updatedAt,
'A converged PHP-only finalizer repeated its side effects.');
// A missing/corrupt encrypted escrow remains a concrete payload incident;
// it must not degrade into a generic ledger-conflict message.
Capsule::table('cloud_instances')->insert([
'id' => 502,
'uuid' => '00000000-0000-4000-8000-000000000502',
'host_id' => 7,
'user_id' => 1,
'name' => 'local-finalizer-corrupt-payload',
'status' => 'active',
'service_status' => 'exempt',
'network_status' => 'active',
'spec_json' => json_encode(['managed' => true], JSON_THROW_ON_ERROR),
'sealed_credential' => null,
'created_at' => $now - 3600,
'updated_at' => $now - 3600,
]);
$corruptOperationId = str_repeat('ae', 16);
Capsule::table('cloud_jobs')->insert([
'id' => 502,
'operation_id' => $corruptOperationId,
'request_idempotency_key' => $corruptOperationId,
'request_fingerprint' => hash('sha256', $corruptOperationId),
'parent_job_id' => null,
'order_id' => null,
'invoice_id' => null,
'instance_id' => 502,
'snapshot_id' => null,
'user_id' => 1,
'host_id' => 7,
'kind' => 'instance_settings',
'execution_mode' => 'agent',
'phase' => 'terminal',
'outcome' => 'failed',
'status' => 'failed',
'sealed_payload' => 'undispatched-finalizer-test:missing',
'attempts' => 0,
'dispatch_attempts' => 0,
'dispatch_deadline_at' => $now - 30,
'terminal_at' => $now - 20,
'effect_may_have_applied' => 0,
'safe_to_resubmit' => 0,
'effect_resolution' => 'not_applied',
'effect_resolution_reason' => '',
'lease_until' => 0,
'available_at' => 0,
'error_code' => 'local_finalization_failed',
'finalization_incident' => 1,
'finalization_error_code' => 'recovery_payload_unavailable',
'result_json' => $originalResult,
'created_at' => $now - 30,
'updated_at' => $now - 20,
]);
$assert($service->tick(),
'The corrupt undispatched recovery payload was not checked.');
$corruptPayloadJob = CloudJob::findOrFail(502);
$assert((bool) $corruptPayloadJob->finalization_incident
&& (string) $corruptPayloadJob->finalization_error_code
=== 'recovery_payload_unavailable'
&& (string) $corruptPayloadJob->sealed_payload
=== 'undispatched-finalizer-test:missing'
&& (int) $corruptPayloadJob->lease_until >= time() + 290
&& (int) $corruptPayloadJob->available_at >= time() + 290
&& CloudJobService::blockingQueryForInstance(502)->exists(),
'A corrupt PHP-only recovery payload lost its concrete incident or resource fence.');
// A create that never crossed the dispatch boundary has definitive
// no-effect proof, but its paid ledgers still require the create-specific
// atomic finalizer. Temporary key or database damage must retain escrow
// and every fence; once repaired, the worker refunds/releases exactly
// once and never sends the original create request to an Agent.
$createOperationId = str_repeat('af', 16);
$createPayload = [
'method' => 'POST',
'request_target' => '/v1/vms',
'body' => [
'operation_id' => $createOperationId,
'source' => 'cloud',
'name' => 'undispatched-create',
'root_password' => 'Secret9!Password',
],
'credential' => [
'password' => 'Secret9!Password',
'login_name' => 'root',
'auth_mode' => 'password',
],
];
$createEnvelope = CloudSecrets::encrypt(
$createPayload,
'job-payload'
);
Capsule::table('user')->where('id', 1)->update(['money' => '0.00']);
Capsule::table('cloud_hosts')->insert(['id' => 7]);
Capsule::table('cloud_host_network_attachments')->insert([
'id' => 601,
'host_id' => 7,
'line_id' => 601,
'network_selector' => 'bridge:br0',
'state' => 'active',
'priority' => 0,
]);
$publicPool = [
'host_id' => 7,
'network_attachment_id' => 601,
'gateway_on_link' => 0,
'reserved_ips_json' => '[]',
'allocation_cursor' => null,
'state' => 'active',
'priority' => 0,
'desired_revision' => 1,
'observed_revision' => 1,
'sync_state' => 'applied',
];
Capsule::table('cloud_public_ip_pools')->insert([
array_merge($publicPool, [
'id' => 601,
'pool_key' => str_repeat('1', 32),
'address_family' => 4,
'cidr' => '8.8.8.0/24',
'gateway' => '8.8.8.1',
'guest_prefix_length' => 24,
'dns_servers_json' => '["1.1.1.1"]',
'allocation_start' => null,
'allocation_end' => null,
'max_allocations' => 0,
]),
array_merge($publicPool, [
'id' => 602,
'pool_key' => str_repeat('2', 32),
'address_family' => 6,
'cidr' => '2404:c140:156:3a::/64',
'gateway' => '2404:c140:156::1',
'gateway_on_link' => 1,
'guest_prefix_length' => 64,
'dns_servers_json' => '["2606:4700:4700::1111"]',
'allocation_start' => '2404:c140:156:3a::10',
'allocation_end' => '2404:c140:156:3a::ff',
'max_allocations' => 8,
]),
]);
Capsule::table('cloud_offers')->insert([
'id' => 601,
'stock' => 0,
'updated_at' => $now - 30,
]);
Capsule::table('cloud_orders')->insert([
'id' => 601,
'user_id' => 1,
'plan_id' => 601,
'offer_id' => 601,
'resource_pool_id' => 601,
'host_id' => 7,
'reservation_id' => 601,
'idempotency_key' => 'undispatched-create-order',
'request_hash' => hash('sha256', 'undispatched-create-order'),
'amount' => '12.34',
'currency' => 'USD',
'status' => 'queued',
'spec_json' => '{}',
'sealed_payload' => null,
'instance_id' => null,
'refunded_at' => null,
'stock_reserved_at' => $now - 60,
'stock_released_at' => null,
'created_at' => $now - 120,
'updated_at' => $now - 30,
]);
Capsule::table('cloud_resource_reservations')->insert([
'id' => 601,
'reservation_key' => 'undispatched-create-reservation',
'operation_id' => $createOperationId,
'order_id' => 601,
'backup_restore_id' => null,
'offer_id' => 601,
'resource_pool_id' => 601,
'host_id' => 7,
'user_id' => 1,
'instance_id' => null,
'network_attachment_id' => null,
'public_ip_pool_id' => null,
'public_ip_id' => null,
'cpu_cores' => 1,
'memory_mib' => 512,
'disk_gib' => 10,
'state' => 'held',
'expires_at' => null,
'released_at' => null,
'release_reason' => '',
'created_at' => $now - 120,
'updated_at' => $now - 30,
]);
Capsule::table('cloud_public_ips')->insert([
[
'id' => 601,
'host_id' => 7,
'public_ip_pool_id' => 601,
'address_family' => 4,
'address' => '8.8.8.61',
'address_binary' => inet_pton('8.8.8.61'),
'prefix_length' => 24,
'state' => 'reserved',
'reservation_id' => 601,
'reservation_ip_claim_id' => 601,
'instance_id' => null,
'operation_id' => $createOperationId,
'created_at' => $now - 120,
'updated_at' => $now - 30,
],
[
'id' => 602,
'host_id' => 7,
'public_ip_pool_id' => 602,
'address_family' => 6,
'address' => '2404:c140:156:3a::61',
'address_binary' => inet_pton('2404:c140:156:3a::61'),
'prefix_length' => 64,
'state' => 'reserved',
'reservation_id' => 601,
'reservation_ip_claim_id' => 602,
'instance_id' => null,
'operation_id' => $createOperationId,
'created_at' => $now - 120,
'updated_at' => $now - 30,
],
]);
Capsule::table('cloud_resource_reservation_ips')->insert([
[
'id' => 601,
'reservation_id' => 601,
'address_family' => 4,
'claim_generation' => 1,
'active_claim' => 1,
'network_attachment_id' => 601,
'public_ip_pool_id' => 601,
'public_ip_id' => 601,
'operation_id' => $createOperationId,
'state' => 'reserved',
'created_at' => $now - 120,
'updated_at' => $now - 30,
'released_at' => null,
'release_reason' => '',
],
[
'id' => 602,
'reservation_id' => 601,
'address_family' => 6,
'claim_generation' => 1,
'active_claim' => 1,
'network_attachment_id' => 601,
'public_ip_pool_id' => 602,
'public_ip_id' => 602,
'operation_id' => $createOperationId,
'state' => 'reserved',
'created_at' => $now - 120,
'updated_at' => $now - 30,
'released_at' => null,
'release_reason' => '',
],
]);
Capsule::table('cloud_jobs')->insert([
'id' => 601,
'operation_id' => $createOperationId,
'request_idempotency_key' => $createOperationId,
'request_fingerprint' => hash('sha256', $createOperationId),
'parent_job_id' => null,
'order_id' => 601,
'invoice_id' => null,
'instance_id' => null,
'snapshot_id' => null,
'user_id' => 1,
'host_id' => 7,
'kind' => 'create',
'execution_mode' => 'agent',
'phase' => 'accepted',
'outcome' => null,
'status' => 'queued',
'sealed_payload' => $createEnvelope,
'attempts' => 0,
'dispatch_attempts' => 0,
'dispatch_deadline_at' => $now - 30,
'dispatch_started_at' => null,
'agent_accepted_at' => null,
'receipt_deadline_at' => null,
'terminal_at' => null,
'effect_may_have_applied' => null,
'safe_to_resubmit' => null,
'effect_resolution' => 'not_applicable',
'effect_resolved_at' => null,
'effect_resolved_by' => null,
'effect_resolution_reason' => '',
'completion_event_id' => null,
'read_at' => null,
'dispatched_at' => null,
'lease_token' => str_repeat('6', 32),
'lease_until' => $now + 180,
'available_at' => 0,
'error_code' => null,
'finalization_incident' => 0,
'finalization_error_code' => null,
'result_json' => $originalResult,
'created_at' => $now - 30,
'updated_at' => $now - 20,
]);
CloudSecrets::setAvailable(false);
$forceTerminalFailure = new ReflectionMethod(
CloudWorkerService::class,
'forceTerminalFailure'
);
$forceTerminalFailure->invoke(
null,
601,
str_repeat('6', 32),
false,
true,
'dispatch_not_sent'
);
$createAfterEmergency = CloudJob::findOrFail(601);
$assert((string) $createAfterEmergency->phase === 'terminal'
&& (string) $createAfterEmergency->outcome === 'failed'
&& (int) $createAfterEmergency->dispatch_attempts === 0
&& !(bool) $createAfterEmergency->effect_may_have_applied
&& !(bool) $createAfterEmergency->safe_to_resubmit
&& (string) $createAfterEmergency->effect_resolution
=== 'not_applied'
&& (bool) $createAfterEmergency->finalization_incident
&& (string) $createAfterEmergency->finalization_error_code
=== 'recovery_payload_unavailable'
&& (string) $createAfterEmergency->sealed_payload === $createEnvelope
&& (int) $createAfterEmergency->available_at >= time() + 290,
'Create emergency terminalization lost its generation-zero proof or escrow.');
Capsule::table('cloud_jobs')->where('id', 601)->update([
'available_at' => 0,
'lease_token' => null,
'lease_until' => 0,
]);
$createService = new CloudAutomaticOperationRecoveryService();
$assert($createService->tick(),
'An undispatched create with unavailable escrow was not claimed.');
$createAfterKeyFailure = CloudJob::findOrFail(601);
$assert((bool) $createAfterKeyFailure->finalization_incident
&& (string) $createAfterKeyFailure->finalization_error_code
=== 'recovery_payload_unavailable'
&& (string) $createAfterKeyFailure->sealed_payload
=== $createEnvelope
&& (string) Capsule::table('user')->where('id', 1)
->value('money') === '0.00'
&& (string) Capsule::table('cloud_resource_reservations')
->where('id', 601)->value('state') === 'held'
&& CloudAgentSettingsService::$requests === 0,
'Unavailable create escrow released money/resources or contacted Agent.');
CloudSecrets::setAvailable(true);
Capsule::table('cloud_jobs')->where('id', 601)->update([
'available_at' => 0,
'lease_token' => null,
'lease_until' => 0,
]);
$assert($createService->tick(),
'An undispatched create with a missing invoice was not retried.');
$createAfterLedgerFailure = CloudJob::findOrFail(601);
$assert((bool) $createAfterLedgerFailure->finalization_incident
&& (string) Capsule::table('user')->where('id', 1)
->value('money') === '0.00'
&& (string) Capsule::table('cloud_resource_reservations')
->where('id', 601)->value('state') === 'held'
&& Capsule::table('cloud_public_ips')
->where('reservation_id', 601)
->where('state', 'reserved')->count() === 2
&& CloudAgentSettingsService::$requests === 0,
'Missing create ledger data caused a partial refund/release or Agent call.');
Capsule::table('cloud_invoices')->insert([
'id' => 601,
'invoice_no' => 'VC-UNDISPATCHED-601',
'purpose' => 'provision',
'instance_id' => null,
'order_id' => 601,
'user_id' => 1,
'plan_id' => 601,
'offer_id' => 601,
'renewal_id' => null,
'renews_expires_at' => null,
'due_at' => $now - 120,
'amount' => '12.34',
'currency' => 'USD',
'validity_days' => 30,
'traffic_quota_bytes' => 0,
'status' => 'paid',
'paid_via' => 'balance',
'payment_reference' => 'undispatched-create-payment',
'paid_at' => $now - 60,
'created_at' => $now - 120,
'updated_at' => $now - 60,
]);
Capsule::table('cloud_jobs')->where('id', 601)->update([
'available_at' => 0,
'lease_token' => null,
'lease_until' => 0,
]);
$beforeCreateAudit = CloudJob::findOrFail(601);
$assert($createService->tick(),
'A repaired undispatched create was not finalized locally.');
$createAfterRecovery = CloudJob::findOrFail(601);
$createSafePayload = CloudSecrets::decrypt(
(string) $createAfterRecovery->sealed_payload,
'job-payload'
);
$assert((string) $createAfterRecovery->phase
=== (string) $beforeCreateAudit->phase
&& (string) $createAfterRecovery->outcome
=== (string) $beforeCreateAudit->outcome
&& (int) $createAfterRecovery->terminal_at
=== (int) $beforeCreateAudit->terminal_at
&& (string) $createAfterRecovery->error_code
=== (string) $beforeCreateAudit->error_code
&& !(bool) $createAfterRecovery->effect_may_have_applied
&& !(bool) $createAfterRecovery->safe_to_resubmit
&& (string) $createAfterRecovery->effect_resolution === 'not_applied'
&& (string) $createAfterRecovery->effect_resolution_reason
=== 'automatic_local_not_dispatched'
&& !(bool) $createAfterRecovery->finalization_incident
&& $createAfterRecovery->finalization_error_code === null
&& !isset($createSafePayload['credential'])
&& !isset($createSafePayload['body']['root_password'])
&& (string) Capsule::table('user')->where('id', 1)
->value('money') === '12.34'
&& (string) Capsule::table('cloud_invoices')->where('id', 601)
->value('status') === 'refunded'
&& Capsule::table('cloud_orders')->where('id', 601)
->value('refunded_at') !== null
&& Capsule::table('cloud_orders')->where('id', 601)
->value('stock_released_at') !== null
&& (int) Capsule::table('cloud_offers')->where('id', 601)
->value('stock') === 1
&& (string) Capsule::table('cloud_resource_reservations')
->where('id', 601)->value('state') === 'released'
&& Capsule::table('cloud_public_ips')
->whereIn('id', [601, 602])
->where('state', 'available')->count() === 2
&& Capsule::table('cloud_public_ips')
->whereIn('id', [601, 602])
->whereNotNull('reservation_id')->count() === 0
&& Capsule::table('cloud_resource_reservation_ips')
->where('reservation_id', 601)
->where('state', 'released')->count() === 2
&& CloudAgentSettingsService::$requests === 0,
'Recovered create did not atomically refund, release, scrub, and close.');
$createUpdatedAt = (int) $createAfterRecovery->updated_at;
$assert(!$createService->tick(),
'A converged undispatched create was claimed a second time.');
$assert((string) Capsule::table('user')->where('id', 1)
->value('money') === '12.34'
&& (int) Capsule::table('cloud_offers')->where('id', 601)
->value('stock') === 1
&& (int) CloudJob::findOrFail(601)->updated_at === $createUpdatedAt
&& CloudAgentSettingsService::$requests === 0,
'Undispatched create recovery repeated refund/release or contacted Agent.');
echo "Cloud undispatched finalizer recovery tests passed.\n";
}