You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
959 lines
38 KiB
PHP
959 lines
38 KiB
PHP
<?php
|
|
declare(strict_types=1);
|
|
|
|
namespace App\Services\Cloud {
|
|
/** Isolated encrypted envelope for PHP-only finalizer recovery tests. */
|
|
final class CloudSecrets
|
|
{
|
|
private static array $values = [];
|
|
private static bool $available = true;
|
|
|
|
public static function setAvailable(bool $available): void
|
|
{
|
|
self::$available = $available;
|
|
}
|
|
|
|
public static function encrypt(array $value, string $purpose): string
|
|
{
|
|
$key = bin2hex(random_bytes(16));
|
|
self::$values[$key] = $value;
|
|
return 'undispatched-finalizer-test:' . $key;
|
|
}
|
|
|
|
public static function decrypt(string $value, string $purpose): array
|
|
{
|
|
if (!self::$available) {
|
|
throw new \RuntimeException(
|
|
'Finalizer test key is temporarily unavailable.'
|
|
);
|
|
}
|
|
$key = substr(
|
|
$value,
|
|
strlen('undispatched-finalizer-test:')
|
|
);
|
|
if (!array_key_exists($key, self::$values)) {
|
|
throw new \RuntimeException('Missing finalizer test payload.');
|
|
}
|
|
return self::$values[$key];
|
|
}
|
|
}
|
|
|
|
/** Any Agent lookup is a test failure for generation-zero recovery. */
|
|
final class CloudAgentSettingsService
|
|
{
|
|
public static int $requests = 0;
|
|
|
|
public static function clientForHost(
|
|
int $hostId,
|
|
float $timeoutSeconds = 120.0
|
|
): object {
|
|
self::$requests++;
|
|
throw new \RuntimeException(
|
|
'Undispatched finalizer recovery contacted the Agent.'
|
|
);
|
|
}
|
|
|
|
public static function clientForConfiguration(
|
|
int $hostId,
|
|
float $timeoutSeconds = 120.0
|
|
): object {
|
|
return self::clientForHost($hostId, $timeoutSeconds);
|
|
}
|
|
}
|
|
}
|
|
|
|
namespace {
|
|
require dirname(__DIR__) . '/vendor/autoload.php';
|
|
|
|
use App\Models\CloudJob;
|
|
use App\Models\CloudRescueTemplateImport;
|
|
use App\Services\Cloud\CloudAutomaticOperationRecoveryService;
|
|
use App\Services\Cloud\CloudAgentSettingsService;
|
|
use App\Services\Cloud\CloudDispatchLedger;
|
|
use App\Services\Cloud\CloudJobService;
|
|
use App\Services\Cloud\CloudRescueTemplateImportService;
|
|
use App\Services\Cloud\CloudSecrets;
|
|
use App\Services\Cloud\CloudWorkerService;
|
|
use Illuminate\Database\Capsule\Manager as Capsule;
|
|
use Illuminate\Database\Schema\Blueprint;
|
|
|
|
$assert = static function (bool $condition, string $message): void {
|
|
if (!$condition) {
|
|
throw new RuntimeException($message);
|
|
}
|
|
};
|
|
|
|
$capsule = new Capsule();
|
|
$capsule->addConnection([
|
|
'driver' => 'sqlite', 'database' => ':memory:',
|
|
], 'default');
|
|
$capsule->setAsGlobal();
|
|
$capsule->bootEloquent();
|
|
$schema = $capsule->schema('default');
|
|
$schema->create('user', static function (Blueprint $table): void {
|
|
$table->increments('id');
|
|
$table->string('money')->default('0.00');
|
|
$table->integer('enable')->default(1);
|
|
$table->integer('is_admin')->default(0);
|
|
});
|
|
$schema->create('cloud_jobs', static function (Blueprint $table): void {
|
|
$table->increments('id');
|
|
$table->string('operation_id')->unique();
|
|
$table->string('request_idempotency_key')->unique();
|
|
$table->string('request_fingerprint');
|
|
$table->integer('parent_job_id')->nullable();
|
|
$table->integer('order_id')->nullable();
|
|
$table->integer('invoice_id')->nullable();
|
|
$table->integer('instance_id')->nullable();
|
|
$table->integer('snapshot_id')->nullable();
|
|
$table->integer('user_id');
|
|
$table->integer('host_id')->nullable();
|
|
$table->string('kind');
|
|
$table->string('execution_mode');
|
|
$table->string('phase');
|
|
$table->string('outcome')->nullable();
|
|
$table->string('status');
|
|
$table->text('sealed_payload');
|
|
$table->integer('attempts')->default(0);
|
|
$table->integer('dispatch_attempts')->default(0);
|
|
$table->integer('dispatch_deadline_at')->default(0);
|
|
$table->integer('dispatch_started_at')->nullable();
|
|
$table->integer('agent_accepted_at')->nullable();
|
|
$table->integer('receipt_deadline_at')->nullable();
|
|
$table->integer('terminal_at')->nullable();
|
|
$table->boolean('effect_may_have_applied')->nullable();
|
|
$table->boolean('safe_to_resubmit')->nullable();
|
|
$table->string('effect_resolution');
|
|
$table->integer('effect_resolved_at')->nullable();
|
|
$table->integer('effect_resolved_by')->nullable();
|
|
$table->string('effect_resolution_reason')->default('');
|
|
$table->string('completion_event_id')->nullable();
|
|
$table->integer('read_at')->nullable();
|
|
$table->integer('dispatched_at')->nullable();
|
|
$table->string('lease_token')->nullable();
|
|
$table->integer('lease_until')->default(0);
|
|
$table->integer('available_at')->default(0);
|
|
$table->string('error_code')->nullable();
|
|
$table->boolean('finalization_incident')->default(0);
|
|
$table->string('finalization_error_code')->nullable();
|
|
$table->text('result_json')->nullable();
|
|
$table->integer('created_at');
|
|
$table->integer('updated_at');
|
|
});
|
|
$schema->create('cloud_instances', static function (Blueprint $table): void {
|
|
$table->increments('id');
|
|
$table->string('uuid');
|
|
$table->integer('host_id');
|
|
$table->integer('user_id');
|
|
$table->integer('order_id')->nullable();
|
|
$table->string('name');
|
|
$table->string('status');
|
|
$table->string('service_status');
|
|
$table->string('network_status');
|
|
$table->text('spec_json');
|
|
$table->text('sealed_credential')->nullable();
|
|
$table->integer('created_at');
|
|
$table->integer('updated_at');
|
|
});
|
|
$schema->create('cloud_agent_completion_events',
|
|
static function (Blueprint $table): void {
|
|
$table->string('event_id')->primary();
|
|
$table->integer('host_id');
|
|
$table->string('operation_id');
|
|
$table->string('payload_sha256');
|
|
$table->string('outcome');
|
|
$table->boolean('effect_may_have_applied');
|
|
$table->boolean('safe_to_resubmit');
|
|
$table->string('error_code')->default('');
|
|
$table->text('result_json')->nullable();
|
|
$table->string('disposition')->default('received');
|
|
$table->integer('received_at');
|
|
$table->integer('applied_at')->nullable();
|
|
$table->unique(['host_id', 'operation_id']);
|
|
});
|
|
$schema->create('cloud_templates',
|
|
static function (Blueprint $table): void {
|
|
$table->string('id')->primary();
|
|
$table->integer('active_job_id')->nullable();
|
|
$table->string('state')->default('active');
|
|
});
|
|
$schema->create('cloud_user_isos',
|
|
static function (Blueprint $table): void {
|
|
$table->string('id')->primary();
|
|
$table->integer('active_job_id')->nullable();
|
|
$table->string('state')->default('active');
|
|
});
|
|
$schema->create('cloud_rescue_template_imports',
|
|
static function (Blueprint $table): void {
|
|
$table->string('operation_id')->primary();
|
|
$table->integer('host_id');
|
|
$table->string('template_id', 32);
|
|
$table->integer('job_id');
|
|
$table->string('import_type');
|
|
$table->string('name');
|
|
$table->integer('size_bytes')->default(0);
|
|
$table->string('sha256', 64)->default('');
|
|
$table->string('state');
|
|
$table->string('error_code')->default('');
|
|
$table->integer('created_at');
|
|
$table->integer('updated_at');
|
|
});
|
|
$schema->create('cloud_template_replicas',
|
|
static function (Blueprint $table): void {
|
|
$table->integer('host_id');
|
|
$table->string('template_id');
|
|
$table->integer('revision')->default(0);
|
|
$table->string('generation_token')->default('');
|
|
$table->string('state')->default('pending');
|
|
$table->integer('image_bytes')->default(0);
|
|
$table->string('sha256')->default('');
|
|
$table->string('operation_id')->default('');
|
|
$table->integer('job_id')->nullable();
|
|
$table->string('error_code')->default('');
|
|
$table->integer('created_at')->default(0);
|
|
$table->integer('updated_at')->default(0);
|
|
$table->primary(['host_id', 'template_id']);
|
|
});
|
|
$schema->create('cloud_user_iso_replicas',
|
|
static function (Blueprint $table): void {
|
|
$table->integer('host_id');
|
|
$table->string('user_iso_id');
|
|
$table->integer('user_id');
|
|
$table->string('state')->default('pending');
|
|
$table->integer('size_bytes')->default(0);
|
|
$table->string('sha256')->default('');
|
|
$table->string('operation_id')->default('');
|
|
$table->integer('job_id')->nullable();
|
|
$table->string('error_code')->default('');
|
|
$table->integer('created_at')->default(0);
|
|
$table->integer('updated_at')->default(0);
|
|
$table->primary(['host_id', 'user_iso_id']);
|
|
});
|
|
$schema->create('cloud_hosts', static function (Blueprint $table): void {
|
|
$table->increments('id');
|
|
});
|
|
$schema->create('cloud_offers', static function (Blueprint $table): void {
|
|
$table->increments('id');
|
|
$table->integer('stock')->default(-1);
|
|
$table->integer('updated_at')->default(0);
|
|
});
|
|
$schema->create('cloud_orders', static function (Blueprint $table): void {
|
|
$table->increments('id');
|
|
$table->integer('user_id');
|
|
$table->integer('plan_id');
|
|
$table->integer('offer_id');
|
|
$table->integer('resource_pool_id');
|
|
$table->integer('host_id')->nullable();
|
|
$table->integer('reservation_id')->nullable();
|
|
$table->string('idempotency_key');
|
|
$table->string('request_hash');
|
|
$table->string('amount');
|
|
$table->string('currency')->default('USD');
|
|
$table->string('status');
|
|
$table->text('spec_json');
|
|
$table->text('sealed_payload')->nullable();
|
|
$table->integer('instance_id')->nullable();
|
|
$table->integer('refunded_at')->nullable();
|
|
$table->integer('stock_reserved_at')->nullable();
|
|
$table->integer('stock_released_at')->nullable();
|
|
$table->integer('created_at');
|
|
$table->integer('updated_at');
|
|
});
|
|
$schema->create('cloud_resource_reservations',
|
|
static function (Blueprint $table): void {
|
|
$table->increments('id');
|
|
$table->string('reservation_key');
|
|
$table->string('operation_id');
|
|
$table->integer('order_id');
|
|
$table->integer('offer_id');
|
|
$table->integer('resource_pool_id');
|
|
$table->integer('host_id');
|
|
$table->integer('user_id');
|
|
$table->integer('instance_id')->nullable();
|
|
$table->integer('network_attachment_id')->nullable();
|
|
$table->integer('public_ip_pool_id')->nullable();
|
|
$table->integer('public_ip_id')->nullable();
|
|
$table->integer('cpu_cores');
|
|
$table->integer('memory_mib');
|
|
$table->integer('disk_gib');
|
|
$table->string('state');
|
|
$table->integer('expires_at')->nullable();
|
|
$table->integer('released_at')->nullable();
|
|
$table->string('release_reason')->default('');
|
|
$table->integer('created_at');
|
|
$table->integer('updated_at');
|
|
});
|
|
$schema->create('cloud_public_ips',
|
|
static function (Blueprint $table): void {
|
|
$table->increments('id');
|
|
$table->integer('host_id');
|
|
$table->integer('public_ip_pool_id')->nullable();
|
|
$table->string('address');
|
|
$table->integer('prefix_length');
|
|
$table->string('state');
|
|
$table->integer('reservation_id')->nullable();
|
|
$table->integer('instance_id')->nullable();
|
|
$table->string('operation_id')->nullable();
|
|
$table->integer('created_at');
|
|
$table->integer('updated_at');
|
|
});
|
|
$schema->create('cloud_invoices',
|
|
static function (Blueprint $table): void {
|
|
$table->increments('id');
|
|
$table->string('invoice_no');
|
|
$table->string('purpose');
|
|
$table->integer('instance_id')->nullable();
|
|
$table->integer('order_id')->nullable();
|
|
$table->integer('user_id');
|
|
$table->integer('plan_id')->nullable();
|
|
$table->integer('offer_id')->nullable();
|
|
$table->integer('renewal_id')->nullable();
|
|
$table->integer('renews_expires_at')->nullable();
|
|
$table->integer('due_at');
|
|
$table->string('amount');
|
|
$table->string('currency')->default('USD');
|
|
$table->integer('validity_days');
|
|
$table->integer('traffic_quota_bytes');
|
|
$table->string('status');
|
|
$table->string('paid_via')->nullable();
|
|
$table->string('payment_reference')->nullable();
|
|
$table->integer('paid_at')->nullable();
|
|
$table->integer('created_at');
|
|
$table->integer('updated_at');
|
|
});
|
|
|
|
$now = time();
|
|
$operationId = str_repeat('ad', 16);
|
|
$uuid = '00000000-0000-4000-8000-000000000501';
|
|
$payload = [
|
|
'method' => 'POST',
|
|
'request_target' => '/v1/vms/' . $uuid . '/settings',
|
|
'body' => [
|
|
'hostname' => 'local-finalizer.example',
|
|
'root_password' => 'Secret9!Password',
|
|
'confirm_restart' => true,
|
|
],
|
|
'owner_id' => 1,
|
|
'can_manage' => false,
|
|
'credential' => [
|
|
'password' => 'Secret9!Password',
|
|
'login_name' => 'root',
|
|
'auth_mode' => 'password',
|
|
],
|
|
'settings_fields' => ['hostname', 'password'],
|
|
];
|
|
Capsule::table('user')->insert([
|
|
'id' => 1, 'enable' => 1, 'is_admin' => 0,
|
|
]);
|
|
Capsule::table('cloud_instances')->insert([
|
|
'id' => 501,
|
|
'uuid' => $uuid,
|
|
'host_id' => 7,
|
|
'user_id' => 1,
|
|
'name' => 'local-finalizer',
|
|
'status' => 'active',
|
|
'service_status' => 'exempt',
|
|
'network_status' => 'active',
|
|
'spec_json' => json_encode(
|
|
['managed' => true],
|
|
JSON_THROW_ON_ERROR
|
|
),
|
|
'sealed_credential' => null,
|
|
'created_at' => $now - 3600,
|
|
'updated_at' => $now - 3600,
|
|
]);
|
|
$originalEnvelope = CloudSecrets::encrypt($payload, 'job-payload');
|
|
$originalResult = json_encode(
|
|
['cause_code' => 'local_finalization_failed'],
|
|
JSON_THROW_ON_ERROR
|
|
);
|
|
Capsule::table('cloud_jobs')->insert([
|
|
'id' => 501,
|
|
'operation_id' => $operationId,
|
|
'request_idempotency_key' => $operationId,
|
|
'request_fingerprint' => hash('sha256', $operationId),
|
|
'parent_job_id' => null,
|
|
'order_id' => null,
|
|
'invoice_id' => null,
|
|
'instance_id' => 501,
|
|
'snapshot_id' => null,
|
|
'user_id' => 1,
|
|
'host_id' => 7,
|
|
'kind' => 'instance_settings',
|
|
'execution_mode' => 'agent',
|
|
'phase' => 'terminal',
|
|
'outcome' => 'failed',
|
|
'status' => 'failed',
|
|
'sealed_payload' => $originalEnvelope,
|
|
'attempts' => 0,
|
|
'dispatch_attempts' => 0,
|
|
'dispatch_deadline_at' => $now - 30,
|
|
'terminal_at' => $now - 20,
|
|
'effect_may_have_applied' => 0,
|
|
'safe_to_resubmit' => 0,
|
|
'effect_resolution' => 'not_applied',
|
|
'effect_resolution_reason' => '',
|
|
'lease_until' => 0,
|
|
'available_at' => 0,
|
|
'error_code' => 'local_finalization_failed',
|
|
'finalization_incident' => 1,
|
|
'finalization_error_code' => 'recovery_ledger_conflict',
|
|
'result_json' => $originalResult,
|
|
'created_at' => $now - 30,
|
|
'updated_at' => $now - 20,
|
|
]);
|
|
|
|
$before = CloudJob::findOrFail(501);
|
|
$assert(CloudJobService::blockingQueryForInstance(501)->exists(),
|
|
'An undispatched local finalizer incident did not block mutations.');
|
|
$service = new CloudAutomaticOperationRecoveryService();
|
|
$assert($service->tick(),
|
|
'The undispatched PHP-only finalizer was not claimed.');
|
|
$after = CloudJob::findOrFail(501);
|
|
$scrubbed = CloudSecrets::decrypt(
|
|
(string) $after->sealed_payload,
|
|
'job-payload'
|
|
);
|
|
$assert((string) $after->phase === (string) $before->phase
|
|
&& (string) $after->outcome === (string) $before->outcome
|
|
&& (string) $after->status === (string) $before->status
|
|
&& (int) $after->terminal_at === (int) $before->terminal_at
|
|
&& (string) $after->error_code === (string) $before->error_code
|
|
&& (string) $after->result_json === (string) $before->result_json
|
|
&& $after->completion_event_id === $before->completion_event_id
|
|
&& !(bool) $after->effect_may_have_applied
|
|
&& (bool) $after->safe_to_resubmit
|
|
&& (string) $after->effect_resolution === 'not_applied'
|
|
&& (string) $after->effect_resolution_reason
|
|
=== 'automatic_local_not_dispatched'
|
|
&& (int) $after->effect_resolved_at > 0
|
|
&& !(bool) $after->finalization_incident
|
|
&& $after->finalization_error_code === null
|
|
&& $after->lease_token === null
|
|
&& (int) $after->lease_until === 0
|
|
&& (int) $after->available_at === 0
|
|
&& (string) $after->sealed_payload !== $originalEnvelope
|
|
&& !isset($scrubbed['credential'])
|
|
&& !isset($scrubbed['body']['root_password'])
|
|
&& !CloudJobService::blockingQueryForInstance(501)->exists(),
|
|
'The PHP-only finalizer changed terminal audit data or failed to converge safely.');
|
|
$updatedAt = (int) $after->updated_at;
|
|
$assert(!$service->tick(),
|
|
'A converged PHP-only finalizer was claimed a second time.');
|
|
$afterSecondTick = CloudJob::findOrFail(501);
|
|
$assert((int) $afterSecondTick->updated_at === $updatedAt,
|
|
'A converged PHP-only finalizer repeated its side effects.');
|
|
|
|
// A missing/corrupt encrypted escrow remains a concrete payload incident;
|
|
// it must not degrade into a generic ledger-conflict message.
|
|
Capsule::table('cloud_instances')->insert([
|
|
'id' => 502,
|
|
'uuid' => '00000000-0000-4000-8000-000000000502',
|
|
'host_id' => 7,
|
|
'user_id' => 1,
|
|
'name' => 'local-finalizer-corrupt-payload',
|
|
'status' => 'active',
|
|
'service_status' => 'exempt',
|
|
'network_status' => 'active',
|
|
'spec_json' => json_encode(['managed' => true], JSON_THROW_ON_ERROR),
|
|
'sealed_credential' => null,
|
|
'created_at' => $now - 3600,
|
|
'updated_at' => $now - 3600,
|
|
]);
|
|
$corruptOperationId = str_repeat('ae', 16);
|
|
Capsule::table('cloud_jobs')->insert([
|
|
'id' => 502,
|
|
'operation_id' => $corruptOperationId,
|
|
'request_idempotency_key' => $corruptOperationId,
|
|
'request_fingerprint' => hash('sha256', $corruptOperationId),
|
|
'parent_job_id' => null,
|
|
'order_id' => null,
|
|
'invoice_id' => null,
|
|
'instance_id' => 502,
|
|
'snapshot_id' => null,
|
|
'user_id' => 1,
|
|
'host_id' => 7,
|
|
'kind' => 'instance_settings',
|
|
'execution_mode' => 'agent',
|
|
'phase' => 'terminal',
|
|
'outcome' => 'failed',
|
|
'status' => 'failed',
|
|
'sealed_payload' => 'undispatched-finalizer-test:missing',
|
|
'attempts' => 0,
|
|
'dispatch_attempts' => 0,
|
|
'dispatch_deadline_at' => $now - 30,
|
|
'terminal_at' => $now - 20,
|
|
'effect_may_have_applied' => 0,
|
|
'safe_to_resubmit' => 0,
|
|
'effect_resolution' => 'not_applied',
|
|
'effect_resolution_reason' => '',
|
|
'lease_until' => 0,
|
|
'available_at' => 0,
|
|
'error_code' => 'local_finalization_failed',
|
|
'finalization_incident' => 1,
|
|
'finalization_error_code' => 'recovery_payload_unavailable',
|
|
'result_json' => $originalResult,
|
|
'created_at' => $now - 30,
|
|
'updated_at' => $now - 20,
|
|
]);
|
|
$assert($service->tick(),
|
|
'The corrupt undispatched recovery payload was not checked.');
|
|
$corruptPayloadJob = CloudJob::findOrFail(502);
|
|
$assert((bool) $corruptPayloadJob->finalization_incident
|
|
&& (string) $corruptPayloadJob->finalization_error_code
|
|
=== 'recovery_payload_unavailable'
|
|
&& (string) $corruptPayloadJob->sealed_payload
|
|
=== 'undispatched-finalizer-test:missing'
|
|
&& (int) $corruptPayloadJob->lease_until >= time() + 290
|
|
&& (int) $corruptPayloadJob->available_at >= time() + 290
|
|
&& CloudJobService::blockingQueryForInstance(502)->exists(),
|
|
'A corrupt PHP-only recovery payload lost its concrete incident or resource fence.');
|
|
|
|
// A create that never crossed the dispatch boundary has definitive
|
|
// no-effect proof, but its paid ledgers still require the create-specific
|
|
// atomic finalizer. Temporary key or database damage must retain escrow
|
|
// and every fence; once repaired, the worker refunds/releases exactly
|
|
// once and never sends the original create request to an Agent.
|
|
$createOperationId = str_repeat('af', 16);
|
|
$createPayload = [
|
|
'method' => 'POST',
|
|
'request_target' => '/v1/vms',
|
|
'body' => [
|
|
'operation_id' => $createOperationId,
|
|
'source' => 'cloud',
|
|
'name' => 'undispatched-create',
|
|
'root_password' => 'Secret9!Password',
|
|
],
|
|
'credential' => [
|
|
'password' => 'Secret9!Password',
|
|
'login_name' => 'root',
|
|
'auth_mode' => 'password',
|
|
],
|
|
];
|
|
$createEnvelope = CloudSecrets::encrypt(
|
|
$createPayload,
|
|
'job-payload'
|
|
);
|
|
Capsule::table('user')->where('id', 1)->update(['money' => '0.00']);
|
|
Capsule::table('cloud_hosts')->insert(['id' => 7]);
|
|
Capsule::table('cloud_offers')->insert([
|
|
'id' => 601,
|
|
'stock' => 0,
|
|
'updated_at' => $now - 30,
|
|
]);
|
|
Capsule::table('cloud_orders')->insert([
|
|
'id' => 601,
|
|
'user_id' => 1,
|
|
'plan_id' => 601,
|
|
'offer_id' => 601,
|
|
'resource_pool_id' => 601,
|
|
'host_id' => 7,
|
|
'reservation_id' => 601,
|
|
'idempotency_key' => 'undispatched-create-order',
|
|
'request_hash' => hash('sha256', 'undispatched-create-order'),
|
|
'amount' => '12.34',
|
|
'currency' => 'USD',
|
|
'status' => 'queued',
|
|
'spec_json' => '{}',
|
|
'sealed_payload' => null,
|
|
'instance_id' => null,
|
|
'refunded_at' => null,
|
|
'stock_reserved_at' => $now - 60,
|
|
'stock_released_at' => null,
|
|
'created_at' => $now - 120,
|
|
'updated_at' => $now - 30,
|
|
]);
|
|
Capsule::table('cloud_resource_reservations')->insert([
|
|
'id' => 601,
|
|
'reservation_key' => 'undispatched-create-reservation',
|
|
'operation_id' => $createOperationId,
|
|
'order_id' => 601,
|
|
'offer_id' => 601,
|
|
'resource_pool_id' => 601,
|
|
'host_id' => 7,
|
|
'user_id' => 1,
|
|
'instance_id' => null,
|
|
'network_attachment_id' => 601,
|
|
'public_ip_pool_id' => 601,
|
|
'public_ip_id' => 601,
|
|
'cpu_cores' => 1,
|
|
'memory_mib' => 512,
|
|
'disk_gib' => 10,
|
|
'state' => 'held',
|
|
'expires_at' => null,
|
|
'released_at' => null,
|
|
'release_reason' => '',
|
|
'created_at' => $now - 120,
|
|
'updated_at' => $now - 30,
|
|
]);
|
|
Capsule::table('cloud_public_ips')->insert([
|
|
'id' => 601,
|
|
'host_id' => 7,
|
|
'public_ip_pool_id' => 601,
|
|
'address' => '203.0.113.61',
|
|
'prefix_length' => 24,
|
|
'state' => 'reserved',
|
|
'reservation_id' => 601,
|
|
'instance_id' => null,
|
|
'operation_id' => $createOperationId,
|
|
'created_at' => $now - 120,
|
|
'updated_at' => $now - 30,
|
|
]);
|
|
Capsule::table('cloud_jobs')->insert([
|
|
'id' => 601,
|
|
'operation_id' => $createOperationId,
|
|
'request_idempotency_key' => $createOperationId,
|
|
'request_fingerprint' => hash('sha256', $createOperationId),
|
|
'parent_job_id' => null,
|
|
'order_id' => 601,
|
|
'invoice_id' => null,
|
|
'instance_id' => null,
|
|
'snapshot_id' => null,
|
|
'user_id' => 1,
|
|
'host_id' => 7,
|
|
'kind' => 'create',
|
|
'execution_mode' => 'agent',
|
|
'phase' => 'accepted',
|
|
'outcome' => null,
|
|
'status' => 'queued',
|
|
'sealed_payload' => $createEnvelope,
|
|
'attempts' => 0,
|
|
'dispatch_attempts' => 0,
|
|
'dispatch_deadline_at' => $now - 30,
|
|
'dispatch_started_at' => null,
|
|
'agent_accepted_at' => null,
|
|
'receipt_deadline_at' => null,
|
|
'terminal_at' => null,
|
|
'effect_may_have_applied' => null,
|
|
'safe_to_resubmit' => null,
|
|
'effect_resolution' => 'not_applicable',
|
|
'effect_resolved_at' => null,
|
|
'effect_resolved_by' => null,
|
|
'effect_resolution_reason' => '',
|
|
'completion_event_id' => null,
|
|
'read_at' => null,
|
|
'dispatched_at' => null,
|
|
'lease_token' => str_repeat('6', 32),
|
|
'lease_until' => $now + 180,
|
|
'available_at' => 0,
|
|
'error_code' => null,
|
|
'finalization_incident' => 0,
|
|
'finalization_error_code' => null,
|
|
'result_json' => $originalResult,
|
|
'created_at' => $now - 30,
|
|
'updated_at' => $now - 20,
|
|
]);
|
|
|
|
CloudSecrets::setAvailable(false);
|
|
$forceTerminalFailure = new ReflectionMethod(
|
|
CloudWorkerService::class,
|
|
'forceTerminalFailure'
|
|
);
|
|
$forceTerminalFailure->invoke(
|
|
null,
|
|
601,
|
|
str_repeat('6', 32),
|
|
false,
|
|
true,
|
|
'dispatch_not_sent'
|
|
);
|
|
$createAfterEmergency = CloudJob::findOrFail(601);
|
|
$assert((string) $createAfterEmergency->phase === 'terminal'
|
|
&& (string) $createAfterEmergency->outcome === 'failed'
|
|
&& (int) $createAfterEmergency->dispatch_attempts === 0
|
|
&& !(bool) $createAfterEmergency->effect_may_have_applied
|
|
&& !(bool) $createAfterEmergency->safe_to_resubmit
|
|
&& (string) $createAfterEmergency->effect_resolution
|
|
=== 'not_applied'
|
|
&& (bool) $createAfterEmergency->finalization_incident
|
|
&& (string) $createAfterEmergency->finalization_error_code
|
|
=== 'recovery_payload_unavailable'
|
|
&& (string) $createAfterEmergency->sealed_payload === $createEnvelope
|
|
&& (int) $createAfterEmergency->available_at >= time() + 290,
|
|
'Create emergency terminalization lost its generation-zero proof or escrow.');
|
|
Capsule::table('cloud_jobs')->where('id', 601)->update([
|
|
'available_at' => 0,
|
|
'lease_token' => null,
|
|
'lease_until' => 0,
|
|
]);
|
|
|
|
$createService = new CloudAutomaticOperationRecoveryService();
|
|
$assert($createService->tick(),
|
|
'An undispatched create with unavailable escrow was not claimed.');
|
|
$createAfterKeyFailure = CloudJob::findOrFail(601);
|
|
$assert((bool) $createAfterKeyFailure->finalization_incident
|
|
&& (string) $createAfterKeyFailure->finalization_error_code
|
|
=== 'recovery_payload_unavailable'
|
|
&& (string) $createAfterKeyFailure->sealed_payload
|
|
=== $createEnvelope
|
|
&& (string) Capsule::table('user')->where('id', 1)
|
|
->value('money') === '0.00'
|
|
&& (string) Capsule::table('cloud_resource_reservations')
|
|
->where('id', 601)->value('state') === 'held'
|
|
&& CloudAgentSettingsService::$requests === 0,
|
|
'Unavailable create escrow released money/resources or contacted Agent.');
|
|
|
|
CloudSecrets::setAvailable(true);
|
|
Capsule::table('cloud_jobs')->where('id', 601)->update([
|
|
'available_at' => 0,
|
|
'lease_token' => null,
|
|
'lease_until' => 0,
|
|
]);
|
|
$assert($createService->tick(),
|
|
'An undispatched create with a missing invoice was not retried.');
|
|
$createAfterLedgerFailure = CloudJob::findOrFail(601);
|
|
$assert((bool) $createAfterLedgerFailure->finalization_incident
|
|
&& (string) Capsule::table('user')->where('id', 1)
|
|
->value('money') === '0.00'
|
|
&& (string) Capsule::table('cloud_resource_reservations')
|
|
->where('id', 601)->value('state') === 'held'
|
|
&& (string) Capsule::table('cloud_public_ips')
|
|
->where('id', 601)->value('state') === 'reserved'
|
|
&& CloudAgentSettingsService::$requests === 0,
|
|
'Missing create ledger data caused a partial refund/release or Agent call.');
|
|
|
|
Capsule::table('cloud_invoices')->insert([
|
|
'id' => 601,
|
|
'invoice_no' => 'VC-UNDISPATCHED-601',
|
|
'purpose' => 'provision',
|
|
'instance_id' => null,
|
|
'order_id' => 601,
|
|
'user_id' => 1,
|
|
'plan_id' => 601,
|
|
'offer_id' => 601,
|
|
'renewal_id' => null,
|
|
'renews_expires_at' => null,
|
|
'due_at' => $now - 120,
|
|
'amount' => '12.34',
|
|
'currency' => 'USD',
|
|
'validity_days' => 30,
|
|
'traffic_quota_bytes' => 0,
|
|
'status' => 'paid',
|
|
'paid_via' => 'balance',
|
|
'payment_reference' => 'undispatched-create-payment',
|
|
'paid_at' => $now - 60,
|
|
'created_at' => $now - 120,
|
|
'updated_at' => $now - 60,
|
|
]);
|
|
Capsule::table('cloud_jobs')->where('id', 601)->update([
|
|
'available_at' => 0,
|
|
'lease_token' => null,
|
|
'lease_until' => 0,
|
|
]);
|
|
$beforeCreateAudit = CloudJob::findOrFail(601);
|
|
$assert($createService->tick(),
|
|
'A repaired undispatched create was not finalized locally.');
|
|
$createAfterRecovery = CloudJob::findOrFail(601);
|
|
$createSafePayload = CloudSecrets::decrypt(
|
|
(string) $createAfterRecovery->sealed_payload,
|
|
'job-payload'
|
|
);
|
|
$assert((string) $createAfterRecovery->phase
|
|
=== (string) $beforeCreateAudit->phase
|
|
&& (string) $createAfterRecovery->outcome
|
|
=== (string) $beforeCreateAudit->outcome
|
|
&& (int) $createAfterRecovery->terminal_at
|
|
=== (int) $beforeCreateAudit->terminal_at
|
|
&& (string) $createAfterRecovery->error_code
|
|
=== (string) $beforeCreateAudit->error_code
|
|
&& !(bool) $createAfterRecovery->effect_may_have_applied
|
|
&& !(bool) $createAfterRecovery->safe_to_resubmit
|
|
&& (string) $createAfterRecovery->effect_resolution === 'not_applied'
|
|
&& (string) $createAfterRecovery->effect_resolution_reason
|
|
=== 'automatic_local_not_dispatched'
|
|
&& !(bool) $createAfterRecovery->finalization_incident
|
|
&& $createAfterRecovery->finalization_error_code === null
|
|
&& !isset($createSafePayload['credential'])
|
|
&& !isset($createSafePayload['body']['root_password'])
|
|
&& (string) Capsule::table('user')->where('id', 1)
|
|
->value('money') === '12.34'
|
|
&& (string) Capsule::table('cloud_invoices')->where('id', 601)
|
|
->value('status') === 'refunded'
|
|
&& Capsule::table('cloud_orders')->where('id', 601)
|
|
->value('refunded_at') !== null
|
|
&& Capsule::table('cloud_orders')->where('id', 601)
|
|
->value('stock_released_at') !== null
|
|
&& (int) Capsule::table('cloud_offers')->where('id', 601)
|
|
->value('stock') === 1
|
|
&& (string) Capsule::table('cloud_resource_reservations')
|
|
->where('id', 601)->value('state') === 'released'
|
|
&& (string) Capsule::table('cloud_public_ips')
|
|
->where('id', 601)->value('state') === 'available'
|
|
&& Capsule::table('cloud_public_ips')->where('id', 601)
|
|
->value('reservation_id') === null
|
|
&& CloudAgentSettingsService::$requests === 0,
|
|
'Recovered create did not atomically refund, release, scrub, and close.');
|
|
|
|
$createUpdatedAt = (int) $createAfterRecovery->updated_at;
|
|
$assert(!$createService->tick(),
|
|
'A converged undispatched create was claimed a second time.');
|
|
$assert((string) Capsule::table('user')->where('id', 1)
|
|
->value('money') === '12.34'
|
|
&& (int) Capsule::table('cloud_offers')->where('id', 601)
|
|
->value('stock') === 1
|
|
&& (int) CloudJob::findOrFail(601)->updated_at === $createUpdatedAt
|
|
&& CloudAgentSettingsService::$requests === 0,
|
|
'Undispatched create recovery repeated refund/release or contacted Agent.');
|
|
|
|
// The dispatch CAS may already be generation one when an exact Agent
|
|
// validation response proves that its operation journal was never entered.
|
|
// If PHP's catalog finalizer then fails, preserve that proof without
|
|
// rewriting the immutable counter, recover only the local ledger, and
|
|
// allow a new same-name import generation afterward.
|
|
$transportOperationId = str_repeat('ba', 16);
|
|
$transportTemplateId = str_repeat('c', 32);
|
|
$transportName = 'transport-not-dispatched-rescue';
|
|
$transportBytes = 2 * 1024 * 1024;
|
|
$transportPayload = [
|
|
'host_id' => 7,
|
|
'import_type' => 'upload',
|
|
'name' => $transportName,
|
|
'source' => '',
|
|
'sha256' => '',
|
|
'source_size_bytes' => $transportBytes,
|
|
'can_manage' => true,
|
|
'rescue_template_id' => $transportTemplateId,
|
|
'method' => 'POST',
|
|
'request_target' => '/v1/rescue-templates',
|
|
'body' => [
|
|
'operation_id' => $transportOperationId,
|
|
'id' => $transportTemplateId,
|
|
'name' => $transportName,
|
|
'source_url' => 'https://panel.example.test/staging/rescue.iso',
|
|
'source_size_bytes' => $transportBytes,
|
|
'sha256' => '',
|
|
],
|
|
'owner_id' => 1,
|
|
];
|
|
$transportEnvelope = CloudSecrets::encrypt(
|
|
$transportPayload,
|
|
'job-payload'
|
|
);
|
|
Capsule::table('cloud_jobs')->insert([
|
|
'id' => 701,
|
|
'operation_id' => $transportOperationId,
|
|
'request_idempotency_key' => $transportOperationId,
|
|
'request_fingerprint' => hash('sha256', $transportOperationId),
|
|
'parent_job_id' => null,
|
|
'order_id' => null,
|
|
'invoice_id' => null,
|
|
'instance_id' => null,
|
|
'snapshot_id' => null,
|
|
'user_id' => 1,
|
|
'host_id' => 7,
|
|
'kind' => 'rescue_template_import',
|
|
'execution_mode' => 'agent',
|
|
'phase' => 'dispatching',
|
|
'outcome' => null,
|
|
'status' => 'running',
|
|
'sealed_payload' => $transportEnvelope,
|
|
'attempts' => 1,
|
|
'dispatch_attempts' => 1,
|
|
'dispatch_deadline_at' => $now + 60,
|
|
'dispatch_started_at' => $now,
|
|
'agent_accepted_at' => null,
|
|
'receipt_deadline_at' => $now + 300,
|
|
'terminal_at' => null,
|
|
'effect_may_have_applied' => null,
|
|
'safe_to_resubmit' => null,
|
|
'effect_resolution' => 'unknown',
|
|
'effect_resolved_at' => null,
|
|
'effect_resolved_by' => null,
|
|
'effect_resolution_reason' => '',
|
|
'completion_event_id' => null,
|
|
'read_at' => null,
|
|
'dispatched_at' => $now,
|
|
'lease_token' => str_repeat('7', 32),
|
|
'lease_until' => $now + 180,
|
|
'available_at' => 0,
|
|
'error_code' => null,
|
|
'finalization_incident' => 0,
|
|
'finalization_error_code' => null,
|
|
'result_json' => null,
|
|
'created_at' => $now - 1,
|
|
'updated_at' => $now,
|
|
]);
|
|
|
|
$forceTerminalFailure->invoke(
|
|
null,
|
|
701,
|
|
str_repeat('7', 32),
|
|
false,
|
|
true,
|
|
'local_finalization_failed',
|
|
true
|
|
);
|
|
$transportIncident = CloudJob::findOrFail(701);
|
|
$assert((int) $transportIncident->dispatch_attempts === 1
|
|
&& CloudDispatchLedger::hasTransportNotDispatchedProof(
|
|
$transportIncident
|
|
)
|
|
&& (string) $transportIncident->sealed_payload === $transportEnvelope
|
|
&& (int) $transportIncident->available_at >= time() + 290
|
|
&& CloudAgentSettingsService::$requests === 0,
|
|
'A strict transport not-dispatched response lost its proof or immutable generation.');
|
|
|
|
Capsule::table('cloud_rescue_template_imports')->insert([
|
|
'operation_id' => $transportOperationId,
|
|
'host_id' => 7,
|
|
'template_id' => $transportTemplateId,
|
|
'job_id' => 701,
|
|
'import_type' => 'upload',
|
|
'name' => $transportName,
|
|
'size_bytes' => 0,
|
|
'sha256' => '',
|
|
'state' => 'importing',
|
|
'error_code' => '',
|
|
'created_at' => $now,
|
|
'updated_at' => $now,
|
|
]);
|
|
Capsule::table('cloud_jobs')->where('id', 701)->update([
|
|
'available_at' => 0,
|
|
'lease_token' => null,
|
|
'lease_until' => 0,
|
|
]);
|
|
$transportService = new CloudAutomaticOperationRecoveryService();
|
|
$assert($transportService->tick(),
|
|
'The strict transport not-dispatched finalizer was not claimed.');
|
|
$transportRecovered = CloudJob::findOrFail(701);
|
|
$assert((int) $transportRecovered->dispatch_attempts === 1
|
|
&& !(bool) $transportRecovered->effect_may_have_applied
|
|
&& (bool) $transportRecovered->safe_to_resubmit
|
|
&& (string) $transportRecovered->effect_resolution === 'not_applied'
|
|
&& (string) $transportRecovered->effect_resolution_reason
|
|
=== 'automatic_local_not_dispatched'
|
|
&& !(bool) $transportRecovered->finalization_incident
|
|
&& $transportRecovered->finalization_error_code === null
|
|
&& (string) CloudRescueTemplateImport::query()
|
|
->where('operation_id', $transportOperationId)
|
|
->value('state') === 'failed'
|
|
&& CloudAgentSettingsService::$requests === 0,
|
|
'Transport not-dispatched recovery did not converge the rescue ledger locally.');
|
|
|
|
$retryActor = [
|
|
'user_id' => 1,
|
|
'is_admin' => true,
|
|
'is_super_admin' => true,
|
|
'can_manage' => true,
|
|
];
|
|
$transportRetry = CloudRescueTemplateImportService::start(
|
|
7,
|
|
'upload',
|
|
$transportName,
|
|
'https://panel.example.test/staging/rescue-retry.iso',
|
|
'',
|
|
'',
|
|
$transportBytes,
|
|
$retryActor
|
|
);
|
|
$assert((int) $transportRetry['task_id'] !== 701
|
|
&& CloudRescueTemplateImport::query()
|
|
->where('host_id', 7)
|
|
->where('name', $transportName)
|
|
->count() === 2
|
|
&& CloudAgentSettingsService::$requests === 0,
|
|
'A locally converged rescue import still blocked its same-name retry.');
|
|
|
|
echo "Cloud undispatched finalizer recovery tests passed.\n";
|
|
}
|