You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
956 lines
38 KiB
PHP
956 lines
38 KiB
PHP
<?php
|
|
declare(strict_types=1);
|
|
|
|
namespace App\Services\Cloud {
|
|
/** Isolated encrypted envelope for PHP-only finalizer recovery tests. */
|
|
final class CloudSecrets
|
|
{
|
|
private static array $values = [];
|
|
private static bool $available = true;
|
|
|
|
public static function setAvailable(bool $available): void
|
|
{
|
|
self::$available = $available;
|
|
}
|
|
|
|
public static function encrypt(array $value, string $purpose): string
|
|
{
|
|
$key = bin2hex(random_bytes(16));
|
|
self::$values[$key] = $value;
|
|
return 'undispatched-finalizer-test:' . $key;
|
|
}
|
|
|
|
public static function decrypt(string $value, string $purpose): array
|
|
{
|
|
if (!self::$available) {
|
|
throw new \RuntimeException(
|
|
'Finalizer test key is temporarily unavailable.'
|
|
);
|
|
}
|
|
$key = substr(
|
|
$value,
|
|
strlen('undispatched-finalizer-test:')
|
|
);
|
|
if (!array_key_exists($key, self::$values)) {
|
|
throw new \RuntimeException('Missing finalizer test payload.');
|
|
}
|
|
return self::$values[$key];
|
|
}
|
|
}
|
|
|
|
/** Any Agent lookup is a test failure for generation-zero recovery. */
|
|
final class CloudAgentSettingsService
|
|
{
|
|
public static int $requests = 0;
|
|
|
|
public static function clientForHost(
|
|
int $hostId,
|
|
float $timeoutSeconds = 120.0
|
|
): object {
|
|
self::$requests++;
|
|
throw new \RuntimeException(
|
|
'Undispatched finalizer recovery contacted the Agent.'
|
|
);
|
|
}
|
|
|
|
public static function clientForConfiguration(
|
|
int $hostId,
|
|
float $timeoutSeconds = 120.0
|
|
): object {
|
|
return self::clientForHost($hostId, $timeoutSeconds);
|
|
}
|
|
}
|
|
}
|
|
namespace {
|
|
require dirname(__DIR__) . '/vendor/autoload.php';
|
|
|
|
use App\Models\CloudJob;
|
|
use App\Services\Cloud\CloudAutomaticOperationRecoveryService;
|
|
use App\Services\Cloud\CloudAgentSettingsService;
|
|
use App\Services\Cloud\CloudDispatchLedger;
|
|
use App\Services\Cloud\CloudJobService;
|
|
use App\Services\Cloud\CloudSecrets;
|
|
use App\Services\Cloud\CloudWorkerService;
|
|
use Illuminate\Database\Capsule\Manager as Capsule;
|
|
use Illuminate\Database\Schema\Blueprint;
|
|
|
|
$assert = static function (bool $condition, string $message): void {
|
|
if (!$condition) {
|
|
throw new RuntimeException($message);
|
|
}
|
|
};
|
|
|
|
$capsule = new Capsule();
|
|
$capsule->addConnection([
|
|
'driver' => 'sqlite', 'database' => ':memory:',
|
|
], 'default');
|
|
$capsule->setAsGlobal();
|
|
$capsule->bootEloquent();
|
|
$schema = $capsule->schema('default');
|
|
$schema->create('user', static function (Blueprint $table): void {
|
|
$table->increments('id');
|
|
$table->string('money')->default('0.00');
|
|
$table->integer('enable')->default(1);
|
|
$table->integer('is_admin')->default(0);
|
|
});
|
|
$schema->create('cloud_jobs', static function (Blueprint $table): void {
|
|
$table->increments('id');
|
|
$table->string('operation_id')->unique();
|
|
$table->string('request_idempotency_key')->unique();
|
|
$table->string('request_fingerprint');
|
|
$table->integer('parent_job_id')->nullable();
|
|
$table->integer('order_id')->nullable();
|
|
$table->integer('invoice_id')->nullable();
|
|
$table->integer('instance_id')->nullable();
|
|
$table->integer('snapshot_id')->nullable();
|
|
$table->integer('public_network_transition_id')->nullable();
|
|
$table->integer('vpc_repair_root_job_id')->nullable();
|
|
$table->integer('user_id');
|
|
$table->integer('host_id')->nullable();
|
|
$table->string('kind');
|
|
$table->string('execution_mode');
|
|
$table->string('phase');
|
|
$table->string('outcome')->nullable();
|
|
$table->string('status');
|
|
$table->text('sealed_payload');
|
|
$table->integer('attempts')->default(0);
|
|
$table->integer('dispatch_attempts')->default(0);
|
|
$table->integer('dispatch_deadline_at')->default(0);
|
|
$table->integer('dispatch_started_at')->nullable();
|
|
$table->integer('agent_accepted_at')->nullable();
|
|
$table->integer('receipt_deadline_at')->nullable();
|
|
$table->integer('terminal_at')->nullable();
|
|
$table->boolean('effect_may_have_applied')->nullable();
|
|
$table->boolean('safe_to_resubmit')->nullable();
|
|
$table->string('effect_resolution');
|
|
$table->integer('effect_resolved_at')->nullable();
|
|
$table->integer('effect_resolved_by')->nullable();
|
|
$table->string('effect_resolution_reason')->default('');
|
|
$table->string('completion_event_id')->nullable();
|
|
$table->integer('read_at')->nullable();
|
|
$table->integer('dispatched_at')->nullable();
|
|
$table->string('lease_token')->nullable();
|
|
$table->integer('lease_until')->default(0);
|
|
$table->integer('available_at')->default(0);
|
|
$table->string('error_code')->nullable();
|
|
$table->boolean('finalization_incident')->default(0);
|
|
$table->string('finalization_error_code')->nullable();
|
|
$table->text('result_json')->nullable();
|
|
$table->integer('created_at');
|
|
$table->integer('updated_at');
|
|
});
|
|
$schema->create('cloud_public_network_transitions',
|
|
static function (Blueprint $table): void {
|
|
$table->increments('id');
|
|
$table->string('state');
|
|
$table->string('effect_resolution');
|
|
$table->string('last_error_code')->default('');
|
|
});
|
|
$schema->create('cloud_instances', static function (Blueprint $table): void {
|
|
$table->increments('id');
|
|
$table->string('uuid');
|
|
$table->integer('host_id');
|
|
$table->integer('user_id');
|
|
$table->integer('order_id')->nullable();
|
|
$table->string('name');
|
|
$table->string('status');
|
|
$table->string('service_status');
|
|
$table->string('network_status');
|
|
$table->text('spec_json');
|
|
$table->text('sealed_credential')->nullable();
|
|
$table->integer('created_at');
|
|
$table->integer('updated_at');
|
|
});
|
|
$schema->create('cloud_agent_completion_events',
|
|
static function (Blueprint $table): void {
|
|
$table->string('event_id')->primary();
|
|
$table->integer('host_id');
|
|
$table->string('operation_id');
|
|
$table->string('payload_sha256');
|
|
$table->string('outcome');
|
|
$table->boolean('effect_may_have_applied');
|
|
$table->boolean('safe_to_resubmit');
|
|
$table->string('error_code')->default('');
|
|
$table->text('result_json')->nullable();
|
|
$table->string('disposition')->default('received');
|
|
$table->integer('received_at');
|
|
$table->integer('applied_at')->nullable();
|
|
$table->unique(['host_id', 'operation_id']);
|
|
});
|
|
$schema->create('cloud_templates',
|
|
static function (Blueprint $table): void {
|
|
$table->string('id')->primary();
|
|
$table->string('catalog_key')->default('test-catalog');
|
|
$table->string('version_label')->default('test-version');
|
|
$table->boolean('is_default')->default(true);
|
|
$table->integer('active_job_id')->nullable();
|
|
$table->string('state')->default('active');
|
|
});
|
|
$schema->create('cloud_user_isos',
|
|
static function (Blueprint $table): void {
|
|
$table->string('id')->primary();
|
|
$table->integer('active_job_id')->nullable();
|
|
$table->string('state')->default('active');
|
|
});
|
|
$schema->create('cloud_template_replicas',
|
|
static function (Blueprint $table): void {
|
|
$table->integer('host_id');
|
|
$table->string('template_id');
|
|
$table->integer('revision')->default(0);
|
|
$table->string('generation_token')->default('');
|
|
$table->string('state')->default('pending');
|
|
$table->integer('image_bytes')->default(0);
|
|
$table->string('sha256')->default('');
|
|
$table->string('operation_id')->default('');
|
|
$table->integer('job_id')->nullable();
|
|
$table->string('error_code')->default('');
|
|
$table->integer('created_at')->default(0);
|
|
$table->integer('updated_at')->default(0);
|
|
$table->primary(['host_id', 'template_id']);
|
|
});
|
|
$schema->create('cloud_user_iso_replicas',
|
|
static function (Blueprint $table): void {
|
|
$table->integer('host_id');
|
|
$table->string('user_iso_id');
|
|
$table->integer('user_id');
|
|
$table->string('state')->default('pending');
|
|
$table->integer('size_bytes')->default(0);
|
|
$table->string('sha256')->default('');
|
|
$table->string('operation_id')->default('');
|
|
$table->integer('job_id')->nullable();
|
|
$table->string('error_code')->default('');
|
|
$table->integer('created_at')->default(0);
|
|
$table->integer('updated_at')->default(0);
|
|
$table->primary(['host_id', 'user_iso_id']);
|
|
});
|
|
$schema->create('cloud_hosts', static function (Blueprint $table): void {
|
|
$table->increments('id');
|
|
});
|
|
$schema->create('cloud_host_network_attachments',
|
|
static function (Blueprint $table): void {
|
|
$table->increments('id');
|
|
$table->integer('host_id');
|
|
$table->integer('line_id');
|
|
$table->string('network_selector');
|
|
$table->string('state');
|
|
$table->integer('priority')->default(0);
|
|
});
|
|
$schema->create('cloud_public_ip_pools',
|
|
static function (Blueprint $table): void {
|
|
$table->increments('id');
|
|
$table->string('pool_key');
|
|
$table->integer('host_id');
|
|
$table->integer('network_attachment_id');
|
|
$table->integer('address_family');
|
|
$table->string('cidr');
|
|
$table->string('gateway');
|
|
$table->boolean('gateway_on_link');
|
|
$table->integer('guest_prefix_length');
|
|
$table->text('dns_servers_json');
|
|
$table->text('reserved_ips_json');
|
|
$table->string('allocation_start')->nullable();
|
|
$table->string('allocation_end')->nullable();
|
|
$table->string('allocation_cursor')->nullable();
|
|
$table->integer('max_allocations');
|
|
$table->string('state');
|
|
$table->integer('priority')->default(0);
|
|
$table->integer('desired_revision');
|
|
$table->integer('observed_revision');
|
|
$table->string('sync_state');
|
|
});
|
|
$schema->create('cloud_offers', static function (Blueprint $table): void {
|
|
$table->increments('id');
|
|
$table->integer('stock')->default(-1);
|
|
$table->integer('updated_at')->default(0);
|
|
});
|
|
$schema->create('cloud_orders', static function (Blueprint $table): void {
|
|
$table->increments('id');
|
|
$table->integer('user_id');
|
|
$table->integer('plan_id');
|
|
$table->integer('offer_id');
|
|
$table->integer('resource_pool_id');
|
|
$table->integer('host_id')->nullable();
|
|
$table->integer('reservation_id')->nullable();
|
|
$table->string('idempotency_key');
|
|
$table->string('request_hash');
|
|
$table->string('amount');
|
|
$table->string('currency')->default('USD');
|
|
$table->string('status');
|
|
$table->text('spec_json');
|
|
$table->text('sealed_payload')->nullable();
|
|
$table->integer('instance_id')->nullable();
|
|
$table->integer('refunded_at')->nullable();
|
|
$table->integer('stock_reserved_at')->nullable();
|
|
$table->integer('stock_released_at')->nullable();
|
|
$table->integer('created_at');
|
|
$table->integer('updated_at');
|
|
});
|
|
$schema->create('cloud_resource_reservations',
|
|
static function (Blueprint $table): void {
|
|
$table->increments('id');
|
|
$table->string('reservation_key');
|
|
$table->string('operation_id');
|
|
$table->integer('order_id');
|
|
$table->integer('backup_restore_id')->nullable();
|
|
$table->integer('offer_id');
|
|
$table->integer('resource_pool_id');
|
|
$table->integer('host_id');
|
|
$table->integer('user_id');
|
|
$table->integer('instance_id')->nullable();
|
|
$table->integer('network_attachment_id')->nullable();
|
|
$table->integer('public_ip_pool_id')->nullable();
|
|
$table->integer('public_ip_id')->nullable();
|
|
$table->integer('cpu_cores');
|
|
$table->integer('memory_mib');
|
|
$table->integer('disk_gib');
|
|
$table->string('state');
|
|
$table->integer('expires_at')->nullable();
|
|
$table->integer('released_at')->nullable();
|
|
$table->string('release_reason')->default('');
|
|
$table->integer('created_at');
|
|
$table->integer('updated_at');
|
|
});
|
|
$schema->create('cloud_public_ips',
|
|
static function (Blueprint $table): void {
|
|
$table->increments('id');
|
|
$table->integer('host_id');
|
|
$table->integer('public_ip_pool_id')->nullable();
|
|
$table->integer('address_family');
|
|
$table->string('address');
|
|
$table->binary('address_binary');
|
|
$table->integer('prefix_length');
|
|
$table->string('state');
|
|
$table->integer('reservation_id')->nullable();
|
|
$table->integer('reservation_ip_claim_id')->nullable();
|
|
$table->integer('instance_id')->nullable();
|
|
$table->string('operation_id')->nullable();
|
|
$table->integer('created_at');
|
|
$table->integer('updated_at');
|
|
});
|
|
$schema->create('cloud_resource_reservation_ips',
|
|
static function (Blueprint $table): void {
|
|
$table->increments('id');
|
|
$table->integer('reservation_id');
|
|
$table->integer('address_family');
|
|
$table->integer('claim_generation')->default(1);
|
|
$table->integer('active_claim')->nullable()->default(1);
|
|
$table->integer('network_attachment_id');
|
|
$table->integer('public_ip_pool_id');
|
|
$table->integer('public_ip_id');
|
|
$table->string('operation_id');
|
|
$table->string('state');
|
|
$table->integer('created_at');
|
|
$table->integer('updated_at');
|
|
$table->integer('released_at')->nullable();
|
|
$table->string('release_reason')->default('');
|
|
});
|
|
$schema->create('cloud_invoices',
|
|
static function (Blueprint $table): void {
|
|
$table->increments('id');
|
|
$table->string('invoice_no');
|
|
$table->string('purpose');
|
|
$table->integer('instance_id')->nullable();
|
|
$table->integer('order_id')->nullable();
|
|
$table->integer('user_id');
|
|
$table->integer('plan_id')->nullable();
|
|
$table->integer('offer_id')->nullable();
|
|
$table->integer('renewal_id')->nullable();
|
|
$table->integer('renews_expires_at')->nullable();
|
|
$table->integer('due_at');
|
|
$table->string('amount');
|
|
$table->string('currency')->default('USD');
|
|
$table->integer('validity_days');
|
|
$table->integer('traffic_quota_bytes');
|
|
$table->string('status');
|
|
$table->string('paid_via')->nullable();
|
|
$table->string('payment_reference')->nullable();
|
|
$table->integer('paid_at')->nullable();
|
|
$table->integer('created_at');
|
|
$table->integer('updated_at');
|
|
});
|
|
|
|
$now = time();
|
|
$operationId = str_repeat('ad', 16);
|
|
$uuid = '00000000-0000-4000-8000-000000000501';
|
|
$payload = [
|
|
'method' => 'POST',
|
|
'request_target' => '/v1/vms/' . $uuid . '/settings',
|
|
'body' => [
|
|
'hostname' => 'local-finalizer.example',
|
|
'root_password' => 'Secret9!Password',
|
|
'confirm_restart' => true,
|
|
],
|
|
'owner_id' => 1,
|
|
'can_manage' => false,
|
|
'credential' => [
|
|
'password' => 'Secret9!Password',
|
|
'login_name' => 'root',
|
|
'auth_mode' => 'password',
|
|
],
|
|
'settings_fields' => ['hostname', 'password'],
|
|
];
|
|
Capsule::table('user')->insert([
|
|
'id' => 1, 'enable' => 1, 'is_admin' => 0,
|
|
]);
|
|
Capsule::table('cloud_instances')->insert([
|
|
'id' => 501,
|
|
'uuid' => $uuid,
|
|
'host_id' => 7,
|
|
'user_id' => 1,
|
|
'name' => 'local-finalizer',
|
|
'status' => 'active',
|
|
'service_status' => 'exempt',
|
|
'network_status' => 'active',
|
|
'spec_json' => json_encode(
|
|
['managed' => true],
|
|
JSON_THROW_ON_ERROR
|
|
),
|
|
'sealed_credential' => null,
|
|
'created_at' => $now - 3600,
|
|
'updated_at' => $now - 3600,
|
|
]);
|
|
$originalEnvelope = CloudSecrets::encrypt($payload, 'job-payload');
|
|
$originalResult = json_encode(
|
|
['cause_code' => 'local_finalization_failed'],
|
|
JSON_THROW_ON_ERROR
|
|
);
|
|
Capsule::table('cloud_jobs')->insert([
|
|
'id' => 501,
|
|
'operation_id' => $operationId,
|
|
'request_idempotency_key' => $operationId,
|
|
'request_fingerprint' => hash('sha256', $operationId),
|
|
'parent_job_id' => null,
|
|
'order_id' => null,
|
|
'invoice_id' => null,
|
|
'instance_id' => 501,
|
|
'snapshot_id' => null,
|
|
'user_id' => 1,
|
|
'host_id' => 7,
|
|
'kind' => 'instance_settings',
|
|
'execution_mode' => 'agent',
|
|
'phase' => 'terminal',
|
|
'outcome' => 'failed',
|
|
'status' => 'failed',
|
|
'sealed_payload' => $originalEnvelope,
|
|
'attempts' => 0,
|
|
'dispatch_attempts' => 0,
|
|
'dispatch_deadline_at' => $now - 30,
|
|
'terminal_at' => $now - 20,
|
|
'effect_may_have_applied' => 0,
|
|
'safe_to_resubmit' => 0,
|
|
'effect_resolution' => 'not_applied',
|
|
'effect_resolution_reason' => '',
|
|
'lease_until' => 0,
|
|
'available_at' => 0,
|
|
'error_code' => 'local_finalization_failed',
|
|
'finalization_incident' => 1,
|
|
'finalization_error_code' => 'recovery_ledger_conflict',
|
|
'result_json' => $originalResult,
|
|
'created_at' => $now - 30,
|
|
'updated_at' => $now - 20,
|
|
]);
|
|
|
|
$before = CloudJob::findOrFail(501);
|
|
$assert(CloudJobService::blockingQueryForInstance(501)->exists(),
|
|
'An undispatched local finalizer incident did not block mutations.');
|
|
$service = new CloudAutomaticOperationRecoveryService();
|
|
$assert($service->tick(),
|
|
'The undispatched PHP-only finalizer was not claimed.');
|
|
$after = CloudJob::findOrFail(501);
|
|
$scrubbed = CloudSecrets::decrypt(
|
|
(string) $after->sealed_payload,
|
|
'job-payload'
|
|
);
|
|
$assert((string) $after->phase === (string) $before->phase
|
|
&& (string) $after->outcome === (string) $before->outcome
|
|
&& (string) $after->status === (string) $before->status
|
|
&& (int) $after->terminal_at === (int) $before->terminal_at
|
|
&& (string) $after->error_code === (string) $before->error_code
|
|
&& (string) $after->result_json === (string) $before->result_json
|
|
&& $after->completion_event_id === $before->completion_event_id
|
|
&& !(bool) $after->effect_may_have_applied
|
|
&& (bool) $after->safe_to_resubmit
|
|
&& (string) $after->effect_resolution === 'not_applied'
|
|
&& (string) $after->effect_resolution_reason
|
|
=== 'automatic_local_not_dispatched'
|
|
&& (int) $after->effect_resolved_at > 0
|
|
&& !(bool) $after->finalization_incident
|
|
&& $after->finalization_error_code === null
|
|
&& $after->lease_token === null
|
|
&& (int) $after->lease_until === 0
|
|
&& (int) $after->available_at === 0
|
|
&& (string) $after->sealed_payload !== $originalEnvelope
|
|
&& !isset($scrubbed['credential'])
|
|
&& !isset($scrubbed['body']['root_password'])
|
|
&& !CloudJobService::blockingQueryForInstance(501)->exists(),
|
|
'The PHP-only finalizer changed terminal audit data or failed to converge safely.');
|
|
$updatedAt = (int) $after->updated_at;
|
|
$assert(!$service->tick(),
|
|
'A converged PHP-only finalizer was claimed a second time.');
|
|
$afterSecondTick = CloudJob::findOrFail(501);
|
|
$assert((int) $afterSecondTick->updated_at === $updatedAt,
|
|
'A converged PHP-only finalizer repeated its side effects.');
|
|
|
|
// A missing/corrupt encrypted escrow remains a concrete payload incident;
|
|
// it must not degrade into a generic ledger-conflict message.
|
|
Capsule::table('cloud_instances')->insert([
|
|
'id' => 502,
|
|
'uuid' => '00000000-0000-4000-8000-000000000502',
|
|
'host_id' => 7,
|
|
'user_id' => 1,
|
|
'name' => 'local-finalizer-corrupt-payload',
|
|
'status' => 'active',
|
|
'service_status' => 'exempt',
|
|
'network_status' => 'active',
|
|
'spec_json' => json_encode(['managed' => true], JSON_THROW_ON_ERROR),
|
|
'sealed_credential' => null,
|
|
'created_at' => $now - 3600,
|
|
'updated_at' => $now - 3600,
|
|
]);
|
|
$corruptOperationId = str_repeat('ae', 16);
|
|
Capsule::table('cloud_jobs')->insert([
|
|
'id' => 502,
|
|
'operation_id' => $corruptOperationId,
|
|
'request_idempotency_key' => $corruptOperationId,
|
|
'request_fingerprint' => hash('sha256', $corruptOperationId),
|
|
'parent_job_id' => null,
|
|
'order_id' => null,
|
|
'invoice_id' => null,
|
|
'instance_id' => 502,
|
|
'snapshot_id' => null,
|
|
'user_id' => 1,
|
|
'host_id' => 7,
|
|
'kind' => 'instance_settings',
|
|
'execution_mode' => 'agent',
|
|
'phase' => 'terminal',
|
|
'outcome' => 'failed',
|
|
'status' => 'failed',
|
|
'sealed_payload' => 'undispatched-finalizer-test:missing',
|
|
'attempts' => 0,
|
|
'dispatch_attempts' => 0,
|
|
'dispatch_deadline_at' => $now - 30,
|
|
'terminal_at' => $now - 20,
|
|
'effect_may_have_applied' => 0,
|
|
'safe_to_resubmit' => 0,
|
|
'effect_resolution' => 'not_applied',
|
|
'effect_resolution_reason' => '',
|
|
'lease_until' => 0,
|
|
'available_at' => 0,
|
|
'error_code' => 'local_finalization_failed',
|
|
'finalization_incident' => 1,
|
|
'finalization_error_code' => 'recovery_payload_unavailable',
|
|
'result_json' => $originalResult,
|
|
'created_at' => $now - 30,
|
|
'updated_at' => $now - 20,
|
|
]);
|
|
$assert($service->tick(),
|
|
'The corrupt undispatched recovery payload was not checked.');
|
|
$corruptPayloadJob = CloudJob::findOrFail(502);
|
|
$assert((bool) $corruptPayloadJob->finalization_incident
|
|
&& (string) $corruptPayloadJob->finalization_error_code
|
|
=== 'recovery_payload_unavailable'
|
|
&& (string) $corruptPayloadJob->sealed_payload
|
|
=== 'undispatched-finalizer-test:missing'
|
|
&& (int) $corruptPayloadJob->lease_until >= time() + 290
|
|
&& (int) $corruptPayloadJob->available_at >= time() + 290
|
|
&& CloudJobService::blockingQueryForInstance(502)->exists(),
|
|
'A corrupt PHP-only recovery payload lost its concrete incident or resource fence.');
|
|
|
|
// A create that never crossed the dispatch boundary has definitive
|
|
// no-effect proof, but its paid ledgers still require the create-specific
|
|
// atomic finalizer. Temporary key or database damage must retain escrow
|
|
// and every fence; once repaired, the worker refunds/releases exactly
|
|
// once and never sends the original create request to an Agent.
|
|
$createOperationId = str_repeat('af', 16);
|
|
$createPayload = [
|
|
'method' => 'POST',
|
|
'request_target' => '/v1/vms',
|
|
'body' => [
|
|
'operation_id' => $createOperationId,
|
|
'source' => 'cloud',
|
|
'name' => 'undispatched-create',
|
|
'root_password' => 'Secret9!Password',
|
|
],
|
|
'credential' => [
|
|
'password' => 'Secret9!Password',
|
|
'login_name' => 'root',
|
|
'auth_mode' => 'password',
|
|
],
|
|
];
|
|
$createEnvelope = CloudSecrets::encrypt(
|
|
$createPayload,
|
|
'job-payload'
|
|
);
|
|
Capsule::table('user')->where('id', 1)->update(['money' => '0.00']);
|
|
Capsule::table('cloud_hosts')->insert(['id' => 7]);
|
|
Capsule::table('cloud_host_network_attachments')->insert([
|
|
'id' => 601,
|
|
'host_id' => 7,
|
|
'line_id' => 601,
|
|
'network_selector' => 'bridge:br0',
|
|
'state' => 'active',
|
|
'priority' => 0,
|
|
]);
|
|
$publicPool = [
|
|
'host_id' => 7,
|
|
'network_attachment_id' => 601,
|
|
'gateway_on_link' => 0,
|
|
'reserved_ips_json' => '[]',
|
|
'allocation_cursor' => null,
|
|
'state' => 'active',
|
|
'priority' => 0,
|
|
'desired_revision' => 1,
|
|
'observed_revision' => 1,
|
|
'sync_state' => 'applied',
|
|
];
|
|
Capsule::table('cloud_public_ip_pools')->insert([
|
|
array_merge($publicPool, [
|
|
'id' => 601,
|
|
'pool_key' => str_repeat('1', 32),
|
|
'address_family' => 4,
|
|
'cidr' => '8.8.8.0/24',
|
|
'gateway' => '8.8.8.1',
|
|
'guest_prefix_length' => 24,
|
|
'dns_servers_json' => '["1.1.1.1"]',
|
|
'allocation_start' => null,
|
|
'allocation_end' => null,
|
|
'max_allocations' => 0,
|
|
]),
|
|
array_merge($publicPool, [
|
|
'id' => 602,
|
|
'pool_key' => str_repeat('2', 32),
|
|
'address_family' => 6,
|
|
'cidr' => '2404:c140:156:3a::/64',
|
|
'gateway' => '2404:c140:156::1',
|
|
'gateway_on_link' => 1,
|
|
'guest_prefix_length' => 64,
|
|
'dns_servers_json' => '["2606:4700:4700::1111"]',
|
|
'allocation_start' => '2404:c140:156:3a::10',
|
|
'allocation_end' => '2404:c140:156:3a::ff',
|
|
'max_allocations' => 8,
|
|
]),
|
|
]);
|
|
Capsule::table('cloud_offers')->insert([
|
|
'id' => 601,
|
|
'stock' => 0,
|
|
'updated_at' => $now - 30,
|
|
]);
|
|
Capsule::table('cloud_orders')->insert([
|
|
'id' => 601,
|
|
'user_id' => 1,
|
|
'plan_id' => 601,
|
|
'offer_id' => 601,
|
|
'resource_pool_id' => 601,
|
|
'host_id' => 7,
|
|
'reservation_id' => 601,
|
|
'idempotency_key' => 'undispatched-create-order',
|
|
'request_hash' => hash('sha256', 'undispatched-create-order'),
|
|
'amount' => '12.34',
|
|
'currency' => 'USD',
|
|
'status' => 'queued',
|
|
'spec_json' => '{}',
|
|
'sealed_payload' => null,
|
|
'instance_id' => null,
|
|
'refunded_at' => null,
|
|
'stock_reserved_at' => $now - 60,
|
|
'stock_released_at' => null,
|
|
'created_at' => $now - 120,
|
|
'updated_at' => $now - 30,
|
|
]);
|
|
Capsule::table('cloud_resource_reservations')->insert([
|
|
'id' => 601,
|
|
'reservation_key' => 'undispatched-create-reservation',
|
|
'operation_id' => $createOperationId,
|
|
'order_id' => 601,
|
|
'backup_restore_id' => null,
|
|
'offer_id' => 601,
|
|
'resource_pool_id' => 601,
|
|
'host_id' => 7,
|
|
'user_id' => 1,
|
|
'instance_id' => null,
|
|
'network_attachment_id' => null,
|
|
'public_ip_pool_id' => null,
|
|
'public_ip_id' => null,
|
|
'cpu_cores' => 1,
|
|
'memory_mib' => 512,
|
|
'disk_gib' => 10,
|
|
'state' => 'held',
|
|
'expires_at' => null,
|
|
'released_at' => null,
|
|
'release_reason' => '',
|
|
'created_at' => $now - 120,
|
|
'updated_at' => $now - 30,
|
|
]);
|
|
Capsule::table('cloud_public_ips')->insert([
|
|
[
|
|
'id' => 601,
|
|
'host_id' => 7,
|
|
'public_ip_pool_id' => 601,
|
|
'address_family' => 4,
|
|
'address' => '8.8.8.61',
|
|
'address_binary' => inet_pton('8.8.8.61'),
|
|
'prefix_length' => 24,
|
|
'state' => 'reserved',
|
|
'reservation_id' => 601,
|
|
'reservation_ip_claim_id' => 601,
|
|
'instance_id' => null,
|
|
'operation_id' => $createOperationId,
|
|
'created_at' => $now - 120,
|
|
'updated_at' => $now - 30,
|
|
],
|
|
[
|
|
'id' => 602,
|
|
'host_id' => 7,
|
|
'public_ip_pool_id' => 602,
|
|
'address_family' => 6,
|
|
'address' => '2404:c140:156:3a::61',
|
|
'address_binary' => inet_pton('2404:c140:156:3a::61'),
|
|
'prefix_length' => 64,
|
|
'state' => 'reserved',
|
|
'reservation_id' => 601,
|
|
'reservation_ip_claim_id' => 602,
|
|
'instance_id' => null,
|
|
'operation_id' => $createOperationId,
|
|
'created_at' => $now - 120,
|
|
'updated_at' => $now - 30,
|
|
],
|
|
]);
|
|
Capsule::table('cloud_resource_reservation_ips')->insert([
|
|
[
|
|
'id' => 601,
|
|
'reservation_id' => 601,
|
|
'address_family' => 4,
|
|
'claim_generation' => 1,
|
|
'active_claim' => 1,
|
|
'network_attachment_id' => 601,
|
|
'public_ip_pool_id' => 601,
|
|
'public_ip_id' => 601,
|
|
'operation_id' => $createOperationId,
|
|
'state' => 'reserved',
|
|
'created_at' => $now - 120,
|
|
'updated_at' => $now - 30,
|
|
'released_at' => null,
|
|
'release_reason' => '',
|
|
],
|
|
[
|
|
'id' => 602,
|
|
'reservation_id' => 601,
|
|
'address_family' => 6,
|
|
'claim_generation' => 1,
|
|
'active_claim' => 1,
|
|
'network_attachment_id' => 601,
|
|
'public_ip_pool_id' => 602,
|
|
'public_ip_id' => 602,
|
|
'operation_id' => $createOperationId,
|
|
'state' => 'reserved',
|
|
'created_at' => $now - 120,
|
|
'updated_at' => $now - 30,
|
|
'released_at' => null,
|
|
'release_reason' => '',
|
|
],
|
|
]);
|
|
Capsule::table('cloud_jobs')->insert([
|
|
'id' => 601,
|
|
'operation_id' => $createOperationId,
|
|
'request_idempotency_key' => $createOperationId,
|
|
'request_fingerprint' => hash('sha256', $createOperationId),
|
|
'parent_job_id' => null,
|
|
'order_id' => 601,
|
|
'invoice_id' => null,
|
|
'instance_id' => null,
|
|
'snapshot_id' => null,
|
|
'user_id' => 1,
|
|
'host_id' => 7,
|
|
'kind' => 'create',
|
|
'execution_mode' => 'agent',
|
|
'phase' => 'accepted',
|
|
'outcome' => null,
|
|
'status' => 'queued',
|
|
'sealed_payload' => $createEnvelope,
|
|
'attempts' => 0,
|
|
'dispatch_attempts' => 0,
|
|
'dispatch_deadline_at' => $now - 30,
|
|
'dispatch_started_at' => null,
|
|
'agent_accepted_at' => null,
|
|
'receipt_deadline_at' => null,
|
|
'terminal_at' => null,
|
|
'effect_may_have_applied' => null,
|
|
'safe_to_resubmit' => null,
|
|
'effect_resolution' => 'not_applicable',
|
|
'effect_resolved_at' => null,
|
|
'effect_resolved_by' => null,
|
|
'effect_resolution_reason' => '',
|
|
'completion_event_id' => null,
|
|
'read_at' => null,
|
|
'dispatched_at' => null,
|
|
'lease_token' => str_repeat('6', 32),
|
|
'lease_until' => $now + 180,
|
|
'available_at' => 0,
|
|
'error_code' => null,
|
|
'finalization_incident' => 0,
|
|
'finalization_error_code' => null,
|
|
'result_json' => $originalResult,
|
|
'created_at' => $now - 30,
|
|
'updated_at' => $now - 20,
|
|
]);
|
|
|
|
CloudSecrets::setAvailable(false);
|
|
$forceTerminalFailure = new ReflectionMethod(
|
|
CloudWorkerService::class,
|
|
'forceTerminalFailure'
|
|
);
|
|
$forceTerminalFailure->invoke(
|
|
null,
|
|
601,
|
|
str_repeat('6', 32),
|
|
false,
|
|
true,
|
|
'dispatch_not_sent'
|
|
);
|
|
$createAfterEmergency = CloudJob::findOrFail(601);
|
|
$assert((string) $createAfterEmergency->phase === 'terminal'
|
|
&& (string) $createAfterEmergency->outcome === 'failed'
|
|
&& (int) $createAfterEmergency->dispatch_attempts === 0
|
|
&& !(bool) $createAfterEmergency->effect_may_have_applied
|
|
&& !(bool) $createAfterEmergency->safe_to_resubmit
|
|
&& (string) $createAfterEmergency->effect_resolution
|
|
=== 'not_applied'
|
|
&& (bool) $createAfterEmergency->finalization_incident
|
|
&& (string) $createAfterEmergency->finalization_error_code
|
|
=== 'recovery_payload_unavailable'
|
|
&& (string) $createAfterEmergency->sealed_payload === $createEnvelope
|
|
&& (int) $createAfterEmergency->available_at >= time() + 290,
|
|
'Create emergency terminalization lost its generation-zero proof or escrow.');
|
|
Capsule::table('cloud_jobs')->where('id', 601)->update([
|
|
'available_at' => 0,
|
|
'lease_token' => null,
|
|
'lease_until' => 0,
|
|
]);
|
|
|
|
$createService = new CloudAutomaticOperationRecoveryService();
|
|
$assert($createService->tick(),
|
|
'An undispatched create with unavailable escrow was not claimed.');
|
|
$createAfterKeyFailure = CloudJob::findOrFail(601);
|
|
$assert((bool) $createAfterKeyFailure->finalization_incident
|
|
&& (string) $createAfterKeyFailure->finalization_error_code
|
|
=== 'recovery_payload_unavailable'
|
|
&& (string) $createAfterKeyFailure->sealed_payload
|
|
=== $createEnvelope
|
|
&& (string) Capsule::table('user')->where('id', 1)
|
|
->value('money') === '0.00'
|
|
&& (string) Capsule::table('cloud_resource_reservations')
|
|
->where('id', 601)->value('state') === 'held'
|
|
&& CloudAgentSettingsService::$requests === 0,
|
|
'Unavailable create escrow released money/resources or contacted Agent.');
|
|
|
|
CloudSecrets::setAvailable(true);
|
|
Capsule::table('cloud_jobs')->where('id', 601)->update([
|
|
'available_at' => 0,
|
|
'lease_token' => null,
|
|
'lease_until' => 0,
|
|
]);
|
|
$assert($createService->tick(),
|
|
'An undispatched create with a missing invoice was not retried.');
|
|
$createAfterLedgerFailure = CloudJob::findOrFail(601);
|
|
$assert((bool) $createAfterLedgerFailure->finalization_incident
|
|
&& (string) Capsule::table('user')->where('id', 1)
|
|
->value('money') === '0.00'
|
|
&& (string) Capsule::table('cloud_resource_reservations')
|
|
->where('id', 601)->value('state') === 'held'
|
|
&& Capsule::table('cloud_public_ips')
|
|
->where('reservation_id', 601)
|
|
->where('state', 'reserved')->count() === 2
|
|
&& CloudAgentSettingsService::$requests === 0,
|
|
'Missing create ledger data caused a partial refund/release or Agent call.');
|
|
|
|
Capsule::table('cloud_invoices')->insert([
|
|
'id' => 601,
|
|
'invoice_no' => 'VC-UNDISPATCHED-601',
|
|
'purpose' => 'provision',
|
|
'instance_id' => null,
|
|
'order_id' => 601,
|
|
'user_id' => 1,
|
|
'plan_id' => 601,
|
|
'offer_id' => 601,
|
|
'renewal_id' => null,
|
|
'renews_expires_at' => null,
|
|
'due_at' => $now - 120,
|
|
'amount' => '12.34',
|
|
'currency' => 'USD',
|
|
'validity_days' => 30,
|
|
'traffic_quota_bytes' => 0,
|
|
'status' => 'paid',
|
|
'paid_via' => 'balance',
|
|
'payment_reference' => 'undispatched-create-payment',
|
|
'paid_at' => $now - 60,
|
|
'created_at' => $now - 120,
|
|
'updated_at' => $now - 60,
|
|
]);
|
|
Capsule::table('cloud_jobs')->where('id', 601)->update([
|
|
'available_at' => 0,
|
|
'lease_token' => null,
|
|
'lease_until' => 0,
|
|
]);
|
|
$beforeCreateAudit = CloudJob::findOrFail(601);
|
|
$assert($createService->tick(),
|
|
'A repaired undispatched create was not finalized locally.');
|
|
$createAfterRecovery = CloudJob::findOrFail(601);
|
|
$createSafePayload = CloudSecrets::decrypt(
|
|
(string) $createAfterRecovery->sealed_payload,
|
|
'job-payload'
|
|
);
|
|
$assert((string) $createAfterRecovery->phase
|
|
=== (string) $beforeCreateAudit->phase
|
|
&& (string) $createAfterRecovery->outcome
|
|
=== (string) $beforeCreateAudit->outcome
|
|
&& (int) $createAfterRecovery->terminal_at
|
|
=== (int) $beforeCreateAudit->terminal_at
|
|
&& (string) $createAfterRecovery->error_code
|
|
=== (string) $beforeCreateAudit->error_code
|
|
&& !(bool) $createAfterRecovery->effect_may_have_applied
|
|
&& !(bool) $createAfterRecovery->safe_to_resubmit
|
|
&& (string) $createAfterRecovery->effect_resolution === 'not_applied'
|
|
&& (string) $createAfterRecovery->effect_resolution_reason
|
|
=== 'automatic_local_not_dispatched'
|
|
&& !(bool) $createAfterRecovery->finalization_incident
|
|
&& $createAfterRecovery->finalization_error_code === null
|
|
&& !isset($createSafePayload['credential'])
|
|
&& !isset($createSafePayload['body']['root_password'])
|
|
&& (string) Capsule::table('user')->where('id', 1)
|
|
->value('money') === '12.34'
|
|
&& (string) Capsule::table('cloud_invoices')->where('id', 601)
|
|
->value('status') === 'refunded'
|
|
&& Capsule::table('cloud_orders')->where('id', 601)
|
|
->value('refunded_at') !== null
|
|
&& Capsule::table('cloud_orders')->where('id', 601)
|
|
->value('stock_released_at') !== null
|
|
&& (int) Capsule::table('cloud_offers')->where('id', 601)
|
|
->value('stock') === 1
|
|
&& (string) Capsule::table('cloud_resource_reservations')
|
|
->where('id', 601)->value('state') === 'released'
|
|
&& Capsule::table('cloud_public_ips')
|
|
->whereIn('id', [601, 602])
|
|
->where('state', 'available')->count() === 2
|
|
&& Capsule::table('cloud_public_ips')
|
|
->whereIn('id', [601, 602])
|
|
->whereNotNull('reservation_id')->count() === 0
|
|
&& Capsule::table('cloud_resource_reservation_ips')
|
|
->where('reservation_id', 601)
|
|
->where('state', 'released')->count() === 2
|
|
&& CloudAgentSettingsService::$requests === 0,
|
|
'Recovered create did not atomically refund, release, scrub, and close.');
|
|
|
|
$createUpdatedAt = (int) $createAfterRecovery->updated_at;
|
|
$assert(!$createService->tick(),
|
|
'A converged undispatched create was claimed a second time.');
|
|
$assert((string) Capsule::table('user')->where('id', 1)
|
|
->value('money') === '12.34'
|
|
&& (int) Capsule::table('cloud_offers')->where('id', 601)
|
|
->value('stock') === 1
|
|
&& (int) CloudJob::findOrFail(601)->updated_at === $createUpdatedAt
|
|
&& CloudAgentSettingsService::$requests === 0,
|
|
'Undispatched create recovery repeated refund/release or contacted Agent.');
|
|
|
|
echo "Cloud undispatched finalizer recovery tests passed.\n";
|
|
}
|