You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
366 lines
15 KiB
PHP
366 lines
15 KiB
PHP
<?php
|
|
declare(strict_types=1);
|
|
|
|
namespace App\Services\Cloud {
|
|
/** Isolated encrypted envelope for PHP-only finalizer recovery tests. */
|
|
final class CloudSecrets
|
|
{
|
|
private static array $values = [];
|
|
|
|
public static function encrypt(array $value, string $purpose): string
|
|
{
|
|
$key = bin2hex(random_bytes(16));
|
|
self::$values[$key] = $value;
|
|
return 'undispatched-finalizer-test:' . $key;
|
|
}
|
|
|
|
public static function decrypt(string $value, string $purpose): array
|
|
{
|
|
$key = substr(
|
|
$value,
|
|
strlen('undispatched-finalizer-test:')
|
|
);
|
|
if (!array_key_exists($key, self::$values)) {
|
|
throw new \RuntimeException('Missing finalizer test payload.');
|
|
}
|
|
return self::$values[$key];
|
|
}
|
|
}
|
|
}
|
|
|
|
namespace {
|
|
require dirname(__DIR__) . '/vendor/autoload.php';
|
|
|
|
use App\Models\CloudJob;
|
|
use App\Services\Cloud\CloudAutomaticOperationRecoveryService;
|
|
use App\Services\Cloud\CloudJobService;
|
|
use App\Services\Cloud\CloudSecrets;
|
|
use Illuminate\Database\Capsule\Manager as Capsule;
|
|
use Illuminate\Database\Schema\Blueprint;
|
|
|
|
$assert = static function (bool $condition, string $message): void {
|
|
if (!$condition) {
|
|
throw new RuntimeException($message);
|
|
}
|
|
};
|
|
|
|
$capsule = new Capsule();
|
|
$capsule->addConnection([
|
|
'driver' => 'sqlite', 'database' => ':memory:',
|
|
], 'default');
|
|
$capsule->setAsGlobal();
|
|
$capsule->bootEloquent();
|
|
$schema = $capsule->schema('default');
|
|
$schema->create('user', static function (Blueprint $table): void {
|
|
$table->increments('id');
|
|
$table->integer('enable')->default(1);
|
|
$table->integer('is_admin')->default(0);
|
|
});
|
|
$schema->create('cloud_jobs', static function (Blueprint $table): void {
|
|
$table->increments('id');
|
|
$table->string('operation_id')->unique();
|
|
$table->string('request_idempotency_key')->unique();
|
|
$table->string('request_fingerprint');
|
|
$table->integer('parent_job_id')->nullable();
|
|
$table->integer('order_id')->nullable();
|
|
$table->integer('invoice_id')->nullable();
|
|
$table->integer('instance_id')->nullable();
|
|
$table->integer('snapshot_id')->nullable();
|
|
$table->integer('user_id');
|
|
$table->integer('host_id')->nullable();
|
|
$table->string('kind');
|
|
$table->string('execution_mode');
|
|
$table->string('phase');
|
|
$table->string('outcome')->nullable();
|
|
$table->string('status');
|
|
$table->text('sealed_payload');
|
|
$table->integer('attempts')->default(0);
|
|
$table->integer('dispatch_attempts')->default(0);
|
|
$table->integer('dispatch_deadline_at')->default(0);
|
|
$table->integer('dispatch_started_at')->nullable();
|
|
$table->integer('agent_accepted_at')->nullable();
|
|
$table->integer('receipt_deadline_at')->nullable();
|
|
$table->integer('terminal_at')->nullable();
|
|
$table->boolean('effect_may_have_applied')->nullable();
|
|
$table->boolean('safe_to_resubmit')->nullable();
|
|
$table->string('effect_resolution');
|
|
$table->integer('effect_resolved_at')->nullable();
|
|
$table->integer('effect_resolved_by')->nullable();
|
|
$table->string('effect_resolution_reason')->default('');
|
|
$table->string('completion_event_id')->nullable();
|
|
$table->integer('read_at')->nullable();
|
|
$table->integer('dispatched_at')->nullable();
|
|
$table->string('lease_token')->nullable();
|
|
$table->integer('lease_until')->default(0);
|
|
$table->integer('available_at')->default(0);
|
|
$table->string('error_code')->nullable();
|
|
$table->boolean('finalization_incident')->default(0);
|
|
$table->string('finalization_error_code')->nullable();
|
|
$table->text('result_json')->nullable();
|
|
$table->integer('created_at');
|
|
$table->integer('updated_at');
|
|
});
|
|
$schema->create('cloud_instances', static function (Blueprint $table): void {
|
|
$table->increments('id');
|
|
$table->string('uuid');
|
|
$table->integer('host_id');
|
|
$table->integer('user_id');
|
|
$table->string('name');
|
|
$table->string('status');
|
|
$table->string('service_status');
|
|
$table->string('network_status');
|
|
$table->text('spec_json');
|
|
$table->text('sealed_credential')->nullable();
|
|
$table->integer('created_at');
|
|
$table->integer('updated_at');
|
|
});
|
|
$schema->create('cloud_agent_completion_events',
|
|
static function (Blueprint $table): void {
|
|
$table->string('event_id')->primary();
|
|
$table->integer('host_id');
|
|
$table->string('operation_id');
|
|
$table->string('payload_sha256');
|
|
$table->string('outcome');
|
|
$table->boolean('effect_may_have_applied');
|
|
$table->boolean('safe_to_resubmit');
|
|
$table->string('error_code')->default('');
|
|
$table->text('result_json')->nullable();
|
|
$table->string('disposition')->default('received');
|
|
$table->integer('received_at');
|
|
$table->integer('applied_at')->nullable();
|
|
$table->unique(['host_id', 'operation_id']);
|
|
});
|
|
$schema->create('cloud_templates',
|
|
static function (Blueprint $table): void {
|
|
$table->string('id')->primary();
|
|
$table->integer('active_job_id')->nullable();
|
|
$table->string('state')->default('active');
|
|
});
|
|
$schema->create('cloud_user_isos',
|
|
static function (Blueprint $table): void {
|
|
$table->string('id')->primary();
|
|
$table->integer('active_job_id')->nullable();
|
|
$table->string('state')->default('active');
|
|
});
|
|
$schema->create('cloud_template_replicas',
|
|
static function (Blueprint $table): void {
|
|
$table->integer('host_id');
|
|
$table->string('template_id');
|
|
$table->integer('revision')->default(0);
|
|
$table->string('generation_token')->default('');
|
|
$table->string('state')->default('pending');
|
|
$table->integer('image_bytes')->default(0);
|
|
$table->string('sha256')->default('');
|
|
$table->string('operation_id')->default('');
|
|
$table->integer('job_id')->nullable();
|
|
$table->string('error_code')->default('');
|
|
$table->integer('created_at')->default(0);
|
|
$table->integer('updated_at')->default(0);
|
|
$table->primary(['host_id', 'template_id']);
|
|
});
|
|
$schema->create('cloud_user_iso_replicas',
|
|
static function (Blueprint $table): void {
|
|
$table->integer('host_id');
|
|
$table->string('user_iso_id');
|
|
$table->integer('user_id');
|
|
$table->string('state')->default('pending');
|
|
$table->integer('size_bytes')->default(0);
|
|
$table->string('sha256')->default('');
|
|
$table->string('operation_id')->default('');
|
|
$table->integer('job_id')->nullable();
|
|
$table->string('error_code')->default('');
|
|
$table->integer('created_at')->default(0);
|
|
$table->integer('updated_at')->default(0);
|
|
$table->primary(['host_id', 'user_iso_id']);
|
|
});
|
|
|
|
$now = time();
|
|
$operationId = str_repeat('ad', 16);
|
|
$uuid = '00000000-0000-4000-8000-000000000501';
|
|
$payload = [
|
|
'method' => 'POST',
|
|
'request_target' => '/v1/vms/' . $uuid . '/settings',
|
|
'body' => [
|
|
'hostname' => 'local-finalizer.example',
|
|
'root_password' => 'Secret9!Password',
|
|
'confirm_restart' => true,
|
|
],
|
|
'owner_id' => 1,
|
|
'can_manage' => false,
|
|
'credential' => [
|
|
'password' => 'Secret9!Password',
|
|
'login_name' => 'root',
|
|
'auth_mode' => 'password',
|
|
],
|
|
'settings_fields' => ['hostname', 'password'],
|
|
];
|
|
Capsule::table('user')->insert([
|
|
'id' => 1, 'enable' => 1, 'is_admin' => 0,
|
|
]);
|
|
Capsule::table('cloud_instances')->insert([
|
|
'id' => 501,
|
|
'uuid' => $uuid,
|
|
'host_id' => 7,
|
|
'user_id' => 1,
|
|
'name' => 'local-finalizer',
|
|
'status' => 'active',
|
|
'service_status' => 'exempt',
|
|
'network_status' => 'active',
|
|
'spec_json' => json_encode(
|
|
['managed' => true],
|
|
JSON_THROW_ON_ERROR
|
|
),
|
|
'sealed_credential' => null,
|
|
'created_at' => $now - 3600,
|
|
'updated_at' => $now - 3600,
|
|
]);
|
|
$originalEnvelope = CloudSecrets::encrypt($payload, 'job-payload');
|
|
$originalResult = json_encode(
|
|
['cause_code' => 'local_finalization_failed'],
|
|
JSON_THROW_ON_ERROR
|
|
);
|
|
Capsule::table('cloud_jobs')->insert([
|
|
'id' => 501,
|
|
'operation_id' => $operationId,
|
|
'request_idempotency_key' => $operationId,
|
|
'request_fingerprint' => hash('sha256', $operationId),
|
|
'parent_job_id' => null,
|
|
'order_id' => null,
|
|
'invoice_id' => null,
|
|
'instance_id' => 501,
|
|
'snapshot_id' => null,
|
|
'user_id' => 1,
|
|
'host_id' => 7,
|
|
'kind' => 'instance_settings',
|
|
'execution_mode' => 'agent',
|
|
'phase' => 'terminal',
|
|
'outcome' => 'failed',
|
|
'status' => 'failed',
|
|
'sealed_payload' => $originalEnvelope,
|
|
'attempts' => 0,
|
|
'dispatch_attempts' => 0,
|
|
'dispatch_deadline_at' => $now - 30,
|
|
'terminal_at' => $now - 20,
|
|
'effect_may_have_applied' => 0,
|
|
'safe_to_resubmit' => 0,
|
|
'effect_resolution' => 'not_applied',
|
|
'effect_resolution_reason' => '',
|
|
'lease_until' => 0,
|
|
'available_at' => 0,
|
|
'error_code' => 'local_finalization_failed',
|
|
'finalization_incident' => 1,
|
|
'finalization_error_code' => 'recovery_ledger_conflict',
|
|
'result_json' => $originalResult,
|
|
'created_at' => $now - 30,
|
|
'updated_at' => $now - 20,
|
|
]);
|
|
|
|
$before = CloudJob::findOrFail(501);
|
|
$assert(CloudJobService::blockingQueryForInstance(501)->exists(),
|
|
'An undispatched local finalizer incident did not block mutations.');
|
|
$service = new CloudAutomaticOperationRecoveryService();
|
|
$assert($service->tick(),
|
|
'The undispatched PHP-only finalizer was not claimed.');
|
|
$after = CloudJob::findOrFail(501);
|
|
$scrubbed = CloudSecrets::decrypt(
|
|
(string) $after->sealed_payload,
|
|
'job-payload'
|
|
);
|
|
$assert((string) $after->phase === (string) $before->phase
|
|
&& (string) $after->outcome === (string) $before->outcome
|
|
&& (string) $after->status === (string) $before->status
|
|
&& (int) $after->terminal_at === (int) $before->terminal_at
|
|
&& (string) $after->error_code === (string) $before->error_code
|
|
&& (string) $after->result_json === (string) $before->result_json
|
|
&& $after->completion_event_id === $before->completion_event_id
|
|
&& !(bool) $after->effect_may_have_applied
|
|
&& (bool) $after->safe_to_resubmit
|
|
&& (string) $after->effect_resolution === 'not_applied'
|
|
&& (string) $after->effect_resolution_reason
|
|
=== 'automatic_local_not_dispatched'
|
|
&& (int) $after->effect_resolved_at > 0
|
|
&& !(bool) $after->finalization_incident
|
|
&& $after->finalization_error_code === null
|
|
&& $after->lease_token === null
|
|
&& (int) $after->lease_until === 0
|
|
&& (int) $after->available_at === 0
|
|
&& (string) $after->sealed_payload !== $originalEnvelope
|
|
&& !isset($scrubbed['credential'])
|
|
&& !isset($scrubbed['body']['root_password'])
|
|
&& !CloudJobService::blockingQueryForInstance(501)->exists(),
|
|
'The PHP-only finalizer changed terminal audit data or failed to converge safely.');
|
|
$updatedAt = (int) $after->updated_at;
|
|
$assert(!$service->tick(),
|
|
'A converged PHP-only finalizer was claimed a second time.');
|
|
$afterSecondTick = CloudJob::findOrFail(501);
|
|
$assert((int) $afterSecondTick->updated_at === $updatedAt,
|
|
'A converged PHP-only finalizer repeated its side effects.');
|
|
|
|
// A missing/corrupt encrypted escrow remains a concrete payload incident;
|
|
// it must not degrade into a generic ledger-conflict message.
|
|
Capsule::table('cloud_instances')->insert([
|
|
'id' => 502,
|
|
'uuid' => '00000000-0000-4000-8000-000000000502',
|
|
'host_id' => 7,
|
|
'user_id' => 1,
|
|
'name' => 'local-finalizer-corrupt-payload',
|
|
'status' => 'active',
|
|
'service_status' => 'exempt',
|
|
'network_status' => 'active',
|
|
'spec_json' => json_encode(['managed' => true], JSON_THROW_ON_ERROR),
|
|
'sealed_credential' => null,
|
|
'created_at' => $now - 3600,
|
|
'updated_at' => $now - 3600,
|
|
]);
|
|
$corruptOperationId = str_repeat('ae', 16);
|
|
Capsule::table('cloud_jobs')->insert([
|
|
'id' => 502,
|
|
'operation_id' => $corruptOperationId,
|
|
'request_idempotency_key' => $corruptOperationId,
|
|
'request_fingerprint' => hash('sha256', $corruptOperationId),
|
|
'parent_job_id' => null,
|
|
'order_id' => null,
|
|
'invoice_id' => null,
|
|
'instance_id' => 502,
|
|
'snapshot_id' => null,
|
|
'user_id' => 1,
|
|
'host_id' => 7,
|
|
'kind' => 'instance_settings',
|
|
'execution_mode' => 'agent',
|
|
'phase' => 'terminal',
|
|
'outcome' => 'failed',
|
|
'status' => 'failed',
|
|
'sealed_payload' => 'undispatched-finalizer-test:missing',
|
|
'attempts' => 0,
|
|
'dispatch_attempts' => 0,
|
|
'dispatch_deadline_at' => $now - 30,
|
|
'terminal_at' => $now - 20,
|
|
'effect_may_have_applied' => 0,
|
|
'safe_to_resubmit' => 0,
|
|
'effect_resolution' => 'not_applied',
|
|
'effect_resolution_reason' => '',
|
|
'lease_until' => 0,
|
|
'available_at' => 0,
|
|
'error_code' => 'local_finalization_failed',
|
|
'finalization_incident' => 1,
|
|
'finalization_error_code' => 'recovery_payload_unavailable',
|
|
'result_json' => $originalResult,
|
|
'created_at' => $now - 30,
|
|
'updated_at' => $now - 20,
|
|
]);
|
|
$assert($service->tick(),
|
|
'The corrupt undispatched recovery payload was not checked.');
|
|
$corruptPayloadJob = CloudJob::findOrFail(502);
|
|
$assert((bool) $corruptPayloadJob->finalization_incident
|
|
&& (string) $corruptPayloadJob->finalization_error_code
|
|
=== 'recovery_payload_unavailable'
|
|
&& (string) $corruptPayloadJob->sealed_payload
|
|
=== 'undispatched-finalizer-test:missing'
|
|
&& (int) $corruptPayloadJob->lease_until >= time() + 290
|
|
&& (int) $corruptPayloadJob->available_at >= time() + 290
|
|
&& CloudJobService::blockingQueryForInstance(502)->exists(),
|
|
'A corrupt PHP-only recovery payload lost its concrete incident or resource fence.');
|
|
|
|
echo "Cloud undispatched finalizer recovery tests passed.\n";
|
|
}
|