|
|
|
|
<?php
|
|
|
|
|
|
|
|
|
|
namespace App\Services\Cloud {
|
|
|
|
|
/** In-memory payload escrow for execution-mode boundary tests. */
|
|
|
|
|
final class CloudSecrets
|
|
|
|
|
{
|
|
|
|
|
private static array $values = [];
|
|
|
|
|
|
|
|
|
|
public static function encrypt(array $value, string $purpose): string
|
|
|
|
|
{
|
|
|
|
|
$key = bin2hex(random_bytes(16));
|
|
|
|
|
self::$values[$key] = $value;
|
|
|
|
|
return 'execution-mode-test:' . $key;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
public static function decrypt(string $value, string $purpose): array
|
|
|
|
|
{
|
|
|
|
|
$key = substr($value, strlen('execution-mode-test:'));
|
|
|
|
|
if (!array_key_exists($key, self::$values)) {
|
|
|
|
|
throw new \RuntimeException('Missing test payload');
|
|
|
|
|
}
|
|
|
|
|
return self::$values[$key];
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
namespace {
|
|
|
|
|
require dirname(__DIR__) . '/vendor/autoload.php';
|
|
|
|
|
|
|
|
|
|
use App\Services\Cloud\CloudAgentClient;
|
|
|
|
|
use App\Services\Cloud\CloudSecrets;
|
|
|
|
|
use App\Services\Cloud\CloudWorkerService;
|
|
|
|
|
use GuzzleHttp\Client;
|
|
|
|
|
use Illuminate\Database\Capsule\Manager as Capsule;
|
|
|
|
|
use Illuminate\Database\Schema\Blueprint;
|
|
|
|
|
|
|
|
|
|
$assert = static function (bool $condition, string $message): void {
|
|
|
|
|
if (!$condition) {
|
|
|
|
|
throw new RuntimeException($message);
|
|
|
|
|
}
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
$capsule = new Capsule();
|
|
|
|
|
$capsule->addConnection(['driver' => 'sqlite', 'database' => ':memory:'],
|
|
|
|
|
'default');
|
|
|
|
|
$capsule->setAsGlobal();
|
|
|
|
|
$capsule->bootEloquent();
|
|
|
|
|
$capsule->schema('default')->create('user',
|
|
|
|
|
static function (Blueprint $table): void {
|
|
|
|
|
$table->increments('id');
|
|
|
|
|
$table->integer('enable')->default(1);
|
|
|
|
|
$table->integer('is_admin')->default(0);
|
|
|
|
|
});
|
|
|
|
|
$capsule->schema('default')->create('cloud_jobs',
|
|
|
|
|
static function (Blueprint $table): void {
|
|
|
|
|
$table->increments('id');
|
|
|
|
|
$table->string('operation_id')->unique();
|
|
|
|
|
$table->string('request_idempotency_key');
|
|
|
|
|
$table->string('request_fingerprint');
|
|
|
|
|
$table->integer('parent_job_id')->nullable();
|
|
|
|
|
$table->integer('order_id')->nullable();
|
|
|
|
|
$table->integer('invoice_id')->nullable();
|
|
|
|
|
$table->integer('instance_id')->nullable();
|
|
|
|
|
$table->integer('snapshot_id')->nullable();
|
|
|
|
|
$table->integer('vpc_repair_root_job_id')->nullable();
|
|
|
|
|
$table->integer('user_id');
|
|
|
|
|
$table->integer('host_id')->nullable();
|
|
|
|
|
$table->string('kind');
|
|
|
|
|
$table->string('execution_mode');
|
|
|
|
|
$table->string('phase');
|
|
|
|
|
$table->string('outcome')->nullable();
|
|
|
|
|
$table->string('status');
|
|
|
|
|
$table->text('sealed_payload');
|
|
|
|
|
$table->integer('attempts')->default(0);
|
|
|
|
|
$table->integer('dispatch_attempts')->default(0);
|
|
|
|
|
$table->integer('dispatch_deadline_at');
|
|
|
|
|
$table->integer('dispatch_started_at')->nullable();
|
|
|
|
|
$table->integer('agent_accepted_at')->nullable();
|
|
|
|
|
$table->integer('receipt_deadline_at')->nullable();
|
|
|
|
|
$table->integer('terminal_at')->nullable();
|
|
|
|
|
$table->boolean('effect_may_have_applied')->nullable();
|
|
|
|
|
$table->boolean('safe_to_resubmit')->nullable();
|
|
|
|
|
$table->string('effect_resolution')->default('not_applicable');
|
|
|
|
|
$table->integer('effect_resolved_at')->nullable();
|
|
|
|
|
$table->integer('effect_resolved_by')->nullable();
|
|
|
|
|
$table->string('effect_resolution_reason')->default('');
|
|
|
|
|
$table->string('completion_event_id')->nullable();
|
|
|
|
|
$table->integer('read_at')->nullable();
|
|
|
|
|
$table->integer('dispatched_at')->nullable();
|
|
|
|
|
$table->string('lease_token')->nullable();
|
|
|
|
|
$table->integer('lease_until')->default(0);
|
|
|
|
|
$table->integer('available_at')->default(0);
|
|
|
|
|
$table->string('error_code')->nullable();
|
|
|
|
|
$table->boolean('finalization_incident')->default(0);
|
|
|
|
|
$table->string('finalization_error_code')->nullable();
|
|
|
|
|
$table->text('result_json')->nullable();
|
|
|
|
|
$table->integer('created_at');
|
|
|
|
|
$table->integer('updated_at');
|
|
|
|
|
});
|
|
|
|
|
$capsule->schema('default')->create('cloud_agent_completion_events',
|
|
|
|
|
static function (Blueprint $table): void {
|
|
|
|
|
$table->string('event_id')->primary();
|
|
|
|
|
$table->integer('host_id');
|
|
|
|
|
$table->string('operation_id');
|
|
|
|
|
$table->string('payload_sha256');
|
|
|
|
|
$table->string('outcome');
|
|
|
|
|
$table->boolean('effect_may_have_applied');
|
|
|
|
|
$table->boolean('safe_to_resubmit');
|
|
|
|
|
$table->string('error_code')->default('');
|
|
|
|
|
$table->text('result_json')->nullable();
|
|
|
|
|
$table->string('disposition')->default('received');
|
|
|
|
|
$table->integer('received_at');
|
|
|
|
|
$table->integer('applied_at')->nullable();
|
|
|
|
|
$table->unique(['host_id', 'operation_id']);
|
|
|
|
|
});
|
|
|
|
|
|
|
|
|
|
$now = time();
|
|
|
|
|
$base = [
|
|
|
|
|
'request_fingerprint' => str_repeat('f', 64),
|
|
|
|
|
'parent_job_id' => null,
|
|
|
|
|
'order_id' => null,
|
|
|
|
|
'invoice_id' => null,
|
|
|
|
|
'instance_id' => null,
|
|
|
|
|
'snapshot_id' => null,
|
|
|
|
|
'user_id' => 1,
|
|
|
|
|
'host_id' => null,
|
|
|
|
|
'outcome' => null,
|
|
|
|
|
'sealed_payload' => 'intentionally-invalid-test-envelope',
|
|
|
|
|
'attempts' => 0,
|
|
|
|
|
'dispatch_attempts' => 0,
|
|
|
|
|
'dispatch_deadline_at' => $now + 300,
|
|
|
|
|
'receipt_deadline_at' => null,
|
|
|
|
|
'lease_until' => 0,
|
|
|
|
|
'available_at' => 0,
|
|
|
|
|
'created_at' => $now,
|
|
|
|
|
'updated_at' => $now,
|
|
|
|
|
];
|
|
|
|
|
Capsule::table('cloud_jobs')->insert(array_merge($base, [
|
|
|
|
|
'id' => 1,
|
|
|
|
|
'operation_id' => str_repeat('a', 32),
|
|
|
|
|
'request_idempotency_key' => str_repeat('a', 32),
|
|
|
|
|
'kind' => 'template_import',
|
|
|
|
|
'execution_mode' => 'aggregate',
|
|
|
|
|
'phase' => 'awaiting_receipt',
|
|
|
|
|
'status' => 'running',
|
|
|
|
|
'receipt_deadline_at' => $now + 8400,
|
|
|
|
|
]));
|
|
|
|
|
Capsule::table('cloud_jobs')->insert(array_merge($base, [
|
|
|
|
|
'id' => 2,
|
|
|
|
|
'operation_id' => str_repeat('b', 32),
|
|
|
|
|
'request_idempotency_key' => str_repeat('b', 32),
|
|
|
|
|
'kind' => 'payment_create',
|
|
|
|
|
'execution_mode' => 'local',
|
|
|
|
|
'phase' => 'accepted',
|
|
|
|
|
'status' => 'queued',
|
|
|
|
|
]));
|
|
|
|
|
Capsule::table('user')->insert(['id' => 1, 'enable' => 1, 'is_admin' => 0]);
|
|
|
|
|
Capsule::table('cloud_jobs')->insert(array_merge($base, [
|
|
|
|
|
'id' => 3,
|
|
|
|
|
'operation_id' => str_repeat('c', 32),
|
|
|
|
|
'request_idempotency_key' => str_repeat('c', 32),
|
|
|
|
|
'host_id' => 7,
|
|
|
|
|
'kind' => 'instance_settings',
|
|
|
|
|
'execution_mode' => 'local',
|
|
|
|
|
'phase' => 'accepted',
|
|
|
|
|
'status' => 'queued',
|
|
|
|
|
'sealed_payload' => CloudSecrets::encrypt([
|
|
|
|
|
'method' => 'POST',
|
|
|
|
|
'request_target' => '/v1/vms/11111111-2222-4333-8444-555555555555/settings',
|
|
|
|
|
'body' => ['hostname' => 'mode-conflict.example'],
|
|
|
|
|
'owner_id' => 1,
|
|
|
|
|
], 'job-payload'),
|
|
|
|
|
]));
|
|
|
|
|
|
|
|
|
|
$agentCalls = 0;
|
|
|
|
|
$transport = new Client(['handler' => static function () use (
|
|
|
|
|
&$agentCalls
|
|
|
|
|
) {
|
|
|
|
|
$agentCalls++;
|
|
|
|
|
throw new RuntimeException('Mode-conflict task reached Agent');
|
|
|
|
|
}]);
|
|
|
|
|
$tripwireAgent = new CloudAgentClient(
|
|
|
|
|
'7', str_repeat('a', 32), $transport, 120.0,
|
|
|
|
|
'http://127.0.0.1:8001/internal/agent/request',
|
|
|
|
|
str_repeat('b', 32)
|
|
|
|
|
);
|
|
|
|
|
$worker = new CloudWorkerService($tripwireAgent);
|
|
|
|
|
$assert($worker->tick() === true,
|
|
|
|
|
'The mode-conflict task was not consumed by the local quarantine path.');
|
|
|
|
|
$corruptLeaf = Capsule::table('cloud_jobs')->where('id', 3)->first();
|
|
|
|
|
$assert($agentCalls === 0
|
|
|
|
|
&& $corruptLeaf !== null
|
|
|
|
|
&& $corruptLeaf->phase === 'terminal'
|
|
|
|
|
&& $corruptLeaf->outcome === 'failed'
|
|
|
|
|
&& $corruptLeaf->effect_resolution === 'unknown'
|
|
|
|
|
&& (int) $corruptLeaf->finalization_incident === 1
|
|
|
|
|
&& $corruptLeaf->finalization_error_code
|
|
|
|
|
=== 'recovery_instance_state_conflict',
|
|
|
|
|
'A supported Agent task with a corrupt local mode was dispatched or not quarantined.');
|
|
|
|
|
$assert($worker->tick() === true,
|
|
|
|
|
'The host-null local payment task was not claimed.');
|
|
|
|
|
$local = Capsule::table('cloud_jobs')->where('id', 2)->first();
|
|
|
|
|
$aggregate = Capsule::table('cloud_jobs')->where('id', 1)->first();
|
|
|
|
|
$assert($local !== null && $local->phase === 'terminal'
|
|
|
|
|
&& $local->outcome === 'failed' && $local->status === 'failed'
|
|
|
|
|
&& (int) $local->dispatch_attempts === 0,
|
|
|
|
|
'A local payment failure did not close once without an Agent dispatch.');
|
|
|
|
|
$assert($aggregate !== null && $aggregate->phase === 'awaiting_receipt'
|
|
|
|
|
&& $aggregate->outcome === null && $aggregate->status === 'running'
|
|
|
|
|
&& (int) $aggregate->dispatch_attempts === 0,
|
|
|
|
|
'The worker incorrectly claimed an aggregate parent task.');
|
|
|
|
|
$assert($worker->tick() === false,
|
|
|
|
|
'The terminal local failure or aggregate parent remained worker-claimable.');
|
|
|
|
|
|
|
|
|
|
echo "Explicit execution-mode worker claim tests passed.\n";
|
|
|
|
|
}
|