You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
352 lines
16 KiB
PHP
352 lines
16 KiB
PHP
<?php
|
|
|
|
namespace App\Services {
|
|
final class Config
|
|
{
|
|
public static string $key = '';
|
|
|
|
public static function get($name, $default = null)
|
|
{
|
|
return $name === 'cloud_data_key' ? self::$key : $default;
|
|
}
|
|
}
|
|
}
|
|
|
|
namespace {
|
|
require dirname(__DIR__) . '/vendor/autoload.php';
|
|
|
|
use App\Models\CloudJob;
|
|
use App\Models\CloudTemplate;
|
|
use App\Models\CloudTemplateArtifact;
|
|
use App\Services\Cloud\CloudObjectStorageS3Client;
|
|
use App\Services\Cloud\CloudSecrets;
|
|
use App\Services\Cloud\CloudTemplateDeletionService;
|
|
use App\Services\Config;
|
|
use GuzzleHttp\Client;
|
|
use GuzzleHttp\Exception\ConnectException;
|
|
use GuzzleHttp\Handler\MockHandler;
|
|
use GuzzleHttp\HandlerStack;
|
|
use GuzzleHttp\Middleware;
|
|
use GuzzleHttp\Psr7\Request;
|
|
use GuzzleHttp\Psr7\Response;
|
|
use Illuminate\Database\Capsule\Manager as Capsule;
|
|
use Illuminate\Database\Schema\Blueprint;
|
|
|
|
$assert = static function (bool $condition, string $message): void {
|
|
if (!$condition) {
|
|
throw new RuntimeException($message);
|
|
}
|
|
};
|
|
Config::$key = base64_encode(str_repeat('d', 32));
|
|
$capsule = new Capsule();
|
|
$capsule->addConnection([
|
|
'driver' => 'sqlite', 'database' => ':memory:',
|
|
], 'default');
|
|
$capsule->setAsGlobal();
|
|
$capsule->bootEloquent();
|
|
$schema = $capsule->schema('default');
|
|
$schema->create('cloud_jobs', static function (Blueprint $table): void {
|
|
$table->increments('id');
|
|
$table->string('operation_id');
|
|
$table->string('request_idempotency_key');
|
|
$table->string('request_fingerprint');
|
|
$table->integer('parent_job_id')->nullable();
|
|
$table->integer('user_id');
|
|
$table->integer('host_id')->nullable();
|
|
$table->string('kind');
|
|
$table->string('execution_mode');
|
|
$table->string('phase');
|
|
$table->string('outcome')->nullable();
|
|
$table->string('status');
|
|
$table->text('sealed_payload');
|
|
$table->integer('attempts')->default(0);
|
|
$table->integer('dispatch_attempts')->default(0);
|
|
$table->integer('dispatch_deadline_at')->default(1);
|
|
$table->integer('dispatch_started_at')->nullable();
|
|
$table->integer('agent_accepted_at')->nullable();
|
|
$table->integer('dispatched_at')->nullable();
|
|
$table->integer('receipt_deadline_at')->nullable();
|
|
$table->integer('terminal_at')->nullable();
|
|
$table->boolean('effect_may_have_applied')->nullable();
|
|
$table->boolean('safe_to_resubmit')->nullable();
|
|
$table->string('effect_resolution')->default('not_applicable');
|
|
$table->integer('effect_resolved_at')->nullable();
|
|
$table->integer('effect_resolved_by')->nullable();
|
|
$table->string('effect_resolution_reason')->default('');
|
|
$table->string('completion_event_id')->nullable();
|
|
$table->integer('read_at')->nullable();
|
|
$table->string('lease_token')->nullable();
|
|
$table->integer('lease_until')->default(0);
|
|
$table->integer('available_at')->default(0);
|
|
$table->string('error_code')->nullable();
|
|
$table->boolean('finalization_incident')->default(0);
|
|
$table->string('finalization_error_code')->nullable();
|
|
$table->text('result_json')->nullable();
|
|
$table->integer('created_at');
|
|
$table->integer('updated_at');
|
|
});
|
|
$schema->create('cloud_templates', static function (Blueprint $table): void {
|
|
$table->string('id')->primary();
|
|
$table->string('catalog_key');
|
|
$table->string('version_label');
|
|
$table->boolean('is_default')->default(false);
|
|
$table->string('name');
|
|
$table->string('os_family');
|
|
$table->string('os_version');
|
|
$table->integer('image_bytes');
|
|
$table->string('sha256');
|
|
$table->text('startup_script');
|
|
$table->integer('revision');
|
|
$table->string('generation_token');
|
|
$table->string('state');
|
|
$table->integer('active_job_id')->nullable();
|
|
$table->integer('created_at');
|
|
$table->integer('updated_at');
|
|
});
|
|
$schema->create('cloud_template_replicas', static function (Blueprint $table): void {
|
|
$table->integer('host_id');
|
|
$table->string('template_id');
|
|
$table->integer('revision');
|
|
$table->string('generation_token');
|
|
$table->string('state');
|
|
$table->integer('image_bytes');
|
|
$table->string('sha256');
|
|
$table->string('operation_id');
|
|
$table->integer('job_id')->nullable();
|
|
$table->string('error_code')->default('');
|
|
$table->integer('created_at');
|
|
$table->integer('updated_at');
|
|
$table->primary(['host_id', 'template_id']);
|
|
});
|
|
$schema->create('cloud_user_iso_replicas', static function (Blueprint $table): void {
|
|
$table->integer('job_id')->nullable();
|
|
});
|
|
$schema->create('cloud_template_artifacts', static function (Blueprint $table): void {
|
|
$table->string('template_id')->primary();
|
|
$table->integer('storage_revision');
|
|
$table->text('sealed_storage_runtime');
|
|
$table->string('object_key');
|
|
$table->string('object_version_id')->default('');
|
|
$table->string('object_etag');
|
|
$table->integer('size_bytes');
|
|
$table->string('sha256');
|
|
$table->string('state');
|
|
$table->string('error_code')->default('');
|
|
$table->string('delete_operation_id')->default('');
|
|
$table->integer('delete_generation')->default(0);
|
|
$table->integer('delete_cycle_attempts')->default(0);
|
|
$table->integer('delete_presence_confirmations')->default(0);
|
|
$table->integer('delete_last_checked_at')->default(0);
|
|
$table->integer('delete_next_check_at')->default(0);
|
|
$table->integer('delete_attention_at')->nullable();
|
|
$table->integer('verified_at');
|
|
$table->integer('created_at');
|
|
$table->integer('updated_at');
|
|
});
|
|
|
|
$runtime = [
|
|
'config' => [
|
|
'endpoint' => 'https://objects.example.com', 'region' => 'ewr',
|
|
'bucket' => 'templates', 'prefix' => 'virt', 'path_style' => true,
|
|
],
|
|
'credentials' => [
|
|
'access_key_id' => 'DELETEACCESSKEY',
|
|
'secret_access_key' => 'delete-secret',
|
|
],
|
|
'revision' => 1,
|
|
];
|
|
$now = time();
|
|
$fixture = static function (string $suffix) use ($runtime, $now): array {
|
|
$parentId = (int) Capsule::table('cloud_jobs')->insertGetId([
|
|
'operation_id' => md5('parent-' . $suffix),
|
|
'request_idempotency_key' => md5('parent-' . $suffix),
|
|
'request_fingerprint' => hash('sha256', 'parent-' . $suffix),
|
|
'parent_job_id' => null, 'user_id' => 1, 'host_id' => null,
|
|
'kind' => 'template_delete', 'execution_mode' => 'aggregate',
|
|
'phase' => 'awaiting_receipt', 'outcome' => null,
|
|
'status' => 'running',
|
|
'sealed_payload' => CloudSecrets::encrypt([], 'job-payload'),
|
|
'attempts' => 0, 'dispatch_attempts' => 0,
|
|
'dispatch_deadline_at' => $now + 300,
|
|
'receipt_deadline_at' => $now + 3600,
|
|
'effect_resolution' => 'not_applicable',
|
|
'effect_resolution_reason' => '', 'lease_until' => 0,
|
|
'available_at' => $now, 'finalization_incident' => 0,
|
|
'created_at' => $now, 'updated_at' => $now,
|
|
]);
|
|
$childOperation = md5('child-' . $suffix);
|
|
$childId = (int) Capsule::table('cloud_jobs')->insertGetId([
|
|
'operation_id' => $childOperation,
|
|
'request_idempotency_key' => $childOperation,
|
|
'request_fingerprint' => hash('sha256', 'child-' . $suffix),
|
|
'parent_job_id' => $parentId, 'user_id' => 1, 'host_id' => 7,
|
|
'kind' => 'template_delete_replica', 'execution_mode' => 'agent',
|
|
'phase' => 'terminal', 'outcome' => 'succeeded',
|
|
'status' => 'succeeded',
|
|
'sealed_payload' => CloudSecrets::encrypt([], 'job-payload'),
|
|
'attempts' => 1, 'dispatch_attempts' => 1,
|
|
'dispatch_deadline_at' => $now - 20,
|
|
'dispatch_started_at' => $now - 19,
|
|
'agent_accepted_at' => $now - 18,
|
|
'dispatched_at' => $now - 17,
|
|
'receipt_deadline_at' => $now + 900,
|
|
'terminal_at' => $now - 1,
|
|
'effect_may_have_applied' => 0, 'safe_to_resubmit' => 0,
|
|
'effect_resolution' => 'applied',
|
|
'effect_resolution_reason' => '', 'lease_until' => 0,
|
|
'available_at' => 0, 'finalization_incident' => 0,
|
|
'created_at' => $now - 20, 'updated_at' => $now - 1,
|
|
]);
|
|
$templateId = 'delete-template-' . $suffix;
|
|
$generation = md5('generation-' . $suffix);
|
|
$sha = hash('sha256', 'image-' . $suffix);
|
|
Capsule::table('cloud_templates')->insert([
|
|
'id' => $templateId, 'catalog_key' => 'debian-13',
|
|
'version_label' => $suffix, 'is_default' => 0,
|
|
'name' => 'Delete ' . $suffix, 'os_family' => 'Debian',
|
|
'os_version' => '13', 'image_bytes' => 4096,
|
|
'sha256' => $sha, 'startup_script' => '', 'revision' => 1,
|
|
'generation_token' => $generation, 'state' => 'deleting',
|
|
'active_job_id' => $parentId, 'created_at' => $now,
|
|
'updated_at' => $now,
|
|
]);
|
|
Capsule::table('cloud_template_replicas')->insert([
|
|
'host_id' => 7, 'template_id' => $templateId, 'revision' => 1,
|
|
'generation_token' => $generation, 'state' => 'delete_succeeded',
|
|
'image_bytes' => 4096, 'sha256' => $sha,
|
|
'operation_id' => $childOperation, 'job_id' => $childId,
|
|
'error_code' => '', 'created_at' => $now, 'updated_at' => $now,
|
|
]);
|
|
Capsule::table('cloud_template_artifacts')->insert([
|
|
'template_id' => $templateId, 'storage_revision' => 1,
|
|
'sealed_storage_runtime' => CloudSecrets::encrypt(
|
|
$runtime, 'template-artifact-runtime'
|
|
),
|
|
'object_key' => 'virt/cloud-templates/' . $templateId
|
|
. '/' . $generation . '/system.qcow2',
|
|
'object_version_id' => 'version-' . $suffix,
|
|
'object_etag' => '"etag-' . $suffix . '"',
|
|
'size_bytes' => 4096, 'sha256' => $sha, 'state' => 'ready',
|
|
'error_code' => '', 'delete_operation_id' => '',
|
|
'delete_generation' => 0, 'delete_cycle_attempts' => 0,
|
|
'delete_presence_confirmations' => 0,
|
|
'delete_last_checked_at' => 0, 'delete_next_check_at' => 0,
|
|
'delete_attention_at' => null, 'verified_at' => $now,
|
|
'created_at' => $now, 'updated_at' => $now,
|
|
]);
|
|
$template = CloudTemplate::findOrFail($templateId);
|
|
$parent = CloudJob::findOrFail($parentId);
|
|
$template->getConnection()->transaction(
|
|
static function () use ($templateId, $parentId): void {
|
|
$parent = CloudJob::where('id', $parentId)
|
|
->lockForUpdate()->firstOrFail();
|
|
$template = CloudTemplate::where('id', $templateId)
|
|
->lockForUpdate()->firstOrFail();
|
|
CloudTemplateDeletionService::armLocked($template, $parent);
|
|
},
|
|
3
|
|
);
|
|
return [$templateId, $parentId];
|
|
};
|
|
|
|
[$normalId, $normalParentId] = $fixture('normal');
|
|
$normalHistory = [];
|
|
$normalStack = HandlerStack::create(new MockHandler([
|
|
new Response(200, [
|
|
'Content-Length' => '4096', 'ETag' => '"etag-normal"',
|
|
'x-amz-version-id' => 'version-normal',
|
|
]),
|
|
new Response(204),
|
|
]));
|
|
$normalStack->push(Middleware::history($normalHistory));
|
|
$normalClient = new CloudObjectStorageS3Client(new Client([
|
|
'handler' => $normalStack,
|
|
]));
|
|
$normal = CloudTemplateDeletionService::processDue(1, $normalClient);
|
|
$assert($normal['completed'] === 1
|
|
&& CloudTemplateArtifact::find($normalId) === null
|
|
&& (string) CloudTemplate::findOrFail($normalId)->state === 'retired'
|
|
&& (int) CloudTemplate::findOrFail($normalId)->revision === 2
|
|
&& (string) CloudJob::findOrFail($normalParentId)->outcome
|
|
=== 'succeeded'
|
|
&& array_map(
|
|
static fn (array $item): string => $item['request']->getMethod(),
|
|
$normalHistory
|
|
) === ['HEAD', 'DELETE'],
|
|
'Confirmed master deletion did not retire the exact catalog version.');
|
|
|
|
[$unknownId, $unknownParentId] = $fixture('unknown');
|
|
$unknownHistory = [];
|
|
$unknownStack = HandlerStack::create(new MockHandler([
|
|
new Response(200, [
|
|
'Content-Length' => '4096', 'ETag' => '"etag-unknown"',
|
|
'x-amz-version-id' => 'version-unknown',
|
|
]),
|
|
new ConnectException('lost after delete', new Request(
|
|
'DELETE', 'https://objects.example.com'
|
|
)),
|
|
new Response(200, [
|
|
'Content-Length' => '4096', 'ETag' => '"etag-unknown"',
|
|
'x-amz-version-id' => 'version-unknown',
|
|
]),
|
|
new Response(200, [
|
|
'Content-Length' => '4096', 'ETag' => '"etag-unknown"',
|
|
'x-amz-version-id' => 'version-unknown',
|
|
]),
|
|
new Response(204),
|
|
]));
|
|
$unknownStack->push(Middleware::history($unknownHistory));
|
|
$unknownClient = new CloudObjectStorageS3Client(new Client([
|
|
'handler' => $unknownStack,
|
|
]));
|
|
CloudTemplateDeletionService::processDue(1, $unknownClient);
|
|
$unknownArtifact = CloudTemplateArtifact::findOrFail($unknownId);
|
|
$assert((string) $unknownArtifact->state === 'delete_uncertain'
|
|
&& (int) $unknownArtifact->delete_generation === 1,
|
|
'Ambiguous DELETE was not fenced on its original generation.');
|
|
Capsule::table('cloud_template_artifacts')->where('template_id', $unknownId)
|
|
->update(['delete_next_check_at' => 0]);
|
|
CloudTemplateDeletionService::processDue(1, $unknownClient);
|
|
$unknownArtifact = CloudTemplateArtifact::findOrFail($unknownId);
|
|
$assert((int) $unknownArtifact->delete_presence_confirmations === 1
|
|
&& (int) $unknownArtifact->delete_generation === 1,
|
|
'One presence probe incorrectly replayed an ambiguous DELETE.');
|
|
Capsule::table('cloud_template_artifacts')->where('template_id', $unknownId)
|
|
->update([
|
|
'delete_last_checked_at' => time() - 301,
|
|
'delete_next_check_at' => 0,
|
|
]);
|
|
$converged = CloudTemplateDeletionService::processDue(1, $unknownClient);
|
|
$assert($converged['completed'] === 1
|
|
&& CloudTemplateArtifact::find($unknownId) === null
|
|
&& (string) CloudTemplate::findOrFail($unknownId)->state === 'retired'
|
|
&& (string) CloudJob::findOrFail($unknownParentId)->outcome
|
|
=== 'succeeded'
|
|
&& array_map(
|
|
static fn (array $item): string => $item['request']->getMethod(),
|
|
$unknownHistory
|
|
) === ['HEAD', 'DELETE', 'HEAD', 'HEAD', 'DELETE'],
|
|
'Stable two-probe evidence did not create exactly one fresh delete generation.');
|
|
|
|
[$attentionId, $attentionParentId] = $fixture('attention');
|
|
$attentionClient = new CloudObjectStorageS3Client(new Client([
|
|
'handler' => HandlerStack::create(new MockHandler([
|
|
new Response(200, [
|
|
'Content-Length' => '8192', 'ETag' => '"different"',
|
|
'x-amz-version-id' => 'version-attention',
|
|
]),
|
|
])),
|
|
]));
|
|
$attention = CloudTemplateDeletionService::processDue(1, $attentionClient);
|
|
$assert($attention['attention'] === 1
|
|
&& (string) CloudTemplateArtifact::findOrFail($attentionId)->state
|
|
=== 'delete_attention'
|
|
&& (string) CloudTemplate::findOrFail($attentionId)->state
|
|
=== 'delete_failed'
|
|
&& (string) CloudJob::findOrFail($attentionParentId)->outcome
|
|
=== 'failed',
|
|
'Changed object identity was not fenced for explicit administrator review.');
|
|
|
|
echo "Cloud-template two-phase deletion and recovery behavior passed.\n";
|
|
}
|