You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
virt/web/tests/CloudTemplateDeletionServic...

352 lines
16 KiB
PHP

<?php
namespace App\Services {
final class Config
{
public static string $key = '';
public static function get($name, $default = null)
{
return $name === 'cloud_data_key' ? self::$key : $default;
}
}
}
namespace {
require dirname(__DIR__) . '/vendor/autoload.php';
use App\Models\CloudJob;
use App\Models\CloudTemplate;
use App\Models\CloudTemplateArtifact;
use App\Services\Cloud\CloudObjectStorageS3Client;
use App\Services\Cloud\CloudSecrets;
use App\Services\Cloud\CloudTemplateDeletionService;
use App\Services\Config;
use GuzzleHttp\Client;
use GuzzleHttp\Exception\ConnectException;
use GuzzleHttp\Handler\MockHandler;
use GuzzleHttp\HandlerStack;
use GuzzleHttp\Middleware;
use GuzzleHttp\Psr7\Request;
use GuzzleHttp\Psr7\Response;
use Illuminate\Database\Capsule\Manager as Capsule;
use Illuminate\Database\Schema\Blueprint;
$assert = static function (bool $condition, string $message): void {
if (!$condition) {
throw new RuntimeException($message);
}
};
Config::$key = base64_encode(str_repeat('d', 32));
$capsule = new Capsule();
$capsule->addConnection([
'driver' => 'sqlite', 'database' => ':memory:',
], 'default');
$capsule->setAsGlobal();
$capsule->bootEloquent();
$schema = $capsule->schema('default');
$schema->create('cloud_jobs', static function (Blueprint $table): void {
$table->increments('id');
$table->string('operation_id');
$table->string('request_idempotency_key');
$table->string('request_fingerprint');
$table->integer('parent_job_id')->nullable();
$table->integer('user_id');
$table->integer('host_id')->nullable();
$table->string('kind');
$table->string('execution_mode');
$table->string('phase');
$table->string('outcome')->nullable();
$table->string('status');
$table->text('sealed_payload');
$table->integer('attempts')->default(0);
$table->integer('dispatch_attempts')->default(0);
$table->integer('dispatch_deadline_at')->default(1);
$table->integer('dispatch_started_at')->nullable();
$table->integer('agent_accepted_at')->nullable();
$table->integer('dispatched_at')->nullable();
$table->integer('receipt_deadline_at')->nullable();
$table->integer('terminal_at')->nullable();
$table->boolean('effect_may_have_applied')->nullable();
$table->boolean('safe_to_resubmit')->nullable();
$table->string('effect_resolution')->default('not_applicable');
$table->integer('effect_resolved_at')->nullable();
$table->integer('effect_resolved_by')->nullable();
$table->string('effect_resolution_reason')->default('');
$table->string('completion_event_id')->nullable();
$table->integer('read_at')->nullable();
$table->string('lease_token')->nullable();
$table->integer('lease_until')->default(0);
$table->integer('available_at')->default(0);
$table->string('error_code')->nullable();
$table->boolean('finalization_incident')->default(0);
$table->string('finalization_error_code')->nullable();
$table->text('result_json')->nullable();
$table->integer('created_at');
$table->integer('updated_at');
});
$schema->create('cloud_templates', static function (Blueprint $table): void {
$table->string('id')->primary();
$table->string('catalog_key');
$table->string('version_label');
$table->boolean('is_default')->default(false);
$table->string('name');
$table->string('os_family');
$table->string('os_version');
$table->integer('image_bytes');
$table->string('sha256');
$table->text('startup_script');
$table->integer('revision');
$table->string('generation_token');
$table->string('state');
$table->integer('active_job_id')->nullable();
$table->integer('created_at');
$table->integer('updated_at');
});
$schema->create('cloud_template_replicas', static function (Blueprint $table): void {
$table->integer('host_id');
$table->string('template_id');
$table->integer('revision');
$table->string('generation_token');
$table->string('state');
$table->integer('image_bytes');
$table->string('sha256');
$table->string('operation_id');
$table->integer('job_id')->nullable();
$table->string('error_code')->default('');
$table->integer('created_at');
$table->integer('updated_at');
$table->primary(['host_id', 'template_id']);
});
$schema->create('cloud_user_iso_replicas', static function (Blueprint $table): void {
$table->integer('job_id')->nullable();
});
$schema->create('cloud_template_artifacts', static function (Blueprint $table): void {
$table->string('template_id')->primary();
$table->integer('storage_revision');
$table->text('sealed_storage_runtime');
$table->string('object_key');
$table->string('object_version_id')->default('');
$table->string('object_etag');
$table->integer('size_bytes');
$table->string('sha256');
$table->string('state');
$table->string('error_code')->default('');
$table->string('delete_operation_id')->default('');
$table->integer('delete_generation')->default(0);
$table->integer('delete_cycle_attempts')->default(0);
$table->integer('delete_presence_confirmations')->default(0);
$table->integer('delete_last_checked_at')->default(0);
$table->integer('delete_next_check_at')->default(0);
$table->integer('delete_attention_at')->nullable();
$table->integer('verified_at');
$table->integer('created_at');
$table->integer('updated_at');
});
$runtime = [
'config' => [
'endpoint' => 'https://objects.example.com', 'region' => 'ewr',
'bucket' => 'templates', 'prefix' => 'virt', 'path_style' => true,
],
'credentials' => [
'access_key_id' => 'DELETEACCESSKEY',
'secret_access_key' => 'delete-secret',
],
'revision' => 1,
];
$now = time();
$fixture = static function (string $suffix) use ($runtime, $now): array {
$parentId = (int) Capsule::table('cloud_jobs')->insertGetId([
'operation_id' => md5('parent-' . $suffix),
'request_idempotency_key' => md5('parent-' . $suffix),
'request_fingerprint' => hash('sha256', 'parent-' . $suffix),
'parent_job_id' => null, 'user_id' => 1, 'host_id' => null,
'kind' => 'template_delete', 'execution_mode' => 'aggregate',
'phase' => 'awaiting_receipt', 'outcome' => null,
'status' => 'running',
'sealed_payload' => CloudSecrets::encrypt([], 'job-payload'),
'attempts' => 0, 'dispatch_attempts' => 0,
'dispatch_deadline_at' => $now + 300,
'receipt_deadline_at' => $now + 3600,
'effect_resolution' => 'not_applicable',
'effect_resolution_reason' => '', 'lease_until' => 0,
'available_at' => $now, 'finalization_incident' => 0,
'created_at' => $now, 'updated_at' => $now,
]);
$childOperation = md5('child-' . $suffix);
$childId = (int) Capsule::table('cloud_jobs')->insertGetId([
'operation_id' => $childOperation,
'request_idempotency_key' => $childOperation,
'request_fingerprint' => hash('sha256', 'child-' . $suffix),
'parent_job_id' => $parentId, 'user_id' => 1, 'host_id' => 7,
'kind' => 'template_delete_replica', 'execution_mode' => 'agent',
'phase' => 'terminal', 'outcome' => 'succeeded',
'status' => 'succeeded',
'sealed_payload' => CloudSecrets::encrypt([], 'job-payload'),
'attempts' => 1, 'dispatch_attempts' => 1,
'dispatch_deadline_at' => $now - 20,
'dispatch_started_at' => $now - 19,
'agent_accepted_at' => $now - 18,
'dispatched_at' => $now - 17,
'receipt_deadline_at' => $now + 900,
'terminal_at' => $now - 1,
'effect_may_have_applied' => 0, 'safe_to_resubmit' => 0,
'effect_resolution' => 'applied',
'effect_resolution_reason' => '', 'lease_until' => 0,
'available_at' => 0, 'finalization_incident' => 0,
'created_at' => $now - 20, 'updated_at' => $now - 1,
]);
$templateId = 'delete-template-' . $suffix;
$generation = md5('generation-' . $suffix);
$sha = hash('sha256', 'image-' . $suffix);
Capsule::table('cloud_templates')->insert([
'id' => $templateId, 'catalog_key' => 'debian-13',
'version_label' => $suffix, 'is_default' => 0,
'name' => 'Delete ' . $suffix, 'os_family' => 'Debian',
'os_version' => '13', 'image_bytes' => 4096,
'sha256' => $sha, 'startup_script' => '', 'revision' => 1,
'generation_token' => $generation, 'state' => 'deleting',
'active_job_id' => $parentId, 'created_at' => $now,
'updated_at' => $now,
]);
Capsule::table('cloud_template_replicas')->insert([
'host_id' => 7, 'template_id' => $templateId, 'revision' => 1,
'generation_token' => $generation, 'state' => 'delete_succeeded',
'image_bytes' => 4096, 'sha256' => $sha,
'operation_id' => $childOperation, 'job_id' => $childId,
'error_code' => '', 'created_at' => $now, 'updated_at' => $now,
]);
Capsule::table('cloud_template_artifacts')->insert([
'template_id' => $templateId, 'storage_revision' => 1,
'sealed_storage_runtime' => CloudSecrets::encrypt(
$runtime, 'template-artifact-runtime'
),
'object_key' => 'virt/cloud-templates/' . $templateId
. '/' . $generation . '/system.qcow2',
'object_version_id' => 'version-' . $suffix,
'object_etag' => '"etag-' . $suffix . '"',
'size_bytes' => 4096, 'sha256' => $sha, 'state' => 'ready',
'error_code' => '', 'delete_operation_id' => '',
'delete_generation' => 0, 'delete_cycle_attempts' => 0,
'delete_presence_confirmations' => 0,
'delete_last_checked_at' => 0, 'delete_next_check_at' => 0,
'delete_attention_at' => null, 'verified_at' => $now,
'created_at' => $now, 'updated_at' => $now,
]);
$template = CloudTemplate::findOrFail($templateId);
$parent = CloudJob::findOrFail($parentId);
$template->getConnection()->transaction(
static function () use ($templateId, $parentId): void {
$parent = CloudJob::where('id', $parentId)
->lockForUpdate()->firstOrFail();
$template = CloudTemplate::where('id', $templateId)
->lockForUpdate()->firstOrFail();
CloudTemplateDeletionService::armLocked($template, $parent);
},
3
);
return [$templateId, $parentId];
};
[$normalId, $normalParentId] = $fixture('normal');
$normalHistory = [];
$normalStack = HandlerStack::create(new MockHandler([
new Response(200, [
'Content-Length' => '4096', 'ETag' => '"etag-normal"',
'x-amz-version-id' => 'version-normal',
]),
new Response(204),
]));
$normalStack->push(Middleware::history($normalHistory));
$normalClient = new CloudObjectStorageS3Client(new Client([
'handler' => $normalStack,
]));
$normal = CloudTemplateDeletionService::processDue(1, $normalClient);
$assert($normal['completed'] === 1
&& CloudTemplateArtifact::find($normalId) === null
&& (string) CloudTemplate::findOrFail($normalId)->state === 'retired'
&& (int) CloudTemplate::findOrFail($normalId)->revision === 2
&& (string) CloudJob::findOrFail($normalParentId)->outcome
=== 'succeeded'
&& array_map(
static fn (array $item): string => $item['request']->getMethod(),
$normalHistory
) === ['HEAD', 'DELETE'],
'Confirmed master deletion did not retire the exact catalog version.');
[$unknownId, $unknownParentId] = $fixture('unknown');
$unknownHistory = [];
$unknownStack = HandlerStack::create(new MockHandler([
new Response(200, [
'Content-Length' => '4096', 'ETag' => '"etag-unknown"',
'x-amz-version-id' => 'version-unknown',
]),
new ConnectException('lost after delete', new Request(
'DELETE', 'https://objects.example.com'
)),
new Response(200, [
'Content-Length' => '4096', 'ETag' => '"etag-unknown"',
'x-amz-version-id' => 'version-unknown',
]),
new Response(200, [
'Content-Length' => '4096', 'ETag' => '"etag-unknown"',
'x-amz-version-id' => 'version-unknown',
]),
new Response(204),
]));
$unknownStack->push(Middleware::history($unknownHistory));
$unknownClient = new CloudObjectStorageS3Client(new Client([
'handler' => $unknownStack,
]));
CloudTemplateDeletionService::processDue(1, $unknownClient);
$unknownArtifact = CloudTemplateArtifact::findOrFail($unknownId);
$assert((string) $unknownArtifact->state === 'delete_uncertain'
&& (int) $unknownArtifact->delete_generation === 1,
'Ambiguous DELETE was not fenced on its original generation.');
Capsule::table('cloud_template_artifacts')->where('template_id', $unknownId)
->update(['delete_next_check_at' => 0]);
CloudTemplateDeletionService::processDue(1, $unknownClient);
$unknownArtifact = CloudTemplateArtifact::findOrFail($unknownId);
$assert((int) $unknownArtifact->delete_presence_confirmations === 1
&& (int) $unknownArtifact->delete_generation === 1,
'One presence probe incorrectly replayed an ambiguous DELETE.');
Capsule::table('cloud_template_artifacts')->where('template_id', $unknownId)
->update([
'delete_last_checked_at' => time() - 301,
'delete_next_check_at' => 0,
]);
$converged = CloudTemplateDeletionService::processDue(1, $unknownClient);
$assert($converged['completed'] === 1
&& CloudTemplateArtifact::find($unknownId) === null
&& (string) CloudTemplate::findOrFail($unknownId)->state === 'retired'
&& (string) CloudJob::findOrFail($unknownParentId)->outcome
=== 'succeeded'
&& array_map(
static fn (array $item): string => $item['request']->getMethod(),
$unknownHistory
) === ['HEAD', 'DELETE', 'HEAD', 'HEAD', 'DELETE'],
'Stable two-probe evidence did not create exactly one fresh delete generation.');
[$attentionId, $attentionParentId] = $fixture('attention');
$attentionClient = new CloudObjectStorageS3Client(new Client([
'handler' => HandlerStack::create(new MockHandler([
new Response(200, [
'Content-Length' => '8192', 'ETag' => '"different"',
'x-amz-version-id' => 'version-attention',
]),
])),
]));
$attention = CloudTemplateDeletionService::processDue(1, $attentionClient);
$assert($attention['attention'] === 1
&& (string) CloudTemplateArtifact::findOrFail($attentionId)->state
=== 'delete_attention'
&& (string) CloudTemplate::findOrFail($attentionId)->state
=== 'delete_failed'
&& (string) CloudJob::findOrFail($attentionParentId)->outcome
=== 'failed',
'Changed object identity was not fenced for explicit administrator review.');
echo "Cloud-template two-phase deletion and recovery behavior passed.\n";
}