You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
virt/web/tests/CloudTemplateMultipartImpor...

399 lines
17 KiB
PHP

<?php
namespace App\Services {
final class Config
{
public static string $key = '';
public static function get($name, $default = null)
{
return $name === 'cloud_data_key' ? self::$key : $default;
}
}
}
namespace {
require dirname(__DIR__) . '/vendor/autoload.php';
use App\Models\CloudTemplateImportIntent;
use App\Services\Cloud\CloudObjectStorageS3Client;
use App\Services\Cloud\CloudSecrets;
use App\Services\Cloud\CloudTemplateImportWorkflow;
use App\Services\Config;
use GuzzleHttp\Client;
use GuzzleHttp\Exception\ConnectException;
use GuzzleHttp\Handler\MockHandler;
use GuzzleHttp\HandlerStack;
use GuzzleHttp\Middleware;
use GuzzleHttp\Psr7\Request;
use GuzzleHttp\Psr7\Response;
use Illuminate\Database\Capsule\Manager as Capsule;
use Illuminate\Database\Schema\Blueprint;
$assert = static function (bool $condition, string $message): void {
if (!$condition) {
throw new \RuntimeException($message);
}
};
Config::$key = base64_encode(str_repeat('m', 32));
$capsule = new Capsule();
$capsule->addConnection([
'driver' => 'sqlite', 'database' => ':memory:',
], 'default');
$capsule->setAsGlobal();
$capsule->bootEloquent();
$schema = $capsule->schema('default');
$schema->create('cloud_template_import_intents',
static function (Blueprint $table): void {
$table->string('id')->primary();
$table->integer('user_id');
$table->string('request_idempotency_key');
$table->string('request_fingerprint');
$table->string('source_kind');
$table->string('name');
$table->string('os_family');
$table->string('os_version');
$table->string('source_filename')->default('');
$table->integer('source_size_bytes')->default(0);
$table->integer('source_last_modified')->default(0);
$table->string('source_sha256')->default('');
$table->text('sealed_source');
$table->string('template_id')->unique();
$table->string('catalog_key');
$table->string('version_label');
$table->string('generation_token')->unique();
$table->integer('storage_revision');
$table->text('sealed_storage_runtime');
$table->string('object_key')->unique();
$table->string('multipart_upload_id')->default('');
$table->integer('multipart_part_size_bytes')->default(0);
$table->integer('multipart_part_count')->default(0);
$table->string('object_version_id')->default('');
$table->string('object_etag')->default('');
$table->string('state');
$table->integer('active_job_id')->nullable();
$table->string('error_code')->default('');
$table->integer('expires_at');
$table->integer('completed_at')->nullable();
$table->integer('created_at');
$table->integer('updated_at');
$table->unique(['user_id', 'request_idempotency_key']);
});
$schema->create('cloud_template_import_parts',
static function (Blueprint $table): void {
$table->string('intent_id');
$table->integer('part_number');
$table->integer('size_bytes');
$table->string('etag');
$table->integer('created_at');
$table->integer('updated_at');
$table->primary(['intent_id', 'part_number']);
});
$schema->create('cloud_templates',
static function (Blueprint $table): void {
$table->string('id')->primary();
$table->string('catalog_key');
$table->string('version_label');
$table->boolean('is_default')->default(false);
$table->string('name');
$table->string('os_family');
$table->string('os_version');
$table->integer('image_bytes');
$table->string('sha256');
$table->text('startup_script');
$table->integer('revision');
$table->string('generation_token')->unique();
$table->string('state');
$table->integer('active_job_id')->nullable();
$table->integer('created_at');
$table->integer('updated_at');
});
$schema->create('cloud_template_artifacts',
static function (Blueprint $table): void {
$table->string('template_id')->primary();
$table->integer('storage_revision');
$table->text('sealed_storage_runtime');
$table->string('object_key');
$table->string('object_version_id')->default('');
$table->string('object_etag')->default('');
$table->integer('size_bytes');
$table->string('sha256');
$table->string('state');
$table->string('error_code')->default('');
$table->string('delete_operation_id')->default('');
$table->integer('delete_generation')->default(0);
$table->integer('delete_cycle_attempts')->default(0);
$table->integer('delete_presence_confirmations')->default(0);
$table->integer('delete_last_checked_at')->default(0);
$table->integer('delete_next_check_at')->default(0);
$table->integer('delete_attention_at')->nullable();
$table->integer('verified_at')->default(0);
$table->integer('created_at');
$table->integer('updated_at');
});
$schema->create('cloud_template_replicas',
static function (Blueprint $table): void {
$table->integer('host_id');
$table->string('template_id');
$table->integer('revision');
$table->string('generation_token');
$table->string('state');
$table->integer('image_bytes');
$table->string('sha256');
$table->string('operation_id');
$table->integer('job_id')->nullable();
$table->string('error_code')->default('');
$table->integer('created_at');
$table->integer('updated_at');
$table->primary(['host_id', 'template_id']);
});
$schema->create('cloud_jobs',
static function (Blueprint $table): void {
$table->increments('id');
$table->string('operation_id')->unique();
$table->string('request_idempotency_key');
$table->string('request_fingerprint');
$table->integer('parent_job_id')->nullable();
$table->integer('user_id');
$table->integer('host_id')->nullable();
$table->string('kind');
$table->string('execution_mode');
$table->string('phase');
$table->string('outcome')->nullable();
$table->string('status');
$table->text('sealed_payload');
$table->integer('attempts')->default(0);
$table->integer('dispatch_attempts')->default(0);
$table->integer('dispatch_deadline_at')->default(1);
$table->integer('dispatch_started_at')->nullable();
$table->integer('agent_accepted_at')->nullable();
$table->integer('dispatched_at')->nullable();
$table->integer('receipt_deadline_at')->nullable();
$table->integer('terminal_at')->nullable();
$table->boolean('effect_may_have_applied')->nullable();
$table->boolean('safe_to_resubmit')->nullable();
$table->string('effect_resolution')->default('not_applicable');
$table->integer('effect_resolved_at')->nullable();
$table->integer('effect_resolved_by')->nullable();
$table->string('effect_resolution_reason')->default('');
$table->boolean('finalization_incident')->default(0);
$table->string('finalization_error_code')->nullable();
$table->string('completion_event_id')->nullable();
$table->integer('read_at')->nullable();
$table->string('lease_token')->nullable();
$table->integer('lease_until')->default(0);
$table->integer('available_at')->default(0);
$table->string('error_code')->nullable();
$table->text('result_json')->nullable();
$table->integer('created_at');
$table->integer('updated_at');
$table->unique(['user_id', 'request_idempotency_key']);
});
$schema->create('cloud_hosts', static function (Blueprint $table): void {
$table->increments('id');
$table->integer('enabled');
$table->string('default_network');
});
$schema->create('cloud_host_agents',
static function (Blueprint $table): void {
$table->integer('host_id')->primary();
$table->integer('enabled');
$table->integer('revision');
$table->integer('tested_revision');
$table->string('last_test_status');
});
$schema->create('cloud_host_storage_configurations',
static function (Blueprint $table): void {
$table->integer('host_id')->primary();
$table->string('state');
});
Capsule::table('cloud_hosts')->insert([
'id' => 1,
'enabled' => 1,
'default_network' => 'bridge:br0',
]);
Capsule::table('cloud_host_agents')->insert([
'host_id' => 1,
'enabled' => 1,
'revision' => 3,
'tested_revision' => 3,
'last_test_status' => 'healthy',
]);
Capsule::table('cloud_host_storage_configurations')->insert([
'host_id' => 1,
'state' => 'applied',
]);
$runtime = [
'config' => [
'endpoint' => 'https://objects.example.test',
'region' => 'ewr',
'bucket' => 'templates',
'prefix' => 'virt-cloud',
'path_style' => true,
],
'credentials' => [
'access_key_id' => 'MULTIPARTACCESS',
'secret_access_key' => 'multipart-secret',
],
'revision' => 11,
];
$clock = 2000001000;
$workflow = new CloudTemplateImportWorkflow(
static fn (): array => $runtime,
static fn (): int => $clock
);
$actor = [
'user_id' => 9,
'is_admin' => true,
'is_super_admin' => true,
'can_manage' => true,
'request_idempotency_key' => 'multipart-template-intent-0001',
];
$input = [
'name' => 'Debian 13 multipart',
'os_family' => 'Debian',
'os_version' => '13',
'filename' => 'debian-13.qcow2',
'size_bytes' => 64 * 1024 * 1024,
'last_modified' => 2000000000000,
];
$digest = str_repeat('a', 64);
$partEtag = '"part-etag-1"';
$partsXml = '<ListPartsResult><IsTruncated>false</IsTruncated>'
. '<Part><PartNumber>1</PartNumber><Size>67108864</Size>'
. '<ETag>' . $partEtag . '</ETag></Part></ListPartsResult>';
$history = [];
$stack = HandlerStack::create(new MockHandler([
new Response(200, [], '<InitiateMultipartUploadResult>'
. '<UploadId>intent-upload-1</UploadId>'
. '</InitiateMultipartUploadResult>'),
new Response(200, [], $partsXml),
new Response(200, [], '<CompleteMultipartUploadResult/>'),
new Response(200, [
'Content-Length' => '67108864',
'ETag' => '"object-etag-1"',
'x-amz-version-id' => 'version-1',
]),
new Response(200, [
'Content-Length' => '67108864',
'ETag' => '"object-etag-1"',
'x-amz-version-id' => 'version-1',
]),
]));
$stack->push(Middleware::history($history));
$client = new CloudObjectStorageS3Client(new Client(['handler' => $stack]));
$intent = $workflow->beginUpload($input, $actor)['intent'];
$plan = $workflow->startUpload((string) $intent['id'], $actor, $client);
$assert(count($plan['part_urls']) === 1
&& !array_key_exists('upload_id', $plan)
&& !str_contains(json_encode($plan, JSON_THROW_ON_ERROR),
'multipart-secret'),
'The browser upload plan leaked provider authority or credentials.');
$workflow->recordPart((string) $intent['id'], 1, [
'size_bytes' => 64 * 1024 * 1024,
'etag' => $partEtag,
], $actor);
$ack = $workflow->completeUpload(
(string) $intent['id'], ['sha256' => $digest], $actor, $client
);
$stored = CloudTemplateImportIntent::findOrFail((string) $intent['id']);
$assert((string) $stored->state === 'distributing'
&& (string) $stored->object_version_id === 'version-1'
&& Capsule::table('cloud_templates')->count() === 1
&& Capsule::table('cloud_template_artifacts')->count() === 1
&& Capsule::table('cloud_template_replicas')->count() === 1
&& Capsule::table('cloud_jobs')->count() === 2
&& (int) $ack['task_id'] === (int) $stored->active_job_id,
'A completed multipart source was not adopted and distributed once.');
$artifactRuntime = CloudSecrets::decrypt((string) Capsule::table(
'cloud_template_artifacts'
)->value('sealed_storage_runtime'), 'template-artifact-runtime');
$assert(($artifactRuntime['revision'] ?? null) === 11
&& ($artifactRuntime['credentials']['access_key_id'] ?? null)
=== 'MULTIPARTACCESS',
'The adopted artifact retained import-purpose ciphertext.');
$assert(array_map(static fn (array $item): string =>
$item['request']->getMethod(), $history) === [
'POST', 'GET', 'POST', 'HEAD',
'HEAD',
], 'Multipart import used an unexpected S3 control sequence.');
// Simulate a process stop after the aggregate/children committed but
// before the intent projection recorded its task. A page retry must bind
// the existing task without contacting S3 or allocating another one.
Capsule::table('cloud_template_import_intents')
->where('id', (string) $intent['id'])->update([
'state' => 'source_ready',
'active_job_id' => null,
]);
$emptyClient = new CloudObjectStorageS3Client(new Client([
'handler' => HandlerStack::create(new MockHandler([])),
]));
$replayAck = $workflow->completeUpload(
(string) $intent['id'], ['sha256' => $digest], $actor, $emptyClient
);
$assert(Capsule::table('cloud_templates')->count() === 1
&& Capsule::table('cloud_template_artifacts')->count() === 1
&& Capsule::table('cloud_jobs')->count() === 2
&& $replayAck === $ack
&& (string) CloudTemplateImportIntent::findOrFail(
(string) $intent['id']
)->state === 'distributing',
'Completion replay allocated a second template or task generation.');
// CompleteMultipartUpload may commit even when its response is lost. The
// workflow must reconcile HEAD and must never issue Complete a second time.
$actor['request_idempotency_key'] = 'multipart-template-intent-0002';
$lost = $workflow->beginUpload(array_replace($input, [
'name' => 'Debian 13 lost response',
'filename' => 'debian-13-lost.qcow2',
]), $actor)['intent'];
$lostHistory = [];
$lostStack = HandlerStack::create(new MockHandler([
new Response(200, [], '<InitiateMultipartUploadResult>'
. '<UploadId>intent-upload-2</UploadId>'
. '</InitiateMultipartUploadResult>'),
new Response(200, [], $partsXml),
new ConnectException('response lost',
new Request('POST', 'https://objects.example.test')),
new Response(200, [
'Content-Length' => '67108864',
'ETag' => '"object-etag-2"',
'x-amz-version-id' => 'version-2',
]),
new Response(200, [
'Content-Length' => '67108864',
'ETag' => '"object-etag-2"',
'x-amz-version-id' => 'version-2',
]),
]));
$lostStack->push(Middleware::history($lostHistory));
$lostClient = new CloudObjectStorageS3Client(new Client([
'handler' => $lostStack,
]));
$workflow->startUpload((string) $lost['id'], $actor, $lostClient);
$workflow->recordPart((string) $lost['id'], 1, [
'size_bytes' => 64 * 1024 * 1024,
'etag' => $partEtag,
], $actor);
$lostAck = $workflow->completeUpload(
(string) $lost['id'], ['sha256' => $digest], $actor, $lostClient
);
$lostStored = CloudTemplateImportIntent::findOrFail((string) $lost['id']);
$assert((string) $lostStored->state === 'distributing'
&& (string) $lostStored->object_version_id === 'version-2'
&& (int) $lostAck['task_id'] === (int) $lostStored->active_job_id
&& array_map(static fn (array $item): string =>
$item['request']->getMethod(), $lostHistory) === [
'POST', 'GET', 'POST', 'HEAD',
'HEAD',
],
'A lost completion response was replayed instead of reconciled.');
echo "Cloud-template resumable multipart import behavior passed.\n";
}