You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
399 lines
17 KiB
PHP
399 lines
17 KiB
PHP
<?php
|
|
|
|
namespace App\Services {
|
|
final class Config
|
|
{
|
|
public static string $key = '';
|
|
|
|
public static function get($name, $default = null)
|
|
{
|
|
return $name === 'cloud_data_key' ? self::$key : $default;
|
|
}
|
|
}
|
|
}
|
|
|
|
namespace {
|
|
require dirname(__DIR__) . '/vendor/autoload.php';
|
|
|
|
use App\Models\CloudTemplateImportIntent;
|
|
use App\Services\Cloud\CloudObjectStorageS3Client;
|
|
use App\Services\Cloud\CloudSecrets;
|
|
use App\Services\Cloud\CloudTemplateImportWorkflow;
|
|
use App\Services\Config;
|
|
use GuzzleHttp\Client;
|
|
use GuzzleHttp\Exception\ConnectException;
|
|
use GuzzleHttp\Handler\MockHandler;
|
|
use GuzzleHttp\HandlerStack;
|
|
use GuzzleHttp\Middleware;
|
|
use GuzzleHttp\Psr7\Request;
|
|
use GuzzleHttp\Psr7\Response;
|
|
use Illuminate\Database\Capsule\Manager as Capsule;
|
|
use Illuminate\Database\Schema\Blueprint;
|
|
|
|
$assert = static function (bool $condition, string $message): void {
|
|
if (!$condition) {
|
|
throw new \RuntimeException($message);
|
|
}
|
|
};
|
|
|
|
Config::$key = base64_encode(str_repeat('m', 32));
|
|
$capsule = new Capsule();
|
|
$capsule->addConnection([
|
|
'driver' => 'sqlite', 'database' => ':memory:',
|
|
], 'default');
|
|
$capsule->setAsGlobal();
|
|
$capsule->bootEloquent();
|
|
$schema = $capsule->schema('default');
|
|
$schema->create('cloud_template_import_intents',
|
|
static function (Blueprint $table): void {
|
|
$table->string('id')->primary();
|
|
$table->integer('user_id');
|
|
$table->string('request_idempotency_key');
|
|
$table->string('request_fingerprint');
|
|
$table->string('source_kind');
|
|
$table->string('name');
|
|
$table->string('os_family');
|
|
$table->string('os_version');
|
|
$table->string('source_filename')->default('');
|
|
$table->integer('source_size_bytes')->default(0);
|
|
$table->integer('source_last_modified')->default(0);
|
|
$table->string('source_sha256')->default('');
|
|
$table->text('sealed_source');
|
|
$table->string('template_id')->unique();
|
|
$table->string('catalog_key');
|
|
$table->string('version_label');
|
|
$table->string('generation_token')->unique();
|
|
$table->integer('storage_revision');
|
|
$table->text('sealed_storage_runtime');
|
|
$table->string('object_key')->unique();
|
|
$table->string('multipart_upload_id')->default('');
|
|
$table->integer('multipart_part_size_bytes')->default(0);
|
|
$table->integer('multipart_part_count')->default(0);
|
|
$table->string('object_version_id')->default('');
|
|
$table->string('object_etag')->default('');
|
|
$table->string('state');
|
|
$table->integer('active_job_id')->nullable();
|
|
$table->string('error_code')->default('');
|
|
$table->integer('expires_at');
|
|
$table->integer('completed_at')->nullable();
|
|
$table->integer('created_at');
|
|
$table->integer('updated_at');
|
|
$table->unique(['user_id', 'request_idempotency_key']);
|
|
});
|
|
$schema->create('cloud_template_import_parts',
|
|
static function (Blueprint $table): void {
|
|
$table->string('intent_id');
|
|
$table->integer('part_number');
|
|
$table->integer('size_bytes');
|
|
$table->string('etag');
|
|
$table->integer('created_at');
|
|
$table->integer('updated_at');
|
|
$table->primary(['intent_id', 'part_number']);
|
|
});
|
|
$schema->create('cloud_templates',
|
|
static function (Blueprint $table): void {
|
|
$table->string('id')->primary();
|
|
$table->string('catalog_key');
|
|
$table->string('version_label');
|
|
$table->boolean('is_default')->default(false);
|
|
$table->string('name');
|
|
$table->string('os_family');
|
|
$table->string('os_version');
|
|
$table->integer('image_bytes');
|
|
$table->string('sha256');
|
|
$table->text('startup_script');
|
|
$table->integer('revision');
|
|
$table->string('generation_token')->unique();
|
|
$table->string('state');
|
|
$table->integer('active_job_id')->nullable();
|
|
$table->integer('created_at');
|
|
$table->integer('updated_at');
|
|
});
|
|
$schema->create('cloud_template_artifacts',
|
|
static function (Blueprint $table): void {
|
|
$table->string('template_id')->primary();
|
|
$table->integer('storage_revision');
|
|
$table->text('sealed_storage_runtime');
|
|
$table->string('object_key');
|
|
$table->string('object_version_id')->default('');
|
|
$table->string('object_etag')->default('');
|
|
$table->integer('size_bytes');
|
|
$table->string('sha256');
|
|
$table->string('state');
|
|
$table->string('error_code')->default('');
|
|
$table->string('delete_operation_id')->default('');
|
|
$table->integer('delete_generation')->default(0);
|
|
$table->integer('delete_cycle_attempts')->default(0);
|
|
$table->integer('delete_presence_confirmations')->default(0);
|
|
$table->integer('delete_last_checked_at')->default(0);
|
|
$table->integer('delete_next_check_at')->default(0);
|
|
$table->integer('delete_attention_at')->nullable();
|
|
$table->integer('verified_at')->default(0);
|
|
$table->integer('created_at');
|
|
$table->integer('updated_at');
|
|
});
|
|
$schema->create('cloud_template_replicas',
|
|
static function (Blueprint $table): void {
|
|
$table->integer('host_id');
|
|
$table->string('template_id');
|
|
$table->integer('revision');
|
|
$table->string('generation_token');
|
|
$table->string('state');
|
|
$table->integer('image_bytes');
|
|
$table->string('sha256');
|
|
$table->string('operation_id');
|
|
$table->integer('job_id')->nullable();
|
|
$table->string('error_code')->default('');
|
|
$table->integer('created_at');
|
|
$table->integer('updated_at');
|
|
$table->primary(['host_id', 'template_id']);
|
|
});
|
|
$schema->create('cloud_jobs',
|
|
static function (Blueprint $table): void {
|
|
$table->increments('id');
|
|
$table->string('operation_id')->unique();
|
|
$table->string('request_idempotency_key');
|
|
$table->string('request_fingerprint');
|
|
$table->integer('parent_job_id')->nullable();
|
|
$table->integer('user_id');
|
|
$table->integer('host_id')->nullable();
|
|
$table->string('kind');
|
|
$table->string('execution_mode');
|
|
$table->string('phase');
|
|
$table->string('outcome')->nullable();
|
|
$table->string('status');
|
|
$table->text('sealed_payload');
|
|
$table->integer('attempts')->default(0);
|
|
$table->integer('dispatch_attempts')->default(0);
|
|
$table->integer('dispatch_deadline_at')->default(1);
|
|
$table->integer('dispatch_started_at')->nullable();
|
|
$table->integer('agent_accepted_at')->nullable();
|
|
$table->integer('dispatched_at')->nullable();
|
|
$table->integer('receipt_deadline_at')->nullable();
|
|
$table->integer('terminal_at')->nullable();
|
|
$table->boolean('effect_may_have_applied')->nullable();
|
|
$table->boolean('safe_to_resubmit')->nullable();
|
|
$table->string('effect_resolution')->default('not_applicable');
|
|
$table->integer('effect_resolved_at')->nullable();
|
|
$table->integer('effect_resolved_by')->nullable();
|
|
$table->string('effect_resolution_reason')->default('');
|
|
$table->boolean('finalization_incident')->default(0);
|
|
$table->string('finalization_error_code')->nullable();
|
|
$table->string('completion_event_id')->nullable();
|
|
$table->integer('read_at')->nullable();
|
|
$table->string('lease_token')->nullable();
|
|
$table->integer('lease_until')->default(0);
|
|
$table->integer('available_at')->default(0);
|
|
$table->string('error_code')->nullable();
|
|
$table->text('result_json')->nullable();
|
|
$table->integer('created_at');
|
|
$table->integer('updated_at');
|
|
$table->unique(['user_id', 'request_idempotency_key']);
|
|
});
|
|
$schema->create('cloud_hosts', static function (Blueprint $table): void {
|
|
$table->increments('id');
|
|
$table->integer('enabled');
|
|
$table->string('default_network');
|
|
});
|
|
$schema->create('cloud_host_agents',
|
|
static function (Blueprint $table): void {
|
|
$table->integer('host_id')->primary();
|
|
$table->integer('enabled');
|
|
$table->integer('revision');
|
|
$table->integer('tested_revision');
|
|
$table->string('last_test_status');
|
|
});
|
|
$schema->create('cloud_host_storage_configurations',
|
|
static function (Blueprint $table): void {
|
|
$table->integer('host_id')->primary();
|
|
$table->string('state');
|
|
});
|
|
Capsule::table('cloud_hosts')->insert([
|
|
'id' => 1,
|
|
'enabled' => 1,
|
|
'default_network' => 'bridge:br0',
|
|
]);
|
|
Capsule::table('cloud_host_agents')->insert([
|
|
'host_id' => 1,
|
|
'enabled' => 1,
|
|
'revision' => 3,
|
|
'tested_revision' => 3,
|
|
'last_test_status' => 'healthy',
|
|
]);
|
|
Capsule::table('cloud_host_storage_configurations')->insert([
|
|
'host_id' => 1,
|
|
'state' => 'applied',
|
|
]);
|
|
|
|
$runtime = [
|
|
'config' => [
|
|
'endpoint' => 'https://objects.example.test',
|
|
'region' => 'ewr',
|
|
'bucket' => 'templates',
|
|
'prefix' => 'virt-cloud',
|
|
'path_style' => true,
|
|
],
|
|
'credentials' => [
|
|
'access_key_id' => 'MULTIPARTACCESS',
|
|
'secret_access_key' => 'multipart-secret',
|
|
],
|
|
'revision' => 11,
|
|
];
|
|
$clock = 2000001000;
|
|
$workflow = new CloudTemplateImportWorkflow(
|
|
static fn (): array => $runtime,
|
|
static fn (): int => $clock
|
|
);
|
|
$actor = [
|
|
'user_id' => 9,
|
|
'is_admin' => true,
|
|
'is_super_admin' => true,
|
|
'can_manage' => true,
|
|
'request_idempotency_key' => 'multipart-template-intent-0001',
|
|
];
|
|
$input = [
|
|
'name' => 'Debian 13 multipart',
|
|
'os_family' => 'Debian',
|
|
'os_version' => '13',
|
|
'filename' => 'debian-13.qcow2',
|
|
'size_bytes' => 64 * 1024 * 1024,
|
|
'last_modified' => 2000000000000,
|
|
];
|
|
$digest = str_repeat('a', 64);
|
|
$partEtag = '"part-etag-1"';
|
|
$partsXml = '<ListPartsResult><IsTruncated>false</IsTruncated>'
|
|
. '<Part><PartNumber>1</PartNumber><Size>67108864</Size>'
|
|
. '<ETag>' . $partEtag . '</ETag></Part></ListPartsResult>';
|
|
|
|
$history = [];
|
|
$stack = HandlerStack::create(new MockHandler([
|
|
new Response(200, [], '<InitiateMultipartUploadResult>'
|
|
. '<UploadId>intent-upload-1</UploadId>'
|
|
. '</InitiateMultipartUploadResult>'),
|
|
new Response(200, [], $partsXml),
|
|
new Response(200, [], '<CompleteMultipartUploadResult/>'),
|
|
new Response(200, [
|
|
'Content-Length' => '67108864',
|
|
'ETag' => '"object-etag-1"',
|
|
'x-amz-version-id' => 'version-1',
|
|
]),
|
|
new Response(200, [
|
|
'Content-Length' => '67108864',
|
|
'ETag' => '"object-etag-1"',
|
|
'x-amz-version-id' => 'version-1',
|
|
]),
|
|
]));
|
|
$stack->push(Middleware::history($history));
|
|
$client = new CloudObjectStorageS3Client(new Client(['handler' => $stack]));
|
|
|
|
$intent = $workflow->beginUpload($input, $actor)['intent'];
|
|
$plan = $workflow->startUpload((string) $intent['id'], $actor, $client);
|
|
$assert(count($plan['part_urls']) === 1
|
|
&& !array_key_exists('upload_id', $plan)
|
|
&& !str_contains(json_encode($plan, JSON_THROW_ON_ERROR),
|
|
'multipart-secret'),
|
|
'The browser upload plan leaked provider authority or credentials.');
|
|
$workflow->recordPart((string) $intent['id'], 1, [
|
|
'size_bytes' => 64 * 1024 * 1024,
|
|
'etag' => $partEtag,
|
|
], $actor);
|
|
$ack = $workflow->completeUpload(
|
|
(string) $intent['id'], ['sha256' => $digest], $actor, $client
|
|
);
|
|
$stored = CloudTemplateImportIntent::findOrFail((string) $intent['id']);
|
|
$assert((string) $stored->state === 'distributing'
|
|
&& (string) $stored->object_version_id === 'version-1'
|
|
&& Capsule::table('cloud_templates')->count() === 1
|
|
&& Capsule::table('cloud_template_artifacts')->count() === 1
|
|
&& Capsule::table('cloud_template_replicas')->count() === 1
|
|
&& Capsule::table('cloud_jobs')->count() === 2
|
|
&& (int) $ack['task_id'] === (int) $stored->active_job_id,
|
|
'A completed multipart source was not adopted and distributed once.');
|
|
$artifactRuntime = CloudSecrets::decrypt((string) Capsule::table(
|
|
'cloud_template_artifacts'
|
|
)->value('sealed_storage_runtime'), 'template-artifact-runtime');
|
|
$assert(($artifactRuntime['revision'] ?? null) === 11
|
|
&& ($artifactRuntime['credentials']['access_key_id'] ?? null)
|
|
=== 'MULTIPARTACCESS',
|
|
'The adopted artifact retained import-purpose ciphertext.');
|
|
$assert(array_map(static fn (array $item): string =>
|
|
$item['request']->getMethod(), $history) === [
|
|
'POST', 'GET', 'POST', 'HEAD',
|
|
'HEAD',
|
|
], 'Multipart import used an unexpected S3 control sequence.');
|
|
|
|
// Simulate a process stop after the aggregate/children committed but
|
|
// before the intent projection recorded its task. A page retry must bind
|
|
// the existing task without contacting S3 or allocating another one.
|
|
Capsule::table('cloud_template_import_intents')
|
|
->where('id', (string) $intent['id'])->update([
|
|
'state' => 'source_ready',
|
|
'active_job_id' => null,
|
|
]);
|
|
$emptyClient = new CloudObjectStorageS3Client(new Client([
|
|
'handler' => HandlerStack::create(new MockHandler([])),
|
|
]));
|
|
$replayAck = $workflow->completeUpload(
|
|
(string) $intent['id'], ['sha256' => $digest], $actor, $emptyClient
|
|
);
|
|
$assert(Capsule::table('cloud_templates')->count() === 1
|
|
&& Capsule::table('cloud_template_artifacts')->count() === 1
|
|
&& Capsule::table('cloud_jobs')->count() === 2
|
|
&& $replayAck === $ack
|
|
&& (string) CloudTemplateImportIntent::findOrFail(
|
|
(string) $intent['id']
|
|
)->state === 'distributing',
|
|
'Completion replay allocated a second template or task generation.');
|
|
|
|
// CompleteMultipartUpload may commit even when its response is lost. The
|
|
// workflow must reconcile HEAD and must never issue Complete a second time.
|
|
$actor['request_idempotency_key'] = 'multipart-template-intent-0002';
|
|
$lost = $workflow->beginUpload(array_replace($input, [
|
|
'name' => 'Debian 13 lost response',
|
|
'filename' => 'debian-13-lost.qcow2',
|
|
]), $actor)['intent'];
|
|
$lostHistory = [];
|
|
$lostStack = HandlerStack::create(new MockHandler([
|
|
new Response(200, [], '<InitiateMultipartUploadResult>'
|
|
. '<UploadId>intent-upload-2</UploadId>'
|
|
. '</InitiateMultipartUploadResult>'),
|
|
new Response(200, [], $partsXml),
|
|
new ConnectException('response lost',
|
|
new Request('POST', 'https://objects.example.test')),
|
|
new Response(200, [
|
|
'Content-Length' => '67108864',
|
|
'ETag' => '"object-etag-2"',
|
|
'x-amz-version-id' => 'version-2',
|
|
]),
|
|
new Response(200, [
|
|
'Content-Length' => '67108864',
|
|
'ETag' => '"object-etag-2"',
|
|
'x-amz-version-id' => 'version-2',
|
|
]),
|
|
]));
|
|
$lostStack->push(Middleware::history($lostHistory));
|
|
$lostClient = new CloudObjectStorageS3Client(new Client([
|
|
'handler' => $lostStack,
|
|
]));
|
|
$workflow->startUpload((string) $lost['id'], $actor, $lostClient);
|
|
$workflow->recordPart((string) $lost['id'], 1, [
|
|
'size_bytes' => 64 * 1024 * 1024,
|
|
'etag' => $partEtag,
|
|
], $actor);
|
|
$lostAck = $workflow->completeUpload(
|
|
(string) $lost['id'], ['sha256' => $digest], $actor, $lostClient
|
|
);
|
|
$lostStored = CloudTemplateImportIntent::findOrFail((string) $lost['id']);
|
|
$assert((string) $lostStored->state === 'distributing'
|
|
&& (string) $lostStored->object_version_id === 'version-2'
|
|
&& (int) $lostAck['task_id'] === (int) $lostStored->active_job_id
|
|
&& array_map(static fn (array $item): string =>
|
|
$item['request']->getMethod(), $lostHistory) === [
|
|
'POST', 'GET', 'POST', 'HEAD',
|
|
'HEAD',
|
|
],
|
|
'A lost completion response was replayed instead of reconciled.');
|
|
|
|
echo "Cloud-template resumable multipart import behavior passed.\n";
|
|
}
|