You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
virt/web/tests/CloudTemplateMultipartImpor...

904 lines
39 KiB
PHP

<?php
namespace App\Services {
final class Config
{
public static string $key = '';
public static function get($name, $default = null)
{
return $name === 'cloud_data_key' ? self::$key : $default;
}
}
}
namespace {
require dirname(__DIR__) . '/vendor/autoload.php';
use App\Models\CloudTemplateImportIntent;
use App\Services\Cloud\CloudError;
use App\Services\Cloud\CloudObjectStorageS3Client;
use App\Services\Cloud\CloudSecrets;
use App\Services\Cloud\CloudTemplateImportWorkflow;
use App\Services\Cloud\CloudTemplateRemoteFetcher;
use App\Services\Config;
use GuzzleHttp\Client;
use GuzzleHttp\Exception\ConnectException;
use GuzzleHttp\Handler\MockHandler;
use GuzzleHttp\HandlerStack;
use GuzzleHttp\Middleware;
use GuzzleHttp\Psr7\Request;
use GuzzleHttp\Psr7\Response;
use Illuminate\Database\Capsule\Manager as Capsule;
use Illuminate\Database\Schema\Blueprint;
$assert = static function (bool $condition, string $message): void {
if (!$condition) {
throw new \RuntimeException($message);
}
};
Config::$key = base64_encode(str_repeat('m', 32));
$capsule = new Capsule();
$capsule->addConnection([
'driver' => 'sqlite', 'database' => ':memory:',
], 'default');
$capsule->setAsGlobal();
$capsule->bootEloquent();
$schema = $capsule->schema('default');
$schema->create('cloud_template_import_intents',
static function (Blueprint $table): void {
$table->string('id')->primary();
$table->integer('user_id');
$table->string('request_idempotency_key');
$table->string('request_fingerprint');
$table->string('source_kind');
$table->string('name');
$table->string('os_family');
$table->string('os_version');
$table->string('source_filename')->default('');
$table->integer('source_size_bytes')->default(0);
$table->integer('source_last_modified')->default(0);
$table->string('source_sha256')->default('');
$table->text('sealed_source');
$table->string('template_id')->unique();
$table->string('catalog_key');
$table->string('version_label');
$table->string('generation_token')->unique();
$table->integer('storage_revision');
$table->text('sealed_storage_runtime');
$table->string('object_key')->unique();
$table->string('multipart_upload_id')->default('');
$table->integer('multipart_part_size_bytes')->default(0);
$table->integer('multipart_part_count')->default(0);
$table->string('object_version_id')->default('');
$table->string('object_etag')->default('');
$table->string('state');
$table->integer('active_job_id')->nullable();
$table->string('error_code')->default('');
$table->integer('expires_at');
$table->integer('completed_at')->nullable();
$table->integer('created_at');
$table->integer('updated_at');
$table->unique(['user_id', 'request_idempotency_key']);
});
$schema->create('cloud_template_import_parts',
static function (Blueprint $table): void {
$table->string('intent_id');
$table->integer('part_number');
$table->integer('size_bytes');
$table->string('etag');
$table->integer('created_at');
$table->integer('updated_at');
$table->primary(['intent_id', 'part_number']);
});
$schema->create('cloud_templates',
static function (Blueprint $table): void {
$table->string('id')->primary();
$table->string('catalog_key');
$table->string('version_label');
$table->boolean('is_default')->default(false);
$table->string('name');
$table->string('os_family');
$table->string('os_version');
$table->integer('image_bytes');
$table->string('sha256');
$table->text('startup_script');
$table->integer('revision');
$table->string('generation_token')->unique();
$table->string('state');
$table->integer('active_job_id')->nullable();
$table->integer('created_at');
$table->integer('updated_at');
});
$schema->create('cloud_template_artifacts',
static function (Blueprint $table): void {
$table->string('template_id')->primary();
$table->integer('storage_revision');
$table->text('sealed_storage_runtime');
$table->string('object_key');
$table->string('object_version_id')->default('');
$table->string('object_etag')->default('');
$table->integer('size_bytes');
$table->string('sha256');
$table->string('state');
$table->string('error_code')->default('');
$table->string('delete_operation_id')->default('');
$table->integer('delete_generation')->default(0);
$table->integer('delete_cycle_attempts')->default(0);
$table->integer('delete_presence_confirmations')->default(0);
$table->integer('delete_last_checked_at')->default(0);
$table->integer('delete_next_check_at')->default(0);
$table->integer('delete_attention_at')->nullable();
$table->integer('verified_at')->default(0);
$table->integer('created_at');
$table->integer('updated_at');
});
$schema->create('cloud_template_replicas',
static function (Blueprint $table): void {
$table->integer('host_id');
$table->string('template_id');
$table->integer('revision');
$table->string('generation_token');
$table->string('state');
$table->integer('image_bytes');
$table->string('sha256');
$table->string('operation_id');
$table->integer('job_id')->nullable();
$table->string('error_code')->default('');
$table->integer('created_at');
$table->integer('updated_at');
$table->primary(['host_id', 'template_id']);
});
$schema->create('cloud_jobs',
static function (Blueprint $table): void {
$table->increments('id');
$table->string('operation_id')->unique();
$table->string('request_idempotency_key');
$table->string('request_fingerprint');
$table->integer('parent_job_id')->nullable();
$table->integer('user_id');
$table->integer('host_id')->nullable();
$table->string('kind');
$table->string('execution_mode');
$table->string('phase');
$table->string('outcome')->nullable();
$table->string('status');
$table->text('sealed_payload');
$table->integer('attempts')->default(0);
$table->integer('dispatch_attempts')->default(0);
$table->integer('dispatch_deadline_at')->default(1);
$table->integer('dispatch_started_at')->nullable();
$table->integer('agent_accepted_at')->nullable();
$table->integer('dispatched_at')->nullable();
$table->integer('receipt_deadline_at')->nullable();
$table->integer('terminal_at')->nullable();
$table->boolean('effect_may_have_applied')->nullable();
$table->boolean('safe_to_resubmit')->nullable();
$table->string('effect_resolution')->default('not_applicable');
$table->integer('effect_resolved_at')->nullable();
$table->integer('effect_resolved_by')->nullable();
$table->string('effect_resolution_reason')->default('');
$table->boolean('finalization_incident')->default(0);
$table->string('finalization_error_code')->nullable();
$table->string('completion_event_id')->nullable();
$table->integer('read_at')->nullable();
$table->string('lease_token')->nullable();
$table->integer('lease_until')->default(0);
$table->integer('available_at')->default(0);
$table->string('error_code')->nullable();
$table->text('result_json')->nullable();
$table->integer('created_at');
$table->integer('updated_at');
$table->unique(['user_id', 'request_idempotency_key']);
});
$schema->create('cloud_hosts', static function (Blueprint $table): void {
$table->increments('id');
$table->integer('enabled');
$table->string('default_network');
});
$schema->create('cloud_host_agents',
static function (Blueprint $table): void {
$table->integer('host_id')->primary();
$table->integer('enabled');
$table->integer('revision');
$table->integer('tested_revision');
$table->string('last_test_status');
});
$schema->create('cloud_host_storage_configurations',
static function (Blueprint $table): void {
$table->integer('host_id')->primary();
$table->string('state');
});
Capsule::table('cloud_hosts')->insert([
'id' => 1,
'enabled' => 1,
'default_network' => 'bridge:br0',
]);
Capsule::table('cloud_host_agents')->insert([
'host_id' => 1,
'enabled' => 1,
'revision' => 3,
'tested_revision' => 3,
'last_test_status' => 'healthy',
]);
Capsule::table('cloud_host_storage_configurations')->insert([
'host_id' => 1,
'state' => 'applied',
]);
$runtime = [
'config' => [
'endpoint' => 'https://objects.example.test',
'region' => 'ewr',
'bucket' => 'templates',
'prefix' => 'virt-cloud',
'path_style' => true,
],
'credentials' => [
'access_key_id' => 'MULTIPARTACCESS',
'secret_access_key' => 'multipart-secret',
],
'revision' => 11,
];
$clock = 2000001000;
$workflow = new CloudTemplateImportWorkflow(
static fn (): array => $runtime,
static fn (): int => $clock
);
$actor = [
'user_id' => 9,
'is_admin' => true,
'is_super_admin' => true,
'can_manage' => true,
'request_idempotency_key' => 'multipart-template-intent-0001',
];
$input = [
'name' => 'Debian 13 multipart',
'os_family' => 'Debian',
'os_version' => '13',
'filename' => 'debian-13.qcow2',
'size_bytes' => 64 * 1024 * 1024,
'last_modified' => 2000000000000,
];
$digest = str_repeat('a', 64);
$partEtag = '"part-etag-1"';
$partsXml = '<ListPartsResult><IsTruncated>false</IsTruncated>'
. '<Part><PartNumber>1</PartNumber><Size>67108864</Size>'
. '<ETag>' . $partEtag . '</ETag></Part></ListPartsResult>';
$history = [];
$stack = HandlerStack::create(new MockHandler([
new Response(200, [], '<InitiateMultipartUploadResult>'
. '<UploadId>intent-upload-1</UploadId>'
. '</InitiateMultipartUploadResult>'),
new Response(200, [], $partsXml),
new Response(200, [], '<CompleteMultipartUploadResult/>'),
new Response(200, [
'Content-Length' => '67108864',
'ETag' => '"object-etag-1"',
'x-amz-version-id' => 'version-1',
]),
new Response(200, [
'Content-Length' => '67108864',
'ETag' => '"object-etag-1"',
'x-amz-version-id' => 'version-1',
]),
]));
$stack->push(Middleware::history($history));
$client = new CloudObjectStorageS3Client(new Client(['handler' => $stack]));
$intent = $workflow->beginUpload($input, $actor)['intent'];
$plan = $workflow->startUpload((string) $intent['id'], $actor, $client);
$assert(count($plan['part_urls']) === 1
&& !array_key_exists('upload_id', $plan)
&& !str_contains(json_encode($plan, JSON_THROW_ON_ERROR),
'multipart-secret'),
'The browser upload plan leaked provider authority or credentials.');
$workflow->recordPart((string) $intent['id'], 1, [
'size_bytes' => 64 * 1024 * 1024,
'etag' => $partEtag,
], $actor);
$ack = $workflow->completeUpload(
(string) $intent['id'], ['sha256' => $digest], $actor, $client
);
$stored = CloudTemplateImportIntent::findOrFail((string) $intent['id']);
$assert((string) $stored->state === 'distributing'
&& (string) $stored->object_version_id === 'version-1'
&& Capsule::table('cloud_templates')->count() === 1
&& Capsule::table('cloud_template_artifacts')->count() === 1
&& Capsule::table('cloud_template_replicas')->count() === 1
&& Capsule::table('cloud_jobs')->count() === 2
&& (int) $ack['task_id'] === (int) $stored->active_job_id,
'A completed multipart source was not adopted and distributed once.');
$artifactRuntime = CloudSecrets::decrypt((string) Capsule::table(
'cloud_template_artifacts'
)->value('sealed_storage_runtime'), 'template-artifact-runtime');
$assert(($artifactRuntime['revision'] ?? null) === 11
&& ($artifactRuntime['credentials']['access_key_id'] ?? null)
=== 'MULTIPARTACCESS',
'The adopted artifact retained import-purpose ciphertext.');
$assert(array_map(static fn (array $item): string =>
$item['request']->getMethod(), $history) === [
'POST', 'GET', 'POST', 'HEAD',
'HEAD',
], 'Multipart import used an unexpected S3 control sequence.');
// Simulate a process stop after the aggregate/children committed but
// before the intent projection recorded its task. A page retry must bind
// the existing task without contacting S3 or allocating another one.
Capsule::table('cloud_template_import_intents')
->where('id', (string) $intent['id'])->update([
'state' => 'source_ready',
'active_job_id' => null,
]);
$emptyClient = new CloudObjectStorageS3Client(new Client([
'handler' => HandlerStack::create(new MockHandler([])),
]));
$replayAck = $workflow->completeUpload(
(string) $intent['id'], ['sha256' => $digest], $actor, $emptyClient
);
$assert(Capsule::table('cloud_templates')->count() === 1
&& Capsule::table('cloud_template_artifacts')->count() === 1
&& Capsule::table('cloud_jobs')->count() === 2
&& $replayAck === $ack
&& (string) CloudTemplateImportIntent::findOrFail(
(string) $intent['id']
)->state === 'distributing',
'Completion replay allocated a second template or task generation.');
// CompleteMultipartUpload may commit even when its response is lost. The
// workflow must reconcile HEAD and must never issue Complete a second time.
$actor['request_idempotency_key'] = 'multipart-template-intent-0002';
$lost = $workflow->beginUpload(array_replace($input, [
'name' => 'Debian 13 lost response',
'filename' => 'debian-13-lost.qcow2',
]), $actor)['intent'];
$lostHistory = [];
$lostStack = HandlerStack::create(new MockHandler([
new Response(200, [], '<InitiateMultipartUploadResult>'
. '<UploadId>intent-upload-2</UploadId>'
. '</InitiateMultipartUploadResult>'),
new Response(200, [], $partsXml),
new ConnectException('response lost',
new Request('POST', 'https://objects.example.test')),
new Response(200, [
'Content-Length' => '67108864',
'ETag' => '"object-etag-2"',
'x-amz-version-id' => 'version-2',
]),
new Response(200, [
'Content-Length' => '67108864',
'ETag' => '"object-etag-2"',
'x-amz-version-id' => 'version-2',
]),
]));
$lostStack->push(Middleware::history($lostHistory));
$lostClient = new CloudObjectStorageS3Client(new Client([
'handler' => $lostStack,
]));
$workflow->startUpload((string) $lost['id'], $actor, $lostClient);
$workflow->recordPart((string) $lost['id'], 1, [
'size_bytes' => 64 * 1024 * 1024,
'etag' => $partEtag,
], $actor);
$lostAck = $workflow->completeUpload(
(string) $lost['id'], ['sha256' => $digest], $actor, $lostClient
);
$lostStored = CloudTemplateImportIntent::findOrFail((string) $lost['id']);
$assert((string) $lostStored->state === 'distributing'
&& (string) $lostStored->object_version_id === 'version-2'
&& (int) $lostAck['task_id'] === (int) $lostStored->active_job_id
&& array_map(static fn (array $item): string =>
$item['request']->getMethod(), $lostHistory) === [
'POST', 'GET', 'POST', 'HEAD',
'HEAD',
],
'A lost completion response was replayed instead of reconciled.');
// Refreshing the page must expose the exact server-confirmed part ledger;
// re-signing an existing upload must not allocate a second upload ID.
$actor['request_idempotency_key'] = 'multipart-template-intent-0003';
$resumable = $workflow->beginUpload(array_replace($input, [
'name' => 'Debian 13 resumable',
'filename' => 'debian-13-resumable.qcow2',
]), $actor)['intent'];
$resumeHistory = [];
$resumeStack = HandlerStack::create(new MockHandler([
new Response(200, [], '<InitiateMultipartUploadResult>'
. '<UploadId>intent-upload-3</UploadId>'
. '</InitiateMultipartUploadResult>'),
new Response(204),
]));
$resumeStack->push(Middleware::history($resumeHistory));
$resumeClient = new CloudObjectStorageS3Client(new Client([
'handler' => $resumeStack,
]));
$workflow->startUpload((string) $resumable['id'], $actor, $resumeClient);
$workflow->recordPart((string) $resumable['id'], 1, [
'size_bytes' => 64 * 1024 * 1024,
'etag' => $partEtag,
], $actor);
$resigned = $workflow->startUpload(
(string) $resumable['id'], $actor, $resumeClient
);
$assert(($resigned['intent']['parts'] ?? null) === [[
'part_number' => 1,
'size_bytes' => 64 * 1024 * 1024,
]]
&& array_map(static fn (array $item): string =>
$item['request']->getMethod(), $resumeHistory) === ['POST'],
'Resume did not expose the exact part ledger or created a new upload.');
$cancelled = $workflow->cancel(
(string) $resumable['id'], $actor, $resumeClient
);
$assert(($cancelled['state'] ?? null) === 'cancelled'
&& array_map(static fn (array $item): string =>
$item['request']->getMethod(), $resumeHistory) === [
'POST', 'DELETE',
],
'A confirmed multipart abort did not settle the intent once.');
// A lost Abort response is reconciled with HEAD + ListParts. A 404 for
// the exact upload ID proves cancellation; DELETE must not be replayed.
$actor['request_idempotency_key'] = 'multipart-template-intent-0004';
$abortLost = $workflow->beginUpload(array_replace($input, [
'name' => 'Debian 13 abort lost',
'filename' => 'debian-13-abort-lost.qcow2',
]), $actor)['intent'];
$abortHistory = [];
$abortStack = HandlerStack::create(new MockHandler([
new Response(200, [], '<InitiateMultipartUploadResult>'
. '<UploadId>intent-upload-4</UploadId>'
. '</InitiateMultipartUploadResult>'),
new ConnectException('abort response lost',
new Request('DELETE', 'https://objects.example.test')),
new Response(404),
new Response(404),
]));
$abortStack->push(Middleware::history($abortHistory));
$abortClient = new CloudObjectStorageS3Client(new Client([
'handler' => $abortStack,
]));
$workflow->startUpload((string) $abortLost['id'], $actor, $abortClient);
$abortResult = $workflow->cancel(
(string) $abortLost['id'], $actor, $abortClient
);
$assert(($abortResult['state'] ?? null) === 'cancelled'
&& array_map(static fn (array $item): string =>
$item['request']->getMethod(), $abortHistory) === [
'POST', 'DELETE', 'HEAD', 'GET',
],
'A lost Abort response was replayed instead of read-only reconciled.');
// If read-only evidence proves the upload still exists, cancellation is
// retryable with a new explicit action rather than reported as complete.
$actor['request_idempotency_key'] = 'multipart-template-intent-0005';
$abortActive = $workflow->beginUpload(array_replace($input, [
'name' => 'Debian 13 abort active',
'filename' => 'debian-13-abort-active.qcow2',
]), $actor)['intent'];
$activeStack = HandlerStack::create(new MockHandler([
new Response(200, [], '<InitiateMultipartUploadResult>'
. '<UploadId>intent-upload-5</UploadId>'
. '</InitiateMultipartUploadResult>'),
new ConnectException('abort response lost',
new Request('DELETE', 'https://objects.example.test')),
new Response(404),
new Response(200, [], '<ListPartsResult><IsTruncated>false</IsTruncated>'
. '</ListPartsResult>'),
]));
$activeClient = new CloudObjectStorageS3Client(new Client([
'handler' => $activeStack,
]));
$workflow->startUpload((string) $abortActive['id'], $actor, $activeClient);
try {
$workflow->cancel((string) $abortActive['id'], $actor, $activeClient);
throw new \RuntimeException('A still-active abort was reported complete.');
} catch (CloudError $error) {
$assert($error->httpStatus === 409,
'A retryable abort failure returned the wrong HTTP status.');
}
$activeStored = CloudTemplateImportIntent::findOrFail(
(string) $abortActive['id']
);
$assert((string) $activeStored->state === 'source_unknown'
&& (string) $activeStored->error_code
=== 'multipart_abort_not_applied'
&& CloudTemplateImportWorkflow::dto($activeStored)['can_cancel']
=== true,
'Provider proof that Abort did not apply was not made retryable.');
// Losing the CreateMultipartUpload response must never be treated as a
// proven no-effect failure. The panel can abandon the unique object key,
// while bucket lifecycle removes any provider-side orphan.
$actor['request_idempotency_key'] = 'multipart-template-intent-0006';
$createUnknown = $workflow->beginUpload(array_replace($input, [
'name' => 'Debian 13 create unknown',
'filename' => 'debian-13-create-unknown.qcow2',
]), $actor)['intent'];
$createUnknownClient = new CloudObjectStorageS3Client(new Client([
'handler' => HandlerStack::create(new MockHandler([
new ConnectException('create response lost',
new Request('POST', 'https://objects.example.test')),
])),
]));
try {
$workflow->startUpload(
(string) $createUnknown['id'], $actor, $createUnknownClient
);
throw new \RuntimeException('Unknown multipart create was retryable.');
} catch (CloudError $error) {
$assert($error->httpStatus === 502,
'Unknown multipart create returned the wrong HTTP status.');
}
$unknownStored = CloudTemplateImportIntent::findOrFail(
(string) $createUnknown['id']
);
$assert((string) $unknownStored->state === 'source_unknown'
&& CloudTemplateImportWorkflow::dto($unknownStored)['can_cancel']
=== true,
'Unknown multipart create was not fenced for lifecycle abandonment.');
$abandoned = $workflow->cancel(
(string) $createUnknown['id'], $actor, $createUnknownClient
);
$assert(($abandoned['state'] ?? null) === 'cancelled'
&& ($abandoned['error_code'] ?? null)
=== 'multipart_abandoned_lifecycle',
'Unknown multipart create was not abandoned for lifecycle cleanup.');
// An expired intent that never crossed multipart creation is cleaned
// locally without contacting object storage.
$actor['request_idempotency_key'] = 'multipart-template-intent-0007';
$expired = $workflow->beginUpload(array_replace($input, [
'name' => 'Debian 13 expired',
'filename' => 'debian-13-expired.qcow2',
]), $actor)['intent'];
Capsule::table('cloud_template_import_intents')
->where('id', (string) $expired['id'])
->update(['expires_at' => $clock - 1]);
$assert($workflow->cleanupOneExpired(new CloudObjectStorageS3Client(
new Client(['handler' => HandlerStack::create(new MockHandler([]))])
))
&& (string) CloudTemplateImportIntent::findOrFail(
(string) $expired['id']
)->state === 'cancelled',
'Expired prepared upload was not cleaned without provider mutation.');
// Official URL imports use the same intent/object/task generation. A
// response replay must return the original task without a second download
// or immutable PUT.
$actor['request_idempotency_key'] = 'multipart-template-intent-0008';
$remoteBody = str_repeat('remote-qcow2-', 90000);
$remoteDigest = hash('sha256', $remoteBody);
$remoteFetchHistory = [];
$remoteFetchStack = HandlerStack::create(new MockHandler([
new Response(200, [
'Content-Length' => (string) strlen($remoteBody),
'Content-Encoding' => 'identity',
], $remoteBody),
]));
$remoteFetchStack->push(Middleware::history($remoteFetchHistory));
$remoteFetcher = new CloudTemplateRemoteFetcher(
new Client(['handler' => $remoteFetchStack]),
static fn (string $host): array => ['93.184.216.34']
);
$remoteStorageHistory = [];
$remoteStorageStack = HandlerStack::create(new MockHandler([
new Response(200, [
'ETag' => '"remote-object-etag"',
'x-amz-version-id' => 'remote-version-1',
]),
new Response(200, [
'Content-Length' => (string) strlen($remoteBody),
'ETag' => '"remote-object-etag"',
'x-amz-version-id' => 'remote-version-1',
]),
]));
$remoteStorageStack->push(Middleware::history($remoteStorageHistory));
$remoteClient = new CloudObjectStorageS3Client(new Client([
'handler' => $remoteStorageStack,
]));
$remoteInput = [
'name' => 'Debian 13 official',
'url' => 'https://images.example.test/debian-13.qcow2',
'sha256' => $remoteDigest,
];
$remoteAck = $workflow->importRemote(
$remoteInput,
$actor,
$remoteFetcher,
$remoteClient
);
$remoteStored = CloudTemplateImportIntent::query()
->where('user_id', 9)
->where(
'request_idempotency_key',
'multipart-template-intent-0008'
)->firstOrFail();
$assert((string) $remoteStored->source_kind === 'remote'
&& (string) $remoteStored->state === 'distributing'
&& (int) $remoteStored->source_size_bytes === strlen($remoteBody)
&& hash_equals(
$remoteDigest,
(string) $remoteStored->source_sha256
)
&& count($remoteFetchHistory) === 1
&& array_map(static fn (array $item): string =>
$item['request']->getMethod(), $remoteStorageHistory) === [
'PUT', 'HEAD',
],
'Official URL import did not converge through the durable intent.');
$emptyFetcher = new CloudTemplateRemoteFetcher(
new Client(['handler' => HandlerStack::create(new MockHandler([]))]),
static fn (string $host): array => ['93.184.216.34']
);
$emptyStorage = new CloudObjectStorageS3Client(new Client([
'handler' => HandlerStack::create(new MockHandler([])),
]));
$remoteReplay = $workflow->importRemote(
$remoteInput,
$actor,
$emptyFetcher,
$emptyStorage
);
$assert($remoteReplay === $remoteAck,
'Official URL replay did not return the original distribution task.');
// A failed official download remains one resumable intent. Retrying that
// intent must reuse its frozen object key and create only one task.
$actor['request_idempotency_key'] = 'multipart-template-intent-0009';
$failedFetcher = new CloudTemplateRemoteFetcher(
new Client(['handler' => HandlerStack::create(new MockHandler([
new Response(503),
]))]),
static fn (string $host): array => ['93.184.216.34']
);
try {
$workflow->importRemote(
array_replace($remoteInput, ['name' => 'Debian 13 retry']),
$actor,
$failedFetcher,
$emptyStorage
);
throw new \RuntimeException('A rejected remote download succeeded.');
} catch (CloudError $error) {
$assert($error->httpStatus === 502,
'A rejected remote download returned the wrong status.');
}
$failedRemote = CloudTemplateImportIntent::query()
->where('request_idempotency_key',
'multipart-template-intent-0009')->firstOrFail();
$failedDto = CloudTemplateImportWorkflow::dto($failedRemote);
$assert((string) $failedRemote->state === 'source_failed'
&& $failedDto['can_resume'] === true
&& $failedDto['can_cancel'] === true,
'A failed official download did not expose a safe recovery exit.');
$retryFetchHistory = [];
$retryFetchStack = HandlerStack::create(new MockHandler([
new Response(200, [
'Content-Length' => (string) strlen($remoteBody),
'Content-Encoding' => 'identity',
], $remoteBody),
]));
$retryFetchStack->push(Middleware::history($retryFetchHistory));
$retryStorageHistory = [];
$retryStorageStack = HandlerStack::create(new MockHandler([
new Response(200, [
'ETag' => '"remote-retry-etag"',
'x-amz-version-id' => 'remote-retry-version',
]),
new Response(200, [
'Content-Length' => (string) strlen($remoteBody),
'ETag' => '"remote-retry-etag"',
'x-amz-version-id' => 'remote-retry-version',
]),
]));
$retryStorageStack->push(Middleware::history($retryStorageHistory));
$retryAck = $workflow->resumeRemote(
(string) $failedRemote->id,
$actor,
new CloudTemplateRemoteFetcher(
new Client(['handler' => $retryFetchStack]),
static fn (string $host): array => ['93.184.216.34']
),
new CloudObjectStorageS3Client(new Client([
'handler' => $retryStorageStack,
]))
);
$assert((int) ($retryAck['task_id'] ?? 0) > 0
&& count($retryFetchHistory) === 1
&& array_map(static fn (array $item): string =>
$item['request']->getMethod(), $retryStorageHistory) === [
'PUT', 'HEAD',
],
'Resuming an official download did not reuse its durable intent.');
// An expired remote fetch that never crossed object publication can be
// cancelled locally; no object-storage mutation is needed.
$actor['request_idempotency_key'] = 'multipart-template-intent-0010';
try {
$workflow->importRemote(
array_replace($remoteInput, ['name' => 'Debian 13 expired remote']),
$actor,
new CloudTemplateRemoteFetcher(
new Client(['handler' => HandlerStack::create(
new MockHandler([new Response(503)])
)]),
static fn (string $host): array => ['93.184.216.34']
),
$emptyStorage
);
} catch (CloudError $error) {
// The durable source_failed intent is the expected test fixture.
}
$expiredRemote = CloudTemplateImportIntent::query()
->where('request_idempotency_key',
'multipart-template-intent-0010')->firstOrFail();
Capsule::table('cloud_template_import_intents')
->where('id', (string) $expiredRemote->id)
->update(['expires_at' => $clock - 1]);
$assert($workflow->cleanupOneExpired($emptyStorage)
&& (string) CloudTemplateImportIntent::findOrFail(
(string) $expiredRemote->id
)->state === 'cancelled',
'Expired official download was not reaped without provider mutation.');
// A lost immutable PUT response plus an unavailable HEAD is effect
// unknown. The same intent must stay fenced until a later HEAD proves the
// exact object; recovery must not download or PUT the image again.
$actor['request_idempotency_key'] = 'multipart-template-intent-0011';
$unknownFetchHistory = [];
$unknownFetchStack = HandlerStack::create(new MockHandler([
new Response(200, [
'Content-Length' => (string) strlen($remoteBody),
'Content-Encoding' => 'identity',
], $remoteBody),
]));
$unknownFetchStack->push(Middleware::history($unknownFetchHistory));
$unknownStorageHistory = [];
$unknownStorageStack = HandlerStack::create(new MockHandler([
new ConnectException('put response lost',
new Request('PUT', 'https://objects.example.test')),
new ConnectException('head unavailable',
new Request('HEAD', 'https://objects.example.test')),
]));
$unknownStorageStack->push(Middleware::history($unknownStorageHistory));
try {
$workflow->importRemote(
array_replace($remoteInput, ['name' => 'Debian 13 unknown put']),
$actor,
new CloudTemplateRemoteFetcher(
new Client(['handler' => $unknownFetchStack]),
static fn (string $host): array => ['93.184.216.34']
),
new CloudObjectStorageS3Client(new Client([
'handler' => $unknownStorageStack,
]))
);
throw new \RuntimeException('An unknown immutable PUT was accepted.');
} catch (CloudError $error) {
$assert($error->httpStatus === 502,
'Unknown immutable PUT returned the wrong status.');
}
$unknownRemote = CloudTemplateImportIntent::query()
->where('request_idempotency_key',
'multipart-template-intent-0011')->firstOrFail();
$unknownDto = CloudTemplateImportWorkflow::dto($unknownRemote);
$assert((string) $unknownRemote->state === 'source_unknown'
&& $unknownDto['can_resume'] === true
&& $unknownDto['can_cancel'] === false
&& count($unknownFetchHistory) === 1
&& array_map(static fn (array $item): string =>
$item['request']->getMethod(), $unknownStorageHistory) === [
'PUT', 'HEAD',
],
'Unknown immutable PUT was not fenced on its original object key.');
$reconcileHistory = [];
$reconcileStack = HandlerStack::create(new MockHandler([
new Response(200, [
'Content-Length' => (string) strlen($remoteBody),
'ETag' => '"remote-unknown-etag"',
'x-amz-version-id' => 'remote-unknown-version',
]),
new Response(200, [
'Content-Length' => (string) strlen($remoteBody),
'ETag' => '"remote-unknown-etag"',
'x-amz-version-id' => 'remote-unknown-version',
]),
]));
$reconcileStack->push(Middleware::history($reconcileHistory));
$unknownAck = $workflow->resumeRemote(
(string) $unknownRemote->id,
$actor,
$emptyFetcher,
new CloudObjectStorageS3Client(new Client([
'handler' => $reconcileStack,
]))
);
$assert((int) ($unknownAck['task_id'] ?? 0) > 0
&& count($unknownFetchHistory) === 1
&& array_map(static fn (array $item): string =>
$item['request']->getMethod(), $reconcileHistory) === [
'HEAD', 'HEAD',
],
'Unknown immutable PUT recovery replayed a download or write.');
// If the reconciliation HEAD proves the PUT did not apply, retrying the
// same intent is safe and may fetch and publish that frozen key again.
$actor['request_idempotency_key'] = 'multipart-template-intent-0012';
$absentFetchStack = HandlerStack::create(new MockHandler([
new Response(200, [
'Content-Length' => (string) strlen($remoteBody),
'Content-Encoding' => 'identity',
], $remoteBody),
]));
$absentStorageHistory = [];
$absentStorageStack = HandlerStack::create(new MockHandler([
new ConnectException('put response lost',
new Request('PUT', 'https://objects.example.test')),
new Response(404),
]));
$absentStorageStack->push(Middleware::history($absentStorageHistory));
try {
$workflow->importRemote(
array_replace($remoteInput, ['name' => 'Debian 13 absent put']),
$actor,
new CloudTemplateRemoteFetcher(
new Client(['handler' => $absentFetchStack]),
static fn (string $host): array => ['93.184.216.34']
),
new CloudObjectStorageS3Client(new Client([
'handler' => $absentStorageStack,
]))
);
throw new \RuntimeException('A proven absent PUT was accepted.');
} catch (CloudError $error) {
$assert($error->httpStatus === 502,
'Proven absent immutable PUT returned the wrong status.');
}
$absentRemote = CloudTemplateImportIntent::query()
->where('request_idempotency_key',
'multipart-template-intent-0012')->firstOrFail();
$assert((string) $absentRemote->state === 'source_failed'
&& array_map(static fn (array $item): string =>
$item['request']->getMethod(), $absentStorageHistory) === [
'PUT', 'HEAD',
],
'Proven absent immutable PUT was not made safely retryable.');
$absentRetryStorage = new CloudObjectStorageS3Client(new Client([
'handler' => HandlerStack::create(new MockHandler([
new Response(200, [
'ETag' => '"remote-absent-retry-etag"',
'x-amz-version-id' => 'remote-absent-retry-version',
]),
new Response(200, [
'Content-Length' => (string) strlen($remoteBody),
'ETag' => '"remote-absent-retry-etag"',
'x-amz-version-id' => 'remote-absent-retry-version',
]),
])),
]));
$absentRetryAck = $workflow->resumeRemote(
(string) $absentRemote->id,
$actor,
new CloudTemplateRemoteFetcher(
new Client(['handler' => HandlerStack::create(new MockHandler([
new Response(200, [
'Content-Length' => (string) strlen($remoteBody),
'Content-Encoding' => 'identity',
], $remoteBody),
]))]),
static fn (string $host): array => ['93.184.216.34']
),
$absentRetryStorage
);
$assert((int) ($absentRetryAck['task_id'] ?? 0) > 0,
'A proven absent immutable PUT could not resume on the same intent.');
echo "Cloud-template resumable multipart import behavior passed.\n";
}