You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
virt/web/tests/CloudStorageArtifactService...

1146 lines
47 KiB
PHP

<?php
require dirname(__DIR__) . '/vendor/autoload.php';
use App\Models\CloudStorageArtifact;
use App\Models\CloudStorageBackend;
use App\Models\CloudStorageOperation;
use App\Models\CloudStoragePublication;
use App\Models\CloudStorageReplica;
use App\Services\Cloud\CloudError;
use App\Services\Cloud\CloudStorageArtifactService;
use App\Services\Cloud\CloudStorageDurabilityPolicy;
use App\Services\Cloud\CloudStoragePublicationFinalizerService;
use App\Services\Cloud\CloudObjectStorageS3Client;
use App\Services\Cloud\CloudStorageReplicaHelper;
use App\Services\Cloud\CloudStorageReplicaAttentionService;
use App\Services\Cloud\CloudStorageReplicaWorkerService;
use GuzzleHttp\Client;
use GuzzleHttp\Handler\MockHandler;
use GuzzleHttp\HandlerStack;
use GuzzleHttp\Psr7\Response;
use Illuminate\Database\Capsule\Manager as Capsule;
use Illuminate\Database\Schema\Blueprint;
$assert = static function (bool $condition, string $message): void {
if (!$condition) {
throw new RuntimeException($message);
}
};
$expect = static function (callable $callback, int $status) use ($assert): void {
try {
$callback();
} catch (CloudError $error) {
$assert($error->httpStatus === $status,
'Unexpected CloudError status: ' . $error->httpStatus);
return;
}
throw new RuntimeException('Expected CloudError was not raised.');
};
$db = new Capsule();
$db->addConnection(['driver' => 'sqlite', 'database' => ':memory:']);
$db->setAsGlobal();
$db->bootEloquent();
$db->schema()->create('cloud_storage_backends', static function (Blueprint $t): void {
$t->increments('id');
$t->string('backend_key'); $t->string('identity_sha256');
$t->string('name'); $t->string('driver'); $t->string('endpoint');
$t->string('region'); $t->string('bucket'); $t->string('prefix');
$t->boolean('path_style'); $t->string('failure_domain');
$t->text('sealed_credentials'); $t->unsignedBigInteger('config_revision');
$t->unsignedBigInteger('tested_revision'); $t->string('admin_state');
$t->string('health_state'); $t->unsignedInteger('write_priority');
$t->unsignedInteger('consecutive_failures')->default(0);
$t->unsignedInteger('consecutive_successes')->default(0);
$t->string('last_error_code')->default('');
$t->unsignedBigInteger('last_tested_at')->default(0);
$t->unsignedBigInteger('last_healthy_at')->default(0);
$t->unsignedBigInteger('next_probe_at')->default(0);
$t->unsignedBigInteger('created_at'); $t->unsignedBigInteger('updated_at');
});
$db->schema()->create('cloud_storage_artifacts', static function (Blueprint $t): void {
$t->increments('id'); $t->string('artifact_key')->unique();
$t->string('kind'); $t->unsignedBigInteger('maximum_size_bytes');
$t->unsignedBigInteger('size_bytes');
$t->string('sha256'); $t->string('durability_policy');
$t->unsignedTinyInteger('desired_copies');
$t->unsignedTinyInteger('required_copies'); $t->string('state');
$t->string('last_operation_id')->nullable();
$t->unsignedBigInteger('verified_at');
$t->unsignedBigInteger('created_at'); $t->unsignedBigInteger('updated_at');
});
$db->schema()->create('cloud_storage_replicas', static function (Blueprint $t): void {
$t->increments('id'); $t->string('replica_key')->unique();
$t->unsignedBigInteger('artifact_id'); $t->unsignedBigInteger('backend_id');
$t->string('role'); $t->string('object_key');
$t->string('object_version_id')->default('');
$t->string('object_etag')->default('');
$t->unsignedBigInteger('size_bytes')->default(0); $t->string('sha256')->default('');
$t->string('state'); $t->string('publish_operation_id')->nullable();
$t->string('delete_operation_id')->nullable();
$t->unsignedInteger('delete_generation')->default(0);
$t->unsignedInteger('delete_cycle_attempts')->default(0);
$t->unsignedInteger('delete_presence_confirmations')->default(0);
$t->unsignedBigInteger('delete_last_checked_at')->default(0);
$t->unsignedBigInteger('delete_next_check_at')->default(0);
$t->unsignedBigInteger('delete_attention_at')->nullable();
$t->string('lease_token')->nullable(); $t->unsignedBigInteger('lease_until')->default(0);
$t->string('error_code')->default(''); $t->unsignedBigInteger('verified_at')->default(0);
$t->unsignedBigInteger('created_at'); $t->unsignedBigInteger('updated_at');
});
$db->schema()->create('cloud_storage_operations', static function (Blueprint $t): void {
$t->increments('id'); $t->string('operation_id')->unique();
$t->string('request_idempotency_key')->unique();
$t->string('request_fingerprint'); $t->unsignedBigInteger('artifact_id');
$t->unsignedBigInteger('replica_id')->nullable();
$t->unsignedBigInteger('source_backend_id')->nullable();
$t->unsignedBigInteger('target_backend_id')->nullable();
$t->string('kind'); $t->string('phase'); $t->string('outcome')->nullable();
$t->string('effect_resolution'); $t->boolean('effect_may_have_applied')->nullable();
$t->boolean('safe_to_resubmit')->nullable(); $t->unsignedInteger('attempts');
$t->string('lease_token')->nullable(); $t->unsignedBigInteger('lease_until');
$t->unsignedBigInteger('available_at'); $t->unsignedBigInteger('next_check_at');
$t->string('error_code'); $t->text('result_json')->nullable();
$t->unsignedBigInteger('terminal_at')->nullable();
$t->unsignedBigInteger('created_at'); $t->unsignedBigInteger('updated_at');
});
$db->schema()->create('cloud_storage_publications', static function (Blueprint $t): void {
$t->increments('id'); $t->string('publication_key')->unique();
$t->unsignedBigInteger('artifact_id')->unique();
$t->unsignedBigInteger('backup_id')->nullable()->unique();
$t->string('template_import_intent_id')->nullable()->unique();
$t->string('rescue_template_id')->nullable()->unique();
$t->string('state'); $t->unsignedInteger('attempts')->default(0);
$t->unsignedBigInteger('available_at')->default(0);
$t->string('lease_token')->nullable();
$t->unsignedBigInteger('lease_until')->default(0);
$t->string('last_error_code')->default('');
$t->unsignedBigInteger('coverage_ready_at')->nullable();
$t->unsignedBigInteger('finalized_at')->nullable();
$t->unsignedBigInteger('created_at'); $t->unsignedBigInteger('updated_at');
});
$now = time();
foreach ([
[1, 'provider-a', 10, 'primary'],
[2, 'provider-b', 20, 'secondary'],
] as [$id, $domain, $priority, $name]) {
Capsule::table('cloud_storage_backends')->insert([
'id' => $id, 'backend_key' => str_repeat((string) $id, 32),
'identity_sha256' => str_repeat((string) $id, 64),
'name' => $name, 'driver' => 's3',
'endpoint' => 'https://' . $name . '.example.test',
'region' => 'us-east-1', 'bucket' => $name . '-bucket',
'prefix' => $name, 'path_style' => 1,
'failure_domain' => $domain, 'sealed_credentials' => 'sealed',
'config_revision' => 1, 'tested_revision' => 1,
'admin_state' => 'active', 'health_state' => 'healthy',
'write_priority' => $priority, 'consecutive_failures' => 0,
'consecutive_successes' => 1, 'last_error_code' => '',
'last_tested_at' => $now, 'last_healthy_at' => $now,
'next_probe_at' => $now + 300, 'created_at' => $now,
'updated_at' => $now,
]);
}
$requestKey = 'backup-publication-0001';
$fingerprint = hash('sha256', 'fixture');
$view = CloudStorageArtifactService::beginPublication(
'backup', 'backups/fixture/system.qcow2', 4096,
str_repeat('a', 64), $requestKey, $fingerprint,
['backup_id' => 1]
);
$assert($view['can_publish'] && $view['backend_id'] === 1
&& CloudStorageArtifact::count() === 1
&& CloudStorageReplica::count() === 2
&& CloudStorageOperation::where('kind', 'copy')->count() === 1,
'Publication did not reserve one primary and one secondary operation.');
$again = CloudStorageArtifactService::beginPublication(
'backup', 'backups/fixture/system.qcow2', 4096,
str_repeat('a', 64), $requestKey, $fingerprint,
['backup_id' => 1]
);
$assert($again['operation_id'] === $view['operation_id']
&& CloudStorageArtifact::count() === 1,
'Idempotent publication allocated a second artifact.');
$expect(static fn () => CloudStorageArtifactService::beginPublication(
'backup', 'backups/other/system.qcow2', 4096,
str_repeat('a', 64), $requestKey, hash('sha256', 'different'),
['backup_id' => 1]
), 409);
$assert(CloudStorageArtifactService::claimReplicaWork(time()) === null,
'Secondary copy became claimable before the primary was verified.');
CloudStorageArtifactService::markPrimaryDispatching($view['operation_id']);
$expect(static fn () => CloudStorageArtifactService::markPrimaryDispatching(
$view['operation_id']
), 409);
$complete = CloudStorageArtifactService::completePrimary(
$view['operation_id'],
(int) $view['artifact_id'],
[
'size_bytes' => 4096, 'sha256' => str_repeat('a', 64),
'etag' => '"etag"', 'version_id' => '',
]
);
$artifact = CloudStorageArtifact::firstOrFail();
$primary = CloudStorageReplica::where('role', 'primary')->firstOrFail();
$copy = CloudStorageOperation::where('kind', 'copy')->firstOrFail();
$assert($complete['outcome'] === 'succeeded'
&& (string) $artifact->state === 'available'
&& (string) $primary->state === 'ready'
&& (string) $copy->phase === 'accepted',
'Primary settlement did not expose an async backup or retain copy work.');
CloudStorageArtifactService::completePrimary(
$view['operation_id'],
(int) $view['artifact_id'],
[
'size_bytes' => 4096, 'sha256' => str_repeat('a', 64),
'etag' => '"etag"', 'version_id' => '',
]);
$expect(static fn () => CloudStorageArtifactService::completePrimary(
$view['operation_id'], (int) $view['artifact_id'], [
'size_bytes' => 4096, 'sha256' => str_repeat('a', 64),
'etag' => '"different"', 'version_id' => '',
]
), 409);
$publication = CloudStoragePublication::firstOrFail();
$assert((string) $publication->state === 'ready_to_finalize'
&& (int) $publication->coverage_ready_at > 0,
'Required primary coverage did not release the durable finalizer signal.');
$publicationBeforeReject = $publication->getAttributes();
$expect(static fn () => CloudStoragePublicationFinalizerService::complete(
'backup', 999, (int) $artifact->id, $view['operation_id']
), 409);
$expect(static fn () => CloudStoragePublicationFinalizerService::complete(
'backup', 1, 999, $view['operation_id']
), 409);
$expect(static fn () => CloudStoragePublicationFinalizerService::complete(
'backup', 1, (int) $artifact->id, (string) $copy->operation_id
), 409);
$publication->refresh();
$assert($publication->getAttributes() === $publicationBeforeReject,
'Rejected publication finalization changed the durable outbox.');
$finalized = CloudStoragePublicationFinalizerService::complete(
'backup', 1, (int) $artifact->id, $view['operation_id']
);
$publication->refresh();
$finalizedAt = (int) $publication->finalized_at;
$assert($finalized['state'] === 'completed'
&& (string) $publication->state === 'completed'
&& (int) $publication->attempts === 1
&& $finalizedAt > 0,
'Owner finalization did not complete the durable publication outbox.');
$againFinalized = CloudStoragePublicationFinalizerService::complete(
'backup', 1, (int) $artifact->id, $view['operation_id']
);
$publication->refresh();
$assert($againFinalized['state'] === 'completed'
&& (int) $publication->attempts === 1
&& (int) $publication->finalized_at === $finalizedAt,
'Repeated owner finalization was not idempotent.');
$claimed = CloudStorageArtifactService::claimReplicaWork(time());
$assert(is_array($claimed) && $claimed['mode'] === 'dispatch'
&& $claimed['source_version_id'] === '',
'Versionless secondary copy was not leased from the durable operation.');
$assert(CloudStorageArtifactService::claimReplicaWork(time() + 21600) === null,
'A second worker reclaimed a copy before the helper deadline elapsed.');
$metadata = [
'upload_id' => 'upload-1', 'part_size_bytes' => 67108864,
'part_count' => 1, 'maximum_size_bytes' => 4096,
'absence_confirmations' => 0,
];
CloudStorageArtifactService::markReplicaAwaitingReceipt(
$claimed['operation_id'], $claimed['lease_token'], $metadata, time()
);
$stored = (string) CloudStorageOperation::where(
'operation_id', $claimed['operation_id']
)->value('result_json');
$assert(!str_contains($stored, 'http') && str_contains($stored, 'upload-1'),
'Durable operation persisted a capability URL or lost its upload identity.');
CloudStorageArtifactService::completeReplica(
$claimed['operation_id'], $claimed['lease_token'], [
'size_bytes' => 4096, 'sha256' => str_repeat('a', 64),
'etag' => '"secondary"', 'version_id' => '',
], time()
);
CloudStorageArtifactService::completeReplica(
$claimed['operation_id'], $claimed['lease_token'], [
'size_bytes' => 4096, 'sha256' => str_repeat('a', 64),
'etag' => '"secondary"', 'version_id' => '',
], time()
);
$expect(static fn () => CloudStorageArtifactService::completeReplica(
$claimed['operation_id'], $claimed['lease_token'], [
'size_bytes' => 4096, 'sha256' => str_repeat('a', 64),
'etag' => '"secondary"', 'version_id' => 'unexpected-version',
], time()
), 409);
$assert((string) CloudStorageArtifact::firstOrFail()->state === 'protected'
&& CloudStorageReplica::where('state', 'ready')->count() === 2,
'Versionless secondary receipt did not protect the artifact.');
$agentView = CloudStorageArtifactService::beginAgentPublication(
'backup', 'backups/agent/system.qcow2', 8192,
'backup-publication-0002', hash('sha256', 'agent'),
['backup_id' => 2]
);
CloudStorageArtifactService::markPrimaryDispatching($agentView['operation_id']);
CloudStorageArtifactService::completePrimary(
$agentView['operation_id'], (int) $agentView['artifact_id'], [
'size_bytes' => 6144, 'sha256' => str_repeat('c', 64),
'etag' => '"agent"', 'version_id' => '',
]);
$agentArtifact = CloudStorageArtifact::whereKey(
$agentView['artifact_id']
)->firstOrFail();
$assert((int) $agentArtifact->maximum_size_bytes === 8192
&& (int) $agentArtifact->size_bytes === 6144
&& (string) $agentArtifact->sha256 === str_repeat('c', 64),
'Agent publication did not freeze its bounded receipt identity.');
$capturedRequest = null;
$mock = new MockHandler([
new Response(200, [], '<InitiateMultipartUploadResult>'
. '<UploadId>worker-upload</UploadId>'
. '</InitiateMultipartUploadResult>'),
new Response(200, [
'Content-Length' => '6144',
'ETag' => '"worker"',
]),
]);
$s3 = new CloudObjectStorageS3Client(new Client([
'handler' => HandlerStack::create($mock),
]));
$helper = new CloudStorageReplicaHelper(null,
static function (array $request) use (&$capturedRequest): array {
$capturedRequest = $request;
return [
'protocol' => $request['protocol'],
'operation_id' => $request['operation_id'],
'outcome' => 'succeeded',
'effect_resolution' => 'applied',
'size_bytes' => 6144,
'sha256' => str_repeat('c', 64),
'etag' => '"worker"',
'version_id' => '',
];
}
);
$runtime = static fn (int $backendId, bool $healthy): array => [
'config' => [
'endpoint' => 'https://storage-' . $backendId . '.example.test',
'region' => 'us-east-1', 'bucket' => 'bucket-' . $backendId,
'path_style' => true, 'prefix' => '',
],
'credentials' => [
'access_key_id' => 'access-key',
'secret_access_key' => 'secret-key',
],
'revision' => 1, 'backend_id' => $backendId,
];
$worker = new CloudStorageReplicaWorkerService($helper, $s3, $runtime);
$assert($worker->tick() && is_array($capturedRequest)
&& $capturedRequest['protocol'] === 'object-replica-v1'
&& !str_contains($capturedRequest['source']['get_url'], 'versionId='),
'Replica worker did not stream a versionless exact-key source.');
$agentCopy = CloudStorageOperation::where('artifact_id', $agentArtifact->id)
->where('kind', 'copy')->firstOrFail();
$assert((string) $agentCopy->phase === 'terminal'
&& (string) $agentCopy->outcome === 'succeeded'
&& !str_contains((string) $agentCopy->result_json, 'https://'),
'Replica worker did not settle safely or persisted a capability URL.');
$unknownView = CloudStorageArtifactService::beginPublication(
'template', 'cloud-templates/unknown/system.qcow2', 4096,
str_repeat('e', 64), 'template-publication-0003',
hash('sha256', 'unknown'), [
'template_import_intent_id' => str_repeat('e', 32),
]
);
CloudStorageArtifactService::markPrimaryDispatching(
$unknownView['operation_id']
);
CloudStorageArtifactService::completePrimary(
$unknownView['operation_id'], (int) $unknownView['artifact_id'], [
'size_bytes' => 4096, 'sha256' => str_repeat('e', 64),
'etag' => '"unknown-primary"',
'version_id' => '',
]);
$copyCalls = 0;
$unknownHelper = new CloudStorageReplicaHelper(null,
static function (array $request) use (&$copyCalls): array {
$copyCalls++;
return [
'protocol' => $request['protocol'],
// A mismatched receipt must become unknown, never success.
'operation_id' => str_repeat('f', 32),
'outcome' => 'succeeded', 'effect_resolution' => 'applied',
'size_bytes' => 4096, 'sha256' => str_repeat('e', 64),
'etag' => '"wrong-operation"', 'version_id' => '',
];
}
);
$unknownHTTP = new MockHandler([
new Response(200, [], '<InitiateMultipartUploadResult>'
. '<UploadId>unknown-upload</UploadId>'
. '</InitiateMultipartUploadResult>'),
new Response(404),
new Response(200, [], '<ListMultipartUploadsResult>'
. '<IsTruncated>false</IsTruncated>'
. '</ListMultipartUploadsResult>'),
new Response(404),
new Response(200, [], '<ListMultipartUploadsResult>'
. '<IsTruncated>false</IsTruncated>'
. '</ListMultipartUploadsResult>'),
]);
$unknownWorker = new CloudStorageReplicaWorkerService(
$unknownHelper,
new CloudObjectStorageS3Client(new Client([
'handler' => HandlerStack::create($unknownHTTP),
])),
$runtime
);
$unknownWorker->tick();
$unknownCopy = CloudStorageOperation::where(
'artifact_id', $unknownView['artifact_id']
)->where('kind', 'copy')->firstOrFail();
$assert((string) $unknownCopy->phase === 'reconciling'
&& (string) $unknownCopy->effect_resolution === 'unknown',
'Mismatched helper receipt did not preserve unknown effect.');
$unknownCopy->next_check_at = 0;
$unknownCopy->save();
$unknownWorker->tick();
$unknownCopy->refresh();
$unknownMetadata = json_decode((string) $unknownCopy->result_json, true);
$assert($copyCalls === 1 && (string) $unknownCopy->phase === 'reconciling'
&& ($unknownMetadata['multipart_absence_confirmations'] ?? null) === 1,
'Unknown copy was replayed instead of read-only reconciliation.');
$unknownCopy->next_check_at = 0;
$unknownCopy->save();
$unknownWorker->tick();
$unknownCopy->refresh();
$unknownPublication = CloudStoragePublication::where(
'artifact_id', $unknownView['artifact_id']
)->firstOrFail();
$assert($copyCalls === 1 && (string) $unknownCopy->phase === 'accepted'
&& (int) $unknownCopy->attempts === 1
&& $unknownCopy->result_json === null
&& in_array(
'replica_retry_scheduled',
$unknownWorker->observationErrors(),
true
)
&& (string) $unknownPublication->state === 'waiting_protection',
'Required-two copy was not safely rescheduled after read-only absence proof: '
. json_encode([
'calls' => $copyCalls, 'phase' => $unknownCopy->phase,
'attempts' => $unknownCopy->attempts,
'result' => $unknownCopy->result_json,
'publication' => $unknownPublication->state,
'error' => $unknownCopy->error_code,
]));
$unknownCopy->available_at = 253402300799;
$unknownCopy->save();
$retryView = CloudStorageArtifactService::beginPublication(
'backup', 'backups/retry/system.qcow2', 4096,
str_repeat('f', 64), 'backup-publication-0005',
hash('sha256', 'retry'), ['backup_id' => 5]
);
CloudStorageArtifactService::markPrimaryDispatching(
$retryView['operation_id']
);
CloudStorageArtifactService::completePrimary(
$retryView['operation_id'], (int) $retryView['artifact_id'], [
'size_bytes' => 4096, 'sha256' => str_repeat('f', 64),
'etag' => '"retry-primary"',
'version_id' => '',
]);
$retryClaim = CloudStorageArtifactService::claimReplicaWork(time());
$assert(is_array($retryClaim)
&& (int) $retryClaim['artifact_id'] === (int) $retryView['artifact_id'],
'Recoverable copy fixture was not claimable.');
CloudStorageArtifactService::rescheduleReplicaNotApplied(
$retryClaim['operation_id'],
$retryClaim['lease_token'],
'replica_source_download_failed',
time(),
60
);
$retryOperation = CloudStorageOperation::where(
'operation_id', $retryClaim['operation_id']
)->firstOrFail();
$assert((string) $retryOperation->phase === 'accepted'
&& $retryOperation->outcome === null
&& $retryOperation->terminal_at === null
&& $retryOperation->result_json === null,
'Definite no-effect copy failure exhausted the only copy intent.');
$retryOperation->available_at = 253402300799;
$retryOperation->save();
$adoptView = CloudStorageArtifactService::beginPublication(
'backup', 'backups/adopt/system.qcow2', 4096,
str_repeat('1', 64), 'backup-publication-0006',
hash('sha256', 'adopt'), ['backup_id' => 6]
);
CloudStorageArtifactService::markPrimaryDispatching(
$adoptView['operation_id']
);
CloudStorageArtifactService::completePrimary(
$adoptView['operation_id'], (int) $adoptView['artifact_id'], [
'size_bytes' => 4096, 'sha256' => str_repeat('1', 64),
'etag' => '"adopt-primary"',
'version_id' => '',
]);
$adoptCopy = CloudStorageOperation::where(
'artifact_id', $adoptView['artifact_id']
)->where('kind', 'copy')->firstOrFail();
$adoptTarget = CloudStorageReplica::whereKey(
(int) $adoptCopy->replica_id
)->firstOrFail();
$adoptCalls = 0;
$adoptHelper = new CloudStorageReplicaHelper(null,
static function (array $request) use (&$adoptCalls): array {
$adoptCalls++;
return [
'protocol' => $request['protocol'],
'operation_id' => $request['operation_id'],
'outcome' => 'succeeded', 'effect_resolution' => 'applied',
'size_bytes' => 4096, 'sha256' => str_repeat('1', 64),
'etag' => '"adopted"', 'version_id' => '',
];
}
);
$adoptHTTP = new MockHandler([
// The provider may create the session even though the response is lost.
new Response(500),
new Response(404),
new Response(200, [], '<ListMultipartUploadsResult>'
. '<IsTruncated>false</IsTruncated>'
. '<Upload><Key>' . htmlspecialchars(
(string) $adoptTarget->object_key,
ENT_XML1 | ENT_QUOTES,
'UTF-8'
) . '</Key><UploadId>adopt-upload</UploadId><Initiated>'
. gmdate('Y-m-d\\TH:i:s\\Z', time())
. '</Initiated></Upload></ListMultipartUploadsResult>'),
new Response(200, [
'Content-Length' => '4096',
'ETag' => '"adopted"',
]),
]);
$adoptWorker = new CloudStorageReplicaWorkerService(
$adoptHelper,
new CloudObjectStorageS3Client(new Client([
'handler' => HandlerStack::create($adoptHTTP),
])),
$runtime
);
$adoptWorker->tick();
$adoptCopy->refresh();
$assert((string) $adoptCopy->phase === 'reconciling'
&& $adoptCopy->result_json === '[]'
&& $adoptCalls === 0,
'Ambiguous multipart creation was replayed before read-only lookup.');
$adoptCopy->next_check_at = 0;
$adoptCopy->save();
$adoptWorker->tick();
$adoptCopy->refresh();
$assert($adoptCalls === 1
&& (string) $adoptCopy->phase === 'terminal'
&& (string) $adoptCopy->outcome === 'succeeded',
'One exact multipart session was not adopted exactly once.');
// Crash after the upload_id is durable but before helper startup. The same
// open session is safe to resume only after two stable read-only observations;
// no new multipart session or Abort is allowed.
$resumeView = CloudStorageArtifactService::beginPublication(
'backup', 'backups/resume/system.qcow2', 4096,
str_repeat('2', 64), 'backup-publication-0007',
hash('sha256', 'resume'), ['backup_id' => 7]
);
CloudStorageArtifactService::markPrimaryDispatching(
$resumeView['operation_id']
);
CloudStorageArtifactService::completePrimary(
$resumeView['operation_id'], (int) $resumeView['artifact_id'], [
'size_bytes' => 4096, 'sha256' => str_repeat('2', 64),
'etag' => '"resume-primary"', 'version_id' => '',
]);
$resumeWork = CloudStorageArtifactService::claimReplicaWork(time());
$assert(is_array($resumeWork), 'Resume crash fixture was not claimable.');
CloudStorageArtifactService::markReplicaAwaitingReceipt(
$resumeWork['operation_id'],
$resumeWork['lease_token'],
[
'upload_id' => 'resume-upload',
'part_size_bytes' => 67108864,
'part_count' => 1,
'maximum_size_bytes' => 4096,
'multipart_absence_confirmations' => 0,
'multipart_presence_confirmations' => 0,
],
time()
);
$resumeOperation = CloudStorageOperation::where(
'operation_id', $resumeWork['operation_id']
)->firstOrFail();
$resumeOperation->lease_until = 0;
$resumeOperation->save();
$resumeTarget = CloudStorageReplica::whereKey(
(int) $resumeOperation->replica_id
)->firstOrFail();
$resumeTarget->lease_until = 0;
$resumeTarget->save();
$resumeXML = '<ListMultipartUploadsResult><IsTruncated>false</IsTruncated>'
. '<Upload><Key>' . htmlspecialchars(
(string) $resumeTarget->object_key,
ENT_XML1 | ENT_QUOTES,
'UTF-8'
) . '</Key><UploadId>resume-upload</UploadId><Initiated>'
. gmdate('Y-m-d\\TH:i:s\\Z', time())
. '</Initiated></Upload></ListMultipartUploadsResult>';
$resumeHTTP = new MockHandler([
new Response(404), new Response(200, [], $resumeXML),
new Response(404), new Response(200, [], $resumeXML),
new Response(200, [
'Content-Length' => '4096', 'ETag' => '"resumed"',
]),
]);
$resumeCalls = 0;
$resumeHelper = new CloudStorageReplicaHelper(null,
static function (array $request) use (&$resumeCalls): array {
$resumeCalls++;
return [
'protocol' => $request['protocol'],
'operation_id' => $request['operation_id'],
'outcome' => 'succeeded', 'effect_resolution' => 'applied',
'size_bytes' => 4096, 'sha256' => str_repeat('2', 64),
'etag' => '"resumed"', 'version_id' => '',
];
}
);
$resumeWorker = new CloudStorageReplicaWorkerService(
$resumeHelper,
new CloudObjectStorageS3Client(new Client([
'handler' => HandlerStack::create($resumeHTTP),
])),
$runtime
);
$resumeWorker->tick();
$resumeOperation->refresh();
$resumeEvidence = json_decode((string) $resumeOperation->result_json, true);
$assert($resumeCalls === 0
&& ($resumeEvidence['multipart_presence_confirmations'] ?? 0) === 1,
'A persisted multipart session resumed without two stable observations.');
$resumeOperation->next_check_at = 0;
$resumeOperation->save();
$resumeWorker->tick();
$resumeOperation->refresh();
$assert($resumeCalls === 1
&& (string) $resumeOperation->phase === 'terminal'
&& (string) $resumeOperation->outcome === 'succeeded',
'A stable persisted multipart session did not resume exactly once.');
// A successful helper receipt is not authoritative until target HEAD agrees
// on both ETag and version identity.
$mixedView = CloudStorageArtifactService::beginPublication(
'backup', 'backups/mixed-version/system.qcow2', 4096,
str_repeat('3', 64), 'backup-publication-0008',
hash('sha256', 'mixed-version'), ['backup_id' => 8]
);
CloudStorageArtifactService::markPrimaryDispatching(
$mixedView['operation_id']
);
CloudStorageArtifactService::completePrimary(
$mixedView['operation_id'], (int) $mixedView['artifact_id'], [
'size_bytes' => 4096, 'sha256' => str_repeat('3', 64),
'etag' => '"mixed-primary"', 'version_id' => '',
]);
$mixedHelper = new CloudStorageReplicaHelper(null,
static fn (array $request): array => [
'protocol' => $request['protocol'],
'operation_id' => $request['operation_id'],
'outcome' => 'succeeded', 'effect_resolution' => 'applied',
'size_bytes' => 4096, 'sha256' => str_repeat('3', 64),
'etag' => '"mixed"', 'version_id' => '',
]
);
$mixedHTTP = new MockHandler([
new Response(200, [], '<InitiateMultipartUploadResult>'
. '<UploadId>mixed-upload</UploadId>'
. '</InitiateMultipartUploadResult>'),
new Response(200, [
'Content-Length' => '4096', 'ETag' => '"mixed"',
'x-amz-version-id' => 'versioned-object',
]),
]);
$mixedWorker = new CloudStorageReplicaWorkerService(
$mixedHelper,
new CloudObjectStorageS3Client(new Client([
'handler' => HandlerStack::create($mixedHTTP),
])),
$runtime
);
$mixedWorker->tick();
$mixedCopy = CloudStorageOperation::where(
'artifact_id', $mixedView['artifact_id']
)->where('kind', 'copy')->firstOrFail();
$assert((string) $mixedCopy->phase === 'terminal'
&& (string) $mixedCopy->outcome === 'failed'
&& (string) $mixedCopy->error_code === 'replica_identity_changed',
'Mixed versioned/versionless completion was accepted.');
$assert(in_array(
'replica_identity_changed',
$mixedWorker->observationErrors(),
true
), 'A terminal identity mismatch was hidden from worker health.');
CloudStorageArtifactService::failReplicaAppliedMismatch(
(string) $mixedCopy->operation_id,
str_repeat('0', 32),
'replica_identity_changed',
time()
);
$expect(static fn () =>
CloudStorageArtifactService::failReplicaAppliedMismatch(
(string) $mixedCopy->operation_id,
str_repeat('0', 32),
'replica_identity_changed_again',
time()
), 409);
$expect(static fn () => CloudStorageArtifactService::failReplicaNotApplied(
(string) $mixedCopy->operation_id,
str_repeat('0', 32),
'replica_identity_changed',
time()
), 409);
// Stable multipart identity ambiguity must stop automatic polling and expose
// one administrator recovery entry for the original operation only.
$attentionView = CloudStorageArtifactService::beginPublication(
'template', 'cloud-templates/attention/system.qcow2', 4096,
str_repeat('4', 64), 'template-publication-attention',
hash('sha256', 'attention'), [
'template_import_intent_id' => str_repeat('4', 32),
]
);
CloudStorageArtifactService::markPrimaryDispatching(
$attentionView['operation_id']
);
CloudStorageArtifactService::completePrimary(
$attentionView['operation_id'], (int) $attentionView['artifact_id'], [
'size_bytes' => 4096, 'sha256' => str_repeat('4', 64),
'etag' => '"attention-primary"', 'version_id' => '',
]);
$attentionCopy = CloudStorageOperation::where(
'artifact_id', $attentionView['artifact_id']
)->where('kind', 'copy')->firstOrFail();
$attentionTarget = CloudStorageReplica::whereKey(
(int) $attentionCopy->replica_id
)->firstOrFail();
$attentionKey = htmlspecialchars(
(string) $attentionTarget->object_key,
ENT_XML1 | ENT_QUOTES,
'UTF-8'
);
$attentionSessions = '<ListMultipartUploadsResult>'
. '<IsTruncated>false</IsTruncated>'
. '<Upload><Key>' . $attentionKey . '</Key><UploadId>attention-a</UploadId>'
. '<Initiated>' . gmdate('Y-m-d\\TH:i:s\\Z', time())
. '</Initiated></Upload>'
. '<Upload><Key>' . $attentionKey . '</Key><UploadId>attention-b</UploadId>'
. '<Initiated>' . gmdate('Y-m-d\\TH:i:s\\Z', time())
. '</Initiated></Upload></ListMultipartUploadsResult>';
$attentionHTTP = new MockHandler([
new Response(500),
new Response(404), new Response(200, [], $attentionSessions),
new Response(404), new Response(200, [], $attentionSessions),
]);
$attentionHelperCalls = 0;
$attentionWorker = new CloudStorageReplicaWorkerService(
new CloudStorageReplicaHelper(null,
static function (array $request) use (&$attentionHelperCalls): array {
$attentionHelperCalls++;
throw new RuntimeException('Ambiguous upload must not start helper');
}
),
new CloudObjectStorageS3Client(new Client([
'handler' => HandlerStack::create($attentionHTTP),
])),
$runtime
);
$attentionWorker->tick();
$attentionCopy->refresh();
$attentionCopy->next_check_at = 0;
$attentionCopy->save();
$attentionWorker->tick();
$attentionCopy->refresh();
$firstAttentionEvidence = json_decode(
(string) $attentionCopy->result_json,
true
);
$assert((string) $attentionCopy->phase === 'reconciling'
&& ($firstAttentionEvidence['multipart_attention_confirmations'] ?? 0)
=== 1,
'First multipart ambiguity observation did not remain reconciling.');
$attentionCopy->next_check_at = 0;
$attentionCopy->save();
$attentionWorker->tick();
$attentionCopy->refresh();
$attentionEvidence = json_decode((string) $attentionCopy->result_json, true);
$attentionList = CloudStorageReplicaAttentionService::listView();
$operationCountBeforeAttentionRetry = CloudStorageOperation::count();
$assert($attentionHelperCalls === 0
&& (string) $attentionCopy->phase === 'terminal'
&& (string) $attentionCopy->outcome === 'failed'
&& (string) $attentionCopy->effect_resolution === 'unknown'
&& (bool) $attentionCopy->effect_may_have_applied
&& !(bool) $attentionCopy->safe_to_resubmit
&& ($attentionEvidence['multipart_attention_confirmations'] ?? 0) === 2
&& ($attentionList['summary']['count'] ?? 0) === 1
&& !str_contains(json_encode($attentionList), $attentionKey)
&& !str_contains(json_encode($attentionList), 'attention-a')
&& !str_contains(json_encode($attentionList), 'attention-b'),
'Stable multipart ambiguity did not converge to administrator attention.');
$attentionTarget->refresh();
$attentionTarget->state = 'ready';
$attentionTarget->size_bytes = 4096;
$attentionTarget->sha256 = str_repeat('4', 64);
$attentionTarget->verified_at = time();
$attentionTarget->error_code = '';
$attentionTarget->save();
$staleAttentionList = CloudStorageReplicaAttentionService::listView();
$assert(($staleAttentionList['items'][0]['can_retry'] ?? true) === false,
'A stale attention operation remained retryable after target repair.');
$expect(static fn () => CloudStorageReplicaAttentionService::retry(
(string) $attentionCopy->operation_id
), 409);
$attentionTarget->state = 'degraded';
$attentionTarget->error_code = 'replica_multipart_identity_ambiguous';
$attentionTarget->save();
$attentionArtifact = CloudStorageArtifact::whereKey(
(int) $attentionView['artifact_id']
)->firstOrFail();
$attentionArtifact->state = 'delete_attention';
$attentionArtifact->save();
$deleteAttentionList = CloudStorageReplicaAttentionService::listView();
$deleteAttentionItem = $deleteAttentionList['items'][0] ?? [];
$assert(($deleteAttentionItem['operation_id'] ?? '')
=== (string) $attentionCopy->operation_id
&& ($deleteAttentionItem['can_retry'] ?? false) === true,
'Delete-attention copy did not expose read-only reconciliation.');
CloudStorageReplicaAttentionService::retry(
(string) $attentionCopy->operation_id
);
$attentionCopy->refresh();
$attentionArtifact->refresh();
$reopenedEvidence = json_decode((string) $attentionCopy->result_json, true);
$assert((string) $attentionCopy->phase === 'reconciling'
&& (string) $attentionCopy->error_code === 'admin_reconcile_requested'
&& ($reopenedEvidence['multipart_attention_confirmations'] ?? -1) === 0
&& (string) $attentionArtifact->state === 'delete_attention'
&& CloudStorageOperation::count() === $operationCountBeforeAttentionRetry
&& $attentionHelperCalls === 0,
'Delete-attention retry replaced or dispatched the original copy intent.');
$attentionCopy->next_check_at = 253402300799;
$attentionCopy->save();
// A definite no-effect failure is idempotent only for the exact same terminal
// identity; conflicting late receipts must be rejected.
$failedCopyView = CloudStorageArtifactService::beginPublication(
'backup', 'backups/failed-copy/system.qcow2', 4096,
str_repeat('5', 64), 'backup-publication-failed-copy',
hash('sha256', 'failed-copy'), ['backup_id' => 9]
);
CloudStorageArtifactService::markPrimaryDispatching(
$failedCopyView['operation_id']
);
CloudStorageArtifactService::completePrimary(
$failedCopyView['operation_id'], (int) $failedCopyView['artifact_id'], [
'size_bytes' => 4096, 'sha256' => str_repeat('5', 64),
'etag' => '"failed-copy-primary"', 'version_id' => '',
]);
$failedCopyClaim = CloudStorageArtifactService::claimReplicaWork(time());
$assert(is_array($failedCopyClaim), 'Failed-copy fixture was not claimable.');
CloudStorageArtifactService::failReplicaNotApplied(
$failedCopyClaim['operation_id'],
$failedCopyClaim['lease_token'],
'replica_copy_rejected',
time()
);
CloudStorageArtifactService::failReplicaNotApplied(
$failedCopyClaim['operation_id'],
$failedCopyClaim['lease_token'],
'replica_copy_rejected',
time()
);
$expect(static fn () => CloudStorageArtifactService::failReplicaNotApplied(
$failedCopyClaim['operation_id'],
$failedCopyClaim['lease_token'],
'replica_copy_rejected_again',
time()
), 409);
$pendingView = CloudStorageArtifactService::beginAgentPublication(
'backup', 'backups/pending/system.qcow2', 8192,
'backup-publication-0004', hash('sha256', 'pending'),
['backup_id' => 4]
);
CloudStorageArtifactService::markPrimaryDispatching(
$pendingView['operation_id']
);
CloudStorageArtifactService::failPrimary(
$pendingView['operation_id'],
(int) $pendingView['artifact_id'],
'not_applied',
'primary_upload_rejected'
);
CloudStorageArtifactService::failPrimary(
$pendingView['operation_id'],
(int) $pendingView['artifact_id'],
'not_applied',
'primary_upload_rejected'
);
$expect(static fn () => CloudStorageArtifactService::failPrimary(
$pendingView['operation_id'],
(int) $pendingView['artifact_id'],
'not_applied',
'primary_upload_rejected_again'
), 409);
$pendingArtifact = CloudStorageArtifact::whereKey(
$pendingView['artifact_id']
)->firstOrFail();
$pendingPublication = CloudStoragePublication::where(
'artifact_id', $pendingArtifact->id
)->firstOrFail();
$pendingCopy = CloudStorageOperation::where(
'artifact_id', $pendingArtifact->id
)->where('kind', 'copy')->firstOrFail();
$assert((string) $pendingArtifact->state === 'publishing'
&& (int) $pendingArtifact->size_bytes === 0
&& (string) $pendingArtifact->sha256 === ''
&& (string) $pendingPublication->state === 'failed'
&& (string) $pendingCopy->phase === 'terminal'
&& (string) $pendingCopy->effect_resolution === 'not_applied',
'Failed pending publication violated the blank-identity state contract.');
// Copy settlement must never overwrite a deletion projection or deletion
// worker lease, even if an older accepted copy operation remains claimable.
$deletedView = CloudStorageArtifactService::beginPublication(
'backup', 'backups/deleted-copy/system.qcow2', 4096,
str_repeat('6', 64), 'backup-publication-deleted-copy',
hash('sha256', 'deleted-copy'), ['backup_id' => 10]
);
CloudStorageArtifactService::markPrimaryDispatching(
$deletedView['operation_id']
);
CloudStorageArtifactService::completePrimary(
$deletedView['operation_id'], (int) $deletedView['artifact_id'], [
'size_bytes' => 4096, 'sha256' => str_repeat('6', 64),
'etag' => '"deleted-copy-primary"', 'version_id' => '',
]);
$deletedArtifact = CloudStorageArtifact::whereKey(
$deletedView['artifact_id']
)->firstOrFail();
$deletedCopy = CloudStorageOperation::where(
'artifact_id', $deletedArtifact->id
)->where('kind', 'copy')->firstOrFail();
$deletedReplica = CloudStorageReplica::whereKey(
(int) $deletedCopy->replica_id
)->firstOrFail();
$deletedArtifact->state = 'deleted';
$deletedArtifact->save();
$deletedReplica->state = 'deleted';
$deletedReplica->lease_token = str_repeat('d', 32);
$deletedReplica->lease_until = time() + 3600;
$deletedReplica->error_code = 'delete_projection_sentinel';
$deletedReplica->save();
$assert(CloudStorageArtifactService::claimReplicaWork(time()) === null,
'Deleted artifact unexpectedly returned copy work.');
$deletedCopy->refresh();
$deletedReplica->refresh();
$deletedArtifact->refresh();
$assert((string) $deletedCopy->phase === 'terminal'
&& (string) $deletedCopy->effect_resolution === 'not_applied'
&& (string) $deletedArtifact->state === 'deleted'
&& (string) $deletedReplica->state === 'deleted'
&& (string) $deletedReplica->lease_token === str_repeat('d', 32)
&& (string) $deletedReplica->error_code
=== 'delete_projection_sentinel',
'Copy fence overwrote an existing deletion projection.');
// A receipt must be bound to the caller's frozen artifact before any row is
// changed, even when two artifacts intentionally share the same bytes.
$crossA = CloudStorageArtifactService::beginPublication(
'backup', 'backups/cross-a/system.qcow2', 4096,
str_repeat('7', 64), 'backup-publication-cross-a',
hash('sha256', 'cross-a'), ['backup_id' => 110]
);
$crossB = CloudStorageArtifactService::beginPublication(
'backup', 'backups/cross-b/system.qcow2', 4096,
str_repeat('7', 64), 'backup-publication-cross-b',
hash('sha256', 'cross-b'), ['backup_id' => 111]
);
CloudStorageArtifactService::markPrimaryDispatching(
$crossA['operation_id']
);
CloudStorageArtifactService::markPrimaryDispatching(
$crossB['operation_id']
);
$expect(static fn () => CloudStorageArtifactService::completePrimary(
$crossA['operation_id'], (int) $crossB['artifact_id'], [
'size_bytes' => 4096, 'sha256' => str_repeat('7', 64),
'etag' => '"cross-a"', 'version_id' => '',
]
), 409);
$crossAOperation = CloudStorageOperation::where(
'operation_id', $crossA['operation_id']
)->firstOrFail();
$crossAPrimary = CloudStorageReplica::whereKey(
(int) $crossAOperation->replica_id
)->firstOrFail();
$crossAArtifact = CloudStorageArtifact::whereKey(
(int) $crossA['artifact_id']
)->firstOrFail();
$crossBOperation = CloudStorageOperation::where(
'operation_id', $crossB['operation_id']
)->firstOrFail();
$crossBPrimary = CloudStorageReplica::whereKey(
(int) $crossBOperation->replica_id
)->firstOrFail();
$crossBArtifact = CloudStorageArtifact::whereKey(
(int) $crossB['artifact_id']
)->firstOrFail();
$assert((string) $crossAOperation->phase === 'dispatching'
&& $crossAOperation->outcome === null
&& (string) $crossAPrimary->state === 'publishing'
&& (int) $crossAPrimary->size_bytes === 0
&& (string) $crossAArtifact->state === 'publishing'
&& (string) $crossBOperation->phase === 'dispatching'
&& $crossBOperation->outcome === null
&& (string) $crossBPrimary->state === 'publishing'
&& (int) $crossBPrimary->size_bytes === 0
&& (string) $crossBArtifact->state === 'publishing',
'Cross-artifact primary receipt changed one of the two ledgers.');
// A required-two artifact must cross the initial publication gate with both
// replicas. Once the owner has finalized it, later loss of one replica is a
// degraded-but-readable repair condition and must not reopen the outbox.
$runtimeReadView = CloudStorageArtifactService::beginPublication(
'template',
'cloud-templates/runtime-read/system.qcow2',
4096,
str_repeat('6', 64),
'template-runtime-read-policy',
hash('sha256', 'template-runtime-read-policy'),
['template_import_intent_id' => str_repeat('6', 32)]
);
CloudStorageArtifactService::markPrimaryDispatching(
$runtimeReadView['operation_id']
);
CloudStorageArtifactService::completePrimary(
$runtimeReadView['operation_id'],
(int) $runtimeReadView['artifact_id'],
[
'size_bytes' => 4096,
'sha256' => str_repeat('6', 64),
'etag' => '"runtime-primary"',
'version_id' => '',
]
);
$runtimePublication = CloudStoragePublication::where(
'artifact_id', (int) $runtimeReadView['artifact_id']
)->firstOrFail();
$assert((string) $runtimePublication->state === 'waiting_protection',
'One template replica prematurely released the required-two publication.');
$runtimeArtifact = CloudStorageArtifact::whereKey(
(int) $runtimeReadView['artifact_id']
)->firstOrFail();
$runtimeReplicas = CloudStorageReplica::where(
'artifact_id', (int) $runtimeArtifact->id
)->orderBy('id')->get();
$runtimeSecondary = $runtimeReplicas->firstWhere('role', 'secondary');
$assert($runtimeSecondary instanceof CloudStorageReplica,
'Required-two fixture is missing its secondary replica.');
$runtimeSecondary->state = 'ready';
$runtimeSecondary->size_bytes = 4096;
$runtimeSecondary->sha256 = str_repeat('6', 64);
$runtimeSecondary->object_etag = '"runtime-secondary"';
$runtimeSecondary->verified_at = time();
$runtimeSecondary->error_code = '';
$runtimeSecondary->save();
$project = (new ReflectionClass(CloudStorageArtifactService::class))
->getMethod('projectArtifactLocked');
$project->invoke(
null,
$runtimeArtifact,
$runtimeReplicas->all(),
time()
);
$runtimePublication->refresh();
$assert((string) $runtimePublication->state === 'ready_to_finalize',
'Two verified template replicas did not release publication.');
CloudStoragePublicationFinalizerService::complete(
'template',
str_repeat('6', 32),
(int) $runtimeArtifact->id,
(string) $runtimeReadView['operation_id']
);
$runtimeSecondary->state = 'degraded';
$runtimeSecondary->verified_at = 0;
$runtimeSecondary->error_code = 'replica_unavailable';
$runtimeSecondary->save();
$runtimeArtifact->refresh();
$runtimeReplicas = CloudStorageReplica::where(
'artifact_id', (int) $runtimeArtifact->id
)->orderBy('id')->get();
$project->invoke(
null,
$runtimeArtifact,
$runtimeReplicas->all(),
time()
);
$runtimeArtifact->refresh();
$runtimePublication->refresh();
$runtimeProjection = CloudStorageDurabilityPolicy::project(
[
'state' => (string) $runtimeArtifact->state,
'size_bytes' => (int) $runtimeArtifact->size_bytes,
'sha256' => (string) $runtimeArtifact->sha256,
'desired_copies' => (int) $runtimeArtifact->desired_copies,
'required_copies' => (int) $runtimeArtifact->required_copies,
],
$runtimeReplicas->map(static fn (CloudStorageReplica $replica): array => [
'backend_id' => (int) $replica->backend_id,
'state' => (string) $replica->state,
'size_bytes' => (int) $replica->size_bytes,
'sha256' => (string) $replica->sha256,
'verified_at' => (int) $replica->verified_at,
])->values()->all()
);
$assert((string) $runtimeArtifact->state === 'degraded'
&& $runtimeProjection['can_read']
&& !$runtimeProjection['meets_publication_requirement']
&& (string) $runtimePublication->state === 'completed'
&& (int) $runtimePublication->finalized_at > 0,
'Replica degradation made a published template unreadable or reopened its outbox.');
CloudStorageBackend::whereKey(2)->update([
'admin_state' => 'disabled', 'health_state' => 'unreachable',
'tested_revision' => 0,
]);
$artifactCountBeforeRejectedPublication = CloudStorageArtifact::count();
$expect(static fn () => CloudStorageArtifactService::beginPublication(
'template', 'cloud-templates/t/g/system.qcow2', 8192,
str_repeat('b', 64), 'template-publication-0001', hash('sha256', 't'),
['template_import_intent_id' => str_repeat('d', 32)]
), 503);
$assert(CloudStorageArtifact::count()
=== $artifactCountBeforeRejectedPublication,
'Required-two preflight left a partial template artifact.');
echo "Storage artifact service tests passed.\n";