|
|
|
|
<?php
|
|
|
|
|
|
|
|
|
|
require dirname(__DIR__) . '/vendor/autoload.php';
|
|
|
|
|
|
|
|
|
|
use App\Models\CloudStorageArtifact;
|
|
|
|
|
use App\Models\CloudStorageBackend;
|
|
|
|
|
use App\Models\CloudStorageOperation;
|
|
|
|
|
use App\Models\CloudStoragePublication;
|
|
|
|
|
use App\Models\CloudStorageReplica;
|
|
|
|
|
use App\Services\Cloud\CloudError;
|
|
|
|
|
use App\Services\Cloud\CloudStorageArtifactService;
|
|
|
|
|
use App\Services\Cloud\CloudStorageDurabilityPolicy;
|
|
|
|
|
use App\Services\Cloud\CloudStoragePublicationFinalizerService;
|
|
|
|
|
use App\Services\Cloud\CloudObjectStorageS3Client;
|
|
|
|
|
use App\Services\Cloud\CloudStorageReplicaHelper;
|
|
|
|
|
use App\Services\Cloud\CloudStorageReplicaAttentionService;
|
|
|
|
|
use App\Services\Cloud\CloudStorageReplicaWorkerService;
|
|
|
|
|
use GuzzleHttp\Client;
|
|
|
|
|
use GuzzleHttp\Handler\MockHandler;
|
|
|
|
|
use GuzzleHttp\HandlerStack;
|
|
|
|
|
use GuzzleHttp\Psr7\Response;
|
|
|
|
|
use Illuminate\Database\Capsule\Manager as Capsule;
|
|
|
|
|
use Illuminate\Database\Schema\Blueprint;
|
|
|
|
|
|
|
|
|
|
$assert = static function (bool $condition, string $message): void {
|
|
|
|
|
if (!$condition) {
|
|
|
|
|
throw new RuntimeException($message);
|
|
|
|
|
}
|
|
|
|
|
};
|
|
|
|
|
$expect = static function (callable $callback, int $status) use ($assert): void {
|
|
|
|
|
try {
|
|
|
|
|
$callback();
|
|
|
|
|
} catch (CloudError $error) {
|
|
|
|
|
$assert($error->httpStatus === $status,
|
|
|
|
|
'Unexpected CloudError status: ' . $error->httpStatus);
|
|
|
|
|
return;
|
|
|
|
|
}
|
|
|
|
|
throw new RuntimeException('Expected CloudError was not raised.');
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
$db = new Capsule();
|
|
|
|
|
$db->addConnection(['driver' => 'sqlite', 'database' => ':memory:']);
|
|
|
|
|
$db->setAsGlobal();
|
|
|
|
|
$db->bootEloquent();
|
|
|
|
|
$db->schema()->create('cloud_storage_backends', static function (Blueprint $t): void {
|
|
|
|
|
$t->increments('id');
|
|
|
|
|
$t->string('backend_key'); $t->string('identity_sha256');
|
|
|
|
|
$t->string('name'); $t->string('driver'); $t->string('endpoint');
|
|
|
|
|
$t->string('region'); $t->string('bucket'); $t->string('prefix');
|
|
|
|
|
$t->boolean('path_style'); $t->string('failure_domain');
|
|
|
|
|
$t->text('sealed_credentials'); $t->unsignedBigInteger('config_revision');
|
|
|
|
|
$t->unsignedBigInteger('tested_revision'); $t->string('admin_state');
|
|
|
|
|
$t->string('health_state'); $t->unsignedInteger('write_priority');
|
|
|
|
|
$t->unsignedInteger('consecutive_failures')->default(0);
|
|
|
|
|
$t->unsignedInteger('consecutive_successes')->default(0);
|
|
|
|
|
$t->string('last_error_code')->default('');
|
|
|
|
|
$t->unsignedBigInteger('last_tested_at')->default(0);
|
|
|
|
|
$t->unsignedBigInteger('last_healthy_at')->default(0);
|
|
|
|
|
$t->unsignedBigInteger('next_probe_at')->default(0);
|
|
|
|
|
$t->unsignedBigInteger('created_at'); $t->unsignedBigInteger('updated_at');
|
|
|
|
|
});
|
|
|
|
|
$db->schema()->create('cloud_storage_artifacts', static function (Blueprint $t): void {
|
|
|
|
|
$t->increments('id'); $t->string('artifact_key')->unique();
|
|
|
|
|
$t->string('kind'); $t->unsignedBigInteger('maximum_size_bytes');
|
|
|
|
|
$t->unsignedBigInteger('size_bytes');
|
|
|
|
|
$t->string('sha256'); $t->string('durability_policy');
|
|
|
|
|
$t->unsignedTinyInteger('desired_copies');
|
|
|
|
|
$t->unsignedTinyInteger('required_copies'); $t->string('state');
|
|
|
|
|
$t->string('last_operation_id')->nullable();
|
|
|
|
|
$t->unsignedBigInteger('verified_at');
|
|
|
|
|
$t->unsignedBigInteger('created_at'); $t->unsignedBigInteger('updated_at');
|
|
|
|
|
});
|
|
|
|
|
$db->schema()->create('cloud_storage_replicas', static function (Blueprint $t): void {
|
|
|
|
|
$t->increments('id'); $t->string('replica_key')->unique();
|
|
|
|
|
$t->unsignedBigInteger('artifact_id'); $t->unsignedBigInteger('backend_id');
|
|
|
|
|
$t->string('role'); $t->string('object_key');
|
|
|
|
|
$t->string('object_version_id')->default('');
|
|
|
|
|
$t->string('object_etag')->default('');
|
|
|
|
|
$t->unsignedBigInteger('size_bytes')->default(0); $t->string('sha256')->default('');
|
|
|
|
|
$t->string('state'); $t->string('publish_operation_id')->nullable();
|
|
|
|
|
$t->string('delete_operation_id')->nullable();
|
|
|
|
|
$t->unsignedInteger('delete_generation')->default(0);
|
|
|
|
|
$t->unsignedInteger('delete_cycle_attempts')->default(0);
|
|
|
|
|
$t->unsignedInteger('delete_presence_confirmations')->default(0);
|
|
|
|
|
$t->unsignedBigInteger('delete_last_checked_at')->default(0);
|
|
|
|
|
$t->unsignedBigInteger('delete_next_check_at')->default(0);
|
|
|
|
|
$t->unsignedBigInteger('delete_attention_at')->nullable();
|
|
|
|
|
$t->string('lease_token')->nullable(); $t->unsignedBigInteger('lease_until')->default(0);
|
|
|
|
|
$t->string('error_code')->default(''); $t->unsignedBigInteger('verified_at')->default(0);
|
|
|
|
|
$t->unsignedBigInteger('created_at'); $t->unsignedBigInteger('updated_at');
|
|
|
|
|
});
|
|
|
|
|
$db->schema()->create('cloud_storage_operations', static function (Blueprint $t): void {
|
|
|
|
|
$t->increments('id'); $t->string('operation_id')->unique();
|
|
|
|
|
$t->string('request_idempotency_key')->unique();
|
|
|
|
|
$t->string('request_fingerprint'); $t->unsignedBigInteger('artifact_id');
|
|
|
|
|
$t->unsignedBigInteger('replica_id')->nullable();
|
|
|
|
|
$t->unsignedBigInteger('source_backend_id')->nullable();
|
|
|
|
|
$t->unsignedBigInteger('target_backend_id')->nullable();
|
|
|
|
|
$t->string('kind'); $t->string('phase'); $t->string('outcome')->nullable();
|
|
|
|
|
$t->string('effect_resolution'); $t->boolean('effect_may_have_applied')->nullable();
|
|
|
|
|
$t->boolean('safe_to_resubmit')->nullable(); $t->unsignedInteger('attempts');
|
|
|
|
|
$t->string('lease_token')->nullable(); $t->unsignedBigInteger('lease_until');
|
|
|
|
|
$t->unsignedBigInteger('available_at'); $t->unsignedBigInteger('next_check_at');
|
|
|
|
|
$t->string('error_code'); $t->text('result_json')->nullable();
|
|
|
|
|
$t->unsignedBigInteger('terminal_at')->nullable();
|
|
|
|
|
$t->unsignedBigInteger('created_at'); $t->unsignedBigInteger('updated_at');
|
|
|
|
|
});
|
|
|
|
|
$db->schema()->create('cloud_storage_publications', static function (Blueprint $t): void {
|
|
|
|
|
$t->increments('id'); $t->string('publication_key')->unique();
|
|
|
|
|
$t->unsignedBigInteger('artifact_id')->unique();
|
|
|
|
|
$t->unsignedBigInteger('backup_id')->nullable()->unique();
|
|
|
|
|
$t->string('template_import_intent_id')->nullable()->unique();
|
|
|
|
|
$t->string('rescue_template_id')->nullable()->unique();
|
|
|
|
|
$t->string('state'); $t->unsignedInteger('attempts')->default(0);
|
|
|
|
|
$t->unsignedBigInteger('available_at')->default(0);
|
|
|
|
|
$t->string('lease_token')->nullable();
|
|
|
|
|
$t->unsignedBigInteger('lease_until')->default(0);
|
|
|
|
|
$t->string('last_error_code')->default('');
|
|
|
|
|
$t->unsignedBigInteger('coverage_ready_at')->nullable();
|
|
|
|
|
$t->unsignedBigInteger('finalized_at')->nullable();
|
|
|
|
|
$t->unsignedBigInteger('created_at'); $t->unsignedBigInteger('updated_at');
|
|
|
|
|
});
|
|
|
|
|
|
|
|
|
|
$now = time();
|
|
|
|
|
foreach ([
|
|
|
|
|
[1, 'provider-a', 10, 'primary'],
|
|
|
|
|
[2, 'provider-b', 20, 'secondary'],
|
|
|
|
|
] as [$id, $domain, $priority, $name]) {
|
|
|
|
|
Capsule::table('cloud_storage_backends')->insert([
|
|
|
|
|
'id' => $id, 'backend_key' => str_repeat((string) $id, 32),
|
|
|
|
|
'identity_sha256' => str_repeat((string) $id, 64),
|
|
|
|
|
'name' => $name, 'driver' => 's3',
|
|
|
|
|
'endpoint' => 'https://' . $name . '.example.test',
|
|
|
|
|
'region' => 'us-east-1', 'bucket' => $name . '-bucket',
|
|
|
|
|
'prefix' => $name, 'path_style' => 1,
|
|
|
|
|
'failure_domain' => $domain, 'sealed_credentials' => 'sealed',
|
|
|
|
|
'config_revision' => 1, 'tested_revision' => 1,
|
|
|
|
|
'admin_state' => 'active', 'health_state' => 'healthy',
|
|
|
|
|
'write_priority' => $priority, 'consecutive_failures' => 0,
|
|
|
|
|
'consecutive_successes' => 1, 'last_error_code' => '',
|
|
|
|
|
'last_tested_at' => $now, 'last_healthy_at' => $now,
|
|
|
|
|
'next_probe_at' => $now + 300, 'created_at' => $now,
|
|
|
|
|
'updated_at' => $now,
|
|
|
|
|
]);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
$requestKey = 'backup-publication-0001';
|
|
|
|
|
$fingerprint = hash('sha256', 'fixture');
|
|
|
|
|
$view = CloudStorageArtifactService::beginPublication(
|
|
|
|
|
'backup', 'backups/fixture/system.qcow2', 4096,
|
|
|
|
|
str_repeat('a', 64), $requestKey, $fingerprint,
|
|
|
|
|
['backup_id' => 1]
|
|
|
|
|
);
|
|
|
|
|
$assert($view['can_publish'] && $view['backend_id'] === 1
|
|
|
|
|
&& CloudStorageArtifact::count() === 1
|
|
|
|
|
&& CloudStorageReplica::count() === 2
|
|
|
|
|
&& CloudStorageOperation::where('kind', 'copy')->count() === 1,
|
|
|
|
|
'Publication did not reserve one primary and one secondary operation.');
|
|
|
|
|
$again = CloudStorageArtifactService::beginPublication(
|
|
|
|
|
'backup', 'backups/fixture/system.qcow2', 4096,
|
|
|
|
|
str_repeat('a', 64), $requestKey, $fingerprint,
|
|
|
|
|
['backup_id' => 1]
|
|
|
|
|
);
|
|
|
|
|
$assert($again['operation_id'] === $view['operation_id']
|
|
|
|
|
&& CloudStorageArtifact::count() === 1,
|
|
|
|
|
'Idempotent publication allocated a second artifact.');
|
|
|
|
|
$expect(static fn () => CloudStorageArtifactService::beginPublication(
|
|
|
|
|
'backup', 'backups/other/system.qcow2', 4096,
|
|
|
|
|
str_repeat('a', 64), $requestKey, hash('sha256', 'different'),
|
|
|
|
|
['backup_id' => 1]
|
|
|
|
|
), 409);
|
|
|
|
|
$assert(CloudStorageArtifactService::claimReplicaWork(time()) === null,
|
|
|
|
|
'Secondary copy became claimable before the primary was verified.');
|
|
|
|
|
|
|
|
|
|
CloudStorageArtifactService::markPrimaryDispatching($view['operation_id']);
|
|
|
|
|
$expect(static fn () => CloudStorageArtifactService::markPrimaryDispatching(
|
|
|
|
|
$view['operation_id']
|
|
|
|
|
), 409);
|
|
|
|
|
$complete = CloudStorageArtifactService::completePrimary(
|
|
|
|
|
$view['operation_id'],
|
|
|
|
|
(int) $view['artifact_id'],
|
|
|
|
|
[
|
|
|
|
|
'size_bytes' => 4096, 'sha256' => str_repeat('a', 64),
|
|
|
|
|
'etag' => '"etag"', 'version_id' => '',
|
|
|
|
|
]
|
|
|
|
|
);
|
|
|
|
|
$artifact = CloudStorageArtifact::firstOrFail();
|
|
|
|
|
$primary = CloudStorageReplica::where('role', 'primary')->firstOrFail();
|
|
|
|
|
$copy = CloudStorageOperation::where('kind', 'copy')->firstOrFail();
|
|
|
|
|
$assert($complete['outcome'] === 'succeeded'
|
|
|
|
|
&& (string) $artifact->state === 'available'
|
|
|
|
|
&& (string) $primary->state === 'ready'
|
|
|
|
|
&& (string) $copy->phase === 'accepted',
|
|
|
|
|
'Primary settlement did not expose an async backup or retain copy work.');
|
|
|
|
|
CloudStorageArtifactService::completePrimary(
|
|
|
|
|
$view['operation_id'],
|
|
|
|
|
(int) $view['artifact_id'],
|
|
|
|
|
[
|
|
|
|
|
'size_bytes' => 4096, 'sha256' => str_repeat('a', 64),
|
|
|
|
|
'etag' => '"etag"', 'version_id' => '',
|
|
|
|
|
]);
|
|
|
|
|
$expect(static fn () => CloudStorageArtifactService::completePrimary(
|
|
|
|
|
$view['operation_id'], (int) $view['artifact_id'], [
|
|
|
|
|
'size_bytes' => 4096, 'sha256' => str_repeat('a', 64),
|
|
|
|
|
'etag' => '"different"', 'version_id' => '',
|
|
|
|
|
]
|
|
|
|
|
), 409);
|
|
|
|
|
$publication = CloudStoragePublication::firstOrFail();
|
|
|
|
|
$assert((string) $publication->state === 'ready_to_finalize'
|
|
|
|
|
&& (int) $publication->coverage_ready_at > 0,
|
|
|
|
|
'Required primary coverage did not release the durable finalizer signal.');
|
|
|
|
|
$publicationBeforeReject = $publication->getAttributes();
|
|
|
|
|
$expect(static fn () => CloudStoragePublicationFinalizerService::complete(
|
|
|
|
|
'backup', 999, (int) $artifact->id, $view['operation_id']
|
|
|
|
|
), 409);
|
|
|
|
|
$expect(static fn () => CloudStoragePublicationFinalizerService::complete(
|
|
|
|
|
'backup', 1, 999, $view['operation_id']
|
|
|
|
|
), 409);
|
|
|
|
|
$expect(static fn () => CloudStoragePublicationFinalizerService::complete(
|
|
|
|
|
'backup', 1, (int) $artifact->id, (string) $copy->operation_id
|
|
|
|
|
), 409);
|
|
|
|
|
$publication->refresh();
|
|
|
|
|
$assert($publication->getAttributes() === $publicationBeforeReject,
|
|
|
|
|
'Rejected publication finalization changed the durable outbox.');
|
|
|
|
|
$finalized = CloudStoragePublicationFinalizerService::complete(
|
|
|
|
|
'backup', 1, (int) $artifact->id, $view['operation_id']
|
|
|
|
|
);
|
|
|
|
|
$publication->refresh();
|
|
|
|
|
$finalizedAt = (int) $publication->finalized_at;
|
|
|
|
|
$assert($finalized['state'] === 'completed'
|
|
|
|
|
&& (string) $publication->state === 'completed'
|
|
|
|
|
&& (int) $publication->attempts === 1
|
|
|
|
|
&& $finalizedAt > 0,
|
|
|
|
|
'Owner finalization did not complete the durable publication outbox.');
|
|
|
|
|
$againFinalized = CloudStoragePublicationFinalizerService::complete(
|
|
|
|
|
'backup', 1, (int) $artifact->id, $view['operation_id']
|
|
|
|
|
);
|
|
|
|
|
$publication->refresh();
|
|
|
|
|
$assert($againFinalized['state'] === 'completed'
|
|
|
|
|
&& (int) $publication->attempts === 1
|
|
|
|
|
&& (int) $publication->finalized_at === $finalizedAt,
|
|
|
|
|
'Repeated owner finalization was not idempotent.');
|
|
|
|
|
|
|
|
|
|
$claimed = CloudStorageArtifactService::claimReplicaWork(time());
|
|
|
|
|
$assert(is_array($claimed) && $claimed['mode'] === 'dispatch'
|
|
|
|
|
&& $claimed['source_version_id'] === '',
|
|
|
|
|
'Versionless secondary copy was not leased from the durable operation.');
|
|
|
|
|
$assert(CloudStorageArtifactService::claimReplicaWork(time() + 21600) === null,
|
|
|
|
|
'A second worker reclaimed a copy before the helper deadline elapsed.');
|
|
|
|
|
$metadata = [
|
|
|
|
|
'upload_id' => 'upload-1', 'part_size_bytes' => 67108864,
|
|
|
|
|
'part_count' => 1, 'maximum_size_bytes' => 4096,
|
|
|
|
|
'absence_confirmations' => 0,
|
|
|
|
|
];
|
|
|
|
|
CloudStorageArtifactService::markReplicaAwaitingReceipt(
|
|
|
|
|
$claimed['operation_id'], $claimed['lease_token'], $metadata, time()
|
|
|
|
|
);
|
|
|
|
|
$stored = (string) CloudStorageOperation::where(
|
|
|
|
|
'operation_id', $claimed['operation_id']
|
|
|
|
|
)->value('result_json');
|
|
|
|
|
$assert(!str_contains($stored, 'http') && str_contains($stored, 'upload-1'),
|
|
|
|
|
'Durable operation persisted a capability URL or lost its upload identity.');
|
|
|
|
|
CloudStorageArtifactService::completeReplica(
|
|
|
|
|
$claimed['operation_id'], $claimed['lease_token'], [
|
|
|
|
|
'size_bytes' => 4096, 'sha256' => str_repeat('a', 64),
|
|
|
|
|
'etag' => '"secondary"', 'version_id' => '',
|
|
|
|
|
], time()
|
|
|
|
|
);
|
|
|
|
|
CloudStorageArtifactService::completeReplica(
|
|
|
|
|
$claimed['operation_id'], $claimed['lease_token'], [
|
|
|
|
|
'size_bytes' => 4096, 'sha256' => str_repeat('a', 64),
|
|
|
|
|
'etag' => '"secondary"', 'version_id' => '',
|
|
|
|
|
], time()
|
|
|
|
|
);
|
|
|
|
|
$expect(static fn () => CloudStorageArtifactService::completeReplica(
|
|
|
|
|
$claimed['operation_id'], $claimed['lease_token'], [
|
|
|
|
|
'size_bytes' => 4096, 'sha256' => str_repeat('a', 64),
|
|
|
|
|
'etag' => '"secondary"', 'version_id' => 'unexpected-version',
|
|
|
|
|
], time()
|
|
|
|
|
), 409);
|
|
|
|
|
$assert((string) CloudStorageArtifact::firstOrFail()->state === 'protected'
|
|
|
|
|
&& CloudStorageReplica::where('state', 'ready')->count() === 2,
|
|
|
|
|
'Versionless secondary receipt did not protect the artifact.');
|
|
|
|
|
|
|
|
|
|
$agentView = CloudStorageArtifactService::beginAgentPublication(
|
|
|
|
|
'backup', 'backups/agent/system.qcow2', 8192,
|
|
|
|
|
'backup-publication-0002', hash('sha256', 'agent'),
|
|
|
|
|
['backup_id' => 2]
|
|
|
|
|
);
|
|
|
|
|
CloudStorageArtifactService::markPrimaryDispatching($agentView['operation_id']);
|
|
|
|
|
CloudStorageArtifactService::completePrimary(
|
|
|
|
|
$agentView['operation_id'], (int) $agentView['artifact_id'], [
|
|
|
|
|
'size_bytes' => 6144, 'sha256' => str_repeat('c', 64),
|
|
|
|
|
'etag' => '"agent"', 'version_id' => '',
|
|
|
|
|
]);
|
|
|
|
|
$agentArtifact = CloudStorageArtifact::whereKey(
|
|
|
|
|
$agentView['artifact_id']
|
|
|
|
|
)->firstOrFail();
|
|
|
|
|
$assert((int) $agentArtifact->maximum_size_bytes === 8192
|
|
|
|
|
&& (int) $agentArtifact->size_bytes === 6144
|
|
|
|
|
&& (string) $agentArtifact->sha256 === str_repeat('c', 64),
|
|
|
|
|
'Agent publication did not freeze its bounded receipt identity.');
|
|
|
|
|
|
|
|
|
|
$capturedRequest = null;
|
|
|
|
|
$mock = new MockHandler([
|
|
|
|
|
new Response(200, [], '<InitiateMultipartUploadResult>'
|
|
|
|
|
. '<UploadId>worker-upload</UploadId>'
|
|
|
|
|
. '</InitiateMultipartUploadResult>'),
|
|
|
|
|
new Response(200, [
|
|
|
|
|
'Content-Length' => '6144',
|
|
|
|
|
'ETag' => '"worker"',
|
|
|
|
|
]),
|
|
|
|
|
]);
|
|
|
|
|
$s3 = new CloudObjectStorageS3Client(new Client([
|
|
|
|
|
'handler' => HandlerStack::create($mock),
|
|
|
|
|
]));
|
|
|
|
|
$helper = new CloudStorageReplicaHelper(null,
|
|
|
|
|
static function (array $request) use (&$capturedRequest): array {
|
|
|
|
|
$capturedRequest = $request;
|
|
|
|
|
return [
|
|
|
|
|
'protocol' => $request['protocol'],
|
|
|
|
|
'operation_id' => $request['operation_id'],
|
|
|
|
|
'outcome' => 'succeeded',
|
|
|
|
|
'effect_resolution' => 'applied',
|
|
|
|
|
'size_bytes' => 6144,
|
|
|
|
|
'sha256' => str_repeat('c', 64),
|
|
|
|
|
'etag' => '"worker"',
|
|
|
|
|
'version_id' => '',
|
|
|
|
|
];
|
|
|
|
|
}
|
|
|
|
|
);
|
|
|
|
|
$runtime = static fn (int $backendId, bool $healthy): array => [
|
|
|
|
|
'config' => [
|
|
|
|
|
'endpoint' => 'https://storage-' . $backendId . '.example.test',
|
|
|
|
|
'region' => 'us-east-1', 'bucket' => 'bucket-' . $backendId,
|
|
|
|
|
'path_style' => true, 'prefix' => '',
|
|
|
|
|
],
|
|
|
|
|
'credentials' => [
|
|
|
|
|
'access_key_id' => 'access-key',
|
|
|
|
|
'secret_access_key' => 'secret-key',
|
|
|
|
|
],
|
|
|
|
|
'revision' => 1, 'backend_id' => $backendId,
|
|
|
|
|
];
|
|
|
|
|
$worker = new CloudStorageReplicaWorkerService($helper, $s3, $runtime);
|
|
|
|
|
$assert($worker->tick() && is_array($capturedRequest)
|
|
|
|
|
&& $capturedRequest['protocol'] === 'object-replica-v1'
|
|
|
|
|
&& !str_contains($capturedRequest['source']['get_url'], 'versionId='),
|
|
|
|
|
'Replica worker did not stream a versionless exact-key source.');
|
|
|
|
|
$agentCopy = CloudStorageOperation::where('artifact_id', $agentArtifact->id)
|
|
|
|
|
->where('kind', 'copy')->firstOrFail();
|
|
|
|
|
$assert((string) $agentCopy->phase === 'terminal'
|
|
|
|
|
&& (string) $agentCopy->outcome === 'succeeded'
|
|
|
|
|
&& !str_contains((string) $agentCopy->result_json, 'https://'),
|
|
|
|
|
'Replica worker did not settle safely or persisted a capability URL.');
|
|
|
|
|
|
|
|
|
|
$unknownView = CloudStorageArtifactService::beginPublication(
|
|
|
|
|
'template', 'cloud-templates/unknown/system.qcow2', 4096,
|
|
|
|
|
str_repeat('e', 64), 'template-publication-0003',
|
|
|
|
|
hash('sha256', 'unknown'), [
|
|
|
|
|
'template_import_intent_id' => str_repeat('e', 32),
|
|
|
|
|
]
|
|
|
|
|
);
|
|
|
|
|
CloudStorageArtifactService::markPrimaryDispatching(
|
|
|
|
|
$unknownView['operation_id']
|
|
|
|
|
);
|
|
|
|
|
CloudStorageArtifactService::completePrimary(
|
|
|
|
|
$unknownView['operation_id'], (int) $unknownView['artifact_id'], [
|
|
|
|
|
'size_bytes' => 4096, 'sha256' => str_repeat('e', 64),
|
|
|
|
|
'etag' => '"unknown-primary"',
|
|
|
|
|
'version_id' => '',
|
|
|
|
|
]);
|
|
|
|
|
$copyCalls = 0;
|
|
|
|
|
$unknownHelper = new CloudStorageReplicaHelper(null,
|
|
|
|
|
static function (array $request) use (&$copyCalls): array {
|
|
|
|
|
$copyCalls++;
|
|
|
|
|
return [
|
|
|
|
|
'protocol' => $request['protocol'],
|
|
|
|
|
// A mismatched receipt must become unknown, never success.
|
|
|
|
|
'operation_id' => str_repeat('f', 32),
|
|
|
|
|
'outcome' => 'succeeded', 'effect_resolution' => 'applied',
|
|
|
|
|
'size_bytes' => 4096, 'sha256' => str_repeat('e', 64),
|
|
|
|
|
'etag' => '"wrong-operation"', 'version_id' => '',
|
|
|
|
|
];
|
|
|
|
|
}
|
|
|
|
|
);
|
|
|
|
|
$unknownHTTP = new MockHandler([
|
|
|
|
|
new Response(200, [], '<InitiateMultipartUploadResult>'
|
|
|
|
|
. '<UploadId>unknown-upload</UploadId>'
|
|
|
|
|
. '</InitiateMultipartUploadResult>'),
|
|
|
|
|
new Response(404),
|
|
|
|
|
new Response(200, [], '<ListMultipartUploadsResult>'
|
|
|
|
|
. '<IsTruncated>false</IsTruncated>'
|
|
|
|
|
. '</ListMultipartUploadsResult>'),
|
|
|
|
|
new Response(404),
|
|
|
|
|
new Response(200, [], '<ListMultipartUploadsResult>'
|
|
|
|
|
. '<IsTruncated>false</IsTruncated>'
|
|
|
|
|
. '</ListMultipartUploadsResult>'),
|
|
|
|
|
]);
|
|
|
|
|
$unknownWorker = new CloudStorageReplicaWorkerService(
|
|
|
|
|
$unknownHelper,
|
|
|
|
|
new CloudObjectStorageS3Client(new Client([
|
|
|
|
|
'handler' => HandlerStack::create($unknownHTTP),
|
|
|
|
|
])),
|
|
|
|
|
$runtime
|
|
|
|
|
);
|
|
|
|
|
$unknownWorker->tick();
|
|
|
|
|
$unknownCopy = CloudStorageOperation::where(
|
|
|
|
|
'artifact_id', $unknownView['artifact_id']
|
|
|
|
|
)->where('kind', 'copy')->firstOrFail();
|
|
|
|
|
$assert((string) $unknownCopy->phase === 'reconciling'
|
|
|
|
|
&& (string) $unknownCopy->effect_resolution === 'unknown',
|
|
|
|
|
'Mismatched helper receipt did not preserve unknown effect.');
|
|
|
|
|
$unknownCopy->next_check_at = 0;
|
|
|
|
|
$unknownCopy->save();
|
|
|
|
|
$unknownWorker->tick();
|
|
|
|
|
$unknownCopy->refresh();
|
|
|
|
|
$unknownMetadata = json_decode((string) $unknownCopy->result_json, true);
|
|
|
|
|
$assert($copyCalls === 1 && (string) $unknownCopy->phase === 'reconciling'
|
|
|
|
|
&& ($unknownMetadata['multipart_absence_confirmations'] ?? null) === 1,
|
|
|
|
|
'Unknown copy was replayed instead of read-only reconciliation.');
|
|
|
|
|
$unknownCopy->next_check_at = 0;
|
|
|
|
|
$unknownCopy->save();
|
|
|
|
|
$unknownWorker->tick();
|
|
|
|
|
$unknownCopy->refresh();
|
|
|
|
|
$unknownPublication = CloudStoragePublication::where(
|
|
|
|
|
'artifact_id', $unknownView['artifact_id']
|
|
|
|
|
)->firstOrFail();
|
|
|
|
|
$assert($copyCalls === 1 && (string) $unknownCopy->phase === 'accepted'
|
|
|
|
|
&& (int) $unknownCopy->attempts === 1
|
|
|
|
|
&& $unknownCopy->result_json === null
|
|
|
|
|
&& in_array(
|
|
|
|
|
'replica_retry_scheduled',
|
|
|
|
|
$unknownWorker->observationErrors(),
|
|
|
|
|
true
|
|
|
|
|
)
|
|
|
|
|
&& (string) $unknownPublication->state === 'waiting_protection',
|
|
|
|
|
'Required-two copy was not safely rescheduled after read-only absence proof: '
|
|
|
|
|
. json_encode([
|
|
|
|
|
'calls' => $copyCalls, 'phase' => $unknownCopy->phase,
|
|
|
|
|
'attempts' => $unknownCopy->attempts,
|
|
|
|
|
'result' => $unknownCopy->result_json,
|
|
|
|
|
'publication' => $unknownPublication->state,
|
|
|
|
|
'error' => $unknownCopy->error_code,
|
|
|
|
|
]));
|
|
|
|
|
$unknownCopy->available_at = 253402300799;
|
|
|
|
|
$unknownCopy->save();
|
|
|
|
|
|
|
|
|
|
$retryView = CloudStorageArtifactService::beginPublication(
|
|
|
|
|
'backup', 'backups/retry/system.qcow2', 4096,
|
|
|
|
|
str_repeat('f', 64), 'backup-publication-0005',
|
|
|
|
|
hash('sha256', 'retry'), ['backup_id' => 5]
|
|
|
|
|
);
|
|
|
|
|
CloudStorageArtifactService::markPrimaryDispatching(
|
|
|
|
|
$retryView['operation_id']
|
|
|
|
|
);
|
|
|
|
|
CloudStorageArtifactService::completePrimary(
|
|
|
|
|
$retryView['operation_id'], (int) $retryView['artifact_id'], [
|
|
|
|
|
'size_bytes' => 4096, 'sha256' => str_repeat('f', 64),
|
|
|
|
|
'etag' => '"retry-primary"',
|
|
|
|
|
'version_id' => '',
|
|
|
|
|
]);
|
|
|
|
|
$retryClaim = CloudStorageArtifactService::claimReplicaWork(time());
|
|
|
|
|
$assert(is_array($retryClaim)
|
|
|
|
|
&& (int) $retryClaim['artifact_id'] === (int) $retryView['artifact_id'],
|
|
|
|
|
'Recoverable copy fixture was not claimable.');
|
|
|
|
|
CloudStorageArtifactService::rescheduleReplicaNotApplied(
|
|
|
|
|
$retryClaim['operation_id'],
|
|
|
|
|
$retryClaim['lease_token'],
|
|
|
|
|
'replica_source_download_failed',
|
|
|
|
|
time(),
|
|
|
|
|
60
|
|
|
|
|
);
|
|
|
|
|
$retryOperation = CloudStorageOperation::where(
|
|
|
|
|
'operation_id', $retryClaim['operation_id']
|
|
|
|
|
)->firstOrFail();
|
|
|
|
|
$assert((string) $retryOperation->phase === 'accepted'
|
|
|
|
|
&& $retryOperation->outcome === null
|
|
|
|
|
&& $retryOperation->terminal_at === null
|
|
|
|
|
&& $retryOperation->result_json === null,
|
|
|
|
|
'Definite no-effect copy failure exhausted the only copy intent.');
|
|
|
|
|
$retryOperation->available_at = 253402300799;
|
|
|
|
|
$retryOperation->save();
|
|
|
|
|
|
|
|
|
|
$adoptView = CloudStorageArtifactService::beginPublication(
|
|
|
|
|
'backup', 'backups/adopt/system.qcow2', 4096,
|
|
|
|
|
str_repeat('1', 64), 'backup-publication-0006',
|
|
|
|
|
hash('sha256', 'adopt'), ['backup_id' => 6]
|
|
|
|
|
);
|
|
|
|
|
CloudStorageArtifactService::markPrimaryDispatching(
|
|
|
|
|
$adoptView['operation_id']
|
|
|
|
|
);
|
|
|
|
|
CloudStorageArtifactService::completePrimary(
|
|
|
|
|
$adoptView['operation_id'], (int) $adoptView['artifact_id'], [
|
|
|
|
|
'size_bytes' => 4096, 'sha256' => str_repeat('1', 64),
|
|
|
|
|
'etag' => '"adopt-primary"',
|
|
|
|
|
'version_id' => '',
|
|
|
|
|
]);
|
|
|
|
|
$adoptCopy = CloudStorageOperation::where(
|
|
|
|
|
'artifact_id', $adoptView['artifact_id']
|
|
|
|
|
)->where('kind', 'copy')->firstOrFail();
|
|
|
|
|
$adoptTarget = CloudStorageReplica::whereKey(
|
|
|
|
|
(int) $adoptCopy->replica_id
|
|
|
|
|
)->firstOrFail();
|
|
|
|
|
$adoptCalls = 0;
|
|
|
|
|
$adoptHelper = new CloudStorageReplicaHelper(null,
|
|
|
|
|
static function (array $request) use (&$adoptCalls): array {
|
|
|
|
|
$adoptCalls++;
|
|
|
|
|
return [
|
|
|
|
|
'protocol' => $request['protocol'],
|
|
|
|
|
'operation_id' => $request['operation_id'],
|
|
|
|
|
'outcome' => 'succeeded', 'effect_resolution' => 'applied',
|
|
|
|
|
'size_bytes' => 4096, 'sha256' => str_repeat('1', 64),
|
|
|
|
|
'etag' => '"adopted"', 'version_id' => '',
|
|
|
|
|
];
|
|
|
|
|
}
|
|
|
|
|
);
|
|
|
|
|
$adoptHTTP = new MockHandler([
|
|
|
|
|
// The provider may create the session even though the response is lost.
|
|
|
|
|
new Response(500),
|
|
|
|
|
new Response(404),
|
|
|
|
|
new Response(200, [], '<ListMultipartUploadsResult>'
|
|
|
|
|
. '<IsTruncated>false</IsTruncated>'
|
|
|
|
|
. '<Upload><Key>' . htmlspecialchars(
|
|
|
|
|
(string) $adoptTarget->object_key,
|
|
|
|
|
ENT_XML1 | ENT_QUOTES,
|
|
|
|
|
'UTF-8'
|
|
|
|
|
) . '</Key><UploadId>adopt-upload</UploadId><Initiated>'
|
|
|
|
|
. gmdate('Y-m-d\\TH:i:s\\Z', time())
|
|
|
|
|
. '</Initiated></Upload></ListMultipartUploadsResult>'),
|
|
|
|
|
new Response(200, [
|
|
|
|
|
'Content-Length' => '4096',
|
|
|
|
|
'ETag' => '"adopted"',
|
|
|
|
|
]),
|
|
|
|
|
]);
|
|
|
|
|
$adoptWorker = new CloudStorageReplicaWorkerService(
|
|
|
|
|
$adoptHelper,
|
|
|
|
|
new CloudObjectStorageS3Client(new Client([
|
|
|
|
|
'handler' => HandlerStack::create($adoptHTTP),
|
|
|
|
|
])),
|
|
|
|
|
$runtime
|
|
|
|
|
);
|
|
|
|
|
$adoptWorker->tick();
|
|
|
|
|
$adoptCopy->refresh();
|
|
|
|
|
$assert((string) $adoptCopy->phase === 'reconciling'
|
|
|
|
|
&& $adoptCopy->result_json === '[]'
|
|
|
|
|
&& $adoptCalls === 0,
|
|
|
|
|
'Ambiguous multipart creation was replayed before read-only lookup.');
|
|
|
|
|
$adoptCopy->next_check_at = 0;
|
|
|
|
|
$adoptCopy->save();
|
|
|
|
|
$adoptWorker->tick();
|
|
|
|
|
$adoptCopy->refresh();
|
|
|
|
|
$assert($adoptCalls === 1
|
|
|
|
|
&& (string) $adoptCopy->phase === 'terminal'
|
|
|
|
|
&& (string) $adoptCopy->outcome === 'succeeded',
|
|
|
|
|
'One exact multipart session was not adopted exactly once.');
|
|
|
|
|
|
|
|
|
|
// Crash after the upload_id is durable but before helper startup. The same
|
|
|
|
|
// open session is safe to resume only after two stable read-only observations;
|
|
|
|
|
// no new multipart session or Abort is allowed.
|
|
|
|
|
$resumeView = CloudStorageArtifactService::beginPublication(
|
|
|
|
|
'backup', 'backups/resume/system.qcow2', 4096,
|
|
|
|
|
str_repeat('2', 64), 'backup-publication-0007',
|
|
|
|
|
hash('sha256', 'resume'), ['backup_id' => 7]
|
|
|
|
|
);
|
|
|
|
|
CloudStorageArtifactService::markPrimaryDispatching(
|
|
|
|
|
$resumeView['operation_id']
|
|
|
|
|
);
|
|
|
|
|
CloudStorageArtifactService::completePrimary(
|
|
|
|
|
$resumeView['operation_id'], (int) $resumeView['artifact_id'], [
|
|
|
|
|
'size_bytes' => 4096, 'sha256' => str_repeat('2', 64),
|
|
|
|
|
'etag' => '"resume-primary"', 'version_id' => '',
|
|
|
|
|
]);
|
|
|
|
|
$resumeWork = CloudStorageArtifactService::claimReplicaWork(time());
|
|
|
|
|
$assert(is_array($resumeWork), 'Resume crash fixture was not claimable.');
|
|
|
|
|
CloudStorageArtifactService::markReplicaAwaitingReceipt(
|
|
|
|
|
$resumeWork['operation_id'],
|
|
|
|
|
$resumeWork['lease_token'],
|
|
|
|
|
[
|
|
|
|
|
'upload_id' => 'resume-upload',
|
|
|
|
|
'part_size_bytes' => 67108864,
|
|
|
|
|
'part_count' => 1,
|
|
|
|
|
'maximum_size_bytes' => 4096,
|
|
|
|
|
'multipart_absence_confirmations' => 0,
|
|
|
|
|
'multipart_presence_confirmations' => 0,
|
|
|
|
|
],
|
|
|
|
|
time()
|
|
|
|
|
);
|
|
|
|
|
$resumeOperation = CloudStorageOperation::where(
|
|
|
|
|
'operation_id', $resumeWork['operation_id']
|
|
|
|
|
)->firstOrFail();
|
|
|
|
|
$resumeOperation->lease_until = 0;
|
|
|
|
|
$resumeOperation->save();
|
|
|
|
|
$resumeTarget = CloudStorageReplica::whereKey(
|
|
|
|
|
(int) $resumeOperation->replica_id
|
|
|
|
|
)->firstOrFail();
|
|
|
|
|
$resumeTarget->lease_until = 0;
|
|
|
|
|
$resumeTarget->save();
|
|
|
|
|
$resumeXML = '<ListMultipartUploadsResult><IsTruncated>false</IsTruncated>'
|
|
|
|
|
. '<Upload><Key>' . htmlspecialchars(
|
|
|
|
|
(string) $resumeTarget->object_key,
|
|
|
|
|
ENT_XML1 | ENT_QUOTES,
|
|
|
|
|
'UTF-8'
|
|
|
|
|
) . '</Key><UploadId>resume-upload</UploadId><Initiated>'
|
|
|
|
|
. gmdate('Y-m-d\\TH:i:s\\Z', time())
|
|
|
|
|
. '</Initiated></Upload></ListMultipartUploadsResult>';
|
|
|
|
|
$resumeHTTP = new MockHandler([
|
|
|
|
|
new Response(404), new Response(200, [], $resumeXML),
|
|
|
|
|
new Response(404), new Response(200, [], $resumeXML),
|
|
|
|
|
new Response(200, [
|
|
|
|
|
'Content-Length' => '4096', 'ETag' => '"resumed"',
|
|
|
|
|
]),
|
|
|
|
|
]);
|
|
|
|
|
$resumeCalls = 0;
|
|
|
|
|
$resumeHelper = new CloudStorageReplicaHelper(null,
|
|
|
|
|
static function (array $request) use (&$resumeCalls): array {
|
|
|
|
|
$resumeCalls++;
|
|
|
|
|
return [
|
|
|
|
|
'protocol' => $request['protocol'],
|
|
|
|
|
'operation_id' => $request['operation_id'],
|
|
|
|
|
'outcome' => 'succeeded', 'effect_resolution' => 'applied',
|
|
|
|
|
'size_bytes' => 4096, 'sha256' => str_repeat('2', 64),
|
|
|
|
|
'etag' => '"resumed"', 'version_id' => '',
|
|
|
|
|
];
|
|
|
|
|
}
|
|
|
|
|
);
|
|
|
|
|
$resumeWorker = new CloudStorageReplicaWorkerService(
|
|
|
|
|
$resumeHelper,
|
|
|
|
|
new CloudObjectStorageS3Client(new Client([
|
|
|
|
|
'handler' => HandlerStack::create($resumeHTTP),
|
|
|
|
|
])),
|
|
|
|
|
$runtime
|
|
|
|
|
);
|
|
|
|
|
$resumeWorker->tick();
|
|
|
|
|
$resumeOperation->refresh();
|
|
|
|
|
$resumeEvidence = json_decode((string) $resumeOperation->result_json, true);
|
|
|
|
|
$assert($resumeCalls === 0
|
|
|
|
|
&& ($resumeEvidence['multipart_presence_confirmations'] ?? 0) === 1,
|
|
|
|
|
'A persisted multipart session resumed without two stable observations.');
|
|
|
|
|
$resumeOperation->next_check_at = 0;
|
|
|
|
|
$resumeOperation->save();
|
|
|
|
|
$resumeWorker->tick();
|
|
|
|
|
$resumeOperation->refresh();
|
|
|
|
|
$assert($resumeCalls === 1
|
|
|
|
|
&& (string) $resumeOperation->phase === 'terminal'
|
|
|
|
|
&& (string) $resumeOperation->outcome === 'succeeded',
|
|
|
|
|
'A stable persisted multipart session did not resume exactly once.');
|
|
|
|
|
|
|
|
|
|
// A successful helper receipt is not authoritative until target HEAD agrees
|
|
|
|
|
// on both ETag and version identity.
|
|
|
|
|
$mixedView = CloudStorageArtifactService::beginPublication(
|
|
|
|
|
'backup', 'backups/mixed-version/system.qcow2', 4096,
|
|
|
|
|
str_repeat('3', 64), 'backup-publication-0008',
|
|
|
|
|
hash('sha256', 'mixed-version'), ['backup_id' => 8]
|
|
|
|
|
);
|
|
|
|
|
CloudStorageArtifactService::markPrimaryDispatching(
|
|
|
|
|
$mixedView['operation_id']
|
|
|
|
|
);
|
|
|
|
|
CloudStorageArtifactService::completePrimary(
|
|
|
|
|
$mixedView['operation_id'], (int) $mixedView['artifact_id'], [
|
|
|
|
|
'size_bytes' => 4096, 'sha256' => str_repeat('3', 64),
|
|
|
|
|
'etag' => '"mixed-primary"', 'version_id' => '',
|
|
|
|
|
]);
|
|
|
|
|
$mixedHelper = new CloudStorageReplicaHelper(null,
|
|
|
|
|
static fn (array $request): array => [
|
|
|
|
|
'protocol' => $request['protocol'],
|
|
|
|
|
'operation_id' => $request['operation_id'],
|
|
|
|
|
'outcome' => 'succeeded', 'effect_resolution' => 'applied',
|
|
|
|
|
'size_bytes' => 4096, 'sha256' => str_repeat('3', 64),
|
|
|
|
|
'etag' => '"mixed"', 'version_id' => '',
|
|
|
|
|
]
|
|
|
|
|
);
|
|
|
|
|
$mixedHTTP = new MockHandler([
|
|
|
|
|
new Response(200, [], '<InitiateMultipartUploadResult>'
|
|
|
|
|
. '<UploadId>mixed-upload</UploadId>'
|
|
|
|
|
. '</InitiateMultipartUploadResult>'),
|
|
|
|
|
new Response(200, [
|
|
|
|
|
'Content-Length' => '4096', 'ETag' => '"mixed"',
|
|
|
|
|
'x-amz-version-id' => 'versioned-object',
|
|
|
|
|
]),
|
|
|
|
|
]);
|
|
|
|
|
$mixedWorker = new CloudStorageReplicaWorkerService(
|
|
|
|
|
$mixedHelper,
|
|
|
|
|
new CloudObjectStorageS3Client(new Client([
|
|
|
|
|
'handler' => HandlerStack::create($mixedHTTP),
|
|
|
|
|
])),
|
|
|
|
|
$runtime
|
|
|
|
|
);
|
|
|
|
|
$mixedWorker->tick();
|
|
|
|
|
$mixedCopy = CloudStorageOperation::where(
|
|
|
|
|
'artifact_id', $mixedView['artifact_id']
|
|
|
|
|
)->where('kind', 'copy')->firstOrFail();
|
|
|
|
|
$assert((string) $mixedCopy->phase === 'terminal'
|
|
|
|
|
&& (string) $mixedCopy->outcome === 'failed'
|
|
|
|
|
&& (string) $mixedCopy->error_code === 'replica_identity_changed',
|
|
|
|
|
'Mixed versioned/versionless completion was accepted.');
|
|
|
|
|
$assert(in_array(
|
|
|
|
|
'replica_identity_changed',
|
|
|
|
|
$mixedWorker->observationErrors(),
|
|
|
|
|
true
|
|
|
|
|
), 'A terminal identity mismatch was hidden from worker health.');
|
|
|
|
|
CloudStorageArtifactService::failReplicaAppliedMismatch(
|
|
|
|
|
(string) $mixedCopy->operation_id,
|
|
|
|
|
str_repeat('0', 32),
|
|
|
|
|
'replica_identity_changed',
|
|
|
|
|
time()
|
|
|
|
|
);
|
|
|
|
|
$expect(static fn () =>
|
|
|
|
|
CloudStorageArtifactService::failReplicaAppliedMismatch(
|
|
|
|
|
(string) $mixedCopy->operation_id,
|
|
|
|
|
str_repeat('0', 32),
|
|
|
|
|
'replica_identity_changed_again',
|
|
|
|
|
time()
|
|
|
|
|
), 409);
|
|
|
|
|
$expect(static fn () => CloudStorageArtifactService::failReplicaNotApplied(
|
|
|
|
|
(string) $mixedCopy->operation_id,
|
|
|
|
|
str_repeat('0', 32),
|
|
|
|
|
'replica_identity_changed',
|
|
|
|
|
time()
|
|
|
|
|
), 409);
|
|
|
|
|
|
|
|
|
|
// Stable multipart identity ambiguity must stop automatic polling and expose
|
|
|
|
|
// one administrator recovery entry for the original operation only.
|
|
|
|
|
$attentionView = CloudStorageArtifactService::beginPublication(
|
|
|
|
|
'template', 'cloud-templates/attention/system.qcow2', 4096,
|
|
|
|
|
str_repeat('4', 64), 'template-publication-attention',
|
|
|
|
|
hash('sha256', 'attention'), [
|
|
|
|
|
'template_import_intent_id' => str_repeat('4', 32),
|
|
|
|
|
]
|
|
|
|
|
);
|
|
|
|
|
CloudStorageArtifactService::markPrimaryDispatching(
|
|
|
|
|
$attentionView['operation_id']
|
|
|
|
|
);
|
|
|
|
|
CloudStorageArtifactService::completePrimary(
|
|
|
|
|
$attentionView['operation_id'], (int) $attentionView['artifact_id'], [
|
|
|
|
|
'size_bytes' => 4096, 'sha256' => str_repeat('4', 64),
|
|
|
|
|
'etag' => '"attention-primary"', 'version_id' => '',
|
|
|
|
|
]);
|
|
|
|
|
$attentionCopy = CloudStorageOperation::where(
|
|
|
|
|
'artifact_id', $attentionView['artifact_id']
|
|
|
|
|
)->where('kind', 'copy')->firstOrFail();
|
|
|
|
|
$attentionTarget = CloudStorageReplica::whereKey(
|
|
|
|
|
(int) $attentionCopy->replica_id
|
|
|
|
|
)->firstOrFail();
|
|
|
|
|
$attentionKey = htmlspecialchars(
|
|
|
|
|
(string) $attentionTarget->object_key,
|
|
|
|
|
ENT_XML1 | ENT_QUOTES,
|
|
|
|
|
'UTF-8'
|
|
|
|
|
);
|
|
|
|
|
$attentionSessions = '<ListMultipartUploadsResult>'
|
|
|
|
|
. '<IsTruncated>false</IsTruncated>'
|
|
|
|
|
. '<Upload><Key>' . $attentionKey . '</Key><UploadId>attention-a</UploadId>'
|
|
|
|
|
. '<Initiated>' . gmdate('Y-m-d\\TH:i:s\\Z', time())
|
|
|
|
|
. '</Initiated></Upload>'
|
|
|
|
|
. '<Upload><Key>' . $attentionKey . '</Key><UploadId>attention-b</UploadId>'
|
|
|
|
|
. '<Initiated>' . gmdate('Y-m-d\\TH:i:s\\Z', time())
|
|
|
|
|
. '</Initiated></Upload></ListMultipartUploadsResult>';
|
|
|
|
|
$attentionHTTP = new MockHandler([
|
|
|
|
|
new Response(500),
|
|
|
|
|
new Response(404), new Response(200, [], $attentionSessions),
|
|
|
|
|
new Response(404), new Response(200, [], $attentionSessions),
|
|
|
|
|
]);
|
|
|
|
|
$attentionHelperCalls = 0;
|
|
|
|
|
$attentionWorker = new CloudStorageReplicaWorkerService(
|
|
|
|
|
new CloudStorageReplicaHelper(null,
|
|
|
|
|
static function (array $request) use (&$attentionHelperCalls): array {
|
|
|
|
|
$attentionHelperCalls++;
|
|
|
|
|
throw new RuntimeException('Ambiguous upload must not start helper');
|
|
|
|
|
}
|
|
|
|
|
),
|
|
|
|
|
new CloudObjectStorageS3Client(new Client([
|
|
|
|
|
'handler' => HandlerStack::create($attentionHTTP),
|
|
|
|
|
])),
|
|
|
|
|
$runtime
|
|
|
|
|
);
|
|
|
|
|
$attentionWorker->tick();
|
|
|
|
|
$attentionCopy->refresh();
|
|
|
|
|
$attentionCopy->next_check_at = 0;
|
|
|
|
|
$attentionCopy->save();
|
|
|
|
|
$attentionWorker->tick();
|
|
|
|
|
$attentionCopy->refresh();
|
|
|
|
|
$firstAttentionEvidence = json_decode(
|
|
|
|
|
(string) $attentionCopy->result_json,
|
|
|
|
|
true
|
|
|
|
|
);
|
|
|
|
|
$assert((string) $attentionCopy->phase === 'reconciling'
|
|
|
|
|
&& ($firstAttentionEvidence['multipart_attention_confirmations'] ?? 0)
|
|
|
|
|
=== 1,
|
|
|
|
|
'First multipart ambiguity observation did not remain reconciling.');
|
|
|
|
|
$attentionCopy->next_check_at = 0;
|
|
|
|
|
$attentionCopy->save();
|
|
|
|
|
$attentionWorker->tick();
|
|
|
|
|
$attentionCopy->refresh();
|
|
|
|
|
$attentionEvidence = json_decode((string) $attentionCopy->result_json, true);
|
|
|
|
|
$attentionList = CloudStorageReplicaAttentionService::listView();
|
|
|
|
|
$operationCountBeforeAttentionRetry = CloudStorageOperation::count();
|
|
|
|
|
$assert($attentionHelperCalls === 0
|
|
|
|
|
&& (string) $attentionCopy->phase === 'terminal'
|
|
|
|
|
&& (string) $attentionCopy->outcome === 'failed'
|
|
|
|
|
&& (string) $attentionCopy->effect_resolution === 'unknown'
|
|
|
|
|
&& (bool) $attentionCopy->effect_may_have_applied
|
|
|
|
|
&& !(bool) $attentionCopy->safe_to_resubmit
|
|
|
|
|
&& ($attentionEvidence['multipart_attention_confirmations'] ?? 0) === 2
|
|
|
|
|
&& ($attentionList['summary']['count'] ?? 0) === 1
|
|
|
|
|
&& !str_contains(json_encode($attentionList), $attentionKey)
|
|
|
|
|
&& !str_contains(json_encode($attentionList), 'attention-a')
|
|
|
|
|
&& !str_contains(json_encode($attentionList), 'attention-b'),
|
|
|
|
|
'Stable multipart ambiguity did not converge to administrator attention.');
|
|
|
|
|
$attentionTarget->refresh();
|
|
|
|
|
$attentionTarget->state = 'ready';
|
|
|
|
|
$attentionTarget->size_bytes = 4096;
|
|
|
|
|
$attentionTarget->sha256 = str_repeat('4', 64);
|
|
|
|
|
$attentionTarget->verified_at = time();
|
|
|
|
|
$attentionTarget->error_code = '';
|
|
|
|
|
$attentionTarget->save();
|
|
|
|
|
$staleAttentionList = CloudStorageReplicaAttentionService::listView();
|
|
|
|
|
$assert(($staleAttentionList['items'][0]['can_retry'] ?? true) === false,
|
|
|
|
|
'A stale attention operation remained retryable after target repair.');
|
|
|
|
|
$expect(static fn () => CloudStorageReplicaAttentionService::retry(
|
|
|
|
|
(string) $attentionCopy->operation_id
|
|
|
|
|
), 409);
|
|
|
|
|
$attentionTarget->state = 'degraded';
|
|
|
|
|
$attentionTarget->error_code = 'replica_multipart_identity_ambiguous';
|
|
|
|
|
$attentionTarget->save();
|
|
|
|
|
$attentionArtifact = CloudStorageArtifact::whereKey(
|
|
|
|
|
(int) $attentionView['artifact_id']
|
|
|
|
|
)->firstOrFail();
|
|
|
|
|
$attentionArtifact->state = 'delete_attention';
|
|
|
|
|
$attentionArtifact->save();
|
|
|
|
|
$deleteAttentionList = CloudStorageReplicaAttentionService::listView();
|
|
|
|
|
$deleteAttentionItem = $deleteAttentionList['items'][0] ?? [];
|
|
|
|
|
$assert(($deleteAttentionItem['operation_id'] ?? '')
|
|
|
|
|
=== (string) $attentionCopy->operation_id
|
|
|
|
|
&& ($deleteAttentionItem['can_retry'] ?? false) === true,
|
|
|
|
|
'Delete-attention copy did not expose read-only reconciliation.');
|
|
|
|
|
CloudStorageReplicaAttentionService::retry(
|
|
|
|
|
(string) $attentionCopy->operation_id
|
|
|
|
|
);
|
|
|
|
|
$attentionCopy->refresh();
|
|
|
|
|
$attentionArtifact->refresh();
|
|
|
|
|
$reopenedEvidence = json_decode((string) $attentionCopy->result_json, true);
|
|
|
|
|
$assert((string) $attentionCopy->phase === 'reconciling'
|
|
|
|
|
&& (string) $attentionCopy->error_code === 'admin_reconcile_requested'
|
|
|
|
|
&& ($reopenedEvidence['multipart_attention_confirmations'] ?? -1) === 0
|
|
|
|
|
&& (string) $attentionArtifact->state === 'delete_attention'
|
|
|
|
|
&& CloudStorageOperation::count() === $operationCountBeforeAttentionRetry
|
|
|
|
|
&& $attentionHelperCalls === 0,
|
|
|
|
|
'Delete-attention retry replaced or dispatched the original copy intent.');
|
|
|
|
|
$attentionCopy->next_check_at = 253402300799;
|
|
|
|
|
$attentionCopy->save();
|
|
|
|
|
|
|
|
|
|
// A definite no-effect failure is idempotent only for the exact same terminal
|
|
|
|
|
// identity; conflicting late receipts must be rejected.
|
|
|
|
|
$failedCopyView = CloudStorageArtifactService::beginPublication(
|
|
|
|
|
'backup', 'backups/failed-copy/system.qcow2', 4096,
|
|
|
|
|
str_repeat('5', 64), 'backup-publication-failed-copy',
|
|
|
|
|
hash('sha256', 'failed-copy'), ['backup_id' => 9]
|
|
|
|
|
);
|
|
|
|
|
CloudStorageArtifactService::markPrimaryDispatching(
|
|
|
|
|
$failedCopyView['operation_id']
|
|
|
|
|
);
|
|
|
|
|
CloudStorageArtifactService::completePrimary(
|
|
|
|
|
$failedCopyView['operation_id'], (int) $failedCopyView['artifact_id'], [
|
|
|
|
|
'size_bytes' => 4096, 'sha256' => str_repeat('5', 64),
|
|
|
|
|
'etag' => '"failed-copy-primary"', 'version_id' => '',
|
|
|
|
|
]);
|
|
|
|
|
$failedCopyClaim = CloudStorageArtifactService::claimReplicaWork(time());
|
|
|
|
|
$assert(is_array($failedCopyClaim), 'Failed-copy fixture was not claimable.');
|
|
|
|
|
CloudStorageArtifactService::failReplicaNotApplied(
|
|
|
|
|
$failedCopyClaim['operation_id'],
|
|
|
|
|
$failedCopyClaim['lease_token'],
|
|
|
|
|
'replica_copy_rejected',
|
|
|
|
|
time()
|
|
|
|
|
);
|
|
|
|
|
CloudStorageArtifactService::failReplicaNotApplied(
|
|
|
|
|
$failedCopyClaim['operation_id'],
|
|
|
|
|
$failedCopyClaim['lease_token'],
|
|
|
|
|
'replica_copy_rejected',
|
|
|
|
|
time()
|
|
|
|
|
);
|
|
|
|
|
$expect(static fn () => CloudStorageArtifactService::failReplicaNotApplied(
|
|
|
|
|
$failedCopyClaim['operation_id'],
|
|
|
|
|
$failedCopyClaim['lease_token'],
|
|
|
|
|
'replica_copy_rejected_again',
|
|
|
|
|
time()
|
|
|
|
|
), 409);
|
|
|
|
|
|
|
|
|
|
$pendingView = CloudStorageArtifactService::beginAgentPublication(
|
|
|
|
|
'backup', 'backups/pending/system.qcow2', 8192,
|
|
|
|
|
'backup-publication-0004', hash('sha256', 'pending'),
|
|
|
|
|
['backup_id' => 4]
|
|
|
|
|
);
|
|
|
|
|
CloudStorageArtifactService::markPrimaryDispatching(
|
|
|
|
|
$pendingView['operation_id']
|
|
|
|
|
);
|
|
|
|
|
CloudStorageArtifactService::failPrimary(
|
|
|
|
|
$pendingView['operation_id'],
|
|
|
|
|
(int) $pendingView['artifact_id'],
|
|
|
|
|
'not_applied',
|
|
|
|
|
'primary_upload_rejected'
|
|
|
|
|
);
|
|
|
|
|
CloudStorageArtifactService::failPrimary(
|
|
|
|
|
$pendingView['operation_id'],
|
|
|
|
|
(int) $pendingView['artifact_id'],
|
|
|
|
|
'not_applied',
|
|
|
|
|
'primary_upload_rejected'
|
|
|
|
|
);
|
|
|
|
|
$expect(static fn () => CloudStorageArtifactService::failPrimary(
|
|
|
|
|
$pendingView['operation_id'],
|
|
|
|
|
(int) $pendingView['artifact_id'],
|
|
|
|
|
'not_applied',
|
|
|
|
|
'primary_upload_rejected_again'
|
|
|
|
|
), 409);
|
|
|
|
|
$pendingArtifact = CloudStorageArtifact::whereKey(
|
|
|
|
|
$pendingView['artifact_id']
|
|
|
|
|
)->firstOrFail();
|
|
|
|
|
$pendingPublication = CloudStoragePublication::where(
|
|
|
|
|
'artifact_id', $pendingArtifact->id
|
|
|
|
|
)->firstOrFail();
|
|
|
|
|
$pendingCopy = CloudStorageOperation::where(
|
|
|
|
|
'artifact_id', $pendingArtifact->id
|
|
|
|
|
)->where('kind', 'copy')->firstOrFail();
|
|
|
|
|
$assert((string) $pendingArtifact->state === 'publishing'
|
|
|
|
|
&& (int) $pendingArtifact->size_bytes === 0
|
|
|
|
|
&& (string) $pendingArtifact->sha256 === ''
|
|
|
|
|
&& (string) $pendingPublication->state === 'failed'
|
|
|
|
|
&& (string) $pendingCopy->phase === 'terminal'
|
|
|
|
|
&& (string) $pendingCopy->effect_resolution === 'not_applied',
|
|
|
|
|
'Failed pending publication violated the blank-identity state contract.');
|
|
|
|
|
|
|
|
|
|
// Copy settlement must never overwrite a deletion projection or deletion
|
|
|
|
|
// worker lease, even if an older accepted copy operation remains claimable.
|
|
|
|
|
$deletedView = CloudStorageArtifactService::beginPublication(
|
|
|
|
|
'backup', 'backups/deleted-copy/system.qcow2', 4096,
|
|
|
|
|
str_repeat('6', 64), 'backup-publication-deleted-copy',
|
|
|
|
|
hash('sha256', 'deleted-copy'), ['backup_id' => 10]
|
|
|
|
|
);
|
|
|
|
|
CloudStorageArtifactService::markPrimaryDispatching(
|
|
|
|
|
$deletedView['operation_id']
|
|
|
|
|
);
|
|
|
|
|
CloudStorageArtifactService::completePrimary(
|
|
|
|
|
$deletedView['operation_id'], (int) $deletedView['artifact_id'], [
|
|
|
|
|
'size_bytes' => 4096, 'sha256' => str_repeat('6', 64),
|
|
|
|
|
'etag' => '"deleted-copy-primary"', 'version_id' => '',
|
|
|
|
|
]);
|
|
|
|
|
$deletedArtifact = CloudStorageArtifact::whereKey(
|
|
|
|
|
$deletedView['artifact_id']
|
|
|
|
|
)->firstOrFail();
|
|
|
|
|
$deletedCopy = CloudStorageOperation::where(
|
|
|
|
|
'artifact_id', $deletedArtifact->id
|
|
|
|
|
)->where('kind', 'copy')->firstOrFail();
|
|
|
|
|
$deletedReplica = CloudStorageReplica::whereKey(
|
|
|
|
|
(int) $deletedCopy->replica_id
|
|
|
|
|
)->firstOrFail();
|
|
|
|
|
$deletedArtifact->state = 'deleted';
|
|
|
|
|
$deletedArtifact->save();
|
|
|
|
|
$deletedReplica->state = 'deleted';
|
|
|
|
|
$deletedReplica->lease_token = str_repeat('d', 32);
|
|
|
|
|
$deletedReplica->lease_until = time() + 3600;
|
|
|
|
|
$deletedReplica->error_code = 'delete_projection_sentinel';
|
|
|
|
|
$deletedReplica->save();
|
|
|
|
|
$assert(CloudStorageArtifactService::claimReplicaWork(time()) === null,
|
|
|
|
|
'Deleted artifact unexpectedly returned copy work.');
|
|
|
|
|
$deletedCopy->refresh();
|
|
|
|
|
$deletedReplica->refresh();
|
|
|
|
|
$deletedArtifact->refresh();
|
|
|
|
|
$assert((string) $deletedCopy->phase === 'terminal'
|
|
|
|
|
&& (string) $deletedCopy->effect_resolution === 'not_applied'
|
|
|
|
|
&& (string) $deletedArtifact->state === 'deleted'
|
|
|
|
|
&& (string) $deletedReplica->state === 'deleted'
|
|
|
|
|
&& (string) $deletedReplica->lease_token === str_repeat('d', 32)
|
|
|
|
|
&& (string) $deletedReplica->error_code
|
|
|
|
|
=== 'delete_projection_sentinel',
|
|
|
|
|
'Copy fence overwrote an existing deletion projection.');
|
|
|
|
|
|
|
|
|
|
// A receipt must be bound to the caller's frozen artifact before any row is
|
|
|
|
|
// changed, even when two artifacts intentionally share the same bytes.
|
|
|
|
|
$crossA = CloudStorageArtifactService::beginPublication(
|
|
|
|
|
'backup', 'backups/cross-a/system.qcow2', 4096,
|
|
|
|
|
str_repeat('7', 64), 'backup-publication-cross-a',
|
|
|
|
|
hash('sha256', 'cross-a'), ['backup_id' => 110]
|
|
|
|
|
);
|
|
|
|
|
$crossB = CloudStorageArtifactService::beginPublication(
|
|
|
|
|
'backup', 'backups/cross-b/system.qcow2', 4096,
|
|
|
|
|
str_repeat('7', 64), 'backup-publication-cross-b',
|
|
|
|
|
hash('sha256', 'cross-b'), ['backup_id' => 111]
|
|
|
|
|
);
|
|
|
|
|
CloudStorageArtifactService::markPrimaryDispatching(
|
|
|
|
|
$crossA['operation_id']
|
|
|
|
|
);
|
|
|
|
|
CloudStorageArtifactService::markPrimaryDispatching(
|
|
|
|
|
$crossB['operation_id']
|
|
|
|
|
);
|
|
|
|
|
$expect(static fn () => CloudStorageArtifactService::completePrimary(
|
|
|
|
|
$crossA['operation_id'], (int) $crossB['artifact_id'], [
|
|
|
|
|
'size_bytes' => 4096, 'sha256' => str_repeat('7', 64),
|
|
|
|
|
'etag' => '"cross-a"', 'version_id' => '',
|
|
|
|
|
]
|
|
|
|
|
), 409);
|
|
|
|
|
$crossAOperation = CloudStorageOperation::where(
|
|
|
|
|
'operation_id', $crossA['operation_id']
|
|
|
|
|
)->firstOrFail();
|
|
|
|
|
$crossAPrimary = CloudStorageReplica::whereKey(
|
|
|
|
|
(int) $crossAOperation->replica_id
|
|
|
|
|
)->firstOrFail();
|
|
|
|
|
$crossAArtifact = CloudStorageArtifact::whereKey(
|
|
|
|
|
(int) $crossA['artifact_id']
|
|
|
|
|
)->firstOrFail();
|
|
|
|
|
$crossBOperation = CloudStorageOperation::where(
|
|
|
|
|
'operation_id', $crossB['operation_id']
|
|
|
|
|
)->firstOrFail();
|
|
|
|
|
$crossBPrimary = CloudStorageReplica::whereKey(
|
|
|
|
|
(int) $crossBOperation->replica_id
|
|
|
|
|
)->firstOrFail();
|
|
|
|
|
$crossBArtifact = CloudStorageArtifact::whereKey(
|
|
|
|
|
(int) $crossB['artifact_id']
|
|
|
|
|
)->firstOrFail();
|
|
|
|
|
$assert((string) $crossAOperation->phase === 'dispatching'
|
|
|
|
|
&& $crossAOperation->outcome === null
|
|
|
|
|
&& (string) $crossAPrimary->state === 'publishing'
|
|
|
|
|
&& (int) $crossAPrimary->size_bytes === 0
|
|
|
|
|
&& (string) $crossAArtifact->state === 'publishing'
|
|
|
|
|
&& (string) $crossBOperation->phase === 'dispatching'
|
|
|
|
|
&& $crossBOperation->outcome === null
|
|
|
|
|
&& (string) $crossBPrimary->state === 'publishing'
|
|
|
|
|
&& (int) $crossBPrimary->size_bytes === 0
|
|
|
|
|
&& (string) $crossBArtifact->state === 'publishing',
|
|
|
|
|
'Cross-artifact primary receipt changed one of the two ledgers.');
|
|
|
|
|
|
|
|
|
|
// A required-two artifact must cross the initial publication gate with both
|
|
|
|
|
// replicas. Once the owner has finalized it, later loss of one replica is a
|
|
|
|
|
// degraded-but-readable repair condition and must not reopen the outbox.
|
|
|
|
|
$runtimeReadView = CloudStorageArtifactService::beginPublication(
|
|
|
|
|
'template',
|
|
|
|
|
'cloud-templates/runtime-read/system.qcow2',
|
|
|
|
|
4096,
|
|
|
|
|
str_repeat('6', 64),
|
|
|
|
|
'template-runtime-read-policy',
|
|
|
|
|
hash('sha256', 'template-runtime-read-policy'),
|
|
|
|
|
['template_import_intent_id' => str_repeat('6', 32)]
|
|
|
|
|
);
|
|
|
|
|
CloudStorageArtifactService::markPrimaryDispatching(
|
|
|
|
|
$runtimeReadView['operation_id']
|
|
|
|
|
);
|
|
|
|
|
CloudStorageArtifactService::completePrimary(
|
|
|
|
|
$runtimeReadView['operation_id'],
|
|
|
|
|
(int) $runtimeReadView['artifact_id'],
|
|
|
|
|
[
|
|
|
|
|
'size_bytes' => 4096,
|
|
|
|
|
'sha256' => str_repeat('6', 64),
|
|
|
|
|
'etag' => '"runtime-primary"',
|
|
|
|
|
'version_id' => '',
|
|
|
|
|
]
|
|
|
|
|
);
|
|
|
|
|
$runtimePublication = CloudStoragePublication::where(
|
|
|
|
|
'artifact_id', (int) $runtimeReadView['artifact_id']
|
|
|
|
|
)->firstOrFail();
|
|
|
|
|
$assert((string) $runtimePublication->state === 'waiting_protection',
|
|
|
|
|
'One template replica prematurely released the required-two publication.');
|
|
|
|
|
$runtimeArtifact = CloudStorageArtifact::whereKey(
|
|
|
|
|
(int) $runtimeReadView['artifact_id']
|
|
|
|
|
)->firstOrFail();
|
|
|
|
|
$runtimeReplicas = CloudStorageReplica::where(
|
|
|
|
|
'artifact_id', (int) $runtimeArtifact->id
|
|
|
|
|
)->orderBy('id')->get();
|
|
|
|
|
$runtimeSecondary = $runtimeReplicas->firstWhere('role', 'secondary');
|
|
|
|
|
$assert($runtimeSecondary instanceof CloudStorageReplica,
|
|
|
|
|
'Required-two fixture is missing its secondary replica.');
|
|
|
|
|
$runtimeSecondary->state = 'ready';
|
|
|
|
|
$runtimeSecondary->size_bytes = 4096;
|
|
|
|
|
$runtimeSecondary->sha256 = str_repeat('6', 64);
|
|
|
|
|
$runtimeSecondary->object_etag = '"runtime-secondary"';
|
|
|
|
|
$runtimeSecondary->verified_at = time();
|
|
|
|
|
$runtimeSecondary->error_code = '';
|
|
|
|
|
$runtimeSecondary->save();
|
|
|
|
|
$project = (new ReflectionClass(CloudStorageArtifactService::class))
|
|
|
|
|
->getMethod('projectArtifactLocked');
|
|
|
|
|
$project->invoke(
|
|
|
|
|
null,
|
|
|
|
|
$runtimeArtifact,
|
|
|
|
|
$runtimeReplicas->all(),
|
|
|
|
|
time()
|
|
|
|
|
);
|
|
|
|
|
$runtimePublication->refresh();
|
|
|
|
|
$assert((string) $runtimePublication->state === 'ready_to_finalize',
|
|
|
|
|
'Two verified template replicas did not release publication.');
|
|
|
|
|
CloudStoragePublicationFinalizerService::complete(
|
|
|
|
|
'template',
|
|
|
|
|
str_repeat('6', 32),
|
|
|
|
|
(int) $runtimeArtifact->id,
|
|
|
|
|
(string) $runtimeReadView['operation_id']
|
|
|
|
|
);
|
|
|
|
|
$runtimeSecondary->state = 'degraded';
|
|
|
|
|
$runtimeSecondary->verified_at = 0;
|
|
|
|
|
$runtimeSecondary->error_code = 'replica_unavailable';
|
|
|
|
|
$runtimeSecondary->save();
|
|
|
|
|
$runtimeArtifact->refresh();
|
|
|
|
|
$runtimeReplicas = CloudStorageReplica::where(
|
|
|
|
|
'artifact_id', (int) $runtimeArtifact->id
|
|
|
|
|
)->orderBy('id')->get();
|
|
|
|
|
$project->invoke(
|
|
|
|
|
null,
|
|
|
|
|
$runtimeArtifact,
|
|
|
|
|
$runtimeReplicas->all(),
|
|
|
|
|
time()
|
|
|
|
|
);
|
|
|
|
|
$runtimeArtifact->refresh();
|
|
|
|
|
$runtimePublication->refresh();
|
|
|
|
|
$runtimeProjection = CloudStorageDurabilityPolicy::project(
|
|
|
|
|
[
|
|
|
|
|
'state' => (string) $runtimeArtifact->state,
|
|
|
|
|
'size_bytes' => (int) $runtimeArtifact->size_bytes,
|
|
|
|
|
'sha256' => (string) $runtimeArtifact->sha256,
|
|
|
|
|
'desired_copies' => (int) $runtimeArtifact->desired_copies,
|
|
|
|
|
'required_copies' => (int) $runtimeArtifact->required_copies,
|
|
|
|
|
],
|
|
|
|
|
$runtimeReplicas->map(static fn (CloudStorageReplica $replica): array => [
|
|
|
|
|
'backend_id' => (int) $replica->backend_id,
|
|
|
|
|
'state' => (string) $replica->state,
|
|
|
|
|
'size_bytes' => (int) $replica->size_bytes,
|
|
|
|
|
'sha256' => (string) $replica->sha256,
|
|
|
|
|
'verified_at' => (int) $replica->verified_at,
|
|
|
|
|
])->values()->all()
|
|
|
|
|
);
|
|
|
|
|
$assert((string) $runtimeArtifact->state === 'degraded'
|
|
|
|
|
&& $runtimeProjection['can_read']
|
|
|
|
|
&& !$runtimeProjection['meets_publication_requirement']
|
|
|
|
|
&& (string) $runtimePublication->state === 'completed'
|
|
|
|
|
&& (int) $runtimePublication->finalized_at > 0,
|
|
|
|
|
'Replica degradation made a published template unreadable or reopened its outbox.');
|
|
|
|
|
|
|
|
|
|
CloudStorageBackend::whereKey(2)->update([
|
|
|
|
|
'admin_state' => 'disabled', 'health_state' => 'unreachable',
|
|
|
|
|
'tested_revision' => 0,
|
|
|
|
|
]);
|
|
|
|
|
$artifactCountBeforeRejectedPublication = CloudStorageArtifact::count();
|
|
|
|
|
$expect(static fn () => CloudStorageArtifactService::beginPublication(
|
|
|
|
|
'template', 'cloud-templates/t/g/system.qcow2', 8192,
|
|
|
|
|
str_repeat('b', 64), 'template-publication-0001', hash('sha256', 't'),
|
|
|
|
|
['template_import_intent_id' => str_repeat('d', 32)]
|
|
|
|
|
), 503);
|
|
|
|
|
$assert(CloudStorageArtifact::count()
|
|
|
|
|
=== $artifactCountBeforeRejectedPublication,
|
|
|
|
|
'Required-two preflight left a partial template artifact.');
|
|
|
|
|
|
|
|
|
|
echo "Storage artifact service tests passed.\n";
|