|
|
|
|
<?php
|
|
|
|
|
|
|
|
|
|
require dirname(__DIR__) . '/vendor/autoload.php';
|
|
|
|
|
|
|
|
|
|
use App\Models\CloudStorageArtifact;
|
|
|
|
|
use App\Models\CloudStorageOperation;
|
|
|
|
|
use App\Models\CloudStoragePublication;
|
|
|
|
|
use App\Models\CloudStorageReplica;
|
|
|
|
|
use App\Services\Cloud\CloudError;
|
|
|
|
|
use App\Services\Cloud\CloudObjectStorageS3Client;
|
|
|
|
|
use App\Services\Cloud\CloudStorageArtifactService;
|
|
|
|
|
use App\Services\Cloud\CloudStorageImmutablePrimaryPublication;
|
|
|
|
|
use App\Services\Cloud\CloudStorageReplicaAttentionService;
|
|
|
|
|
use GuzzleHttp\Client;
|
|
|
|
|
use GuzzleHttp\HandlerStack;
|
|
|
|
|
use GuzzleHttp\Promise\Create;
|
|
|
|
|
use GuzzleHttp\Psr7\Response;
|
|
|
|
|
use Illuminate\Database\Capsule\Manager as Capsule;
|
|
|
|
|
use Illuminate\Database\Schema\Blueprint;
|
|
|
|
|
use Psr\Http\Message\RequestInterface;
|
|
|
|
|
|
|
|
|
|
$assert = static function (bool $condition, string $message): void {
|
|
|
|
|
if (!$condition) {
|
|
|
|
|
throw new RuntimeException($message);
|
|
|
|
|
}
|
|
|
|
|
};
|
|
|
|
|
$expectCloudError = static function (
|
|
|
|
|
callable $callback,
|
|
|
|
|
int $status
|
|
|
|
|
) use ($assert): void {
|
|
|
|
|
try {
|
|
|
|
|
$callback();
|
|
|
|
|
} catch (CloudError $error) {
|
|
|
|
|
$assert($error->httpStatus === $status,
|
|
|
|
|
'Unexpected CloudError status: ' . $error->httpStatus);
|
|
|
|
|
return;
|
|
|
|
|
}
|
|
|
|
|
throw new RuntimeException('Expected CloudError was not raised.');
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
$db = new Capsule();
|
|
|
|
|
$db->addConnection(['driver' => 'sqlite', 'database' => ':memory:']);
|
|
|
|
|
$db->setAsGlobal();
|
|
|
|
|
$db->bootEloquent();
|
|
|
|
|
$db->schema()->create('cloud_storage_backends', static function (
|
|
|
|
|
Blueprint $table
|
|
|
|
|
): void {
|
|
|
|
|
$table->increments('id');
|
|
|
|
|
$table->string('backend_key');
|
|
|
|
|
$table->string('identity_sha256');
|
|
|
|
|
$table->string('name');
|
|
|
|
|
$table->string('driver');
|
|
|
|
|
$table->string('endpoint');
|
|
|
|
|
$table->string('region');
|
|
|
|
|
$table->string('bucket');
|
|
|
|
|
$table->string('prefix');
|
|
|
|
|
$table->boolean('path_style');
|
|
|
|
|
$table->string('failure_domain');
|
|
|
|
|
$table->text('sealed_credentials');
|
|
|
|
|
$table->unsignedBigInteger('config_revision');
|
|
|
|
|
$table->unsignedBigInteger('tested_revision');
|
|
|
|
|
$table->string('admin_state');
|
|
|
|
|
$table->string('health_state');
|
|
|
|
|
$table->unsignedInteger('write_priority');
|
|
|
|
|
$table->unsignedInteger('consecutive_failures')->default(0);
|
|
|
|
|
$table->unsignedInteger('consecutive_successes')->default(0);
|
|
|
|
|
$table->string('last_error_code')->default('');
|
|
|
|
|
$table->unsignedBigInteger('last_tested_at')->default(0);
|
|
|
|
|
$table->unsignedBigInteger('last_healthy_at')->default(0);
|
|
|
|
|
$table->unsignedBigInteger('next_probe_at')->default(0);
|
|
|
|
|
$table->unsignedBigInteger('created_at');
|
|
|
|
|
$table->unsignedBigInteger('updated_at');
|
|
|
|
|
});
|
|
|
|
|
$db->schema()->create('cloud_storage_artifacts', static function (
|
|
|
|
|
Blueprint $table
|
|
|
|
|
): void {
|
|
|
|
|
$table->increments('id');
|
|
|
|
|
$table->string('artifact_key')->unique();
|
|
|
|
|
$table->string('kind');
|
|
|
|
|
$table->unsignedBigInteger('maximum_size_bytes');
|
|
|
|
|
$table->unsignedBigInteger('size_bytes');
|
|
|
|
|
$table->string('sha256');
|
|
|
|
|
$table->string('durability_policy');
|
|
|
|
|
$table->unsignedTinyInteger('desired_copies');
|
|
|
|
|
$table->unsignedTinyInteger('required_copies');
|
|
|
|
|
$table->string('state');
|
|
|
|
|
$table->string('last_operation_id')->nullable();
|
|
|
|
|
$table->unsignedBigInteger('verified_at');
|
|
|
|
|
$table->unsignedBigInteger('created_at');
|
|
|
|
|
$table->unsignedBigInteger('updated_at');
|
|
|
|
|
});
|
|
|
|
|
$db->schema()->create('cloud_storage_replicas', static function (
|
|
|
|
|
Blueprint $table
|
|
|
|
|
): void {
|
|
|
|
|
$table->increments('id');
|
|
|
|
|
$table->string('replica_key')->unique();
|
|
|
|
|
$table->unsignedBigInteger('artifact_id');
|
|
|
|
|
$table->unsignedBigInteger('backend_id');
|
|
|
|
|
$table->string('role');
|
|
|
|
|
$table->string('object_key');
|
|
|
|
|
$table->string('object_version_id')->default('');
|
|
|
|
|
$table->string('object_etag')->default('');
|
|
|
|
|
$table->unsignedBigInteger('size_bytes')->default(0);
|
|
|
|
|
$table->string('sha256')->default('');
|
|
|
|
|
$table->string('state');
|
|
|
|
|
$table->string('publish_operation_id')->nullable();
|
|
|
|
|
$table->string('delete_operation_id')->nullable();
|
|
|
|
|
$table->unsignedInteger('delete_generation')->default(0);
|
|
|
|
|
$table->unsignedInteger('delete_cycle_attempts')->default(0);
|
|
|
|
|
$table->unsignedInteger('delete_presence_confirmations')->default(0);
|
|
|
|
|
$table->unsignedBigInteger('delete_last_checked_at')->default(0);
|
|
|
|
|
$table->unsignedBigInteger('delete_next_check_at')->default(0);
|
|
|
|
|
$table->unsignedBigInteger('delete_attention_at')->nullable();
|
|
|
|
|
$table->string('lease_token')->nullable();
|
|
|
|
|
$table->unsignedBigInteger('lease_until')->default(0);
|
|
|
|
|
$table->string('error_code')->default('');
|
|
|
|
|
$table->unsignedBigInteger('verified_at')->default(0);
|
|
|
|
|
$table->unsignedBigInteger('created_at');
|
|
|
|
|
$table->unsignedBigInteger('updated_at');
|
|
|
|
|
});
|
|
|
|
|
$db->schema()->create('cloud_storage_operations', static function (
|
|
|
|
|
Blueprint $table
|
|
|
|
|
): void {
|
|
|
|
|
$table->increments('id');
|
|
|
|
|
$table->string('operation_id')->unique();
|
|
|
|
|
$table->string('request_idempotency_key')->unique();
|
|
|
|
|
$table->string('request_fingerprint');
|
|
|
|
|
$table->unsignedBigInteger('artifact_id');
|
|
|
|
|
$table->unsignedBigInteger('replica_id')->nullable();
|
|
|
|
|
$table->unsignedBigInteger('source_backend_id')->nullable();
|
|
|
|
|
$table->unsignedBigInteger('target_backend_id')->nullable();
|
|
|
|
|
$table->string('kind');
|
|
|
|
|
$table->string('phase');
|
|
|
|
|
$table->string('outcome')->nullable();
|
|
|
|
|
$table->string('effect_resolution');
|
|
|
|
|
$table->boolean('effect_may_have_applied')->nullable();
|
|
|
|
|
$table->boolean('safe_to_resubmit')->nullable();
|
|
|
|
|
$table->unsignedInteger('attempts');
|
|
|
|
|
$table->string('lease_token')->nullable();
|
|
|
|
|
$table->unsignedBigInteger('lease_until');
|
|
|
|
|
$table->unsignedBigInteger('available_at');
|
|
|
|
|
$table->unsignedBigInteger('next_check_at');
|
|
|
|
|
$table->string('error_code');
|
|
|
|
|
$table->text('result_json')->nullable();
|
|
|
|
|
$table->unsignedBigInteger('terminal_at')->nullable();
|
|
|
|
|
$table->unsignedBigInteger('created_at');
|
|
|
|
|
$table->unsignedBigInteger('updated_at');
|
|
|
|
|
});
|
|
|
|
|
$db->schema()->create('cloud_storage_publications', static function (
|
|
|
|
|
Blueprint $table
|
|
|
|
|
): void {
|
|
|
|
|
$table->increments('id');
|
|
|
|
|
$table->string('publication_key')->unique();
|
|
|
|
|
$table->unsignedBigInteger('artifact_id')->unique();
|
|
|
|
|
$table->unsignedBigInteger('backup_id')->nullable()->unique();
|
|
|
|
|
$table->string('template_import_intent_id')->nullable()->unique();
|
|
|
|
|
$table->string('rescue_template_id')->nullable()->unique();
|
|
|
|
|
$table->string('state');
|
|
|
|
|
$table->unsignedInteger('attempts')->default(0);
|
|
|
|
|
$table->unsignedBigInteger('available_at')->default(0);
|
|
|
|
|
$table->string('lease_token')->nullable();
|
|
|
|
|
$table->unsignedBigInteger('lease_until')->default(0);
|
|
|
|
|
$table->string('last_error_code')->default('');
|
|
|
|
|
$table->unsignedBigInteger('coverage_ready_at')->nullable();
|
|
|
|
|
$table->unsignedBigInteger('finalized_at')->nullable();
|
|
|
|
|
$table->unsignedBigInteger('created_at');
|
|
|
|
|
$table->unsignedBigInteger('updated_at');
|
|
|
|
|
});
|
|
|
|
|
|
|
|
|
|
$databaseNow = time();
|
|
|
|
|
foreach ([
|
|
|
|
|
[1, 'primary', 'provider-a', 10],
|
|
|
|
|
[2, 'secondary', 'provider-b', 20],
|
|
|
|
|
] as [$id, $name, $domain, $priority]) {
|
|
|
|
|
Capsule::table('cloud_storage_backends')->insert([
|
|
|
|
|
'id' => $id,
|
|
|
|
|
'backend_key' => str_repeat((string) $id, 32),
|
|
|
|
|
'identity_sha256' => str_repeat((string) $id, 64),
|
|
|
|
|
'name' => $name,
|
|
|
|
|
'driver' => 's3',
|
|
|
|
|
'endpoint' => 'https://' . $name . '.example.test',
|
|
|
|
|
'region' => 'us-east-1',
|
|
|
|
|
'bucket' => $name . '-bucket',
|
|
|
|
|
'prefix' => $name,
|
|
|
|
|
'path_style' => 1,
|
|
|
|
|
'failure_domain' => $domain,
|
|
|
|
|
'sealed_credentials' => 'sealed',
|
|
|
|
|
'config_revision' => 1,
|
|
|
|
|
'tested_revision' => 1,
|
|
|
|
|
'admin_state' => 'active',
|
|
|
|
|
'health_state' => 'healthy',
|
|
|
|
|
'write_priority' => $priority,
|
|
|
|
|
'consecutive_failures' => 0,
|
|
|
|
|
'consecutive_successes' => 1,
|
|
|
|
|
'last_error_code' => '',
|
|
|
|
|
'last_tested_at' => $databaseNow,
|
|
|
|
|
'last_healthy_at' => $databaseNow,
|
|
|
|
|
'next_probe_at' => $databaseNow + 300,
|
|
|
|
|
'created_at' => $databaseNow,
|
|
|
|
|
'updated_at' => $databaseNow,
|
|
|
|
|
]);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
$temporaryPaths = [];
|
|
|
|
|
register_shutdown_function(static function () use (&$temporaryPaths): void {
|
|
|
|
|
foreach ($temporaryPaths as $path) {
|
|
|
|
|
@unlink($path);
|
|
|
|
|
}
|
|
|
|
|
});
|
|
|
|
|
$fixtureSequence = 0;
|
|
|
|
|
$fixture = static function (
|
|
|
|
|
string $kind = 'template'
|
|
|
|
|
) use (&$fixtureSequence, &$temporaryPaths): array {
|
|
|
|
|
$fixtureSequence++;
|
|
|
|
|
$path = tempnam(sys_get_temp_dir(), 'virt-primary-publication-');
|
|
|
|
|
if ($path === false) {
|
|
|
|
|
throw new RuntimeException('Cannot create publication fixture.');
|
|
|
|
|
}
|
|
|
|
|
file_put_contents(
|
|
|
|
|
$path,
|
|
|
|
|
str_repeat('immutable-primary-' . $fixtureSequence . '-', 64)
|
|
|
|
|
);
|
|
|
|
|
$temporaryPaths[] = $path;
|
|
|
|
|
$size = filesize($path);
|
|
|
|
|
$sha256 = hash_file('sha256', $path);
|
|
|
|
|
if (!is_int($size) || !is_string($sha256)) {
|
|
|
|
|
throw new RuntimeException('Cannot identify publication fixture.');
|
|
|
|
|
}
|
|
|
|
|
$ownerId = str_pad(dechex($fixtureSequence), 32, '0', STR_PAD_LEFT);
|
|
|
|
|
$owner = $kind === 'rescue'
|
|
|
|
|
? ['rescue_template_id' => $ownerId]
|
|
|
|
|
: ['template_import_intent_id' => $ownerId];
|
|
|
|
|
$view = CloudStorageArtifactService::beginPublication(
|
|
|
|
|
$kind,
|
|
|
|
|
$kind . '/fixture-' . $fixtureSequence . '/system.qcow2',
|
|
|
|
|
$size,
|
|
|
|
|
$sha256,
|
|
|
|
|
'immutable-primary-' . str_pad(
|
|
|
|
|
(string) $fixtureSequence,
|
|
|
|
|
4,
|
|
|
|
|
'0',
|
|
|
|
|
STR_PAD_LEFT
|
|
|
|
|
),
|
|
|
|
|
hash('sha256', $kind . '|' . $fixtureSequence),
|
|
|
|
|
$owner
|
|
|
|
|
);
|
|
|
|
|
return [
|
|
|
|
|
'path' => $path,
|
|
|
|
|
'size_bytes' => $size,
|
|
|
|
|
'sha256' => $sha256,
|
|
|
|
|
'operation_id' => (string) $view['operation_id'],
|
|
|
|
|
'artifact_id' => (int) $view['artifact_id'],
|
|
|
|
|
'replica_id' => (int) $view['replica_id'],
|
|
|
|
|
'object_key' => (string) $view['object_key'],
|
|
|
|
|
];
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
$publisher = static function (
|
|
|
|
|
array &$responses,
|
|
|
|
|
array &$requests,
|
|
|
|
|
array &$transactionLevels,
|
|
|
|
|
int &$clock
|
|
|
|
|
): CloudStorageImmutablePrimaryPublication {
|
|
|
|
|
$handler = static function (
|
|
|
|
|
RequestInterface $request,
|
|
|
|
|
array $options
|
|
|
|
|
) use (&$responses, &$requests, &$transactionLevels) {
|
|
|
|
|
$transactionLevels[] = Capsule::connection()->transactionLevel();
|
|
|
|
|
$requests[] = $request;
|
|
|
|
|
if ($responses === []) {
|
|
|
|
|
throw new RuntimeException('Unexpected object-storage request.');
|
|
|
|
|
}
|
|
|
|
|
$response = array_shift($responses);
|
|
|
|
|
if (!$response instanceof Response) {
|
|
|
|
|
throw new RuntimeException('Invalid object-storage response fixture.');
|
|
|
|
|
}
|
|
|
|
|
return Create::promiseFor($response);
|
|
|
|
|
};
|
|
|
|
|
$client = new CloudObjectStorageS3Client(new Client([
|
|
|
|
|
'handler' => HandlerStack::create($handler),
|
|
|
|
|
]));
|
|
|
|
|
return new CloudStorageImmutablePrimaryPublication(
|
|
|
|
|
$client,
|
|
|
|
|
static fn (int $backendId, bool $healthy): array => [
|
|
|
|
|
'backend_id' => $backendId,
|
|
|
|
|
'revision' => 1,
|
|
|
|
|
'config' => [
|
|
|
|
|
'endpoint' => 'https://objects.example.test',
|
|
|
|
|
'region' => 'us-east-1',
|
|
|
|
|
'bucket' => 'immutable-fixtures',
|
|
|
|
|
'prefix' => 'unused-runtime-prefix',
|
|
|
|
|
'path_style' => true,
|
|
|
|
|
],
|
|
|
|
|
'credentials' => [
|
|
|
|
|
'access_key_id' => 'IMMUTABLEACCESSKEY',
|
|
|
|
|
'secret_access_key' => 'immutable-secret-must-not-persist',
|
|
|
|
|
],
|
|
|
|
|
],
|
|
|
|
|
static function () use (&$clock): int {
|
|
|
|
|
return $clock;
|
|
|
|
|
}
|
|
|
|
|
);
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
// A successful immutable PUT settles only the primary. Template publication
|
|
|
|
|
// remains behind the required-two protection gate until its copy completes.
|
|
|
|
|
$applied = $fixture('template');
|
|
|
|
|
$clock = time() + 10;
|
|
|
|
|
$responses = [new Response(200, [
|
|
|
|
|
'ETag' => '"applied-etag"',
|
|
|
|
|
'x-amz-version-id' => 'applied-version',
|
|
|
|
|
])];
|
|
|
|
|
$requests = [];
|
|
|
|
|
$transactionLevels = [];
|
|
|
|
|
$module = $publisher($responses, $requests, $transactionLevels, $clock);
|
|
|
|
|
$appliedResult = $module->publish(
|
|
|
|
|
$applied['operation_id'],
|
|
|
|
|
$applied['artifact_id'],
|
|
|
|
|
$applied
|
|
|
|
|
);
|
|
|
|
|
$assert($appliedResult['primary_completed'] === true
|
|
|
|
|
&& $appliedResult['publication_state'] === 'waiting_protection'
|
|
|
|
|
&& $appliedResult['ready_to_finalize'] === false,
|
|
|
|
|
'Primary success bypassed the template publication quorum.');
|
|
|
|
|
$assert(count($requests) === 1 && $requests[0]->getMethod() === 'PUT'
|
|
|
|
|
&& $requests[0]->getHeaderLine('If-None-Match') === '*'
|
|
|
|
|
&& $requests[0]->getHeaderLine('x-amz-meta-virt-sha256')
|
|
|
|
|
=== $applied['sha256']
|
|
|
|
|
&& str_contains(
|
|
|
|
|
$requests[0]->getHeaderLine('Authorization'),
|
|
|
|
|
'x-amz-meta-virt-sha256'
|
|
|
|
|
), 'Immutable primary PUT lost its conditional signed identity.');
|
|
|
|
|
$assert($transactionLevels === [0],
|
|
|
|
|
'Immutable primary PUT ran inside a database transaction.');
|
|
|
|
|
|
|
|
|
|
// A provider rejection proves non-application and schedules a bounded retry;
|
|
|
|
|
// it never recursively replays inside the same call.
|
|
|
|
|
$rejected = $fixture('rescue');
|
|
|
|
|
$clock += 1000;
|
|
|
|
|
$responses = [new Response(403)];
|
|
|
|
|
$requests = [];
|
|
|
|
|
$transactionLevels = [];
|
|
|
|
|
$module = $publisher($responses, $requests, $transactionLevels, $clock);
|
|
|
|
|
$rejectedResult = $module->publish(
|
|
|
|
|
$rejected['operation_id'],
|
|
|
|
|
$rejected['artifact_id'],
|
|
|
|
|
$rejected
|
|
|
|
|
);
|
|
|
|
|
$rejectedOperation = CloudStorageOperation::where(
|
|
|
|
|
'operation_id',
|
|
|
|
|
$rejected['operation_id']
|
|
|
|
|
)->firstOrFail();
|
|
|
|
|
$rejectedReplica = CloudStorageReplica::whereKey(
|
|
|
|
|
$rejected['replica_id']
|
|
|
|
|
)->firstOrFail();
|
|
|
|
|
$assert($rejectedResult['phase'] === 'accepted'
|
|
|
|
|
&& (int) $rejectedOperation->attempts === 1
|
|
|
|
|
&& $rejectedOperation->outcome === null
|
|
|
|
|
&& (int) $rejectedOperation->available_at > $clock
|
|
|
|
|
&& (string) $rejectedReplica->state === 'reserved'
|
|
|
|
|
&& count($requests) === 1,
|
|
|
|
|
'Proven non-application was not delayed without replay.');
|
|
|
|
|
|
|
|
|
|
// A lost PUT receipt is settled only by exact-key size plus signed SHA
|
|
|
|
|
// metadata. The temporary local file is deliberately removed before HEAD.
|
|
|
|
|
$unknownApplied = $fixture('template');
|
|
|
|
|
$clock += 1000;
|
|
|
|
|
$responses = [
|
|
|
|
|
new Response(503),
|
|
|
|
|
new Response(200, [
|
|
|
|
|
'Content-Length' => (string) $unknownApplied['size_bytes'],
|
|
|
|
|
'ETag' => '"unknown-applied-etag"',
|
|
|
|
|
'x-amz-version-id' => 'unknown-applied-version',
|
|
|
|
|
'x-amz-meta-virt-sha256' => $unknownApplied['sha256'],
|
|
|
|
|
]),
|
|
|
|
|
];
|
|
|
|
|
$requests = [];
|
|
|
|
|
$transactionLevels = [];
|
|
|
|
|
$module = $publisher($responses, $requests, $transactionLevels, $clock);
|
|
|
|
|
$unknownResult = $module->publish(
|
|
|
|
|
$unknownApplied['operation_id'],
|
|
|
|
|
$unknownApplied['artifact_id'],
|
|
|
|
|
$unknownApplied
|
|
|
|
|
);
|
|
|
|
|
$assert($unknownResult['phase'] === 'reconciling',
|
|
|
|
|
'Unknown PUT did not enter read-only reconciliation.');
|
|
|
|
|
@unlink($unknownApplied['path']);
|
|
|
|
|
$clock += 60;
|
|
|
|
|
$unknownResult = $module->publish(
|
|
|
|
|
$unknownApplied['operation_id'],
|
|
|
|
|
$unknownApplied['artifact_id'],
|
|
|
|
|
$unknownApplied
|
|
|
|
|
);
|
|
|
|
|
$assert($unknownResult['primary_completed'] === true
|
|
|
|
|
&& array_map(
|
|
|
|
|
static fn (RequestInterface $request): string => $request->getMethod(),
|
|
|
|
|
$requests
|
|
|
|
|
) === ['PUT', 'HEAD']
|
|
|
|
|
&& (string) $requests[0]->getUri() === (string) $requests[1]->getUri(),
|
|
|
|
|
'Unknown PUT was replayed or reconciled against a different key.');
|
|
|
|
|
$assert($transactionLevels === [0, 0],
|
|
|
|
|
'Immutable PUT reconciliation ran remote I/O inside a transaction.');
|
|
|
|
|
|
|
|
|
|
// Two spaced exact-key absences are required before the original operation,
|
|
|
|
|
// artifact and key may become dispatchable again.
|
|
|
|
|
$absent = $fixture('template');
|
|
|
|
|
$clock += 1000;
|
|
|
|
|
$responses = [
|
|
|
|
|
new Response(503),
|
|
|
|
|
new Response(404),
|
|
|
|
|
new Response(404),
|
|
|
|
|
new Response(200, [
|
|
|
|
|
'ETag' => '"retried-etag"',
|
|
|
|
|
'x-amz-version-id' => 'retried-version',
|
|
|
|
|
]),
|
|
|
|
|
];
|
|
|
|
|
$requests = [];
|
|
|
|
|
$transactionLevels = [];
|
|
|
|
|
$module = $publisher($responses, $requests, $transactionLevels, $clock);
|
|
|
|
|
$artifactCount = CloudStorageArtifact::count();
|
|
|
|
|
$operationCount = CloudStorageOperation::count();
|
|
|
|
|
$module->publish($absent['operation_id'], $absent['artifact_id'], $absent);
|
|
|
|
|
$clock += 60;
|
|
|
|
|
$firstAbsent = $module->publish(
|
|
|
|
|
$absent['operation_id'], $absent['artifact_id'], $absent
|
|
|
|
|
);
|
|
|
|
|
$assert($firstAbsent['phase'] === 'reconciling',
|
|
|
|
|
'One absence observation incorrectly authorized a retry.');
|
|
|
|
|
$clock += 60;
|
|
|
|
|
$secondAbsent = $module->publish(
|
|
|
|
|
$absent['operation_id'], $absent['artifact_id'], $absent
|
|
|
|
|
);
|
|
|
|
|
$assert($secondAbsent['phase'] === 'accepted',
|
|
|
|
|
'Two stable absence observations did not release the original intent.');
|
|
|
|
|
$clock += 60;
|
|
|
|
|
$retried = $module->publish(
|
|
|
|
|
$absent['operation_id'], $absent['artifact_id'], $absent
|
|
|
|
|
);
|
|
|
|
|
$methods = array_map(
|
|
|
|
|
static fn (RequestInterface $request): string => $request->getMethod(),
|
|
|
|
|
$requests
|
|
|
|
|
);
|
|
|
|
|
$assert($retried['primary_completed'] === true
|
|
|
|
|
&& $methods === ['PUT', 'HEAD', 'HEAD', 'PUT']
|
|
|
|
|
&& (string) $requests[0]->getUri() === (string) $requests[1]->getUri()
|
|
|
|
|
&& (string) $requests[0]->getUri() === (string) $requests[2]->getUri()
|
|
|
|
|
&& (string) $requests[0]->getUri() === (string) $requests[3]->getUri()
|
|
|
|
|
&& (int) CloudStorageOperation::where(
|
|
|
|
|
'operation_id', $absent['operation_id']
|
|
|
|
|
)->value('attempts') === 2
|
|
|
|
|
&& CloudStorageArtifact::count() === $artifactCount
|
|
|
|
|
&& CloudStorageOperation::count() === $operationCount,
|
|
|
|
|
'Stable absence did not retry the same frozen publication intent.');
|
|
|
|
|
|
|
|
|
|
// Existing content with a different signed identity is an attention state,
|
|
|
|
|
// never evidence for success and never permission to overwrite the key.
|
|
|
|
|
$conflict = $fixture('rescue');
|
|
|
|
|
$clock += 1000;
|
|
|
|
|
$responses = [
|
|
|
|
|
new Response(503),
|
|
|
|
|
new Response(200, [
|
|
|
|
|
'Content-Length' => (string) $conflict['size_bytes'],
|
|
|
|
|
'ETag' => '"conflicting-etag"',
|
|
|
|
|
'x-amz-version-id' => 'conflicting-version',
|
|
|
|
|
'x-amz-meta-virt-sha256' => str_repeat('f', 64),
|
|
|
|
|
]),
|
|
|
|
|
];
|
|
|
|
|
$requests = [];
|
|
|
|
|
$transactionLevels = [];
|
|
|
|
|
$module = $publisher($responses, $requests, $transactionLevels, $clock);
|
|
|
|
|
$module->publish($conflict['operation_id'], $conflict['artifact_id'], $conflict);
|
|
|
|
|
$clock += 60;
|
|
|
|
|
$conflictResult = $module->publish(
|
|
|
|
|
$conflict['operation_id'], $conflict['artifact_id'], $conflict
|
|
|
|
|
);
|
|
|
|
|
$conflictOperation = CloudStorageOperation::where(
|
|
|
|
|
'operation_id', $conflict['operation_id']
|
|
|
|
|
)->firstOrFail();
|
|
|
|
|
$assert($conflictResult['requires_attention'] === true
|
|
|
|
|
&& (string) $conflictOperation->phase === 'terminal'
|
|
|
|
|
&& (string) $conflictOperation->outcome === 'failed'
|
|
|
|
|
&& (string) $conflictOperation->effect_resolution === 'unknown'
|
|
|
|
|
&& (bool) $conflictOperation->safe_to_resubmit === false
|
|
|
|
|
&& (string) CloudStorageArtifact::whereKey(
|
|
|
|
|
$conflict['artifact_id']
|
|
|
|
|
)->value('state') === 'degraded'
|
|
|
|
|
&& (string) CloudStoragePublication::where(
|
|
|
|
|
'artifact_id', $conflict['artifact_id']
|
|
|
|
|
)->value('state') === 'failed'
|
|
|
|
|
&& count($requests) === 2,
|
|
|
|
|
'Conflicting immutable identity did not stop in operator attention.');
|
|
|
|
|
|
|
|
|
|
// An administrator may reopen only the same unknown-effect operation for
|
|
|
|
|
// read-only HEAD reconciliation. Secondary work stopped by the primary fence
|
|
|
|
|
// is restored but remains blocked until that exact primary succeeds.
|
|
|
|
|
$artifactCountBeforeReopen = CloudStorageArtifact::count();
|
|
|
|
|
$operationCountBeforeReopen = CloudStorageOperation::count();
|
|
|
|
|
$clock += 1;
|
|
|
|
|
$attention = CloudStorageReplicaAttentionService::listView();
|
|
|
|
|
$attentionItem = collect($attention['items'])->firstWhere(
|
|
|
|
|
'operation_id',
|
|
|
|
|
$conflict['operation_id']
|
|
|
|
|
);
|
|
|
|
|
$assert(is_array($attentionItem)
|
|
|
|
|
&& ($attentionItem['operation_type'] ?? null)
|
|
|
|
|
=== '不可变主副本发布'
|
|
|
|
|
&& ($attentionItem['owner_type'] ?? null) === 'rescue'
|
|
|
|
|
&& ($attentionItem['can_retry'] ?? false) === true,
|
|
|
|
|
'Immutable primary attention was not exposed to administrators.');
|
|
|
|
|
$attentionResult = CloudStorageReplicaAttentionService::retry(
|
|
|
|
|
$conflict['operation_id'],
|
|
|
|
|
'publish'
|
|
|
|
|
);
|
|
|
|
|
$assert(($attentionResult['phase'] ?? null) === 'reconciling',
|
|
|
|
|
'Immutable primary attention did not reopen through the admin service.');
|
|
|
|
|
$reopenedConflict = CloudStorageOperation::where(
|
|
|
|
|
'operation_id', $conflict['operation_id']
|
|
|
|
|
)->firstOrFail();
|
|
|
|
|
$reopenedCopy = CloudStorageOperation::where(
|
|
|
|
|
'artifact_id', $conflict['artifact_id']
|
|
|
|
|
)->where('kind', 'copy')->firstOrFail();
|
|
|
|
|
$reopenedPrimary = CloudStorageReplica::whereKey(
|
|
|
|
|
$conflict['replica_id']
|
|
|
|
|
)->firstOrFail();
|
|
|
|
|
$reopenedSecondary = CloudStorageReplica::whereKey(
|
|
|
|
|
(int) $reopenedCopy->replica_id
|
|
|
|
|
)->firstOrFail();
|
|
|
|
|
$assert((string) $reopenedConflict->phase === 'reconciling'
|
|
|
|
|
&& (string) $reopenedConflict->effect_resolution === 'unknown'
|
|
|
|
|
&& $reopenedConflict->outcome === null
|
|
|
|
|
&& (string) $reopenedPrimary->state === 'publishing'
|
|
|
|
|
&& (string) $reopenedCopy->phase === 'accepted'
|
|
|
|
|
&& (int) $reopenedCopy->available_at === 253402300799
|
|
|
|
|
&& (string) $reopenedSecondary->state === 'reserved'
|
|
|
|
|
&& (string) CloudStorageArtifact::whereKey(
|
|
|
|
|
$conflict['artifact_id']
|
|
|
|
|
)->value('state') === 'publishing'
|
|
|
|
|
&& (string) CloudStoragePublication::where(
|
|
|
|
|
'artifact_id', $conflict['artifact_id']
|
|
|
|
|
)->value('state') === 'waiting_primary'
|
|
|
|
|
&& CloudStorageArtifact::count() === $artifactCountBeforeReopen
|
|
|
|
|
&& CloudStorageOperation::count() === $operationCountBeforeReopen,
|
|
|
|
|
'Administrator reconciliation did not reopen the original fenced ledger.');
|
|
|
|
|
|
|
|
|
|
$responses = [new Response(404), new Response(404)];
|
|
|
|
|
$requests = [];
|
|
|
|
|
$transactionLevels = [];
|
|
|
|
|
$module = $publisher($responses, $requests, $transactionLevels, $clock);
|
|
|
|
|
$module->publish(
|
|
|
|
|
$conflict['operation_id'], $conflict['artifact_id'], $conflict
|
|
|
|
|
);
|
|
|
|
|
$clock += 60;
|
|
|
|
|
$reconciledAbsence = $module->publish(
|
|
|
|
|
$conflict['operation_id'], $conflict['artifact_id'], $conflict
|
|
|
|
|
);
|
|
|
|
|
$assert($reconciledAbsence['phase'] === 'accepted'
|
|
|
|
|
&& array_map(
|
|
|
|
|
static fn (RequestInterface $request): string => $request->getMethod(),
|
|
|
|
|
$requests
|
|
|
|
|
) === ['HEAD', 'HEAD'],
|
|
|
|
|
'Administrator recovery replayed a write before stable absence evidence.');
|
|
|
|
|
|
|
|
|
|
// Repeated proven non-application also stops for operator review. Reopening
|
|
|
|
|
// authorizes one new dispatch of the same operation and immutable key only.
|
|
|
|
|
$manualRetry = $fixture('template');
|
|
|
|
|
$clock += 1000;
|
|
|
|
|
$responses = [new Response(403), new Response(403), new Response(403)];
|
|
|
|
|
$requests = [];
|
|
|
|
|
$transactionLevels = [];
|
|
|
|
|
$module = $publisher($responses, $requests, $transactionLevels, $clock);
|
|
|
|
|
for ($attempt = 0; $attempt < 3; $attempt++) {
|
|
|
|
|
$module->publish(
|
|
|
|
|
$manualRetry['operation_id'],
|
|
|
|
|
$manualRetry['artifact_id'],
|
|
|
|
|
$manualRetry
|
|
|
|
|
);
|
|
|
|
|
$clock += 300;
|
|
|
|
|
}
|
|
|
|
|
$manualRetryOperation = CloudStorageOperation::where(
|
|
|
|
|
'operation_id', $manualRetry['operation_id']
|
|
|
|
|
)->firstOrFail();
|
|
|
|
|
$assert((string) $manualRetryOperation->phase === 'terminal'
|
|
|
|
|
&& (string) $manualRetryOperation->effect_resolution === 'not_applied'
|
|
|
|
|
&& (string) $manualRetryOperation->error_code
|
|
|
|
|
=== 'immutable_primary_publish_retry_exhausted',
|
|
|
|
|
'Bounded proven failures did not enter manual retry attention.');
|
|
|
|
|
$manualAttention = CloudStorageReplicaAttentionService::listView();
|
|
|
|
|
$manualAttentionItem = collect($manualAttention['items'])->firstWhere(
|
|
|
|
|
'operation_id',
|
|
|
|
|
$manualRetry['operation_id']
|
|
|
|
|
);
|
|
|
|
|
$assert(is_array($manualAttentionItem)
|
|
|
|
|
&& ($manualAttentionItem['can_retry'] ?? false) === true,
|
|
|
|
|
'Proven-not-applied primary attention was not exposed for retry.');
|
|
|
|
|
CloudStorageReplicaAttentionService::retry(
|
|
|
|
|
$manualRetry['operation_id'],
|
|
|
|
|
'publish'
|
|
|
|
|
);
|
|
|
|
|
$responses = [new Response(200, [
|
|
|
|
|
'ETag' => '"manual-retry-etag"',
|
|
|
|
|
'x-amz-version-id' => 'manual-retry-version',
|
|
|
|
|
])];
|
|
|
|
|
$requests = [];
|
|
|
|
|
$transactionLevels = [];
|
|
|
|
|
$module = $publisher($responses, $requests, $transactionLevels, $clock);
|
|
|
|
|
$manualRetryResult = $module->publish(
|
|
|
|
|
$manualRetry['operation_id'],
|
|
|
|
|
$manualRetry['artifact_id'],
|
|
|
|
|
$manualRetry
|
|
|
|
|
);
|
|
|
|
|
$assert($manualRetryResult['primary_completed'] === true
|
|
|
|
|
&& count($requests) === 1
|
|
|
|
|
&& $requests[0]->getMethod() === 'PUT'
|
|
|
|
|
&& (int) CloudStorageOperation::where(
|
|
|
|
|
'operation_id', $manualRetry['operation_id']
|
|
|
|
|
)->value('attempts') === 4,
|
|
|
|
|
'Manual retry did not reuse the exact proven-not-applied operation.');
|
|
|
|
|
|
|
|
|
|
// The expected artifact fence runs before source validation or provider I/O.
|
|
|
|
|
$fencedA = $fixture('template');
|
|
|
|
|
$fencedB = $fixture('rescue');
|
|
|
|
|
$clock += 1000;
|
|
|
|
|
$responses = [new Response(200)];
|
|
|
|
|
$requests = [];
|
|
|
|
|
$transactionLevels = [];
|
|
|
|
|
$module = $publisher($responses, $requests, $transactionLevels, $clock);
|
|
|
|
|
$beforeA = CloudStorageOperation::where(
|
|
|
|
|
'operation_id', $fencedA['operation_id']
|
|
|
|
|
)->firstOrFail()->toArray();
|
|
|
|
|
$expectCloudError(static fn () => $module->publish(
|
|
|
|
|
$fencedA['operation_id'], $fencedB['artifact_id'], $fencedA
|
|
|
|
|
), 409);
|
|
|
|
|
$afterA = CloudStorageOperation::where(
|
|
|
|
|
'operation_id', $fencedA['operation_id']
|
|
|
|
|
)->firstOrFail()->toArray();
|
|
|
|
|
$assert($requests === [] && $beforeA === $afterA,
|
|
|
|
|
'Wrong expected artifact crossed the durable or remote mutation fence.');
|
|
|
|
|
|
|
|
|
|
// A settled publication is idempotent even after its local source is gone.
|
|
|
|
|
@unlink($applied['path']);
|
|
|
|
|
$requestsBefore = count($requests);
|
|
|
|
|
$terminalAgain = new CloudStorageImmutablePrimaryPublication(
|
|
|
|
|
new CloudObjectStorageS3Client(new Client([
|
|
|
|
|
'handler' => HandlerStack::create(static function () {
|
|
|
|
|
throw new RuntimeException('Terminal publication used HTTP.');
|
|
|
|
|
}),
|
|
|
|
|
])),
|
|
|
|
|
static function (): array {
|
|
|
|
|
throw new RuntimeException('Terminal publication resolved runtime.');
|
|
|
|
|
},
|
|
|
|
|
static fn (): int => time()
|
|
|
|
|
);
|
|
|
|
|
$idempotent = $terminalAgain->publish(
|
|
|
|
|
$applied['operation_id'], $applied['artifact_id'], $applied
|
|
|
|
|
);
|
|
|
|
|
$assert($idempotent['primary_completed'] === true
|
|
|
|
|
&& (int) CloudStorageOperation::where(
|
|
|
|
|
'operation_id', $applied['operation_id']
|
|
|
|
|
)->value('attempts') === 1
|
|
|
|
|
&& count($requests) === $requestsBefore,
|
|
|
|
|
'Terminal publication was not idempotent without its local file.');
|
|
|
|
|
|
|
|
|
|
foreach (CloudStorageOperation::query()->get() as $operation) {
|
|
|
|
|
$stored = (string) $operation->result_json;
|
|
|
|
|
$assert(!str_contains($stored, 'https://')
|
|
|
|
|
&& !str_contains($stored, 'IMMUTABLEACCESSKEY')
|
|
|
|
|
&& !str_contains($stored, 'immutable-secret-must-not-persist'),
|
|
|
|
|
'Publication ledger persisted a credential or capability URL.');
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
echo "Immutable primary publication behavior passed.\n";
|