You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
virt/web/tests/CloudStorageImmutablePrimar...

672 lines
26 KiB
PHP

<?php
require dirname(__DIR__) . '/vendor/autoload.php';
use App\Models\CloudStorageArtifact;
use App\Models\CloudStorageOperation;
use App\Models\CloudStoragePublication;
use App\Models\CloudStorageReplica;
use App\Services\Cloud\CloudError;
use App\Services\Cloud\CloudObjectStorageS3Client;
use App\Services\Cloud\CloudStorageArtifactService;
use App\Services\Cloud\CloudStorageImmutablePrimaryPublication;
use App\Services\Cloud\CloudStorageReplicaAttentionService;
use GuzzleHttp\Client;
use GuzzleHttp\HandlerStack;
use GuzzleHttp\Promise\Create;
use GuzzleHttp\Psr7\Response;
use Illuminate\Database\Capsule\Manager as Capsule;
use Illuminate\Database\Schema\Blueprint;
use Psr\Http\Message\RequestInterface;
$assert = static function (bool $condition, string $message): void {
if (!$condition) {
throw new RuntimeException($message);
}
};
$expectCloudError = static function (
callable $callback,
int $status
) use ($assert): void {
try {
$callback();
} catch (CloudError $error) {
$assert($error->httpStatus === $status,
'Unexpected CloudError status: ' . $error->httpStatus);
return;
}
throw new RuntimeException('Expected CloudError was not raised.');
};
$db = new Capsule();
$db->addConnection(['driver' => 'sqlite', 'database' => ':memory:']);
$db->setAsGlobal();
$db->bootEloquent();
$db->schema()->create('cloud_storage_backends', static function (
Blueprint $table
): void {
$table->increments('id');
$table->string('backend_key');
$table->string('identity_sha256');
$table->string('name');
$table->string('driver');
$table->string('endpoint');
$table->string('region');
$table->string('bucket');
$table->string('prefix');
$table->boolean('path_style');
$table->string('failure_domain');
$table->text('sealed_credentials');
$table->unsignedBigInteger('config_revision');
$table->unsignedBigInteger('tested_revision');
$table->string('admin_state');
$table->string('health_state');
$table->unsignedInteger('write_priority');
$table->unsignedInteger('consecutive_failures')->default(0);
$table->unsignedInteger('consecutive_successes')->default(0);
$table->string('last_error_code')->default('');
$table->unsignedBigInteger('last_tested_at')->default(0);
$table->unsignedBigInteger('last_healthy_at')->default(0);
$table->unsignedBigInteger('next_probe_at')->default(0);
$table->unsignedBigInteger('created_at');
$table->unsignedBigInteger('updated_at');
});
$db->schema()->create('cloud_storage_artifacts', static function (
Blueprint $table
): void {
$table->increments('id');
$table->string('artifact_key')->unique();
$table->string('kind');
$table->unsignedBigInteger('maximum_size_bytes');
$table->unsignedBigInteger('size_bytes');
$table->string('sha256');
$table->string('durability_policy');
$table->unsignedTinyInteger('desired_copies');
$table->unsignedTinyInteger('required_copies');
$table->string('state');
$table->string('last_operation_id')->nullable();
$table->unsignedBigInteger('verified_at');
$table->unsignedBigInteger('created_at');
$table->unsignedBigInteger('updated_at');
});
$db->schema()->create('cloud_storage_replicas', static function (
Blueprint $table
): void {
$table->increments('id');
$table->string('replica_key')->unique();
$table->unsignedBigInteger('artifact_id');
$table->unsignedBigInteger('backend_id');
$table->string('role');
$table->string('object_key');
$table->string('object_version_id')->default('');
$table->string('object_etag')->default('');
$table->unsignedBigInteger('size_bytes')->default(0);
$table->string('sha256')->default('');
$table->string('state');
$table->string('publish_operation_id')->nullable();
$table->string('delete_operation_id')->nullable();
$table->unsignedInteger('delete_generation')->default(0);
$table->unsignedInteger('delete_cycle_attempts')->default(0);
$table->unsignedInteger('delete_presence_confirmations')->default(0);
$table->unsignedBigInteger('delete_last_checked_at')->default(0);
$table->unsignedBigInteger('delete_next_check_at')->default(0);
$table->unsignedBigInteger('delete_attention_at')->nullable();
$table->string('lease_token')->nullable();
$table->unsignedBigInteger('lease_until')->default(0);
$table->string('error_code')->default('');
$table->unsignedBigInteger('verified_at')->default(0);
$table->unsignedBigInteger('created_at');
$table->unsignedBigInteger('updated_at');
});
$db->schema()->create('cloud_storage_operations', static function (
Blueprint $table
): void {
$table->increments('id');
$table->string('operation_id')->unique();
$table->string('request_idempotency_key')->unique();
$table->string('request_fingerprint');
$table->unsignedBigInteger('artifact_id');
$table->unsignedBigInteger('replica_id')->nullable();
$table->unsignedBigInteger('source_backend_id')->nullable();
$table->unsignedBigInteger('target_backend_id')->nullable();
$table->string('kind');
$table->string('phase');
$table->string('outcome')->nullable();
$table->string('effect_resolution');
$table->boolean('effect_may_have_applied')->nullable();
$table->boolean('safe_to_resubmit')->nullable();
$table->unsignedInteger('attempts');
$table->string('lease_token')->nullable();
$table->unsignedBigInteger('lease_until');
$table->unsignedBigInteger('available_at');
$table->unsignedBigInteger('next_check_at');
$table->string('error_code');
$table->text('result_json')->nullable();
$table->unsignedBigInteger('terminal_at')->nullable();
$table->unsignedBigInteger('created_at');
$table->unsignedBigInteger('updated_at');
});
$db->schema()->create('cloud_storage_publications', static function (
Blueprint $table
): void {
$table->increments('id');
$table->string('publication_key')->unique();
$table->unsignedBigInteger('artifact_id')->unique();
$table->unsignedBigInteger('backup_id')->nullable()->unique();
$table->string('template_import_intent_id')->nullable()->unique();
$table->string('rescue_template_id')->nullable()->unique();
$table->string('state');
$table->unsignedInteger('attempts')->default(0);
$table->unsignedBigInteger('available_at')->default(0);
$table->string('lease_token')->nullable();
$table->unsignedBigInteger('lease_until')->default(0);
$table->string('last_error_code')->default('');
$table->unsignedBigInteger('coverage_ready_at')->nullable();
$table->unsignedBigInteger('finalized_at')->nullable();
$table->unsignedBigInteger('created_at');
$table->unsignedBigInteger('updated_at');
});
$databaseNow = time();
foreach ([
[1, 'primary', 'provider-a', 10],
[2, 'secondary', 'provider-b', 20],
] as [$id, $name, $domain, $priority]) {
Capsule::table('cloud_storage_backends')->insert([
'id' => $id,
'backend_key' => str_repeat((string) $id, 32),
'identity_sha256' => str_repeat((string) $id, 64),
'name' => $name,
'driver' => 's3',
'endpoint' => 'https://' . $name . '.example.test',
'region' => 'us-east-1',
'bucket' => $name . '-bucket',
'prefix' => $name,
'path_style' => 1,
'failure_domain' => $domain,
'sealed_credentials' => 'sealed',
'config_revision' => 1,
'tested_revision' => 1,
'admin_state' => 'active',
'health_state' => 'healthy',
'write_priority' => $priority,
'consecutive_failures' => 0,
'consecutive_successes' => 1,
'last_error_code' => '',
'last_tested_at' => $databaseNow,
'last_healthy_at' => $databaseNow,
'next_probe_at' => $databaseNow + 300,
'created_at' => $databaseNow,
'updated_at' => $databaseNow,
]);
}
$temporaryPaths = [];
register_shutdown_function(static function () use (&$temporaryPaths): void {
foreach ($temporaryPaths as $path) {
@unlink($path);
}
});
$fixtureSequence = 0;
$fixture = static function (
string $kind = 'template'
) use (&$fixtureSequence, &$temporaryPaths): array {
$fixtureSequence++;
$path = tempnam(sys_get_temp_dir(), 'virt-primary-publication-');
if ($path === false) {
throw new RuntimeException('Cannot create publication fixture.');
}
file_put_contents(
$path,
str_repeat('immutable-primary-' . $fixtureSequence . '-', 64)
);
$temporaryPaths[] = $path;
$size = filesize($path);
$sha256 = hash_file('sha256', $path);
if (!is_int($size) || !is_string($sha256)) {
throw new RuntimeException('Cannot identify publication fixture.');
}
$ownerId = str_pad(dechex($fixtureSequence), 32, '0', STR_PAD_LEFT);
$owner = $kind === 'rescue'
? ['rescue_template_id' => $ownerId]
: ['template_import_intent_id' => $ownerId];
$view = CloudStorageArtifactService::beginPublication(
$kind,
$kind . '/fixture-' . $fixtureSequence . '/system.qcow2',
$size,
$sha256,
'immutable-primary-' . str_pad(
(string) $fixtureSequence,
4,
'0',
STR_PAD_LEFT
),
hash('sha256', $kind . '|' . $fixtureSequence),
$owner
);
return [
'path' => $path,
'size_bytes' => $size,
'sha256' => $sha256,
'operation_id' => (string) $view['operation_id'],
'artifact_id' => (int) $view['artifact_id'],
'replica_id' => (int) $view['replica_id'],
'object_key' => (string) $view['object_key'],
];
};
$publisher = static function (
array &$responses,
array &$requests,
array &$transactionLevels,
int &$clock
): CloudStorageImmutablePrimaryPublication {
$handler = static function (
RequestInterface $request,
array $options
) use (&$responses, &$requests, &$transactionLevels) {
$transactionLevels[] = Capsule::connection()->transactionLevel();
$requests[] = $request;
if ($responses === []) {
throw new RuntimeException('Unexpected object-storage request.');
}
$response = array_shift($responses);
if (!$response instanceof Response) {
throw new RuntimeException('Invalid object-storage response fixture.');
}
return Create::promiseFor($response);
};
$client = new CloudObjectStorageS3Client(new Client([
'handler' => HandlerStack::create($handler),
]));
return new CloudStorageImmutablePrimaryPublication(
$client,
static fn (int $backendId, bool $healthy): array => [
'backend_id' => $backendId,
'revision' => 1,
'config' => [
'endpoint' => 'https://objects.example.test',
'region' => 'us-east-1',
'bucket' => 'immutable-fixtures',
'prefix' => 'unused-runtime-prefix',
'path_style' => true,
],
'credentials' => [
'access_key_id' => 'IMMUTABLEACCESSKEY',
'secret_access_key' => 'immutable-secret-must-not-persist',
],
],
static function () use (&$clock): int {
return $clock;
}
);
};
// A successful immutable PUT settles only the primary. Template publication
// remains behind the required-two protection gate until its copy completes.
$applied = $fixture('template');
$clock = time() + 10;
$responses = [new Response(200, [
'ETag' => '"applied-etag"',
'x-amz-version-id' => 'applied-version',
])];
$requests = [];
$transactionLevels = [];
$module = $publisher($responses, $requests, $transactionLevels, $clock);
$appliedResult = $module->publish(
$applied['operation_id'],
$applied['artifact_id'],
$applied
);
$assert($appliedResult['primary_completed'] === true
&& $appliedResult['publication_state'] === 'waiting_protection'
&& $appliedResult['ready_to_finalize'] === false,
'Primary success bypassed the template publication quorum.');
$assert(count($requests) === 1 && $requests[0]->getMethod() === 'PUT'
&& $requests[0]->getHeaderLine('If-None-Match') === '*'
&& $requests[0]->getHeaderLine('x-amz-meta-virt-sha256')
=== $applied['sha256']
&& str_contains(
$requests[0]->getHeaderLine('Authorization'),
'x-amz-meta-virt-sha256'
), 'Immutable primary PUT lost its conditional signed identity.');
$assert($transactionLevels === [0],
'Immutable primary PUT ran inside a database transaction.');
// A provider rejection proves non-application and schedules a bounded retry;
// it never recursively replays inside the same call.
$rejected = $fixture('rescue');
$clock += 1000;
$responses = [new Response(403)];
$requests = [];
$transactionLevels = [];
$module = $publisher($responses, $requests, $transactionLevels, $clock);
$rejectedResult = $module->publish(
$rejected['operation_id'],
$rejected['artifact_id'],
$rejected
);
$rejectedOperation = CloudStorageOperation::where(
'operation_id',
$rejected['operation_id']
)->firstOrFail();
$rejectedReplica = CloudStorageReplica::whereKey(
$rejected['replica_id']
)->firstOrFail();
$assert($rejectedResult['phase'] === 'accepted'
&& (int) $rejectedOperation->attempts === 1
&& $rejectedOperation->outcome === null
&& (int) $rejectedOperation->available_at > $clock
&& (string) $rejectedReplica->state === 'reserved'
&& count($requests) === 1,
'Proven non-application was not delayed without replay.');
// A lost PUT receipt is settled only by exact-key size plus signed SHA
// metadata. The temporary local file is deliberately removed before HEAD.
$unknownApplied = $fixture('template');
$clock += 1000;
$responses = [
new Response(503),
new Response(200, [
'Content-Length' => (string) $unknownApplied['size_bytes'],
'ETag' => '"unknown-applied-etag"',
'x-amz-version-id' => 'unknown-applied-version',
'x-amz-meta-virt-sha256' => $unknownApplied['sha256'],
]),
];
$requests = [];
$transactionLevels = [];
$module = $publisher($responses, $requests, $transactionLevels, $clock);
$unknownResult = $module->publish(
$unknownApplied['operation_id'],
$unknownApplied['artifact_id'],
$unknownApplied
);
$assert($unknownResult['phase'] === 'reconciling',
'Unknown PUT did not enter read-only reconciliation.');
@unlink($unknownApplied['path']);
$clock += 60;
$unknownResult = $module->publish(
$unknownApplied['operation_id'],
$unknownApplied['artifact_id'],
$unknownApplied
);
$assert($unknownResult['primary_completed'] === true
&& array_map(
static fn (RequestInterface $request): string => $request->getMethod(),
$requests
) === ['PUT', 'HEAD']
&& (string) $requests[0]->getUri() === (string) $requests[1]->getUri(),
'Unknown PUT was replayed or reconciled against a different key.');
$assert($transactionLevels === [0, 0],
'Immutable PUT reconciliation ran remote I/O inside a transaction.');
// Two spaced exact-key absences are required before the original operation,
// artifact and key may become dispatchable again.
$absent = $fixture('template');
$clock += 1000;
$responses = [
new Response(503),
new Response(404),
new Response(404),
new Response(200, [
'ETag' => '"retried-etag"',
'x-amz-version-id' => 'retried-version',
]),
];
$requests = [];
$transactionLevels = [];
$module = $publisher($responses, $requests, $transactionLevels, $clock);
$artifactCount = CloudStorageArtifact::count();
$operationCount = CloudStorageOperation::count();
$module->publish($absent['operation_id'], $absent['artifact_id'], $absent);
$clock += 60;
$firstAbsent = $module->publish(
$absent['operation_id'], $absent['artifact_id'], $absent
);
$assert($firstAbsent['phase'] === 'reconciling',
'One absence observation incorrectly authorized a retry.');
$clock += 60;
$secondAbsent = $module->publish(
$absent['operation_id'], $absent['artifact_id'], $absent
);
$assert($secondAbsent['phase'] === 'accepted',
'Two stable absence observations did not release the original intent.');
$clock += 60;
$retried = $module->publish(
$absent['operation_id'], $absent['artifact_id'], $absent
);
$methods = array_map(
static fn (RequestInterface $request): string => $request->getMethod(),
$requests
);
$assert($retried['primary_completed'] === true
&& $methods === ['PUT', 'HEAD', 'HEAD', 'PUT']
&& (string) $requests[0]->getUri() === (string) $requests[1]->getUri()
&& (string) $requests[0]->getUri() === (string) $requests[2]->getUri()
&& (string) $requests[0]->getUri() === (string) $requests[3]->getUri()
&& (int) CloudStorageOperation::where(
'operation_id', $absent['operation_id']
)->value('attempts') === 2
&& CloudStorageArtifact::count() === $artifactCount
&& CloudStorageOperation::count() === $operationCount,
'Stable absence did not retry the same frozen publication intent.');
// Existing content with a different signed identity is an attention state,
// never evidence for success and never permission to overwrite the key.
$conflict = $fixture('rescue');
$clock += 1000;
$responses = [
new Response(503),
new Response(200, [
'Content-Length' => (string) $conflict['size_bytes'],
'ETag' => '"conflicting-etag"',
'x-amz-version-id' => 'conflicting-version',
'x-amz-meta-virt-sha256' => str_repeat('f', 64),
]),
];
$requests = [];
$transactionLevels = [];
$module = $publisher($responses, $requests, $transactionLevels, $clock);
$module->publish($conflict['operation_id'], $conflict['artifact_id'], $conflict);
$clock += 60;
$conflictResult = $module->publish(
$conflict['operation_id'], $conflict['artifact_id'], $conflict
);
$conflictOperation = CloudStorageOperation::where(
'operation_id', $conflict['operation_id']
)->firstOrFail();
$assert($conflictResult['requires_attention'] === true
&& (string) $conflictOperation->phase === 'terminal'
&& (string) $conflictOperation->outcome === 'failed'
&& (string) $conflictOperation->effect_resolution === 'unknown'
&& (bool) $conflictOperation->safe_to_resubmit === false
&& (string) CloudStorageArtifact::whereKey(
$conflict['artifact_id']
)->value('state') === 'degraded'
&& (string) CloudStoragePublication::where(
'artifact_id', $conflict['artifact_id']
)->value('state') === 'failed'
&& count($requests) === 2,
'Conflicting immutable identity did not stop in operator attention.');
// An administrator may reopen only the same unknown-effect operation for
// read-only HEAD reconciliation. Secondary work stopped by the primary fence
// is restored but remains blocked until that exact primary succeeds.
$artifactCountBeforeReopen = CloudStorageArtifact::count();
$operationCountBeforeReopen = CloudStorageOperation::count();
$clock += 1;
$attention = CloudStorageReplicaAttentionService::listView();
$attentionItem = collect($attention['items'])->firstWhere(
'operation_id',
$conflict['operation_id']
);
$assert(is_array($attentionItem)
&& ($attentionItem['operation_type'] ?? null)
=== '不可变主副本发布'
&& ($attentionItem['owner_type'] ?? null) === 'rescue'
&& ($attentionItem['can_retry'] ?? false) === true,
'Immutable primary attention was not exposed to administrators.');
$attentionResult = CloudStorageReplicaAttentionService::retry(
$conflict['operation_id'],
'publish'
);
$assert(($attentionResult['phase'] ?? null) === 'reconciling',
'Immutable primary attention did not reopen through the admin service.');
$reopenedConflict = CloudStorageOperation::where(
'operation_id', $conflict['operation_id']
)->firstOrFail();
$reopenedCopy = CloudStorageOperation::where(
'artifact_id', $conflict['artifact_id']
)->where('kind', 'copy')->firstOrFail();
$reopenedPrimary = CloudStorageReplica::whereKey(
$conflict['replica_id']
)->firstOrFail();
$reopenedSecondary = CloudStorageReplica::whereKey(
(int) $reopenedCopy->replica_id
)->firstOrFail();
$assert((string) $reopenedConflict->phase === 'reconciling'
&& (string) $reopenedConflict->effect_resolution === 'unknown'
&& $reopenedConflict->outcome === null
&& (string) $reopenedPrimary->state === 'publishing'
&& (string) $reopenedCopy->phase === 'accepted'
&& (int) $reopenedCopy->available_at === 253402300799
&& (string) $reopenedSecondary->state === 'reserved'
&& (string) CloudStorageArtifact::whereKey(
$conflict['artifact_id']
)->value('state') === 'publishing'
&& (string) CloudStoragePublication::where(
'artifact_id', $conflict['artifact_id']
)->value('state') === 'waiting_primary'
&& CloudStorageArtifact::count() === $artifactCountBeforeReopen
&& CloudStorageOperation::count() === $operationCountBeforeReopen,
'Administrator reconciliation did not reopen the original fenced ledger.');
$responses = [new Response(404), new Response(404)];
$requests = [];
$transactionLevels = [];
$module = $publisher($responses, $requests, $transactionLevels, $clock);
$module->publish(
$conflict['operation_id'], $conflict['artifact_id'], $conflict
);
$clock += 60;
$reconciledAbsence = $module->publish(
$conflict['operation_id'], $conflict['artifact_id'], $conflict
);
$assert($reconciledAbsence['phase'] === 'accepted'
&& array_map(
static fn (RequestInterface $request): string => $request->getMethod(),
$requests
) === ['HEAD', 'HEAD'],
'Administrator recovery replayed a write before stable absence evidence.');
// Repeated proven non-application also stops for operator review. Reopening
// authorizes one new dispatch of the same operation and immutable key only.
$manualRetry = $fixture('template');
$clock += 1000;
$responses = [new Response(403), new Response(403), new Response(403)];
$requests = [];
$transactionLevels = [];
$module = $publisher($responses, $requests, $transactionLevels, $clock);
for ($attempt = 0; $attempt < 3; $attempt++) {
$module->publish(
$manualRetry['operation_id'],
$manualRetry['artifact_id'],
$manualRetry
);
$clock += 300;
}
$manualRetryOperation = CloudStorageOperation::where(
'operation_id', $manualRetry['operation_id']
)->firstOrFail();
$assert((string) $manualRetryOperation->phase === 'terminal'
&& (string) $manualRetryOperation->effect_resolution === 'not_applied'
&& (string) $manualRetryOperation->error_code
=== 'immutable_primary_publish_retry_exhausted',
'Bounded proven failures did not enter manual retry attention.');
$manualAttention = CloudStorageReplicaAttentionService::listView();
$manualAttentionItem = collect($manualAttention['items'])->firstWhere(
'operation_id',
$manualRetry['operation_id']
);
$assert(is_array($manualAttentionItem)
&& ($manualAttentionItem['can_retry'] ?? false) === true,
'Proven-not-applied primary attention was not exposed for retry.');
CloudStorageReplicaAttentionService::retry(
$manualRetry['operation_id'],
'publish'
);
$responses = [new Response(200, [
'ETag' => '"manual-retry-etag"',
'x-amz-version-id' => 'manual-retry-version',
])];
$requests = [];
$transactionLevels = [];
$module = $publisher($responses, $requests, $transactionLevels, $clock);
$manualRetryResult = $module->publish(
$manualRetry['operation_id'],
$manualRetry['artifact_id'],
$manualRetry
);
$assert($manualRetryResult['primary_completed'] === true
&& count($requests) === 1
&& $requests[0]->getMethod() === 'PUT'
&& (int) CloudStorageOperation::where(
'operation_id', $manualRetry['operation_id']
)->value('attempts') === 4,
'Manual retry did not reuse the exact proven-not-applied operation.');
// The expected artifact fence runs before source validation or provider I/O.
$fencedA = $fixture('template');
$fencedB = $fixture('rescue');
$clock += 1000;
$responses = [new Response(200)];
$requests = [];
$transactionLevels = [];
$module = $publisher($responses, $requests, $transactionLevels, $clock);
$beforeA = CloudStorageOperation::where(
'operation_id', $fencedA['operation_id']
)->firstOrFail()->toArray();
$expectCloudError(static fn () => $module->publish(
$fencedA['operation_id'], $fencedB['artifact_id'], $fencedA
), 409);
$afterA = CloudStorageOperation::where(
'operation_id', $fencedA['operation_id']
)->firstOrFail()->toArray();
$assert($requests === [] && $beforeA === $afterA,
'Wrong expected artifact crossed the durable or remote mutation fence.');
// A settled publication is idempotent even after its local source is gone.
@unlink($applied['path']);
$requestsBefore = count($requests);
$terminalAgain = new CloudStorageImmutablePrimaryPublication(
new CloudObjectStorageS3Client(new Client([
'handler' => HandlerStack::create(static function () {
throw new RuntimeException('Terminal publication used HTTP.');
}),
])),
static function (): array {
throw new RuntimeException('Terminal publication resolved runtime.');
},
static fn (): int => time()
);
$idempotent = $terminalAgain->publish(
$applied['operation_id'], $applied['artifact_id'], $applied
);
$assert($idempotent['primary_completed'] === true
&& (int) CloudStorageOperation::where(
'operation_id', $applied['operation_id']
)->value('attempts') === 1
&& count($requests) === $requestsBefore,
'Terminal publication was not idempotent without its local file.');
foreach (CloudStorageOperation::query()->get() as $operation) {
$stored = (string) $operation->result_json;
$assert(!str_contains($stored, 'https://')
&& !str_contains($stored, 'IMMUTABLEACCESSKEY')
&& !str_contains($stored, 'immutable-secret-must-not-persist'),
'Publication ledger persisted a credential or capability URL.');
}
echo "Immutable primary publication behavior passed.\n";