|
|
|
|
<?php
|
|
|
|
|
declare(strict_types=1);
|
|
|
|
|
|
|
|
|
|
namespace App\Services\Cloud {
|
|
|
|
|
/** Test-only sealing stub; the state-machine assertions do not inspect ciphertext. */
|
|
|
|
|
final class CloudSecrets
|
|
|
|
|
{
|
|
|
|
|
public static function encrypt(array $payload, string $purpose): string
|
|
|
|
|
{
|
|
|
|
|
return 'test-sealed:' . $purpose;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
final class CatalogImportFakeAgent
|
|
|
|
|
{
|
|
|
|
|
public array $requests = [];
|
|
|
|
|
/** @var null|callable */
|
|
|
|
|
public $beforeResponse = null;
|
|
|
|
|
public ?array $getResponse = null;
|
|
|
|
|
public ?array $postResponse = null;
|
|
|
|
|
public ?array $reconcileResponse = null;
|
|
|
|
|
public bool $ambiguousReplay = false;
|
|
|
|
|
public bool $preDispatchBusy = false;
|
|
|
|
|
|
|
|
|
|
public function request(string $method, string $target, array $actor,
|
|
|
|
|
?array $payload = null): array
|
|
|
|
|
{
|
|
|
|
|
$this->requests[] = compact('method', 'target', 'actor', 'payload');
|
|
|
|
|
if (is_callable($this->beforeResponse)) {
|
|
|
|
|
($this->beforeResponse)($method, $target, $actor, $payload);
|
|
|
|
|
}
|
|
|
|
|
$operationId = (string) ($payload['operation_id']
|
|
|
|
|
?? $actor['operation_id'] ?? basename($target));
|
|
|
|
|
if (preg_match('#^/v1/operations/([A-Za-z0-9_-]+)/reconcile$#D',
|
|
|
|
|
$target, $match) === 1) {
|
|
|
|
|
$operationId = rawurldecode($match[1]);
|
|
|
|
|
}
|
|
|
|
|
if ($method === 'GET') {
|
|
|
|
|
return ['operation_id' => $operationId]
|
|
|
|
|
+ ($this->getResponse ?? ['status' => 'running']);
|
|
|
|
|
}
|
|
|
|
|
if ($method === 'POST'
|
|
|
|
|
&& preg_match('#^/v1/operations/[A-Za-z0-9_-]+/reconcile$#D',
|
|
|
|
|
$target) === 1) {
|
|
|
|
|
return ['operation_id' => $operationId]
|
|
|
|
|
+ ($this->reconcileResponse ?? $this->postResponse
|
|
|
|
|
?? ['status' => 'uncertain']);
|
|
|
|
|
}
|
|
|
|
|
if ($method === 'POST' && $this->preDispatchBusy
|
|
|
|
|
&& in_array($target,
|
|
|
|
|
['/v1/templates', '/v1/user-isos'], true)) {
|
|
|
|
|
throw new CloudAgentError('request was not dispatched', 503,
|
|
|
|
|
'agent_busy', true, 503, false, true);
|
|
|
|
|
}
|
|
|
|
|
if ($method === 'POST' && $this->ambiguousReplay) {
|
|
|
|
|
throw new CloudAgentError('nonce may have dispatched', 502,
|
|
|
|
|
'replayed_request', false, 401);
|
|
|
|
|
}
|
|
|
|
|
if ($this->postResponse !== null) {
|
|
|
|
|
return ['operation_id' => $operationId]
|
|
|
|
|
+ $this->postResponse;
|
|
|
|
|
}
|
|
|
|
|
if ($target === '/v1/templates') {
|
|
|
|
|
return [
|
|
|
|
|
'operation_id' => $operationId,
|
|
|
|
|
'status' => 'succeeded',
|
|
|
|
|
'result' => [
|
|
|
|
|
'id' => (string) $payload['id'],
|
|
|
|
|
'revision' => (int) $payload['revision'],
|
|
|
|
|
'generation_token' =>
|
|
|
|
|
(string) $payload['generation_token'],
|
|
|
|
|
'size_bytes' => 4096,
|
|
|
|
|
'sha256' => str_repeat('a', 64),
|
|
|
|
|
],
|
|
|
|
|
];
|
|
|
|
|
}
|
|
|
|
|
if ($target === '/v1/user-isos') {
|
|
|
|
|
return [
|
|
|
|
|
'operation_id' => $operationId,
|
|
|
|
|
'status' => 'succeeded',
|
|
|
|
|
'result' => [
|
|
|
|
|
'id' => (string) $payload['id'],
|
|
|
|
|
'owner_id' => (int) $payload['owner_id'],
|
|
|
|
|
'size_bytes' => 8192,
|
|
|
|
|
'sha256' => str_repeat('b', 64),
|
|
|
|
|
],
|
|
|
|
|
];
|
|
|
|
|
}
|
|
|
|
|
throw new \RuntimeException('Unexpected fake Agent request');
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
final class CloudAgentSettingsService
|
|
|
|
|
{
|
|
|
|
|
public static array $hostIds = [101];
|
|
|
|
|
public static ?CatalogImportFakeAgent $client = null;
|
|
|
|
|
|
|
|
|
|
public static function enabledHostIds(): array
|
|
|
|
|
{
|
|
|
|
|
return self::$hostIds;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
public static function clientForHost(int $hostId,
|
|
|
|
|
float $timeoutSeconds = 20.0): CatalogImportFakeAgent
|
|
|
|
|
{
|
|
|
|
|
if (!in_array($hostId, self::$hostIds, true)
|
|
|
|
|
|| self::$client === null) {
|
|
|
|
|
throw new \RuntimeException('Missing fake Agent');
|
|
|
|
|
}
|
|
|
|
|
return self::$client;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
namespace {
|
|
|
|
|
require dirname(__DIR__) . '/vendor/autoload.php';
|
|
|
|
|
|
|
|
|
|
use App\Models\CloudTemplate;
|
|
|
|
|
use App\Models\CloudTemplateReplica;
|
|
|
|
|
use App\Models\CloudUserIso;
|
|
|
|
|
use App\Models\CloudUserIsoReplica;
|
|
|
|
|
use App\Services\Cloud\CatalogImportFakeAgent;
|
|
|
|
|
use App\Services\Cloud\CloudAgentSettingsService;
|
|
|
|
|
use App\Services\Cloud\CloudTemplateCatalogService;
|
|
|
|
|
use App\Services\Cloud\CloudUserIsoService;
|
|
|
|
|
use Illuminate\Database\Capsule\Manager as Capsule;
|
|
|
|
|
use Illuminate\Database\Schema\Blueprint;
|
|
|
|
|
|
|
|
|
|
$assert = static function (bool $condition, string $message): void {
|
|
|
|
|
if (!$condition) {
|
|
|
|
|
throw new RuntimeException($message);
|
|
|
|
|
}
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
$capsule = new Capsule();
|
|
|
|
|
$capsule->addConnection(['driver' => 'sqlite', 'database' => ':memory:'],
|
|
|
|
|
'default');
|
|
|
|
|
$capsule->setAsGlobal();
|
|
|
|
|
$capsule->bootEloquent();
|
|
|
|
|
$schema = $capsule->schema('default');
|
|
|
|
|
|
|
|
|
|
$schema->create('cloud_templates', static function (Blueprint $table): void {
|
|
|
|
|
$table->string('id')->primary();
|
|
|
|
|
$table->string('name');
|
|
|
|
|
$table->string('os_family');
|
|
|
|
|
$table->string('os_version');
|
|
|
|
|
$table->string('source_type');
|
|
|
|
|
$table->text('sealed_source_url');
|
|
|
|
|
$table->integer('image_bytes')->default(0);
|
|
|
|
|
$table->string('sha256')->default('');
|
|
|
|
|
$table->text('startup_script')->default('');
|
|
|
|
|
$table->integer('revision');
|
|
|
|
|
$table->string('generation_token')->unique();
|
|
|
|
|
$table->string('state');
|
|
|
|
|
$table->integer('created_at');
|
|
|
|
|
$table->integer('updated_at');
|
|
|
|
|
});
|
|
|
|
|
$schema->create('cloud_template_replicas',
|
|
|
|
|
static function (Blueprint $table): void {
|
|
|
|
|
$table->integer('host_id');
|
|
|
|
|
$table->string('template_id');
|
|
|
|
|
$table->integer('revision');
|
|
|
|
|
$table->string('generation_token');
|
|
|
|
|
$table->string('state');
|
|
|
|
|
$table->integer('image_bytes')->default(0);
|
|
|
|
|
$table->string('sha256')->default('');
|
|
|
|
|
$table->string('operation_id')->default('');
|
|
|
|
|
$table->string('error_code')->default('');
|
|
|
|
|
$table->integer('created_at');
|
|
|
|
|
$table->integer('updated_at');
|
|
|
|
|
$table->primary(['host_id', 'template_id']);
|
|
|
|
|
});
|
|
|
|
|
$schema->create('cloud_orders', static function (Blueprint $table): void {
|
|
|
|
|
$table->increments('id');
|
|
|
|
|
$table->string('status');
|
|
|
|
|
$table->text('spec_json');
|
|
|
|
|
});
|
|
|
|
|
$schema->create('cloud_jobs', static function (Blueprint $table): void {
|
|
|
|
|
$table->increments('id');
|
|
|
|
|
$table->string('kind');
|
|
|
|
|
$table->string('status');
|
|
|
|
|
$table->text('sealed_payload');
|
|
|
|
|
});
|
|
|
|
|
$schema->create('cloud_user_isos', static function (Blueprint $table): void {
|
|
|
|
|
$table->string('id')->primary();
|
|
|
|
|
$table->integer('user_id');
|
|
|
|
|
$table->string('name');
|
|
|
|
|
$table->text('sealed_source_url');
|
|
|
|
|
$table->integer('size_bytes')->default(0);
|
|
|
|
|
$table->string('sha256')->default('');
|
|
|
|
|
$table->string('state');
|
|
|
|
|
$table->string('import_operation_id');
|
|
|
|
|
$table->integer('created_at');
|
|
|
|
|
$table->integer('updated_at');
|
|
|
|
|
});
|
|
|
|
|
$schema->create('cloud_user_iso_replicas',
|
|
|
|
|
static function (Blueprint $table): void {
|
|
|
|
|
$table->integer('host_id');
|
|
|
|
|
$table->string('user_iso_id');
|
|
|
|
|
$table->integer('user_id');
|
|
|
|
|
$table->string('state');
|
|
|
|
|
$table->integer('size_bytes')->default(0);
|
|
|
|
|
$table->string('sha256')->default('');
|
|
|
|
|
$table->string('operation_id')->default('');
|
|
|
|
|
$table->string('error_code')->default('');
|
|
|
|
|
$table->integer('created_at');
|
|
|
|
|
$table->integer('updated_at');
|
|
|
|
|
$table->primary(['host_id', 'user_iso_id']);
|
|
|
|
|
});
|
|
|
|
|
|
|
|
|
|
$agent = new CatalogImportFakeAgent();
|
|
|
|
|
CloudAgentSettingsService::$client = $agent;
|
|
|
|
|
|
|
|
|
|
// An Agent success followed by a local metadata write failure is uncertain,
|
|
|
|
|
// never a definite dispatch failure. The transaction leaves the replica
|
|
|
|
|
// pending long enough for the guarded uncertainty transition to latch.
|
|
|
|
|
Capsule::statement("CREATE TRIGGER reject_template_digest "
|
|
|
|
|
. "BEFORE UPDATE OF sha256 ON cloud_templates "
|
|
|
|
|
. "WHEN NEW.sha256 <> OLD.sha256 BEGIN "
|
|
|
|
|
. "SELECT RAISE(ABORT, 'template persistence failure'); END");
|
|
|
|
|
$templateJob = CloudTemplateCatalogService::importRemote([
|
|
|
|
|
'name' => 'Debian 12 persistence test',
|
|
|
|
|
'url' => 'https://images.example/debian-12.qcow2',
|
|
|
|
|
'sha256' => '',
|
|
|
|
|
], ['user_id' => 1, 'can_manage' => true]);
|
|
|
|
|
$template = CloudTemplate::query()->where('id',
|
|
|
|
|
(string) $templateJob['job_id'])->first();
|
|
|
|
|
$templateReplica = CloudTemplateReplica::query()
|
|
|
|
|
->where('template_id', (string) $template->id)->first();
|
|
|
|
|
$assert($templateJob['status'] === 'uncertain'
|
|
|
|
|
&& (string) $template->state === 'error'
|
|
|
|
|
&& (string) $templateReplica->state === 'uncertain'
|
|
|
|
|
&& (string) $templateReplica->error_code === 'panel_persist_failed',
|
|
|
|
|
'Template Agent success was mislabeled after panel persistence failed.');
|
|
|
|
|
Capsule::statement('DROP TRIGGER reject_template_digest');
|
|
|
|
|
|
|
|
|
|
Capsule::statement("CREATE TRIGGER reject_iso_digest "
|
|
|
|
|
. "BEFORE UPDATE OF sha256 ON cloud_user_isos "
|
|
|
|
|
. "WHEN NEW.sha256 <> OLD.sha256 BEGIN "
|
|
|
|
|
. "SELECT RAISE(ABORT, 'ISO persistence failure'); END");
|
|
|
|
|
$isoJob = CloudUserIsoService::import([
|
|
|
|
|
'name' => 'Recovery ISO',
|
|
|
|
|
'url' => 'https://images.example/recovery.iso',
|
|
|
|
|
], ['user_id' => 42, 'can_manage' => false]);
|
|
|
|
|
$iso = CloudUserIso::query()->where('import_operation_id',
|
|
|
|
|
(string) $isoJob['job_id'])->first();
|
|
|
|
|
$isoReplica = CloudUserIsoReplica::query()
|
|
|
|
|
->where('user_iso_id', (string) $iso->id)->first();
|
|
|
|
|
$assert($isoJob['status'] === 'uncertain'
|
|
|
|
|
&& (string) $iso->state === 'failed'
|
|
|
|
|
&& (string) $isoReplica->state === 'uncertain'
|
|
|
|
|
&& (string) $isoReplica->error_code === 'panel_persist_failed',
|
|
|
|
|
'ISO Agent success was mislabeled after panel persistence failed.');
|
|
|
|
|
Capsule::statement('DROP TRIGGER reject_iso_digest');
|
|
|
|
|
|
|
|
|
|
// A process interruption after the last replica update must be repaired by
|
|
|
|
|
// status lookup even when there is no pending operation left to poll.
|
|
|
|
|
Capsule::table('cloud_templates')->where('id', (string) $template->id)
|
|
|
|
|
->update(['state' => 'importing']);
|
|
|
|
|
Capsule::table('cloud_template_replicas')
|
|
|
|
|
->where('template_id', (string) $template->id)->update([
|
|
|
|
|
'state' => 'active',
|
|
|
|
|
'image_bytes' => 4096,
|
|
|
|
|
'sha256' => str_repeat('a', 64),
|
|
|
|
|
'error_code' => '',
|
|
|
|
|
]);
|
|
|
|
|
$requestsBeforeTemplateRepair = count($agent->requests);
|
|
|
|
|
$repairedTemplate = CloudTemplateCatalogService::importJob(
|
|
|
|
|
(string) $template->id, ['user_id' => 1, 'can_manage' => true]
|
|
|
|
|
);
|
|
|
|
|
$assert($repairedTemplate['status'] === 'succeeded'
|
|
|
|
|
&& (string) CloudTemplate::find((string) $template->id)->state
|
|
|
|
|
=== 'active'
|
|
|
|
|
&& count($agent->requests) === $requestsBeforeTemplateRepair,
|
|
|
|
|
'Template aggregate state did not self-heal without pending replicas.');
|
|
|
|
|
|
|
|
|
|
Capsule::table('cloud_user_isos')->where('id', (string) $iso->id)
|
|
|
|
|
->update([
|
|
|
|
|
'state' => 'importing',
|
|
|
|
|
'size_bytes' => 8192,
|
|
|
|
|
'sha256' => str_repeat('b', 64),
|
|
|
|
|
]);
|
|
|
|
|
Capsule::table('cloud_user_iso_replicas')
|
|
|
|
|
->where('user_iso_id', (string) $iso->id)->update([
|
|
|
|
|
'state' => 'active',
|
|
|
|
|
'size_bytes' => 8192,
|
|
|
|
|
'sha256' => str_repeat('b', 64),
|
|
|
|
|
'error_code' => '',
|
|
|
|
|
]);
|
|
|
|
|
$requestsBeforeIsoRepair = count($agent->requests);
|
|
|
|
|
$repairedIso = CloudUserIsoService::job((string) $iso->import_operation_id,
|
|
|
|
|
['user_id' => 42, 'can_manage' => false]);
|
|
|
|
|
$assert($repairedIso['status'] === 'succeeded'
|
|
|
|
|
&& (string) CloudUserIso::find((string) $iso->id)->state === 'active'
|
|
|
|
|
&& count($agent->requests) === $requestsBeforeIsoRepair,
|
|
|
|
|
'ISO aggregate state did not self-heal without pending replicas.');
|
|
|
|
|
|
|
|
|
|
// `dispatching` is a normal durable Agent state. It must remain pollable
|
|
|
|
|
// rather than being surfaced as an ambiguous import result.
|
|
|
|
|
$agent->postResponse = ['status' => 'dispatching'];
|
|
|
|
|
$dispatchTemplateJob = CloudTemplateCatalogService::importRemote([
|
|
|
|
|
'name' => 'Debian 13 dispatching template',
|
|
|
|
|
'url' => 'https://images.example/debian-13.qcow2',
|
|
|
|
|
'sha256' => '',
|
|
|
|
|
], ['user_id' => 1, 'can_manage' => true]);
|
|
|
|
|
$dispatchTemplate = CloudTemplate::find(
|
|
|
|
|
(string) $dispatchTemplateJob['job_id']
|
|
|
|
|
);
|
|
|
|
|
$dispatchTemplateReplica = CloudTemplateReplica::query()
|
|
|
|
|
->where('template_id', (string) $dispatchTemplate->id)->first();
|
|
|
|
|
$assert($dispatchTemplateJob['status'] === 'running'
|
|
|
|
|
&& (string) $dispatchTemplate->state === 'importing'
|
|
|
|
|
&& (string) $dispatchTemplateReplica->state === 'pending'
|
|
|
|
|
&& (string) $dispatchTemplateReplica->error_code === '',
|
|
|
|
|
'Template Agent dispatching response was made uncertain.');
|
|
|
|
|
|
|
|
|
|
$dispatchIsoJob = CloudUserIsoService::import([
|
|
|
|
|
'name' => 'Dispatching ISO',
|
|
|
|
|
'url' => 'https://images.example/dispatching.iso',
|
|
|
|
|
], ['user_id' => 52, 'can_manage' => false]);
|
|
|
|
|
$dispatchIso = CloudUserIso::query()->where('import_operation_id',
|
|
|
|
|
(string) $dispatchIsoJob['job_id'])->first();
|
|
|
|
|
$dispatchIsoReplica = CloudUserIsoReplica::query()
|
|
|
|
|
->where('user_iso_id', (string) $dispatchIso->id)->first();
|
|
|
|
|
$assert($dispatchIsoJob['status'] === 'running'
|
|
|
|
|
&& (string) $dispatchIso->state === 'importing'
|
|
|
|
|
&& (string) $dispatchIsoReplica->state === 'pending'
|
|
|
|
|
&& (string) $dispatchIsoReplica->error_code === '',
|
|
|
|
|
'ISO Agent dispatching response was made uncertain.');
|
|
|
|
|
$agent->postResponse = null;
|
|
|
|
|
|
|
|
|
|
// Catalog imports intentionally get one POST only. Even exact
|
|
|
|
|
// not_dispatched backpressure is terminal so expiring upload/source
|
|
|
|
|
// capabilities are never reused by an automatic retry; an administrator
|
|
|
|
|
// must create a fresh import with a fresh operation and source snapshot.
|
|
|
|
|
$agent->preDispatchBusy = true;
|
|
|
|
|
$preDispatchTemplate = CloudTemplateCatalogService::importRemote([
|
|
|
|
|
'name' => 'Debian 12 pre-dispatch terminal template',
|
|
|
|
|
'url' => 'https://images.example/debian-12-predispatch.qcow2',
|
|
|
|
|
'sha256' => '',
|
|
|
|
|
], ['user_id' => 1, 'can_manage' => true]);
|
|
|
|
|
$preDispatchTemplateRow = CloudTemplate::find(
|
|
|
|
|
(string) $preDispatchTemplate['job_id']
|
|
|
|
|
);
|
|
|
|
|
$preDispatchTemplateReplica = CloudTemplateReplica::query()
|
|
|
|
|
->where('template_id', (string) $preDispatchTemplateRow->id)->first();
|
|
|
|
|
$preDispatchIso = CloudUserIsoService::import([
|
|
|
|
|
'name' => 'Pre-dispatch terminal ISO',
|
|
|
|
|
'url' => 'https://images.example/predispatch.iso',
|
|
|
|
|
], ['user_id' => 72, 'can_manage' => false]);
|
|
|
|
|
$preDispatchIsoRow = CloudUserIso::query()->where('import_operation_id',
|
|
|
|
|
(string) $preDispatchIso['job_id'])->first();
|
|
|
|
|
$preDispatchIsoReplica = CloudUserIsoReplica::query()
|
|
|
|
|
->where('user_iso_id', (string) $preDispatchIsoRow->id)->first();
|
|
|
|
|
$requestsAfterPreDispatch = count($agent->requests);
|
|
|
|
|
CloudTemplateCatalogService::importJob(
|
|
|
|
|
(string) $preDispatchTemplateRow->id,
|
|
|
|
|
['user_id' => 1, 'can_manage' => true]
|
|
|
|
|
);
|
|
|
|
|
CloudUserIsoService::job((string) $preDispatchIsoRow->import_operation_id,
|
|
|
|
|
['user_id' => 72, 'can_manage' => false]);
|
|
|
|
|
$assert($preDispatchTemplate['status'] === 'failed'
|
|
|
|
|
&& (string) $preDispatchTemplateReplica->state === 'failed'
|
|
|
|
|
&& (string) $preDispatchTemplateReplica->error_code === 'agent_busy'
|
|
|
|
|
&& $preDispatchIso['status'] === 'failed'
|
|
|
|
|
&& (string) $preDispatchIsoReplica->state === 'failed'
|
|
|
|
|
&& (string) $preDispatchIsoReplica->error_code === 'agent_busy'
|
|
|
|
|
&& count($agent->requests) === $requestsAfterPreDispatch,
|
|
|
|
|
'Definitive pre-dispatch catalog import was retried automatically.');
|
|
|
|
|
$agent->preDispatchBusy = false;
|
|
|
|
|
|
|
|
|
|
// A non-retryable response without explicit not-dispatched proof is still
|
|
|
|
|
// ambiguous. Retain both imports under their original operation IDs so a
|
|
|
|
|
// later status lookup can reconcile them; never label them failed here.
|
|
|
|
|
$agent->ambiguousReplay = true;
|
|
|
|
|
$replayedTemplateJob = CloudTemplateCatalogService::importRemote([
|
|
|
|
|
'name' => 'Debian 12 ambiguous replay template',
|
|
|
|
|
'url' => 'https://images.example/debian-12-ambiguous-replay.qcow2',
|
|
|
|
|
'sha256' => '',
|
|
|
|
|
], ['user_id' => 1, 'can_manage' => true]);
|
|
|
|
|
$replayedTemplate = CloudTemplate::find(
|
|
|
|
|
(string) $replayedTemplateJob['job_id']
|
|
|
|
|
);
|
|
|
|
|
$replayedTemplateReplica = CloudTemplateReplica::query()
|
|
|
|
|
->where('template_id', (string) $replayedTemplate->id)->first();
|
|
|
|
|
$replayedIsoJob = CloudUserIsoService::import([
|
|
|
|
|
'name' => 'Ambiguous replay ISO',
|
|
|
|
|
'url' => 'https://images.example/ambiguous-replay.iso',
|
|
|
|
|
], ['user_id' => 62, 'can_manage' => false]);
|
|
|
|
|
$replayedIso = CloudUserIso::query()->where('import_operation_id',
|
|
|
|
|
(string) $replayedIsoJob['job_id'])->first();
|
|
|
|
|
$replayedIsoReplica = CloudUserIsoReplica::query()
|
|
|
|
|
->where('user_iso_id', (string) $replayedIso->id)->first();
|
|
|
|
|
$assert($replayedTemplateJob['status'] === 'running'
|
|
|
|
|
&& (string) $replayedTemplateReplica->state === 'pending'
|
|
|
|
|
&& (string) $replayedTemplateReplica->error_code
|
|
|
|
|
=== 'replayed_request'
|
|
|
|
|
&& $replayedIsoJob['status'] === 'running'
|
|
|
|
|
&& (string) $replayedIsoReplica->state === 'pending'
|
|
|
|
|
&& (string) $replayedIsoReplica->error_code === 'replayed_request',
|
|
|
|
|
'Ambiguous non-retryable catalog import was made terminal.');
|
|
|
|
|
$agent->ambiguousReplay = false;
|
|
|
|
|
|
|
|
|
|
// An uncertain GET is inspected through the same operation's read-only
|
|
|
|
|
// reconcile route. Neither catalog import may be replayed.
|
|
|
|
|
$agent->postResponse = ['status' => 'dispatching'];
|
|
|
|
|
$reconcileTemplateJob = CloudTemplateCatalogService::importRemote([
|
|
|
|
|
'name' => 'Debian 12 template reconcile test',
|
|
|
|
|
'url' => 'https://images.example/debian-12-reconcile.qcow2',
|
|
|
|
|
'sha256' => '',
|
|
|
|
|
], ['user_id' => 1, 'can_manage' => true]);
|
|
|
|
|
$reconcileTemplate = CloudTemplate::find(
|
|
|
|
|
(string) $reconcileTemplateJob['job_id']
|
|
|
|
|
);
|
|
|
|
|
$reconcileTemplateReplica = CloudTemplateReplica::query()
|
|
|
|
|
->where('template_id', (string) $reconcileTemplate->id)->first();
|
|
|
|
|
$agent->postResponse = null;
|
|
|
|
|
$agent->getResponse = [
|
|
|
|
|
'status' => 'uncertain',
|
|
|
|
|
'error' => ['code' => 'operation_uncertain'],
|
|
|
|
|
];
|
|
|
|
|
$agent->reconcileResponse = [
|
|
|
|
|
'status' => 'succeeded',
|
|
|
|
|
'result' => [
|
|
|
|
|
'id' => (string) $reconcileTemplate->id,
|
|
|
|
|
'revision' => (int) $reconcileTemplate->revision,
|
|
|
|
|
'generation_token' =>
|
|
|
|
|
(string) $reconcileTemplate->generation_token,
|
|
|
|
|
'size_bytes' => 4096,
|
|
|
|
|
'sha256' => str_repeat('a', 64),
|
|
|
|
|
],
|
|
|
|
|
];
|
|
|
|
|
$templateRequestsBeforeReconcile = count($agent->requests);
|
|
|
|
|
$reconciledTemplate = CloudTemplateCatalogService::importJob(
|
|
|
|
|
(string) $reconcileTemplate->id,
|
|
|
|
|
['user_id' => 1, 'can_manage' => true]
|
|
|
|
|
);
|
|
|
|
|
$templateReconcileRequests = array_slice(
|
|
|
|
|
$agent->requests, $templateRequestsBeforeReconcile
|
|
|
|
|
);
|
|
|
|
|
$assert($reconciledTemplate['status'] === 'succeeded'
|
|
|
|
|
&& count($templateReconcileRequests) === 2
|
|
|
|
|
&& $templateReconcileRequests[0]['method'] === 'GET'
|
|
|
|
|
&& $templateReconcileRequests[0]['target']
|
|
|
|
|
=== '/v1/operations/'
|
|
|
|
|
. (string) $reconcileTemplateReplica->operation_id
|
|
|
|
|
&& $templateReconcileRequests[1]['method'] === 'POST'
|
|
|
|
|
&& $templateReconcileRequests[1]['target']
|
|
|
|
|
=== '/v1/operations/'
|
|
|
|
|
. (string) $reconcileTemplateReplica->operation_id
|
|
|
|
|
. '/reconcile',
|
|
|
|
|
'Template uncertainty did not reconcile the same operation without replay.');
|
|
|
|
|
|
|
|
|
|
$agent->postResponse = ['status' => 'dispatching'];
|
|
|
|
|
$reconcileIsoJob = CloudUserIsoService::import([
|
|
|
|
|
'name' => 'ISO reconcile test',
|
|
|
|
|
'url' => 'https://images.example/iso-reconcile.iso',
|
|
|
|
|
], ['user_id' => 54, 'can_manage' => false]);
|
|
|
|
|
$reconcileIso = CloudUserIso::query()->where('import_operation_id',
|
|
|
|
|
(string) $reconcileIsoJob['job_id'])->first();
|
|
|
|
|
$reconcileIsoReplica = CloudUserIsoReplica::query()
|
|
|
|
|
->where('user_iso_id', (string) $reconcileIso->id)->first();
|
|
|
|
|
$agent->postResponse = null;
|
|
|
|
|
$agent->reconcileResponse = [
|
|
|
|
|
'status' => 'succeeded',
|
|
|
|
|
'result' => [
|
|
|
|
|
'id' => (string) $reconcileIso->id,
|
|
|
|
|
'owner_id' => (int) $reconcileIso->user_id,
|
|
|
|
|
'size_bytes' => 8192,
|
|
|
|
|
'sha256' => str_repeat('b', 64),
|
|
|
|
|
],
|
|
|
|
|
];
|
|
|
|
|
$isoRequestsBeforeReconcile = count($agent->requests);
|
|
|
|
|
$reconciledIso = CloudUserIsoService::job(
|
|
|
|
|
(string) $reconcileIso->import_operation_id,
|
|
|
|
|
['user_id' => 54, 'can_manage' => false]
|
|
|
|
|
);
|
|
|
|
|
$isoReconcileRequests = array_slice(
|
|
|
|
|
$agent->requests, $isoRequestsBeforeReconcile
|
|
|
|
|
);
|
|
|
|
|
$assert($reconciledIso['status'] === 'succeeded'
|
|
|
|
|
&& count($isoReconcileRequests) === 2
|
|
|
|
|
&& $isoReconcileRequests[0]['method'] === 'GET'
|
|
|
|
|
&& $isoReconcileRequests[0]['target']
|
|
|
|
|
=== '/v1/operations/' . (string) $reconcileIsoReplica->operation_id
|
|
|
|
|
&& $isoReconcileRequests[1]['method'] === 'POST'
|
|
|
|
|
&& $isoReconcileRequests[1]['target']
|
|
|
|
|
=== '/v1/operations/' . (string) $reconcileIsoReplica->operation_id
|
|
|
|
|
. '/reconcile',
|
|
|
|
|
'User ISO uncertainty did not reconcile the same operation without replay.');
|
|
|
|
|
$agent->reconcileResponse = null;
|
|
|
|
|
|
|
|
|
|
// Repair only the historical false-positive marker. The old operation ID
|
|
|
|
|
// is polled exactly once; the import POST is never replayed.
|
|
|
|
|
$templateOperationId = (string) $dispatchTemplateReplica->operation_id;
|
|
|
|
|
Capsule::table('cloud_templates')->where('id',
|
|
|
|
|
(string) $dispatchTemplate->id)->update(['state' => 'error']);
|
|
|
|
|
Capsule::table('cloud_template_replicas')->where('template_id',
|
|
|
|
|
(string) $dispatchTemplate->id)->update([
|
|
|
|
|
'state' => 'uncertain',
|
|
|
|
|
'error_code' => 'agent_import_uncertain',
|
|
|
|
|
]);
|
|
|
|
|
$agent->getResponse = [
|
|
|
|
|
'status' => 'succeeded',
|
|
|
|
|
'result' => [
|
|
|
|
|
'id' => (string) $dispatchTemplate->id,
|
|
|
|
|
'revision' => (int) $dispatchTemplate->revision,
|
|
|
|
|
'generation_token' => (string) $dispatchTemplate->generation_token,
|
|
|
|
|
'size_bytes' => 4096,
|
|
|
|
|
'sha256' => str_repeat('a', 64),
|
|
|
|
|
],
|
|
|
|
|
];
|
|
|
|
|
$requestsBeforeTemplateRecovery = count($agent->requests);
|
|
|
|
|
$recoveredTemplate = CloudTemplateCatalogService::importJob(
|
|
|
|
|
(string) $dispatchTemplate->id,
|
|
|
|
|
['user_id' => 1, 'can_manage' => true]
|
|
|
|
|
);
|
|
|
|
|
$templateRecoveryRequest = $agent->requests[
|
|
|
|
|
$requestsBeforeTemplateRecovery
|
|
|
|
|
] ?? [];
|
|
|
|
|
$dispatchTemplateReplica->refresh();
|
|
|
|
|
$assert($recoveredTemplate['status'] === 'succeeded'
|
|
|
|
|
&& count($agent->requests) === $requestsBeforeTemplateRecovery + 1
|
|
|
|
|
&& ($templateRecoveryRequest['method'] ?? null) === 'GET'
|
|
|
|
|
&& ($templateRecoveryRequest['target'] ?? null)
|
|
|
|
|
=== '/v1/operations/' . $templateOperationId
|
|
|
|
|
&& (string) $dispatchTemplateReplica->operation_id
|
|
|
|
|
=== $templateOperationId
|
|
|
|
|
&& (string) $dispatchTemplateReplica->state === 'active',
|
|
|
|
|
'Historical template uncertainty did not reconcile its saved operation.');
|
|
|
|
|
|
|
|
|
|
$isoOperationId = (string) $dispatchIsoReplica->operation_id;
|
|
|
|
|
Capsule::table('cloud_user_isos')->where('id',
|
|
|
|
|
(string) $dispatchIso->id)->update(['state' => 'failed']);
|
|
|
|
|
Capsule::table('cloud_user_iso_replicas')->where('user_iso_id',
|
|
|
|
|
(string) $dispatchIso->id)->update([
|
|
|
|
|
'state' => 'uncertain',
|
|
|
|
|
'error_code' => 'agent_import_uncertain',
|
|
|
|
|
]);
|
|
|
|
|
$agent->getResponse = [
|
|
|
|
|
'status' => 'succeeded',
|
|
|
|
|
'result' => [
|
|
|
|
|
'id' => (string) $dispatchIso->id,
|
|
|
|
|
'owner_id' => (int) $dispatchIso->user_id,
|
|
|
|
|
'size_bytes' => 8192,
|
|
|
|
|
'sha256' => str_repeat('b', 64),
|
|
|
|
|
],
|
|
|
|
|
];
|
|
|
|
|
$requestsBeforeIsoRecovery = count($agent->requests);
|
|
|
|
|
$recoveredIso = CloudUserIsoService::job(
|
|
|
|
|
(string) $dispatchIso->import_operation_id,
|
|
|
|
|
['user_id' => 52, 'can_manage' => false]
|
|
|
|
|
);
|
|
|
|
|
$isoRecoveryRequest = $agent->requests[$requestsBeforeIsoRecovery] ?? [];
|
|
|
|
|
$dispatchIsoReplica->refresh();
|
|
|
|
|
$assert($recoveredIso['status'] === 'succeeded'
|
|
|
|
|
&& count($agent->requests) === $requestsBeforeIsoRecovery + 1
|
|
|
|
|
&& ($isoRecoveryRequest['method'] ?? null) === 'GET'
|
|
|
|
|
&& ($isoRecoveryRequest['target'] ?? null)
|
|
|
|
|
=== '/v1/operations/' . $isoOperationId
|
|
|
|
|
&& (string) $dispatchIsoReplica->operation_id === $isoOperationId
|
|
|
|
|
&& (string) $dispatchIsoReplica->state === 'active',
|
|
|
|
|
'Historical ISO uncertainty did not reconcile its saved operation.');
|
|
|
|
|
|
|
|
|
|
// A genuine Agent uncertainty gets a new marker and stays terminal. Only
|
|
|
|
|
// the exact legacy dispatching marker is eligible for recovery.
|
|
|
|
|
$agent->postResponse = [
|
|
|
|
|
'status' => 'uncertain',
|
|
|
|
|
'error' => ['code' => 'agent_import_uncertain'],
|
|
|
|
|
];
|
|
|
|
|
$lockedTemplateJob = CloudTemplateCatalogService::importRemote([
|
|
|
|
|
'name' => 'Debian 11 genuinely uncertain template',
|
|
|
|
|
'url' => 'https://images.example/debian-11.qcow2',
|
|
|
|
|
'sha256' => '',
|
|
|
|
|
], ['user_id' => 1, 'can_manage' => true]);
|
|
|
|
|
$lockedTemplate = CloudTemplate::find(
|
|
|
|
|
(string) $lockedTemplateJob['job_id']
|
|
|
|
|
);
|
|
|
|
|
$lockedTemplateReplica = CloudTemplateReplica::query()
|
|
|
|
|
->where('template_id', (string) $lockedTemplate->id)->first();
|
|
|
|
|
$agent->postResponse = null;
|
|
|
|
|
$requestsBeforeLockedTemplate = count($agent->requests);
|
|
|
|
|
$lockedTemplateAgain = CloudTemplateCatalogService::importJob(
|
|
|
|
|
(string) $lockedTemplate->id,
|
|
|
|
|
['user_id' => 1, 'can_manage' => true]
|
|
|
|
|
);
|
|
|
|
|
$lockedTemplateReplica->refresh();
|
|
|
|
|
$assert($lockedTemplateJob['status'] === 'uncertain'
|
|
|
|
|
&& $lockedTemplateAgain['status'] === 'uncertain'
|
|
|
|
|
&& count($agent->requests) === $requestsBeforeLockedTemplate
|
|
|
|
|
&& (string) $lockedTemplateReplica->state === 'uncertain'
|
|
|
|
|
&& (string) $lockedTemplateReplica->error_code
|
|
|
|
|
=== 'agent_operation_uncertain',
|
|
|
|
|
'A genuine template uncertainty was unlocked as legacy dispatching.');
|
|
|
|
|
|
|
|
|
|
$agent->postResponse = [
|
|
|
|
|
'status' => 'uncertain',
|
|
|
|
|
'error' => ['code' => 'agent_import_uncertain'],
|
|
|
|
|
];
|
|
|
|
|
$lockedIsoJob = CloudUserIsoService::import([
|
|
|
|
|
'name' => 'Genuinely uncertain ISO',
|
|
|
|
|
'url' => 'https://images.example/genuinely-uncertain.iso',
|
|
|
|
|
], ['user_id' => 53, 'can_manage' => false]);
|
|
|
|
|
$lockedIso = CloudUserIso::query()->where('import_operation_id',
|
|
|
|
|
(string) $lockedIsoJob['job_id'])->first();
|
|
|
|
|
$lockedIsoReplica = CloudUserIsoReplica::query()
|
|
|
|
|
->where('user_iso_id', (string) $lockedIso->id)->first();
|
|
|
|
|
$agent->postResponse = null;
|
|
|
|
|
$requestsBeforeLockedIso = count($agent->requests);
|
|
|
|
|
$lockedIsoAgain = CloudUserIsoService::job(
|
|
|
|
|
(string) $lockedIso->import_operation_id,
|
|
|
|
|
['user_id' => 53, 'can_manage' => false]
|
|
|
|
|
);
|
|
|
|
|
$lockedIsoReplica->refresh();
|
|
|
|
|
$assert($lockedIsoJob['status'] === 'uncertain'
|
|
|
|
|
&& $lockedIsoAgain['status'] === 'uncertain'
|
|
|
|
|
&& count($agent->requests) === $requestsBeforeLockedIso
|
|
|
|
|
&& (string) $lockedIsoReplica->state === 'uncertain'
|
|
|
|
|
&& (string) $lockedIsoReplica->error_code
|
|
|
|
|
=== 'agent_operation_uncertain',
|
|
|
|
|
'A genuine ISO uncertainty was unlocked as legacy dispatching.');
|
|
|
|
|
|
|
|
|
|
// Quota aggregation must use the bytes committed through the locked model,
|
|
|
|
|
// not the stale outer object that existed before the Agent response.
|
|
|
|
|
$quotaOwner = 77;
|
|
|
|
|
$quotaBytes = 10 * 1024 * 1024 * 1024;
|
|
|
|
|
Capsule::table('cloud_user_isos')->insert([
|
|
|
|
|
'id' => str_repeat('9', 32),
|
|
|
|
|
'user_id' => $quotaOwner,
|
|
|
|
|
'name' => 'Existing quota usage',
|
|
|
|
|
'sealed_source_url' => 'test-sealed:user-iso-source',
|
|
|
|
|
'size_bytes' => $quotaBytes - 4096,
|
|
|
|
|
'sha256' => str_repeat('9', 64),
|
|
|
|
|
'state' => 'active',
|
|
|
|
|
'import_operation_id' => str_repeat('8', 32),
|
|
|
|
|
'created_at' => time(),
|
|
|
|
|
'updated_at' => time(),
|
|
|
|
|
]);
|
|
|
|
|
$quotaJob = CloudUserIsoService::import([
|
|
|
|
|
'name' => 'Quota overflow ISO',
|
|
|
|
|
'url' => 'https://images.example/quota-overflow.iso',
|
|
|
|
|
], ['user_id' => $quotaOwner, 'can_manage' => false]);
|
|
|
|
|
$quotaIso = CloudUserIso::query()->where('import_operation_id',
|
|
|
|
|
(string) $quotaJob['job_id'])->first();
|
|
|
|
|
$assert($quotaJob['status'] === 'failed'
|
|
|
|
|
&& (int) $quotaIso->size_bytes === 8192
|
|
|
|
|
&& (string) $quotaIso->state === 'failed',
|
|
|
|
|
'ISO quota aggregation ignored the persisted Agent result size.');
|
|
|
|
|
|
|
|
|
|
// A successful stale template response must not overwrite a concurrently
|
|
|
|
|
// installed delete operation or move the parent out of deleting.
|
|
|
|
|
$staleTemplateId = 'debian-12-stale-import';
|
|
|
|
|
$oldTemplateOperation = str_repeat('c', 32);
|
|
|
|
|
$newTemplateOperation = str_repeat('d', 32);
|
|
|
|
|
$generation = str_repeat('e', 32);
|
|
|
|
|
$now = time();
|
|
|
|
|
Capsule::table('cloud_templates')->insert([
|
|
|
|
|
'id' => $staleTemplateId,
|
|
|
|
|
'name' => 'Debian 12 stale import',
|
|
|
|
|
'os_family' => 'Debian',
|
|
|
|
|
'os_version' => '12',
|
|
|
|
|
'source_type' => 'remote',
|
|
|
|
|
'sealed_source_url' => 'test-sealed:template-source',
|
|
|
|
|
'image_bytes' => 0,
|
|
|
|
|
'sha256' => '',
|
|
|
|
|
'startup_script' => '',
|
|
|
|
|
'revision' => 1,
|
|
|
|
|
'generation_token' => $generation,
|
|
|
|
|
'state' => 'importing',
|
|
|
|
|
'created_at' => $now,
|
|
|
|
|
'updated_at' => $now,
|
|
|
|
|
]);
|
|
|
|
|
Capsule::table('cloud_template_replicas')->insert([
|
|
|
|
|
'host_id' => 101,
|
|
|
|
|
'template_id' => $staleTemplateId,
|
|
|
|
|
'revision' => 1,
|
|
|
|
|
'generation_token' => $generation,
|
|
|
|
|
'state' => 'pending',
|
|
|
|
|
'image_bytes' => 0,
|
|
|
|
|
'sha256' => '',
|
|
|
|
|
'operation_id' => $oldTemplateOperation,
|
|
|
|
|
'error_code' => '',
|
|
|
|
|
'created_at' => $now,
|
|
|
|
|
'updated_at' => $now,
|
|
|
|
|
]);
|
|
|
|
|
$agent->beforeResponse = static function (string $method) use (
|
|
|
|
|
$staleTemplateId, $newTemplateOperation
|
|
|
|
|
): void {
|
|
|
|
|
if ($method !== 'GET') {
|
|
|
|
|
return;
|
|
|
|
|
}
|
|
|
|
|
Capsule::table('cloud_templates')->where('id', $staleTemplateId)
|
|
|
|
|
->update(['state' => 'deleting']);
|
|
|
|
|
Capsule::table('cloud_template_replicas')
|
|
|
|
|
->where('template_id', $staleTemplateId)->update([
|
|
|
|
|
'state' => 'delete_succeeded',
|
|
|
|
|
'operation_id' => $newTemplateOperation,
|
|
|
|
|
'error_code' => '',
|
|
|
|
|
]);
|
|
|
|
|
};
|
|
|
|
|
$agent->getResponse = [
|
|
|
|
|
'status' => 'succeeded',
|
|
|
|
|
'result' => [
|
|
|
|
|
'id' => $staleTemplateId,
|
|
|
|
|
'revision' => 1,
|
|
|
|
|
'generation_token' => $generation,
|
|
|
|
|
'size_bytes' => 4096,
|
|
|
|
|
'sha256' => str_repeat('a', 64),
|
|
|
|
|
],
|
|
|
|
|
];
|
|
|
|
|
CloudTemplateCatalogService::importJob($staleTemplateId,
|
|
|
|
|
['user_id' => 1, 'can_manage' => true]);
|
|
|
|
|
$staleTemplate = CloudTemplate::find($staleTemplateId);
|
|
|
|
|
$staleTemplateReplica = CloudTemplateReplica::query()
|
|
|
|
|
->where('template_id', $staleTemplateId)->first();
|
|
|
|
|
$assert((string) $staleTemplate->state === 'deleting'
|
|
|
|
|
&& (string) $staleTemplateReplica->state === 'delete_succeeded'
|
|
|
|
|
&& (string) $staleTemplateReplica->operation_id
|
|
|
|
|
=== $newTemplateOperation,
|
|
|
|
|
'Stale template polling rolled back a concurrent terminal delete.');
|
|
|
|
|
|
|
|
|
|
// The state predicate is independently required: even the same operation
|
|
|
|
|
// ID cannot move a terminal ISO replica back to pending.
|
|
|
|
|
$staleIsoId = str_repeat('f', 32);
|
|
|
|
|
$staleIsoJob = str_repeat('1', 32);
|
|
|
|
|
$staleIsoOperation = str_repeat('2', 32);
|
|
|
|
|
Capsule::table('cloud_user_isos')->insert([
|
|
|
|
|
'id' => $staleIsoId,
|
|
|
|
|
'user_id' => 42,
|
|
|
|
|
'name' => 'Stale ISO',
|
|
|
|
|
'sealed_source_url' => 'test-sealed:user-iso-source',
|
|
|
|
|
'size_bytes' => 0,
|
|
|
|
|
'sha256' => '',
|
|
|
|
|
'state' => 'importing',
|
|
|
|
|
'import_operation_id' => $staleIsoJob,
|
|
|
|
|
'created_at' => $now,
|
|
|
|
|
'updated_at' => $now,
|
|
|
|
|
]);
|
|
|
|
|
Capsule::table('cloud_user_iso_replicas')->insert([
|
|
|
|
|
'host_id' => 101,
|
|
|
|
|
'user_iso_id' => $staleIsoId,
|
|
|
|
|
'user_id' => 42,
|
|
|
|
|
'state' => 'pending',
|
|
|
|
|
'size_bytes' => 0,
|
|
|
|
|
'sha256' => '',
|
|
|
|
|
'operation_id' => $staleIsoOperation,
|
|
|
|
|
'error_code' => '',
|
|
|
|
|
'created_at' => $now,
|
|
|
|
|
'updated_at' => $now,
|
|
|
|
|
]);
|
|
|
|
|
$agent->beforeResponse = static function (string $method) use (
|
|
|
|
|
$staleIsoId
|
|
|
|
|
): void {
|
|
|
|
|
if ($method !== 'GET') {
|
|
|
|
|
return;
|
|
|
|
|
}
|
|
|
|
|
Capsule::table('cloud_user_isos')->where('id', $staleIsoId)
|
|
|
|
|
->update(['state' => 'active']);
|
|
|
|
|
Capsule::table('cloud_user_iso_replicas')
|
|
|
|
|
->where('user_iso_id', $staleIsoId)->update([
|
|
|
|
|
'state' => 'active',
|
|
|
|
|
'error_code' => '',
|
|
|
|
|
]);
|
|
|
|
|
};
|
|
|
|
|
$agent->getResponse = null;
|
|
|
|
|
CloudUserIsoService::job($staleIsoJob,
|
|
|
|
|
['user_id' => 42, 'can_manage' => false]);
|
|
|
|
|
$staleIso = CloudUserIso::find($staleIsoId);
|
|
|
|
|
$staleIsoReplica = CloudUserIsoReplica::query()
|
|
|
|
|
->where('user_iso_id', $staleIsoId)->first();
|
|
|
|
|
$assert((string) $staleIso->state === 'active'
|
|
|
|
|
&& (string) $staleIsoReplica->state === 'active'
|
|
|
|
|
&& (string) $staleIsoReplica->operation_id === $staleIsoOperation,
|
|
|
|
|
'Stale ISO polling moved a terminal replica back to pending.');
|
|
|
|
|
|
|
|
|
|
echo "Catalog import persistence failures and stale-response CAS passed.\n";
|
|
|
|
|
}
|