You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
993 lines
48 KiB
PHP
993 lines
48 KiB
PHP
<?php
|
|
declare(strict_types=1);
|
|
|
|
namespace App\Services\Cloud {
|
|
/** Deterministic in-process Agent used to verify durable delete behavior. */
|
|
final class CatalogDeletionFakeAgent
|
|
{
|
|
public array $requests = [];
|
|
public array $operations = [];
|
|
public string $mode = 'running';
|
|
public ?string $reconcileMode = null;
|
|
|
|
public function request(string $method, string $target, array $actor,
|
|
?array $payload = null): array
|
|
{
|
|
$method = strtoupper($method);
|
|
$this->requests[] = compact('method', 'target', 'actor', 'payload');
|
|
if ($method === 'DELETE'
|
|
&& in_array($this->mode,
|
|
['pre_dispatch_busy', 'pre_dispatch_draining'], true)) {
|
|
$code = $this->mode === 'pre_dispatch_busy'
|
|
? 'agent_busy' : 'gateway_draining';
|
|
throw new CloudAgentError(
|
|
'request was not dispatched', 503, $code, true, 503,
|
|
false, true
|
|
);
|
|
}
|
|
if ($method === 'DELETE') {
|
|
$operationId = (string) ($payload['operation_id'] ?? '');
|
|
if (preg_match('#^/v1/templates/([^?]+)$#D', $target, $match)) {
|
|
$resource = ['resource_type' => 'template',
|
|
'id' => rawurldecode($match[1]), 'owner_id' => 0];
|
|
} elseif (preg_match('#^/v1/user-isos/([^?]+)\?owner_id=([0-9]+)$#D',
|
|
$target, $match)) {
|
|
$resource = ['resource_type' => 'user_iso',
|
|
'id' => rawurldecode($match[1]),
|
|
'owner_id' => (int) $match[2]];
|
|
} elseif (preg_match('#^/v1/rescue-templates/([^?]+)$#D',
|
|
$target, $match)) {
|
|
$resource = ['resource_type' => 'rescue_template',
|
|
'id' => rawurldecode($match[1]), 'owner_id' => 0];
|
|
} else {
|
|
throw new \RuntimeException('Unexpected fake Agent DELETE: ' . $target);
|
|
}
|
|
$this->operations[$operationId] = $resource;
|
|
} elseif ($method === 'POST'
|
|
&& preg_match(
|
|
'#^/v1/operations/([A-Za-z0-9_-]+)/reconcile$#D',
|
|
$target, $match)) {
|
|
$operationId = rawurldecode($match[1]);
|
|
$resource = $this->operations[$operationId] ?? null;
|
|
if (!is_array($resource)) {
|
|
throw new CloudAgentError('operation missing', 404,
|
|
'operation_not_found', false, 404);
|
|
}
|
|
return $this->response($operationId, $resource,
|
|
$this->reconcileMode ?? $this->mode);
|
|
} elseif ($method === 'GET'
|
|
&& preg_match('#^/v1/operations/([A-Za-z0-9_-]+)$#D',
|
|
$target, $match)) {
|
|
$operationId = rawurldecode($match[1]);
|
|
$resource = $this->operations[$operationId] ?? null;
|
|
if (!is_array($resource)) {
|
|
throw new CloudAgentError('operation missing', 404,
|
|
'operation_not_found', false, 404);
|
|
}
|
|
} else {
|
|
throw new \RuntimeException('Unexpected fake Agent request: '
|
|
. $method . ' ' . $target);
|
|
}
|
|
return $this->response($operationId, $resource, $this->mode);
|
|
}
|
|
|
|
private function response(string $operationId, array $resource,
|
|
string $mode): array
|
|
{
|
|
$notFoundCode = $resource['resource_type'] . '_not_found';
|
|
if ($mode === 'direct_not_found') {
|
|
throw new CloudAgentError('Direct Agent 404 without operation',
|
|
502, $notFoundCode, false, 404);
|
|
}
|
|
if ($mode === 'durable_not_found') {
|
|
return ['operation_id' => $operationId, 'status' => 'failed',
|
|
'retryable' => false,
|
|
'error' => ['code' => $notFoundCode]];
|
|
}
|
|
if ($mode === 'running') {
|
|
return ['operation_id' => $operationId, 'status' => 'running'];
|
|
}
|
|
if ($mode === 'failed') {
|
|
return ['operation_id' => $operationId, 'status' => 'failed',
|
|
'error' => ['code' => 'resource_busy']];
|
|
}
|
|
if ($mode === 'uncertain') {
|
|
return ['operation_id' => $operationId, 'status' => 'uncertain',
|
|
'error' => ['code' => 'delete_uncertain']];
|
|
}
|
|
if ($mode === 'missing_evidence') {
|
|
return ['operation_id' => $operationId, 'status' => 'succeeded',
|
|
'result' => ['deleted' => true]];
|
|
}
|
|
if (!in_array($mode, ['success', 'wrong_owner'], true)) {
|
|
throw new \RuntimeException('Unknown fake Agent mode: ' . $mode);
|
|
}
|
|
$result = [
|
|
'resource_type' => $resource['resource_type'],
|
|
'id' => $resource['id'],
|
|
'deleted' => true,
|
|
'managed_image_absent' => true,
|
|
'managed_metadata_absent' => true,
|
|
'managed_artifacts_absent' => true,
|
|
];
|
|
if ($resource['resource_type'] === 'user_iso') {
|
|
$result['owner_id'] = $resource['owner_id']
|
|
+ ($mode === 'wrong_owner' ? 1 : 0);
|
|
}
|
|
return ['operation_id' => $operationId, 'status' => 'succeeded',
|
|
'retryable' => false,
|
|
'result' => $result];
|
|
}
|
|
}
|
|
|
|
/** Replaces only Agent lookup in this isolated process. */
|
|
final class CloudAgentSettingsService
|
|
{
|
|
/** @var array<int, CatalogDeletionFakeAgent> */
|
|
public static array $clients = [];
|
|
|
|
public static function clientForHost(int $hostId,
|
|
float $timeoutSeconds = 20.0): CatalogDeletionFakeAgent
|
|
{
|
|
if (!isset(self::$clients[$hostId])) {
|
|
throw new \RuntimeException('Missing fake Agent for host ' . $hostId);
|
|
}
|
|
return self::$clients[$hostId];
|
|
}
|
|
}
|
|
}
|
|
|
|
namespace {
|
|
require dirname(__DIR__) . '/vendor/autoload.php';
|
|
|
|
use App\Models\CloudRescueTemplateDeletion;
|
|
use App\Models\CloudTemplate;
|
|
use App\Models\CloudTemplateReplica;
|
|
use App\Models\CloudUserIso;
|
|
use App\Models\CloudUserIsoReplica;
|
|
use App\Services\Cloud\CatalogDeletionFakeAgent;
|
|
use App\Services\Cloud\CloudAgentCatalogDeletion;
|
|
use App\Services\Cloud\CloudAgentSettingsService;
|
|
use App\Services\Cloud\CloudError;
|
|
use App\Services\Cloud\CloudRescueTemplateDeletionService;
|
|
use App\Services\Cloud\CloudTemplateCatalogService;
|
|
use App\Services\Cloud\CloudUserIsoService;
|
|
use Illuminate\Database\Capsule\Manager as Capsule;
|
|
use Illuminate\Database\Schema\Blueprint;
|
|
|
|
$assert = static function (bool $condition, string $message): void {
|
|
if (!$condition) {
|
|
throw new \RuntimeException($message);
|
|
}
|
|
};
|
|
$expectCloudError = static function (callable $callable, int $status,
|
|
string $message) use ($assert): void {
|
|
try {
|
|
$callable();
|
|
} catch (CloudError $error) {
|
|
$assert($error->httpStatus === $status,
|
|
$message . ' Unexpected status: ' . $error->httpStatus);
|
|
return;
|
|
}
|
|
throw new \RuntimeException($message);
|
|
};
|
|
$requestCount = static fn (CatalogDeletionFakeAgent $agent,
|
|
string $method): int => count(array_filter($agent->requests,
|
|
static fn (array $request): bool => $request['method'] === $method));
|
|
$proof = static function (string $type, string $id,
|
|
?int $ownerId = null): array {
|
|
$result = [
|
|
'resource_type' => $type,
|
|
'id' => $id,
|
|
'deleted' => true,
|
|
'managed_image_absent' => true,
|
|
'managed_metadata_absent' => true,
|
|
'managed_artifacts_absent' => true,
|
|
];
|
|
if ($ownerId !== null) {
|
|
$result['owner_id'] = $ownerId;
|
|
}
|
|
return ['status' => 'succeeded', 'retryable' => false,
|
|
'result' => $result];
|
|
};
|
|
|
|
$capsule = new Capsule();
|
|
$capsule->addConnection(['driver' => 'sqlite', 'database' => ':memory:'],
|
|
'default');
|
|
$capsule->setAsGlobal();
|
|
$capsule->bootEloquent();
|
|
$schema = $capsule->schema('default');
|
|
|
|
$schema->create('cloud_templates', static function (Blueprint $table): void {
|
|
$table->string('id')->primary();
|
|
$table->string('name');
|
|
$table->string('os_family');
|
|
$table->string('os_version');
|
|
$table->string('source_type');
|
|
$table->text('sealed_source_url');
|
|
$table->integer('image_bytes')->default(0);
|
|
$table->string('sha256')->default('');
|
|
$table->text('startup_script')->default('');
|
|
$table->integer('revision');
|
|
$table->string('generation_token')->unique();
|
|
$table->string('state');
|
|
$table->integer('created_at');
|
|
$table->integer('updated_at');
|
|
});
|
|
$schema->create('cloud_template_replicas',
|
|
static function (Blueprint $table): void {
|
|
$table->integer('host_id');
|
|
$table->string('template_id');
|
|
$table->integer('revision');
|
|
$table->string('generation_token');
|
|
$table->string('state');
|
|
$table->integer('image_bytes')->default(0);
|
|
$table->string('sha256')->default('');
|
|
$table->string('operation_id')->default('');
|
|
$table->string('error_code')->default('');
|
|
$table->integer('created_at');
|
|
$table->integer('updated_at');
|
|
$table->primary(['host_id', 'template_id']);
|
|
});
|
|
$schema->create('cloud_orders', static function (Blueprint $table): void {
|
|
$table->increments('id');
|
|
$table->string('status');
|
|
$table->text('spec_json');
|
|
});
|
|
$schema->create('cloud_jobs', static function (Blueprint $table): void {
|
|
$table->increments('id');
|
|
$table->string('kind');
|
|
$table->string('status');
|
|
$table->text('sealed_payload');
|
|
});
|
|
$schema->create('cloud_user_isos', static function (Blueprint $table): void {
|
|
$table->string('id')->primary();
|
|
$table->integer('user_id');
|
|
$table->string('name');
|
|
$table->text('sealed_source_url');
|
|
$table->integer('size_bytes')->default(0);
|
|
$table->string('sha256')->default('');
|
|
$table->string('state');
|
|
$table->string('import_operation_id')->default('');
|
|
$table->integer('created_at');
|
|
$table->integer('updated_at');
|
|
});
|
|
$schema->create('cloud_user_iso_replicas',
|
|
static function (Blueprint $table): void {
|
|
$table->integer('host_id');
|
|
$table->string('user_iso_id');
|
|
$table->integer('user_id');
|
|
$table->string('state');
|
|
$table->integer('size_bytes')->default(0);
|
|
$table->string('sha256')->default('');
|
|
$table->string('operation_id')->default('');
|
|
$table->string('error_code')->default('');
|
|
$table->integer('created_at');
|
|
$table->integer('updated_at');
|
|
$table->primary(['host_id', 'user_iso_id']);
|
|
});
|
|
$schema->create('cloud_rescue_template_deletions',
|
|
static function (Blueprint $table): void {
|
|
$table->integer('host_id');
|
|
$table->string('template_id');
|
|
$table->string('operation_id');
|
|
$table->string('state');
|
|
$table->string('error_code')->default('');
|
|
$table->integer('created_at');
|
|
$table->integer('updated_at');
|
|
$table->primary(['host_id', 'template_id']);
|
|
$table->unique(['host_id', 'operation_id']);
|
|
});
|
|
|
|
$seedTemplate = static function (string $id, int $hostId,
|
|
string $generation): void {
|
|
$now = time();
|
|
Capsule::table('cloud_templates')->insert([
|
|
'id' => $id, 'name' => $id, 'os_family' => 'Debian',
|
|
'os_version' => '12', 'source_type' => 'remote',
|
|
'sealed_source_url' => 'test-only-sealed-source',
|
|
'image_bytes' => 1024, 'sha256' => str_repeat('a', 64),
|
|
'startup_script' => '', 'revision' => 1,
|
|
'generation_token' => $generation, 'state' => 'active',
|
|
'created_at' => $now, 'updated_at' => $now,
|
|
]);
|
|
Capsule::table('cloud_template_replicas')->insert([
|
|
'host_id' => $hostId, 'template_id' => $id, 'revision' => 1,
|
|
'generation_token' => $generation, 'state' => 'active',
|
|
'image_bytes' => 1024, 'sha256' => str_repeat('a', 64),
|
|
'operation_id' => bin2hex(random_bytes(16)), 'error_code' => '',
|
|
'created_at' => $now, 'updated_at' => $now,
|
|
]);
|
|
};
|
|
$seedIso = static function (string $id, int $hostId,
|
|
int $ownerId): void {
|
|
$now = time();
|
|
Capsule::table('cloud_user_isos')->insert([
|
|
'id' => $id, 'user_id' => $ownerId, 'name' => $id,
|
|
'sealed_source_url' => 'test-only-sealed-source',
|
|
'size_bytes' => 2048, 'sha256' => str_repeat('b', 64),
|
|
'state' => 'active', 'import_operation_id' => bin2hex(random_bytes(16)),
|
|
'created_at' => $now, 'updated_at' => $now,
|
|
]);
|
|
Capsule::table('cloud_user_iso_replicas')->insert([
|
|
'host_id' => $hostId, 'user_iso_id' => $id,
|
|
'user_id' => $ownerId, 'state' => 'active',
|
|
'size_bytes' => 2048, 'sha256' => str_repeat('b', 64),
|
|
'operation_id' => bin2hex(random_bytes(16)), 'error_code' => '',
|
|
'created_at' => $now, 'updated_at' => $now,
|
|
]);
|
|
};
|
|
|
|
// Exact Agent cleanup evidence is fail-closed, including ISO ownership.
|
|
$templateId = 'debian-12-delete-test';
|
|
$assert(CloudAgentCatalogDeletion::confirms(
|
|
$proof('template', $templateId), 'template', $templateId
|
|
), 'Exact template cleanup evidence was rejected.');
|
|
$missing = $proof('template', $templateId);
|
|
unset($missing['result']['managed_artifacts_absent']);
|
|
$assert(!CloudAgentCatalogDeletion::confirms(
|
|
$missing, 'template', $templateId
|
|
), 'Incomplete cleanup evidence was accepted.');
|
|
$isoProofId = str_repeat('1', 32);
|
|
$assert(CloudAgentCatalogDeletion::confirms(
|
|
$proof('user_iso', $isoProofId, 42), 'user_iso', $isoProofId, 42
|
|
) && !CloudAgentCatalogDeletion::confirms(
|
|
$proof('user_iso', $isoProofId, 43), 'user_iso', $isoProofId, 42
|
|
), 'User ISO cleanup evidence was not bound to its exact owner.');
|
|
|
|
// Template: one destructive dispatch, persisted operation polling, and
|
|
// terminal uncertain lock when a succeeded response lacks proof.
|
|
$templateAgent = new CatalogDeletionFakeAgent();
|
|
CloudAgentSettingsService::$clients[10] = $templateAgent;
|
|
$generation = str_repeat('a', 32);
|
|
$seedTemplate($templateId, 10, $generation);
|
|
$templateJob = CloudTemplateCatalogService::delete(
|
|
$templateId, 1, $generation, ['user_id' => 1, 'can_manage' => true]
|
|
);
|
|
$replica = CloudTemplateReplica::query()->where('host_id', 10)
|
|
->where('template_id', $templateId)->first();
|
|
$operationId = (string) $replica->operation_id;
|
|
$assert($templateJob['deleted'] === false
|
|
&& $templateJob['status'] === 'running'
|
|
&& $templateJob['status_url']
|
|
=== '/api/v1/admin/cloud-templates/delete-jobs/' . $templateId
|
|
&& preg_match('/^[a-f0-9]{32}$/D', $operationId) === 1
|
|
&& $requestCount($templateAgent, 'DELETE') === 1,
|
|
'Template delete did not persist one durable operation before dispatch.');
|
|
CloudTemplateCatalogService::delete(
|
|
$templateId, 1, $generation, ['user_id' => 1, 'can_manage' => true]
|
|
);
|
|
$assert($requestCount($templateAgent, 'DELETE') === 1
|
|
&& str_ends_with((string) end($templateAgent->requests)['target'],
|
|
'/v1/operations/' . $operationId),
|
|
'Repeated template DELETE replayed destruction or changed operation ID.');
|
|
$templateAgent->mode = 'missing_evidence';
|
|
$uncertainTemplate = CloudTemplateCatalogService::deleteJob(
|
|
$templateId, ['user_id' => 1, 'can_manage' => true]
|
|
);
|
|
$requestsBeforeLockCheck = count($templateAgent->requests);
|
|
$lockedTemplate = CloudTemplateCatalogService::deleteJob(
|
|
$templateId, ['user_id' => 1, 'can_manage' => true]
|
|
);
|
|
$replica->refresh();
|
|
$assert($uncertainTemplate['status'] === 'uncertain'
|
|
&& $lockedTemplate['status'] === 'uncertain'
|
|
&& (string) $replica->state === 'delete_uncertain'
|
|
&& (string) $replica->error_code === 'delete_evidence_missing'
|
|
&& count($templateAgent->requests) === $requestsBeforeLockCheck,
|
|
'Template missing-proof outcome was not latched without replay.');
|
|
|
|
// Exact not_dispatched backpressure is the only case where DELETE may be
|
|
// sent again. It must retain and reuse the already-persisted operation ID.
|
|
$busyTemplateId = 'debian-12-delete-busy';
|
|
$busyTemplateAgent = new CatalogDeletionFakeAgent();
|
|
$busyTemplateAgent->mode = 'pre_dispatch_busy';
|
|
CloudAgentSettingsService::$clients[17] = $busyTemplateAgent;
|
|
$busyTemplateGeneration = str_repeat('1', 32);
|
|
$seedTemplate($busyTemplateId, 17, $busyTemplateGeneration);
|
|
$busyTemplate = CloudTemplateCatalogService::delete(
|
|
$busyTemplateId, 1, $busyTemplateGeneration,
|
|
['user_id' => 1, 'can_manage' => true]
|
|
);
|
|
$busyTemplateReplica = CloudTemplateReplica::query()
|
|
->where('template_id', $busyTemplateId)->first();
|
|
$busyTemplateOperation = (string) $busyTemplateReplica->operation_id;
|
|
$assert($busyTemplate['status'] === 'running'
|
|
&& (string) $busyTemplateReplica->state === 'delete_dispatch'
|
|
&& $busyTemplateAgent->operations === []
|
|
&& $requestCount($busyTemplateAgent, 'DELETE') === 1,
|
|
'Pre-dispatch template backpressure became a terminal deletion.');
|
|
$busyTemplateReplica->updated_at = time() - 6;
|
|
$busyTemplateReplica->save();
|
|
$busyTemplateAgent->mode = 'success';
|
|
$retriedBusyTemplate = CloudTemplateCatalogService::deleteJob(
|
|
$busyTemplateId, ['user_id' => 1, 'can_manage' => true]
|
|
);
|
|
$busyTemplateDeletes = array_values(array_filter(
|
|
$busyTemplateAgent->requests,
|
|
static fn (array $request): bool => $request['method'] === 'DELETE'
|
|
));
|
|
$assert($retriedBusyTemplate['status'] === 'succeeded'
|
|
&& count($busyTemplateDeletes) === 2
|
|
&& ($busyTemplateDeletes[0]['payload']['operation_id'] ?? null)
|
|
=== $busyTemplateOperation
|
|
&& ($busyTemplateDeletes[1]['payload']['operation_id'] ?? null)
|
|
=== $busyTemplateOperation,
|
|
'Template backpressure retry changed operation ID or did not finish.');
|
|
|
|
$sendingTemplateId = 'debian-12-delete-sending';
|
|
$sendingTemplateAgent = new CatalogDeletionFakeAgent();
|
|
CloudAgentSettingsService::$clients[12] = $sendingTemplateAgent;
|
|
$sendingGeneration = str_repeat('c', 32);
|
|
$seedTemplate($sendingTemplateId, 12, $sendingGeneration);
|
|
Capsule::table('cloud_templates')->where('id', $sendingTemplateId)
|
|
->update(['state' => 'deleting']);
|
|
Capsule::table('cloud_template_replicas')
|
|
->where('template_id', $sendingTemplateId)->update([
|
|
'state' => 'delete_sending',
|
|
'operation_id' => bin2hex(random_bytes(16)),
|
|
'updated_at' => time(),
|
|
]);
|
|
$freshSendingTemplate = CloudTemplateCatalogService::deleteJob(
|
|
$sendingTemplateId, ['user_id' => 1, 'can_manage' => true]
|
|
);
|
|
$assert($freshSendingTemplate['status'] === 'running'
|
|
&& $sendingTemplateAgent->requests === [],
|
|
'Fresh template delete_sending claim was polled before dispatch settled.');
|
|
Capsule::table('cloud_template_replicas')
|
|
->where('template_id', $sendingTemplateId)
|
|
->update(['updated_at' => time() - 91]);
|
|
$staleSendingTemplate = CloudTemplateCatalogService::deleteJob(
|
|
$sendingTemplateId, ['user_id' => 1, 'can_manage' => true]
|
|
);
|
|
$sendingTemplateReplica = CloudTemplateReplica::query()
|
|
->where('template_id', $sendingTemplateId)->first();
|
|
$assert($staleSendingTemplate['status'] === 'uncertain'
|
|
&& (string) $sendingTemplateReplica->error_code
|
|
=== 'operation_not_found'
|
|
&& count($sendingTemplateAgent->requests) === 1
|
|
&& $sendingTemplateAgent->requests[0]['method'] === 'GET'
|
|
&& $sendingTemplateAgent->requests[0]['target']
|
|
=== '/v1/operations/'
|
|
. (string) $sendingTemplateReplica->operation_id
|
|
&& $requestCount($sendingTemplateAgent, 'DELETE') === 0,
|
|
'Stale template sending claim was not queried safely by operation ID.');
|
|
|
|
$reconcileTemplateId = 'debian-12-delete-reconcile';
|
|
$reconcileTemplateAgent = new CatalogDeletionFakeAgent();
|
|
CloudAgentSettingsService::$clients[15] = $reconcileTemplateAgent;
|
|
$reconcileTemplateGeneration = str_repeat('f', 32);
|
|
$seedTemplate($reconcileTemplateId, 15, $reconcileTemplateGeneration);
|
|
CloudTemplateCatalogService::delete(
|
|
$reconcileTemplateId, 1, $reconcileTemplateGeneration,
|
|
['user_id' => 1, 'can_manage' => true]
|
|
);
|
|
$reconcileTemplateReplica = CloudTemplateReplica::query()
|
|
->where('template_id', $reconcileTemplateId)->first();
|
|
$reconcileTemplateOperation =
|
|
(string) $reconcileTemplateReplica->operation_id;
|
|
$reconcileTemplateAgent->mode = 'uncertain';
|
|
$reconcileTemplateAgent->reconcileMode = 'success';
|
|
$beforeTemplateReconcile = count($reconcileTemplateAgent->requests);
|
|
$reconciledTemplateDelete = CloudTemplateCatalogService::deleteJob(
|
|
$reconcileTemplateId, ['user_id' => 1, 'can_manage' => true]
|
|
);
|
|
$templateReconcileRequests = array_slice(
|
|
$reconcileTemplateAgent->requests, $beforeTemplateReconcile
|
|
);
|
|
$assert($reconciledTemplateDelete['status'] === 'succeeded'
|
|
&& $requestCount($reconcileTemplateAgent, 'DELETE') === 1
|
|
&& count($templateReconcileRequests) === 2
|
|
&& $templateReconcileRequests[0]['method'] === 'GET'
|
|
&& $templateReconcileRequests[0]['target']
|
|
=== '/v1/operations/' . $reconcileTemplateOperation
|
|
&& $templateReconcileRequests[1]['method'] === 'POST'
|
|
&& $templateReconcileRequests[1]['target']
|
|
=== '/v1/operations/' . $reconcileTemplateOperation
|
|
. '/reconcile',
|
|
'Template delete uncertainty replayed DELETE or changed operation ID.');
|
|
|
|
$trueUncertainTemplateId = 'debian-12-delete-uncertain';
|
|
$trueUncertainTemplateAgent = new CatalogDeletionFakeAgent();
|
|
CloudAgentSettingsService::$clients[16] = $trueUncertainTemplateAgent;
|
|
$trueUncertainGeneration = str_repeat('0', 32);
|
|
$seedTemplate(
|
|
$trueUncertainTemplateId, 16, $trueUncertainGeneration
|
|
);
|
|
CloudTemplateCatalogService::delete(
|
|
$trueUncertainTemplateId, 1, $trueUncertainGeneration,
|
|
['user_id' => 1, 'can_manage' => true]
|
|
);
|
|
$trueUncertainTemplateAgent->mode = 'uncertain';
|
|
$trueUncertainTemplateAgent->reconcileMode = 'uncertain';
|
|
$trueUncertainDelete = CloudTemplateCatalogService::deleteJob(
|
|
$trueUncertainTemplateId, ['user_id' => 1, 'can_manage' => true]
|
|
);
|
|
$requestsAtTrueUncertain = count($trueUncertainTemplateAgent->requests);
|
|
$trueUncertainDeleteAgain = CloudTemplateCatalogService::deleteJob(
|
|
$trueUncertainTemplateId, ['user_id' => 1, 'can_manage' => true]
|
|
);
|
|
$assert($trueUncertainDelete['status'] === 'uncertain'
|
|
&& $trueUncertainDeleteAgain['status'] === 'uncertain'
|
|
&& $requestCount($trueUncertainTemplateAgent, 'DELETE') === 1
|
|
&& count($trueUncertainTemplateAgent->requests)
|
|
=== $requestsAtTrueUncertain,
|
|
'A genuinely uncertain template delete was replayed or left pollable.');
|
|
|
|
$successfulTemplateId = 'debian-12-delete-success';
|
|
$successfulTemplateAgent = new CatalogDeletionFakeAgent();
|
|
$successfulTemplateAgent->mode = 'success';
|
|
CloudAgentSettingsService::$clients[11] = $successfulTemplateAgent;
|
|
$successfulGeneration = str_repeat('b', 32);
|
|
$seedTemplate($successfulTemplateId, 11, $successfulGeneration);
|
|
$deletedTemplate = CloudTemplateCatalogService::delete(
|
|
$successfulTemplateId, 1, $successfulGeneration,
|
|
['user_id' => 1, 'can_manage' => true]
|
|
);
|
|
$assert($deletedTemplate['deleted'] === true
|
|
&& $deletedTemplate['status'] === 'succeeded'
|
|
&& (string) CloudTemplate::find($successfulTemplateId)->state === 'retired'
|
|
&& !CloudTemplateReplica::query()->where('template_id',
|
|
$successfulTemplateId)->exists()
|
|
&& $requestCount($successfulTemplateAgent, 'DELETE') === 1,
|
|
'Exact template cleanup proof did not safely finalize retirement.');
|
|
|
|
$directTemplateId = 'debian-12-direct-404';
|
|
$directTemplateAgent = new CatalogDeletionFakeAgent();
|
|
$directTemplateAgent->mode = 'direct_not_found';
|
|
CloudAgentSettingsService::$clients[13] = $directTemplateAgent;
|
|
$directTemplateGeneration = str_repeat('d', 32);
|
|
$seedTemplate($directTemplateId, 13, $directTemplateGeneration);
|
|
$directTemplate = CloudTemplateCatalogService::delete(
|
|
$directTemplateId, 1, $directTemplateGeneration,
|
|
['user_id' => 1, 'can_manage' => true]
|
|
);
|
|
$directTemplateReplica = CloudTemplateReplica::query()
|
|
->where('template_id', $directTemplateId)->first();
|
|
$assert($directTemplate['deleted'] === false
|
|
&& $directTemplate['status'] === 'uncertain'
|
|
&& (string) CloudTemplate::find($directTemplateId)->state === 'deleting'
|
|
&& (string) $directTemplateReplica->state === 'delete_uncertain'
|
|
&& (string) $directTemplateReplica->error_code === 'operation_not_found',
|
|
'Unmarked template 404 was made terminal without no-dispatch proof.');
|
|
$durableTemplateId = 'debian-12-durable-404';
|
|
$durableTemplateAgent = new CatalogDeletionFakeAgent();
|
|
$durableTemplateAgent->mode = 'durable_not_found';
|
|
CloudAgentSettingsService::$clients[14] = $durableTemplateAgent;
|
|
$durableTemplateGeneration = str_repeat('e', 32);
|
|
$seedTemplate($durableTemplateId, 14, $durableTemplateGeneration);
|
|
$durableTemplate = CloudTemplateCatalogService::delete(
|
|
$durableTemplateId, 1, $durableTemplateGeneration,
|
|
['user_id' => 1, 'can_manage' => true]
|
|
);
|
|
$assert($durableTemplate['deleted'] === true
|
|
&& (string) CloudTemplate::find($durableTemplateId)->state === 'retired',
|
|
'Durable non-retryable template_not_found operation did not complete deletion.');
|
|
|
|
// User ISO: owner-scoped status plus strict owner evidence.
|
|
$isoId = str_repeat('2', 32);
|
|
$isoAgent = new CatalogDeletionFakeAgent();
|
|
CloudAgentSettingsService::$clients[20] = $isoAgent;
|
|
$seedIso($isoId, 20, 42);
|
|
$isoJob = CloudUserIsoService::delete($isoId,
|
|
['user_id' => 42, 'can_manage' => false]);
|
|
$isoReplica = CloudUserIsoReplica::query()->where('host_id', 20)
|
|
->where('user_iso_id', $isoId)->first();
|
|
$isoOperation = (string) $isoReplica->operation_id;
|
|
$assert($isoJob['status'] === 'running'
|
|
&& $isoJob['status_url'] === '/api/v1/user-isos/delete-jobs/' . $isoId
|
|
&& $requestCount($isoAgent, 'DELETE') === 1,
|
|
'User ISO delete did not return its durable owner-scoped job.');
|
|
$requestsBeforeWrongOwner = count($isoAgent->requests);
|
|
$expectCloudError(static fn () => CloudUserIsoService::deleteJob(
|
|
$isoId, ['user_id' => 43, 'can_manage' => false]
|
|
), 404, 'Another user read the ISO delete job.');
|
|
$assert(count($isoAgent->requests) === $requestsBeforeWrongOwner,
|
|
'Wrong-owner ISO status lookup contacted an Agent.');
|
|
CloudUserIsoService::delete($isoId,
|
|
['user_id' => 42, 'can_manage' => false]);
|
|
$assert($requestCount($isoAgent, 'DELETE') === 1
|
|
&& str_ends_with((string) end($isoAgent->requests)['target'],
|
|
'/v1/operations/' . $isoOperation),
|
|
'Repeated ISO DELETE replayed destruction or changed operation ID.');
|
|
$isoAgent->mode = 'wrong_owner';
|
|
$uncertainIso = CloudUserIsoService::deleteJob(
|
|
$isoId, ['user_id' => 42, 'can_manage' => false]
|
|
);
|
|
$requestsBeforeIsoLockCheck = count($isoAgent->requests);
|
|
$lockedIso = CloudUserIsoService::delete(
|
|
$isoId, ['user_id' => 42, 'can_manage' => false]
|
|
);
|
|
$isoReplica->refresh();
|
|
$assert($uncertainIso['status'] === 'uncertain'
|
|
&& $lockedIso['status'] === 'uncertain'
|
|
&& (string) $isoReplica->state === 'delete_uncertain'
|
|
&& (string) $isoReplica->error_code === 'delete_evidence_missing'
|
|
&& count($isoAgent->requests) === $requestsBeforeIsoLockCheck,
|
|
'ISO owner mismatch was not latched without destructive replay.');
|
|
|
|
$busyIsoId = str_repeat('b', 32);
|
|
$busyIsoAgent = new CatalogDeletionFakeAgent();
|
|
$busyIsoAgent->mode = 'pre_dispatch_draining';
|
|
CloudAgentSettingsService::$clients[26] = $busyIsoAgent;
|
|
$seedIso($busyIsoId, 26, 42);
|
|
$busyIso = CloudUserIsoService::delete(
|
|
$busyIsoId, ['user_id' => 42, 'can_manage' => false]
|
|
);
|
|
$busyIsoReplica = CloudUserIsoReplica::query()
|
|
->where('user_iso_id', $busyIsoId)->first();
|
|
$busyIsoOperation = (string) $busyIsoReplica->operation_id;
|
|
$assert($busyIso['status'] === 'running'
|
|
&& (string) $busyIsoReplica->state === 'delete_dispatch'
|
|
&& $busyIsoAgent->operations === []
|
|
&& $requestCount($busyIsoAgent, 'DELETE') === 1,
|
|
'Pre-dispatch ISO backpressure became a terminal deletion.');
|
|
$busyIsoReplica->updated_at = time() - 6;
|
|
$busyIsoReplica->save();
|
|
$busyIsoAgent->mode = 'success';
|
|
$retriedBusyIso = CloudUserIsoService::deleteJob(
|
|
$busyIsoId, ['user_id' => 42, 'can_manage' => false]
|
|
);
|
|
$busyIsoDeletes = array_values(array_filter(
|
|
$busyIsoAgent->requests,
|
|
static fn (array $request): bool => $request['method'] === 'DELETE'
|
|
));
|
|
$assert($retriedBusyIso['status'] === 'succeeded'
|
|
&& count($busyIsoDeletes) === 2
|
|
&& ($busyIsoDeletes[0]['payload']['operation_id'] ?? null)
|
|
=== $busyIsoOperation
|
|
&& ($busyIsoDeletes[1]['payload']['operation_id'] ?? null)
|
|
=== $busyIsoOperation,
|
|
'ISO backpressure retry changed operation ID or did not finish.');
|
|
|
|
$sendingIsoId = str_repeat('6', 32);
|
|
$sendingIsoAgent = new CatalogDeletionFakeAgent();
|
|
CloudAgentSettingsService::$clients[22] = $sendingIsoAgent;
|
|
$seedIso($sendingIsoId, 22, 42);
|
|
Capsule::table('cloud_user_isos')->where('id', $sendingIsoId)
|
|
->update(['state' => 'deleting']);
|
|
Capsule::table('cloud_user_iso_replicas')
|
|
->where('user_iso_id', $sendingIsoId)->update([
|
|
'state' => 'delete_sending',
|
|
'operation_id' => bin2hex(random_bytes(16)),
|
|
'updated_at' => time(),
|
|
]);
|
|
$freshSendingIso = CloudUserIsoService::deleteJob(
|
|
$sendingIsoId, ['user_id' => 42, 'can_manage' => false]
|
|
);
|
|
$assert($freshSendingIso['status'] === 'running'
|
|
&& $sendingIsoAgent->requests === [],
|
|
'Fresh ISO delete_sending claim was polled before dispatch settled.');
|
|
Capsule::table('cloud_user_iso_replicas')
|
|
->where('user_iso_id', $sendingIsoId)
|
|
->update(['updated_at' => time() - 91]);
|
|
$staleSendingIso = CloudUserIsoService::deleteJob(
|
|
$sendingIsoId, ['user_id' => 42, 'can_manage' => false]
|
|
);
|
|
$sendingIsoReplica = CloudUserIsoReplica::query()
|
|
->where('user_iso_id', $sendingIsoId)->first();
|
|
$assert($staleSendingIso['status'] === 'uncertain'
|
|
&& (string) $sendingIsoReplica->error_code === 'operation_not_found'
|
|
&& count($sendingIsoAgent->requests) === 1
|
|
&& $sendingIsoAgent->requests[0]['method'] === 'GET'
|
|
&& $sendingIsoAgent->requests[0]['target']
|
|
=== '/v1/operations/' . (string) $sendingIsoReplica->operation_id
|
|
&& $requestCount($sendingIsoAgent, 'DELETE') === 0,
|
|
'Stale ISO sending claim was not queried safely by operation ID.');
|
|
|
|
$reconcileIsoId = str_repeat('d', 32);
|
|
$reconcileIsoAgent = new CatalogDeletionFakeAgent();
|
|
CloudAgentSettingsService::$clients[25] = $reconcileIsoAgent;
|
|
$seedIso($reconcileIsoId, 25, 42);
|
|
CloudUserIsoService::delete(
|
|
$reconcileIsoId, ['user_id' => 42, 'can_manage' => false]
|
|
);
|
|
$reconcileIsoReplica = CloudUserIsoReplica::query()
|
|
->where('user_iso_id', $reconcileIsoId)->first();
|
|
$reconcileIsoOperation = (string) $reconcileIsoReplica->operation_id;
|
|
$reconcileIsoAgent->mode = 'uncertain';
|
|
$reconcileIsoAgent->reconcileMode = 'success';
|
|
$beforeIsoReconcile = count($reconcileIsoAgent->requests);
|
|
$reconciledIsoDelete = CloudUserIsoService::deleteJob(
|
|
$reconcileIsoId, ['user_id' => 42, 'can_manage' => false]
|
|
);
|
|
$isoReconcileRequests = array_slice(
|
|
$reconcileIsoAgent->requests, $beforeIsoReconcile
|
|
);
|
|
$assert($reconciledIsoDelete['status'] === 'succeeded'
|
|
&& $requestCount($reconcileIsoAgent, 'DELETE') === 1
|
|
&& count($isoReconcileRequests) === 2
|
|
&& $isoReconcileRequests[0]['method'] === 'GET'
|
|
&& $isoReconcileRequests[0]['target']
|
|
=== '/v1/operations/' . $reconcileIsoOperation
|
|
&& $isoReconcileRequests[1]['method'] === 'POST'
|
|
&& $isoReconcileRequests[1]['target']
|
|
=== '/v1/operations/' . $reconcileIsoOperation . '/reconcile',
|
|
'User ISO delete uncertainty replayed DELETE or changed operation ID.');
|
|
|
|
$successfulIsoId = str_repeat('3', 32);
|
|
$successfulIsoAgent = new CatalogDeletionFakeAgent();
|
|
$successfulIsoAgent->mode = 'success';
|
|
CloudAgentSettingsService::$clients[21] = $successfulIsoAgent;
|
|
$seedIso($successfulIsoId, 21, 42);
|
|
$deletedIso = CloudUserIsoService::delete($successfulIsoId,
|
|
['user_id' => 42, 'can_manage' => false]);
|
|
$successfulIsoTombstone = CloudUserIso::find($successfulIsoId);
|
|
$polledDeletedIso = CloudUserIsoService::deleteJob($successfulIsoId,
|
|
['user_id' => 42, 'can_manage' => false]);
|
|
$repeatedDeletedIso = CloudUserIsoService::delete($successfulIsoId,
|
|
['user_id' => 42, 'can_manage' => false]);
|
|
$historicalImport = CloudUserIsoService::job(
|
|
(string) $successfulIsoTombstone->import_operation_id,
|
|
['user_id' => 42, 'can_manage' => false]
|
|
);
|
|
$listedAfterDelete = CloudUserIsoService::listing(42);
|
|
$assert($deletedIso['deleted'] === true
|
|
&& $deletedIso['status'] === 'succeeded'
|
|
&& $polledDeletedIso['status'] === 'succeeded'
|
|
&& $repeatedDeletedIso['status'] === 'succeeded'
|
|
&& $historicalImport['status'] === 'succeeded'
|
|
&& $historicalImport['deleted'] === true
|
|
&& !in_array($successfulIsoId, array_column(
|
|
$listedAfterDelete['isos'], 'id'
|
|
), true)
|
|
&& $successfulIsoTombstone !== null
|
|
&& (string) $successfulIsoTombstone->state === 'retired'
|
|
&& (string) $successfulIsoTombstone->name === ''
|
|
&& (string) $successfulIsoTombstone->sealed_source_url === ''
|
|
&& (string) $successfulIsoTombstone->sha256 === ''
|
|
&& (int) $successfulIsoTombstone->size_bytes === 0
|
|
&& !CloudUserIsoReplica::query()->where('user_iso_id',
|
|
$successfulIsoId)->exists()
|
|
&& $requestCount($successfulIsoAgent, 'DELETE') === 1,
|
|
'Exact ISO cleanup did not leave a pollable, data-free tombstone.');
|
|
|
|
$directIsoId = str_repeat('7', 32);
|
|
$directIsoAgent = new CatalogDeletionFakeAgent();
|
|
$directIsoAgent->mode = 'direct_not_found';
|
|
CloudAgentSettingsService::$clients[23] = $directIsoAgent;
|
|
$seedIso($directIsoId, 23, 42);
|
|
$directIso = CloudUserIsoService::delete($directIsoId,
|
|
['user_id' => 42, 'can_manage' => false]);
|
|
$directIsoReplica = CloudUserIsoReplica::query()
|
|
->where('user_iso_id', $directIsoId)->first();
|
|
$assert($directIso['deleted'] === false
|
|
&& $directIso['status'] === 'uncertain'
|
|
&& (string) CloudUserIso::find($directIsoId)->state === 'deleting'
|
|
&& (string) $directIsoReplica->state === 'delete_uncertain'
|
|
&& (string) $directIsoReplica->error_code === 'operation_not_found',
|
|
'Unmarked ISO 404 was made terminal without no-dispatch proof.');
|
|
$durableIsoId = str_repeat('8', 32);
|
|
$durableIsoAgent = new CatalogDeletionFakeAgent();
|
|
$durableIsoAgent->mode = 'durable_not_found';
|
|
CloudAgentSettingsService::$clients[24] = $durableIsoAgent;
|
|
$seedIso($durableIsoId, 24, 42);
|
|
$durableIso = CloudUserIsoService::delete($durableIsoId,
|
|
['user_id' => 42, 'can_manage' => false]);
|
|
$durableIsoTombstone = CloudUserIso::find($durableIsoId);
|
|
$assert($durableIso['deleted'] === true
|
|
&& (string) $durableIsoTombstone->state === 'retired',
|
|
'Durable non-retryable user_iso_not_found operation did not complete deletion.');
|
|
|
|
// Rescue: terminal tombstones survive retries and prevent redispatch.
|
|
$busyRescueId = str_repeat('c', 32);
|
|
$busyRescueAgent = new CatalogDeletionFakeAgent();
|
|
$busyRescueAgent->mode = 'pre_dispatch_busy';
|
|
CloudAgentSettingsService::$clients[35] = $busyRescueAgent;
|
|
$busyRescue = CloudRescueTemplateDeletionService::delete(
|
|
35, $busyRescueId, ['user_id' => 1, 'can_manage' => true]
|
|
);
|
|
$busyRescueRecord = CloudRescueTemplateDeletion::query()
|
|
->where('host_id', 35)->where('template_id', $busyRescueId)->first();
|
|
$busyRescueOperation = (string) $busyRescueRecord->operation_id;
|
|
$assert($busyRescue['status'] === 'running'
|
|
&& (string) $busyRescueRecord->state === 'dispatching'
|
|
&& $busyRescueAgent->operations === []
|
|
&& $requestCount($busyRescueAgent, 'DELETE') === 1,
|
|
'Pre-dispatch rescue backpressure became a terminal deletion.');
|
|
// This model has a composite key and deliberately must not use save().
|
|
Capsule::table('cloud_rescue_template_deletions')
|
|
->where('host_id', 35)->where('template_id', $busyRescueId)
|
|
->where('operation_id', $busyRescueOperation)
|
|
->update(['updated_at' => time() - 6]);
|
|
$busyRescueAgent->mode = 'success';
|
|
$retriedBusyRescue = CloudRescueTemplateDeletionService::job(
|
|
35, $busyRescueOperation,
|
|
['user_id' => 1, 'can_manage' => true]
|
|
);
|
|
$busyRescueDeletes = array_values(array_filter(
|
|
$busyRescueAgent->requests,
|
|
static fn (array $request): bool => $request['method'] === 'DELETE'
|
|
));
|
|
$assert($retriedBusyRescue['status'] === 'succeeded'
|
|
&& count($busyRescueDeletes) === 2
|
|
&& ($busyRescueDeletes[0]['payload']['operation_id'] ?? null)
|
|
=== $busyRescueOperation
|
|
&& ($busyRescueDeletes[1]['payload']['operation_id'] ?? null)
|
|
=== $busyRescueOperation,
|
|
'Rescue backpressure retry changed operation ID or did not finish.');
|
|
|
|
$rescueId = str_repeat('4', 32);
|
|
$rescueAgent = new CatalogDeletionFakeAgent();
|
|
CloudAgentSettingsService::$clients[30] = $rescueAgent;
|
|
$rescueJob = CloudRescueTemplateDeletionService::delete(
|
|
30, $rescueId, ['user_id' => 1, 'can_manage' => true]
|
|
);
|
|
$rescueRecord = CloudRescueTemplateDeletion::query()
|
|
->where('host_id', 30)->where('template_id', $rescueId)->first();
|
|
$rescueOperation = (string) $rescueRecord->operation_id;
|
|
$assert($rescueJob['status'] === 'running'
|
|
&& $rescueJob['operation_id'] === $rescueOperation
|
|
&& $rescueJob['status_url']
|
|
=== '/api/v1/admin/rescue-templates/delete-jobs/'
|
|
. $rescueOperation . '?host_id=30'
|
|
&& $requestCount($rescueAgent, 'DELETE') === 1,
|
|
'Rescue delete did not persist its host-scoped operation tombstone.');
|
|
CloudRescueTemplateDeletionService::delete(
|
|
30, $rescueId, ['user_id' => 1, 'can_manage' => true]
|
|
);
|
|
$assert($requestCount($rescueAgent, 'DELETE') === 1
|
|
&& str_ends_with((string) end($rescueAgent->requests)['target'],
|
|
'/v1/operations/' . $rescueOperation),
|
|
'Repeated rescue DELETE replayed destruction or changed operation ID.');
|
|
$rescueAgent->mode = 'missing_evidence';
|
|
$uncertainRescue = CloudRescueTemplateDeletionService::job(
|
|
30, $rescueOperation, ['user_id' => 1, 'can_manage' => true]
|
|
);
|
|
$requestsBeforeRescueLockCheck = count($rescueAgent->requests);
|
|
$lockedRescue = CloudRescueTemplateDeletionService::delete(
|
|
30, $rescueId, ['user_id' => 1, 'can_manage' => true]
|
|
);
|
|
$rescueRecord = CloudRescueTemplateDeletion::query()
|
|
->where('host_id', 30)->where('template_id', $rescueId)
|
|
->where('operation_id', $rescueOperation)->first();
|
|
$assert($uncertainRescue['status'] === 'uncertain'
|
|
&& $lockedRescue['status'] === 'uncertain'
|
|
&& (string) $rescueRecord->state === 'uncertain'
|
|
&& (string) $rescueRecord->error_code === 'delete_evidence_missing'
|
|
&& count($rescueAgent->requests) === $requestsBeforeRescueLockCheck,
|
|
'Rescue missing-proof result was not retained as a no-replay tombstone.');
|
|
|
|
$reconcileRescueId = str_repeat('e', 32);
|
|
$reconcileRescueAgent = new CatalogDeletionFakeAgent();
|
|
CloudAgentSettingsService::$clients[34] = $reconcileRescueAgent;
|
|
$reconcileRescue = CloudRescueTemplateDeletionService::delete(
|
|
34, $reconcileRescueId, ['user_id' => 1, 'can_manage' => true]
|
|
);
|
|
$reconcileRescueOperation = (string) $reconcileRescue['operation_id'];
|
|
$reconcileRescueAgent->mode = 'uncertain';
|
|
$reconcileRescueAgent->reconcileMode = 'success';
|
|
$beforeRescueReconcile = count($reconcileRescueAgent->requests);
|
|
$reconciledRescueDelete = CloudRescueTemplateDeletionService::job(
|
|
34, $reconcileRescueOperation,
|
|
['user_id' => 1, 'can_manage' => true]
|
|
);
|
|
$rescueReconcileRequests = array_slice(
|
|
$reconcileRescueAgent->requests, $beforeRescueReconcile
|
|
);
|
|
$assert($reconciledRescueDelete['status'] === 'succeeded'
|
|
&& $requestCount($reconcileRescueAgent, 'DELETE') === 1
|
|
&& count($rescueReconcileRequests) === 2
|
|
&& $rescueReconcileRequests[0]['method'] === 'GET'
|
|
&& $rescueReconcileRequests[0]['target']
|
|
=== '/v1/operations/' . $reconcileRescueOperation
|
|
&& $rescueReconcileRequests[1]['method'] === 'POST'
|
|
&& $rescueReconcileRequests[1]['target']
|
|
=== '/v1/operations/' . $reconcileRescueOperation . '/reconcile',
|
|
'Rescue-template delete uncertainty replayed DELETE or changed operation ID.');
|
|
|
|
$successfulRescueId = str_repeat('5', 32);
|
|
$successfulRescueAgent = new CatalogDeletionFakeAgent();
|
|
$successfulRescueAgent->mode = 'success';
|
|
CloudAgentSettingsService::$clients[31] = $successfulRescueAgent;
|
|
$deletedRescue = CloudRescueTemplateDeletionService::delete(
|
|
31, $successfulRescueId, ['user_id' => 1, 'can_manage' => true]
|
|
);
|
|
$requestsAfterRescueSuccess = count($successfulRescueAgent->requests);
|
|
$rescueStatus = CloudRescueTemplateDeletionService::job(
|
|
31, (string) $deletedRescue['operation_id'],
|
|
['user_id' => 1, 'can_manage' => true]
|
|
);
|
|
$rescueRetry = CloudRescueTemplateDeletionService::delete(
|
|
31, $successfulRescueId, ['user_id' => 1, 'can_manage' => true]
|
|
);
|
|
$assert($deletedRescue['deleted'] === true
|
|
&& $rescueStatus['status'] === 'succeeded'
|
|
&& $rescueRetry['status'] === 'succeeded'
|
|
&& CloudRescueTemplateDeletion::query()->where('host_id', 31)
|
|
->where('template_id', $successfulRescueId)->exists()
|
|
&& $requestCount($successfulRescueAgent, 'DELETE') === 1
|
|
&& count($successfulRescueAgent->requests)
|
|
=== $requestsAfterRescueSuccess,
|
|
'Successful rescue tombstone did not make every retry read-only.');
|
|
|
|
// A slower status poll may carry a stale pre-terminal model snapshot.
|
|
// Its result must not move the durable success tombstone backwards.
|
|
$staleRescueRecord = clone CloudRescueTemplateDeletion::query()
|
|
->where('host_id', 31)->where('template_id', $successfulRescueId)
|
|
->first();
|
|
$staleRescueRecord->state = 'running';
|
|
$markRescueDeletion = new ReflectionMethod(
|
|
CloudRescueTemplateDeletionService::class, 'mark'
|
|
);
|
|
$markRescueDeletion->invoke(
|
|
null, $staleRescueRecord, 'running', 'late_poll_result'
|
|
);
|
|
$successfulRescueRecord = CloudRescueTemplateDeletion::query()
|
|
->where('host_id', 31)->where('template_id', $successfulRescueId)
|
|
->first();
|
|
$assert((string) $successfulRescueRecord->state === 'succeeded'
|
|
&& (string) $successfulRescueRecord->error_code === '',
|
|
'A stale rescue deletion poll overwrote its terminal tombstone.');
|
|
|
|
$directRescueId = str_repeat('9', 32);
|
|
$directRescueAgent = new CatalogDeletionFakeAgent();
|
|
$directRescueAgent->mode = 'direct_not_found';
|
|
CloudAgentSettingsService::$clients[32] = $directRescueAgent;
|
|
$directRescue = CloudRescueTemplateDeletionService::delete(
|
|
32, $directRescueId, ['user_id' => 1, 'can_manage' => true]
|
|
);
|
|
$directRescueRecord = CloudRescueTemplateDeletion::query()
|
|
->where('host_id', 32)->where('template_id', $directRescueId)->first();
|
|
$assert($directRescue['deleted'] === false
|
|
&& $directRescue['status'] === 'running'
|
|
&& (string) $directRescueRecord->state === 'running'
|
|
&& (string) $directRescueRecord->error_code
|
|
=== 'rescue_template_not_found',
|
|
'Unmarked rescue-template 404 was made terminal without no-dispatch proof.');
|
|
$durableRescueId = str_repeat('a', 32);
|
|
$durableRescueAgent = new CatalogDeletionFakeAgent();
|
|
$durableRescueAgent->mode = 'durable_not_found';
|
|
CloudAgentSettingsService::$clients[33] = $durableRescueAgent;
|
|
$durableRescue = CloudRescueTemplateDeletionService::delete(
|
|
33, $durableRescueId, ['user_id' => 1, 'can_manage' => true]
|
|
);
|
|
$assert($durableRescue['deleted'] === true
|
|
&& $durableRescue['status'] === 'succeeded'
|
|
&& CloudRescueTemplateDeletion::query()->where('host_id', 33)
|
|
->where('template_id', $durableRescueId)
|
|
->where('state', 'succeeded')->exists(),
|
|
'Durable non-retryable rescue_template_not_found operation did not complete deletion.');
|
|
|
|
// The same rescue template ID may exist on several hosts. Every reload
|
|
// must retain the complete host/template/operation identity.
|
|
$sharedRescueId = str_repeat('b', 32);
|
|
$otherOperation = str_repeat('c', 32);
|
|
Capsule::table('cloud_rescue_template_deletions')->insert([
|
|
'host_id' => 40,
|
|
'template_id' => $sharedRescueId,
|
|
'operation_id' => $otherOperation,
|
|
'state' => 'succeeded',
|
|
'error_code' => '',
|
|
'created_at' => time(),
|
|
'updated_at' => time(),
|
|
]);
|
|
$sharedRescueAgent = new CatalogDeletionFakeAgent();
|
|
$sharedRescueAgent->mode = 'success';
|
|
CloudAgentSettingsService::$clients[41] = $sharedRescueAgent;
|
|
$sharedHostDelete = CloudRescueTemplateDeletionService::delete(
|
|
41, $sharedRescueId, ['user_id' => 1, 'can_manage' => true]
|
|
);
|
|
$otherHostRow = CloudRescueTemplateDeletion::query()
|
|
->where('host_id', 40)->where('template_id', $sharedRescueId)
|
|
->where('operation_id', $otherOperation)->first();
|
|
$targetHostRow = CloudRescueTemplateDeletion::query()
|
|
->where('host_id', 41)->where('template_id', $sharedRescueId)
|
|
->where('operation_id', (string) $sharedHostDelete['operation_id'])
|
|
->first();
|
|
$rescueServiceSource = file_get_contents(dirname(__DIR__)
|
|
. '/app/Services/Cloud/CloudRescueTemplateDeletionService.php');
|
|
$assert($sharedHostDelete['status'] === 'succeeded'
|
|
&& $otherHostRow !== null && (string) $otherHostRow->state === 'succeeded'
|
|
&& (string) $otherHostRow->error_code === ''
|
|
&& $targetHostRow !== null && (string) $targetHostRow->state === 'succeeded'
|
|
&& $requestCount($sharedRescueAgent, 'DELETE') === 1
|
|
&& is_string($rescueServiceSource)
|
|
&& !str_contains($rescueServiceSource, '->refresh()')
|
|
&& str_contains($rescueServiceSource,
|
|
"->where('operation_id', (string) \$record->operation_id)"),
|
|
'Composite rescue-template identity crossed hosts during state reload.');
|
|
|
|
echo "Catalog delete operation persistence, exact cleanup evidence, owner binding and no-replay tombstones passed.\n";
|
|
}
|