You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
virt/web/tests/CloudCatalogDeletionStateMa...

993 lines
48 KiB
PHP

<?php
declare(strict_types=1);
namespace App\Services\Cloud {
/** Deterministic in-process Agent used to verify durable delete behavior. */
final class CatalogDeletionFakeAgent
{
public array $requests = [];
public array $operations = [];
public string $mode = 'running';
public ?string $reconcileMode = null;
public function request(string $method, string $target, array $actor,
?array $payload = null): array
{
$method = strtoupper($method);
$this->requests[] = compact('method', 'target', 'actor', 'payload');
if ($method === 'DELETE'
&& in_array($this->mode,
['pre_dispatch_busy', 'pre_dispatch_draining'], true)) {
$code = $this->mode === 'pre_dispatch_busy'
? 'agent_busy' : 'gateway_draining';
throw new CloudAgentError(
'request was not dispatched', 503, $code, true, 503,
false, true
);
}
if ($method === 'DELETE') {
$operationId = (string) ($payload['operation_id'] ?? '');
if (preg_match('#^/v1/templates/([^?]+)$#D', $target, $match)) {
$resource = ['resource_type' => 'template',
'id' => rawurldecode($match[1]), 'owner_id' => 0];
} elseif (preg_match('#^/v1/user-isos/([^?]+)\?owner_id=([0-9]+)$#D',
$target, $match)) {
$resource = ['resource_type' => 'user_iso',
'id' => rawurldecode($match[1]),
'owner_id' => (int) $match[2]];
} elseif (preg_match('#^/v1/rescue-templates/([^?]+)$#D',
$target, $match)) {
$resource = ['resource_type' => 'rescue_template',
'id' => rawurldecode($match[1]), 'owner_id' => 0];
} else {
throw new \RuntimeException('Unexpected fake Agent DELETE: ' . $target);
}
$this->operations[$operationId] = $resource;
} elseif ($method === 'POST'
&& preg_match(
'#^/v1/operations/([A-Za-z0-9_-]+)/reconcile$#D',
$target, $match)) {
$operationId = rawurldecode($match[1]);
$resource = $this->operations[$operationId] ?? null;
if (!is_array($resource)) {
throw new CloudAgentError('operation missing', 404,
'operation_not_found', false, 404);
}
return $this->response($operationId, $resource,
$this->reconcileMode ?? $this->mode);
} elseif ($method === 'GET'
&& preg_match('#^/v1/operations/([A-Za-z0-9_-]+)$#D',
$target, $match)) {
$operationId = rawurldecode($match[1]);
$resource = $this->operations[$operationId] ?? null;
if (!is_array($resource)) {
throw new CloudAgentError('operation missing', 404,
'operation_not_found', false, 404);
}
} else {
throw new \RuntimeException('Unexpected fake Agent request: '
. $method . ' ' . $target);
}
return $this->response($operationId, $resource, $this->mode);
}
private function response(string $operationId, array $resource,
string $mode): array
{
$notFoundCode = $resource['resource_type'] . '_not_found';
if ($mode === 'direct_not_found') {
throw new CloudAgentError('Direct Agent 404 without operation',
502, $notFoundCode, false, 404);
}
if ($mode === 'durable_not_found') {
return ['operation_id' => $operationId, 'status' => 'failed',
'retryable' => false,
'error' => ['code' => $notFoundCode]];
}
if ($mode === 'running') {
return ['operation_id' => $operationId, 'status' => 'running'];
}
if ($mode === 'failed') {
return ['operation_id' => $operationId, 'status' => 'failed',
'error' => ['code' => 'resource_busy']];
}
if ($mode === 'uncertain') {
return ['operation_id' => $operationId, 'status' => 'uncertain',
'error' => ['code' => 'delete_uncertain']];
}
if ($mode === 'missing_evidence') {
return ['operation_id' => $operationId, 'status' => 'succeeded',
'result' => ['deleted' => true]];
}
if (!in_array($mode, ['success', 'wrong_owner'], true)) {
throw new \RuntimeException('Unknown fake Agent mode: ' . $mode);
}
$result = [
'resource_type' => $resource['resource_type'],
'id' => $resource['id'],
'deleted' => true,
'managed_image_absent' => true,
'managed_metadata_absent' => true,
'managed_artifacts_absent' => true,
];
if ($resource['resource_type'] === 'user_iso') {
$result['owner_id'] = $resource['owner_id']
+ ($mode === 'wrong_owner' ? 1 : 0);
}
return ['operation_id' => $operationId, 'status' => 'succeeded',
'retryable' => false,
'result' => $result];
}
}
/** Replaces only Agent lookup in this isolated process. */
final class CloudAgentSettingsService
{
/** @var array<int, CatalogDeletionFakeAgent> */
public static array $clients = [];
public static function clientForHost(int $hostId,
float $timeoutSeconds = 20.0): CatalogDeletionFakeAgent
{
if (!isset(self::$clients[$hostId])) {
throw new \RuntimeException('Missing fake Agent for host ' . $hostId);
}
return self::$clients[$hostId];
}
}
}
namespace {
require dirname(__DIR__) . '/vendor/autoload.php';
use App\Models\CloudRescueTemplateDeletion;
use App\Models\CloudTemplate;
use App\Models\CloudTemplateReplica;
use App\Models\CloudUserIso;
use App\Models\CloudUserIsoReplica;
use App\Services\Cloud\CatalogDeletionFakeAgent;
use App\Services\Cloud\CloudAgentCatalogDeletion;
use App\Services\Cloud\CloudAgentSettingsService;
use App\Services\Cloud\CloudError;
use App\Services\Cloud\CloudRescueTemplateDeletionService;
use App\Services\Cloud\CloudTemplateCatalogService;
use App\Services\Cloud\CloudUserIsoService;
use Illuminate\Database\Capsule\Manager as Capsule;
use Illuminate\Database\Schema\Blueprint;
$assert = static function (bool $condition, string $message): void {
if (!$condition) {
throw new \RuntimeException($message);
}
};
$expectCloudError = static function (callable $callable, int $status,
string $message) use ($assert): void {
try {
$callable();
} catch (CloudError $error) {
$assert($error->httpStatus === $status,
$message . ' Unexpected status: ' . $error->httpStatus);
return;
}
throw new \RuntimeException($message);
};
$requestCount = static fn (CatalogDeletionFakeAgent $agent,
string $method): int => count(array_filter($agent->requests,
static fn (array $request): bool => $request['method'] === $method));
$proof = static function (string $type, string $id,
?int $ownerId = null): array {
$result = [
'resource_type' => $type,
'id' => $id,
'deleted' => true,
'managed_image_absent' => true,
'managed_metadata_absent' => true,
'managed_artifacts_absent' => true,
];
if ($ownerId !== null) {
$result['owner_id'] = $ownerId;
}
return ['status' => 'succeeded', 'retryable' => false,
'result' => $result];
};
$capsule = new Capsule();
$capsule->addConnection(['driver' => 'sqlite', 'database' => ':memory:'],
'default');
$capsule->setAsGlobal();
$capsule->bootEloquent();
$schema = $capsule->schema('default');
$schema->create('cloud_templates', static function (Blueprint $table): void {
$table->string('id')->primary();
$table->string('name');
$table->string('os_family');
$table->string('os_version');
$table->string('source_type');
$table->text('sealed_source_url');
$table->integer('image_bytes')->default(0);
$table->string('sha256')->default('');
$table->text('startup_script')->default('');
$table->integer('revision');
$table->string('generation_token')->unique();
$table->string('state');
$table->integer('created_at');
$table->integer('updated_at');
});
$schema->create('cloud_template_replicas',
static function (Blueprint $table): void {
$table->integer('host_id');
$table->string('template_id');
$table->integer('revision');
$table->string('generation_token');
$table->string('state');
$table->integer('image_bytes')->default(0);
$table->string('sha256')->default('');
$table->string('operation_id')->default('');
$table->string('error_code')->default('');
$table->integer('created_at');
$table->integer('updated_at');
$table->primary(['host_id', 'template_id']);
});
$schema->create('cloud_orders', static function (Blueprint $table): void {
$table->increments('id');
$table->string('status');
$table->text('spec_json');
});
$schema->create('cloud_jobs', static function (Blueprint $table): void {
$table->increments('id');
$table->string('kind');
$table->string('status');
$table->text('sealed_payload');
});
$schema->create('cloud_user_isos', static function (Blueprint $table): void {
$table->string('id')->primary();
$table->integer('user_id');
$table->string('name');
$table->text('sealed_source_url');
$table->integer('size_bytes')->default(0);
$table->string('sha256')->default('');
$table->string('state');
$table->string('import_operation_id')->default('');
$table->integer('created_at');
$table->integer('updated_at');
});
$schema->create('cloud_user_iso_replicas',
static function (Blueprint $table): void {
$table->integer('host_id');
$table->string('user_iso_id');
$table->integer('user_id');
$table->string('state');
$table->integer('size_bytes')->default(0);
$table->string('sha256')->default('');
$table->string('operation_id')->default('');
$table->string('error_code')->default('');
$table->integer('created_at');
$table->integer('updated_at');
$table->primary(['host_id', 'user_iso_id']);
});
$schema->create('cloud_rescue_template_deletions',
static function (Blueprint $table): void {
$table->integer('host_id');
$table->string('template_id');
$table->string('operation_id');
$table->string('state');
$table->string('error_code')->default('');
$table->integer('created_at');
$table->integer('updated_at');
$table->primary(['host_id', 'template_id']);
$table->unique(['host_id', 'operation_id']);
});
$seedTemplate = static function (string $id, int $hostId,
string $generation): void {
$now = time();
Capsule::table('cloud_templates')->insert([
'id' => $id, 'name' => $id, 'os_family' => 'Debian',
'os_version' => '12', 'source_type' => 'remote',
'sealed_source_url' => 'test-only-sealed-source',
'image_bytes' => 1024, 'sha256' => str_repeat('a', 64),
'startup_script' => '', 'revision' => 1,
'generation_token' => $generation, 'state' => 'active',
'created_at' => $now, 'updated_at' => $now,
]);
Capsule::table('cloud_template_replicas')->insert([
'host_id' => $hostId, 'template_id' => $id, 'revision' => 1,
'generation_token' => $generation, 'state' => 'active',
'image_bytes' => 1024, 'sha256' => str_repeat('a', 64),
'operation_id' => bin2hex(random_bytes(16)), 'error_code' => '',
'created_at' => $now, 'updated_at' => $now,
]);
};
$seedIso = static function (string $id, int $hostId,
int $ownerId): void {
$now = time();
Capsule::table('cloud_user_isos')->insert([
'id' => $id, 'user_id' => $ownerId, 'name' => $id,
'sealed_source_url' => 'test-only-sealed-source',
'size_bytes' => 2048, 'sha256' => str_repeat('b', 64),
'state' => 'active', 'import_operation_id' => bin2hex(random_bytes(16)),
'created_at' => $now, 'updated_at' => $now,
]);
Capsule::table('cloud_user_iso_replicas')->insert([
'host_id' => $hostId, 'user_iso_id' => $id,
'user_id' => $ownerId, 'state' => 'active',
'size_bytes' => 2048, 'sha256' => str_repeat('b', 64),
'operation_id' => bin2hex(random_bytes(16)), 'error_code' => '',
'created_at' => $now, 'updated_at' => $now,
]);
};
// Exact Agent cleanup evidence is fail-closed, including ISO ownership.
$templateId = 'debian-12-delete-test';
$assert(CloudAgentCatalogDeletion::confirms(
$proof('template', $templateId), 'template', $templateId
), 'Exact template cleanup evidence was rejected.');
$missing = $proof('template', $templateId);
unset($missing['result']['managed_artifacts_absent']);
$assert(!CloudAgentCatalogDeletion::confirms(
$missing, 'template', $templateId
), 'Incomplete cleanup evidence was accepted.');
$isoProofId = str_repeat('1', 32);
$assert(CloudAgentCatalogDeletion::confirms(
$proof('user_iso', $isoProofId, 42), 'user_iso', $isoProofId, 42
) && !CloudAgentCatalogDeletion::confirms(
$proof('user_iso', $isoProofId, 43), 'user_iso', $isoProofId, 42
), 'User ISO cleanup evidence was not bound to its exact owner.');
// Template: one destructive dispatch, persisted operation polling, and
// terminal uncertain lock when a succeeded response lacks proof.
$templateAgent = new CatalogDeletionFakeAgent();
CloudAgentSettingsService::$clients[10] = $templateAgent;
$generation = str_repeat('a', 32);
$seedTemplate($templateId, 10, $generation);
$templateJob = CloudTemplateCatalogService::delete(
$templateId, 1, $generation, ['user_id' => 1, 'can_manage' => true]
);
$replica = CloudTemplateReplica::query()->where('host_id', 10)
->where('template_id', $templateId)->first();
$operationId = (string) $replica->operation_id;
$assert($templateJob['deleted'] === false
&& $templateJob['status'] === 'running'
&& $templateJob['status_url']
=== '/api/v1/admin/cloud-templates/delete-jobs/' . $templateId
&& preg_match('/^[a-f0-9]{32}$/D', $operationId) === 1
&& $requestCount($templateAgent, 'DELETE') === 1,
'Template delete did not persist one durable operation before dispatch.');
CloudTemplateCatalogService::delete(
$templateId, 1, $generation, ['user_id' => 1, 'can_manage' => true]
);
$assert($requestCount($templateAgent, 'DELETE') === 1
&& str_ends_with((string) end($templateAgent->requests)['target'],
'/v1/operations/' . $operationId),
'Repeated template DELETE replayed destruction or changed operation ID.');
$templateAgent->mode = 'missing_evidence';
$uncertainTemplate = CloudTemplateCatalogService::deleteJob(
$templateId, ['user_id' => 1, 'can_manage' => true]
);
$requestsBeforeLockCheck = count($templateAgent->requests);
$lockedTemplate = CloudTemplateCatalogService::deleteJob(
$templateId, ['user_id' => 1, 'can_manage' => true]
);
$replica->refresh();
$assert($uncertainTemplate['status'] === 'uncertain'
&& $lockedTemplate['status'] === 'uncertain'
&& (string) $replica->state === 'delete_uncertain'
&& (string) $replica->error_code === 'delete_evidence_missing'
&& count($templateAgent->requests) === $requestsBeforeLockCheck,
'Template missing-proof outcome was not latched without replay.');
// Exact not_dispatched backpressure is the only case where DELETE may be
// sent again. It must retain and reuse the already-persisted operation ID.
$busyTemplateId = 'debian-12-delete-busy';
$busyTemplateAgent = new CatalogDeletionFakeAgent();
$busyTemplateAgent->mode = 'pre_dispatch_busy';
CloudAgentSettingsService::$clients[17] = $busyTemplateAgent;
$busyTemplateGeneration = str_repeat('1', 32);
$seedTemplate($busyTemplateId, 17, $busyTemplateGeneration);
$busyTemplate = CloudTemplateCatalogService::delete(
$busyTemplateId, 1, $busyTemplateGeneration,
['user_id' => 1, 'can_manage' => true]
);
$busyTemplateReplica = CloudTemplateReplica::query()
->where('template_id', $busyTemplateId)->first();
$busyTemplateOperation = (string) $busyTemplateReplica->operation_id;
$assert($busyTemplate['status'] === 'running'
&& (string) $busyTemplateReplica->state === 'delete_dispatch'
&& $busyTemplateAgent->operations === []
&& $requestCount($busyTemplateAgent, 'DELETE') === 1,
'Pre-dispatch template backpressure became a terminal deletion.');
$busyTemplateReplica->updated_at = time() - 6;
$busyTemplateReplica->save();
$busyTemplateAgent->mode = 'success';
$retriedBusyTemplate = CloudTemplateCatalogService::deleteJob(
$busyTemplateId, ['user_id' => 1, 'can_manage' => true]
);
$busyTemplateDeletes = array_values(array_filter(
$busyTemplateAgent->requests,
static fn (array $request): bool => $request['method'] === 'DELETE'
));
$assert($retriedBusyTemplate['status'] === 'succeeded'
&& count($busyTemplateDeletes) === 2
&& ($busyTemplateDeletes[0]['payload']['operation_id'] ?? null)
=== $busyTemplateOperation
&& ($busyTemplateDeletes[1]['payload']['operation_id'] ?? null)
=== $busyTemplateOperation,
'Template backpressure retry changed operation ID or did not finish.');
$sendingTemplateId = 'debian-12-delete-sending';
$sendingTemplateAgent = new CatalogDeletionFakeAgent();
CloudAgentSettingsService::$clients[12] = $sendingTemplateAgent;
$sendingGeneration = str_repeat('c', 32);
$seedTemplate($sendingTemplateId, 12, $sendingGeneration);
Capsule::table('cloud_templates')->where('id', $sendingTemplateId)
->update(['state' => 'deleting']);
Capsule::table('cloud_template_replicas')
->where('template_id', $sendingTemplateId)->update([
'state' => 'delete_sending',
'operation_id' => bin2hex(random_bytes(16)),
'updated_at' => time(),
]);
$freshSendingTemplate = CloudTemplateCatalogService::deleteJob(
$sendingTemplateId, ['user_id' => 1, 'can_manage' => true]
);
$assert($freshSendingTemplate['status'] === 'running'
&& $sendingTemplateAgent->requests === [],
'Fresh template delete_sending claim was polled before dispatch settled.');
Capsule::table('cloud_template_replicas')
->where('template_id', $sendingTemplateId)
->update(['updated_at' => time() - 91]);
$staleSendingTemplate = CloudTemplateCatalogService::deleteJob(
$sendingTemplateId, ['user_id' => 1, 'can_manage' => true]
);
$sendingTemplateReplica = CloudTemplateReplica::query()
->where('template_id', $sendingTemplateId)->first();
$assert($staleSendingTemplate['status'] === 'uncertain'
&& (string) $sendingTemplateReplica->error_code
=== 'operation_not_found'
&& count($sendingTemplateAgent->requests) === 1
&& $sendingTemplateAgent->requests[0]['method'] === 'GET'
&& $sendingTemplateAgent->requests[0]['target']
=== '/v1/operations/'
. (string) $sendingTemplateReplica->operation_id
&& $requestCount($sendingTemplateAgent, 'DELETE') === 0,
'Stale template sending claim was not queried safely by operation ID.');
$reconcileTemplateId = 'debian-12-delete-reconcile';
$reconcileTemplateAgent = new CatalogDeletionFakeAgent();
CloudAgentSettingsService::$clients[15] = $reconcileTemplateAgent;
$reconcileTemplateGeneration = str_repeat('f', 32);
$seedTemplate($reconcileTemplateId, 15, $reconcileTemplateGeneration);
CloudTemplateCatalogService::delete(
$reconcileTemplateId, 1, $reconcileTemplateGeneration,
['user_id' => 1, 'can_manage' => true]
);
$reconcileTemplateReplica = CloudTemplateReplica::query()
->where('template_id', $reconcileTemplateId)->first();
$reconcileTemplateOperation =
(string) $reconcileTemplateReplica->operation_id;
$reconcileTemplateAgent->mode = 'uncertain';
$reconcileTemplateAgent->reconcileMode = 'success';
$beforeTemplateReconcile = count($reconcileTemplateAgent->requests);
$reconciledTemplateDelete = CloudTemplateCatalogService::deleteJob(
$reconcileTemplateId, ['user_id' => 1, 'can_manage' => true]
);
$templateReconcileRequests = array_slice(
$reconcileTemplateAgent->requests, $beforeTemplateReconcile
);
$assert($reconciledTemplateDelete['status'] === 'succeeded'
&& $requestCount($reconcileTemplateAgent, 'DELETE') === 1
&& count($templateReconcileRequests) === 2
&& $templateReconcileRequests[0]['method'] === 'GET'
&& $templateReconcileRequests[0]['target']
=== '/v1/operations/' . $reconcileTemplateOperation
&& $templateReconcileRequests[1]['method'] === 'POST'
&& $templateReconcileRequests[1]['target']
=== '/v1/operations/' . $reconcileTemplateOperation
. '/reconcile',
'Template delete uncertainty replayed DELETE or changed operation ID.');
$trueUncertainTemplateId = 'debian-12-delete-uncertain';
$trueUncertainTemplateAgent = new CatalogDeletionFakeAgent();
CloudAgentSettingsService::$clients[16] = $trueUncertainTemplateAgent;
$trueUncertainGeneration = str_repeat('0', 32);
$seedTemplate(
$trueUncertainTemplateId, 16, $trueUncertainGeneration
);
CloudTemplateCatalogService::delete(
$trueUncertainTemplateId, 1, $trueUncertainGeneration,
['user_id' => 1, 'can_manage' => true]
);
$trueUncertainTemplateAgent->mode = 'uncertain';
$trueUncertainTemplateAgent->reconcileMode = 'uncertain';
$trueUncertainDelete = CloudTemplateCatalogService::deleteJob(
$trueUncertainTemplateId, ['user_id' => 1, 'can_manage' => true]
);
$requestsAtTrueUncertain = count($trueUncertainTemplateAgent->requests);
$trueUncertainDeleteAgain = CloudTemplateCatalogService::deleteJob(
$trueUncertainTemplateId, ['user_id' => 1, 'can_manage' => true]
);
$assert($trueUncertainDelete['status'] === 'uncertain'
&& $trueUncertainDeleteAgain['status'] === 'uncertain'
&& $requestCount($trueUncertainTemplateAgent, 'DELETE') === 1
&& count($trueUncertainTemplateAgent->requests)
=== $requestsAtTrueUncertain,
'A genuinely uncertain template delete was replayed or left pollable.');
$successfulTemplateId = 'debian-12-delete-success';
$successfulTemplateAgent = new CatalogDeletionFakeAgent();
$successfulTemplateAgent->mode = 'success';
CloudAgentSettingsService::$clients[11] = $successfulTemplateAgent;
$successfulGeneration = str_repeat('b', 32);
$seedTemplate($successfulTemplateId, 11, $successfulGeneration);
$deletedTemplate = CloudTemplateCatalogService::delete(
$successfulTemplateId, 1, $successfulGeneration,
['user_id' => 1, 'can_manage' => true]
);
$assert($deletedTemplate['deleted'] === true
&& $deletedTemplate['status'] === 'succeeded'
&& (string) CloudTemplate::find($successfulTemplateId)->state === 'retired'
&& !CloudTemplateReplica::query()->where('template_id',
$successfulTemplateId)->exists()
&& $requestCount($successfulTemplateAgent, 'DELETE') === 1,
'Exact template cleanup proof did not safely finalize retirement.');
$directTemplateId = 'debian-12-direct-404';
$directTemplateAgent = new CatalogDeletionFakeAgent();
$directTemplateAgent->mode = 'direct_not_found';
CloudAgentSettingsService::$clients[13] = $directTemplateAgent;
$directTemplateGeneration = str_repeat('d', 32);
$seedTemplate($directTemplateId, 13, $directTemplateGeneration);
$directTemplate = CloudTemplateCatalogService::delete(
$directTemplateId, 1, $directTemplateGeneration,
['user_id' => 1, 'can_manage' => true]
);
$directTemplateReplica = CloudTemplateReplica::query()
->where('template_id', $directTemplateId)->first();
$assert($directTemplate['deleted'] === false
&& $directTemplate['status'] === 'uncertain'
&& (string) CloudTemplate::find($directTemplateId)->state === 'deleting'
&& (string) $directTemplateReplica->state === 'delete_uncertain'
&& (string) $directTemplateReplica->error_code === 'operation_not_found',
'Unmarked template 404 was made terminal without no-dispatch proof.');
$durableTemplateId = 'debian-12-durable-404';
$durableTemplateAgent = new CatalogDeletionFakeAgent();
$durableTemplateAgent->mode = 'durable_not_found';
CloudAgentSettingsService::$clients[14] = $durableTemplateAgent;
$durableTemplateGeneration = str_repeat('e', 32);
$seedTemplate($durableTemplateId, 14, $durableTemplateGeneration);
$durableTemplate = CloudTemplateCatalogService::delete(
$durableTemplateId, 1, $durableTemplateGeneration,
['user_id' => 1, 'can_manage' => true]
);
$assert($durableTemplate['deleted'] === true
&& (string) CloudTemplate::find($durableTemplateId)->state === 'retired',
'Durable non-retryable template_not_found operation did not complete deletion.');
// User ISO: owner-scoped status plus strict owner evidence.
$isoId = str_repeat('2', 32);
$isoAgent = new CatalogDeletionFakeAgent();
CloudAgentSettingsService::$clients[20] = $isoAgent;
$seedIso($isoId, 20, 42);
$isoJob = CloudUserIsoService::delete($isoId,
['user_id' => 42, 'can_manage' => false]);
$isoReplica = CloudUserIsoReplica::query()->where('host_id', 20)
->where('user_iso_id', $isoId)->first();
$isoOperation = (string) $isoReplica->operation_id;
$assert($isoJob['status'] === 'running'
&& $isoJob['status_url'] === '/api/v1/user-isos/delete-jobs/' . $isoId
&& $requestCount($isoAgent, 'DELETE') === 1,
'User ISO delete did not return its durable owner-scoped job.');
$requestsBeforeWrongOwner = count($isoAgent->requests);
$expectCloudError(static fn () => CloudUserIsoService::deleteJob(
$isoId, ['user_id' => 43, 'can_manage' => false]
), 404, 'Another user read the ISO delete job.');
$assert(count($isoAgent->requests) === $requestsBeforeWrongOwner,
'Wrong-owner ISO status lookup contacted an Agent.');
CloudUserIsoService::delete($isoId,
['user_id' => 42, 'can_manage' => false]);
$assert($requestCount($isoAgent, 'DELETE') === 1
&& str_ends_with((string) end($isoAgent->requests)['target'],
'/v1/operations/' . $isoOperation),
'Repeated ISO DELETE replayed destruction or changed operation ID.');
$isoAgent->mode = 'wrong_owner';
$uncertainIso = CloudUserIsoService::deleteJob(
$isoId, ['user_id' => 42, 'can_manage' => false]
);
$requestsBeforeIsoLockCheck = count($isoAgent->requests);
$lockedIso = CloudUserIsoService::delete(
$isoId, ['user_id' => 42, 'can_manage' => false]
);
$isoReplica->refresh();
$assert($uncertainIso['status'] === 'uncertain'
&& $lockedIso['status'] === 'uncertain'
&& (string) $isoReplica->state === 'delete_uncertain'
&& (string) $isoReplica->error_code === 'delete_evidence_missing'
&& count($isoAgent->requests) === $requestsBeforeIsoLockCheck,
'ISO owner mismatch was not latched without destructive replay.');
$busyIsoId = str_repeat('b', 32);
$busyIsoAgent = new CatalogDeletionFakeAgent();
$busyIsoAgent->mode = 'pre_dispatch_draining';
CloudAgentSettingsService::$clients[26] = $busyIsoAgent;
$seedIso($busyIsoId, 26, 42);
$busyIso = CloudUserIsoService::delete(
$busyIsoId, ['user_id' => 42, 'can_manage' => false]
);
$busyIsoReplica = CloudUserIsoReplica::query()
->where('user_iso_id', $busyIsoId)->first();
$busyIsoOperation = (string) $busyIsoReplica->operation_id;
$assert($busyIso['status'] === 'running'
&& (string) $busyIsoReplica->state === 'delete_dispatch'
&& $busyIsoAgent->operations === []
&& $requestCount($busyIsoAgent, 'DELETE') === 1,
'Pre-dispatch ISO backpressure became a terminal deletion.');
$busyIsoReplica->updated_at = time() - 6;
$busyIsoReplica->save();
$busyIsoAgent->mode = 'success';
$retriedBusyIso = CloudUserIsoService::deleteJob(
$busyIsoId, ['user_id' => 42, 'can_manage' => false]
);
$busyIsoDeletes = array_values(array_filter(
$busyIsoAgent->requests,
static fn (array $request): bool => $request['method'] === 'DELETE'
));
$assert($retriedBusyIso['status'] === 'succeeded'
&& count($busyIsoDeletes) === 2
&& ($busyIsoDeletes[0]['payload']['operation_id'] ?? null)
=== $busyIsoOperation
&& ($busyIsoDeletes[1]['payload']['operation_id'] ?? null)
=== $busyIsoOperation,
'ISO backpressure retry changed operation ID or did not finish.');
$sendingIsoId = str_repeat('6', 32);
$sendingIsoAgent = new CatalogDeletionFakeAgent();
CloudAgentSettingsService::$clients[22] = $sendingIsoAgent;
$seedIso($sendingIsoId, 22, 42);
Capsule::table('cloud_user_isos')->where('id', $sendingIsoId)
->update(['state' => 'deleting']);
Capsule::table('cloud_user_iso_replicas')
->where('user_iso_id', $sendingIsoId)->update([
'state' => 'delete_sending',
'operation_id' => bin2hex(random_bytes(16)),
'updated_at' => time(),
]);
$freshSendingIso = CloudUserIsoService::deleteJob(
$sendingIsoId, ['user_id' => 42, 'can_manage' => false]
);
$assert($freshSendingIso['status'] === 'running'
&& $sendingIsoAgent->requests === [],
'Fresh ISO delete_sending claim was polled before dispatch settled.');
Capsule::table('cloud_user_iso_replicas')
->where('user_iso_id', $sendingIsoId)
->update(['updated_at' => time() - 91]);
$staleSendingIso = CloudUserIsoService::deleteJob(
$sendingIsoId, ['user_id' => 42, 'can_manage' => false]
);
$sendingIsoReplica = CloudUserIsoReplica::query()
->where('user_iso_id', $sendingIsoId)->first();
$assert($staleSendingIso['status'] === 'uncertain'
&& (string) $sendingIsoReplica->error_code === 'operation_not_found'
&& count($sendingIsoAgent->requests) === 1
&& $sendingIsoAgent->requests[0]['method'] === 'GET'
&& $sendingIsoAgent->requests[0]['target']
=== '/v1/operations/' . (string) $sendingIsoReplica->operation_id
&& $requestCount($sendingIsoAgent, 'DELETE') === 0,
'Stale ISO sending claim was not queried safely by operation ID.');
$reconcileIsoId = str_repeat('d', 32);
$reconcileIsoAgent = new CatalogDeletionFakeAgent();
CloudAgentSettingsService::$clients[25] = $reconcileIsoAgent;
$seedIso($reconcileIsoId, 25, 42);
CloudUserIsoService::delete(
$reconcileIsoId, ['user_id' => 42, 'can_manage' => false]
);
$reconcileIsoReplica = CloudUserIsoReplica::query()
->where('user_iso_id', $reconcileIsoId)->first();
$reconcileIsoOperation = (string) $reconcileIsoReplica->operation_id;
$reconcileIsoAgent->mode = 'uncertain';
$reconcileIsoAgent->reconcileMode = 'success';
$beforeIsoReconcile = count($reconcileIsoAgent->requests);
$reconciledIsoDelete = CloudUserIsoService::deleteJob(
$reconcileIsoId, ['user_id' => 42, 'can_manage' => false]
);
$isoReconcileRequests = array_slice(
$reconcileIsoAgent->requests, $beforeIsoReconcile
);
$assert($reconciledIsoDelete['status'] === 'succeeded'
&& $requestCount($reconcileIsoAgent, 'DELETE') === 1
&& count($isoReconcileRequests) === 2
&& $isoReconcileRequests[0]['method'] === 'GET'
&& $isoReconcileRequests[0]['target']
=== '/v1/operations/' . $reconcileIsoOperation
&& $isoReconcileRequests[1]['method'] === 'POST'
&& $isoReconcileRequests[1]['target']
=== '/v1/operations/' . $reconcileIsoOperation . '/reconcile',
'User ISO delete uncertainty replayed DELETE or changed operation ID.');
$successfulIsoId = str_repeat('3', 32);
$successfulIsoAgent = new CatalogDeletionFakeAgent();
$successfulIsoAgent->mode = 'success';
CloudAgentSettingsService::$clients[21] = $successfulIsoAgent;
$seedIso($successfulIsoId, 21, 42);
$deletedIso = CloudUserIsoService::delete($successfulIsoId,
['user_id' => 42, 'can_manage' => false]);
$successfulIsoTombstone = CloudUserIso::find($successfulIsoId);
$polledDeletedIso = CloudUserIsoService::deleteJob($successfulIsoId,
['user_id' => 42, 'can_manage' => false]);
$repeatedDeletedIso = CloudUserIsoService::delete($successfulIsoId,
['user_id' => 42, 'can_manage' => false]);
$historicalImport = CloudUserIsoService::job(
(string) $successfulIsoTombstone->import_operation_id,
['user_id' => 42, 'can_manage' => false]
);
$listedAfterDelete = CloudUserIsoService::listing(42);
$assert($deletedIso['deleted'] === true
&& $deletedIso['status'] === 'succeeded'
&& $polledDeletedIso['status'] === 'succeeded'
&& $repeatedDeletedIso['status'] === 'succeeded'
&& $historicalImport['status'] === 'succeeded'
&& $historicalImport['deleted'] === true
&& !in_array($successfulIsoId, array_column(
$listedAfterDelete['isos'], 'id'
), true)
&& $successfulIsoTombstone !== null
&& (string) $successfulIsoTombstone->state === 'retired'
&& (string) $successfulIsoTombstone->name === ''
&& (string) $successfulIsoTombstone->sealed_source_url === ''
&& (string) $successfulIsoTombstone->sha256 === ''
&& (int) $successfulIsoTombstone->size_bytes === 0
&& !CloudUserIsoReplica::query()->where('user_iso_id',
$successfulIsoId)->exists()
&& $requestCount($successfulIsoAgent, 'DELETE') === 1,
'Exact ISO cleanup did not leave a pollable, data-free tombstone.');
$directIsoId = str_repeat('7', 32);
$directIsoAgent = new CatalogDeletionFakeAgent();
$directIsoAgent->mode = 'direct_not_found';
CloudAgentSettingsService::$clients[23] = $directIsoAgent;
$seedIso($directIsoId, 23, 42);
$directIso = CloudUserIsoService::delete($directIsoId,
['user_id' => 42, 'can_manage' => false]);
$directIsoReplica = CloudUserIsoReplica::query()
->where('user_iso_id', $directIsoId)->first();
$assert($directIso['deleted'] === false
&& $directIso['status'] === 'uncertain'
&& (string) CloudUserIso::find($directIsoId)->state === 'deleting'
&& (string) $directIsoReplica->state === 'delete_uncertain'
&& (string) $directIsoReplica->error_code === 'operation_not_found',
'Unmarked ISO 404 was made terminal without no-dispatch proof.');
$durableIsoId = str_repeat('8', 32);
$durableIsoAgent = new CatalogDeletionFakeAgent();
$durableIsoAgent->mode = 'durable_not_found';
CloudAgentSettingsService::$clients[24] = $durableIsoAgent;
$seedIso($durableIsoId, 24, 42);
$durableIso = CloudUserIsoService::delete($durableIsoId,
['user_id' => 42, 'can_manage' => false]);
$durableIsoTombstone = CloudUserIso::find($durableIsoId);
$assert($durableIso['deleted'] === true
&& (string) $durableIsoTombstone->state === 'retired',
'Durable non-retryable user_iso_not_found operation did not complete deletion.');
// Rescue: terminal tombstones survive retries and prevent redispatch.
$busyRescueId = str_repeat('c', 32);
$busyRescueAgent = new CatalogDeletionFakeAgent();
$busyRescueAgent->mode = 'pre_dispatch_busy';
CloudAgentSettingsService::$clients[35] = $busyRescueAgent;
$busyRescue = CloudRescueTemplateDeletionService::delete(
35, $busyRescueId, ['user_id' => 1, 'can_manage' => true]
);
$busyRescueRecord = CloudRescueTemplateDeletion::query()
->where('host_id', 35)->where('template_id', $busyRescueId)->first();
$busyRescueOperation = (string) $busyRescueRecord->operation_id;
$assert($busyRescue['status'] === 'running'
&& (string) $busyRescueRecord->state === 'dispatching'
&& $busyRescueAgent->operations === []
&& $requestCount($busyRescueAgent, 'DELETE') === 1,
'Pre-dispatch rescue backpressure became a terminal deletion.');
// This model has a composite key and deliberately must not use save().
Capsule::table('cloud_rescue_template_deletions')
->where('host_id', 35)->where('template_id', $busyRescueId)
->where('operation_id', $busyRescueOperation)
->update(['updated_at' => time() - 6]);
$busyRescueAgent->mode = 'success';
$retriedBusyRescue = CloudRescueTemplateDeletionService::job(
35, $busyRescueOperation,
['user_id' => 1, 'can_manage' => true]
);
$busyRescueDeletes = array_values(array_filter(
$busyRescueAgent->requests,
static fn (array $request): bool => $request['method'] === 'DELETE'
));
$assert($retriedBusyRescue['status'] === 'succeeded'
&& count($busyRescueDeletes) === 2
&& ($busyRescueDeletes[0]['payload']['operation_id'] ?? null)
=== $busyRescueOperation
&& ($busyRescueDeletes[1]['payload']['operation_id'] ?? null)
=== $busyRescueOperation,
'Rescue backpressure retry changed operation ID or did not finish.');
$rescueId = str_repeat('4', 32);
$rescueAgent = new CatalogDeletionFakeAgent();
CloudAgentSettingsService::$clients[30] = $rescueAgent;
$rescueJob = CloudRescueTemplateDeletionService::delete(
30, $rescueId, ['user_id' => 1, 'can_manage' => true]
);
$rescueRecord = CloudRescueTemplateDeletion::query()
->where('host_id', 30)->where('template_id', $rescueId)->first();
$rescueOperation = (string) $rescueRecord->operation_id;
$assert($rescueJob['status'] === 'running'
&& $rescueJob['operation_id'] === $rescueOperation
&& $rescueJob['status_url']
=== '/api/v1/admin/rescue-templates/delete-jobs/'
. $rescueOperation . '?host_id=30'
&& $requestCount($rescueAgent, 'DELETE') === 1,
'Rescue delete did not persist its host-scoped operation tombstone.');
CloudRescueTemplateDeletionService::delete(
30, $rescueId, ['user_id' => 1, 'can_manage' => true]
);
$assert($requestCount($rescueAgent, 'DELETE') === 1
&& str_ends_with((string) end($rescueAgent->requests)['target'],
'/v1/operations/' . $rescueOperation),
'Repeated rescue DELETE replayed destruction or changed operation ID.');
$rescueAgent->mode = 'missing_evidence';
$uncertainRescue = CloudRescueTemplateDeletionService::job(
30, $rescueOperation, ['user_id' => 1, 'can_manage' => true]
);
$requestsBeforeRescueLockCheck = count($rescueAgent->requests);
$lockedRescue = CloudRescueTemplateDeletionService::delete(
30, $rescueId, ['user_id' => 1, 'can_manage' => true]
);
$rescueRecord = CloudRescueTemplateDeletion::query()
->where('host_id', 30)->where('template_id', $rescueId)
->where('operation_id', $rescueOperation)->first();
$assert($uncertainRescue['status'] === 'uncertain'
&& $lockedRescue['status'] === 'uncertain'
&& (string) $rescueRecord->state === 'uncertain'
&& (string) $rescueRecord->error_code === 'delete_evidence_missing'
&& count($rescueAgent->requests) === $requestsBeforeRescueLockCheck,
'Rescue missing-proof result was not retained as a no-replay tombstone.');
$reconcileRescueId = str_repeat('e', 32);
$reconcileRescueAgent = new CatalogDeletionFakeAgent();
CloudAgentSettingsService::$clients[34] = $reconcileRescueAgent;
$reconcileRescue = CloudRescueTemplateDeletionService::delete(
34, $reconcileRescueId, ['user_id' => 1, 'can_manage' => true]
);
$reconcileRescueOperation = (string) $reconcileRescue['operation_id'];
$reconcileRescueAgent->mode = 'uncertain';
$reconcileRescueAgent->reconcileMode = 'success';
$beforeRescueReconcile = count($reconcileRescueAgent->requests);
$reconciledRescueDelete = CloudRescueTemplateDeletionService::job(
34, $reconcileRescueOperation,
['user_id' => 1, 'can_manage' => true]
);
$rescueReconcileRequests = array_slice(
$reconcileRescueAgent->requests, $beforeRescueReconcile
);
$assert($reconciledRescueDelete['status'] === 'succeeded'
&& $requestCount($reconcileRescueAgent, 'DELETE') === 1
&& count($rescueReconcileRequests) === 2
&& $rescueReconcileRequests[0]['method'] === 'GET'
&& $rescueReconcileRequests[0]['target']
=== '/v1/operations/' . $reconcileRescueOperation
&& $rescueReconcileRequests[1]['method'] === 'POST'
&& $rescueReconcileRequests[1]['target']
=== '/v1/operations/' . $reconcileRescueOperation . '/reconcile',
'Rescue-template delete uncertainty replayed DELETE or changed operation ID.');
$successfulRescueId = str_repeat('5', 32);
$successfulRescueAgent = new CatalogDeletionFakeAgent();
$successfulRescueAgent->mode = 'success';
CloudAgentSettingsService::$clients[31] = $successfulRescueAgent;
$deletedRescue = CloudRescueTemplateDeletionService::delete(
31, $successfulRescueId, ['user_id' => 1, 'can_manage' => true]
);
$requestsAfterRescueSuccess = count($successfulRescueAgent->requests);
$rescueStatus = CloudRescueTemplateDeletionService::job(
31, (string) $deletedRescue['operation_id'],
['user_id' => 1, 'can_manage' => true]
);
$rescueRetry = CloudRescueTemplateDeletionService::delete(
31, $successfulRescueId, ['user_id' => 1, 'can_manage' => true]
);
$assert($deletedRescue['deleted'] === true
&& $rescueStatus['status'] === 'succeeded'
&& $rescueRetry['status'] === 'succeeded'
&& CloudRescueTemplateDeletion::query()->where('host_id', 31)
->where('template_id', $successfulRescueId)->exists()
&& $requestCount($successfulRescueAgent, 'DELETE') === 1
&& count($successfulRescueAgent->requests)
=== $requestsAfterRescueSuccess,
'Successful rescue tombstone did not make every retry read-only.');
// A slower status poll may carry a stale pre-terminal model snapshot.
// Its result must not move the durable success tombstone backwards.
$staleRescueRecord = clone CloudRescueTemplateDeletion::query()
->where('host_id', 31)->where('template_id', $successfulRescueId)
->first();
$staleRescueRecord->state = 'running';
$markRescueDeletion = new ReflectionMethod(
CloudRescueTemplateDeletionService::class, 'mark'
);
$markRescueDeletion->invoke(
null, $staleRescueRecord, 'running', 'late_poll_result'
);
$successfulRescueRecord = CloudRescueTemplateDeletion::query()
->where('host_id', 31)->where('template_id', $successfulRescueId)
->first();
$assert((string) $successfulRescueRecord->state === 'succeeded'
&& (string) $successfulRescueRecord->error_code === '',
'A stale rescue deletion poll overwrote its terminal tombstone.');
$directRescueId = str_repeat('9', 32);
$directRescueAgent = new CatalogDeletionFakeAgent();
$directRescueAgent->mode = 'direct_not_found';
CloudAgentSettingsService::$clients[32] = $directRescueAgent;
$directRescue = CloudRescueTemplateDeletionService::delete(
32, $directRescueId, ['user_id' => 1, 'can_manage' => true]
);
$directRescueRecord = CloudRescueTemplateDeletion::query()
->where('host_id', 32)->where('template_id', $directRescueId)->first();
$assert($directRescue['deleted'] === false
&& $directRescue['status'] === 'running'
&& (string) $directRescueRecord->state === 'running'
&& (string) $directRescueRecord->error_code
=== 'rescue_template_not_found',
'Unmarked rescue-template 404 was made terminal without no-dispatch proof.');
$durableRescueId = str_repeat('a', 32);
$durableRescueAgent = new CatalogDeletionFakeAgent();
$durableRescueAgent->mode = 'durable_not_found';
CloudAgentSettingsService::$clients[33] = $durableRescueAgent;
$durableRescue = CloudRescueTemplateDeletionService::delete(
33, $durableRescueId, ['user_id' => 1, 'can_manage' => true]
);
$assert($durableRescue['deleted'] === true
&& $durableRescue['status'] === 'succeeded'
&& CloudRescueTemplateDeletion::query()->where('host_id', 33)
->where('template_id', $durableRescueId)
->where('state', 'succeeded')->exists(),
'Durable non-retryable rescue_template_not_found operation did not complete deletion.');
// The same rescue template ID may exist on several hosts. Every reload
// must retain the complete host/template/operation identity.
$sharedRescueId = str_repeat('b', 32);
$otherOperation = str_repeat('c', 32);
Capsule::table('cloud_rescue_template_deletions')->insert([
'host_id' => 40,
'template_id' => $sharedRescueId,
'operation_id' => $otherOperation,
'state' => 'succeeded',
'error_code' => '',
'created_at' => time(),
'updated_at' => time(),
]);
$sharedRescueAgent = new CatalogDeletionFakeAgent();
$sharedRescueAgent->mode = 'success';
CloudAgentSettingsService::$clients[41] = $sharedRescueAgent;
$sharedHostDelete = CloudRescueTemplateDeletionService::delete(
41, $sharedRescueId, ['user_id' => 1, 'can_manage' => true]
);
$otherHostRow = CloudRescueTemplateDeletion::query()
->where('host_id', 40)->where('template_id', $sharedRescueId)
->where('operation_id', $otherOperation)->first();
$targetHostRow = CloudRescueTemplateDeletion::query()
->where('host_id', 41)->where('template_id', $sharedRescueId)
->where('operation_id', (string) $sharedHostDelete['operation_id'])
->first();
$rescueServiceSource = file_get_contents(dirname(__DIR__)
. '/app/Services/Cloud/CloudRescueTemplateDeletionService.php');
$assert($sharedHostDelete['status'] === 'succeeded'
&& $otherHostRow !== null && (string) $otherHostRow->state === 'succeeded'
&& (string) $otherHostRow->error_code === ''
&& $targetHostRow !== null && (string) $targetHostRow->state === 'succeeded'
&& $requestCount($sharedRescueAgent, 'DELETE') === 1
&& is_string($rescueServiceSource)
&& !str_contains($rescueServiceSource, '->refresh()')
&& str_contains($rescueServiceSource,
"->where('operation_id', (string) \$record->operation_id)"),
'Composite rescue-template identity crossed hosts during state reload.');
echo "Catalog delete operation persistence, exact cleanup evidence, owner binding and no-replay tombstones passed.\n";
}