You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
virt/web/tests/CloudRescueTemplateRetryBou...

614 lines
22 KiB
PHP

<?php
declare(strict_types=1);
namespace App\Services\Cloud {
/** Isolated encrypted payload store for this SQLite contract test. */
final class CloudSecrets
{
private static array $values = [];
public static function encrypt(array $value, string $purpose): string
{
$key = bin2hex(random_bytes(16));
self::$values[$key] = $value;
return 'rescue-template-retry-test:' . $key;
}
public static function decrypt(string $value, string $purpose): array
{
$key = substr($value, strlen('rescue-template-retry-test:'));
if (!array_key_exists($key, self::$values)) {
throw new \RuntimeException('Missing test payload.');
}
return self::$values[$key];
}
}
}
namespace {
require dirname(__DIR__) . '/vendor/autoload.php';
use App\Models\CloudJob;
use App\Models\CloudAgentCompletionEvent;
use App\Models\CloudRescueTemplateDeletion;
use App\Models\CloudRescueTemplateImport;
use App\Services\Cloud\CloudError;
use App\Services\Cloud\CloudOperationCompletionService;
use App\Services\Cloud\CloudRescueTemplateDeletionService;
use App\Services\Cloud\CloudRescueTemplateImportService;
use App\Services\Cloud\CloudSecrets;
use Illuminate\Database\Capsule\Manager as Capsule;
use Illuminate\Database\Schema\Blueprint;
$assert = static function (bool $condition, string $message): void {
if (!$condition) {
throw new RuntimeException($message);
}
};
$capsule = new Capsule();
$capsule->addConnection([
'driver' => 'sqlite',
'database' => ':memory:',
], 'default');
$capsule->setAsGlobal();
$capsule->bootEloquent();
$capsule->schema('default')->create(
'cloud_jobs',
static function (Blueprint $table): void {
$table->increments('id');
$table->string('operation_id');
$table->string('request_idempotency_key');
$table->string('request_fingerprint', 64);
$table->integer('parent_job_id')->nullable();
$table->integer('instance_id')->nullable();
$table->integer('order_id')->nullable();
$table->integer('invoice_id')->nullable();
$table->integer('snapshot_id')->nullable();
$table->integer('user_id');
$table->integer('host_id')->nullable();
$table->string('kind');
$table->string('execution_mode');
$table->string('phase');
$table->string('outcome')->nullable();
$table->string('status');
$table->integer('attempts')->default(0);
$table->integer('dispatch_attempts')->default(0);
$table->integer('dispatch_deadline_at')->nullable();
$table->integer('dispatch_started_at')->nullable();
$table->integer('dispatched_at')->nullable();
$table->integer('agent_accepted_at')->nullable();
$table->integer('receipt_deadline_at')->nullable();
$table->text('sealed_payload');
$table->text('result_json')->nullable();
$table->string('error_code')->nullable();
$table->text('error_message')->nullable();
$table->string('lease_token')->nullable();
$table->integer('lease_until')->default(0);
$table->integer('available_at')->default(0);
$table->integer('created_at');
$table->integer('updated_at');
$table->integer('terminal_at')->nullable();
$table->boolean('effect_may_have_applied')->nullable();
$table->boolean('safe_to_resubmit')->nullable();
$table->string('effect_resolution');
$table->integer('effect_resolved_at')->nullable();
$table->integer('effect_resolved_by')->nullable();
$table->string('effect_resolution_reason')->default('');
$table->boolean('finalization_incident')->default(0);
$table->string('finalization_error_code')->nullable();
$table->string('completion_event_id')->nullable();
$table->integer('read_at')->nullable();
}
);
$capsule->schema('default')->create(
'cloud_rescue_template_imports',
static function (Blueprint $table): void {
$table->string('operation_id')->primary();
$table->integer('host_id');
$table->string('template_id', 32);
$table->integer('job_id');
$table->string('import_type');
$table->string('name');
$table->integer('size_bytes')->default(0);
$table->string('sha256', 64)->default('');
$table->string('state');
$table->string('error_code')->default('');
$table->integer('created_at');
$table->integer('updated_at');
}
);
$capsule->schema('default')->create(
'cloud_rescue_template_deletions',
static function (Blueprint $table): void {
$table->integer('host_id');
$table->string('template_id', 32);
$table->string('operation_id');
$table->integer('job_id');
$table->string('state');
$table->string('error_code')->default('');
$table->integer('created_at');
$table->integer('updated_at');
$table->primary(['host_id', 'template_id']);
}
);
$now = time();
$actor = [
'user_id' => 1,
'is_admin' => true,
'is_super_admin' => true,
'can_manage' => true,
];
$insertJob = static function (array $overrides) use ($now): int {
$operationId = (string) ($overrides['operation_id']
?? bin2hex(random_bytes(16)));
$values = [
'operation_id' => $operationId,
'request_idempotency_key' => $operationId,
'request_fingerprint' => hash('sha256', $operationId),
'parent_job_id' => null,
'instance_id' => null,
'order_id' => null,
'invoice_id' => null,
'snapshot_id' => null,
'user_id' => 1,
'host_id' => 1,
'kind' => 'rescue_template_import',
'execution_mode' => 'agent',
'phase' => 'terminal',
'outcome' => 'failed',
'status' => 'failed',
'attempts' => 1,
'dispatch_attempts' => 1,
'dispatch_deadline_at' => $now - 10,
'dispatch_started_at' => $now - 9,
'dispatched_at' => $now - 8,
'agent_accepted_at' => $now - 7,
'receipt_deadline_at' => $now - 1,
'sealed_payload' => '',
'result_json' => null,
'error_code' => 'operation_failed',
'error_message' => '',
'lease_token' => null,
'lease_until' => 0,
'available_at' => 0,
'created_at' => $now - 20,
'updated_at' => $now - 1,
'terminal_at' => $now - 1,
'effect_may_have_applied' => 0,
'safe_to_resubmit' => 1,
'effect_resolution' => 'not_applied',
'effect_resolved_at' => $now - 1,
'effect_resolved_by' => null,
'effect_resolution_reason' => 'test',
'finalization_incident' => 0,
'finalization_error_code' => null,
'completion_event_id' => null,
'read_at' => null,
];
$values = array_replace($values, $overrides);
return (int) Capsule::table('cloud_jobs')->insertGetId($values);
};
$insertImport = static function (
string $name,
string $templateId,
string $state,
string $resolution,
bool $incident = false
) use ($insertJob, $now): array {
$operationId = bin2hex(random_bytes(16));
$payload = [
'can_manage' => true,
'rescue_template_id' => $templateId,
'method' => 'POST',
'request_target' => '/v1/rescue-templates',
'body' => [
'id' => $templateId,
'name' => $name,
'source_url' => 'https://example.invalid/rescue.qcow2',
'sha256' => '',
'operation_id' => $operationId,
],
'owner_id' => 1,
];
$jobId = $insertJob([
'operation_id' => $operationId,
'kind' => 'rescue_template_import',
'outcome' => $resolution === 'applied' ? 'succeeded' : 'failed',
'status' => $resolution === 'applied' ? 'succeeded' : 'failed',
'effect_resolution' => $resolution,
'effect_may_have_applied' => 0,
'safe_to_resubmit' => $resolution === 'applied' ? 0 : 1,
'error_code' => $resolution === 'applied'
? null : 'operation_failed',
'sealed_payload' => CloudSecrets::encrypt(
$payload,
'job-payload'
),
'finalization_incident' => $incident ? 1 : 0,
'finalization_error_code' => $incident
? 'recovery_instance_state_conflict' : null,
]);
Capsule::table('cloud_rescue_template_imports')->insert([
'operation_id' => $operationId,
'host_id' => 1,
'template_id' => $templateId,
'job_id' => $jobId,
'import_type' => 'remote',
'name' => $name,
'size_bytes' => $resolution === 'applied' ? 1024 : 0,
'sha256' => '',
'state' => $state,
'error_code' => $state === 'failed' ? 'import_failed' : '',
'created_at' => $now - 20,
'updated_at' => $now - 1,
]);
return [$jobId, $operationId];
};
$activeTemplateId = str_repeat('a', 32);
[$activeImportJobId] = $insertImport(
'active-rescue',
$activeTemplateId,
'active',
'applied'
);
$firstDelete = CloudRescueTemplateDeletionService::delete(
1,
$activeTemplateId,
$actor
);
$firstDeleteJobId = (int) $firstDelete['task_id'];
$firstDeleteJob = CloudJob::findOrFail($firstDeleteJobId);
$firstDeleteOperation = (string) $firstDeleteJob->operation_id;
Capsule::table('cloud_jobs')->where('id', $firstDeleteJobId)->update([
'phase' => 'terminal',
'outcome' => 'failed',
'status' => 'failed',
'attempts' => 1,
'dispatch_attempts' => 1,
'dispatch_started_at' => $now,
'dispatched_at' => $now,
'agent_accepted_at' => $now,
'receipt_deadline_at' => $now + 60,
'terminal_at' => $now,
'effect_may_have_applied' => 0,
'safe_to_resubmit' => 1,
'effect_resolution' => 'not_applied',
'effect_resolved_at' => $now,
'effect_resolution_reason' => 'test_not_applied',
'error_code' => 'delete_failed',
'finalization_incident' => 0,
'finalization_error_code' => null,
]);
Capsule::table('cloud_rescue_template_deletions')
->where('host_id', 1)
->where('template_id', $activeTemplateId)
->update([
'state' => 'failed',
'error_code' => 'delete_failed',
'updated_at' => $now,
]);
$secondDelete = CloudRescueTemplateDeletionService::delete(
1,
$activeTemplateId,
$actor
);
$secondDeleteJobId = (int) $secondDelete['task_id'];
$secondDeleteRow = CloudRescueTemplateDeletion::query()
->where('host_id', 1)
->where('template_id', $activeTemplateId)
->firstOrFail();
$assert($secondDeleteJobId !== $firstDeleteJobId
&& (int) $secondDeleteRow->job_id === $secondDeleteJobId
&& !hash_equals(
$firstDeleteOperation,
(string) $secondDeleteRow->operation_id
)
&& (string) $secondDeleteRow->state === 'deleting'
&& (string) CloudJob::findOrFail($firstDeleteJobId)->effect_resolution
=== 'not_applied',
'A proven not-applied rescue deletion did not admit one new generation.'
);
Capsule::table('cloud_jobs')->where('id', $secondDeleteJobId)->update([
'phase' => 'terminal',
'outcome' => 'failed',
'status' => 'failed',
'dispatch_attempts' => 1,
'effect_may_have_applied' => 1,
'safe_to_resubmit' => 0,
'effect_resolution' => 'applied',
'finalization_incident' => 0,
'finalization_error_code' => null,
]);
Capsule::table('cloud_rescue_template_deletions')
->where('host_id', 1)
->where('template_id', $activeTemplateId)
->update(['state' => 'failed', 'error_code' => 'delete_failed']);
$beforeBlockedDelete = CloudJob::count();
try {
CloudRescueTemplateDeletionService::delete(
1,
$activeTemplateId,
$actor
);
throw new RuntimeException(
'An applied rescue deletion was admitted as a new generation.'
);
} catch (CloudError $error) {
$assert($error->httpStatus === 409,
'An unsafe rescue deletion returned the wrong status.');
}
$assert(CloudJob::count() === $beforeBlockedDelete
&& (int) CloudRescueTemplateDeletion::query()
->where('host_id', 1)
->where('template_id', $activeTemplateId)
->value('job_id') === $secondDeleteJobId,
'A blocked rescue deletion changed its job or durable slot.');
$retryTemplateId = str_repeat('b', 32);
$insertImport(
'retry-rescue',
$retryTemplateId,
'failed',
'not_applied'
);
$retryImport = CloudRescueTemplateImportService::start(
1,
'remote',
'retry-rescue',
'https://example.invalid/new-rescue.qcow2',
'',
'',
$actor
);
$retryImportJob = CloudJob::findOrFail((int) $retryImport['task_id']);
$assert((string) $retryImportJob->kind === 'rescue_template_import'
&& (string) $retryImportJob->phase === 'accepted'
&& CloudRescueTemplateImport::query()
->where('host_id', 1)
->where('name', 'retry-rescue')
->count() === 2,
'A proven not-applied rescue import did not admit a new operation.');
foreach ([
['applied-rescue', str_repeat('c', 32), 'applied', false],
['incident-rescue', str_repeat('d', 32), 'not_applied', true],
] as [$name, $templateId, $resolution, $incident]) {
$insertImport($name, $templateId, 'failed', $resolution, $incident);
$before = CloudJob::count();
try {
CloudRescueTemplateImportService::start(
1,
'remote',
$name,
'https://example.invalid/blocked-rescue.qcow2',
'',
'',
$actor
);
throw new RuntimeException(
'An unsafe same-name rescue import was admitted: ' . $name
);
} catch (CloudError $error) {
$assert($error->httpStatus === 409,
'An unsafe rescue import returned the wrong status.');
}
$assert(CloudJob::count() === $before,
'A blocked rescue import persisted a new job: ' . $name);
}
$tupleTemplateId = str_repeat('f', 32);
[$tupleJobId] = $insertImport(
'tuple-rescue',
$tupleTemplateId,
'failed',
'not_applied'
);
Capsule::table('cloud_jobs')->where('id', $tupleJobId)->update([
'effect_may_have_applied' => 1,
'safe_to_resubmit' => 1,
]);
$beforeTupleConflict = CloudJob::count();
try {
CloudRescueTemplateImportService::start(
1,
'remote',
'tuple-rescue',
'https://example.invalid/tuple-rescue.qcow2',
'',
'',
$actor
);
throw new RuntimeException(
'A rescue import with a contradictory terminal tuple was admitted.'
);
} catch (CloudError $error) {
$assert($error->httpStatus === 409,
'A rescue import tuple conflict returned the wrong status.');
}
$assert(CloudJob::count() === $beforeTupleConflict,
'A rescue import tuple conflict persisted a new job.');
$corruptActiveTemplateId = str_repeat('1', 32);
[$corruptActiveJobId] = $insertImport(
'corrupt-active-rescue',
$corruptActiveTemplateId,
'active',
'applied'
);
Capsule::table('cloud_jobs')->where('id', $corruptActiveJobId)->update([
'effect_may_have_applied' => 1,
]);
$beforeCorruptActiveDelete = CloudJob::count();
try {
CloudRescueTemplateDeletionService::delete(
1,
$corruptActiveTemplateId,
$actor
);
throw new RuntimeException(
'A deletion was admitted for a corrupt active import tuple.'
);
} catch (CloudError $error) {
$assert($error->httpStatus === 409,
'A corrupt active import tuple returned the wrong status.');
}
$assert(CloudJob::count() === $beforeCorruptActiveDelete,
'A corrupt active import tuple persisted a deletion job.');
$tupleDeleteTemplateId = str_repeat('2', 32);
$insertImport(
'tuple-delete-rescue',
$tupleDeleteTemplateId,
'active',
'applied'
);
$tupleDelete = CloudRescueTemplateDeletionService::delete(
1,
$tupleDeleteTemplateId,
$actor
);
$tupleDeleteJobId = (int) $tupleDelete['task_id'];
Capsule::table('cloud_jobs')->where('id', $tupleDeleteJobId)->update([
'phase' => 'terminal',
'outcome' => 'failed',
'status' => 'failed',
'attempts' => 1,
'dispatch_attempts' => 1,
'dispatch_started_at' => $now,
'dispatched_at' => $now,
'agent_accepted_at' => $now,
'terminal_at' => $now,
'effect_may_have_applied' => 1,
'safe_to_resubmit' => 1,
'effect_resolution' => 'not_applied',
'effect_resolved_at' => $now,
'effect_resolution_reason' => 'invalid_test_tuple',
'error_code' => 'delete_failed',
'finalization_incident' => 0,
'finalization_error_code' => null,
]);
Capsule::table('cloud_rescue_template_deletions')
->where('host_id', 1)
->where('template_id', $tupleDeleteTemplateId)
->update(['state' => 'failed', 'error_code' => 'delete_failed']);
$beforeTupleDeleteRetry = CloudJob::count();
try {
CloudRescueTemplateDeletionService::delete(
1,
$tupleDeleteTemplateId,
$actor
);
throw new RuntimeException(
'A rescue deletion with a contradictory terminal tuple was admitted.'
);
} catch (CloudError $error) {
$assert($error->httpStatus === 409,
'A rescue deletion tuple conflict returned the wrong status.');
}
$assert(CloudJob::count() === $beforeTupleDeleteRetry,
'A rescue deletion tuple conflict persisted a new job.');
$identityTemplateId = str_repeat('e', 32);
[, $identityOperation] = $insertImport(
'identity-rescue',
$identityTemplateId,
'failed',
'not_applied'
);
Capsule::table('cloud_rescue_template_imports')
->where('operation_id', $identityOperation)
->update(['operation_id' => bin2hex(random_bytes(16))]);
$beforeIdentityConflict = CloudJob::count();
try {
CloudRescueTemplateImportService::start(
1,
'remote',
'identity-rescue',
'https://example.invalid/identity-rescue.qcow2',
'',
'',
$actor
);
throw new RuntimeException(
'A rescue import with a corrupt durable identity was admitted.'
);
} catch (CloudError $error) {
$assert($error->httpStatus === 409,
'A rescue import identity conflict returned the wrong status.');
}
$assert(CloudJob::count() === $beforeIdentityConflict,
'A rescue import identity conflict persisted a new job.');
// Rescue-template deletion consumes the same reconstructed completion
// operation contract as template and user-ISO deletion. Exercise its
// actual ledger finalizer so a future adapter change cannot strand this
// standalone catalog operation in a failed state.
$completionTemplateId = str_repeat('3', 32);
$insertImport(
'completion-rescue',
$completionTemplateId,
'active',
'applied'
);
$completionDelete = CloudRescueTemplateDeletionService::delete(
1,
$completionTemplateId,
$actor
);
$completionDeleteJob = CloudJob::findOrFail(
(int) $completionDelete['task_id']
);
$completionResult = [
'resource_type' => 'rescue_template',
'id' => $completionTemplateId,
'deleted' => true,
'managed_image_absent' => true,
'managed_metadata_absent' => true,
'managed_artifacts_absent' => true,
];
$completionEvent = new CloudAgentCompletionEvent();
$completionEvent->operation_id =
(string) $completionDeleteJob->operation_id;
$completionEvent->outcome = 'succeeded';
$completionEvent->effect_may_have_applied = 0;
$completionEvent->safe_to_resubmit = 0;
$completionEvent->error_code = '';
$completionEvent->result_json = json_encode([
'schema' => 'virt_cloud_completion_event',
'version' => 1,
'identity' => [
'action' => 'delete-rescue-template',
'resource_id' => 'rescue-template:' . $completionTemplateId,
'request_fingerprint' => str_repeat('4', 64),
],
'result' => $completionResult,
], JSON_THROW_ON_ERROR);
CloudRescueTemplateDeletionService::finishJobLocked(
$completionDeleteJob,
'succeeded',
$completionResult,
CloudOperationCompletionService::operation($completionEvent)
);
$assert((string) CloudRescueTemplateDeletion::query()
->where('host_id', 1)
->where('template_id', $completionTemplateId)
->value('state') === 'succeeded'
&& (string) CloudRescueTemplateImport::query()
->where('host_id', 1)
->where('template_id', $completionTemplateId)
->value('state') === 'retired',
'A durable rescue-template deletion receipt did not finalize its '
. 'catalog ledgers.');
echo "Cloud rescue-template retry boundary checks passed.\n";
}