You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
virt/web/tests/CloudRescueReplicaWorkflowT...

562 lines
22 KiB
PHP

<?php
namespace App\Services {
final class Config
{
public static string $key = '';
public static function get($name, $default = null)
{
return $name === 'cloud_data_key' ? self::$key : $default;
}
}
}
namespace {
require dirname(__DIR__) . '/vendor/autoload.php';
use App\Models\CloudJob;
use App\Models\CloudRescueTemplate;
use App\Models\CloudRescueTemplateArtifact;
use App\Models\CloudRescueTemplateReplica;
use App\Services\Cloud\CloudObjectStorageS3Client;
use App\Services\Cloud\CloudRescueCatalogService;
use App\Services\Cloud\CloudRescueTemplateReplicaCoordinator;
use App\Services\Cloud\CloudRescueTemplateReconciliationService;
use App\Services\Cloud\CloudRescueTemplateRetirementService;
use App\Services\Cloud\CloudSecrets;
use App\Services\Config;
use GuzzleHttp\Client;
use GuzzleHttp\Handler\MockHandler;
use GuzzleHttp\HandlerStack;
use GuzzleHttp\Psr7\Response;
use Illuminate\Database\Capsule\Manager as Capsule;
use Illuminate\Database\Schema\Blueprint;
$assert = static function (bool $condition, string $message): void {
if (!$condition) {
throw new RuntimeException($message);
}
};
Config::$key = base64_encode(str_repeat('q', 32));
$capsule = new Capsule();
$capsule->addConnection([
'driver' => 'sqlite', 'database' => ':memory:',
], 'default');
$capsule->setAsGlobal();
$capsule->bootEloquent();
$schema = $capsule->schema('default');
$schema->create('cloud_hosts', static function (Blueprint $table): void {
$table->integer('id')->primary();
$table->boolean('enabled');
$table->string('default_network');
});
$schema->create('cloud_host_agents',
static function (Blueprint $table): void {
$table->integer('host_id')->primary();
$table->boolean('enabled');
$table->string('last_test_status');
$table->integer('tested_revision');
$table->integer('revision');
});
$schema->create('cloud_host_storage_configurations',
static function (Blueprint $table): void {
$table->integer('host_id')->primary();
$table->string('state');
});
$schema->create('cloud_jobs', static function (Blueprint $table): void {
$table->increments('id');
$table->string('operation_id')->unique();
$table->string('request_idempotency_key');
$table->string('request_fingerprint');
$table->integer('parent_job_id')->nullable();
$table->integer('user_id');
$table->integer('host_id')->nullable();
$table->string('kind');
$table->string('execution_mode');
$table->string('phase');
$table->string('outcome')->nullable();
$table->string('status');
$table->text('sealed_payload');
$table->integer('attempts')->default(0);
$table->integer('dispatch_attempts')->default(0);
$table->integer('dispatch_deadline_at')->default(1);
$table->integer('dispatch_started_at')->nullable();
$table->integer('agent_accepted_at')->nullable();
$table->integer('dispatched_at')->nullable();
$table->integer('receipt_deadline_at')->nullable();
$table->integer('terminal_at')->nullable();
$table->boolean('effect_may_have_applied')->nullable();
$table->boolean('safe_to_resubmit')->nullable();
$table->string('effect_resolution')->default('not_applicable');
$table->integer('effect_resolved_at')->nullable();
$table->integer('effect_resolved_by')->nullable();
$table->string('effect_resolution_reason')->default('');
$table->boolean('finalization_incident')->default(false);
$table->string('finalization_error_code')->nullable();
$table->string('completion_event_id')->nullable();
$table->integer('read_at')->nullable();
$table->string('lease_token')->nullable();
$table->integer('lease_until')->default(0);
$table->integer('available_at')->default(0);
$table->string('error_code')->nullable();
$table->text('result_json')->nullable();
$table->integer('created_at');
$table->integer('updated_at');
});
$schema->create('cloud_rescue_templates',
static function (Blueprint $table): void {
$table->string('id')->primary();
$table->string('name');
$table->integer('size_bytes');
$table->string('sha256');
$table->integer('revision');
$table->string('generation_token');
$table->string('state');
$table->integer('active_job_id')->nullable();
$table->integer('created_at');
$table->integer('updated_at');
});
$schema->create('cloud_rescue_template_artifacts',
static function (Blueprint $table): void {
$table->string('template_id')->primary();
$table->integer('storage_revision');
$table->text('sealed_storage_runtime');
$table->string('object_key');
$table->string('object_version_id');
$table->string('object_etag');
$table->integer('size_bytes');
$table->string('sha256');
$table->string('state');
$table->string('error_code')->default('');
$table->string('delete_operation_id')->default('');
$table->integer('delete_generation')->default(0);
$table->integer('delete_cycle_attempts')->default(0);
$table->integer('delete_presence_confirmations')->default(0);
$table->integer('delete_last_checked_at')->default(0);
$table->integer('delete_next_check_at')->default(0);
$table->integer('delete_attention_at')->nullable();
$table->integer('verified_at')->default(0);
$table->integer('created_at');
$table->integer('updated_at');
});
$schema->create('cloud_rescue_template_replicas',
static function (Blueprint $table): void {
$table->integer('host_id');
$table->string('template_id');
$table->integer('revision');
$table->string('generation_token');
$table->string('state');
$table->integer('size_bytes');
$table->string('sha256');
$table->string('operation_id');
$table->integer('job_id')->nullable();
$table->string('error_code')->default('');
$table->string('observed_state')->default('never');
$table->integer('observed_revision')->default(0);
$table->string('observed_generation_token')->default('');
$table->integer('observed_size_bytes')->default(0);
$table->string('observed_sha256')->default('');
$table->integer('last_verified_at')->default(0);
$table->integer('cleanup_resolved_at')->nullable();
$table->integer('created_at');
$table->integer('updated_at');
$table->primary(['host_id', 'template_id']);
});
$addHost = static function (int $hostId) use ($capsule): void {
Capsule::table('cloud_hosts')->insert([
'id' => $hostId, 'enabled' => 1,
'default_network' => 'bridge:br0',
]);
Capsule::table('cloud_host_agents')->insert([
'host_id' => $hostId, 'enabled' => 1,
'last_test_status' => 'healthy',
'tested_revision' => 3, 'revision' => 3,
]);
Capsule::table('cloud_host_storage_configurations')->insert([
'host_id' => $hostId, 'state' => 'applied',
]);
};
$addHost(31);
$addHost(32);
$now = time();
$templateId = str_repeat('a', 32);
$generation = str_repeat('b', 32);
$sha256 = str_repeat('c', 64);
$bytes = 2 * 1024 * 1024;
Capsule::table('cloud_rescue_templates')->insert([
'id' => $templateId, 'name' => 'SystemRescue',
'size_bytes' => $bytes, 'sha256' => $sha256,
'revision' => 1, 'generation_token' => $generation,
'state' => 'validating', 'active_job_id' => null,
'created_at' => $now, 'updated_at' => $now,
]);
$runtime = [
'config' => [
'endpoint' => 'https://objects.example.com',
'region' => 'ewr', 'bucket' => 'rescue',
'prefix' => 'virt', 'path_style' => true,
],
'credentials' => [
'access_key_id' => 'ACCESSKEY',
'secret_access_key' => 'secret',
],
'revision' => 1,
];
Capsule::table('cloud_rescue_template_artifacts')->insert([
'template_id' => $templateId, 'storage_revision' => 1,
'sealed_storage_runtime' => CloudSecrets::encrypt(
$runtime,
'rescue-artifact-runtime'
),
'object_key' => 'virt/rescue-isos/' . $templateId . '/'
. $generation . '/rescue.iso',
'object_version_id' => 'version-1',
'object_etag' => '"etag-1"', 'size_bytes' => $bytes,
'sha256' => $sha256, 'state' => 'validating',
'created_at' => $now, 'updated_at' => $now,
]);
$head = static fn (): Response => new Response(200, [
'Content-Length' => (string) $bytes,
'ETag' => '"etag-1"',
'x-amz-version-id' => 'version-1',
]);
$storageClient = new CloudObjectStorageS3Client(new Client([
'handler' => HandlerStack::create(new MockHandler([
$head(), $head(),
])),
]));
$actor = [
'is_super_admin' => true, 'can_manage' => true, 'user_id' => 1,
];
$validation = CloudRescueTemplateReplicaCoordinator::beginValidation(
$templateId,
31,
$actor,
[31, 32],
$storageClient
);
$assert(($validation['scheduled'] ?? false) === true,
'First-host validation was not scheduled.');
$validationJob = CloudJob::findOrFail((int) $validation['task_id']);
$assert((string) $validationJob->kind
=== 'rescue_catalog_validate_replica'
&& (int) CloudRescueTemplate::findOrFail($templateId)->active_job_id
=== (int) $validationJob->id,
'Validation job did not own the quarantine fence.');
$result = [
'id' => $templateId, 'name' => 'SystemRescue',
'size_bytes' => $bytes, 'sha256' => $sha256,
'revision' => 1, 'generation_token' => $generation,
'created_at' => gmdate(DATE_RFC3339),
];
CloudRescueTemplateReplicaCoordinator::finishJobLocked(
$validationJob,
'succeeded',
$result,
['rescue_template_id' => $templateId],
['effect_may_have_applied' => false, 'safe_to_resubmit' => false]
);
$assert((string) CloudRescueTemplateArtifact::findOrFail($templateId)->state
=== 'ready'
&& (string) CloudRescueTemplate::findOrFail($templateId)->state
=== 'distributing'
&& CloudRescueTemplate::findOrFail($templateId)->active_job_id === null,
'Successful first-host validation did not promote the object.');
$distribution = CloudRescueTemplateReplicaCoordinator::distribute(
$templateId,
$actor,
'validation_complete',
[31, 32],
$storageClient
);
$assert((int) $distribution['scheduled'] === 1
&& (int) $distribution['tasks'][0]['host_id'] === 32,
'Distribution did not skip the exact validation host.');
$replicaJob = CloudJob::findOrFail(
(int) $distribution['tasks'][0]['task_id']
);
CloudRescueTemplateReplicaCoordinator::finishJobLocked(
$replicaJob,
'succeeded',
$result,
['rescue_template_id' => $templateId],
['effect_may_have_applied' => false, 'safe_to_resubmit' => false]
);
$assert((string) CloudRescueTemplate::findOrFail($templateId)->state
=== 'active'
&& CloudRescueTemplateReplica::query()
->where('template_id', $templateId)
->where('state', 'active')->count() === 2
&& (int) CloudRescueTemplateReplica::query()
->where('template_id', $templateId)->where('host_id', 31)
->value('job_id') === (int) $validationJob->id
&& (int) CloudRescueTemplateReplica::query()
->where('template_id', $templateId)->where('host_id', 32)
->value('job_id') === (int) $replicaJob->id,
'All exact replicas did not activate the catalog.');
$publicReady = CloudRescueCatalogService::publicCatalogForHost(31);
$publicUnavailable = CloudRescueCatalogService::publicCatalogForHost(99);
$assert(($publicReady['status']['state'] ?? null) === 'ready'
&& ($publicReady['templates'][0]['id'] ?? null) === $templateId
&& array_keys($publicReady['templates'][0]) === ['id', 'name']
&& ($publicUnavailable['status']['state'] ?? null)
=== 'host_unavailable'
&& $publicUnavailable['templates'] === [],
'Public rescue catalog exposed a non-exact or unavailable host copy.');
$addHost(33);
$publicSyncing = CloudRescueCatalogService::publicCatalogForHost(33);
$assert(($publicSyncing['status']['state'] ?? null) === 'syncing'
&& $publicSyncing['templates'] === [],
'Public rescue catalog did not distinguish a pending host replica.');
$adminCatalog = CloudRescueCatalogService::adminCatalog();
$adminEntry = $adminCatalog['templates'][0] ?? null;
$missingHost = is_array($adminEntry)
? array_values(array_filter(
$adminEntry['replicas'],
static fn (array $replica): bool =>
(int) $replica['host_id'] === 33
))[0] ?? null
: null;
$assert(count($adminCatalog['hosts']) === 3
&& is_array($adminEntry)
&& (int) $adminEntry['replica_summary']['eligible'] === 3
&& (int) $adminEntry['replica_summary']['ready'] === 2
&& (int) $adminEntry['replica_summary']['attention'] === 1
&& is_array($missingHost)
&& $missingHost['state'] === 'missing'
&& $missingHost['repair_action'] === 'import'
&& $missingHost['repair_reason'] === 'missing',
'Administrator rescue catalog did not expose the exact host matrix.');
$remoteExact = $result;
$clientFactory = static fn (int $hostId): object => new class(
$hostId,
$remoteExact
) {
public function __construct(
private int $hostId,
private array $exact
) {
}
public function request(string $method, string $path, array $actor): array
{
if ($method !== 'GET' || $path !== '/v1/rescue-templates') {
throw new RuntimeException('Unexpected reconciliation request');
}
return ['templates' => $this->hostId === 33
? [] : [$this->exact]];
}
};
$repairs = [];
$reconciler = new CloudRescueTemplateReconciliationService(
$clientFactory,
static function (
string $id,
int $hostId
) use (&$repairs): array {
$repairs[] = [$id, $hostId];
return ['scheduled' => true];
}
);
$report = $reconciler->reconcile($actor, 3, 3, 0);
$assert($report['hosts_checked'] === 3
&& $report['missing'] === 1
&& $report['scheduled'] === 1
&& $repairs === [[$templateId, 33]]
&& (string) CloudRescueTemplate::findOrFail($templateId)->state
=== 'degraded',
'Periodic reconciliation did not project and repair a missing host.');
$retirement = CloudRescueTemplateRetirementService::retire(
$templateId,
1,
$generation,
$actor
);
$assert((string) CloudRescueTemplate::findOrFail($templateId)->state
=== 'retiring'
&& (int) $retirement['cleanup_debt'] === 2
&& count($retirement['tasks']) === 2
&& (string) CloudRescueTemplateReplica::query()
->where('template_id', $templateId)->where('host_id', 33)
->value('state') === 'deleted',
'Logical retirement did not stop publication or resolve known absence.');
$deleteResult = [
'resource_type' => 'rescue_template',
'id' => $templateId,
'deleted' => true,
'managed_image_absent' => true,
'managed_metadata_absent' => true,
'managed_artifacts_absent' => true,
];
foreach ($retirement['tasks'] as $task) {
$job = CloudJob::findOrFail((int) $task['task_id']);
CloudRescueTemplateRetirementService::finishJobLocked(
$job,
'succeeded',
$deleteResult,
['rescue_template_id' => $templateId],
[
'status' => 'succeeded',
'retryable' => false,
'result' => $deleteResult,
'effect_may_have_applied' => false,
'safe_to_resubmit' => false,
]
);
}
$assert((string) CloudRescueTemplateArtifact::findOrFail($templateId)->state
=== 'delete_pending'
&& CloudRescueTemplateReplica::query()
->where('template_id', $templateId)
->where('state', '<>', 'deleted')->count() === 0,
'Canonical object cleanup was armed before all replicas proved absent.');
$deleteClient = new CloudObjectStorageS3Client(new Client([
'handler' => HandlerStack::create(new MockHandler([
$head(),
new Response(204),
new Response(404),
])),
]));
$cleanup = CloudRescueTemplateRetirementService::processDue(
$actor,
8,
$deleteClient
);
$assert((int) $cleanup['completed'] === 1
&& (string) CloudRescueTemplate::findOrFail($templateId)->state
=== 'retired'
&& (string) CloudRescueTemplateArtifact::findOrFail($templateId)->state
=== 'deleted'
&& CloudRescueTemplateReplica::query()
->where('template_id', $templateId)->count() === 3,
'Exact object deletion did not retain a settled retirement tombstone.');
$unknownId = str_repeat('d', 32);
$unknownGeneration = str_repeat('e', 32);
Capsule::table('cloud_rescue_templates')->insert([
'id' => $unknownId,
'name' => 'Unknown delete rescue',
'size_bytes' => $bytes,
'sha256' => $sha256,
'revision' => 1,
'generation_token' => $unknownGeneration,
'state' => 'active',
'active_job_id' => null,
'created_at' => $now,
'updated_at' => $now,
]);
Capsule::table('cloud_rescue_template_artifacts')->insert([
'template_id' => $unknownId,
'storage_revision' => 1,
'sealed_storage_runtime' => CloudSecrets::encrypt(
$runtime,
'rescue-artifact-runtime'
),
'object_key' => 'virt/rescue-isos/' . $unknownId . '/'
. $unknownGeneration . '/rescue.iso',
'object_version_id' => 'version-1',
'object_etag' => '"etag-1"',
'size_bytes' => $bytes,
'sha256' => $sha256,
'state' => 'ready',
'created_at' => $now,
'updated_at' => $now,
]);
Capsule::table('cloud_rescue_template_replicas')->insert([
'host_id' => 31,
'template_id' => $unknownId,
'revision' => 1,
'generation_token' => $unknownGeneration,
'state' => 'active',
'size_bytes' => $bytes,
'sha256' => $sha256,
'operation_id' => '',
'job_id' => null,
'error_code' => '',
'observed_state' => 'exact',
'observed_revision' => 1,
'observed_generation_token' => $unknownGeneration,
'observed_size_bytes' => $bytes,
'observed_sha256' => $sha256,
'last_verified_at' => $now,
'cleanup_resolved_at' => null,
'created_at' => $now,
'updated_at' => $now,
]);
$unknownRetirement = CloudRescueTemplateRetirementService::retire(
$unknownId,
1,
$unknownGeneration,
$actor
);
$unknownJob = CloudJob::findOrFail(
(int) $unknownRetirement['tasks'][0]['task_id']
);
CloudRescueTemplateRetirementService::finishJobLocked(
$unknownJob,
'failed',
null,
['rescue_template_id' => $unknownId],
[
'effect_may_have_applied' => true,
'safe_to_resubmit' => false,
'cause_code' => 'agent_response_lost',
]
);
$debt = CloudRescueTemplateRetirementService::cleanupState($unknownId);
$assert(($debt['replica_states']['delete_unknown'] ?? 0) === 1
&& (string) CloudRescueTemplateArtifact::findOrFail($unknownId)->state
=== 'ready',
'Unknown host deletion did not retain cleanup debt and the object.');
$absenceReconciler = new CloudRescueTemplateReconciliationService(
static fn (int $hostId): object => new class {
public function request(
string $method,
string $path,
array $actor
): array {
return ['templates' => []];
}
},
static function (): array {
throw new RuntimeException(
'Retiring rescue ISO attempted an import repair.'
);
}
);
$absenceReport = $absenceReconciler->reconcile($actor, 3, 3, 0);
$assert($absenceReport['scheduled'] === 0
&& (string) CloudRescueTemplateReplica::query()
->where('template_id', $unknownId)->where('host_id', 31)
->value('state') === 'deleted',
'Trusted host absence did not resolve unknown deletion debt.');
$unknownDeleteClient = new CloudObjectStorageS3Client(new Client([
'handler' => HandlerStack::create(new MockHandler([
$head(),
new Response(204),
new Response(404),
])),
]));
$unknownCleanup = CloudRescueTemplateRetirementService::processDue(
$actor,
8,
$unknownDeleteClient
);
$assert((int) $unknownCleanup['completed'] === 1
&& (string) CloudRescueTemplate::findOrFail($unknownId)->state
=== 'retired',
'Reconciled host absence did not release exact object cleanup.');
echo "Rescue replica validation, distribution and reconciliation passed.\n";
}