You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
907 lines
38 KiB
PHP
907 lines
38 KiB
PHP
<?php
|
|
|
|
namespace App\Services {
|
|
final class Config
|
|
{
|
|
public static string $key = '';
|
|
|
|
public static function get($name, $default = null)
|
|
{
|
|
return $name === 'cloud_data_key' ? self::$key : $default;
|
|
}
|
|
}
|
|
}
|
|
|
|
namespace {
|
|
require dirname(__DIR__) . '/vendor/autoload.php';
|
|
|
|
use App\Models\CloudJob;
|
|
use App\Models\CloudRescueTemplate;
|
|
use App\Models\CloudRescueTemplateArtifact;
|
|
use App\Models\CloudRescueTemplateReplica;
|
|
use App\Services\Cloud\CloudObjectStorageS3Client;
|
|
use App\Services\Cloud\CloudRescueCatalogService;
|
|
use App\Services\Cloud\CloudRescueTemplateReplicaCoordinator;
|
|
use App\Services\Cloud\CloudRescueTemplateReconciliationService;
|
|
use App\Services\Cloud\CloudRescueTemplateRetirementService;
|
|
use App\Services\Cloud\CloudSecrets;
|
|
use App\Services\Config;
|
|
use GuzzleHttp\Client;
|
|
use GuzzleHttp\Handler\MockHandler;
|
|
use GuzzleHttp\HandlerStack;
|
|
use GuzzleHttp\Psr7\Response;
|
|
use Illuminate\Database\Capsule\Manager as Capsule;
|
|
use Illuminate\Database\Schema\Blueprint;
|
|
|
|
$assert = static function (bool $condition, string $message): void {
|
|
if (!$condition) {
|
|
throw new RuntimeException($message);
|
|
}
|
|
};
|
|
Config::$key = base64_encode(str_repeat('q', 32));
|
|
$capsule = new Capsule();
|
|
$capsule->addConnection([
|
|
'driver' => 'sqlite', 'database' => ':memory:',
|
|
], 'default');
|
|
$capsule->setAsGlobal();
|
|
$capsule->bootEloquent();
|
|
$schema = $capsule->schema('default');
|
|
$schema->create('cloud_hosts', static function (Blueprint $table): void {
|
|
$table->integer('id')->primary();
|
|
$table->boolean('enabled');
|
|
$table->string('default_network');
|
|
});
|
|
$schema->create('cloud_host_agents',
|
|
static function (Blueprint $table): void {
|
|
$table->integer('host_id')->primary();
|
|
$table->boolean('enabled');
|
|
$table->string('last_test_status');
|
|
$table->integer('tested_revision');
|
|
$table->integer('revision');
|
|
});
|
|
$schema->create('cloud_host_storage_configurations',
|
|
static function (Blueprint $table): void {
|
|
$table->integer('host_id')->primary();
|
|
$table->string('state');
|
|
});
|
|
$schema->create('cloud_storage_backends', static function (Blueprint $t): void {
|
|
$t->increments('id'); $t->string('backend_key');
|
|
$t->string('identity_sha256'); $t->string('name');
|
|
$t->string('driver'); $t->string('endpoint'); $t->string('region');
|
|
$t->string('bucket'); $t->string('prefix'); $t->boolean('path_style');
|
|
$t->string('failure_domain'); $t->text('sealed_credentials');
|
|
$t->integer('config_revision'); $t->integer('tested_revision');
|
|
$t->string('admin_state'); $t->string('health_state');
|
|
$t->integer('write_priority'); $t->integer('consecutive_failures')->default(0);
|
|
$t->integer('consecutive_successes')->default(0);
|
|
$t->string('last_error_code')->default('');
|
|
$t->integer('last_tested_at')->default(0);
|
|
$t->integer('last_healthy_at')->default(0);
|
|
$t->integer('next_probe_at')->default(0);
|
|
$t->integer('created_at'); $t->integer('updated_at');
|
|
});
|
|
$schema->create('cloud_storage_artifacts', static function (Blueprint $t): void {
|
|
$t->increments('id'); $t->string('artifact_key')->unique();
|
|
$t->string('kind'); $t->integer('maximum_size_bytes');
|
|
$t->integer('size_bytes'); $t->string('sha256');
|
|
$t->string('durability_policy'); $t->integer('desired_copies');
|
|
$t->integer('required_copies'); $t->string('state');
|
|
$t->string('last_operation_id')->nullable(); $t->integer('verified_at');
|
|
$t->integer('created_at'); $t->integer('updated_at');
|
|
});
|
|
$schema->create('cloud_storage_replicas', static function (Blueprint $t): void {
|
|
$t->increments('id'); $t->string('replica_key')->unique();
|
|
$t->integer('artifact_id'); $t->integer('backend_id');
|
|
$t->string('role'); $t->string('object_key');
|
|
$t->string('object_version_id')->default('');
|
|
$t->string('object_etag')->default('');
|
|
$t->integer('size_bytes')->default(0); $t->string('sha256')->default('');
|
|
$t->string('state'); $t->string('publish_operation_id')->nullable();
|
|
$t->string('delete_operation_id')->nullable();
|
|
$t->integer('delete_generation')->default(0);
|
|
$t->integer('delete_cycle_attempts')->default(0);
|
|
$t->integer('delete_presence_confirmations')->default(0);
|
|
$t->integer('delete_last_checked_at')->default(0);
|
|
$t->integer('delete_next_check_at')->default(0);
|
|
$t->integer('delete_attention_at')->nullable();
|
|
$t->string('lease_token')->nullable(); $t->integer('lease_until')->default(0);
|
|
$t->string('error_code')->default(''); $t->integer('verified_at')->default(0);
|
|
$t->integer('created_at'); $t->integer('updated_at');
|
|
});
|
|
$schema->create('cloud_storage_operations', static function (Blueprint $t): void {
|
|
$t->increments('id'); $t->string('operation_id')->unique();
|
|
$t->string('request_idempotency_key')->unique();
|
|
$t->string('request_fingerprint'); $t->integer('artifact_id');
|
|
$t->integer('replica_id')->nullable();
|
|
$t->integer('source_backend_id')->nullable();
|
|
$t->integer('target_backend_id')->nullable(); $t->string('kind');
|
|
$t->string('phase'); $t->string('outcome')->nullable();
|
|
$t->string('effect_resolution');
|
|
$t->boolean('effect_may_have_applied')->nullable();
|
|
$t->boolean('safe_to_resubmit')->nullable(); $t->integer('attempts');
|
|
$t->string('lease_token')->nullable(); $t->integer('lease_until');
|
|
$t->integer('available_at'); $t->integer('next_check_at');
|
|
$t->string('error_code'); $t->text('result_json')->nullable();
|
|
$t->integer('terminal_at')->nullable(); $t->integer('created_at');
|
|
$t->integer('updated_at');
|
|
});
|
|
$schema->create('cloud_storage_publications', static function (Blueprint $t): void {
|
|
$t->increments('id'); $t->string('publication_key')->unique();
|
|
$t->integer('artifact_id')->unique(); $t->integer('backup_id')->nullable();
|
|
$t->string('template_import_intent_id')->nullable();
|
|
$t->string('rescue_template_id')->nullable()->unique(); $t->string('state');
|
|
$t->integer('attempts')->default(0); $t->integer('available_at')->default(0);
|
|
$t->string('lease_token')->nullable(); $t->integer('lease_until')->default(0);
|
|
$t->string('last_error_code')->default('');
|
|
$t->integer('coverage_ready_at')->nullable();
|
|
$t->integer('finalized_at')->nullable();
|
|
$t->integer('created_at'); $t->integer('updated_at');
|
|
});
|
|
$schema->create('cloud_jobs', static function (Blueprint $table): void {
|
|
$table->increments('id');
|
|
$table->string('operation_id')->unique();
|
|
$table->string('request_idempotency_key');
|
|
$table->string('request_fingerprint');
|
|
$table->integer('parent_job_id')->nullable();
|
|
$table->integer('user_id');
|
|
$table->integer('host_id')->nullable();
|
|
$table->string('kind');
|
|
$table->string('execution_mode');
|
|
$table->string('phase');
|
|
$table->string('outcome')->nullable();
|
|
$table->string('status');
|
|
$table->text('sealed_payload');
|
|
$table->integer('attempts')->default(0);
|
|
$table->integer('dispatch_attempts')->default(0);
|
|
$table->integer('dispatch_deadline_at')->default(1);
|
|
$table->integer('dispatch_started_at')->nullable();
|
|
$table->integer('agent_accepted_at')->nullable();
|
|
$table->integer('dispatched_at')->nullable();
|
|
$table->integer('receipt_deadline_at')->nullable();
|
|
$table->integer('terminal_at')->nullable();
|
|
$table->boolean('effect_may_have_applied')->nullable();
|
|
$table->boolean('safe_to_resubmit')->nullable();
|
|
$table->string('effect_resolution')->default('not_applicable');
|
|
$table->integer('effect_resolved_at')->nullable();
|
|
$table->integer('effect_resolved_by')->nullable();
|
|
$table->string('effect_resolution_reason')->default('');
|
|
$table->boolean('finalization_incident')->default(false);
|
|
$table->string('finalization_error_code')->nullable();
|
|
$table->string('completion_event_id')->nullable();
|
|
$table->integer('read_at')->nullable();
|
|
$table->string('lease_token')->nullable();
|
|
$table->integer('lease_until')->default(0);
|
|
$table->integer('available_at')->default(0);
|
|
$table->string('error_code')->nullable();
|
|
$table->text('result_json')->nullable();
|
|
$table->integer('created_at');
|
|
$table->integer('updated_at');
|
|
});
|
|
$schema->create('cloud_rescue_templates',
|
|
static function (Blueprint $table): void {
|
|
$table->string('id')->primary();
|
|
$table->string('name');
|
|
$table->integer('size_bytes');
|
|
$table->string('sha256');
|
|
$table->integer('revision');
|
|
$table->string('generation_token');
|
|
$table->string('state');
|
|
$table->integer('active_job_id')->nullable();
|
|
$table->integer('created_at');
|
|
$table->integer('updated_at');
|
|
});
|
|
$schema->create('cloud_rescue_template_artifacts',
|
|
static function (Blueprint $table): void {
|
|
$table->string('template_id')->primary();
|
|
$table->integer('storage_artifact_id')->nullable()->unique();
|
|
$table->integer('storage_revision');
|
|
$table->string('object_key');
|
|
$table->string('object_version_id');
|
|
$table->string('object_etag');
|
|
$table->integer('size_bytes');
|
|
$table->string('sha256');
|
|
$table->string('state');
|
|
$table->string('error_code')->default('');
|
|
$table->string('delete_operation_id')->default('');
|
|
$table->integer('delete_generation')->default(0);
|
|
$table->integer('delete_cycle_attempts')->default(0);
|
|
$table->integer('delete_presence_confirmations')->default(0);
|
|
$table->integer('delete_last_checked_at')->default(0);
|
|
$table->integer('delete_next_check_at')->default(0);
|
|
$table->integer('delete_attention_at')->nullable();
|
|
$table->integer('verified_at')->default(0);
|
|
$table->integer('created_at');
|
|
$table->integer('updated_at');
|
|
});
|
|
$schema->create('cloud_rescue_template_replicas',
|
|
static function (Blueprint $table): void {
|
|
$table->integer('host_id');
|
|
$table->string('template_id');
|
|
$table->integer('revision');
|
|
$table->string('generation_token');
|
|
$table->string('state');
|
|
$table->integer('size_bytes');
|
|
$table->string('sha256');
|
|
$table->string('operation_id');
|
|
$table->integer('job_id')->nullable();
|
|
$table->string('error_code')->default('');
|
|
$table->string('observed_state')->default('never');
|
|
$table->integer('observed_revision')->default(0);
|
|
$table->string('observed_generation_token')->default('');
|
|
$table->integer('observed_size_bytes')->default(0);
|
|
$table->string('observed_sha256')->default('');
|
|
$table->integer('last_verified_at')->default(0);
|
|
$table->integer('cleanup_resolved_at')->nullable();
|
|
$table->integer('created_at');
|
|
$table->integer('updated_at');
|
|
$table->primary(['host_id', 'template_id']);
|
|
});
|
|
|
|
$addHost = static function (int $hostId) use ($capsule): void {
|
|
Capsule::table('cloud_hosts')->insert([
|
|
'id' => $hostId, 'enabled' => 1,
|
|
'default_network' => 'bridge:br0',
|
|
]);
|
|
Capsule::table('cloud_host_agents')->insert([
|
|
'host_id' => $hostId, 'enabled' => 1,
|
|
'last_test_status' => 'healthy',
|
|
'tested_revision' => 3, 'revision' => 3,
|
|
]);
|
|
Capsule::table('cloud_host_storage_configurations')->insert([
|
|
'host_id' => $hostId, 'state' => 'applied',
|
|
]);
|
|
};
|
|
$addHost(31);
|
|
$addHost(32);
|
|
|
|
$now = time();
|
|
$templateId = str_repeat('a', 32);
|
|
$generation = str_repeat('b', 32);
|
|
$sha256 = str_repeat('c', 64);
|
|
$bytes = 2 * 1024 * 1024;
|
|
Capsule::table('cloud_rescue_templates')->insert([
|
|
'id' => $templateId, 'name' => 'SystemRescue',
|
|
'size_bytes' => $bytes, 'sha256' => $sha256,
|
|
'revision' => 1, 'generation_token' => $generation,
|
|
'state' => 'validating', 'active_job_id' => null,
|
|
'created_at' => $now, 'updated_at' => $now,
|
|
]);
|
|
$runtime = [
|
|
'config' => [
|
|
'endpoint' => 'https://objects.example.com',
|
|
'region' => 'ewr', 'bucket' => 'rescue',
|
|
'prefix' => 'virt', 'path_style' => true,
|
|
],
|
|
'credentials' => [
|
|
'access_key_id' => 'ACCESSKEY',
|
|
'secret_access_key' => 'secret-access-key',
|
|
],
|
|
'revision' => 1,
|
|
];
|
|
foreach ([
|
|
[1, 'primary', 'https://objects.example.com', 'rescue', 'provider-a', 10],
|
|
[2, 'secondary', 'https://objects-b.example.com', 'rescue-b', 'provider-b', 20],
|
|
] as [$backendId, $backendName, $endpoint, $bucket, $domain, $priority]) {
|
|
Capsule::table('cloud_storage_backends')->insert([
|
|
'id' => $backendId,
|
|
'backend_key' => str_repeat((string) $backendId, 32),
|
|
'identity_sha256' => hash('sha256', $endpoint . '|' . $bucket),
|
|
'name' => $backendName, 'driver' => 's3',
|
|
'endpoint' => $endpoint, 'region' => 'ewr', 'bucket' => $bucket,
|
|
'prefix' => 'virt', 'path_style' => 1,
|
|
'failure_domain' => $domain,
|
|
'sealed_credentials' => CloudSecrets::encrypt(
|
|
$runtime['credentials'], 'object-storage-backend-credentials'
|
|
),
|
|
'config_revision' => 1, 'tested_revision' => 1,
|
|
'admin_state' => 'active', 'health_state' => 'healthy',
|
|
'write_priority' => $priority, 'consecutive_failures' => 0,
|
|
'consecutive_successes' => 1, 'last_error_code' => '',
|
|
'last_tested_at' => $now, 'last_healthy_at' => $now,
|
|
'next_probe_at' => $now + 300,
|
|
'created_at' => $now, 'updated_at' => $now,
|
|
]);
|
|
}
|
|
$storageSequence = 0;
|
|
$seedStorage = static function (
|
|
string $ownerId,
|
|
string $generationToken,
|
|
int $sizeBytes,
|
|
string $digest
|
|
) use (&$storageSequence, $now): int {
|
|
$storageSequence++;
|
|
$artifactId = Capsule::table('cloud_storage_artifacts')->insertGetId([
|
|
'artifact_key' => str_pad((string) $storageSequence, 32, 'a'),
|
|
'kind' => 'rescue', 'maximum_size_bytes' => $sizeBytes,
|
|
'size_bytes' => $sizeBytes, 'sha256' => $digest,
|
|
'durability_policy' => 'dual_region', 'desired_copies' => 2,
|
|
'required_copies' => 2, 'state' => 'protected',
|
|
'last_operation_id' => null, 'verified_at' => $now,
|
|
'created_at' => $now, 'updated_at' => $now,
|
|
]);
|
|
$operations = [];
|
|
foreach ([1, 2] as $index => $backendId) {
|
|
$operationId = str_pad(
|
|
dechex($storageSequence * 10 + $backendId),
|
|
32,
|
|
(string) $backendId,
|
|
STR_PAD_LEFT
|
|
);
|
|
$replicaId = Capsule::table('cloud_storage_replicas')->insertGetId([
|
|
'replica_key' => str_pad(
|
|
dechex($storageSequence * 10 + $backendId), 32, 'b'
|
|
),
|
|
'artifact_id' => $artifactId, 'backend_id' => $backendId,
|
|
'role' => $index === 0 ? 'primary' : 'secondary',
|
|
'object_key' => 'virt/rescue-isos/' . $ownerId . '/'
|
|
. $generationToken . '/rescue.iso',
|
|
'object_version_id' => 'version-1',
|
|
'object_etag' => '"etag-1"',
|
|
'size_bytes' => $sizeBytes, 'sha256' => $digest,
|
|
'state' => 'ready', 'publish_operation_id' => $operationId,
|
|
'delete_operation_id' => null, 'delete_generation' => 0,
|
|
'delete_cycle_attempts' => 0,
|
|
'delete_presence_confirmations' => 0,
|
|
'delete_last_checked_at' => 0, 'delete_next_check_at' => 0,
|
|
'delete_attention_at' => null, 'lease_token' => null,
|
|
'lease_until' => 0, 'error_code' => '', 'verified_at' => $now,
|
|
'created_at' => $now, 'updated_at' => $now,
|
|
]);
|
|
Capsule::table('cloud_storage_operations')->insert([
|
|
'operation_id' => $operationId,
|
|
'request_idempotency_key' => 'rescue-fixture-'
|
|
. $storageSequence . '-' . $backendId,
|
|
'request_fingerprint' => hash(
|
|
'sha256', $ownerId . '|' . $backendId
|
|
),
|
|
'artifact_id' => $artifactId, 'replica_id' => $replicaId,
|
|
'source_backend_id' => $index === 0 ? null : 1,
|
|
'target_backend_id' => $backendId,
|
|
'kind' => $index === 0 ? 'publish' : 'copy',
|
|
'phase' => 'terminal', 'outcome' => 'succeeded',
|
|
'effect_resolution' => 'applied',
|
|
'effect_may_have_applied' => 0, 'safe_to_resubmit' => 0,
|
|
'attempts' => 1, 'lease_token' => null, 'lease_until' => 0,
|
|
'available_at' => 0, 'next_check_at' => 0, 'error_code' => '',
|
|
'result_json' => null, 'terminal_at' => $now,
|
|
'created_at' => $now, 'updated_at' => $now,
|
|
]);
|
|
$operations[] = $operationId;
|
|
}
|
|
Capsule::table('cloud_storage_artifacts')->where('id', $artifactId)
|
|
->update(['last_operation_id' => $operations[1]]);
|
|
Capsule::table('cloud_storage_publications')->insert([
|
|
'publication_key' => str_pad((string) $storageSequence, 32, 'c'),
|
|
'artifact_id' => $artifactId, 'backup_id' => null,
|
|
'template_import_intent_id' => null,
|
|
'rescue_template_id' => $ownerId, 'state' => 'completed',
|
|
'attempts' => 1, 'available_at' => 0, 'lease_token' => null,
|
|
'lease_until' => 0, 'last_error_code' => '',
|
|
'coverage_ready_at' => $now, 'finalized_at' => $now,
|
|
'created_at' => $now, 'updated_at' => $now,
|
|
]);
|
|
return $artifactId;
|
|
};
|
|
$storageArtifactId = $seedStorage(
|
|
$templateId, $generation, $bytes, $sha256
|
|
);
|
|
Capsule::table('cloud_rescue_template_artifacts')->insert([
|
|
'template_id' => $templateId,
|
|
'storage_artifact_id' => $storageArtifactId,
|
|
'storage_revision' => 0,
|
|
'object_key' => 'virt/rescue-isos/' . $templateId . '/'
|
|
. $generation . '/rescue.iso',
|
|
'object_version_id' => 'version-1',
|
|
'object_etag' => '"etag-1"', 'size_bytes' => $bytes,
|
|
'sha256' => $sha256, 'state' => 'validating',
|
|
'created_at' => $now, 'updated_at' => $now,
|
|
]);
|
|
$head = static fn (): Response => new Response(200, [
|
|
'Content-Length' => (string) $bytes,
|
|
'ETag' => '"etag-1"',
|
|
'x-amz-version-id' => 'version-1',
|
|
]);
|
|
$storageClient = new CloudObjectStorageS3Client(new Client([
|
|
'handler' => HandlerStack::create(new MockHandler([
|
|
$head(), $head(),
|
|
])),
|
|
]));
|
|
$actor = [
|
|
'is_super_admin' => true, 'can_manage' => true, 'user_id' => 1,
|
|
];
|
|
|
|
$validation = CloudRescueTemplateReplicaCoordinator::beginValidation(
|
|
$templateId,
|
|
31,
|
|
$actor,
|
|
[31, 32],
|
|
$storageClient
|
|
);
|
|
$assert(($validation['scheduled'] ?? false) === true,
|
|
'First-host validation was not scheduled.');
|
|
$validationJob = CloudJob::findOrFail((int) $validation['task_id']);
|
|
$assert((string) $validationJob->kind
|
|
=== 'rescue_catalog_validate_replica'
|
|
&& (int) CloudRescueTemplate::findOrFail($templateId)->active_job_id
|
|
=== (int) $validationJob->id,
|
|
'Validation job did not own the quarantine fence.');
|
|
$result = [
|
|
'id' => $templateId, 'name' => 'SystemRescue',
|
|
'size_bytes' => $bytes, 'sha256' => $sha256,
|
|
'revision' => 1, 'generation_token' => $generation,
|
|
'created_at' => gmdate(DATE_RFC3339),
|
|
];
|
|
CloudRescueTemplateReplicaCoordinator::finishJobLocked(
|
|
$validationJob,
|
|
'succeeded',
|
|
$result,
|
|
['rescue_template_id' => $templateId],
|
|
['effect_may_have_applied' => false, 'safe_to_resubmit' => false]
|
|
);
|
|
$assert((string) CloudRescueTemplateArtifact::findOrFail($templateId)->state
|
|
=== 'ready'
|
|
&& (string) CloudRescueTemplate::findOrFail($templateId)->state
|
|
=== 'distributing'
|
|
&& CloudRescueTemplate::findOrFail($templateId)->active_job_id === null,
|
|
'Successful first-host validation did not promote the object.');
|
|
|
|
$distribution = CloudRescueTemplateReplicaCoordinator::distribute(
|
|
$templateId,
|
|
$actor,
|
|
'validation_complete',
|
|
[31, 32],
|
|
$storageClient
|
|
);
|
|
$assert((int) $distribution['scheduled'] === 1
|
|
&& (int) $distribution['tasks'][0]['host_id'] === 32,
|
|
'Distribution did not skip the exact validation host.');
|
|
$replicaJob = CloudJob::findOrFail(
|
|
(int) $distribution['tasks'][0]['task_id']
|
|
);
|
|
CloudRescueTemplateReplicaCoordinator::finishJobLocked(
|
|
$replicaJob,
|
|
'succeeded',
|
|
$result,
|
|
['rescue_template_id' => $templateId],
|
|
['effect_may_have_applied' => false, 'safe_to_resubmit' => false]
|
|
);
|
|
$assert((string) CloudRescueTemplate::findOrFail($templateId)->state
|
|
=== 'active'
|
|
&& CloudRescueTemplateReplica::query()
|
|
->where('template_id', $templateId)
|
|
->where('state', 'active')->count() === 2
|
|
&& (int) CloudRescueTemplateReplica::query()
|
|
->where('template_id', $templateId)->where('host_id', 31)
|
|
->value('job_id') === (int) $validationJob->id
|
|
&& (int) CloudRescueTemplateReplica::query()
|
|
->where('template_id', $templateId)->where('host_id', 32)
|
|
->value('job_id') === (int) $replicaJob->id,
|
|
'All exact replicas did not activate the catalog.');
|
|
|
|
$publicReady = CloudRescueCatalogService::publicCatalogForHost(31);
|
|
$publicUnavailable = CloudRescueCatalogService::publicCatalogForHost(99);
|
|
$assert(($publicReady['status']['state'] ?? null) === 'ready'
|
|
&& ($publicReady['templates'][0]['id'] ?? null) === $templateId
|
|
&& array_keys($publicReady['templates'][0]) === ['id', 'name']
|
|
&& ($publicUnavailable['status']['state'] ?? null)
|
|
=== 'host_unavailable'
|
|
&& $publicUnavailable['templates'] === [],
|
|
'Public rescue catalog exposed a non-exact or unavailable host copy.');
|
|
|
|
$addHost(33);
|
|
$publicSyncing = CloudRescueCatalogService::publicCatalogForHost(33);
|
|
$assert(($publicSyncing['status']['state'] ?? null) === 'syncing'
|
|
&& $publicSyncing['templates'] === [],
|
|
'Public rescue catalog did not distinguish a pending host replica.');
|
|
$adminCatalog = CloudRescueCatalogService::adminCatalog();
|
|
$adminEntry = $adminCatalog['templates'][0] ?? null;
|
|
$missingHost = is_array($adminEntry)
|
|
? array_values(array_filter(
|
|
$adminEntry['replicas'],
|
|
static fn (array $replica): bool =>
|
|
(int) $replica['host_id'] === 33
|
|
))[0] ?? null
|
|
: null;
|
|
$assert(count($adminCatalog['hosts']) === 3
|
|
&& is_array($adminEntry)
|
|
&& (int) $adminEntry['replica_summary']['eligible'] === 3
|
|
&& (int) $adminEntry['replica_summary']['ready'] === 2
|
|
&& (int) $adminEntry['replica_summary']['attention'] === 1
|
|
&& is_array($missingHost)
|
|
&& $missingHost['state'] === 'missing'
|
|
&& $missingHost['repair_action'] === 'import'
|
|
&& $missingHost['repair_reason'] === 'missing',
|
|
'Administrator rescue catalog did not expose the exact host matrix.');
|
|
$remoteExact = $result;
|
|
$clientFactory = static fn (int $hostId): object => new class(
|
|
$hostId,
|
|
$remoteExact
|
|
) {
|
|
public function __construct(
|
|
private int $hostId,
|
|
private array $exact
|
|
) {
|
|
}
|
|
|
|
public function request(string $method, string $path, array $actor): array
|
|
{
|
|
if ($method !== 'GET' || $path !== '/v1/rescue-templates') {
|
|
throw new RuntimeException('Unexpected reconciliation request');
|
|
}
|
|
return ['templates' => $this->hostId === 33
|
|
? [] : [$this->exact]];
|
|
}
|
|
};
|
|
$repairs = [];
|
|
$reconciler = new CloudRescueTemplateReconciliationService(
|
|
$clientFactory,
|
|
static function (
|
|
string $id,
|
|
int $hostId
|
|
) use (&$repairs): array {
|
|
$repairs[] = [$id, $hostId];
|
|
return ['scheduled' => true];
|
|
}
|
|
);
|
|
$report = $reconciler->reconcile($actor, 3, 3, 0);
|
|
$assert($report['hosts_checked'] === 3
|
|
&& $report['missing'] === 1
|
|
&& $report['scheduled'] === 1
|
|
&& $repairs === [[$templateId, 33]]
|
|
&& (string) CloudRescueTemplate::findOrFail($templateId)->state
|
|
=== 'degraded',
|
|
'Periodic reconciliation did not project and repair a missing host.');
|
|
|
|
$retirement = CloudRescueTemplateRetirementService::retire(
|
|
$templateId,
|
|
1,
|
|
$generation,
|
|
$actor
|
|
);
|
|
$assert((string) CloudRescueTemplate::findOrFail($templateId)->state
|
|
=== 'retiring'
|
|
&& (int) $retirement['cleanup_debt'] === 2
|
|
&& count($retirement['tasks']) === 2
|
|
&& (string) CloudRescueTemplateReplica::query()
|
|
->where('template_id', $templateId)->where('host_id', 33)
|
|
->value('state') === 'deleted',
|
|
'Logical retirement did not stop publication or resolve known absence.');
|
|
$deleteResult = [
|
|
'resource_type' => 'rescue_template',
|
|
'id' => $templateId,
|
|
'deleted' => true,
|
|
'managed_image_absent' => true,
|
|
'managed_metadata_absent' => true,
|
|
'managed_artifacts_absent' => true,
|
|
];
|
|
foreach ($retirement['tasks'] as $task) {
|
|
$job = CloudJob::findOrFail((int) $task['task_id']);
|
|
CloudRescueTemplateRetirementService::finishJobLocked(
|
|
$job,
|
|
'succeeded',
|
|
$deleteResult,
|
|
['rescue_template_id' => $templateId],
|
|
[
|
|
'status' => 'succeeded',
|
|
'retryable' => false,
|
|
'result' => $deleteResult,
|
|
'effect_may_have_applied' => false,
|
|
'safe_to_resubmit' => false,
|
|
]
|
|
);
|
|
}
|
|
$assert((string) CloudRescueTemplateArtifact::findOrFail($templateId)->state
|
|
=== 'delete_pending'
|
|
&& CloudRescueTemplateReplica::query()
|
|
->where('template_id', $templateId)
|
|
->where('state', '<>', 'deleted')->count() === 0,
|
|
'Canonical object cleanup was armed before all replicas proved absent.');
|
|
|
|
$deleteClient = new CloudObjectStorageS3Client(new Client([
|
|
'handler' => HandlerStack::create(new MockHandler([
|
|
$head(), new Response(204),
|
|
$head(), new Response(204),
|
|
])),
|
|
]));
|
|
$cleanup = CloudRescueTemplateRetirementService::processDue(
|
|
$actor,
|
|
8,
|
|
$deleteClient
|
|
);
|
|
$assert((int) $cleanup['completed'] === 1
|
|
&& (string) CloudRescueTemplate::findOrFail($templateId)->state
|
|
=== 'retired'
|
|
&& (string) CloudRescueTemplateArtifact::findOrFail($templateId)->state
|
|
=== 'deleted'
|
|
&& CloudRescueTemplateReplica::query()
|
|
->where('template_id', $templateId)->count() === 3,
|
|
'Exact object deletion did not retain a settled retirement tombstone.');
|
|
|
|
$attentionId = str_repeat('f', 32);
|
|
$attentionGeneration = str_repeat('1', 32);
|
|
Capsule::table('cloud_rescue_templates')->insert([
|
|
'id' => $attentionId,
|
|
'name' => 'Attention delete rescue',
|
|
'size_bytes' => $bytes,
|
|
'sha256' => $sha256,
|
|
'revision' => 1,
|
|
'generation_token' => $attentionGeneration,
|
|
'state' => 'active',
|
|
'active_job_id' => null,
|
|
'created_at' => $now,
|
|
'updated_at' => $now,
|
|
]);
|
|
$attentionStorageArtifactId = $seedStorage(
|
|
$attentionId,
|
|
$attentionGeneration,
|
|
$bytes,
|
|
$sha256
|
|
);
|
|
Capsule::table('cloud_rescue_template_artifacts')->insert([
|
|
'template_id' => $attentionId,
|
|
'storage_artifact_id' => $attentionStorageArtifactId,
|
|
'storage_revision' => 0,
|
|
'object_key' => 'virt/rescue-isos/' . $attentionId . '/'
|
|
. $attentionGeneration . '/rescue.iso',
|
|
'object_version_id' => 'version-1',
|
|
'object_etag' => '"etag-1"',
|
|
'size_bytes' => $bytes,
|
|
'sha256' => $sha256,
|
|
'state' => 'ready',
|
|
'created_at' => $now,
|
|
'updated_at' => $now,
|
|
]);
|
|
$attentionRetirement = CloudRescueTemplateRetirementService::retire(
|
|
$attentionId,
|
|
1,
|
|
$attentionGeneration,
|
|
$actor
|
|
);
|
|
$assert($attentionRetirement['tasks'] === []
|
|
&& (string) CloudRescueTemplateArtifact::findOrFail(
|
|
$attentionId
|
|
)->state === 'delete_pending',
|
|
'Host-free rescue retirement did not arm provider deletion.');
|
|
$attentionReplicas = Capsule::table('cloud_storage_replicas')
|
|
->where('artifact_id', $attentionStorageArtifactId)
|
|
->orderBy('id')->get();
|
|
$attentionDeleteOperation = str_repeat('9', 32);
|
|
Capsule::table('cloud_storage_operations')->insert([
|
|
'operation_id' => $attentionDeleteOperation,
|
|
'request_idempotency_key' => 'rescue-attention-delete-1',
|
|
'request_fingerprint' => hash(
|
|
'sha256',
|
|
$attentionId . '|delete|1'
|
|
),
|
|
'artifact_id' => $attentionStorageArtifactId,
|
|
'replica_id' => (int) $attentionReplicas[0]->id,
|
|
'source_backend_id' => null,
|
|
'target_backend_id' => (int) $attentionReplicas[0]->backend_id,
|
|
'kind' => 'delete',
|
|
'phase' => 'terminal',
|
|
'outcome' => 'failed',
|
|
'effect_resolution' => 'unknown',
|
|
'effect_may_have_applied' => 1,
|
|
'safe_to_resubmit' => 0,
|
|
'attempts' => 6,
|
|
'lease_token' => null,
|
|
'lease_until' => 0,
|
|
'available_at' => 0,
|
|
'next_check_at' => 0,
|
|
'error_code' => 'object_delete_reconcile_exhausted',
|
|
'result_json' => null,
|
|
'terminal_at' => $now,
|
|
'created_at' => $now,
|
|
'updated_at' => $now,
|
|
]);
|
|
Capsule::table('cloud_storage_replicas')
|
|
->where('id', (int) $attentionReplicas[0]->id)->update([
|
|
'state' => 'delete_attention',
|
|
'delete_operation_id' => $attentionDeleteOperation,
|
|
'delete_generation' => 1,
|
|
'delete_cycle_attempts' => 6,
|
|
'delete_presence_confirmations' => 0,
|
|
'delete_next_check_at' => 0,
|
|
'delete_attention_at' => $now,
|
|
'error_code' => 'object_delete_reconcile_exhausted',
|
|
'updated_at' => $now,
|
|
]);
|
|
Capsule::table('cloud_storage_replicas')
|
|
->where('id', (int) $attentionReplicas[1]->id)->update([
|
|
'state' => 'deleted',
|
|
'delete_next_check_at' => 0,
|
|
'delete_attention_at' => null,
|
|
'error_code' => '',
|
|
'updated_at' => $now,
|
|
]);
|
|
Capsule::table('cloud_storage_artifacts')
|
|
->where('id', $attentionStorageArtifactId)->update([
|
|
'state' => 'delete_attention',
|
|
'last_operation_id' => $attentionDeleteOperation,
|
|
'updated_at' => $now,
|
|
]);
|
|
Capsule::table('cloud_rescue_template_artifacts')
|
|
->where('template_id', $attentionId)->update([
|
|
'state' => 'delete_failed',
|
|
'error_code' => 'rescue_storage_delete_attention',
|
|
'delete_next_check_at' => 0,
|
|
'delete_attention_at' => $now,
|
|
'updated_at' => $now,
|
|
]);
|
|
|
|
$attentionCatalogEntry = collect(
|
|
CloudRescueCatalogService::adminCatalog()['templates']
|
|
)->firstWhere('id', $attentionId);
|
|
$assert(is_array($attentionCatalogEntry)
|
|
&& ($attentionCatalogEntry['artifact']['delete_attention_at'] ?? null)
|
|
!== null
|
|
&& in_array(
|
|
'retry_delete',
|
|
$attentionCatalogEntry['artifact']['recovery_actions'] ?? [],
|
|
true
|
|
),
|
|
'Administrator rescue DTO did not expose the executable recovery action.');
|
|
|
|
$retry = CloudRescueTemplateRetirementService::retryObjectDeletion(
|
|
$attentionId,
|
|
1,
|
|
$attentionGeneration,
|
|
$actor
|
|
);
|
|
$attentionOperation = Capsule::table('cloud_storage_operations')
|
|
->where('operation_id', $attentionDeleteOperation)->first();
|
|
$attentionReplica = Capsule::table('cloud_storage_replicas')
|
|
->where('id', (int) $attentionReplicas[0]->id)->first();
|
|
$attentionArtifact = CloudRescueTemplateArtifact::findOrFail(
|
|
$attentionId
|
|
);
|
|
$assert(($retry['retry_scheduled'] ?? false) === true
|
|
&& (string) $attentionArtifact->state === 'delete_pending'
|
|
&& $attentionArtifact->delete_attention_at === null
|
|
&& (string) $attentionOperation->operation_id
|
|
=== $attentionDeleteOperation
|
|
&& (string) $attentionOperation->phase === 'reconciling'
|
|
&& (string) $attentionOperation->effect_resolution === 'unknown'
|
|
&& (string) $attentionReplica->delete_operation_id
|
|
=== $attentionDeleteOperation
|
|
&& (string) $attentionReplica->state === 'delete_uncertain'
|
|
&& Capsule::table('cloud_storage_operations')
|
|
->where('artifact_id', $attentionStorageArtifactId)
|
|
->where('kind', 'delete')->count() === 1,
|
|
'Manual rescue cleanup recovery did not resume the exact delete intent.');
|
|
$retryRejected = false;
|
|
try {
|
|
CloudRescueTemplateRetirementService::retryObjectDeletion(
|
|
$attentionId,
|
|
1,
|
|
$attentionGeneration,
|
|
$actor
|
|
);
|
|
} catch (\App\Services\Cloud\CloudError $error) {
|
|
$retryRejected = $error->httpStatus === 409;
|
|
}
|
|
$assert($retryRejected,
|
|
'Rescue cleanup recovery accepted an artifact outside attention.');
|
|
|
|
$unknownId = str_repeat('d', 32);
|
|
$unknownGeneration = str_repeat('e', 32);
|
|
Capsule::table('cloud_rescue_templates')->insert([
|
|
'id' => $unknownId,
|
|
'name' => 'Unknown delete rescue',
|
|
'size_bytes' => $bytes,
|
|
'sha256' => $sha256,
|
|
'revision' => 1,
|
|
'generation_token' => $unknownGeneration,
|
|
'state' => 'active',
|
|
'active_job_id' => null,
|
|
'created_at' => $now,
|
|
'updated_at' => $now,
|
|
]);
|
|
$unknownStorageArtifactId = $seedStorage(
|
|
$unknownId,
|
|
$unknownGeneration,
|
|
$bytes,
|
|
$sha256
|
|
);
|
|
Capsule::table('cloud_rescue_template_artifacts')->insert([
|
|
'template_id' => $unknownId,
|
|
'storage_artifact_id' => $unknownStorageArtifactId,
|
|
'storage_revision' => 0,
|
|
'object_key' => 'virt/rescue-isos/' . $unknownId . '/'
|
|
. $unknownGeneration . '/rescue.iso',
|
|
'object_version_id' => 'version-1',
|
|
'object_etag' => '"etag-1"',
|
|
'size_bytes' => $bytes,
|
|
'sha256' => $sha256,
|
|
'state' => 'ready',
|
|
'created_at' => $now,
|
|
'updated_at' => $now,
|
|
]);
|
|
Capsule::table('cloud_rescue_template_replicas')->insert([
|
|
'host_id' => 31,
|
|
'template_id' => $unknownId,
|
|
'revision' => 1,
|
|
'generation_token' => $unknownGeneration,
|
|
'state' => 'active',
|
|
'size_bytes' => $bytes,
|
|
'sha256' => $sha256,
|
|
'operation_id' => '',
|
|
'job_id' => null,
|
|
'error_code' => '',
|
|
'observed_state' => 'exact',
|
|
'observed_revision' => 1,
|
|
'observed_generation_token' => $unknownGeneration,
|
|
'observed_size_bytes' => $bytes,
|
|
'observed_sha256' => $sha256,
|
|
'last_verified_at' => $now,
|
|
'cleanup_resolved_at' => null,
|
|
'created_at' => $now,
|
|
'updated_at' => $now,
|
|
]);
|
|
$unknownRetirement = CloudRescueTemplateRetirementService::retire(
|
|
$unknownId,
|
|
1,
|
|
$unknownGeneration,
|
|
$actor
|
|
);
|
|
$unknownJob = CloudJob::findOrFail(
|
|
(int) $unknownRetirement['tasks'][0]['task_id']
|
|
);
|
|
CloudRescueTemplateRetirementService::finishJobLocked(
|
|
$unknownJob,
|
|
'failed',
|
|
null,
|
|
['rescue_template_id' => $unknownId],
|
|
[
|
|
'effect_may_have_applied' => true,
|
|
'safe_to_resubmit' => false,
|
|
'cause_code' => 'agent_response_lost',
|
|
]
|
|
);
|
|
$debt = CloudRescueTemplateRetirementService::cleanupState($unknownId);
|
|
$assert(($debt['replica_states']['delete_unknown'] ?? 0) === 1
|
|
&& (string) CloudRescueTemplateArtifact::findOrFail($unknownId)->state
|
|
=== 'ready',
|
|
'Unknown host deletion did not retain cleanup debt and the object.');
|
|
|
|
$absenceReconciler = new CloudRescueTemplateReconciliationService(
|
|
static fn (int $hostId): object => new class {
|
|
public function request(
|
|
string $method,
|
|
string $path,
|
|
array $actor
|
|
): array {
|
|
return ['templates' => []];
|
|
}
|
|
},
|
|
static function (): array {
|
|
throw new RuntimeException(
|
|
'Retiring rescue ISO attempted an import repair.'
|
|
);
|
|
}
|
|
);
|
|
$absenceReport = $absenceReconciler->reconcile($actor, 3, 3, 0);
|
|
$assert($absenceReport['scheduled'] === 0
|
|
&& (string) CloudRescueTemplateReplica::query()
|
|
->where('template_id', $unknownId)->where('host_id', 31)
|
|
->value('state') === 'deleted',
|
|
'Trusted host absence did not resolve unknown deletion debt.');
|
|
$unknownDeleteClient = new CloudObjectStorageS3Client(new Client([
|
|
'handler' => HandlerStack::create(new MockHandler([
|
|
$head(), new Response(204),
|
|
$head(), new Response(204),
|
|
])),
|
|
]));
|
|
$unknownCleanup = CloudRescueTemplateRetirementService::processDue(
|
|
$actor,
|
|
8,
|
|
$unknownDeleteClient
|
|
);
|
|
$assert((int) $unknownCleanup['completed'] === 1
|
|
&& (string) CloudRescueTemplate::findOrFail($unknownId)->state
|
|
=== 'retired',
|
|
'Reconciled host absence did not release exact object cleanup.');
|
|
|
|
echo "Rescue replica validation, distribution and reconciliation passed.\n";
|
|
}
|