You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
virt/web/tests/CloudTemplateReconciliation...

364 lines
12 KiB
PHP

<?php
require dirname(__DIR__) . '/vendor/autoload.php';
use App\Models\CloudJob;
use App\Models\CloudTemplate;
use App\Models\CloudTemplateReplica;
use App\Services\Cloud\CloudError;
use App\Services\Cloud\CloudTemplateReplicaCoordinator;
use App\Services\Cloud\CloudTemplateReconciliationService;
use Illuminate\Database\Capsule\Manager as Capsule;
use Illuminate\Database\Schema\Blueprint;
$assert = static function (bool $condition, string $message): void {
if (!$condition) {
throw new RuntimeException($message);
}
};
$capsule = new Capsule();
$capsule->addConnection([
'driver' => 'sqlite', 'database' => ':memory:',
], 'default');
$capsule->setAsGlobal();
$capsule->bootEloquent();
$schema = $capsule->schema('default');
$schema->create('cloud_hosts', static function (Blueprint $table): void {
$table->integer('id')->primary();
$table->boolean('enabled');
$table->string('default_network');
});
$schema->create('cloud_host_agents', static function (Blueprint $table): void {
$table->integer('host_id')->primary();
$table->boolean('enabled');
$table->string('last_test_status');
$table->integer('tested_revision');
$table->integer('revision');
});
$schema->create('cloud_host_storage_configurations', static function (
Blueprint $table
): void {
$table->integer('host_id')->primary();
$table->string('state');
});
$schema->create('cloud_templates', static function (Blueprint $table): void {
$table->string('id')->primary();
$table->string('catalog_key');
$table->string('version_label');
$table->boolean('is_default');
$table->string('name');
$table->string('os_family');
$table->string('os_version');
$table->integer('image_bytes');
$table->string('sha256');
$table->text('startup_script');
$table->integer('revision');
$table->string('generation_token');
$table->string('state');
$table->integer('active_job_id')->nullable();
$table->integer('created_at');
$table->integer('updated_at');
});
$schema->create('cloud_template_replicas', static function (
Blueprint $table
): void {
$table->integer('host_id');
$table->string('template_id');
$table->integer('revision');
$table->string('generation_token');
$table->string('state');
$table->integer('image_bytes');
$table->string('sha256');
$table->string('operation_id');
$table->integer('job_id')->nullable();
$table->string('error_code');
$table->string('observed_state')->default('never');
$table->integer('observed_revision')->default(0);
$table->string('observed_generation_token')->default('');
$table->integer('observed_image_bytes')->default(0);
$table->string('observed_sha256')->default('');
$table->integer('last_verified_at')->default(0);
$table->integer('created_at');
$table->integer('updated_at');
$table->primary(['host_id', 'template_id']);
});
Capsule::table('cloud_hosts')->insert([
'id' => 51, 'enabled' => 1, 'default_network' => 'bridge:br0',
]);
Capsule::table('cloud_host_agents')->insert([
'host_id' => 51, 'enabled' => 1, 'last_test_status' => 'healthy',
'tested_revision' => 3, 'revision' => 3,
]);
Capsule::table('cloud_host_storage_configurations')->insert([
'host_id' => 51, 'state' => 'applied',
]);
$now = time();
$generation = str_repeat('a', 32);
$sha = str_repeat('b', 64);
$template = static function (string $id) use (
$now,
$generation,
$sha
): array {
return [
'id' => $id,
'catalog_key' => $id,
'version_label' => 'v1',
'is_default' => 1,
'name' => 'Template ' . $id,
'os_family' => 'Debian',
'os_version' => '13',
'image_bytes' => 4096,
'sha256' => $sha,
'startup_script' => '',
'revision' => 2,
'generation_token' => $generation,
'state' => 'active',
'active_job_id' => null,
'created_at' => $now,
'updated_at' => $now,
];
};
$templateIds = ['divergent-template', 'exact-template',
'missing-template', 'stale-template'];
foreach ($templateIds as $templateId) {
Capsule::table('cloud_templates')->insert($template($templateId));
}
foreach (['divergent-template', 'exact-template', 'stale-template']
as $templateId) {
Capsule::table('cloud_template_replicas')->insert([
'host_id' => 51,
'template_id' => $templateId,
'revision' => 2,
'generation_token' => $generation,
'state' => 'active',
'image_bytes' => 4096,
'sha256' => $sha,
'operation_id' => str_repeat(substr($templateId, 0, 1), 32),
'job_id' => null,
'error_code' => '',
'created_at' => $now,
'updated_at' => $now,
]);
}
$remote = static function (
string $id,
int $revision,
string $name,
string $rowSha
) use ($generation): array {
return [
'id' => $id,
'name' => $name,
'os_family' => 'Debian',
'version' => '13',
'architecture' => 'x86_64',
'size_bytes' => 4096,
'virtual_size_bytes' => 20 * 1024 * 1024,
'sha256' => $rowSha,
'revision' => $revision,
'generation_token' => $generation,
'created_at' => '2026-10-30T00:00:00Z',
];
};
$payload = ['templates' => [
$remote('divergent-template', 2, 'Template divergent-template',
str_repeat('c', 64)),
$remote('exact-template', 2, 'Template exact-template', $sha),
$remote('orphan-template', 1, 'Orphan template', $sha),
$remote('stale-template', 1, 'Old stale name', $sha),
]];
$repairs = [];
$clientFactory = static fn (int $hostId): object => new class($payload) {
public function __construct(private array $payload)
{
}
public function request(
string $method,
string $target,
array $actor
): array {
if ($method !== 'GET' || $target !== '/v1/templates') {
throw new RuntimeException('Unexpected Agent request.');
}
return $this->payload;
}
};
$repairer = static function (
string $action,
string $templateId,
int $hostId,
array $actor
) use (&$repairs): array {
$repairs[] = compact('action', 'templateId', 'hostId');
return ['scheduled' => true];
};
$actor = [
'user_id' => 1,
'is_admin' => true,
'is_super_admin' => true,
'can_manage' => true,
];
$report = (new CloudTemplateReconciliationService(
$clientFactory,
$repairer
))->reconcile($actor, 4, 8);
$assert($report['hosts_checked'] === 1
&& $report['templates_checked'] === 4
&& $report['exact'] === 1
&& $report['missing'] === 1
&& $report['stale'] === 1
&& $report['divergent'] === 1
&& $report['orphans'] === 1
&& $report['scheduled'] === 2
&& $report['errors'] === 0,
'Periodic reconciliation did not classify the bounded Agent catalog.');
$assert($repairs === [
['action' => 'import', 'templateId' => 'missing-template', 'hostId' => 51],
['action' => 'metadata', 'templateId' => 'stale-template', 'hostId' => 51],
], 'Reconciliation queued an unsafe or incomplete repair set.');
$exact = Capsule::table('cloud_template_replicas')
->where('template_id', 'exact-template')->first();
$stale = Capsule::table('cloud_template_replicas')
->where('template_id', 'stale-template')->first();
$divergent = Capsule::table('cloud_template_replicas')
->where('template_id', 'divergent-template')->first();
$assert((string) $exact->state === 'active'
&& (string) $exact->observed_state === 'exact'
&& (int) $exact->observed_revision === 2
&& (int) $exact->last_verified_at > 0,
'Exact Agent observation changed the ready replica.');
$assert((string) $stale->state === 'stale'
&& (string) $stale->observed_state === 'stale'
&& (int) $stale->observed_revision === 1
&& (string) $stale->error_code === 'template_observed_stale',
'Stale metadata observation was not fenced before repair.');
$assert((string) $divergent->state === 'failed'
&& (string) $divergent->observed_state === 'divergent'
&& (string) $divergent->error_code === 'template_observed_divergent',
'Divergent image content remained eligible or was auto-repaired.');
$assert((string) Capsule::table('cloud_templates')
->where('id', 'exact-template')->value('state') === 'active'
&& (string) Capsule::table('cloud_templates')
->where('id', 'missing-template')->value('state') === 'error'
&& (string) Capsule::table('cloud_templates')
->where('id', 'stale-template')->value('state') === 'error'
&& (string) Capsule::table('cloud_templates')
->where('id', 'divergent-template')->value('state') === 'error',
'Observed host drift did not update template availability.');
// The persisted replica projection may already describe the desired revision
// while the host still exposes older metadata. Fresh host evidence, not the
// replica projection, is the authority for deciding whether PATCH is safe.
$master = new CloudTemplate();
$master->setRawAttributes([
'revision' => 2,
'generation_token' => $generation,
'image_bytes' => 4096,
'sha256' => $sha,
]);
$metadataReplica = new CloudTemplateReplica();
$metadataReplica->setRawAttributes([
'revision' => 2,
'observed_state' => 'stale',
'observed_revision' => 1,
'observed_generation_token' => $generation,
'observed_image_bytes' => 4096,
'observed_sha256' => $sha,
'last_verified_at' => time(),
]);
$coordinatorReflection = new ReflectionClass(
CloudTemplateReplicaCoordinator::class
);
$staleEvidence = $coordinatorReflection->getMethod(
'assertObservedMetadataStale'
);
$staleEvidence->invoke(null, $master, $metadataReplica);
$metadataReplica->observed_sha256 = str_repeat('d', 64);
$rejectedDivergence = false;
try {
$staleEvidence->invoke(null, $master, $metadataReplica);
} catch (CloudError $error) {
$rejectedDivergence = $error->httpStatus === 409;
}
$assert($rejectedDivergence,
'Metadata repair accepted divergent content evidence.');
// Once a fresh read proves the host copy is absent, an older replica ledger
// may be replaced by the current immutable master only when its prior Agent
// generation is terminal and explicitly safe to resubmit.
$missingReplica = new CloudTemplateReplica();
$missingReplica->setRawAttributes([
'host_id' => 51,
'state' => 'failed',
'error_code' => 'template_observed_missing',
'observed_state' => 'missing',
'last_verified_at' => time(),
'revision' => 1,
'generation_token' => $generation,
'image_bytes' => 4096,
'sha256' => $sha,
'operation_id' => str_repeat('e', 32),
]);
$safeJob = new CloudJob();
$safeJob->setRawAttributes([
'kind' => 'template_metadata_replica',
'host_id' => 51,
'execution_mode' => 'agent',
'phase' => 'terminal',
'outcome' => 'failed',
'status' => 'failed',
'effect_resolution' => 'not_applied',
'effect_may_have_applied' => 0,
'safe_to_resubmit' => 1,
'finalization_incident' => 0,
'operation_id' => str_repeat('e', 32),
'request_fingerprint' => str_repeat('f', 64),
'request_idempotency_key' => str_repeat('g', 32),
'dispatch_attempts' => 0,
'attempts' => 0,
'dispatch_deadline_at' => time() + 300,
'dispatch_started_at' => null,
'agent_accepted_at' => null,
'dispatched_at' => null,
'receipt_deadline_at' => null,
'completion_event_id' => null,
]);
$missingEvidence = $coordinatorReflection->getMethod(
'observedMissingIsRetryable'
);
$assert($missingEvidence->invoke(
null,
$master,
$missingReplica,
$safeJob
) === true, 'Proven missing older replica could not be safely re-imported.');
$badClient = static fn (int $hostId): object => new class {
public function request(string $method, string $target, array $actor): array
{
return ['templates' => [['id' => '../invalid']]];
}
};
$before = Capsule::table('cloud_template_replicas')
->where('template_id', 'exact-template')->value('last_verified_at');
$badReport = (new CloudTemplateReconciliationService(
$badClient,
$repairer
))->reconcile($actor, 1, 1);
$assert($badReport['errors'] === 1
&& $badReport['hosts_checked'] === 0
&& Capsule::table('cloud_template_replicas')
->where('template_id', 'exact-template')
->value('last_verified_at') === $before,
'Malformed Agent catalog partially changed the replica projection.');
echo "Periodic template reconciliation behavior passed.\n";