feat(template): backfill ready hosts
parent
d166b56aa4
commit
aa3898a07c
@ -0,0 +1,149 @@
|
||||
<?php
|
||||
|
||||
namespace App\Services\Cloud;
|
||||
|
||||
use App\Models\CloudTemplate;
|
||||
use App\Models\CloudTemplateReplica;
|
||||
|
||||
/**
|
||||
* Fill the current catalog generations onto a host after it becomes ready.
|
||||
*
|
||||
* Host readiness is the admission boundary. This coordinator is deliberately
|
||||
* best-effort across templates: one busy or temporarily unavailable artifact
|
||||
* must not prevent another independent catalog generation from being queued.
|
||||
*/
|
||||
final class CloudTemplateHostBackfillService
|
||||
{
|
||||
private const TEMPLATE_STATES = ['active', 'degraded', 'error'];
|
||||
|
||||
/**
|
||||
* Queue every missing default generation for one ready host.
|
||||
*
|
||||
* @return array{host_id:int,ready:bool,considered:int,already_ready:int,
|
||||
* scheduled:int,deferred:int}
|
||||
*/
|
||||
public static function scheduleReadyHost(
|
||||
int $hostId,
|
||||
array $actor,
|
||||
string $reason = 'host_ready'
|
||||
): array {
|
||||
CloudAccess::manage($actor);
|
||||
self::assertInput($hostId, $reason);
|
||||
$plan = self::planReadyHost($hostId);
|
||||
$report = [
|
||||
'host_id' => $hostId,
|
||||
'ready' => $plan['ready'],
|
||||
'considered' => $plan['considered'],
|
||||
'already_ready' => count($plan['already_ready']),
|
||||
'scheduled' => 0,
|
||||
'deferred' => 0,
|
||||
];
|
||||
if (!$plan['ready']) {
|
||||
return $report;
|
||||
}
|
||||
foreach ($plan['pending'] as $templateId) {
|
||||
try {
|
||||
$result = CloudTemplateReplicaCoordinator::syncHost(
|
||||
$templateId,
|
||||
$hostId,
|
||||
$actor,
|
||||
$reason
|
||||
);
|
||||
if (($result['scheduled'] ?? false) === true) {
|
||||
$report['scheduled']++;
|
||||
} else {
|
||||
$report['already_ready']++;
|
||||
}
|
||||
} catch (\Throwable $error) {
|
||||
// The original host-readiness action has already committed.
|
||||
// Keep this generation eligible for the periodic reconciler
|
||||
// instead of converting an independent backfill into a false
|
||||
// Agent/storage failure.
|
||||
$report['deferred']++;
|
||||
error_log(sprintf(
|
||||
'cloud template backfill host=%d template=%s deferred: %s',
|
||||
$hostId,
|
||||
$templateId,
|
||||
get_class($error)
|
||||
));
|
||||
}
|
||||
}
|
||||
return $report;
|
||||
}
|
||||
|
||||
/**
|
||||
* Read-only plan used by event hooks and future reconciliation passes.
|
||||
* Only the default generation of each catalog entry is auto-filled; older
|
||||
* immutable versions remain available for explicit operator repair.
|
||||
*
|
||||
* @return array{host_id:int,ready:bool,considered:int,
|
||||
* already_ready:list<string>,pending:list<string>}
|
||||
*/
|
||||
public static function planReadyHost(int $hostId): array
|
||||
{
|
||||
if ($hostId < 1) {
|
||||
throw new CloudError('云镜像补齐宿主机参数无效', 422);
|
||||
}
|
||||
$ready = in_array(
|
||||
$hostId,
|
||||
CloudAgentSettingsService::enabledHostIds(),
|
||||
true
|
||||
);
|
||||
if (!$ready) {
|
||||
return [
|
||||
'host_id' => $hostId,
|
||||
'ready' => false,
|
||||
'considered' => 0,
|
||||
'already_ready' => [],
|
||||
'pending' => [],
|
||||
];
|
||||
}
|
||||
$templates = CloudTemplate::query()
|
||||
->where('is_default', 1)
|
||||
->whereIn('state', self::TEMPLATE_STATES)
|
||||
->orderBy('catalog_key')
|
||||
->orderBy('version_label')
|
||||
->orderBy('id')
|
||||
->get();
|
||||
$templateIds = $templates->map(
|
||||
static fn (CloudTemplate $template): string =>
|
||||
(string) $template->id
|
||||
)->all();
|
||||
$replicas = $templateIds === []
|
||||
? collect()
|
||||
: CloudTemplateReplica::query()
|
||||
->where('host_id', $hostId)
|
||||
->whereIn('template_id', $templateIds)
|
||||
->get()->keyBy('template_id');
|
||||
$alreadyReady = [];
|
||||
$pending = [];
|
||||
foreach ($templates as $template) {
|
||||
$templateId = (string) $template->id;
|
||||
$replica = $replicas->get($templateId);
|
||||
if ($replica instanceof CloudTemplateReplica
|
||||
&& CloudTemplateReplicaCoordinator::isExactReplica(
|
||||
$template,
|
||||
$replica
|
||||
)) {
|
||||
$alreadyReady[] = $templateId;
|
||||
continue;
|
||||
}
|
||||
$pending[] = $templateId;
|
||||
}
|
||||
return [
|
||||
'host_id' => $hostId,
|
||||
'ready' => true,
|
||||
'considered' => count($templateIds),
|
||||
'already_ready' => $alreadyReady,
|
||||
'pending' => $pending,
|
||||
];
|
||||
}
|
||||
|
||||
private static function assertInput(int $hostId, string $reason): void
|
||||
{
|
||||
if ($hostId < 1
|
||||
|| preg_match('/^[a-z][a-z0-9_]{1,47}$/D', $reason) !== 1) {
|
||||
throw new CloudError('云镜像补齐参数无效', 422);
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,45 @@
|
||||
<?php
|
||||
|
||||
$root = dirname(__DIR__);
|
||||
$worker = file_get_contents(
|
||||
$root . '/app/Services/Cloud/CloudWorkerService.php'
|
||||
);
|
||||
$agent = file_get_contents(
|
||||
$root . '/app/Services/Cloud/CloudAgentSettingsService.php'
|
||||
);
|
||||
$service = file_get_contents(
|
||||
$root . '/app/Services/Cloud/CloudTemplateHostBackfillService.php'
|
||||
);
|
||||
$assert = static function (bool $condition, string $message): void {
|
||||
if (!$condition) {
|
||||
throw new RuntimeException($message);
|
||||
}
|
||||
};
|
||||
|
||||
$transactionEnd = strpos($worker, '}, 3);', strpos(
|
||||
$worker,
|
||||
'private function finish('
|
||||
));
|
||||
$storageHook = strpos(
|
||||
$worker,
|
||||
'$this->scheduleTemplateBackfillAfterStorage(',
|
||||
$transactionEnd === false ? 0 : $transactionEnd
|
||||
);
|
||||
$assert($transactionEnd !== false && $storageHook !== false
|
||||
&& $storageHook > $transactionEnd,
|
||||
'Storage completion schedules template backfill before its transaction commits.');
|
||||
$assert(str_contains($agent,
|
||||
"scheduleTemplateBackfill(\$hostId, \$actor, 'agent_ready')"),
|
||||
'A successful Agent readiness test does not trigger host backfill.');
|
||||
$assert(str_contains($service, "->where('is_default', 1)")
|
||||
&& str_contains($service,
|
||||
"private const TEMPLATE_STATES = ['active', 'degraded', 'error']")
|
||||
&& str_contains($service, 'CloudAgentSettingsService::enabledHostIds()')
|
||||
&& str_contains($service,
|
||||
'CloudTemplateReplicaCoordinator::isExactReplica('),
|
||||
'Host backfill is not constrained to ready hosts and current generations.');
|
||||
$assert(str_contains($service, 'foreach ($plan[\'pending\'] as $templateId)')
|
||||
&& str_contains($service, '} catch (\\Throwable $error) {'),
|
||||
'One template failure can abort the remaining ready-host backfill plan.');
|
||||
|
||||
echo "Template ready-host backfill contracts passed.\n";
|
||||
@ -0,0 +1,177 @@
|
||||
<?php
|
||||
|
||||
require dirname(__DIR__) . '/vendor/autoload.php';
|
||||
|
||||
use App\Models\CloudTemplate;
|
||||
use App\Services\Cloud\CloudTemplateHostBackfillService;
|
||||
use Illuminate\Database\Capsule\Manager as Capsule;
|
||||
use Illuminate\Database\Schema\Blueprint;
|
||||
|
||||
$assert = static function (bool $condition, string $message): void {
|
||||
if (!$condition) {
|
||||
throw new RuntimeException($message);
|
||||
}
|
||||
};
|
||||
$capsule = new Capsule();
|
||||
$capsule->addConnection([
|
||||
'driver' => 'sqlite', 'database' => ':memory:',
|
||||
], 'default');
|
||||
$capsule->setAsGlobal();
|
||||
$capsule->bootEloquent();
|
||||
$schema = $capsule->schema('default');
|
||||
$schema->create('cloud_hosts', static function (Blueprint $table): void {
|
||||
$table->integer('id')->primary();
|
||||
$table->boolean('enabled');
|
||||
$table->string('default_network');
|
||||
});
|
||||
$schema->create('cloud_host_agents', static function (Blueprint $table): void {
|
||||
$table->integer('host_id')->primary();
|
||||
$table->boolean('enabled');
|
||||
$table->string('last_test_status');
|
||||
$table->integer('tested_revision');
|
||||
$table->integer('revision');
|
||||
});
|
||||
$schema->create('cloud_host_storage_configurations',
|
||||
static function (Blueprint $table): void {
|
||||
$table->integer('host_id')->primary();
|
||||
$table->string('state');
|
||||
});
|
||||
$schema->create('cloud_templates', static function (Blueprint $table): void {
|
||||
$table->string('id')->primary();
|
||||
$table->string('catalog_key');
|
||||
$table->string('version_label');
|
||||
$table->boolean('is_default');
|
||||
$table->string('name');
|
||||
$table->string('os_family');
|
||||
$table->string('os_version');
|
||||
$table->string('source_type');
|
||||
$table->text('sealed_source_url');
|
||||
$table->integer('image_bytes');
|
||||
$table->string('sha256');
|
||||
$table->text('startup_script');
|
||||
$table->integer('revision');
|
||||
$table->string('generation_token');
|
||||
$table->string('state');
|
||||
$table->integer('active_job_id')->nullable();
|
||||
$table->integer('created_at');
|
||||
$table->integer('updated_at');
|
||||
});
|
||||
$schema->create('cloud_template_replicas',
|
||||
static function (Blueprint $table): void {
|
||||
$table->integer('host_id');
|
||||
$table->string('template_id');
|
||||
$table->integer('revision');
|
||||
$table->string('generation_token');
|
||||
$table->string('state');
|
||||
$table->integer('image_bytes');
|
||||
$table->string('sha256');
|
||||
$table->string('operation_id');
|
||||
$table->integer('job_id')->nullable();
|
||||
$table->string('error_code');
|
||||
$table->integer('created_at');
|
||||
$table->integer('updated_at');
|
||||
$table->primary(['host_id', 'template_id']);
|
||||
});
|
||||
|
||||
Capsule::table('cloud_hosts')->insert([
|
||||
['id' => 41, 'enabled' => 1, 'default_network' => 'bridge:br0'],
|
||||
['id' => 42, 'enabled' => 1, 'default_network' => 'bridge:br0'],
|
||||
]);
|
||||
Capsule::table('cloud_host_agents')->insert([
|
||||
['host_id' => 41, 'enabled' => 1, 'last_test_status' => 'healthy',
|
||||
'tested_revision' => 3, 'revision' => 3],
|
||||
['host_id' => 42, 'enabled' => 1, 'last_test_status' => 'healthy',
|
||||
'tested_revision' => 3, 'revision' => 4],
|
||||
]);
|
||||
Capsule::table('cloud_host_storage_configurations')->insert([
|
||||
['host_id' => 41, 'state' => 'applied'],
|
||||
['host_id' => 42, 'state' => 'applied'],
|
||||
]);
|
||||
|
||||
$now = time();
|
||||
$template = static function (
|
||||
string $id,
|
||||
string $catalogKey,
|
||||
bool $default,
|
||||
string $state,
|
||||
string $generation,
|
||||
string $sha
|
||||
) use ($now): array {
|
||||
return [
|
||||
'id' => $id,
|
||||
'catalog_key' => $catalogKey,
|
||||
'version_label' => 'v1',
|
||||
'is_default' => $default ? 1 : 0,
|
||||
'name' => $id,
|
||||
'os_family' => 'Debian',
|
||||
'os_version' => '13',
|
||||
'source_type' => 'object',
|
||||
'sealed_source_url' => 'test-only',
|
||||
'image_bytes' => 4096,
|
||||
'sha256' => $sha,
|
||||
'startup_script' => '',
|
||||
'revision' => 2,
|
||||
'generation_token' => $generation,
|
||||
'state' => $state,
|
||||
'active_job_id' => null,
|
||||
'created_at' => $now,
|
||||
'updated_at' => $now,
|
||||
];
|
||||
};
|
||||
$generationA = str_repeat('a', 32);
|
||||
$generationB = str_repeat('b', 32);
|
||||
$shaA = str_repeat('c', 64);
|
||||
$shaB = str_repeat('d', 64);
|
||||
Capsule::table('cloud_templates')->insert([
|
||||
$template('debian-current', 'debian-13', true, 'active',
|
||||
$generationA, $shaA),
|
||||
$template('ubuntu-current', 'ubuntu-24', true, 'degraded',
|
||||
$generationB, $shaB),
|
||||
$template('debian-old', 'debian-13', false, 'active',
|
||||
str_repeat('e', 32), str_repeat('f', 64)),
|
||||
$template('retired-current', 'rocky-9', true, 'retired',
|
||||
str_repeat('1', 32), str_repeat('2', 64)),
|
||||
]);
|
||||
Capsule::table('cloud_template_replicas')->insert([
|
||||
'host_id' => 41,
|
||||
'template_id' => 'debian-current',
|
||||
'revision' => 2,
|
||||
'generation_token' => $generationA,
|
||||
'state' => 'active',
|
||||
'image_bytes' => 4096,
|
||||
'sha256' => $shaA,
|
||||
'operation_id' => str_repeat('3', 32),
|
||||
'job_id' => 1,
|
||||
'error_code' => '',
|
||||
'created_at' => $now,
|
||||
'updated_at' => $now,
|
||||
]);
|
||||
|
||||
$plan = CloudTemplateHostBackfillService::planReadyHost(41);
|
||||
$assert($plan['ready'] === true
|
||||
&& $plan['considered'] === 2
|
||||
&& $plan['already_ready'] === ['debian-current']
|
||||
&& $plan['pending'] === ['ubuntu-current'],
|
||||
'Ready-host backfill did not select only missing default generations.');
|
||||
|
||||
$notReady = CloudTemplateHostBackfillService::planReadyHost(42);
|
||||
$assert($notReady === [
|
||||
'host_id' => 42,
|
||||
'ready' => false,
|
||||
'considered' => 0,
|
||||
'already_ready' => [],
|
||||
'pending' => [],
|
||||
], 'A stale Agent readiness revision was accepted for template backfill.');
|
||||
|
||||
$exact = CloudTemplate::findOrFail('debian-current');
|
||||
Capsule::table('cloud_template_replicas')
|
||||
->where('host_id', 41)
|
||||
->where('template_id', 'debian-current')
|
||||
->update(['sha256' => str_repeat('9', 64)]);
|
||||
$changed = CloudTemplateHostBackfillService::planReadyHost(41);
|
||||
$assert($changed['already_ready'] === []
|
||||
&& $changed['pending'] === ['debian-current', 'ubuntu-current']
|
||||
&& (string) $exact->sha256 === $shaA,
|
||||
'A content-mismatched host replica was treated as exact.');
|
||||
|
||||
echo "Template ready-host backfill behavior passed.\n";
|
||||
Loading…
Reference in New Issue