feat(template): backfill ready hosts

go
codex 3 weeks ago
parent d166b56aa4
commit aa3898a07c

@ -158,6 +158,7 @@ final class CloudAgentSettingsService
);
self::recordTest($hostId, $expectedRevision, 'healthy', '',
$health['version']);
self::scheduleTemplateBackfill($hostId, $actor, 'agent_ready');
$fresh = CloudHostAgent::find($hostId);
return self::dto($host, $fresh,
CloudHostStorageConfiguration::find($hostId)) + [
@ -370,6 +371,28 @@ final class CloudAgentSettingsService
}
}
/** A completed health test must not be rolled back by optional backfill. */
private static function scheduleTemplateBackfill(
int $hostId,
array $actor,
string $reason
): void {
try {
CloudTemplateHostBackfillService::scheduleReadyHost(
$hostId,
$actor,
$reason
);
} catch (\Throwable $error) {
error_log(sprintf(
'cloud template backfill host=%d trigger=%s unavailable: %s',
$hostId,
$reason,
get_class($error)
));
}
}
private static function host(int $hostId): CloudHost
{
if ($hostId < 1 || ($host = CloudHost::find($hostId)) === null) {

@ -0,0 +1,149 @@
<?php
namespace App\Services\Cloud;
use App\Models\CloudTemplate;
use App\Models\CloudTemplateReplica;
/**
* Fill the current catalog generations onto a host after it becomes ready.
*
* Host readiness is the admission boundary. This coordinator is deliberately
* best-effort across templates: one busy or temporarily unavailable artifact
* must not prevent another independent catalog generation from being queued.
*/
final class CloudTemplateHostBackfillService
{
private const TEMPLATE_STATES = ['active', 'degraded', 'error'];
/**
* Queue every missing default generation for one ready host.
*
* @return array{host_id:int,ready:bool,considered:int,already_ready:int,
* scheduled:int,deferred:int}
*/
public static function scheduleReadyHost(
int $hostId,
array $actor,
string $reason = 'host_ready'
): array {
CloudAccess::manage($actor);
self::assertInput($hostId, $reason);
$plan = self::planReadyHost($hostId);
$report = [
'host_id' => $hostId,
'ready' => $plan['ready'],
'considered' => $plan['considered'],
'already_ready' => count($plan['already_ready']),
'scheduled' => 0,
'deferred' => 0,
];
if (!$plan['ready']) {
return $report;
}
foreach ($plan['pending'] as $templateId) {
try {
$result = CloudTemplateReplicaCoordinator::syncHost(
$templateId,
$hostId,
$actor,
$reason
);
if (($result['scheduled'] ?? false) === true) {
$report['scheduled']++;
} else {
$report['already_ready']++;
}
} catch (\Throwable $error) {
// The original host-readiness action has already committed.
// Keep this generation eligible for the periodic reconciler
// instead of converting an independent backfill into a false
// Agent/storage failure.
$report['deferred']++;
error_log(sprintf(
'cloud template backfill host=%d template=%s deferred: %s',
$hostId,
$templateId,
get_class($error)
));
}
}
return $report;
}
/**
* Read-only plan used by event hooks and future reconciliation passes.
* Only the default generation of each catalog entry is auto-filled; older
* immutable versions remain available for explicit operator repair.
*
* @return array{host_id:int,ready:bool,considered:int,
* already_ready:list<string>,pending:list<string>}
*/
public static function planReadyHost(int $hostId): array
{
if ($hostId < 1) {
throw new CloudError('云镜像补齐宿主机参数无效', 422);
}
$ready = in_array(
$hostId,
CloudAgentSettingsService::enabledHostIds(),
true
);
if (!$ready) {
return [
'host_id' => $hostId,
'ready' => false,
'considered' => 0,
'already_ready' => [],
'pending' => [],
];
}
$templates = CloudTemplate::query()
->where('is_default', 1)
->whereIn('state', self::TEMPLATE_STATES)
->orderBy('catalog_key')
->orderBy('version_label')
->orderBy('id')
->get();
$templateIds = $templates->map(
static fn (CloudTemplate $template): string =>
(string) $template->id
)->all();
$replicas = $templateIds === []
? collect()
: CloudTemplateReplica::query()
->where('host_id', $hostId)
->whereIn('template_id', $templateIds)
->get()->keyBy('template_id');
$alreadyReady = [];
$pending = [];
foreach ($templates as $template) {
$templateId = (string) $template->id;
$replica = $replicas->get($templateId);
if ($replica instanceof CloudTemplateReplica
&& CloudTemplateReplicaCoordinator::isExactReplica(
$template,
$replica
)) {
$alreadyReady[] = $templateId;
continue;
}
$pending[] = $templateId;
}
return [
'host_id' => $hostId,
'ready' => true,
'considered' => count($templateIds),
'already_ready' => $alreadyReady,
'pending' => $pending,
];
}
private static function assertInput(int $hostId, string $reason): void
{
if ($hostId < 1
|| preg_match('/^[a-z][a-z0-9_]{1,47}$/D', $reason) !== 1) {
throw new CloudError('云镜像补齐参数无效', 422);
}
}
}

@ -80,7 +80,8 @@ final class CloudTemplateReplicaCoordinator
->where('host_id', $hostId)
->where('template_id', $templateId)
->lockForUpdate()->first();
if ($replica !== null && self::isExact($template, $replica)) {
if ($replica !== null
&& self::isExactReplica($template, $replica)) {
return null;
}
if ($replica !== null) {
@ -252,7 +253,8 @@ final class CloudTemplateReplicaCoordinator
}
}
private static function isExact(
/** One authority for deciding whether a host already has this generation. */
public static function isExactReplica(
CloudTemplate $template,
CloudTemplateReplica $replica
): bool {

@ -3316,6 +3316,37 @@ final class CloudWorkerService
$order->save();
}
}, 3);
if ((string) $claimed->kind === 'host_storage_configure'
&& $status === 'succeeded') {
$this->scheduleTemplateBackfillAfterStorage(
(int) $claimed->host_id
);
}
}
/** Run object-storage discovery only after the storage transaction commits. */
private function scheduleTemplateBackfillAfterStorage(int $hostId): void
{
try {
$actor = CloudDaemonActor::resolve('cloud-template-backfill');
if ($actor === null) {
throw new CloudError(
'云镜像补齐缺少可用的系统管理员执行者',
503
);
}
CloudTemplateHostBackfillService::scheduleReadyHost(
$hostId,
$actor,
'storage_ready'
);
} catch (\Throwable $error) {
error_log(sprintf(
'cloud template backfill host=%d trigger=storage_ready unavailable: %s',
$hostId,
get_class($error)
));
}
}
private static function throwInvalidSettingsResult(

@ -0,0 +1,45 @@
<?php
$root = dirname(__DIR__);
$worker = file_get_contents(
$root . '/app/Services/Cloud/CloudWorkerService.php'
);
$agent = file_get_contents(
$root . '/app/Services/Cloud/CloudAgentSettingsService.php'
);
$service = file_get_contents(
$root . '/app/Services/Cloud/CloudTemplateHostBackfillService.php'
);
$assert = static function (bool $condition, string $message): void {
if (!$condition) {
throw new RuntimeException($message);
}
};
$transactionEnd = strpos($worker, '}, 3);', strpos(
$worker,
'private function finish('
));
$storageHook = strpos(
$worker,
'$this->scheduleTemplateBackfillAfterStorage(',
$transactionEnd === false ? 0 : $transactionEnd
);
$assert($transactionEnd !== false && $storageHook !== false
&& $storageHook > $transactionEnd,
'Storage completion schedules template backfill before its transaction commits.');
$assert(str_contains($agent,
"scheduleTemplateBackfill(\$hostId, \$actor, 'agent_ready')"),
'A successful Agent readiness test does not trigger host backfill.');
$assert(str_contains($service, "->where('is_default', 1)")
&& str_contains($service,
"private const TEMPLATE_STATES = ['active', 'degraded', 'error']")
&& str_contains($service, 'CloudAgentSettingsService::enabledHostIds()')
&& str_contains($service,
'CloudTemplateReplicaCoordinator::isExactReplica('),
'Host backfill is not constrained to ready hosts and current generations.');
$assert(str_contains($service, 'foreach ($plan[\'pending\'] as $templateId)')
&& str_contains($service, '} catch (\\Throwable $error) {'),
'One template failure can abort the remaining ready-host backfill plan.');
echo "Template ready-host backfill contracts passed.\n";

@ -0,0 +1,177 @@
<?php
require dirname(__DIR__) . '/vendor/autoload.php';
use App\Models\CloudTemplate;
use App\Services\Cloud\CloudTemplateHostBackfillService;
use Illuminate\Database\Capsule\Manager as Capsule;
use Illuminate\Database\Schema\Blueprint;
$assert = static function (bool $condition, string $message): void {
if (!$condition) {
throw new RuntimeException($message);
}
};
$capsule = new Capsule();
$capsule->addConnection([
'driver' => 'sqlite', 'database' => ':memory:',
], 'default');
$capsule->setAsGlobal();
$capsule->bootEloquent();
$schema = $capsule->schema('default');
$schema->create('cloud_hosts', static function (Blueprint $table): void {
$table->integer('id')->primary();
$table->boolean('enabled');
$table->string('default_network');
});
$schema->create('cloud_host_agents', static function (Blueprint $table): void {
$table->integer('host_id')->primary();
$table->boolean('enabled');
$table->string('last_test_status');
$table->integer('tested_revision');
$table->integer('revision');
});
$schema->create('cloud_host_storage_configurations',
static function (Blueprint $table): void {
$table->integer('host_id')->primary();
$table->string('state');
});
$schema->create('cloud_templates', static function (Blueprint $table): void {
$table->string('id')->primary();
$table->string('catalog_key');
$table->string('version_label');
$table->boolean('is_default');
$table->string('name');
$table->string('os_family');
$table->string('os_version');
$table->string('source_type');
$table->text('sealed_source_url');
$table->integer('image_bytes');
$table->string('sha256');
$table->text('startup_script');
$table->integer('revision');
$table->string('generation_token');
$table->string('state');
$table->integer('active_job_id')->nullable();
$table->integer('created_at');
$table->integer('updated_at');
});
$schema->create('cloud_template_replicas',
static function (Blueprint $table): void {
$table->integer('host_id');
$table->string('template_id');
$table->integer('revision');
$table->string('generation_token');
$table->string('state');
$table->integer('image_bytes');
$table->string('sha256');
$table->string('operation_id');
$table->integer('job_id')->nullable();
$table->string('error_code');
$table->integer('created_at');
$table->integer('updated_at');
$table->primary(['host_id', 'template_id']);
});
Capsule::table('cloud_hosts')->insert([
['id' => 41, 'enabled' => 1, 'default_network' => 'bridge:br0'],
['id' => 42, 'enabled' => 1, 'default_network' => 'bridge:br0'],
]);
Capsule::table('cloud_host_agents')->insert([
['host_id' => 41, 'enabled' => 1, 'last_test_status' => 'healthy',
'tested_revision' => 3, 'revision' => 3],
['host_id' => 42, 'enabled' => 1, 'last_test_status' => 'healthy',
'tested_revision' => 3, 'revision' => 4],
]);
Capsule::table('cloud_host_storage_configurations')->insert([
['host_id' => 41, 'state' => 'applied'],
['host_id' => 42, 'state' => 'applied'],
]);
$now = time();
$template = static function (
string $id,
string $catalogKey,
bool $default,
string $state,
string $generation,
string $sha
) use ($now): array {
return [
'id' => $id,
'catalog_key' => $catalogKey,
'version_label' => 'v1',
'is_default' => $default ? 1 : 0,
'name' => $id,
'os_family' => 'Debian',
'os_version' => '13',
'source_type' => 'object',
'sealed_source_url' => 'test-only',
'image_bytes' => 4096,
'sha256' => $sha,
'startup_script' => '',
'revision' => 2,
'generation_token' => $generation,
'state' => $state,
'active_job_id' => null,
'created_at' => $now,
'updated_at' => $now,
];
};
$generationA = str_repeat('a', 32);
$generationB = str_repeat('b', 32);
$shaA = str_repeat('c', 64);
$shaB = str_repeat('d', 64);
Capsule::table('cloud_templates')->insert([
$template('debian-current', 'debian-13', true, 'active',
$generationA, $shaA),
$template('ubuntu-current', 'ubuntu-24', true, 'degraded',
$generationB, $shaB),
$template('debian-old', 'debian-13', false, 'active',
str_repeat('e', 32), str_repeat('f', 64)),
$template('retired-current', 'rocky-9', true, 'retired',
str_repeat('1', 32), str_repeat('2', 64)),
]);
Capsule::table('cloud_template_replicas')->insert([
'host_id' => 41,
'template_id' => 'debian-current',
'revision' => 2,
'generation_token' => $generationA,
'state' => 'active',
'image_bytes' => 4096,
'sha256' => $shaA,
'operation_id' => str_repeat('3', 32),
'job_id' => 1,
'error_code' => '',
'created_at' => $now,
'updated_at' => $now,
]);
$plan = CloudTemplateHostBackfillService::planReadyHost(41);
$assert($plan['ready'] === true
&& $plan['considered'] === 2
&& $plan['already_ready'] === ['debian-current']
&& $plan['pending'] === ['ubuntu-current'],
'Ready-host backfill did not select only missing default generations.');
$notReady = CloudTemplateHostBackfillService::planReadyHost(42);
$assert($notReady === [
'host_id' => 42,
'ready' => false,
'considered' => 0,
'already_ready' => [],
'pending' => [],
], 'A stale Agent readiness revision was accepted for template backfill.');
$exact = CloudTemplate::findOrFail('debian-current');
Capsule::table('cloud_template_replicas')
->where('host_id', 41)
->where('template_id', 'debian-current')
->update(['sha256' => str_repeat('9', 64)]);
$changed = CloudTemplateHostBackfillService::planReadyHost(41);
$assert($changed['already_ready'] === []
&& $changed['pending'] === ['debian-current', 'ubuntu-current']
&& (string) $exact->sha256 === $shaA,
'A content-mismatched host replica was treated as exact.');
echo "Template ready-host backfill behavior passed.\n";
Loading…
Cancel
Save