diff --git a/web/app/Services/Cloud/CloudAgentSettingsService.php b/web/app/Services/Cloud/CloudAgentSettingsService.php index 6b5a9717..b3770d6c 100644 --- a/web/app/Services/Cloud/CloudAgentSettingsService.php +++ b/web/app/Services/Cloud/CloudAgentSettingsService.php @@ -158,6 +158,7 @@ final class CloudAgentSettingsService ); self::recordTest($hostId, $expectedRevision, 'healthy', '', $health['version']); + self::scheduleTemplateBackfill($hostId, $actor, 'agent_ready'); $fresh = CloudHostAgent::find($hostId); return self::dto($host, $fresh, CloudHostStorageConfiguration::find($hostId)) + [ @@ -370,6 +371,28 @@ final class CloudAgentSettingsService } } + /** A completed health test must not be rolled back by optional backfill. */ + private static function scheduleTemplateBackfill( + int $hostId, + array $actor, + string $reason + ): void { + try { + CloudTemplateHostBackfillService::scheduleReadyHost( + $hostId, + $actor, + $reason + ); + } catch (\Throwable $error) { + error_log(sprintf( + 'cloud template backfill host=%d trigger=%s unavailable: %s', + $hostId, + $reason, + get_class($error) + )); + } + } + private static function host(int $hostId): CloudHost { if ($hostId < 1 || ($host = CloudHost::find($hostId)) === null) { diff --git a/web/app/Services/Cloud/CloudTemplateHostBackfillService.php b/web/app/Services/Cloud/CloudTemplateHostBackfillService.php new file mode 100644 index 00000000..0919a0cf --- /dev/null +++ b/web/app/Services/Cloud/CloudTemplateHostBackfillService.php @@ -0,0 +1,149 @@ + $hostId, + 'ready' => $plan['ready'], + 'considered' => $plan['considered'], + 'already_ready' => count($plan['already_ready']), + 'scheduled' => 0, + 'deferred' => 0, + ]; + if (!$plan['ready']) { + return $report; + } + foreach ($plan['pending'] as $templateId) { + try { + $result = CloudTemplateReplicaCoordinator::syncHost( + $templateId, + $hostId, + $actor, + $reason + ); + if (($result['scheduled'] ?? false) === true) { + $report['scheduled']++; + } else { + $report['already_ready']++; + } + } catch (\Throwable $error) { + // The original host-readiness action has already committed. + // Keep this generation eligible for the periodic reconciler + // instead of converting an independent backfill into a false + // Agent/storage failure. + $report['deferred']++; + error_log(sprintf( + 'cloud template backfill host=%d template=%s deferred: %s', + $hostId, + $templateId, + get_class($error) + )); + } + } + return $report; + } + + /** + * Read-only plan used by event hooks and future reconciliation passes. + * Only the default generation of each catalog entry is auto-filled; older + * immutable versions remain available for explicit operator repair. + * + * @return array{host_id:int,ready:bool,considered:int, + * already_ready:list,pending:list} + */ + public static function planReadyHost(int $hostId): array + { + if ($hostId < 1) { + throw new CloudError('云镜像补齐宿主机参数无效', 422); + } + $ready = in_array( + $hostId, + CloudAgentSettingsService::enabledHostIds(), + true + ); + if (!$ready) { + return [ + 'host_id' => $hostId, + 'ready' => false, + 'considered' => 0, + 'already_ready' => [], + 'pending' => [], + ]; + } + $templates = CloudTemplate::query() + ->where('is_default', 1) + ->whereIn('state', self::TEMPLATE_STATES) + ->orderBy('catalog_key') + ->orderBy('version_label') + ->orderBy('id') + ->get(); + $templateIds = $templates->map( + static fn (CloudTemplate $template): string => + (string) $template->id + )->all(); + $replicas = $templateIds === [] + ? collect() + : CloudTemplateReplica::query() + ->where('host_id', $hostId) + ->whereIn('template_id', $templateIds) + ->get()->keyBy('template_id'); + $alreadyReady = []; + $pending = []; + foreach ($templates as $template) { + $templateId = (string) $template->id; + $replica = $replicas->get($templateId); + if ($replica instanceof CloudTemplateReplica + && CloudTemplateReplicaCoordinator::isExactReplica( + $template, + $replica + )) { + $alreadyReady[] = $templateId; + continue; + } + $pending[] = $templateId; + } + return [ + 'host_id' => $hostId, + 'ready' => true, + 'considered' => count($templateIds), + 'already_ready' => $alreadyReady, + 'pending' => $pending, + ]; + } + + private static function assertInput(int $hostId, string $reason): void + { + if ($hostId < 1 + || preg_match('/^[a-z][a-z0-9_]{1,47}$/D', $reason) !== 1) { + throw new CloudError('云镜像补齐参数无效', 422); + } + } +} diff --git a/web/app/Services/Cloud/CloudTemplateReplicaCoordinator.php b/web/app/Services/Cloud/CloudTemplateReplicaCoordinator.php index e57f065e..a8c31750 100644 --- a/web/app/Services/Cloud/CloudTemplateReplicaCoordinator.php +++ b/web/app/Services/Cloud/CloudTemplateReplicaCoordinator.php @@ -80,7 +80,8 @@ final class CloudTemplateReplicaCoordinator ->where('host_id', $hostId) ->where('template_id', $templateId) ->lockForUpdate()->first(); - if ($replica !== null && self::isExact($template, $replica)) { + if ($replica !== null + && self::isExactReplica($template, $replica)) { return null; } if ($replica !== null) { @@ -252,7 +253,8 @@ final class CloudTemplateReplicaCoordinator } } - private static function isExact( + /** One authority for deciding whether a host already has this generation. */ + public static function isExactReplica( CloudTemplate $template, CloudTemplateReplica $replica ): bool { diff --git a/web/app/Services/Cloud/CloudWorkerService.php b/web/app/Services/Cloud/CloudWorkerService.php index 763f8d23..09a60d7b 100644 --- a/web/app/Services/Cloud/CloudWorkerService.php +++ b/web/app/Services/Cloud/CloudWorkerService.php @@ -3316,6 +3316,37 @@ final class CloudWorkerService $order->save(); } }, 3); + if ((string) $claimed->kind === 'host_storage_configure' + && $status === 'succeeded') { + $this->scheduleTemplateBackfillAfterStorage( + (int) $claimed->host_id + ); + } + } + + /** Run object-storage discovery only after the storage transaction commits. */ + private function scheduleTemplateBackfillAfterStorage(int $hostId): void + { + try { + $actor = CloudDaemonActor::resolve('cloud-template-backfill'); + if ($actor === null) { + throw new CloudError( + '云镜像补齐缺少可用的系统管理员执行者', + 503 + ); + } + CloudTemplateHostBackfillService::scheduleReadyHost( + $hostId, + $actor, + 'storage_ready' + ); + } catch (\Throwable $error) { + error_log(sprintf( + 'cloud template backfill host=%d trigger=storage_ready unavailable: %s', + $hostId, + get_class($error) + )); + } } private static function throwInvalidSettingsResult( diff --git a/web/tests/CloudTemplateHostBackfillContractTest.php b/web/tests/CloudTemplateHostBackfillContractTest.php new file mode 100644 index 00000000..fac17028 --- /dev/null +++ b/web/tests/CloudTemplateHostBackfillContractTest.php @@ -0,0 +1,45 @@ +scheduleTemplateBackfillAfterStorage(', + $transactionEnd === false ? 0 : $transactionEnd +); +$assert($transactionEnd !== false && $storageHook !== false + && $storageHook > $transactionEnd, + 'Storage completion schedules template backfill before its transaction commits.'); +$assert(str_contains($agent, + "scheduleTemplateBackfill(\$hostId, \$actor, 'agent_ready')"), + 'A successful Agent readiness test does not trigger host backfill.'); +$assert(str_contains($service, "->where('is_default', 1)") + && str_contains($service, + "private const TEMPLATE_STATES = ['active', 'degraded', 'error']") + && str_contains($service, 'CloudAgentSettingsService::enabledHostIds()') + && str_contains($service, + 'CloudTemplateReplicaCoordinator::isExactReplica('), + 'Host backfill is not constrained to ready hosts and current generations.'); +$assert(str_contains($service, 'foreach ($plan[\'pending\'] as $templateId)') + && str_contains($service, '} catch (\\Throwable $error) {'), + 'One template failure can abort the remaining ready-host backfill plan.'); + +echo "Template ready-host backfill contracts passed.\n"; diff --git a/web/tests/CloudTemplateHostBackfillTest.php b/web/tests/CloudTemplateHostBackfillTest.php new file mode 100644 index 00000000..1a02689f --- /dev/null +++ b/web/tests/CloudTemplateHostBackfillTest.php @@ -0,0 +1,177 @@ +addConnection([ + 'driver' => 'sqlite', 'database' => ':memory:', +], 'default'); +$capsule->setAsGlobal(); +$capsule->bootEloquent(); +$schema = $capsule->schema('default'); +$schema->create('cloud_hosts', static function (Blueprint $table): void { + $table->integer('id')->primary(); + $table->boolean('enabled'); + $table->string('default_network'); +}); +$schema->create('cloud_host_agents', static function (Blueprint $table): void { + $table->integer('host_id')->primary(); + $table->boolean('enabled'); + $table->string('last_test_status'); + $table->integer('tested_revision'); + $table->integer('revision'); +}); +$schema->create('cloud_host_storage_configurations', + static function (Blueprint $table): void { + $table->integer('host_id')->primary(); + $table->string('state'); + }); +$schema->create('cloud_templates', static function (Blueprint $table): void { + $table->string('id')->primary(); + $table->string('catalog_key'); + $table->string('version_label'); + $table->boolean('is_default'); + $table->string('name'); + $table->string('os_family'); + $table->string('os_version'); + $table->string('source_type'); + $table->text('sealed_source_url'); + $table->integer('image_bytes'); + $table->string('sha256'); + $table->text('startup_script'); + $table->integer('revision'); + $table->string('generation_token'); + $table->string('state'); + $table->integer('active_job_id')->nullable(); + $table->integer('created_at'); + $table->integer('updated_at'); +}); +$schema->create('cloud_template_replicas', + static function (Blueprint $table): void { + $table->integer('host_id'); + $table->string('template_id'); + $table->integer('revision'); + $table->string('generation_token'); + $table->string('state'); + $table->integer('image_bytes'); + $table->string('sha256'); + $table->string('operation_id'); + $table->integer('job_id')->nullable(); + $table->string('error_code'); + $table->integer('created_at'); + $table->integer('updated_at'); + $table->primary(['host_id', 'template_id']); + }); + +Capsule::table('cloud_hosts')->insert([ + ['id' => 41, 'enabled' => 1, 'default_network' => 'bridge:br0'], + ['id' => 42, 'enabled' => 1, 'default_network' => 'bridge:br0'], +]); +Capsule::table('cloud_host_agents')->insert([ + ['host_id' => 41, 'enabled' => 1, 'last_test_status' => 'healthy', + 'tested_revision' => 3, 'revision' => 3], + ['host_id' => 42, 'enabled' => 1, 'last_test_status' => 'healthy', + 'tested_revision' => 3, 'revision' => 4], +]); +Capsule::table('cloud_host_storage_configurations')->insert([ + ['host_id' => 41, 'state' => 'applied'], + ['host_id' => 42, 'state' => 'applied'], +]); + +$now = time(); +$template = static function ( + string $id, + string $catalogKey, + bool $default, + string $state, + string $generation, + string $sha +) use ($now): array { + return [ + 'id' => $id, + 'catalog_key' => $catalogKey, + 'version_label' => 'v1', + 'is_default' => $default ? 1 : 0, + 'name' => $id, + 'os_family' => 'Debian', + 'os_version' => '13', + 'source_type' => 'object', + 'sealed_source_url' => 'test-only', + 'image_bytes' => 4096, + 'sha256' => $sha, + 'startup_script' => '', + 'revision' => 2, + 'generation_token' => $generation, + 'state' => $state, + 'active_job_id' => null, + 'created_at' => $now, + 'updated_at' => $now, + ]; +}; +$generationA = str_repeat('a', 32); +$generationB = str_repeat('b', 32); +$shaA = str_repeat('c', 64); +$shaB = str_repeat('d', 64); +Capsule::table('cloud_templates')->insert([ + $template('debian-current', 'debian-13', true, 'active', + $generationA, $shaA), + $template('ubuntu-current', 'ubuntu-24', true, 'degraded', + $generationB, $shaB), + $template('debian-old', 'debian-13', false, 'active', + str_repeat('e', 32), str_repeat('f', 64)), + $template('retired-current', 'rocky-9', true, 'retired', + str_repeat('1', 32), str_repeat('2', 64)), +]); +Capsule::table('cloud_template_replicas')->insert([ + 'host_id' => 41, + 'template_id' => 'debian-current', + 'revision' => 2, + 'generation_token' => $generationA, + 'state' => 'active', + 'image_bytes' => 4096, + 'sha256' => $shaA, + 'operation_id' => str_repeat('3', 32), + 'job_id' => 1, + 'error_code' => '', + 'created_at' => $now, + 'updated_at' => $now, +]); + +$plan = CloudTemplateHostBackfillService::planReadyHost(41); +$assert($plan['ready'] === true + && $plan['considered'] === 2 + && $plan['already_ready'] === ['debian-current'] + && $plan['pending'] === ['ubuntu-current'], + 'Ready-host backfill did not select only missing default generations.'); + +$notReady = CloudTemplateHostBackfillService::planReadyHost(42); +$assert($notReady === [ + 'host_id' => 42, + 'ready' => false, + 'considered' => 0, + 'already_ready' => [], + 'pending' => [], +], 'A stale Agent readiness revision was accepted for template backfill.'); + +$exact = CloudTemplate::findOrFail('debian-current'); +Capsule::table('cloud_template_replicas') + ->where('host_id', 41) + ->where('template_id', 'debian-current') + ->update(['sha256' => str_repeat('9', 64)]); +$changed = CloudTemplateHostBackfillService::planReadyHost(41); +$assert($changed['already_ready'] === [] + && $changed['pending'] === ['debian-current', 'ubuntu-current'] + && (string) $exact->sha256 === $shaA, + 'A content-mismatched host replica was treated as exact.'); + +echo "Template ready-host backfill behavior passed.\n";