diff --git a/engine/tests/test_engine_boundary.py b/engine/tests/test_engine_boundary.py index 4422607c..54a41cb1 100644 --- a/engine/tests/test_engine_boundary.py +++ b/engine/tests/test_engine_boundary.py @@ -1823,6 +1823,10 @@ def test_internal_update_routes_are_super_admin_only_and_never_echo_token(bounda engine.update_trigger = trigger settings_path = "/internal/v1/update/settings" + ready = call(client, "GET", "/internal/v1/health", SUPER) + assert ready.status_code == 200 + assert ready.json() == {"status": "ok", "accepting_mutations": True} + assert client.get(settings_path, cookies={"venix": "admin"}).status_code == 401 assert call(client, "GET", settings_path, USER).status_code == 403 assert call(client, "GET", settings_path, ADMIN).status_code == 403 @@ -1853,7 +1857,9 @@ def test_internal_update_routes_are_super_admin_only_and_never_echo_token(bounda # While the persisted update job is active, reads remain available but no # host/VM mutation may begin and overlap the code/service restart. - assert call(client, "GET", "/internal/v1/health", SUPER).status_code == 200 + maintenance = call(client, "GET", "/internal/v1/health", SUPER) + assert maintenance.status_code == 200 + assert maintenance.json() == {"status": "ok", "accepting_mutations": False} assert call(client, "POST", "/internal/v1/hosts/scan", SUPER, {"host": "host.example.test", "port": 22}).status_code == 409 diff --git a/engine/tests/test_update_worker.py b/engine/tests/test_update_worker.py index ca6e5da8..30839254 100644 --- a/engine/tests/test_update_worker.py +++ b/engine/tests/test_update_worker.py @@ -41,7 +41,108 @@ class FakeUpdater: self.job["status"] = "failed" -class FinalizeIncompleteTests(unittest.TestCase): +class UpdateWorkerTests(unittest.TestCase): + def test_service_consumers_start_only_after_engine_is_healthy(self): + calls = [] + deployment = object.__new__(update_worker.DeploymentTransaction) + deployment.repo_dir = Path("/tmp") + deployment.config = {"service_secret": "unused"} + deployment.units = ( + "virt-cloud-engine.service", + "virt-cloud-worker.service", + "virt-cloud-lifecycle.service", + "virt-cloud-policy-worker@1.service", + "virt-cloud-policy-worker@2.service", + "php-fpm.service", + ) + + def start(units, *, message): + calls.append(("start", units, message)) + + def wait(_repo_dir, units): + calls.append(("active", units)) + + with patch.object(deployment, "_start_services", side_effect=start), \ + patch.object(update_worker, "_wait_units", side_effect=wait), \ + patch.object(update_worker, "_wait_engine_health", + side_effect=lambda config: calls.append(("health", config))): + deployment.start_and_check() + + engine = ("virt-cloud-engine.service",) + consumers = ( + "virt-cloud-worker.service", + "virt-cloud-lifecycle.service", + "virt-cloud-policy-worker@1.service", + "virt-cloud-policy-worker@2.service", + ) + web = ("php-fpm.service",) + self.assertEqual(calls[0][:2], ("start", web)) + self.assertEqual(calls[1], ("active", web)) + self.assertEqual(calls[2][:2], ("start", engine)) + self.assertEqual(calls[3], ("active", engine)) + self.assertEqual(calls[4], ("health", deployment.config)) + self.assertEqual(calls[5][:2], ("start", consumers)) + self.assertEqual(calls[6], ("active", consumers)) + self.assertIn("PHP-FPM", calls[0][2]) + self.assertIn("私有引擎", calls[2][2]) + self.assertIn("后台服务", calls[5][2]) + + def test_engine_health_failure_restores_gated_consumers_before_raising(self): + calls = [] + deployment = object.__new__(update_worker.DeploymentTransaction) + deployment.repo_dir = Path("/tmp") + deployment.config = {"service_secret": "unused"} + deployment.units = ( + "virt-cloud-engine.service", + "virt-cloud-worker.service", + "virt-cloud-lifecycle.service", + "virt-cloud-policy-worker@1.service", + "virt-cloud-policy-worker@2.service", + "php-fpm.service", + ) + + def start(units, *, message): + calls.append(("start", units, message)) + + def unhealthy(config): + calls.append(("health", config)) + raise update_worker.UpdateError("引擎未就绪", 500) + + with patch.object(deployment, "_start_services", side_effect=start), \ + patch.object(update_worker, "_wait_units", + side_effect=lambda _repo, units: calls.append(("active", units))), \ + patch.object(update_worker, "_wait_engine_health", + side_effect=unhealthy): + with self.assertRaisesRegex(update_worker.UpdateError, "引擎未就绪"): + deployment.start_and_check() + + self.assertEqual(calls[0][:2], ("start", ("php-fpm.service",))) + self.assertEqual(calls[2][:2], ("start", ("virt-cloud-engine.service",))) + self.assertEqual(calls[4], ("health", deployment.config)) + self.assertEqual(calls[5][:2], ("start", ( + "virt-cloud-worker.service", + "virt-cloud-lifecycle.service", + "virt-cloud-policy-worker@1.service", + "virt-cloud-policy-worker@2.service", + ))) + + def test_start_services_uses_only_the_explicit_unit_group(self): + deployment = object.__new__(update_worker.DeploymentTransaction) + deployment.repo_dir = Path("/tmp") + with patch.object(update_worker, "_executable", return_value="/usr/bin/systemctl"), \ + patch.object(update_worker, "_run") as run: + deployment._start_services( + ("virt-cloud-engine.service",), message="engine start failed" + ) + + run.assert_called_once_with( + ["/usr/bin/systemctl", "start", "virt-cloud-engine.service"], + cwd=Path("/tmp"), + timeout=180, + message="engine start failed", + environment={"PATH": update_worker._EXECUTABLE_PATH, "LANG": "C", "LC_ALL": "C"}, + ) + def test_running_job_is_marked_failed_once(self): store = FakeStore("job-1234567890123456") updater = FakeUpdater({"status": "running"}) diff --git a/engine/virsh_panel/internal.py b/engine/virsh_panel/internal.py index 699d7548..1e84c410 100644 --- a/engine/virsh_panel/internal.py +++ b/engine/virsh_panel/internal.py @@ -677,7 +677,11 @@ class EngineApplication: return if scope["method"] == "GET" and path == "/internal/v1/health": await asyncio.to_thread(self.store.get, "engine", "readiness") - await json_response(send, 200, {"status": "ok"}) + maintenance = await asyncio.to_thread(self.updater.active_job) + await json_response(send, 200, { + "status": "ok", + "accepting_mutations": maintenance is None, + }) return cloud_template_match = re.fullmatch( r"/internal/v1/(vm-templates(?:/(?:import|upload)|/[A-Za-z0-9_-]+|" diff --git a/engine/virsh_panel/update_worker.py b/engine/virsh_panel/update_worker.py index c0991030..755a11e1 100644 --- a/engine/virsh_panel/update_worker.py +++ b/engine/virsh_panel/update_worker.py @@ -32,8 +32,9 @@ LOG = logging.getLogger(__name__) _CONFIG_ENV = "CLOUD_ENGINE_CONFIG" _DEFAULT_CONFIG = Path(__file__).resolve().parents[2] / "runtime" / "engine.json" _ENGINE_HEALTH_URL = "http://127.0.0.1:8000/internal/v1/health" +_ENGINE_SERVICE_UNIT = "virt-cloud-engine.service" _CORE_SERVICE_UNITS = ( - "virt-cloud-engine.service", + _ENGINE_SERVICE_UNIT, "virt-cloud-worker.service", "virt-cloud-lifecycle.service", "virt-cloud-policy-worker@1.service", @@ -219,19 +220,66 @@ class DeploymentTransaction: mark_schema_ready() self.start_and_check() - def _start_services(self) -> None: + def _start_services(self, units: tuple[str, ...], *, message: str) -> None: + if not units: + return _run( - [_executable("systemctl"), "start", *self.units], + [_executable("systemctl"), "start", *units], cwd=self.repo_dir, timeout=180, - message="新版本服务启动失败,请检查 systemd 日志", + message=message, environment={"PATH": _EXECUTABLE_PATH, "LANG": "C", "LC_ALL": "C"}, ) def start_and_check(self) -> None: - self._start_services() - _wait_units(self.repo_dir, self.units) - _wait_engine_health(self.config) + engine_units = tuple(unit for unit in self.units if unit == _ENGINE_SERVICE_UNIT) + if engine_units != (_ENGINE_SERVICE_UNIT,): + raise UpdateError("更新服务集合缺少私有引擎,已拒绝启动其他服务", 500) + php_fpm = str(self.config.get("update_php_fpm_unit", "php-fpm.service")) + web_units = tuple(unit for unit in self.units if unit == php_fpm) + if web_units != (php_fpm,): + raise UpdateError("更新服务集合缺少 PHP-FPM,已拒绝启动其他服务", 500) + consumer_units = tuple( + unit for unit in self.units if unit not in {_ENGINE_SERVICE_UNIT, php_fpm} + ) + + # ``systemctl is-active`` for a Type=exec unit only proves that uvicorn + # was executed; it does not prove that application startup completed + # or that the signed internal API is accepting requests. Starting the + # lifecycle worker in the same systemctl command used to let its first + # inventory tick race that startup window and mark healthy hosts + # unavailable. Keep every consumer stopped until the engine's signed + # health endpoint confirms actual readiness. + self._start_services( + web_units, + message="新版本 PHP-FPM 启动失败,请检查 systemd 日志", + ) + _wait_units(self.repo_dir, web_units) + self._start_services( + engine_units, + message="新版本私有引擎启动失败,请检查 systemd 日志", + ) + _wait_units(self.repo_dir, engine_units) + try: + _wait_engine_health(self.config) + except Exception: + # The website must remain available to report and retry a failed + # update. Restore the consumers too: the target checkout's PHP + # daemons have their own bounded engine-readiness gate, so they do + # not inventory a half-started engine or claim VM work. Re-raise + # the original health failure after systemd has a recoverable + # service topology instead of leaving the panel SSH-only. + self._start_services( + consumer_units, + message="引擎未就绪且面板后台服务恢复失败,请检查 systemd 日志", + ) + _wait_units(self.repo_dir, consumer_units) + raise + self._start_services( + consumer_units, + message="新版本面板后台服务启动失败,请检查 systemd 日志", + ) + _wait_units(self.repo_dir, consumer_units) def _clean_recorded_checkout(updater: Updater, job: dict, field: str) -> bool: diff --git a/web/app/Command/CloudLifecycle.php b/web/app/Command/CloudLifecycle.php index c0eb78a6..63161039 100644 --- a/web/app/Command/CloudLifecycle.php +++ b/web/app/Command/CloudLifecycle.php @@ -1,6 +1,7 @@ busy('engine_readiness'); + if (!CloudEngineReadiness::wait()) { + $heartbeat->error('engine_readiness_timeout'); + fwrite(STDERR, "cloud lifecycle engine readiness timed out\n"); + // A prolonged engine outage must not stop database-only + // billing and expiry work forever. Skip engine-backed + // observations in the first tick so the startup race + // cannot overwrite the last successful inventory; the + // next daemon pass resumes normal per-stage isolation. + $skipEngineObservations = true; + } + } $service = new CloudLifecycleService($heartbeat); do { $heartbeat->busy('lifecycle_tick'); try { - $service->tick(); + // Skip only the first capacity/runtime observation after + // a startup-readiness timeout. Database-only billing, + // expiry and reconciliation still run; the next daemon + // pass retries ordinary engine observations. + $skipThisTick = $skipEngineObservations; + $skipEngineObservations = false; + $service->tick(200, $skipThisTick); $errors = $service->observationErrors(); if ($errors === []) { $heartbeat->success(); diff --git a/web/app/Command/CloudWorker.php b/web/app/Command/CloudWorker.php index 2b2c3d89..5f89753b 100644 --- a/web/app/Command/CloudWorker.php +++ b/web/app/Command/CloudWorker.php @@ -1,18 +1,39 @@ CloudEngineReadiness::wait(); try { + // Do not claim a durable VM job during the brief interval where + // systemd reports the Type=exec engine active but its signed API + // is not ready. The service manager will retry this daemon if the + // bounded readiness window expires. + if (!$once) { + $heartbeat->busy('engine_readiness'); + if (!$engineReadiness()) { + $heartbeat->error('engine_readiness_timeout'); + fwrite(STDERR, "cloud worker engine readiness timed out\n"); + if (!$policyOnly) { + self::deliverNotifications($heartbeat); + // Notification telemetry must not hide why this VM + // worker is exiting for systemd to retry. + $heartbeat->error('engine_readiness_timeout'); + } + return 1; + } + } $worker = new CloudWorkerService(null, $policyOnly); do { $errors = []; @@ -34,16 +55,7 @@ final class CloudWorker } // SMTP remains isolated from lifecycle and policy workers. if (!$policyOnly) { - $heartbeat->busy('notification'); - try { - CloudInvoiceService::deliverReminders(1); - $errors = array_merge($errors, - CloudInvoiceService::observationErrors('notifications')); - } catch (\Throwable $error) { - $errors[] = 'notification_failed'; - fwrite(STDERR, 'cloud notification mail temporarily unavailable: ' - . get_class($error) . PHP_EOL); - } + $errors = array_merge($errors, self::deliverNotifications($heartbeat)); } if ($errors === []) { $heartbeat->success(); @@ -62,4 +74,17 @@ final class CloudWorker } return 0; } + + private static function deliverNotifications(CloudRuntimeHeartbeat $heartbeat): array + { + $heartbeat->busy('notification'); + try { + CloudInvoiceService::deliverReminders(1); + return CloudInvoiceService::observationErrors('notifications'); + } catch (\Throwable $error) { + fwrite(STDERR, 'cloud notification mail temporarily unavailable: ' + . get_class($error) . PHP_EOL); + return ['notification_failed']; + } + } } diff --git a/web/app/Services/Cloud/CloudEngineClient.php b/web/app/Services/Cloud/CloudEngineClient.php index 81543488..93afd709 100644 --- a/web/app/Services/Cloud/CloudEngineClient.php +++ b/web/app/Services/Cloud/CloudEngineClient.php @@ -80,7 +80,7 @@ final class CloudEngineClient if (!is_array($result)) { throw new CloudError('虚拟化引擎响应不确定,请等待任务核对', 502); } - if ($response->getStatusCode() >= 400) { + if ($response->getStatusCode() < 200 || $response->getStatusCode() >= 300) { // A VM mutation can return a non-2xx HTTP status together with a // durable, signed operation outcome. Only the worker that knows // the exact operation ID may consume it; all ordinary API callers @@ -419,7 +419,7 @@ final class CloudEngineClient } catch (\Throwable $error) { $value = null; } - if ($status >= 400) { + if ($status < 200 || $status >= 300) { $code = match ($status) { 401, 403 => 'engine_auth_rejected', 404 => 'engine_resource_missing', diff --git a/web/app/Services/Cloud/CloudEngineReadiness.php b/web/app/Services/Cloud/CloudEngineReadiness.php new file mode 100644 index 00000000..f808830b --- /dev/null +++ b/web/app/Services/Cloud/CloudEngineReadiness.php @@ -0,0 +1,68 @@ + hrtime(true) / 1000000000; + $sleep ??= static function (int $microseconds): void { + usleep($microseconds); + }; + $probe ??= self::probe(...); + $deadline = $clock() + $timeoutSeconds; + + while (true) { + try { + if ($probe() === true) { + return true; + } + } catch (\Throwable $error) { + // Startup races and an engine that is not listening yet are + // expected here. Never log transport details or secrets. + } + + $remaining = $deadline - $clock(); + if ($remaining <= 0) { + return false; + } + $microseconds = max(1, min( + self::RETRY_MICROSECONDS, + (int) ceil($remaining * 1000000), + )); + $sleep($microseconds); + } + } + + private static function probe(): bool + { + // Health is an authenticated system probe, not an end-user action. + // The engine health route checks the signed request and engine stores; + // it does not authorize against a particular panel account. + $actor = [ + 'user_id' => 1, + 'is_admin' => true, + 'is_super_admin' => true, + 'can_manage' => true, + 'session_id' => 'cloud-engine-readiness', + ]; + $result = (new CloudEngineClient(null, 2.0))->request( + 'GET', '/internal/v1/health', $actor + ); + return ($result['status'] ?? null) === 'ok' + && ($result['accepting_mutations'] ?? null) === true; + } +} diff --git a/web/app/Services/Cloud/CloudLifecycleService.php b/web/app/Services/Cloud/CloudLifecycleService.php index 8a6e92d1..ee0a1d83 100644 --- a/web/app/Services/Cloud/CloudLifecycleService.php +++ b/web/app/Services/Cloud/CloudLifecycleService.php @@ -25,9 +25,12 @@ final class CloudLifecycleService $this->heartbeat?->error($code); } - public function tick(int $limit = 200): int + public function tick(int $limit = 200, bool $skipEngineObservations = false): int { $this->observedErrors = []; + if ($skipEngineObservations) { + $this->observeFailure('engine_readiness_timeout'); + } $batchSize = max(1, min(1000, $limit)); $this->heartbeat?->busy('provision_expiry'); try { @@ -61,31 +64,35 @@ final class CloudLifecycleService // Capacity and the concrete free-IP library are one read-only host // observation. A failed poll preserves the last-known inventory and // must not prevent billing or instance policy enforcement. - $this->heartbeat?->busy('host_inventory_sync'); - try { - $inventory = (new CloudHostInventorySyncService())->sync( - min(10, $batchSize) - ); - foreach (array_unique($inventory['errors'] ?? []) as $code) { - $this->observeFailure((string) $code); + if (!$skipEngineObservations) { + $this->heartbeat?->busy('host_inventory_sync'); + try { + $inventory = (new CloudHostInventorySyncService())->sync( + min(10, $batchSize) + ); + foreach (array_unique($inventory['errors'] ?? []) as $code) { + $this->observeFailure((string) $code); + } + } catch (\Throwable $error) { + $this->observeFailure('host_inventory_sync_failed'); + error_log('cloud host inventory sync unavailable: ' . get_class($error)); } - } catch (\Throwable $error) { - $this->observeFailure('host_inventory_sync_failed'); - error_log('cloud host inventory sync unavailable: ' . get_class($error)); } // Power state is an observation cache, not a billing decision. Keep // it on the lifecycle cadence so list pages can render immediately // from the last successful host batch without waiting on libvirt. - $this->heartbeat?->busy('runtime_status_sync'); - try { - $runtime = (new CloudInstanceRuntimeSyncService())->sync($batchSize); - foreach (array_unique($runtime['errors'] ?? []) as $code) { - $this->observeFailure((string) $code); + if (!$skipEngineObservations) { + $this->heartbeat?->busy('runtime_status_sync'); + try { + $runtime = (new CloudInstanceRuntimeSyncService())->sync($batchSize); + foreach (array_unique($runtime['errors'] ?? []) as $code) { + $this->observeFailure((string) $code); + } + } catch (\Throwable $error) { + $this->observeFailure('runtime_status_sync_failed'); + // A status poll must never prevent invoice or expiry policy work. + error_log('cloud runtime status sync unavailable: ' . get_class($error)); } - } catch (\Throwable $error) { - $this->observeFailure('runtime_status_sync_failed'); - // A status poll must never prevent invoice or expiry policy work. - error_log('cloud runtime status sync unavailable: ' . get_class($error)); } $queued = 0; $this->heartbeat?->busy('instance_reconcile'); diff --git a/web/app/Services/Cloud/CloudRuntimeHeartbeat.php b/web/app/Services/Cloud/CloudRuntimeHeartbeat.php index 3a729cbb..7c895746 100644 --- a/web/app/Services/Cloud/CloudRuntimeHeartbeat.php +++ b/web/app/Services/Cloud/CloudRuntimeHeartbeat.php @@ -7,14 +7,15 @@ final class CloudRuntimeHeartbeat private const KINDS = ['normal' => ['worker', '普通任务 worker'], 'policy' => ['policy', '策略任务 worker'], 'lifecycle' => ['lifecycle', '生命周期调度']]; - private const PHASES = ['startup' => '启动', 'worker_tick' => '任务消费或核对', + private const PHASES = ['startup' => '启动', 'engine_readiness' => '等待私有引擎就绪', + 'worker_tick' => '任务消费或核对', 'notification' => '邮件通知', 'lifecycle_tick' => '生命周期巡检', 'provision_expiry' => '开通账单过期处理', 'invoice_generation' => '续费账单生成', 'traffic_sync' => '流量同步', 'host_inventory_sync' => '宿主容量与公网 IP 同步', 'runtime_status_sync' => '实例状态同步', 'instance_reconcile' => '实例策略核对', 'idle' => '等待下一轮', 'stopped' => '已退出']; - private const ERRORS = ['worker_tick_failed', 'notification_failed', + private const ERRORS = ['engine_readiness_timeout', 'worker_tick_failed', 'notification_failed', 'lifecycle_tick_failed', 'provision_expiry_failed', 'invoice_generation_failed', 'traffic_sync_failed', 'host_inventory_sync_failed', 'host_inventory_sync_unconfigured', 'runtime_status_sync_failed', @@ -78,7 +79,13 @@ final class CloudRuntimeHeartbeat public function stopped(): void { + $discardReadinessFailure = ($this->record['mode'] ?? null) === 'daemon' + && ($this->record['last_success'] ?? null) === null + && ($this->record['last_error'] ?? null) === 'engine_readiness_timeout'; $this->observe('stopped', 'stopped'); + if ($discardReadinessFailure) { + $this->discardCurrentFile(); + } } private function now(): int @@ -147,6 +154,20 @@ final class CloudRuntimeHeartbeat && (!function_exists('posix_geteuid') || $stat['uid'] === posix_geteuid()); } + /** Avoid unbounded restart-churn files before a worker has ever run. */ + private function discardCurrentFile(): void + { + $path = $this->file; + $this->file = null; + if ($path === null || !self::secureDirectory($this->directory)) { + return; + } + $stat = @lstat($path); + if (is_array($stat) && self::privateFile($stat)) { + @unlink($path); + } + } + private function persist(): void { $temporary = null; diff --git a/web/tests/CloudEngineReadinessTest.php b/web/tests/CloudEngineReadinessTest.php new file mode 100644 index 00000000..7da6c976 --- /dev/null +++ b/web/tests/CloudEngineReadinessTest.php @@ -0,0 +1,81 @@ + 'cloud-engine-readiness'"), + 'The production gate is no longer a bounded signed engine health probe.'); +$assert(str_contains($heartbeat, "'engine_readiness' => '等待私有引擎就绪'") + && str_contains($heartbeat, "'engine_readiness_timeout'"), + 'Readiness observations are not allowlisted by runtime health telemetry.'); +$assert(str_contains($lifecycle, '$service->tick(200, $skipThisTick)') + && substr_count($lifecycleService, 'if (!$skipEngineObservations)') === 2 + && str_contains($lifecycleService, "observeFailure('engine_readiness_timeout')"), + 'A readiness timeout can still run first-pass engine observations.'); +foreach ([$lifecycle, $worker] as $source) { + $assert(str_contains($source, 'if (!$once)') + && str_contains($source, 'CloudEngineReadiness::wait()') + && str_contains($source, "engine_readiness_timeout"), + 'A long-running consumer lost its engine readiness gate.'); +} +$assert(str_contains($worker, 'self::deliverNotifications($heartbeat)') + && str_contains($worker, '?callable $engineReadiness = null'), + 'An engine-readiness failure can still suppress DB-backed notifications.'); + +echo "Engine consumer readiness checks passed.\n"; diff --git a/web/tests/CloudInventoryEngineClientTest.php b/web/tests/CloudInventoryEngineClientTest.php index 9c29cbcf..d1984cbc 100644 --- a/web/tests/CloudInventoryEngineClientTest.php +++ b/web/tests/CloudInventoryEngineClientTest.php @@ -61,6 +61,7 @@ namespace { ['errno' => 7]), 'engine_connect_failed', null, 502], [new ConnectException('SECRET timed out', new Request('GET', 'http://localhost')), 'engine_connection_or_timeout', null, 502], + [new Response(302, [], '{"redirect":"SECRET"}'), 'engine_request_rejected', 302, 502], [new Response(401, [], 'SECRET'), 'engine_auth_rejected', 401, 502], [new Response(403, [], 'SECRET'), 'engine_auth_rejected', 403, 502], [new Response(404, [], 'SECRET'), 'engine_resource_missing', 404, 502], @@ -118,5 +119,14 @@ namespace { $assert($error->httpStatus === 502 && !str_contains($error->getMessage(), 'SECRET'), 'Inventory diagnostics changed mutation error semantics'); } + $engine = $client([new Response(302, [], '{"status":"ok","accepting_mutations":true}')], + $history); + try { + $engine->request('GET', '/internal/v1/health', $actor); + throw new \RuntimeException('Redirect-like engine response was accepted'); + } catch (CloudError $error) { + $assert($error->httpStatus === 502 && count($history) === 1, + 'A non-2xx engine response crossed the private client boundary.'); + } echo "Inventory engine diagnostics client checks passed.\n"; } diff --git a/web/tests/CloudRuntimeHeartbeatTest.php b/web/tests/CloudRuntimeHeartbeatTest.php index c0b585c4..7e313d8d 100644 --- a/web/tests/CloudRuntimeHeartbeatTest.php +++ b/web/tests/CloudRuntimeHeartbeatTest.php @@ -63,6 +63,13 @@ try { && $checks($directory)['policy']['active_count'] === 2, 'Starting workers or multiple policy reporters were not represented.'); + $normal->busy('engine_readiness'); + $assert(str_contains($checks($directory)['worker']['message'], '等待私有引擎就绪'), + 'Engine readiness was not exposed as its own bounded startup phase.'); + $normal->error('engine_readiness_timeout'); + $assert($read($directory)[0]['last_error'] === 'engine_readiness_timeout', + 'The fixed readiness timeout code was not retained by private telemetry.'); + $normal->success(); $normal->busy('worker_tick'); $policy->busy('worker_tick'); $policy2->stopped(); @@ -278,6 +285,31 @@ try { && $record['mode'] === 'once', 'Real --once command did not finish as a successful stopped observation.'); } + $gatedDirectory = $testRoot . '/readiness-gated-lifecycle'; + $gatedHeartbeat = CloudRuntimeHeartbeat::start('lifecycle', false, $gatedDirectory, $clock); + $gatedLifecycle = new CloudLifecycleService($gatedHeartbeat); + $assert($gatedLifecycle->tick(200, true) === 0 + && $gatedLifecycle->observationErrors() === ['engine_readiness_timeout'] + && $read($gatedDirectory)[0]['last_error'] === 'engine_readiness_timeout', + 'The first readiness-timeout tick did not preserve DB work while skipping engine observations.'); + + $gatedWorkerDirectory = $testRoot . '/readiness-gated-worker'; + $gatedWorkerHeartbeat = CloudRuntimeHeartbeat::start( + 'normal', false, $gatedWorkerDirectory, $clock + ); + Capsule::table('cloud_jobs')->insert([ + 'kind' => 'create', 'status' => 'queued', 'payload_cipher' => 'must-not-be-read', + 'operation_id' => 'maintenance-gated-job', 'available_at' => 0, + 'lease_until' => 0, 'attempts' => 0, 'user_id' => 1, 'host_id' => 1, + ]); + $assert(CloudWorker::run([], $gatedWorkerHeartbeat, static fn (): bool => false) === 1 + && $files($gatedWorkerDirectory) === [] + && Capsule::table('cloud_jobs')->value('status') === 'queued' + && (int) Capsule::table('cloud_jobs')->value('attempts') === 0 + && Capsule::table('cloud_jobs')->value('dispatched_at') === null, + 'A worker readiness timeout skipped notification fallback or leaked restart-churn files.'); + Capsule::table('cloud_jobs')->delete(); + Capsule::table('cloud_instances')->insert(['id' => 10, 'status' => 'active', 'service_status' => 'active', 'expires_at' => time() + 3600, 'renewal_price' => 'invalid-amount', 'renewal_days' => 30]);