fix: gate update consumers on engine readiness

dev
admin 2 months ago
parent 9bb5d30b0a
commit b8105fab23

@ -1823,6 +1823,10 @@ def test_internal_update_routes_are_super_admin_only_and_never_echo_token(bounda
engine.update_trigger = trigger
settings_path = "/internal/v1/update/settings"
ready = call(client, "GET", "/internal/v1/health", SUPER)
assert ready.status_code == 200
assert ready.json() == {"status": "ok", "accepting_mutations": True}
assert client.get(settings_path, cookies={"venix": "admin"}).status_code == 401
assert call(client, "GET", settings_path, USER).status_code == 403
assert call(client, "GET", settings_path, ADMIN).status_code == 403
@ -1853,7 +1857,9 @@ def test_internal_update_routes_are_super_admin_only_and_never_echo_token(bounda
# While the persisted update job is active, reads remain available but no
# host/VM mutation may begin and overlap the code/service restart.
assert call(client, "GET", "/internal/v1/health", SUPER).status_code == 200
maintenance = call(client, "GET", "/internal/v1/health", SUPER)
assert maintenance.status_code == 200
assert maintenance.json() == {"status": "ok", "accepting_mutations": False}
assert call(client, "POST", "/internal/v1/hosts/scan", SUPER,
{"host": "host.example.test", "port": 22}).status_code == 409

@ -41,7 +41,108 @@ class FakeUpdater:
self.job["status"] = "failed"
class FinalizeIncompleteTests(unittest.TestCase):
class UpdateWorkerTests(unittest.TestCase):
def test_service_consumers_start_only_after_engine_is_healthy(self):
calls = []
deployment = object.__new__(update_worker.DeploymentTransaction)
deployment.repo_dir = Path("/tmp")
deployment.config = {"service_secret": "unused"}
deployment.units = (
"virt-cloud-engine.service",
"virt-cloud-worker.service",
"virt-cloud-lifecycle.service",
"virt-cloud-policy-worker@1.service",
"virt-cloud-policy-worker@2.service",
"php-fpm.service",
)
def start(units, *, message):
calls.append(("start", units, message))
def wait(_repo_dir, units):
calls.append(("active", units))
with patch.object(deployment, "_start_services", side_effect=start), \
patch.object(update_worker, "_wait_units", side_effect=wait), \
patch.object(update_worker, "_wait_engine_health",
side_effect=lambda config: calls.append(("health", config))):
deployment.start_and_check()
engine = ("virt-cloud-engine.service",)
consumers = (
"virt-cloud-worker.service",
"virt-cloud-lifecycle.service",
"virt-cloud-policy-worker@1.service",
"virt-cloud-policy-worker@2.service",
)
web = ("php-fpm.service",)
self.assertEqual(calls[0][:2], ("start", web))
self.assertEqual(calls[1], ("active", web))
self.assertEqual(calls[2][:2], ("start", engine))
self.assertEqual(calls[3], ("active", engine))
self.assertEqual(calls[4], ("health", deployment.config))
self.assertEqual(calls[5][:2], ("start", consumers))
self.assertEqual(calls[6], ("active", consumers))
self.assertIn("PHP-FPM", calls[0][2])
self.assertIn("私有引擎", calls[2][2])
self.assertIn("后台服务", calls[5][2])
def test_engine_health_failure_restores_gated_consumers_before_raising(self):
calls = []
deployment = object.__new__(update_worker.DeploymentTransaction)
deployment.repo_dir = Path("/tmp")
deployment.config = {"service_secret": "unused"}
deployment.units = (
"virt-cloud-engine.service",
"virt-cloud-worker.service",
"virt-cloud-lifecycle.service",
"virt-cloud-policy-worker@1.service",
"virt-cloud-policy-worker@2.service",
"php-fpm.service",
)
def start(units, *, message):
calls.append(("start", units, message))
def unhealthy(config):
calls.append(("health", config))
raise update_worker.UpdateError("引擎未就绪", 500)
with patch.object(deployment, "_start_services", side_effect=start), \
patch.object(update_worker, "_wait_units",
side_effect=lambda _repo, units: calls.append(("active", units))), \
patch.object(update_worker, "_wait_engine_health",
side_effect=unhealthy):
with self.assertRaisesRegex(update_worker.UpdateError, "引擎未就绪"):
deployment.start_and_check()
self.assertEqual(calls[0][:2], ("start", ("php-fpm.service",)))
self.assertEqual(calls[2][:2], ("start", ("virt-cloud-engine.service",)))
self.assertEqual(calls[4], ("health", deployment.config))
self.assertEqual(calls[5][:2], ("start", (
"virt-cloud-worker.service",
"virt-cloud-lifecycle.service",
"virt-cloud-policy-worker@1.service",
"virt-cloud-policy-worker@2.service",
)))
def test_start_services_uses_only_the_explicit_unit_group(self):
deployment = object.__new__(update_worker.DeploymentTransaction)
deployment.repo_dir = Path("/tmp")
with patch.object(update_worker, "_executable", return_value="/usr/bin/systemctl"), \
patch.object(update_worker, "_run") as run:
deployment._start_services(
("virt-cloud-engine.service",), message="engine start failed"
)
run.assert_called_once_with(
["/usr/bin/systemctl", "start", "virt-cloud-engine.service"],
cwd=Path("/tmp"),
timeout=180,
message="engine start failed",
environment={"PATH": update_worker._EXECUTABLE_PATH, "LANG": "C", "LC_ALL": "C"},
)
def test_running_job_is_marked_failed_once(self):
store = FakeStore("job-1234567890123456")
updater = FakeUpdater({"status": "running"})

@ -677,7 +677,11 @@ class EngineApplication:
return
if scope["method"] == "GET" and path == "/internal/v1/health":
await asyncio.to_thread(self.store.get, "engine", "readiness")
await json_response(send, 200, {"status": "ok"})
maintenance = await asyncio.to_thread(self.updater.active_job)
await json_response(send, 200, {
"status": "ok",
"accepting_mutations": maintenance is None,
})
return
cloud_template_match = re.fullmatch(
r"/internal/v1/(vm-templates(?:/(?:import|upload)|/[A-Za-z0-9_-]+|"

@ -32,8 +32,9 @@ LOG = logging.getLogger(__name__)
_CONFIG_ENV = "CLOUD_ENGINE_CONFIG"
_DEFAULT_CONFIG = Path(__file__).resolve().parents[2] / "runtime" / "engine.json"
_ENGINE_HEALTH_URL = "http://127.0.0.1:8000/internal/v1/health"
_ENGINE_SERVICE_UNIT = "virt-cloud-engine.service"
_CORE_SERVICE_UNITS = (
"virt-cloud-engine.service",
_ENGINE_SERVICE_UNIT,
"virt-cloud-worker.service",
"virt-cloud-lifecycle.service",
"virt-cloud-policy-worker@1.service",
@ -219,19 +220,66 @@ class DeploymentTransaction:
mark_schema_ready()
self.start_and_check()
def _start_services(self) -> None:
def _start_services(self, units: tuple[str, ...], *, message: str) -> None:
if not units:
return
_run(
[_executable("systemctl"), "start", *self.units],
[_executable("systemctl"), "start", *units],
cwd=self.repo_dir,
timeout=180,
message="新版本服务启动失败,请检查 systemd 日志",
message=message,
environment={"PATH": _EXECUTABLE_PATH, "LANG": "C", "LC_ALL": "C"},
)
def start_and_check(self) -> None:
self._start_services()
_wait_units(self.repo_dir, self.units)
_wait_engine_health(self.config)
engine_units = tuple(unit for unit in self.units if unit == _ENGINE_SERVICE_UNIT)
if engine_units != (_ENGINE_SERVICE_UNIT,):
raise UpdateError("更新服务集合缺少私有引擎,已拒绝启动其他服务", 500)
php_fpm = str(self.config.get("update_php_fpm_unit", "php-fpm.service"))
web_units = tuple(unit for unit in self.units if unit == php_fpm)
if web_units != (php_fpm,):
raise UpdateError("更新服务集合缺少 PHP-FPM,已拒绝启动其他服务", 500)
consumer_units = tuple(
unit for unit in self.units if unit not in {_ENGINE_SERVICE_UNIT, php_fpm}
)
# ``systemctl is-active`` for a Type=exec unit only proves that uvicorn
# was executed; it does not prove that application startup completed
# or that the signed internal API is accepting requests. Starting the
# lifecycle worker in the same systemctl command used to let its first
# inventory tick race that startup window and mark healthy hosts
# unavailable. Keep every consumer stopped until the engine's signed
# health endpoint confirms actual readiness.
self._start_services(
web_units,
message="新版本 PHP-FPM 启动失败,请检查 systemd 日志",
)
_wait_units(self.repo_dir, web_units)
self._start_services(
engine_units,
message="新版本私有引擎启动失败,请检查 systemd 日志",
)
_wait_units(self.repo_dir, engine_units)
try:
_wait_engine_health(self.config)
except Exception:
# The website must remain available to report and retry a failed
# update. Restore the consumers too: the target checkout's PHP
# daemons have their own bounded engine-readiness gate, so they do
# not inventory a half-started engine or claim VM work. Re-raise
# the original health failure after systemd has a recoverable
# service topology instead of leaving the panel SSH-only.
self._start_services(
consumer_units,
message="引擎未就绪且面板后台服务恢复失败,请检查 systemd 日志",
)
_wait_units(self.repo_dir, consumer_units)
raise
self._start_services(
consumer_units,
message="新版本面板后台服务启动失败,请检查 systemd 日志",
)
_wait_units(self.repo_dir, consumer_units)
def _clean_recorded_checkout(updater: Updater, job: dict, field: str) -> bool:

@ -1,6 +1,7 @@
<?php
namespace App\Command;
use App\Services\Cloud\CloudEngineReadiness;
use App\Services\Cloud\CloudLifecycleService;
use App\Services\Cloud\CloudRuntimeHeartbeat;
@ -10,12 +11,37 @@ final class CloudLifecycle
{
$once = in_array('--once', $argv, true);
$heartbeat ??= CloudRuntimeHeartbeat::start('lifecycle', $once);
$skipEngineObservations = false;
try {
// Online update and boot may execute this PHP process before the
// Type=exec engine has completed its ASGI startup. Keep the first
// automatic inventory tick from turning that startup window into
// a false host-unavailable result. One-shot diagnostics retain
// their existing immediate behavior.
if (!$once) {
$heartbeat->busy('engine_readiness');
if (!CloudEngineReadiness::wait()) {
$heartbeat->error('engine_readiness_timeout');
fwrite(STDERR, "cloud lifecycle engine readiness timed out\n");
// A prolonged engine outage must not stop database-only
// billing and expiry work forever. Skip engine-backed
// observations in the first tick so the startup race
// cannot overwrite the last successful inventory; the
// next daemon pass resumes normal per-stage isolation.
$skipEngineObservations = true;
}
}
$service = new CloudLifecycleService($heartbeat);
do {
$heartbeat->busy('lifecycle_tick');
try {
$service->tick();
// Skip only the first capacity/runtime observation after
// a startup-readiness timeout. Database-only billing,
// expiry and reconciliation still run; the next daemon
// pass retries ordinary engine observations.
$skipThisTick = $skipEngineObservations;
$skipEngineObservations = false;
$service->tick(200, $skipThisTick);
$errors = $service->observationErrors();
if ($errors === []) {
$heartbeat->success();

@ -1,18 +1,39 @@
<?php
namespace App\Command;
use App\Services\Cloud\CloudWorkerService;
use App\Services\Cloud\CloudEngineReadiness;
use App\Services\Cloud\CloudInvoiceService;
use App\Services\Cloud\CloudRuntimeHeartbeat;
use App\Services\Cloud\CloudWorkerService;
final class CloudWorker
{
public static function run(array $argv = [], ?CloudRuntimeHeartbeat $heartbeat = null): int
public static function run(array $argv = [], ?CloudRuntimeHeartbeat $heartbeat = null,
?callable $engineReadiness = null): int
{
$once = in_array('--once', $argv, true);
$policyOnly = in_array('--policy-only', $argv, true);
$heartbeat ??= CloudRuntimeHeartbeat::start($policyOnly ? 'policy' : 'normal', $once);
$engineReadiness ??= static fn (): bool => CloudEngineReadiness::wait();
try {
// Do not claim a durable VM job during the brief interval where
// systemd reports the Type=exec engine active but its signed API
// is not ready. The service manager will retry this daemon if the
// bounded readiness window expires.
if (!$once) {
$heartbeat->busy('engine_readiness');
if (!$engineReadiness()) {
$heartbeat->error('engine_readiness_timeout');
fwrite(STDERR, "cloud worker engine readiness timed out\n");
if (!$policyOnly) {
self::deliverNotifications($heartbeat);
// Notification telemetry must not hide why this VM
// worker is exiting for systemd to retry.
$heartbeat->error('engine_readiness_timeout');
}
return 1;
}
}
$worker = new CloudWorkerService(null, $policyOnly);
do {
$errors = [];
@ -34,16 +55,7 @@ final class CloudWorker
}
// SMTP remains isolated from lifecycle and policy workers.
if (!$policyOnly) {
$heartbeat->busy('notification');
try {
CloudInvoiceService::deliverReminders(1);
$errors = array_merge($errors,
CloudInvoiceService::observationErrors('notifications'));
} catch (\Throwable $error) {
$errors[] = 'notification_failed';
fwrite(STDERR, 'cloud notification mail temporarily unavailable: '
. get_class($error) . PHP_EOL);
}
$errors = array_merge($errors, self::deliverNotifications($heartbeat));
}
if ($errors === []) {
$heartbeat->success();
@ -62,4 +74,17 @@ final class CloudWorker
}
return 0;
}
private static function deliverNotifications(CloudRuntimeHeartbeat $heartbeat): array
{
$heartbeat->busy('notification');
try {
CloudInvoiceService::deliverReminders(1);
return CloudInvoiceService::observationErrors('notifications');
} catch (\Throwable $error) {
fwrite(STDERR, 'cloud notification mail temporarily unavailable: '
. get_class($error) . PHP_EOL);
return ['notification_failed'];
}
}
}

@ -80,7 +80,7 @@ final class CloudEngineClient
if (!is_array($result)) {
throw new CloudError('虚拟化引擎响应不确定,请等待任务核对', 502);
}
if ($response->getStatusCode() >= 400) {
if ($response->getStatusCode() < 200 || $response->getStatusCode() >= 300) {
// A VM mutation can return a non-2xx HTTP status together with a
// durable, signed operation outcome. Only the worker that knows
// the exact operation ID may consume it; all ordinary API callers
@ -419,7 +419,7 @@ final class CloudEngineClient
} catch (\Throwable $error) {
$value = null;
}
if ($status >= 400) {
if ($status < 200 || $status >= 300) {
$code = match ($status) {
401, 403 => 'engine_auth_rejected',
404 => 'engine_resource_missing',

@ -0,0 +1,68 @@
<?php
namespace App\Services\Cloud;
/**
* Gate long-running control-plane consumers on the private engine's real
* application readiness, not merely on systemd having executed uvicorn.
*/
final class CloudEngineReadiness
{
private const DEFAULT_TIMEOUT_SECONDS = 45.0;
private const RETRY_MICROSECONDS = 1000000;
public static function wait(
?callable $probe = null,
float $timeoutSeconds = self::DEFAULT_TIMEOUT_SECONDS,
?callable $clock = null,
?callable $sleep = null,
): bool {
$timeoutSeconds = max(0.001, min(60.0, $timeoutSeconds));
$clock ??= static fn (): float => hrtime(true) / 1000000000;
$sleep ??= static function (int $microseconds): void {
usleep($microseconds);
};
$probe ??= self::probe(...);
$deadline = $clock() + $timeoutSeconds;
while (true) {
try {
if ($probe() === true) {
return true;
}
} catch (\Throwable $error) {
// Startup races and an engine that is not listening yet are
// expected here. Never log transport details or secrets.
}
$remaining = $deadline - $clock();
if ($remaining <= 0) {
return false;
}
$microseconds = max(1, min(
self::RETRY_MICROSECONDS,
(int) ceil($remaining * 1000000),
));
$sleep($microseconds);
}
}
private static function probe(): bool
{
// Health is an authenticated system probe, not an end-user action.
// The engine health route checks the signed request and engine stores;
// it does not authorize against a particular panel account.
$actor = [
'user_id' => 1,
'is_admin' => true,
'is_super_admin' => true,
'can_manage' => true,
'session_id' => 'cloud-engine-readiness',
];
$result = (new CloudEngineClient(null, 2.0))->request(
'GET', '/internal/v1/health', $actor
);
return ($result['status'] ?? null) === 'ok'
&& ($result['accepting_mutations'] ?? null) === true;
}
}

@ -25,9 +25,12 @@ final class CloudLifecycleService
$this->heartbeat?->error($code);
}
public function tick(int $limit = 200): int
public function tick(int $limit = 200, bool $skipEngineObservations = false): int
{
$this->observedErrors = [];
if ($skipEngineObservations) {
$this->observeFailure('engine_readiness_timeout');
}
$batchSize = max(1, min(1000, $limit));
$this->heartbeat?->busy('provision_expiry');
try {
@ -61,31 +64,35 @@ final class CloudLifecycleService
// Capacity and the concrete free-IP library are one read-only host
// observation. A failed poll preserves the last-known inventory and
// must not prevent billing or instance policy enforcement.
$this->heartbeat?->busy('host_inventory_sync');
try {
$inventory = (new CloudHostInventorySyncService())->sync(
min(10, $batchSize)
);
foreach (array_unique($inventory['errors'] ?? []) as $code) {
$this->observeFailure((string) $code);
if (!$skipEngineObservations) {
$this->heartbeat?->busy('host_inventory_sync');
try {
$inventory = (new CloudHostInventorySyncService())->sync(
min(10, $batchSize)
);
foreach (array_unique($inventory['errors'] ?? []) as $code) {
$this->observeFailure((string) $code);
}
} catch (\Throwable $error) {
$this->observeFailure('host_inventory_sync_failed');
error_log('cloud host inventory sync unavailable: ' . get_class($error));
}
} catch (\Throwable $error) {
$this->observeFailure('host_inventory_sync_failed');
error_log('cloud host inventory sync unavailable: ' . get_class($error));
}
// Power state is an observation cache, not a billing decision. Keep
// it on the lifecycle cadence so list pages can render immediately
// from the last successful host batch without waiting on libvirt.
$this->heartbeat?->busy('runtime_status_sync');
try {
$runtime = (new CloudInstanceRuntimeSyncService())->sync($batchSize);
foreach (array_unique($runtime['errors'] ?? []) as $code) {
$this->observeFailure((string) $code);
if (!$skipEngineObservations) {
$this->heartbeat?->busy('runtime_status_sync');
try {
$runtime = (new CloudInstanceRuntimeSyncService())->sync($batchSize);
foreach (array_unique($runtime['errors'] ?? []) as $code) {
$this->observeFailure((string) $code);
}
} catch (\Throwable $error) {
$this->observeFailure('runtime_status_sync_failed');
// A status poll must never prevent invoice or expiry policy work.
error_log('cloud runtime status sync unavailable: ' . get_class($error));
}
} catch (\Throwable $error) {
$this->observeFailure('runtime_status_sync_failed');
// A status poll must never prevent invoice or expiry policy work.
error_log('cloud runtime status sync unavailable: ' . get_class($error));
}
$queued = 0;
$this->heartbeat?->busy('instance_reconcile');

@ -7,14 +7,15 @@ final class CloudRuntimeHeartbeat
private const KINDS = ['normal' => ['worker', '普通任务 worker'],
'policy' => ['policy', '策略任务 worker'],
'lifecycle' => ['lifecycle', '生命周期调度']];
private const PHASES = ['startup' => '启动', 'worker_tick' => '任务消费或核对',
private const PHASES = ['startup' => '启动', 'engine_readiness' => '等待私有引擎就绪',
'worker_tick' => '任务消费或核对',
'notification' => '邮件通知', 'lifecycle_tick' => '生命周期巡检',
'provision_expiry' => '开通账单过期处理', 'invoice_generation' => '续费账单生成',
'traffic_sync' => '流量同步', 'host_inventory_sync' => '宿主容量与公网 IP 同步',
'runtime_status_sync' => '实例状态同步',
'instance_reconcile' => '实例策略核对',
'idle' => '等待下一轮', 'stopped' => '已退出'];
private const ERRORS = ['worker_tick_failed', 'notification_failed',
private const ERRORS = ['engine_readiness_timeout', 'worker_tick_failed', 'notification_failed',
'lifecycle_tick_failed', 'provision_expiry_failed', 'invoice_generation_failed',
'traffic_sync_failed', 'host_inventory_sync_failed',
'host_inventory_sync_unconfigured', 'runtime_status_sync_failed',
@ -78,7 +79,13 @@ final class CloudRuntimeHeartbeat
public function stopped(): void
{
$discardReadinessFailure = ($this->record['mode'] ?? null) === 'daemon'
&& ($this->record['last_success'] ?? null) === null
&& ($this->record['last_error'] ?? null) === 'engine_readiness_timeout';
$this->observe('stopped', 'stopped');
if ($discardReadinessFailure) {
$this->discardCurrentFile();
}
}
private function now(): int
@ -147,6 +154,20 @@ final class CloudRuntimeHeartbeat
&& (!function_exists('posix_geteuid') || $stat['uid'] === posix_geteuid());
}
/** Avoid unbounded restart-churn files before a worker has ever run. */
private function discardCurrentFile(): void
{
$path = $this->file;
$this->file = null;
if ($path === null || !self::secureDirectory($this->directory)) {
return;
}
$stat = @lstat($path);
if (is_array($stat) && self::privateFile($stat)) {
@unlink($path);
}
}
private function persist(): void
{
$temporary = null;

@ -0,0 +1,81 @@
<?php
require dirname(__DIR__) . '/vendor/autoload.php';
use App\Services\Cloud\CloudEngineReadiness;
$assert = static function ($ok, string $message): void {
if (!$ok) {
throw new RuntimeException($message);
}
};
$now = 10.0;
$attempts = 0;
$sleeps = [];
$ready = CloudEngineReadiness::wait(
static function () use (&$attempts): bool {
$attempts++;
return $attempts === 3;
},
3.0,
static function () use (&$now): float {
return $now;
},
static function (int $microseconds) use (&$now, &$sleeps): void {
$sleeps[] = $microseconds;
$now += $microseconds / 1000000;
},
);
$assert($ready && $attempts === 3 && $sleeps === [1000000, 1000000],
'Readiness did not retry a transient startup race with bounded backoff.');
$now = 20.0;
$attempts = 0;
$sleeps = [];
$ready = CloudEngineReadiness::wait(
static function () use (&$attempts): bool {
$attempts++;
throw new RuntimeException('SECRET transport detail');
},
1.5,
static function () use (&$now): float {
return $now;
},
static function (int $microseconds) use (&$now, &$sleeps): void {
$sleeps[] = $microseconds;
$now += $microseconds / 1000000;
},
);
$assert(!$ready && $attempts === 3 && $sleeps === [1000000, 500000],
'Readiness did not stop at its monotonic deadline after probe failures.');
$root = dirname(__DIR__);
$readiness = file_get_contents($root . '/app/Services/Cloud/CloudEngineReadiness.php');
$lifecycle = file_get_contents($root . '/app/Command/CloudLifecycle.php');
$lifecycleService = file_get_contents($root . '/app/Services/Cloud/CloudLifecycleService.php');
$worker = file_get_contents($root . '/app/Command/CloudWorker.php');
$heartbeat = file_get_contents($root . '/app/Services/Cloud/CloudRuntimeHeartbeat.php');
$assert(str_contains($readiness, "new CloudEngineClient(null, 2.0)")
&& str_contains($readiness, "'GET', '/internal/v1/health'")
&& str_contains($readiness, "['accepting_mutations'] ?? null")
&& str_contains($readiness, "'session_id' => 'cloud-engine-readiness'"),
'The production gate is no longer a bounded signed engine health probe.');
$assert(str_contains($heartbeat, "'engine_readiness' => '等待私有引擎就绪'")
&& str_contains($heartbeat, "'engine_readiness_timeout'"),
'Readiness observations are not allowlisted by runtime health telemetry.');
$assert(str_contains($lifecycle, '$service->tick(200, $skipThisTick)')
&& substr_count($lifecycleService, 'if (!$skipEngineObservations)') === 2
&& str_contains($lifecycleService, "observeFailure('engine_readiness_timeout')"),
'A readiness timeout can still run first-pass engine observations.');
foreach ([$lifecycle, $worker] as $source) {
$assert(str_contains($source, 'if (!$once)')
&& str_contains($source, 'CloudEngineReadiness::wait()')
&& str_contains($source, "engine_readiness_timeout"),
'A long-running consumer lost its engine readiness gate.');
}
$assert(str_contains($worker, 'self::deliverNotifications($heartbeat)')
&& str_contains($worker, '?callable $engineReadiness = null'),
'An engine-readiness failure can still suppress DB-backed notifications.');
echo "Engine consumer readiness checks passed.\n";

@ -61,6 +61,7 @@ namespace {
['errno' => 7]), 'engine_connect_failed', null, 502],
[new ConnectException('SECRET timed out', new Request('GET', 'http://localhost')),
'engine_connection_or_timeout', null, 502],
[new Response(302, [], '{"redirect":"SECRET"}'), 'engine_request_rejected', 302, 502],
[new Response(401, [], 'SECRET'), 'engine_auth_rejected', 401, 502],
[new Response(403, [], 'SECRET'), 'engine_auth_rejected', 403, 502],
[new Response(404, [], 'SECRET'), 'engine_resource_missing', 404, 502],
@ -118,5 +119,14 @@ namespace {
$assert($error->httpStatus === 502 && !str_contains($error->getMessage(), 'SECRET'),
'Inventory diagnostics changed mutation error semantics');
}
$engine = $client([new Response(302, [], '{"status":"ok","accepting_mutations":true}')],
$history);
try {
$engine->request('GET', '/internal/v1/health', $actor);
throw new \RuntimeException('Redirect-like engine response was accepted');
} catch (CloudError $error) {
$assert($error->httpStatus === 502 && count($history) === 1,
'A non-2xx engine response crossed the private client boundary.');
}
echo "Inventory engine diagnostics client checks passed.\n";
}

@ -63,6 +63,13 @@ try {
&& $checks($directory)['policy']['active_count'] === 2,
'Starting workers or multiple policy reporters were not represented.');
$normal->busy('engine_readiness');
$assert(str_contains($checks($directory)['worker']['message'], '等待私有引擎就绪'),
'Engine readiness was not exposed as its own bounded startup phase.');
$normal->error('engine_readiness_timeout');
$assert($read($directory)[0]['last_error'] === 'engine_readiness_timeout',
'The fixed readiness timeout code was not retained by private telemetry.');
$normal->success();
$normal->busy('worker_tick');
$policy->busy('worker_tick');
$policy2->stopped();
@ -278,6 +285,31 @@ try {
&& $record['mode'] === 'once', 'Real --once command did not finish as a successful stopped observation.');
}
$gatedDirectory = $testRoot . '/readiness-gated-lifecycle';
$gatedHeartbeat = CloudRuntimeHeartbeat::start('lifecycle', false, $gatedDirectory, $clock);
$gatedLifecycle = new CloudLifecycleService($gatedHeartbeat);
$assert($gatedLifecycle->tick(200, true) === 0
&& $gatedLifecycle->observationErrors() === ['engine_readiness_timeout']
&& $read($gatedDirectory)[0]['last_error'] === 'engine_readiness_timeout',
'The first readiness-timeout tick did not preserve DB work while skipping engine observations.');
$gatedWorkerDirectory = $testRoot . '/readiness-gated-worker';
$gatedWorkerHeartbeat = CloudRuntimeHeartbeat::start(
'normal', false, $gatedWorkerDirectory, $clock
);
Capsule::table('cloud_jobs')->insert([
'kind' => 'create', 'status' => 'queued', 'payload_cipher' => 'must-not-be-read',
'operation_id' => 'maintenance-gated-job', 'available_at' => 0,
'lease_until' => 0, 'attempts' => 0, 'user_id' => 1, 'host_id' => 1,
]);
$assert(CloudWorker::run([], $gatedWorkerHeartbeat, static fn (): bool => false) === 1
&& $files($gatedWorkerDirectory) === []
&& Capsule::table('cloud_jobs')->value('status') === 'queued'
&& (int) Capsule::table('cloud_jobs')->value('attempts') === 0
&& Capsule::table('cloud_jobs')->value('dispatched_at') === null,
'A worker readiness timeout skipped notification fallback or leaked restart-churn files.');
Capsule::table('cloud_jobs')->delete();
Capsule::table('cloud_instances')->insert(['id' => 10, 'status' => 'active',
'service_status' => 'active', 'expires_at' => time() + 3600,
'renewal_price' => 'invalid-amount', 'renewal_days' => 30]);

Loading…
Cancel
Save