fix: gate update consumers on engine readiness
parent
9bb5d30b0a
commit
b8105fab23
@ -0,0 +1,68 @@
|
||||
<?php
|
||||
|
||||
namespace App\Services\Cloud;
|
||||
|
||||
/**
|
||||
* Gate long-running control-plane consumers on the private engine's real
|
||||
* application readiness, not merely on systemd having executed uvicorn.
|
||||
*/
|
||||
final class CloudEngineReadiness
|
||||
{
|
||||
private const DEFAULT_TIMEOUT_SECONDS = 45.0;
|
||||
private const RETRY_MICROSECONDS = 1000000;
|
||||
|
||||
public static function wait(
|
||||
?callable $probe = null,
|
||||
float $timeoutSeconds = self::DEFAULT_TIMEOUT_SECONDS,
|
||||
?callable $clock = null,
|
||||
?callable $sleep = null,
|
||||
): bool {
|
||||
$timeoutSeconds = max(0.001, min(60.0, $timeoutSeconds));
|
||||
$clock ??= static fn (): float => hrtime(true) / 1000000000;
|
||||
$sleep ??= static function (int $microseconds): void {
|
||||
usleep($microseconds);
|
||||
};
|
||||
$probe ??= self::probe(...);
|
||||
$deadline = $clock() + $timeoutSeconds;
|
||||
|
||||
while (true) {
|
||||
try {
|
||||
if ($probe() === true) {
|
||||
return true;
|
||||
}
|
||||
} catch (\Throwable $error) {
|
||||
// Startup races and an engine that is not listening yet are
|
||||
// expected here. Never log transport details or secrets.
|
||||
}
|
||||
|
||||
$remaining = $deadline - $clock();
|
||||
if ($remaining <= 0) {
|
||||
return false;
|
||||
}
|
||||
$microseconds = max(1, min(
|
||||
self::RETRY_MICROSECONDS,
|
||||
(int) ceil($remaining * 1000000),
|
||||
));
|
||||
$sleep($microseconds);
|
||||
}
|
||||
}
|
||||
|
||||
private static function probe(): bool
|
||||
{
|
||||
// Health is an authenticated system probe, not an end-user action.
|
||||
// The engine health route checks the signed request and engine stores;
|
||||
// it does not authorize against a particular panel account.
|
||||
$actor = [
|
||||
'user_id' => 1,
|
||||
'is_admin' => true,
|
||||
'is_super_admin' => true,
|
||||
'can_manage' => true,
|
||||
'session_id' => 'cloud-engine-readiness',
|
||||
];
|
||||
$result = (new CloudEngineClient(null, 2.0))->request(
|
||||
'GET', '/internal/v1/health', $actor
|
||||
);
|
||||
return ($result['status'] ?? null) === 'ok'
|
||||
&& ($result['accepting_mutations'] ?? null) === true;
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,81 @@
|
||||
<?php
|
||||
|
||||
require dirname(__DIR__) . '/vendor/autoload.php';
|
||||
|
||||
use App\Services\Cloud\CloudEngineReadiness;
|
||||
|
||||
$assert = static function ($ok, string $message): void {
|
||||
if (!$ok) {
|
||||
throw new RuntimeException($message);
|
||||
}
|
||||
};
|
||||
|
||||
$now = 10.0;
|
||||
$attempts = 0;
|
||||
$sleeps = [];
|
||||
$ready = CloudEngineReadiness::wait(
|
||||
static function () use (&$attempts): bool {
|
||||
$attempts++;
|
||||
return $attempts === 3;
|
||||
},
|
||||
3.0,
|
||||
static function () use (&$now): float {
|
||||
return $now;
|
||||
},
|
||||
static function (int $microseconds) use (&$now, &$sleeps): void {
|
||||
$sleeps[] = $microseconds;
|
||||
$now += $microseconds / 1000000;
|
||||
},
|
||||
);
|
||||
$assert($ready && $attempts === 3 && $sleeps === [1000000, 1000000],
|
||||
'Readiness did not retry a transient startup race with bounded backoff.');
|
||||
|
||||
$now = 20.0;
|
||||
$attempts = 0;
|
||||
$sleeps = [];
|
||||
$ready = CloudEngineReadiness::wait(
|
||||
static function () use (&$attempts): bool {
|
||||
$attempts++;
|
||||
throw new RuntimeException('SECRET transport detail');
|
||||
},
|
||||
1.5,
|
||||
static function () use (&$now): float {
|
||||
return $now;
|
||||
},
|
||||
static function (int $microseconds) use (&$now, &$sleeps): void {
|
||||
$sleeps[] = $microseconds;
|
||||
$now += $microseconds / 1000000;
|
||||
},
|
||||
);
|
||||
$assert(!$ready && $attempts === 3 && $sleeps === [1000000, 500000],
|
||||
'Readiness did not stop at its monotonic deadline after probe failures.');
|
||||
|
||||
$root = dirname(__DIR__);
|
||||
$readiness = file_get_contents($root . '/app/Services/Cloud/CloudEngineReadiness.php');
|
||||
$lifecycle = file_get_contents($root . '/app/Command/CloudLifecycle.php');
|
||||
$lifecycleService = file_get_contents($root . '/app/Services/Cloud/CloudLifecycleService.php');
|
||||
$worker = file_get_contents($root . '/app/Command/CloudWorker.php');
|
||||
$heartbeat = file_get_contents($root . '/app/Services/Cloud/CloudRuntimeHeartbeat.php');
|
||||
$assert(str_contains($readiness, "new CloudEngineClient(null, 2.0)")
|
||||
&& str_contains($readiness, "'GET', '/internal/v1/health'")
|
||||
&& str_contains($readiness, "['accepting_mutations'] ?? null")
|
||||
&& str_contains($readiness, "'session_id' => 'cloud-engine-readiness'"),
|
||||
'The production gate is no longer a bounded signed engine health probe.');
|
||||
$assert(str_contains($heartbeat, "'engine_readiness' => '等待私有引擎就绪'")
|
||||
&& str_contains($heartbeat, "'engine_readiness_timeout'"),
|
||||
'Readiness observations are not allowlisted by runtime health telemetry.');
|
||||
$assert(str_contains($lifecycle, '$service->tick(200, $skipThisTick)')
|
||||
&& substr_count($lifecycleService, 'if (!$skipEngineObservations)') === 2
|
||||
&& str_contains($lifecycleService, "observeFailure('engine_readiness_timeout')"),
|
||||
'A readiness timeout can still run first-pass engine observations.');
|
||||
foreach ([$lifecycle, $worker] as $source) {
|
||||
$assert(str_contains($source, 'if (!$once)')
|
||||
&& str_contains($source, 'CloudEngineReadiness::wait()')
|
||||
&& str_contains($source, "engine_readiness_timeout"),
|
||||
'A long-running consumer lost its engine readiness gate.');
|
||||
}
|
||||
$assert(str_contains($worker, 'self::deliverNotifications($heartbeat)')
|
||||
&& str_contains($worker, '?callable $engineReadiness = null'),
|
||||
'An engine-readiness failure can still suppress DB-backed notifications.');
|
||||
|
||||
echo "Engine consumer readiness checks passed.\n";
|
||||
Loading…
Reference in New Issue