You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
virt/web/app/Services/Cloud/CloudRuntimeHeartbeat.php

376 lines
16 KiB
PHP

<?php
namespace App\Services\Cloud;
/** Best-effort local observations, never a lease or evidence that a process is alive. */
final class CloudRuntimeHeartbeat
{
private const KINDS = ['normal' => ['worker', '普通任务 worker'],
'policy' => ['policy', '策略任务 worker'],
'lifecycle' => ['lifecycle', '生命周期调度']];
private const PHASES = ['startup' => '启动', 'engine_readiness' => '等待私有引擎就绪',
'worker_tick' => '任务消费或核对',
'notification' => '邮件通知', 'lifecycle_tick' => '生命周期巡检',
'provision_expiry' => '开通账单过期处理', 'invoice_generation' => '续费账单生成',
'traffic_sync' => '流量同步',
'runtime_status_sync' => '实例状态同步',
'instance_reconcile' => '实例策略核对',
'idle' => '等待下一轮', 'stopped' => '已退出'];
private const ERRORS = ['engine_readiness_timeout', 'worker_tick_failed', 'notification_failed',
'lifecycle_tick_failed', 'provision_expiry_failed', 'invoice_generation_failed',
'traffic_sync_failed', 'runtime_status_sync_failed',
'runtime_status_sync_unconfigured', 'instance_reconcile_failed', 'task_failed',
'task_uncertain', 'runtime_error'];
private const SCAN_LIMIT = 512;
private const FILE_LIMIT = 4096;
private const RETENTION_SECONDS = 604800;
private array $record = [];
private ?string $file = null;
private function __construct(private string $directory, private ?\Closure $clock)
{
}
private static function directory(): string
{
return dirname(__DIR__, 3) . '/storage/cloud-runtime';
}
/** The optional directory and clock keep filesystem tests isolated from deployment state. */
public static function start(string $kind, bool $once = false,
?string $directory = null, ?\Closure $clock = null): self
{
$heartbeat = new self($directory ?? self::directory(), $clock);
try {
if (!isset(self::KINDS[$kind]) || !self::secureDirectory($heartbeat->directory, true)) {
return $heartbeat;
}
$now = $heartbeat->now();
$heartbeat->record = ['version' => 1, 'kind' => $kind,
'mode' => $once ? 'once' : 'daemon', 'state' => 'begin', 'phase' => 'startup',
'started_at' => $now, 'last_seen' => $now, 'last_success' => null,
'last_error' => null, 'last_error_at' => null, 'busy_since' => null,
'stopped_at' => null, 'unresolved_error' => false];
$heartbeat->file = $heartbeat->directory . '/' . $kind . '-'
. max(1, (int) getmypid()) . '-' . bin2hex(random_bytes(12)) . '.json';
$heartbeat->cleanup($now);
$heartbeat->persist();
} catch (\Throwable $ignored) {
// Observability must not stop queue consumption or lifecycle enforcement.
}
return $heartbeat;
}
public function busy(string $phase): void
{
$this->observe('busy', isset(self::PHASES[$phase]) ? $phase : 'worker_tick');
}
public function success(): void
{
$this->observe('idle', 'idle');
}
public function error(string $code): void
{
$this->observe('error', null, in_array($code, self::ERRORS, true) ? $code : 'runtime_error');
}
public function stopped(): void
{
$discardReadinessFailure = ($this->record['mode'] ?? null) === 'daemon'
&& ($this->record['last_success'] ?? null) === null
&& ($this->record['last_error'] ?? null) === 'engine_readiness_timeout';
$this->observe('stopped', 'stopped');
if ($discardReadinessFailure) {
$this->discardCurrentFile();
}
}
private function now(): int
{
return max(1, $this->clock === null ? time() : (int) ($this->clock)());
}
private function observe(string $state, ?string $phase, ?string $error = null): void
{
try {
if ($this->file === null) {
return;
}
$now = $this->now();
// Lifecycle may visit thousands of instances; refresh an unchanged
// phase at most once every 15 seconds, without changing its start.
if ($state === 'busy' && $this->record['state'] === 'busy'
&& $phase === $this->record['phase'] && $now - $this->record['last_seen'] < 15) {
return;
}
if ($state === 'busy' && ($this->record['state'] !== 'busy'
|| $phase !== $this->record['phase'])) {
$this->record['busy_since'] = $now;
} elseif ($state !== 'busy') {
$this->record['busy_since'] = null;
}
$this->record['state'] = $state;
$this->record['phase'] = $phase ?? $this->record['phase'];
$this->record['last_seen'] = $now;
if ($state === 'idle') {
$this->record['last_success'] = $now;
$this->record['unresolved_error'] = false;
}
if ($error !== null) {
$this->record['last_error'] = $error;
$this->record['last_error_at'] = $now;
$this->record['unresolved_error'] = true;
}
if ($state === 'stopped') {
$this->record['stopped_at'] = $now;
}
$this->persist();
} catch (\Throwable $ignored) {
}
}
private static function secureDirectory(string $directory, bool $create = false): bool
{
if ($directory === '' || $directory[0] !== '/' || str_contains($directory, "\0")) {
return false;
}
if ($create && !file_exists($directory) && !is_link($directory)) {
@mkdir($directory, 0700, true);
}
clearstatcache(true, $directory);
$stat = @lstat($directory);
return is_array($stat) && ($stat['mode'] & 0170000) === 0040000
&& ($stat['mode'] & 0777) === 0700
&& (!function_exists('posix_geteuid') || $stat['uid'] === posix_geteuid());
}
private static function privateFile(array $stat): bool
{
return ($stat['mode'] & 0170000) === 0100000 && ($stat['mode'] & 0777) === 0600
&& $stat['nlink'] === 1
&& (!function_exists('posix_geteuid') || $stat['uid'] === posix_geteuid());
}
/** Avoid unbounded restart-churn files before a worker has ever run. */
private function discardCurrentFile(): void
{
$path = $this->file;
$this->file = null;
if ($path === null || !self::secureDirectory($this->directory)) {
return;
}
$stat = @lstat($path);
if (is_array($stat) && self::privateFile($stat)) {
@unlink($path);
}
}
private function persist(): void
{
$temporary = null;
$handle = null;
try {
if ($this->file === null || !self::secureDirectory($this->directory)) {
return;
}
$temporary = $this->file . '.tmp';
$handle = @fopen($temporary, 'x+b');
if ($handle === false) {
// Do not remove a pre-existing path, even if it blocks telemetry.
$temporary = null;
return;
}
if (!@chmod($temporary, 0600)) {
return;
}
$encoded = json_encode($this->record, JSON_THROW_ON_ERROR);
if (@fwrite($handle, $encoded) !== strlen($encoded) || !@fflush($handle)) {
return;
}
fclose($handle);
$handle = null;
@rename($temporary, $this->file);
} catch (\Throwable $ignored) {
} finally {
if (is_resource($handle)) {
fclose($handle);
}
if ($temporary !== null) {
@unlink($temporary);
}
}
}
/** Only startup prunes bounded, private heartbeat files older than seven days. */
private function cleanup(int $now): void
{
$scanned = 0;
$removed = 0;
foreach (new \FilesystemIterator($this->directory, \FilesystemIterator::SKIP_DOTS) as $entry) {
if (++$scanned > self::SCAN_LIMIT || $removed >= 32) {
break;
}
if (preg_match('/^(normal|policy|lifecycle)-[1-9][0-9]*-[a-f0-9]{24}\.json(?:\.tmp)?$/D',
$entry->getFilename()) !== 1) {
continue;
}
$stat = @lstat($entry->getPathname());
if (is_array($stat) && self::privateFile($stat)
&& $stat['mtime'] < $now - self::RETENTION_SECONDS) {
if (@unlink($entry->getPathname())) {
$removed++;
}
}
}
}
private static function readRecord(string $path, string $kind, int $now): ?array
{
$stat = @lstat($path);
if (!is_array($stat) || !self::privateFile($stat) || $stat['size'] > self::FILE_LIMIT) {
return null;
}
$handle = @fopen($path, 'rb');
if ($handle === false) {
return null;
}
try {
$opened = fstat($handle);
if (!is_array($opened) || !self::privateFile($opened)
|| $opened['ino'] !== $stat['ino'] || $opened['dev'] !== $stat['dev']) {
return null;
}
$raw = stream_get_contents($handle, self::FILE_LIMIT + 1);
} finally {
fclose($handle);
}
$record = is_string($raw) && strlen($raw) <= self::FILE_LIMIT ? json_decode($raw, true) : null;
if (!is_array($record) || ($record['version'] ?? null) !== 1
|| ($record['kind'] ?? null) !== $kind
|| !in_array($record['mode'] ?? null, ['once', 'daemon'], true)
|| !in_array($record['state'] ?? null, ['begin', 'busy', 'idle', 'error', 'stopped'], true)
|| !is_bool($record['unresolved_error'] ?? null)
|| !isset(self::PHASES[$record['phase'] ?? ''])
|| !is_int($record['last_seen'] ?? null) || $record['last_seen'] < 1
|| $record['last_seen'] > $now + 5) {
return null;
}
if (!is_int($record['started_at'] ?? null) || $record['started_at'] < 1
|| $record['started_at'] > $record['last_seen']) {
return null;
}
foreach (['last_success', 'last_error_at', 'busy_since', 'stopped_at'] as $field) {
if (!array_key_exists($field, $record)) {
return null;
}
if (($record[$field] ?? null) !== null
&& (!is_int($record[$field]) || $record[$field] < 1 || $record[$field] > $record['last_seen'])) {
return null;
}
}
if (($record['last_error'] ?? null) !== null
&& !in_array($record['last_error'], self::ERRORS, true)) {
return null;
}
if (($record['state'] === 'idle' && $record['last_success'] !== $record['last_seen'])
|| ($record['state'] === 'busy' && $record['busy_since'] === null)
|| ($record['state'] === 'stopped' && $record['stopped_at'] !== $record['last_seen'])
|| ($record['state'] === 'error' && (($record['last_error'] ?? null) === null
|| $record['last_error_at'] !== $record['last_seen']))) {
return null;
}
return $record;
}
/** Read-only, bounded public checks: no paths, PID, process token or raw errors. */
public static function snapshot(?string $directory = null, ?int $now = null): array
{
$now ??= time();
$directory ??= self::directory();
$records = array_fill_keys(array_keys(self::KINDS), []);
$available = false;
$incomplete = false;
try {
$available = self::secureDirectory($directory);
if ($available) {
$scanned = 0;
foreach (new \FilesystemIterator($directory, \FilesystemIterator::SKIP_DOTS) as $entry) {
if (++$scanned > self::SCAN_LIMIT) {
$incomplete = true;
break;
}
if (preg_match('/^(normal|policy|lifecycle)-[1-9][0-9]*-[a-f0-9]{24}\.json$/D',
$entry->getFilename(), $match) !== 1) {
continue;
}
$record = self::readRecord($entry->getPathname(), $match[1], $now);
if ($record !== null) {
$records[$match[1]][] = $record;
} else {
$incomplete = true;
}
}
}
} catch (\Throwable $ignored) {
$available = false;
// A partially completed directory read cannot prove healthy coverage.
$records = array_fill_keys(array_keys(self::KINDS), []);
}
$checks = [];
foreach (self::KINDS as $kind => [$key, $label]) {
$rows = $records[$kind];
usort($rows, static fn ($left, $right) => $right['last_seen'] <=> $left['last_seen']);
$latest = $rows[0] ?? null;
$active = array_values(array_filter($rows, static function ($row) use ($kind, $now): bool {
// HTTP work can last 900 seconds; policy calls have a 120s
// timeout. SMTP/lifecycle also need a generous observation window.
$grace = $row['state'] === 'busy' ? ($kind === 'policy' ? 180 : 1200) : 90;
return $row['mode'] === 'daemon' && $row['state'] !== 'stopped'
&& $now - $row['last_seen'] <= $grace;
}));
$status = 'unknown';
$message = $available ? '尚无常驻运行记录;请确认服务已重启为当前版本。'
: '本地运行记录不可用;不能据此确认服务是否运行。';
if ($active !== []) {
$status = 'ok';
$message = '最近一轮已完成;心跳不代表所有任务或宿主机均正常。';
foreach ($active as $row) {
if ($row['state'] === 'error' || $row['unresolved_error']) {
$status = 'warning';
$message = '最近一轮存在异常或部分失败,请结合任务记录核查。';
break;
}
if ($row['state'] === 'busy' || $row['state'] === 'begin') {
$status = 'busy';
$message = '最近上报阶段:' . self::PHASES[$row['phase']]
. ';阶段内没有独立存活探测。';
}
}
} elseif ($latest !== null) {
if ($latest['mode'] === 'once') {
$message = '仅有单次执行记录,不能确认常驻服务运行。';
} elseif ($latest['state'] === 'stopped') {
$status = 'warning';
$message = '已报告退出,未观测到新的常驻心跳。';
} else {
$status = 'warning';
$message = '运行记录已过期,需核查;不能据此确认进程已停止。';
}
}
if ($incomplete) {
$status = 'warning';
$message = '部分运行记录无效或超过读取上限,状态需要核查。';
}
$successes = array_filter(array_column($rows, 'last_success'), 'is_int');
$checks[] = ['key' => $key, 'label' => $label, 'status' => $status,
'message' => $message, 'checked_at' => $now,
'last_seen_at' => $latest['last_seen'] ?? null,
'last_success_at' => $successes === [] ? null : max($successes),
'active_count' => count($active)];
}
return $checks;
}
}