You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
376 lines
16 KiB
PHP
376 lines
16 KiB
PHP
<?php
|
|
namespace App\Services\Cloud;
|
|
|
|
/** Best-effort local observations, never a lease or evidence that a process is alive. */
|
|
final class CloudRuntimeHeartbeat
|
|
{
|
|
private const KINDS = ['normal' => ['worker', '普通任务 worker'],
|
|
'policy' => ['policy', '策略任务 worker'],
|
|
'lifecycle' => ['lifecycle', '生命周期调度']];
|
|
private const PHASES = ['startup' => '启动', 'engine_readiness' => '等待私有引擎就绪',
|
|
'worker_tick' => '任务消费或核对',
|
|
'notification' => '邮件通知', 'lifecycle_tick' => '生命周期巡检',
|
|
'provision_expiry' => '开通账单过期处理', 'invoice_generation' => '续费账单生成',
|
|
'traffic_sync' => '流量同步',
|
|
'runtime_status_sync' => '实例状态同步',
|
|
'instance_reconcile' => '实例策略核对',
|
|
'idle' => '等待下一轮', 'stopped' => '已退出'];
|
|
private const ERRORS = ['engine_readiness_timeout', 'worker_tick_failed', 'notification_failed',
|
|
'lifecycle_tick_failed', 'provision_expiry_failed', 'invoice_generation_failed',
|
|
'traffic_sync_failed', 'runtime_status_sync_failed',
|
|
'runtime_status_sync_unconfigured', 'instance_reconcile_failed', 'task_failed',
|
|
'task_uncertain', 'runtime_error'];
|
|
private const SCAN_LIMIT = 512;
|
|
private const FILE_LIMIT = 4096;
|
|
private const RETENTION_SECONDS = 604800;
|
|
|
|
private array $record = [];
|
|
private ?string $file = null;
|
|
|
|
private function __construct(private string $directory, private ?\Closure $clock)
|
|
{
|
|
}
|
|
|
|
private static function directory(): string
|
|
{
|
|
return dirname(__DIR__, 3) . '/storage/cloud-runtime';
|
|
}
|
|
|
|
/** The optional directory and clock keep filesystem tests isolated from deployment state. */
|
|
public static function start(string $kind, bool $once = false,
|
|
?string $directory = null, ?\Closure $clock = null): self
|
|
{
|
|
$heartbeat = new self($directory ?? self::directory(), $clock);
|
|
try {
|
|
if (!isset(self::KINDS[$kind]) || !self::secureDirectory($heartbeat->directory, true)) {
|
|
return $heartbeat;
|
|
}
|
|
$now = $heartbeat->now();
|
|
$heartbeat->record = ['version' => 1, 'kind' => $kind,
|
|
'mode' => $once ? 'once' : 'daemon', 'state' => 'begin', 'phase' => 'startup',
|
|
'started_at' => $now, 'last_seen' => $now, 'last_success' => null,
|
|
'last_error' => null, 'last_error_at' => null, 'busy_since' => null,
|
|
'stopped_at' => null, 'unresolved_error' => false];
|
|
$heartbeat->file = $heartbeat->directory . '/' . $kind . '-'
|
|
. max(1, (int) getmypid()) . '-' . bin2hex(random_bytes(12)) . '.json';
|
|
$heartbeat->cleanup($now);
|
|
$heartbeat->persist();
|
|
} catch (\Throwable $ignored) {
|
|
// Observability must not stop queue consumption or lifecycle enforcement.
|
|
}
|
|
return $heartbeat;
|
|
}
|
|
|
|
public function busy(string $phase): void
|
|
{
|
|
$this->observe('busy', isset(self::PHASES[$phase]) ? $phase : 'worker_tick');
|
|
}
|
|
|
|
public function success(): void
|
|
{
|
|
$this->observe('idle', 'idle');
|
|
}
|
|
|
|
public function error(string $code): void
|
|
{
|
|
$this->observe('error', null, in_array($code, self::ERRORS, true) ? $code : 'runtime_error');
|
|
}
|
|
|
|
public function stopped(): void
|
|
{
|
|
$discardReadinessFailure = ($this->record['mode'] ?? null) === 'daemon'
|
|
&& ($this->record['last_success'] ?? null) === null
|
|
&& ($this->record['last_error'] ?? null) === 'engine_readiness_timeout';
|
|
$this->observe('stopped', 'stopped');
|
|
if ($discardReadinessFailure) {
|
|
$this->discardCurrentFile();
|
|
}
|
|
}
|
|
|
|
private function now(): int
|
|
{
|
|
return max(1, $this->clock === null ? time() : (int) ($this->clock)());
|
|
}
|
|
|
|
private function observe(string $state, ?string $phase, ?string $error = null): void
|
|
{
|
|
try {
|
|
if ($this->file === null) {
|
|
return;
|
|
}
|
|
$now = $this->now();
|
|
// Lifecycle may visit thousands of instances; refresh an unchanged
|
|
// phase at most once every 15 seconds, without changing its start.
|
|
if ($state === 'busy' && $this->record['state'] === 'busy'
|
|
&& $phase === $this->record['phase'] && $now - $this->record['last_seen'] < 15) {
|
|
return;
|
|
}
|
|
if ($state === 'busy' && ($this->record['state'] !== 'busy'
|
|
|| $phase !== $this->record['phase'])) {
|
|
$this->record['busy_since'] = $now;
|
|
} elseif ($state !== 'busy') {
|
|
$this->record['busy_since'] = null;
|
|
}
|
|
$this->record['state'] = $state;
|
|
$this->record['phase'] = $phase ?? $this->record['phase'];
|
|
$this->record['last_seen'] = $now;
|
|
if ($state === 'idle') {
|
|
$this->record['last_success'] = $now;
|
|
$this->record['unresolved_error'] = false;
|
|
}
|
|
if ($error !== null) {
|
|
$this->record['last_error'] = $error;
|
|
$this->record['last_error_at'] = $now;
|
|
$this->record['unresolved_error'] = true;
|
|
}
|
|
if ($state === 'stopped') {
|
|
$this->record['stopped_at'] = $now;
|
|
}
|
|
$this->persist();
|
|
} catch (\Throwable $ignored) {
|
|
}
|
|
}
|
|
|
|
private static function secureDirectory(string $directory, bool $create = false): bool
|
|
{
|
|
if ($directory === '' || $directory[0] !== '/' || str_contains($directory, "\0")) {
|
|
return false;
|
|
}
|
|
if ($create && !file_exists($directory) && !is_link($directory)) {
|
|
@mkdir($directory, 0700, true);
|
|
}
|
|
clearstatcache(true, $directory);
|
|
$stat = @lstat($directory);
|
|
return is_array($stat) && ($stat['mode'] & 0170000) === 0040000
|
|
&& ($stat['mode'] & 0777) === 0700
|
|
&& (!function_exists('posix_geteuid') || $stat['uid'] === posix_geteuid());
|
|
}
|
|
|
|
private static function privateFile(array $stat): bool
|
|
{
|
|
return ($stat['mode'] & 0170000) === 0100000 && ($stat['mode'] & 0777) === 0600
|
|
&& $stat['nlink'] === 1
|
|
&& (!function_exists('posix_geteuid') || $stat['uid'] === posix_geteuid());
|
|
}
|
|
|
|
/** Avoid unbounded restart-churn files before a worker has ever run. */
|
|
private function discardCurrentFile(): void
|
|
{
|
|
$path = $this->file;
|
|
$this->file = null;
|
|
if ($path === null || !self::secureDirectory($this->directory)) {
|
|
return;
|
|
}
|
|
$stat = @lstat($path);
|
|
if (is_array($stat) && self::privateFile($stat)) {
|
|
@unlink($path);
|
|
}
|
|
}
|
|
|
|
private function persist(): void
|
|
{
|
|
$temporary = null;
|
|
$handle = null;
|
|
try {
|
|
if ($this->file === null || !self::secureDirectory($this->directory)) {
|
|
return;
|
|
}
|
|
$temporary = $this->file . '.tmp';
|
|
$handle = @fopen($temporary, 'x+b');
|
|
if ($handle === false) {
|
|
// Do not remove a pre-existing path, even if it blocks telemetry.
|
|
$temporary = null;
|
|
return;
|
|
}
|
|
if (!@chmod($temporary, 0600)) {
|
|
return;
|
|
}
|
|
$encoded = json_encode($this->record, JSON_THROW_ON_ERROR);
|
|
if (@fwrite($handle, $encoded) !== strlen($encoded) || !@fflush($handle)) {
|
|
return;
|
|
}
|
|
fclose($handle);
|
|
$handle = null;
|
|
@rename($temporary, $this->file);
|
|
} catch (\Throwable $ignored) {
|
|
} finally {
|
|
if (is_resource($handle)) {
|
|
fclose($handle);
|
|
}
|
|
if ($temporary !== null) {
|
|
@unlink($temporary);
|
|
}
|
|
}
|
|
}
|
|
|
|
/** Only startup prunes bounded, private heartbeat files older than seven days. */
|
|
private function cleanup(int $now): void
|
|
{
|
|
$scanned = 0;
|
|
$removed = 0;
|
|
foreach (new \FilesystemIterator($this->directory, \FilesystemIterator::SKIP_DOTS) as $entry) {
|
|
if (++$scanned > self::SCAN_LIMIT || $removed >= 32) {
|
|
break;
|
|
}
|
|
if (preg_match('/^(normal|policy|lifecycle)-[1-9][0-9]*-[a-f0-9]{24}\.json(?:\.tmp)?$/D',
|
|
$entry->getFilename()) !== 1) {
|
|
continue;
|
|
}
|
|
$stat = @lstat($entry->getPathname());
|
|
if (is_array($stat) && self::privateFile($stat)
|
|
&& $stat['mtime'] < $now - self::RETENTION_SECONDS) {
|
|
if (@unlink($entry->getPathname())) {
|
|
$removed++;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
private static function readRecord(string $path, string $kind, int $now): ?array
|
|
{
|
|
$stat = @lstat($path);
|
|
if (!is_array($stat) || !self::privateFile($stat) || $stat['size'] > self::FILE_LIMIT) {
|
|
return null;
|
|
}
|
|
$handle = @fopen($path, 'rb');
|
|
if ($handle === false) {
|
|
return null;
|
|
}
|
|
try {
|
|
$opened = fstat($handle);
|
|
if (!is_array($opened) || !self::privateFile($opened)
|
|
|| $opened['ino'] !== $stat['ino'] || $opened['dev'] !== $stat['dev']) {
|
|
return null;
|
|
}
|
|
$raw = stream_get_contents($handle, self::FILE_LIMIT + 1);
|
|
} finally {
|
|
fclose($handle);
|
|
}
|
|
$record = is_string($raw) && strlen($raw) <= self::FILE_LIMIT ? json_decode($raw, true) : null;
|
|
if (!is_array($record) || ($record['version'] ?? null) !== 1
|
|
|| ($record['kind'] ?? null) !== $kind
|
|
|| !in_array($record['mode'] ?? null, ['once', 'daemon'], true)
|
|
|| !in_array($record['state'] ?? null, ['begin', 'busy', 'idle', 'error', 'stopped'], true)
|
|
|| !is_bool($record['unresolved_error'] ?? null)
|
|
|| !isset(self::PHASES[$record['phase'] ?? ''])
|
|
|| !is_int($record['last_seen'] ?? null) || $record['last_seen'] < 1
|
|
|| $record['last_seen'] > $now + 5) {
|
|
return null;
|
|
}
|
|
if (!is_int($record['started_at'] ?? null) || $record['started_at'] < 1
|
|
|| $record['started_at'] > $record['last_seen']) {
|
|
return null;
|
|
}
|
|
foreach (['last_success', 'last_error_at', 'busy_since', 'stopped_at'] as $field) {
|
|
if (!array_key_exists($field, $record)) {
|
|
return null;
|
|
}
|
|
if (($record[$field] ?? null) !== null
|
|
&& (!is_int($record[$field]) || $record[$field] < 1 || $record[$field] > $record['last_seen'])) {
|
|
return null;
|
|
}
|
|
}
|
|
if (($record['last_error'] ?? null) !== null
|
|
&& !in_array($record['last_error'], self::ERRORS, true)) {
|
|
return null;
|
|
}
|
|
if (($record['state'] === 'idle' && $record['last_success'] !== $record['last_seen'])
|
|
|| ($record['state'] === 'busy' && $record['busy_since'] === null)
|
|
|| ($record['state'] === 'stopped' && $record['stopped_at'] !== $record['last_seen'])
|
|
|| ($record['state'] === 'error' && (($record['last_error'] ?? null) === null
|
|
|| $record['last_error_at'] !== $record['last_seen']))) {
|
|
return null;
|
|
}
|
|
return $record;
|
|
}
|
|
|
|
/** Read-only, bounded public checks: no paths, PID, process token or raw errors. */
|
|
public static function snapshot(?string $directory = null, ?int $now = null): array
|
|
{
|
|
$now ??= time();
|
|
$directory ??= self::directory();
|
|
$records = array_fill_keys(array_keys(self::KINDS), []);
|
|
$available = false;
|
|
$incomplete = false;
|
|
try {
|
|
$available = self::secureDirectory($directory);
|
|
if ($available) {
|
|
$scanned = 0;
|
|
foreach (new \FilesystemIterator($directory, \FilesystemIterator::SKIP_DOTS) as $entry) {
|
|
if (++$scanned > self::SCAN_LIMIT) {
|
|
$incomplete = true;
|
|
break;
|
|
}
|
|
if (preg_match('/^(normal|policy|lifecycle)-[1-9][0-9]*-[a-f0-9]{24}\.json$/D',
|
|
$entry->getFilename(), $match) !== 1) {
|
|
continue;
|
|
}
|
|
$record = self::readRecord($entry->getPathname(), $match[1], $now);
|
|
if ($record !== null) {
|
|
$records[$match[1]][] = $record;
|
|
} else {
|
|
$incomplete = true;
|
|
}
|
|
}
|
|
}
|
|
} catch (\Throwable $ignored) {
|
|
$available = false;
|
|
// A partially completed directory read cannot prove healthy coverage.
|
|
$records = array_fill_keys(array_keys(self::KINDS), []);
|
|
}
|
|
$checks = [];
|
|
foreach (self::KINDS as $kind => [$key, $label]) {
|
|
$rows = $records[$kind];
|
|
usort($rows, static fn ($left, $right) => $right['last_seen'] <=> $left['last_seen']);
|
|
$latest = $rows[0] ?? null;
|
|
$active = array_values(array_filter($rows, static function ($row) use ($kind, $now): bool {
|
|
// HTTP work can last 900 seconds; policy calls have a 120s
|
|
// timeout. SMTP/lifecycle also need a generous observation window.
|
|
$grace = $row['state'] === 'busy' ? ($kind === 'policy' ? 180 : 1200) : 90;
|
|
return $row['mode'] === 'daemon' && $row['state'] !== 'stopped'
|
|
&& $now - $row['last_seen'] <= $grace;
|
|
}));
|
|
$status = 'unknown';
|
|
$message = $available ? '尚无常驻运行记录;请确认服务已重启为当前版本。'
|
|
: '本地运行记录不可用;不能据此确认服务是否运行。';
|
|
if ($active !== []) {
|
|
$status = 'ok';
|
|
$message = '最近一轮已完成;心跳不代表所有任务或宿主机均正常。';
|
|
foreach ($active as $row) {
|
|
if ($row['state'] === 'error' || $row['unresolved_error']) {
|
|
$status = 'warning';
|
|
$message = '最近一轮存在异常或部分失败,请结合任务记录核查。';
|
|
break;
|
|
}
|
|
if ($row['state'] === 'busy' || $row['state'] === 'begin') {
|
|
$status = 'busy';
|
|
$message = '最近上报阶段:' . self::PHASES[$row['phase']]
|
|
. ';阶段内没有独立存活探测。';
|
|
}
|
|
}
|
|
} elseif ($latest !== null) {
|
|
if ($latest['mode'] === 'once') {
|
|
$message = '仅有单次执行记录,不能确认常驻服务运行。';
|
|
} elseif ($latest['state'] === 'stopped') {
|
|
$status = 'warning';
|
|
$message = '已报告退出,未观测到新的常驻心跳。';
|
|
} else {
|
|
$status = 'warning';
|
|
$message = '运行记录已过期,需核查;不能据此确认进程已停止。';
|
|
}
|
|
}
|
|
if ($incomplete) {
|
|
$status = 'warning';
|
|
$message = '部分运行记录无效或超过读取上限,状态需要核查。';
|
|
}
|
|
$successes = array_filter(array_column($rows, 'last_success'), 'is_int');
|
|
$checks[] = ['key' => $key, 'label' => $label, 'status' => $status,
|
|
'message' => $message, 'checked_at' => $now,
|
|
'last_seen_at' => $latest['last_seen'] ?? null,
|
|
'last_success_at' => $successes === [] ? null : max($successes),
|
|
'active_count' => count($active)];
|
|
}
|
|
return $checks;
|
|
}
|
|
}
|