['worker', '普通任务 worker'], 'policy' => ['policy', '策略任务 worker'], 'lifecycle' => ['lifecycle', '生命周期调度']]; private const PHASES = ['startup' => '启动', 'engine_readiness' => '等待私有引擎就绪', 'worker_tick' => '任务消费或核对', 'notification' => '邮件通知', 'lifecycle_tick' => '生命周期巡检', 'provision_expiry' => '开通账单过期处理', 'invoice_generation' => '续费账单生成', 'traffic_sync' => '流量同步', 'runtime_status_sync' => '实例状态同步', 'instance_reconcile' => '实例策略核对', 'idle' => '等待下一轮', 'stopped' => '已退出']; private const ERRORS = ['engine_readiness_timeout', 'worker_tick_failed', 'notification_failed', 'lifecycle_tick_failed', 'provision_expiry_failed', 'invoice_generation_failed', 'traffic_sync_failed', 'runtime_status_sync_failed', 'runtime_status_sync_unconfigured', 'instance_reconcile_failed', 'task_failed', 'task_uncertain', 'runtime_error']; private const SCAN_LIMIT = 512; private const FILE_LIMIT = 4096; private const RETENTION_SECONDS = 604800; private array $record = []; private ?string $file = null; private function __construct(private string $directory, private ?\Closure $clock) { } private static function directory(): string { return dirname(__DIR__, 3) . '/storage/cloud-runtime'; } /** The optional directory and clock keep filesystem tests isolated from deployment state. */ public static function start(string $kind, bool $once = false, ?string $directory = null, ?\Closure $clock = null): self { $heartbeat = new self($directory ?? self::directory(), $clock); try { if (!isset(self::KINDS[$kind]) || !self::secureDirectory($heartbeat->directory, true)) { return $heartbeat; } $now = $heartbeat->now(); $heartbeat->record = ['version' => 1, 'kind' => $kind, 'mode' => $once ? 'once' : 'daemon', 'state' => 'begin', 'phase' => 'startup', 'started_at' => $now, 'last_seen' => $now, 'last_success' => null, 'last_error' => null, 'last_error_at' => null, 'busy_since' => null, 'stopped_at' => null, 'unresolved_error' => false]; $heartbeat->file = $heartbeat->directory . '/' . $kind . '-' . max(1, (int) getmypid()) . '-' . bin2hex(random_bytes(12)) . '.json'; $heartbeat->cleanup($now); $heartbeat->persist(); } catch (\Throwable $ignored) { // Observability must not stop queue consumption or lifecycle enforcement. } return $heartbeat; } public function busy(string $phase): void { $this->observe('busy', isset(self::PHASES[$phase]) ? $phase : 'worker_tick'); } public function success(): void { $this->observe('idle', 'idle'); } public function error(string $code): void { $this->observe('error', null, in_array($code, self::ERRORS, true) ? $code : 'runtime_error'); } public function stopped(): void { $discardReadinessFailure = ($this->record['mode'] ?? null) === 'daemon' && ($this->record['last_success'] ?? null) === null && ($this->record['last_error'] ?? null) === 'engine_readiness_timeout'; $this->observe('stopped', 'stopped'); if ($discardReadinessFailure) { $this->discardCurrentFile(); } } private function now(): int { return max(1, $this->clock === null ? time() : (int) ($this->clock)()); } private function observe(string $state, ?string $phase, ?string $error = null): void { try { if ($this->file === null) { return; } $now = $this->now(); // Lifecycle may visit thousands of instances; refresh an unchanged // phase at most once every 15 seconds, without changing its start. if ($state === 'busy' && $this->record['state'] === 'busy' && $phase === $this->record['phase'] && $now - $this->record['last_seen'] < 15) { return; } if ($state === 'busy' && ($this->record['state'] !== 'busy' || $phase !== $this->record['phase'])) { $this->record['busy_since'] = $now; } elseif ($state !== 'busy') { $this->record['busy_since'] = null; } $this->record['state'] = $state; $this->record['phase'] = $phase ?? $this->record['phase']; $this->record['last_seen'] = $now; if ($state === 'idle') { $this->record['last_success'] = $now; $this->record['unresolved_error'] = false; } if ($error !== null) { $this->record['last_error'] = $error; $this->record['last_error_at'] = $now; $this->record['unresolved_error'] = true; } if ($state === 'stopped') { $this->record['stopped_at'] = $now; } $this->persist(); } catch (\Throwable $ignored) { } } private static function secureDirectory(string $directory, bool $create = false): bool { if ($directory === '' || $directory[0] !== '/' || str_contains($directory, "\0")) { return false; } if ($create && !file_exists($directory) && !is_link($directory)) { @mkdir($directory, 0700, true); } clearstatcache(true, $directory); $stat = @lstat($directory); return is_array($stat) && ($stat['mode'] & 0170000) === 0040000 && ($stat['mode'] & 0777) === 0700 && (!function_exists('posix_geteuid') || $stat['uid'] === posix_geteuid()); } private static function privateFile(array $stat): bool { return ($stat['mode'] & 0170000) === 0100000 && ($stat['mode'] & 0777) === 0600 && $stat['nlink'] === 1 && (!function_exists('posix_geteuid') || $stat['uid'] === posix_geteuid()); } /** Avoid unbounded restart-churn files before a worker has ever run. */ private function discardCurrentFile(): void { $path = $this->file; $this->file = null; if ($path === null || !self::secureDirectory($this->directory)) { return; } $stat = @lstat($path); if (is_array($stat) && self::privateFile($stat)) { @unlink($path); } } private function persist(): void { $temporary = null; $handle = null; try { if ($this->file === null || !self::secureDirectory($this->directory)) { return; } $temporary = $this->file . '.tmp'; $handle = @fopen($temporary, 'x+b'); if ($handle === false) { // Do not remove a pre-existing path, even if it blocks telemetry. $temporary = null; return; } if (!@chmod($temporary, 0600)) { return; } $encoded = json_encode($this->record, JSON_THROW_ON_ERROR); if (@fwrite($handle, $encoded) !== strlen($encoded) || !@fflush($handle)) { return; } fclose($handle); $handle = null; @rename($temporary, $this->file); } catch (\Throwable $ignored) { } finally { if (is_resource($handle)) { fclose($handle); } if ($temporary !== null) { @unlink($temporary); } } } /** Only startup prunes bounded, private heartbeat files older than seven days. */ private function cleanup(int $now): void { $scanned = 0; $removed = 0; foreach (new \FilesystemIterator($this->directory, \FilesystemIterator::SKIP_DOTS) as $entry) { if (++$scanned > self::SCAN_LIMIT || $removed >= 32) { break; } if (preg_match('/^(normal|policy|lifecycle)-[1-9][0-9]*-[a-f0-9]{24}\.json(?:\.tmp)?$/D', $entry->getFilename()) !== 1) { continue; } $stat = @lstat($entry->getPathname()); if (is_array($stat) && self::privateFile($stat) && $stat['mtime'] < $now - self::RETENTION_SECONDS) { if (@unlink($entry->getPathname())) { $removed++; } } } } private static function readRecord(string $path, string $kind, int $now): ?array { $stat = @lstat($path); if (!is_array($stat) || !self::privateFile($stat) || $stat['size'] > self::FILE_LIMIT) { return null; } $handle = @fopen($path, 'rb'); if ($handle === false) { return null; } try { $opened = fstat($handle); if (!is_array($opened) || !self::privateFile($opened) || $opened['ino'] !== $stat['ino'] || $opened['dev'] !== $stat['dev']) { return null; } $raw = stream_get_contents($handle, self::FILE_LIMIT + 1); } finally { fclose($handle); } $record = is_string($raw) && strlen($raw) <= self::FILE_LIMIT ? json_decode($raw, true) : null; if (!is_array($record) || ($record['version'] ?? null) !== 1 || ($record['kind'] ?? null) !== $kind || !in_array($record['mode'] ?? null, ['once', 'daemon'], true) || !in_array($record['state'] ?? null, ['begin', 'busy', 'idle', 'error', 'stopped'], true) || !is_bool($record['unresolved_error'] ?? null) || !isset(self::PHASES[$record['phase'] ?? '']) || !is_int($record['last_seen'] ?? null) || $record['last_seen'] < 1 || $record['last_seen'] > $now + 5) { return null; } if (!is_int($record['started_at'] ?? null) || $record['started_at'] < 1 || $record['started_at'] > $record['last_seen']) { return null; } foreach (['last_success', 'last_error_at', 'busy_since', 'stopped_at'] as $field) { if (!array_key_exists($field, $record)) { return null; } if (($record[$field] ?? null) !== null && (!is_int($record[$field]) || $record[$field] < 1 || $record[$field] > $record['last_seen'])) { return null; } } if (($record['last_error'] ?? null) !== null && !in_array($record['last_error'], self::ERRORS, true)) { return null; } if (($record['state'] === 'idle' && $record['last_success'] !== $record['last_seen']) || ($record['state'] === 'busy' && $record['busy_since'] === null) || ($record['state'] === 'stopped' && $record['stopped_at'] !== $record['last_seen']) || ($record['state'] === 'error' && (($record['last_error'] ?? null) === null || $record['last_error_at'] !== $record['last_seen']))) { return null; } return $record; } /** Read-only, bounded public checks: no paths, PID, process token or raw errors. */ public static function snapshot(?string $directory = null, ?int $now = null): array { $now ??= time(); $directory ??= self::directory(); $records = array_fill_keys(array_keys(self::KINDS), []); $available = false; $incomplete = false; try { $available = self::secureDirectory($directory); if ($available) { $scanned = 0; foreach (new \FilesystemIterator($directory, \FilesystemIterator::SKIP_DOTS) as $entry) { if (++$scanned > self::SCAN_LIMIT) { $incomplete = true; break; } if (preg_match('/^(normal|policy|lifecycle)-[1-9][0-9]*-[a-f0-9]{24}\.json$/D', $entry->getFilename(), $match) !== 1) { continue; } $record = self::readRecord($entry->getPathname(), $match[1], $now); if ($record !== null) { $records[$match[1]][] = $record; } else { $incomplete = true; } } } } catch (\Throwable $ignored) { $available = false; // A partially completed directory read cannot prove healthy coverage. $records = array_fill_keys(array_keys(self::KINDS), []); } $checks = []; foreach (self::KINDS as $kind => [$key, $label]) { $rows = $records[$kind]; usort($rows, static fn ($left, $right) => $right['last_seen'] <=> $left['last_seen']); $latest = $rows[0] ?? null; $active = array_values(array_filter($rows, static function ($row) use ($kind, $now): bool { // HTTP work can last 900 seconds; policy calls have a 120s // timeout. SMTP/lifecycle also need a generous observation window. $grace = $row['state'] === 'busy' ? ($kind === 'policy' ? 180 : 1200) : 90; return $row['mode'] === 'daemon' && $row['state'] !== 'stopped' && $now - $row['last_seen'] <= $grace; })); $status = 'unknown'; $message = $available ? '尚无常驻运行记录;请确认服务已重启为当前版本。' : '本地运行记录不可用;不能据此确认服务是否运行。'; if ($active !== []) { $status = 'ok'; $message = '最近一轮已完成;心跳不代表所有任务或宿主机均正常。'; foreach ($active as $row) { if ($row['state'] === 'error' || $row['unresolved_error']) { $status = 'warning'; $message = '最近一轮存在异常或部分失败,请结合任务记录核查。'; break; } if ($row['state'] === 'busy' || $row['state'] === 'begin') { $status = 'busy'; $message = '最近上报阶段:' . self::PHASES[$row['phase']] . ';阶段内没有独立存活探测。'; } } } elseif ($latest !== null) { if ($latest['mode'] === 'once') { $message = '仅有单次执行记录,不能确认常驻服务运行。'; } elseif ($latest['state'] === 'stopped') { $status = 'warning'; $message = '已报告退出,未观测到新的常驻心跳。'; } else { $status = 'warning'; $message = '运行记录已过期,需核查;不能据此确认进程已停止。'; } } if ($incomplete) { $status = 'warning'; $message = '部分运行记录无效或超过读取上限,状态需要核查。'; } $successes = array_filter(array_column($rows, 'last_success'), 'is_int'); $checks[] = ['key' => $key, 'label' => $label, 'status' => $status, 'message' => $message, 'checked_at' => $now, 'last_seen_at' => $latest['last_seen'] ?? null, 'last_success_at' => $successes === [] ? null : max($successes), 'active_count' => count($active)]; } return $checks; } }