json_decode(file_get_contents($file), true), $files($directory)); }; $removeTree = static function (string $directory) use (&$removeTree): void { foreach (new FilesystemIterator($directory, FilesystemIterator::SKIP_DOTS) as $entry) { if ($entry->isDir() && !$entry->isLink()) { $removeTree($entry->getPathname()); } else { unlink($entry->getPathname()); } } rmdir($directory); }; try { $missing = $testRoot . '/not-created'; $empty = $checks($missing); $assert(array_keys($empty) === ['worker', 'policy', 'lifecycle'] && array_unique(array_column($empty, 'status')) === ['unknown'] && !file_exists($missing), 'Missing snapshot must be unknown, cover all kinds, and never create runtime files.'); $directory = $testRoot . '/workers'; $normal = CloudRuntimeHeartbeat::start('normal', false, $directory, $clock); $policy = CloudRuntimeHeartbeat::start('policy', false, $directory, $clock); $policy2 = CloudRuntimeHeartbeat::start('policy', false, $directory, $clock); $assert(count($files($directory)) === 3, 'Concurrent worker identities overwrite one another.'); $assert((fileperms($directory) & 0777) === 0700, 'Runtime directory is not private.'); foreach ($files($directory) as $path) { $assert((fileperms($path) & 0777) === 0600, 'Heartbeat file is not private.'); } $assert($checks($directory)['worker']['status'] === 'busy' && $checks($directory)['policy']['active_count'] === 2, 'Starting workers or multiple policy reporters were not represented.'); $normal->busy('engine_readiness'); $assert(str_contains($checks($directory)['worker']['message'], '等待私有引擎就绪'), 'Engine readiness was not exposed as its own bounded startup phase.'); $normal->error('engine_readiness_timeout'); $assert($read($directory)[0]['last_error'] === 'engine_readiness_timeout', 'The fixed readiness timeout code was not retained by private telemetry.'); $normal->success(); $normal->busy('worker_tick'); $policy->busy('worker_tick'); $policy2->stopped(); $now += 121; $assert($checks($directory)['worker']['status'] === 'busy' && $checks($directory)['policy']['status'] === 'busy', 'Normal long calls or policy 120-second calls were incorrectly declared stale.'); $now += 100; $assert($checks($directory)['policy']['status'] === 'warning' && $checks($directory)['policy']['active_count'] === 0, 'Expired policy observations must request investigation without claiming a live process.'); $now += 700; $assert($checks($directory)['worker']['status'] === 'busy', 'A normal 900-second engine call was incorrectly declared stopped.'); $normal->busy('notification'); $now += 300; $assert(str_contains($checks($directory)['worker']['message'], '邮件通知'), 'SMTP did not receive its own observation stage and time window.'); $normal->success(); $assert($checks($directory)['worker']['status'] === 'ok' && $checks($directory)['worker']['last_success_at'] === $now, 'Completed cycle did not record its real success time.'); $successfulAt = $now; $now++; $normal->error('secret-password-token-must-not-leak'); $encoded = json_encode($read($directory), JSON_THROW_ON_ERROR); $assert(!str_contains($encoded, 'secret-password-token') && str_contains($encoded, 'runtime_error') && $checks($directory)['worker']['status'] === 'warning' && $checks($directory)['worker']['last_success_at'] === $successfulAt, 'Error was exposed verbatim, hidden, or incorrectly advanced success time.'); $normal->busy('worker_tick'); $assert($checks($directory)['worker']['status'] === 'warning', 'Starting a new pass erased the previous unresolved failure.'); $normal->success(); $assert($checks($directory)['worker']['status'] === 'ok', 'Recovery was not observable.'); $normal->error('worker_tick_failed'); $normal->busy('notification'); $assert($checks($directory)['worker']['status'] === 'warning', 'Same-second success/error/busy events lost their unresolved failure.'); $normal->success(); $assert($checks($directory)['worker']['status'] === 'ok', 'Only a subsequent completed cycle should clear the explicit unresolved failure.'); $now += 91; $assert($checks($directory)['worker']['status'] === 'warning', 'Idle stale records appeared current.'); $normal->stopped(); $assert($checks($directory)['worker']['active_count'] === 0 && str_contains($checks($directory)['worker']['message'], '退出'), 'Stopped daemon still appeared active.'); $normal2 = CloudRuntimeHeartbeat::start('normal', false, $directory, $clock); $normal2->success(); $assert($checks($directory)['worker']['status'] === 'ok' && $checks($directory)['worker']['active_count'] === 1, 'Old process records shadowed a fresh replacement process.'); $public = json_encode($checks($directory), JSON_THROW_ON_ERROR); foreach (['pid', 'token', 'filename', $directory, 'runtime_error'] as $private) { $assert(!str_contains($public, $private), 'Public checks leaked internal telemetry data.'); } $before = array_map('file_get_contents', $files($directory)); $checks($directory); $assert($before === array_map('file_get_contents', $files($directory)), 'Snapshot mutated heartbeat files.'); $atomicDirectory = $testRoot . '/atomic'; $atomic = CloudRuntimeHeartbeat::start('normal', false, $atomicDirectory, $clock); $atomicPath = $files($atomicDirectory)[0]; $atomicBefore = file_get_contents($atomicPath); file_put_contents($atomicPath . '.tmp', 'pre-existing object'); $atomic->success(); $assert(file_get_contents($atomicPath) === $atomicBefore && file_get_contents($atomicPath . '.tmp') === 'pre-existing object', 'Blocked atomic write corrupted the prior snapshot or removed another temporary file.'); $onceDirectory = $testRoot . '/once'; $once = CloudRuntimeHeartbeat::start('lifecycle', true, $onceDirectory, $clock); $once->busy('lifecycle_tick'); $once->success(); $once->stopped(); $assert($checks($onceDirectory)['lifecycle']['status'] === 'unknown' && $checks($onceDirectory)['lifecycle']['active_count'] === 0 && $read($onceDirectory)[0]['state'] === 'stopped', '--once history falsely claimed a running daemon.'); $unsafe = $testRoot . '/unsafe'; mkdir($unsafe, 0755); $silent = CloudRuntimeHeartbeat::start('normal', false, $unsafe, $clock); $silent->busy('worker_tick'); $silent->error('worker_tick_failed'); $silent->stopped(); $assert($files($unsafe) === [] && $checks($unsafe)['worker']['status'] === 'unknown', 'Insecure runtime directory was accepted or errors escaped into the caller.'); $symlinkDirectory = $testRoot . '/directory-link'; if (@symlink($directory, $symlinkDirectory)) { $count = count($files($directory)); CloudRuntimeHeartbeat::start('normal', false, $symlinkDirectory, $clock)->success(); $assert(count($files($directory)) === $count && $checks($symlinkDirectory)['worker']['status'] === 'unknown', 'Symlink runtime directory was followed.'); } $blocked = $testRoot . '/blocked'; file_put_contents($blocked, 'not a directory'); CloudRuntimeHeartbeat::start('normal', false, $blocked . '/child', $clock)->success(); CloudRuntimeHeartbeat::start('unsupported', false, $testRoot . '/invalid-kind', $clock)->success(); $assert(!file_exists($testRoot . '/invalid-kind'), 'Invalid worker kind created filesystem state.'); $corruptDirectory = $testRoot . '/corrupt'; $corrupt = CloudRuntimeHeartbeat::start('normal', false, $corruptDirectory, $clock); $corrupt->success(); $good = file_get_contents($files($corruptDirectory)[0]); $badFile = $corruptDirectory . '/normal-999-' . str_repeat('a', 24) . '.json'; foreach (['{truncated', str_repeat('x', 4097), json_encode(array_replace(json_decode($good, true), ['last_seen' => $now + 600, 'last_success' => $now + 600]))] as $bad) { file_put_contents($badFile, $bad); chmod($badFile, 0600); $assert($checks($corruptDirectory)['worker']['status'] === 'warning', 'Corrupt, oversized or future-dated record falsely produced healthy status.'); } unlink($badFile); $target = $testRoot . '/private-target'; file_put_contents($target, $good); chmod($target, 0600); if (@symlink($target, $badFile)) { $assert($checks($corruptDirectory)['worker']['status'] === 'warning' && file_get_contents($target) === $good, 'Snapshot accepted or modified a symlink heartbeat.'); unlink($badFile); } $busyRecord = json_decode($good, true); $busyRecord['state'] = 'busy'; $busyRecord['busy_since'] = null; file_put_contents($badFile, json_encode($busyRecord)); chmod($badFile, 0600); $assert($checks($corruptDirectory)['worker']['status'] === 'warning', 'Malformed busy record was accepted.'); unlink($badFile); $cappedDirectory = $testRoot . '/capped'; $capped = CloudRuntimeHeartbeat::start('normal', false, $cappedDirectory, $clock); $capped->success(); for ($index = 0; $index < 513; $index++) { file_put_contents($cappedDirectory . '/unrelated-' . $index, 'untouched'); } $assert($checks($cappedDirectory)['worker']['status'] === 'warning', 'A truncated directory scan falsely returned complete healthy status.'); $cleanupDirectory = $testRoot . '/cleanup'; $old = CloudRuntimeHeartbeat::start('normal', false, $cleanupDirectory, $clock); $oldPath = $files($cleanupDirectory)[0]; touch($oldPath, $now - 604801); $unrelated = $cleanupDirectory . '/keep-me.json'; file_put_contents($unrelated, 'unrelated old file'); touch($unrelated, $now - 604801); CloudRuntimeHeartbeat::start('normal', false, $cleanupDirectory, $clock); $assert(!file_exists($oldPath) && file_get_contents($unrelated) === 'unrelated old file', 'Startup retention retained old telemetry or removed unrelated files.'); // Real CLI --once paths and real lifecycle internal catches, using empty SQLite schemas. $capsule = new Capsule(); $capsule->addConnection(['driver' => 'sqlite', 'database' => ':memory:'], 'default'); $capsule->addConnection(['driver' => 'sqlite', 'database' => ':memory:'], 'log'); $capsule->setAsGlobal(); $capsule->bootEloquent(); $schema = $capsule->schema('default'); $schema->create('cloud_jobs', static function (Blueprint $table): void { $table->increments('id'); foreach (['kind', 'status', 'payload_cipher', 'operation_id'] as $column) { $table->string($column); } foreach (['available_at', 'lease_until', 'attempts', 'user_id', 'host_id', 'updated_at'] as $column) { $table->integer($column)->default(0); } foreach (['dispatched_at', 'order_id', 'instance_id'] as $column) { $table->integer($column)->nullable(); } $table->string('lease_token')->nullable(); $table->string('error_code')->nullable(); }); $schema->create('cloud_mail_outbox', static function (Blueprint $table): void { $table->increments('id'); $table->string('status'); $table->integer('available_at'); $table->integer('lease_until'); $table->integer('invoice_id')->default(0); $table->string('kind')->default('renewal_due'); $table->integer('attempts')->default(0); $table->string('lease_token')->nullable(); $table->string('last_error')->default(''); $table->integer('updated_at')->default(0); $table->integer('sent_at')->nullable(); }); $schema->create('cloud_instances', static function (Blueprint $table): void { $table->increments('id'); foreach (['status', 'service_status', 'uuid', 'renewal_price'] as $column) { $table->string($column)->nullable(); } foreach (['host_id', 'user_id', 'expires_at', 'renewal_days'] as $column) { $table->integer($column)->nullable(); } }); $schema->create('cloud_invoices', static function (Blueprint $table): void { $table->increments('id'); $table->string('purpose'); $table->string('status'); foreach (['due_at', 'instance_id', 'renews_expires_at'] as $column) { $table->integer($column)->nullable(); } $table->integer('user_id')->default(0); $table->integer('created_at')->default(0); }); foreach (['normal', 'policy', 'lifecycle'] as $kind) { $commandDirectory = $testRoot . '/command-' . $kind; $heartbeat = CloudRuntimeHeartbeat::start($kind, true, $commandDirectory, $clock); $status = $kind === 'lifecycle' ? CloudLifecycle::run(['--once'], $heartbeat) : CloudWorker::run($kind === 'policy' ? ['--once', '--policy-only'] : ['--once'], $heartbeat); $record = $read($commandDirectory)[0]; $assert($status === 0 && $record['state'] === 'stopped' && $record['last_success'] === $now && $record['last_error'] === null && $record['mode'] === 'once', 'Real --once command did not finish as a successful stopped observation.'); } $gatedDirectory = $testRoot . '/readiness-gated-lifecycle'; $gatedHeartbeat = CloudRuntimeHeartbeat::start('lifecycle', false, $gatedDirectory, $clock); $gatedLifecycle = new CloudLifecycleService($gatedHeartbeat); $assert($gatedLifecycle->tick(200, true) === 0 && $gatedLifecycle->observationErrors() === ['engine_readiness_timeout'] && $read($gatedDirectory)[0]['last_error'] === 'engine_readiness_timeout', 'The first readiness-timeout tick did not preserve DB work while skipping engine observations.'); $gatedWorkerDirectory = $testRoot . '/readiness-gated-worker'; $gatedWorkerHeartbeat = CloudRuntimeHeartbeat::start( 'normal', false, $gatedWorkerDirectory, $clock ); Capsule::table('cloud_jobs')->insert([ 'kind' => 'create', 'status' => 'queued', 'payload_cipher' => 'must-not-be-read', 'operation_id' => 'maintenance-gated-job', 'available_at' => 0, 'lease_until' => 0, 'attempts' => 0, 'user_id' => 1, 'host_id' => 1, ]); $assert(CloudWorker::run([], $gatedWorkerHeartbeat, static fn (): bool => false) === 1 && $files($gatedWorkerDirectory) === [] && Capsule::table('cloud_jobs')->value('status') === 'queued' && (int) Capsule::table('cloud_jobs')->value('attempts') === 0 && Capsule::table('cloud_jobs')->value('dispatched_at') === null, 'A worker readiness timeout skipped notification fallback or leaked restart-churn files.'); Capsule::table('cloud_jobs')->delete(); Capsule::table('cloud_instances')->insert(['id' => 10, 'status' => 'active', 'service_status' => 'active', 'expires_at' => time() + 3600, 'renewal_price' => 'invalid-amount', 'renewal_days' => 30]); $assert(CloudInvoiceService::issueDue() === 0 && CloudInvoiceService::observationErrors('issue_due') === ['invoice_generation_failed'] && Capsule::table('cloud_invoices')->count() === 0, 'Per-invoice generation failure was hidden or changed invoice outcomes.'); Capsule::table('cloud_instances')->delete(); $assert(CloudInvoiceService::issueDue() === 0 && CloudInvoiceService::observationErrors('issue_due') === [], 'Invoice observation failures carried into an unrelated subsequent invocation.'); $schema->create('user', static function (Blueprint $table): void { $table->increments('id'); $table->string('email'); }); Capsule::table('user')->insert(['id' => 1, 'email' => 'invalid-email']); Capsule::table('cloud_instances')->insert(['id' => 10, 'user_id' => 1, 'status' => 'active', 'service_status' => 'active']); Capsule::table('cloud_invoices')->insert(['id' => 10, 'user_id' => 1, 'instance_id' => 10, 'purpose' => 'renewal', 'status' => 'open', 'due_at' => time() + 3600]); Capsule::table('cloud_mail_outbox')->insert(['id' => 10, 'invoice_id' => 10, 'status' => 'pending', 'available_at' => 0, 'lease_until' => 0]); $assert(CloudInvoiceService::deliverReminders(1) === 0 && CloudInvoiceService::observationErrors('notifications') === ['notification_failed'] && Capsule::table('cloud_mail_outbox')->where('id', 10)->value('status') === 'pending' && Capsule::table('cloud_mail_outbox')->where('id', 10)->value('attempts') === 1, 'Non-throwing notification failure was hidden or changed its existing retry behavior.'); $assert(CloudInvoiceService::deliverReminders(1) === 0 && CloudInvoiceService::observationErrors('notifications') === [], 'Notification observation failures carried into the next idle invocation.'); Capsule::table('cloud_mail_outbox')->where('id', 10)->update(['available_at' => 0]); Capsule::table('cloud_instances')->delete(); $assert(CloudInvoiceService::deliverReminders(1) === 0 && CloudInvoiceService::observationErrors('notifications') === [] && Capsule::table('cloud_mail_outbox')->where('id', 10)->value('status') === 'cancelled', 'Expected cancellation was incorrectly reported as a delivery failure.'); Capsule::table('cloud_mail_outbox')->delete(); $schema->drop('user'); $schema->drop('cloud_invoices'); $partialDirectory = $testRoot . '/partial'; $partial = CloudRuntimeHeartbeat::start('lifecycle', false, $partialDirectory, $clock); $lifecycle = new CloudLifecycleService($partial); $assert($lifecycle->tick() === 0 && $lifecycle->observationErrors() === ['provision_expiry_failed', 'invoice_generation_failed'] && $checks($partialDirectory)['lifecycle']['status'] === 'warning' && $checks($partialDirectory)['lifecycle']['last_success_at'] === null, 'Lifecycle swallowed sub-stage failures and falsely reported normal operation.'); $schema->drop('cloud_instances'); $schema->create('cloud_instances', static function (Blueprint $table): void { $table->increments('id'); $table->string('status'); $table->string('service_status'); }); Capsule::table('cloud_instances')->insert(['id' => 1, 'status' => 'active', 'service_status' => 'active']); $assert($lifecycle->tick() === 0 && $lifecycle->observationErrors() === ['provision_expiry_failed', 'invoice_generation_failed', 'traffic_sync_failed', 'instance_reconcile_failed'] && $checks($partialDirectory)['lifecycle']['status'] === 'warning', 'All internally caught lifecycle stages failed without an unhealthy observation.'); $partialOnceDirectory = $testRoot . '/partial-once'; $partialOnce = CloudRuntimeHeartbeat::start('lifecycle', true, $partialOnceDirectory, $clock); $assert(CloudLifecycle::run(['--once'], $partialOnce) === 0 && $read($partialOnceDirectory)[0]['last_success'] === null && $read($partialOnceDirectory)[0]['last_error'] === 'provision_expiry_failed' && $read($partialOnceDirectory)[0]['state'] === 'stopped', 'Partial failure changed legacy CLI exit semantics or was marked successful.'); Capsule::table('cloud_jobs')->insert(['kind' => 'action', 'status' => 'queued', 'payload_cipher' => 'deliberately-invalid-cipher', 'operation_id' => 'test-operation', 'available_at' => 0, 'lease_until' => 0]); $worker = new CloudWorkerService(); $assert($worker->tick() === true && in_array('worker_tick_failed', $worker->observationErrors(), true) && Capsule::table('cloud_jobs')->value('status') === 'uncertain', 'Worker swallowed a per-job exception without retaining its observation.'); $assert($worker->tick() === false && $worker->observationErrors() === [], 'Worker observation errors were not reset for the next idle cycle.'); echo "Cloud runtime private atomic files, bounded snapshots, long-call freshness, multi-worker identity, --once CLI and SQLite partial failures passed.\n"; } finally { ini_set('error_log', $oldLog === false ? '' : $oldLog); $removeTree($testRoot); }