json_decode(file_get_contents($file), true), $files($directory)); }; $removeTree = static function (string $directory) use (&$removeTree): void { foreach (new FilesystemIterator($directory, FilesystemIterator::SKIP_DOTS) as $entry) { if ($entry->isDir() && !$entry->isLink()) { $removeTree($entry->getPathname()); } else { unlink($entry->getPathname()); } } rmdir($directory); }; try { $missing = $testRoot . '/not-created'; $empty = $checks($missing); $assert(array_keys($empty) === ['worker', 'policy', 'vpc', 'recovery', 'lifecycle'] && array_unique(array_column($empty, 'status')) === ['unknown'] && !file_exists($missing), 'Missing snapshot must be unknown, cover all kinds, and never create runtime files.'); $directory = $testRoot . '/workers'; $normal = CloudRuntimeHeartbeat::start('normal', false, $directory, $clock); $policy = CloudRuntimeHeartbeat::start('policy', false, $directory, $clock); $policy2 = CloudRuntimeHeartbeat::start('policy', false, $directory, $clock); $recovery = CloudRuntimeHeartbeat::start('recovery', false, $directory, $clock); $assert(count($files($directory)) === 4, 'Concurrent worker identities overwrite one another.'); $assert(is_readable($directory) && is_writable($directory), 'Runtime directory is not readable and writable.'); foreach ($files($directory) as $path) { $assert(is_file($path) && is_readable($path) && is_writable($path), 'Heartbeat file is not readable and writable.'); } $assert($checks($directory)['worker']['status'] === 'busy' && $checks($directory)['policy']['active_count'] === 2 && $checks($directory)['recovery']['active_count'] === 1, 'Starting workers or multiple policy reporters were not represented.'); $normal->busy('worker_tick'); $assert(str_contains($checks($directory)['worker']['message'], '任务消费或核对'), 'Worker dispatch was not exposed as its bounded runtime phase.'); $policy->busy('worker_tick'); $policy2->stopped(); $now += 121; $assert($checks($directory)['worker']['status'] === 'busy' && $checks($directory)['policy']['status'] === 'busy', 'Normal long calls or policy 120-second calls were incorrectly declared stale.'); $now += 100; $assert($checks($directory)['policy']['status'] === 'warning' && $checks($directory)['policy']['active_count'] === 0, 'Expired policy observations must request investigation without claiming a live process.'); $now += 700; $assert($checks($directory)['worker']['status'] === 'busy', 'A normal long PHP-to-Agent request was incorrectly declared stopped.'); $normal->busy('notification'); $now += 300; $assert(str_contains($checks($directory)['worker']['message'], '邮件通知'), 'SMTP did not receive its own observation stage and time window.'); $normal->success(); $assert($checks($directory)['worker']['status'] === 'ok' && $checks($directory)['worker']['last_success_at'] === $now, 'Completed cycle did not record its real success time.'); $successfulAt = $now; $now++; $normal->error('secret-password-token-must-not-leak'); $encoded = json_encode($read($directory), JSON_THROW_ON_ERROR); $assert(!str_contains($encoded, 'secret-password-token') && str_contains($encoded, 'runtime_error') && $checks($directory)['worker']['status'] === 'warning' && $checks($directory)['worker']['last_success_at'] === $successfulAt, 'Error was exposed verbatim, hidden, or incorrectly advanced success time.'); $normal->busy('worker_tick'); $assert($checks($directory)['worker']['status'] === 'warning', 'Starting a new pass erased the previous unresolved failure.'); $normal->success(); $assert($checks($directory)['worker']['status'] === 'ok', 'Recovery was not observable.'); foreach (['traffic_collection_unconfigured', 'traffic_log_schema_missing', 'traffic_agent_unavailable', 'traffic_metric_unavailable', 'traffic_collection_failed', 'traffic_retention_failed'] as $collectorError) { $normal->error($collectorError); $record = $read($directory)[0]; $assert($record['last_error'] === $collectorError && $record['unresolved_error'] === true, 'Stable traffic collector observation was collapsed to runtime_error: ' . $collectorError); $normal->success(); } $normal->error('worker_tick_failed'); $normal->busy('notification'); $assert($checks($directory)['worker']['status'] === 'warning', 'Same-second success/error/busy events lost their unresolved failure.'); $normal->success(); $assert($checks($directory)['worker']['status'] === 'ok', 'Only a subsequent completed cycle should clear the explicit unresolved failure.'); $now += 91; $assert($checks($directory)['worker']['status'] === 'warning', 'Idle stale records appeared current.'); $normal->stopped(); $assert($checks($directory)['worker']['active_count'] === 0 && str_contains($checks($directory)['worker']['message'], '退出'), 'Stopped daemon still appeared active.'); $normal2 = CloudRuntimeHeartbeat::start('normal', false, $directory, $clock); $normal2->success(); $assert($checks($directory)['worker']['status'] === 'ok' && $checks($directory)['worker']['active_count'] === 1, 'Old process records shadowed a fresh replacement process.'); $public = json_encode($checks($directory), JSON_THROW_ON_ERROR); foreach (['pid', 'token', 'filename', $directory, 'runtime_error'] as $private) { $assert(!str_contains($public, $private), 'Public checks leaked internal telemetry data.'); } $before = array_map('file_get_contents', $files($directory)); $checks($directory); $assert($before === array_map('file_get_contents', $files($directory)), 'Snapshot mutated heartbeat files.'); $atomicDirectory = $testRoot . '/atomic'; $atomic = CloudRuntimeHeartbeat::start('normal', false, $atomicDirectory, $clock); $atomicPath = $files($atomicDirectory)[0]; $atomicBefore = file_get_contents($atomicPath); file_put_contents($atomicPath . '.tmp', 'pre-existing object'); $atomic->success(); $assert(file_get_contents($atomicPath) === $atomicBefore && file_get_contents($atomicPath . '.tmp') === 'pre-existing object', 'Blocked atomic write corrupted the prior snapshot or removed another temporary file.'); $onceDirectory = $testRoot . '/once'; $once = CloudRuntimeHeartbeat::start('lifecycle', true, $onceDirectory, $clock); $once->busy('lifecycle_tick'); $once->success(); $once->stopped(); $assert($checks($onceDirectory)['lifecycle']['status'] === 'unknown' && $checks($onceDirectory)['lifecycle']['active_count'] === 0 && $read($onceDirectory)[0]['state'] === 'stopped', '--once history falsely claimed a running daemon.'); $permissive = $testRoot . '/permissive'; mkdir($permissive, 0755); chmod($permissive, 0755); $permissiveHeartbeat = CloudRuntimeHeartbeat::start('normal', false, $permissive, $clock); $permissiveFiles = $files($permissive); $assert(count($permissiveFiles) === 1, 'Permissive runtime directory blocked startup.'); $permissiveHeartbeat->success(); chmod($permissiveFiles[0], 0644); $assert($checks($permissive)['worker']['status'] === 'ok', 'Permissive heartbeat file mode blocked an otherwise valid observation.'); $symlinkDirectory = $testRoot . '/directory-link'; if (@symlink($directory, $symlinkDirectory)) { $count = count($files($directory)); CloudRuntimeHeartbeat::start('normal', false, $symlinkDirectory, $clock)->success(); $assert(count($files($directory)) === $count && $checks($symlinkDirectory)['worker']['status'] === 'unknown', 'Symlink runtime directory was followed.'); } $blocked = $testRoot . '/blocked'; file_put_contents($blocked, 'not a directory'); CloudRuntimeHeartbeat::start('normal', false, $blocked . '/child', $clock)->success(); CloudRuntimeHeartbeat::start('unsupported', false, $testRoot . '/invalid-kind', $clock)->success(); $assert(!file_exists($testRoot . '/invalid-kind'), 'Invalid worker kind created filesystem state.'); $corruptDirectory = $testRoot . '/corrupt'; $corrupt = CloudRuntimeHeartbeat::start('normal', false, $corruptDirectory, $clock); $corrupt->success(); $good = file_get_contents($files($corruptDirectory)[0]); $badFile = $corruptDirectory . '/normal-999-' . str_repeat('a', 24) . '.json'; foreach (['{truncated', str_repeat('x', 4097), json_encode(array_replace(json_decode($good, true), ['last_seen' => $now + 600, 'last_success' => $now + 600]))] as $bad) { file_put_contents($badFile, $bad); chmod($badFile, 0600); $assert($checks($corruptDirectory)['worker']['status'] === 'warning', 'Corrupt, oversized or future-dated record falsely produced healthy status.'); } unlink($badFile); $target = $testRoot . '/private-target'; file_put_contents($target, $good); chmod($target, 0600); if (@symlink($target, $badFile)) { $assert($checks($corruptDirectory)['worker']['status'] === 'warning' && file_get_contents($target) === $good, 'Snapshot accepted or modified a symlink heartbeat.'); unlink($badFile); } $busyRecord = json_decode($good, true); $busyRecord['state'] = 'busy'; $busyRecord['busy_since'] = null; file_put_contents($badFile, json_encode($busyRecord)); chmod($badFile, 0600); $assert($checks($corruptDirectory)['worker']['status'] === 'warning', 'Malformed busy record was accepted.'); unlink($badFile); $cappedDirectory = $testRoot . '/capped'; $capped = CloudRuntimeHeartbeat::start('normal', false, $cappedDirectory, $clock); $capped->success(); for ($index = 0; $index < 513; $index++) { file_put_contents($cappedDirectory . '/unrelated-' . $index, 'untouched'); } $assert($checks($cappedDirectory)['worker']['status'] === 'warning', 'A truncated directory scan falsely returned complete healthy status.'); $cleanupDirectory = $testRoot . '/cleanup'; $old = CloudRuntimeHeartbeat::start('normal', false, $cleanupDirectory, $clock); $oldPath = $files($cleanupDirectory)[0]; touch($oldPath, $now - 604801); $unrelated = $cleanupDirectory . '/keep-me.json'; file_put_contents($unrelated, 'unrelated old file'); touch($unrelated, $now - 604801); CloudRuntimeHeartbeat::start('normal', false, $cleanupDirectory, $clock); $assert(!file_exists($oldPath) && file_get_contents($unrelated) === 'unrelated old file', 'Startup retention retained old telemetry or removed unrelated files.'); // Real CLI --once paths and real lifecycle internal catches, using empty SQLite schemas. $capsule = new Capsule(); $capsule->addConnection(['driver' => 'sqlite', 'database' => ':memory:'], 'default'); $capsule->addConnection(['driver' => 'sqlite', 'database' => ':memory:'], 'log'); $capsule->setAsGlobal(); $capsule->bootEloquent(); $schema = $capsule->schema('default'); $schema->create('cloud_jobs', static function (Blueprint $table): void { $table->increments('id'); foreach (['kind', 'status', 'sealed_payload', 'operation_id'] as $column) { $table->string($column); } $table->string('request_idempotency_key')->default(''); $table->string('request_fingerprint')->default(''); $table->string('execution_mode')->default('agent'); $table->string('phase')->default('accepted'); $table->string('outcome')->nullable(); foreach (['available_at', 'lease_until', 'attempts', 'user_id', 'host_id', 'created_at', 'updated_at'] as $column) { $table->integer($column)->default(0); } foreach (['dispatched_at', 'order_id', 'invoice_id', 'instance_id', 'snapshot_id', 'parent_job_id', 'dispatch_started_at', 'agent_accepted_at', 'receipt_deadline_at', 'terminal_at', 'effect_resolved_at', 'effect_resolved_by', 'read_at', 'vpc_batch_id', 'vpc_repair_root_job_id'] as $column) { $table->integer($column)->nullable(); } $table->integer('dispatch_attempts')->default(0); $table->integer('dispatch_deadline_at')->default(0); $table->boolean('effect_may_have_applied')->nullable(); $table->boolean('safe_to_resubmit')->nullable(); $table->string('effect_resolution')->default('not_applicable'); $table->string('effect_resolution_reason')->default(''); $table->string('completion_event_id')->nullable(); $table->string('lease_token')->nullable(); $table->string('error_code')->nullable(); $table->boolean('finalization_incident')->default(0); $table->string('finalization_error_code')->nullable(); $table->text('result_json')->nullable(); }); $schema->create('cloud_agent_completion_events', static function (Blueprint $table): void { $table->string('event_id')->primary(); $table->integer('host_id'); $table->string('operation_id'); $table->string('payload_sha256'); $table->string('outcome'); $table->boolean('effect_may_have_applied'); $table->boolean('safe_to_resubmit'); $table->string('error_code')->default(''); $table->text('result_json')->nullable(); $table->string('disposition')->default('received'); $table->integer('received_at'); $table->integer('applied_at')->nullable(); $table->unique(['host_id', 'operation_id']); }); $schema->create('cloud_templates', static function ( Blueprint $table ): void { $table->increments('id'); $table->string('catalog_key')->default('test-catalog'); $table->string('version_label')->default('test-version'); $table->boolean('is_default')->default(true); $table->integer('active_job_id')->nullable(); $table->string('state')->default('active'); }); $schema->create('cloud_template_replicas', static function ( Blueprint $table ): void { $table->integer('template_id'); $table->integer('job_id')->nullable(); $table->string('state')->default('pending'); }); $schema->create('cloud_template_artifacts', static function ( Blueprint $table ): void { $table->integer('template_id')->primary(); $table->string('state'); $table->integer('delete_next_check_at')->default(0); }); $schema->create('cloud_user_isos', static function ( Blueprint $table ): void { $table->increments('id'); $table->integer('active_job_id')->nullable(); $table->string('state')->default('ready'); }); $schema->create('cloud_user_iso_replicas', static function ( Blueprint $table ): void { $table->integer('user_iso_id'); $table->integer('job_id')->nullable(); $table->string('state')->default('pending'); }); $schema->create('cloud_mail_outbox', static function (Blueprint $table): void { $table->increments('id'); $table->string('status'); $table->integer('available_at'); $table->integer('lease_until'); $table->integer('invoice_id')->default(0); $table->string('kind')->default('renewal_due'); $table->integer('attempts')->default(0); $table->string('lease_token')->nullable(); $table->string('last_error')->default(''); $table->integer('updated_at')->default(0); $table->integer('sent_at')->nullable(); }); $schema->create('cloud_instance_backups', static function ( Blueprint $table ): void { $table->increments('id'); $table->string('purpose'); $table->string('source')->default('manual'); $table->integer('subscription_id')->nullable(); $table->integer('user_id')->default(0); $table->string('state'); $table->integer('updated_at'); }); $schema->create('cloud_backup_subscriptions', static function ( Blueprint $table ): void { $table->increments('id'); $table->boolean('enabled')->default(false); $table->integer('next_run_at')->default(0); $table->integer('retention_count')->default(2); }); $schema->create('cloud_backup_period_entitlements', static function ( Blueprint $table ): void { $table->increments('id'); $table->integer('subscription_id'); $table->integer('ends_at'); }); $schema->create('cloud_deployment_snapshots', static function ( Blueprint $table ): void { $table->increments('id'); $table->integer('backup_id'); $table->string('state'); }); $schema->create('cloud_instances', static function (Blueprint $table): void { $table->increments('id'); foreach (['status', 'service_status', 'uuid', 'renewal_price'] as $column) { $table->string($column)->nullable(); } foreach (['host_id', 'user_id', 'expires_at', 'renewal_days'] as $column) { $table->integer($column)->nullable(); } }); $schema->create('cloud_vpc_ports', static function ( Blueprint $table ): void { $table->increments('id'); $table->integer('instance_id'); $table->integer('active_claim')->nullable(); $table->string('state'); }); $schema->create('cloud_invoices', static function (Blueprint $table): void { $table->increments('id'); $table->string('purpose'); $table->string('status'); foreach (['due_at', 'instance_id', 'renews_expires_at'] as $column) { $table->integer($column)->nullable(); } $table->integer('user_id')->default(0); $table->integer('created_at')->default(0); }); foreach (['normal', 'policy', 'vpc', 'recovery', 'lifecycle'] as $kind) { $commandDirectory = $testRoot . '/command-' . $kind; $heartbeat = CloudRuntimeHeartbeat::start($kind, true, $commandDirectory, $clock); $status = $kind === 'lifecycle' ? CloudLifecycle::run(['--once'], $heartbeat) : CloudWorker::run($kind === 'policy' ? ['--once', '--policy-only'] : ($kind === 'vpc' ? ['--once', '--vpc-batch-only'] : ($kind === 'recovery' ? ['--once', '--recovery-only'] : ['--once'])), $heartbeat); $record = $read($commandDirectory)[0]; if ($kind === 'lifecycle') { $assert($status === 0 && $record['state'] === 'stopped' && $record['last_success'] === null && $record['last_error'] === 'traffic_sync_failed' && $record['unresolved_error'] === true && $record['mode'] === 'once', 'Lifecycle --once hid its caught traffic-sync failure: ' . json_encode($record, JSON_UNESCAPED_SLASHES)); } else { $assert($status === 0 && $record['state'] === 'stopped' && $record['last_success'] === $now && $record['last_error'] === null && $record['mode'] === 'once', 'Worker --once did not finish as a successful stopped observation.'); } } $workerCommand = file_get_contents(dirname(__DIR__) . '/app/Command/CloudWorker.php'); $lifecycleCommand = file_get_contents(dirname(__DIR__) . '/app/Command/CloudLifecycle.php'); $assert(!str_contains($workerCommand, 'CloudEngineReadiness') && !str_contains($lifecycleCommand, 'CloudEngineReadiness') && !str_contains($workerCommand, 'engine_readiness') && !str_contains($lifecycleCommand, 'engine_readiness'), 'A retired global engine-readiness gate still blocks per-host Agent work.'); $workerUnit = file_get_contents(dirname(__DIR__, 2) . '/deploy/virt-cloud-recovery-worker.service'); $xcat = file_get_contents(dirname(__DIR__) . '/app/Command/XCat.php'); $assert(str_contains($workerCommand, "'--recovery-only'") && str_contains($workerCommand, 'CloudAutomaticOperationRecoveryService') && str_contains($workerCommand, "if (!\$policyOnly && !\$vpcBatchOnly && !\$recoveryOnly)") && str_contains($workerCommand, '!$policyOnly && !$vpcBatchOnly && !$recoveryOnly') && str_contains($xcat, '--recovery-only') && str_contains($workerUnit, 'cloudWorker --recovery-only'), 'The dedicated recovery worker is missing, sends mail, or is not deployable.'); Capsule::table('cloud_instances')->insert(['id' => 10, 'status' => 'active', 'service_status' => 'active', 'expires_at' => time() + 3600, 'renewal_price' => 'invalid-amount', 'renewal_days' => 30]); $assert(CloudInvoiceService::issueDue() === 0 && CloudInvoiceService::observationErrors('issue_due') === ['invoice_generation_failed'] && Capsule::table('cloud_invoices')->count() === 0, 'Per-invoice generation failure was hidden or changed invoice outcomes.'); Capsule::table('cloud_instances')->delete(); $assert(CloudInvoiceService::issueDue() === 0 && CloudInvoiceService::observationErrors('issue_due') === [], 'Invoice observation failures carried into an unrelated subsequent invocation.'); $schema->create('user', static function (Blueprint $table): void { $table->increments('id'); $table->string('email'); }); Capsule::table('user')->insert(['id' => 1, 'email' => 'invalid-email']); Capsule::table('cloud_instances')->insert(['id' => 10, 'user_id' => 1, 'status' => 'active', 'service_status' => 'active']); Capsule::table('cloud_invoices')->insert(['id' => 10, 'user_id' => 1, 'instance_id' => 10, 'purpose' => 'renewal', 'status' => 'open', 'due_at' => time() + 3600]); Capsule::table('cloud_mail_outbox')->insert(['id' => 10, 'invoice_id' => 10, 'status' => 'pending', 'available_at' => 0, 'lease_until' => 0]); $assert(CloudInvoiceService::deliverReminders(1) === 0 && CloudInvoiceService::observationErrors('notifications') === ['notification_failed'] && Capsule::table('cloud_mail_outbox')->where('id', 10)->value('status') === 'failed' && Capsule::table('cloud_mail_outbox')->where('id', 10)->value('attempts') === 1, 'Non-throwing notification failure was hidden or left eligible for retry.'); $assert(CloudInvoiceService::deliverReminders(1) === 0 && CloudInvoiceService::observationErrors('notifications') === [], 'Notification observation failures carried into the next idle invocation.'); Capsule::table('cloud_mail_outbox')->where('id', 10)->update([ 'status' => 'pending', 'attempts' => 0, 'available_at' => 0, ]); Capsule::table('cloud_instances')->delete(); $assert(CloudInvoiceService::deliverReminders(1) === 0 && CloudInvoiceService::observationErrors('notifications') === [] && Capsule::table('cloud_mail_outbox')->where('id', 10)->value('status') === 'cancelled', 'Expected cancellation was incorrectly reported as a delivery failure.'); Capsule::table('cloud_mail_outbox')->delete(); $schema->drop('user'); $schema->drop('cloud_invoices'); $partialDirectory = $testRoot . '/partial'; $partial = CloudRuntimeHeartbeat::start('lifecycle', false, $partialDirectory, $clock); $lifecycle = new CloudLifecycleService($partial); $assert($lifecycle->tick() === 0 && $lifecycle->observationErrors() === ['provision_expiry_failed', 'invoice_generation_failed', 'traffic_sync_failed', 'traffic_pool_accounting_failed'] && $checks($partialDirectory)['lifecycle']['status'] === 'warning' && $checks($partialDirectory)['lifecycle']['last_success_at'] === null, 'Lifecycle swallowed sub-stage failures and falsely reported normal operation: ' . json_encode($lifecycle->observationErrors(), JSON_UNESCAPED_SLASHES)); $schema->drop('cloud_instances'); $schema->create('cloud_instances', static function (Blueprint $table): void { $table->increments('id'); $table->string('status'); $table->string('service_status'); }); Capsule::table('cloud_instances')->insert(['id' => 1, 'status' => 'active', 'service_status' => 'active']); $assert($lifecycle->tick() === 0 && $lifecycle->observationErrors() === ['provision_expiry_failed', 'invoice_generation_failed', 'traffic_sync_failed', 'traffic_pool_accounting_failed', 'instance_reconcile_failed'] && $checks($partialDirectory)['lifecycle']['status'] === 'warning', 'All internally caught lifecycle stages failed without an unhealthy observation: ' . json_encode($lifecycle->observationErrors(), JSON_UNESCAPED_SLASHES)); $partialOnceDirectory = $testRoot . '/partial-once'; $partialOnce = CloudRuntimeHeartbeat::start('lifecycle', true, $partialOnceDirectory, $clock); $assert(CloudLifecycle::run(['--once'], $partialOnce) === 0 && $read($partialOnceDirectory)[0]['last_success'] === null && $read($partialOnceDirectory)[0]['last_error'] === 'provision_expiry_failed' && $read($partialOnceDirectory)[0]['state'] === 'stopped', 'Partial failure changed legacy CLI exit semantics or was marked successful.'); Capsule::table('cloud_jobs')->insert(['kind' => 'action', 'execution_mode' => 'agent', 'phase' => 'accepted', 'status' => 'queued', 'user_id' => 1, 'host_id' => 1, 'sealed_payload' => 'deliberately-invalid-cipher', 'operation_id' => str_repeat('a', 32), 'request_idempotency_key' => str_repeat('a', 32), 'request_fingerprint' => str_repeat('b', 64), 'dispatch_deadline_at' => time() + 300, 'available_at' => 0, 'lease_until' => 0]); $worker = new CloudWorkerService(); $assert($worker->tick() === true && in_array('worker_tick_failed', $worker->observationErrors(), true) && Capsule::table('cloud_jobs')->value('phase') === 'terminal' && Capsule::table('cloud_jobs')->value('outcome') === 'failed' && Capsule::table('cloud_jobs')->value('status') === 'failed', 'Worker exception did not become one terminal failed receipt.'); $assert($worker->tick() === false && $worker->observationErrors() === [], 'Worker observation errors were not reset for the next idle cycle.'); echo "Cloud runtime private atomic files, bounded snapshots, long-call freshness, multi-worker identity, --once CLI and SQLite partial failures passed.\n"; } finally { ini_set('error_log', $oldLog === false ? '' : $oldLog); $removeTree($testRoot); }