You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
virt/web/tests/CloudRuntimeHeartbeatTest.php

651 lines
32 KiB
PHP

<?php
// Real private files and SQLite only: no deployment bootstrap, services or Agent calls.
require dirname(__DIR__) . '/vendor/autoload.php';
use App\Command\CloudLifecycle;
use App\Command\CloudWorker;
use App\Services\Cloud\CloudLifecycleService;
use App\Services\Cloud\CloudInvoiceService;
use App\Services\Cloud\CloudRuntimeHeartbeat;
use App\Services\Cloud\CloudWorkerService;
use Illuminate\Database\Capsule\Manager as Capsule;
use Illuminate\Database\Schema\Blueprint;
$assert = static function ($condition, string $message): void {
if (!$condition) {
throw new RuntimeException($message);
}
};
$testRoot = sys_get_temp_dir() . '/cloud-heartbeat-test-' . bin2hex(random_bytes(8));
mkdir($testRoot, 0700);
$oldLog = ini_get('error_log');
ini_set('error_log', $testRoot . '/expected-errors.log');
$now = time();
$clock = static function () use (&$now): int { return $now; };
$checks = static function (string $directory) use (&$now): array {
return array_column(CloudRuntimeHeartbeat::snapshot($directory, $now), null, 'key');
};
$files = static function (string $directory): array {
return glob($directory . '/*.json') ?: [];
};
$read = static function (string $directory) use ($files): array {
return array_map(static fn ($file) => json_decode(file_get_contents($file), true), $files($directory));
};
$removeTree = static function (string $directory) use (&$removeTree): void {
foreach (new FilesystemIterator($directory, FilesystemIterator::SKIP_DOTS) as $entry) {
if ($entry->isDir() && !$entry->isLink()) {
$removeTree($entry->getPathname());
} else {
unlink($entry->getPathname());
}
}
rmdir($directory);
};
try {
$missing = $testRoot . '/not-created';
$empty = $checks($missing);
$assert(array_keys($empty) === [
'worker', 'policy', 'vpc', 'recovery', 'notification',
'storage_replica', 'storage_health', 'lifecycle',
]
&& array_unique(array_column($empty, 'status')) === ['unknown'] && !file_exists($missing),
'Missing snapshot must be unknown, cover all kinds, and never create runtime files.');
$directory = $testRoot . '/workers';
$normal = CloudRuntimeHeartbeat::start('normal', false, $directory, $clock);
$policy = CloudRuntimeHeartbeat::start('policy', false, $directory, $clock);
$policy2 = CloudRuntimeHeartbeat::start('policy', false, $directory, $clock);
$recovery = CloudRuntimeHeartbeat::start('recovery', false, $directory, $clock);
$notification = CloudRuntimeHeartbeat::start(
'notification', false, $directory, $clock
);
$storageReplica = CloudRuntimeHeartbeat::start(
'storage_replica', false, $directory, $clock
);
$storageHealth = CloudRuntimeHeartbeat::start(
'storage_health', false, $directory, $clock
);
$assert(count($files($directory)) === 7, 'Concurrent worker identities overwrite one another.');
$assert(is_readable($directory) && is_writable($directory),
'Runtime directory is not readable and writable.');
foreach ($files($directory) as $path) {
$assert(is_file($path) && is_readable($path) && is_writable($path),
'Heartbeat file is not readable and writable.');
}
$assert($checks($directory)['worker']['status'] === 'busy'
&& $checks($directory)['policy']['active_count'] === 2
&& $checks($directory)['recovery']['active_count'] === 1
&& $checks($directory)['notification']['active_count'] === 1
&& $checks($directory)['notification']['label'] === '客户通知 worker'
&& $checks($directory)['storage_replica']['active_count'] === 1
&& $checks($directory)['storage_replica']['label'] === '对象存储副本 worker'
&& $checks($directory)['storage_health']['active_count'] === 1
&& $checks($directory)['storage_health']['label'] === '对象存储健康探测 worker',
'Starting workers or multiple policy reporters were not represented.');
$notification->busy('notification');
$assert(str_contains(
$checks($directory)['notification']['message'],
'邮件通知'
), 'Customer notification delivery did not receive an isolated runtime phase.');
$storageReplica->busy('storage_replica_tick');
$assert(str_contains(
$checks($directory)['storage_replica']['message'],
'对象存储副本复制或核对'
), 'Object replica work did not receive an isolated runtime phase.');
$storageHealth->busy('storage_health_probe');
$assert(str_contains(
$checks($directory)['storage_health']['message'],
'对象存储健康探测'
), 'Object storage health probes did not receive an isolated runtime phase.');
$normal->busy('worker_tick');
$assert(str_contains($checks($directory)['worker']['message'], '任务消费或核对'),
'Worker dispatch was not exposed as its bounded runtime phase.');
$policy->busy('worker_tick');
$policy2->stopped();
$now += 121;
$assert($checks($directory)['worker']['status'] === 'busy'
&& $checks($directory)['policy']['status'] === 'busy',
'Normal long calls or policy 120-second calls were incorrectly declared stale.');
$now += 100;
$assert($checks($directory)['policy']['status'] === 'warning'
&& $checks($directory)['policy']['active_count'] === 0,
'Expired policy observations must request investigation without claiming a live process.');
$now += 700;
$assert($checks($directory)['worker']['status'] === 'busy',
'A normal long PHP-to-Agent request was incorrectly declared stopped.');
$normal->busy('notification');
$now += 300;
$assert(str_contains($checks($directory)['worker']['message'], '邮件通知'),
'SMTP did not receive its own observation stage and time window.');
$normal->success();
$assert($checks($directory)['worker']['status'] === 'ok'
&& $checks($directory)['worker']['last_success_at'] === $now,
'Completed cycle did not record its real success time.');
$successfulAt = $now;
$now++;
$normal->error('secret-password-token-must-not-leak');
$encoded = json_encode($read($directory), JSON_THROW_ON_ERROR);
$assert(!str_contains($encoded, 'secret-password-token') && str_contains($encoded, 'runtime_error')
&& $checks($directory)['worker']['status'] === 'warning'
&& $checks($directory)['worker']['last_success_at'] === $successfulAt,
'Error was exposed verbatim, hidden, or incorrectly advanced success time.');
$normal->busy('worker_tick');
$assert($checks($directory)['worker']['status'] === 'warning',
'Starting a new pass erased the previous unresolved failure.');
$normal->success();
$assert($checks($directory)['worker']['status'] === 'ok', 'Recovery was not observable.');
foreach (['traffic_collection_unconfigured', 'traffic_log_schema_missing',
'traffic_agent_unavailable', 'traffic_metric_unavailable',
'traffic_collection_failed', 'traffic_retention_failed'] as $collectorError) {
$normal->error($collectorError);
$record = $read($directory)[0];
$assert($record['last_error'] === $collectorError
&& $record['unresolved_error'] === true,
'Stable traffic collector observation was collapsed to runtime_error: '
. $collectorError);
$normal->success();
}
$normal->error('worker_tick_failed');
$normal->busy('notification');
$assert($checks($directory)['worker']['status'] === 'warning',
'Same-second success/error/busy events lost their unresolved failure.');
$normal->success();
$assert($checks($directory)['worker']['status'] === 'ok',
'Only a subsequent completed cycle should clear the explicit unresolved failure.');
$now += 91;
$assert($checks($directory)['worker']['status'] === 'warning', 'Idle stale records appeared current.');
$normal->stopped();
$assert($checks($directory)['worker']['active_count'] === 0
&& str_contains($checks($directory)['worker']['message'], '退出'),
'Stopped daemon still appeared active.');
$normal2 = CloudRuntimeHeartbeat::start('normal', false, $directory, $clock);
$normal2->success();
$assert($checks($directory)['worker']['status'] === 'ok'
&& $checks($directory)['worker']['active_count'] === 1,
'Old process records shadowed a fresh replacement process.');
$public = json_encode($checks($directory), JSON_THROW_ON_ERROR);
foreach (['pid', 'token', 'filename', $directory, 'runtime_error'] as $private) {
$assert(!str_contains($public, $private), 'Public checks leaked internal telemetry data.');
}
$before = array_map('file_get_contents', $files($directory));
$checks($directory);
$assert($before === array_map('file_get_contents', $files($directory)), 'Snapshot mutated heartbeat files.');
$atomicDirectory = $testRoot . '/atomic';
$atomic = CloudRuntimeHeartbeat::start('normal', false, $atomicDirectory, $clock);
$atomicPath = $files($atomicDirectory)[0];
$atomicBefore = file_get_contents($atomicPath);
file_put_contents($atomicPath . '.tmp', 'pre-existing object');
$atomic->success();
$assert(file_get_contents($atomicPath) === $atomicBefore
&& file_get_contents($atomicPath . '.tmp') === 'pre-existing object',
'Blocked atomic write corrupted the prior snapshot or removed another temporary file.');
$onceDirectory = $testRoot . '/once';
$once = CloudRuntimeHeartbeat::start('lifecycle', true, $onceDirectory, $clock);
$once->busy('lifecycle_tick');
$once->success();
$once->stopped();
$assert($checks($onceDirectory)['lifecycle']['status'] === 'unknown'
&& $checks($onceDirectory)['lifecycle']['active_count'] === 0
&& $read($onceDirectory)[0]['state'] === 'stopped',
'--once history falsely claimed a running daemon.');
$permissive = $testRoot . '/permissive';
mkdir($permissive, 0755);
chmod($permissive, 0755);
$permissiveHeartbeat = CloudRuntimeHeartbeat::start('normal', false, $permissive, $clock);
$permissiveFiles = $files($permissive);
$assert(count($permissiveFiles) === 1, 'Permissive runtime directory blocked startup.');
$permissiveHeartbeat->success();
chmod($permissiveFiles[0], 0644);
$assert($checks($permissive)['worker']['status'] === 'ok',
'Permissive heartbeat file mode blocked an otherwise valid observation.');
$symlinkDirectory = $testRoot . '/directory-link';
if (@symlink($directory, $symlinkDirectory)) {
$count = count($files($directory));
CloudRuntimeHeartbeat::start('normal', false, $symlinkDirectory, $clock)->success();
$assert(count($files($directory)) === $count
&& $checks($symlinkDirectory)['worker']['status'] === 'unknown',
'Symlink runtime directory was followed.');
}
$blocked = $testRoot . '/blocked';
file_put_contents($blocked, 'not a directory');
CloudRuntimeHeartbeat::start('normal', false, $blocked . '/child', $clock)->success();
CloudRuntimeHeartbeat::start('unsupported', false, $testRoot . '/invalid-kind', $clock)->success();
$assert(!file_exists($testRoot . '/invalid-kind'), 'Invalid worker kind created filesystem state.');
$corruptDirectory = $testRoot . '/corrupt';
$corrupt = CloudRuntimeHeartbeat::start('normal', false, $corruptDirectory, $clock);
$corrupt->success();
$good = file_get_contents($files($corruptDirectory)[0]);
$badFile = $corruptDirectory . '/normal-999-' . str_repeat('a', 24) . '.json';
foreach (['{truncated', str_repeat('x', 4097), json_encode(array_replace(json_decode($good, true),
['last_seen' => $now + 600, 'last_success' => $now + 600]))] as $bad) {
file_put_contents($badFile, $bad);
chmod($badFile, 0600);
$assert($checks($corruptDirectory)['worker']['status'] === 'warning',
'Corrupt, oversized or future-dated record falsely produced healthy status.');
}
unlink($badFile);
$target = $testRoot . '/private-target';
file_put_contents($target, $good);
chmod($target, 0600);
if (@symlink($target, $badFile)) {
$assert($checks($corruptDirectory)['worker']['status'] === 'warning'
&& file_get_contents($target) === $good, 'Snapshot accepted or modified a symlink heartbeat.');
unlink($badFile);
}
$busyRecord = json_decode($good, true);
$busyRecord['state'] = 'busy';
$busyRecord['busy_since'] = null;
file_put_contents($badFile, json_encode($busyRecord));
chmod($badFile, 0600);
$assert($checks($corruptDirectory)['worker']['status'] === 'warning', 'Malformed busy record was accepted.');
unlink($badFile);
$cappedDirectory = $testRoot . '/capped';
$capped = CloudRuntimeHeartbeat::start('normal', false, $cappedDirectory, $clock);
$capped->success();
for ($index = 0; $index < 513; $index++) {
file_put_contents($cappedDirectory . '/unrelated-' . $index, 'untouched');
}
$assert($checks($cappedDirectory)['worker']['status'] === 'warning',
'A truncated directory scan falsely returned complete healthy status.');
$cleanupDirectory = $testRoot . '/cleanup';
$old = CloudRuntimeHeartbeat::start('normal', false, $cleanupDirectory, $clock);
$oldPath = $files($cleanupDirectory)[0];
touch($oldPath, $now - 604801);
$unrelated = $cleanupDirectory . '/keep-me.json';
file_put_contents($unrelated, 'unrelated old file');
touch($unrelated, $now - 604801);
CloudRuntimeHeartbeat::start('normal', false, $cleanupDirectory, $clock);
$assert(!file_exists($oldPath) && file_get_contents($unrelated) === 'unrelated old file',
'Startup retention retained old telemetry or removed unrelated files.');
// Real CLI --once paths and real lifecycle internal catches, using empty SQLite schemas.
$capsule = new Capsule();
$capsule->addConnection(['driver' => 'sqlite', 'database' => ':memory:'], 'default');
$capsule->addConnection(['driver' => 'sqlite', 'database' => ':memory:'], 'log');
$capsule->setAsGlobal();
$capsule->bootEloquent();
$schema = $capsule->schema('default');
$schema->create('cloud_jobs', static function (Blueprint $table): void {
$table->increments('id');
foreach (['kind', 'status', 'sealed_payload', 'operation_id'] as $column) {
$table->string($column);
}
$table->string('request_idempotency_key')->default('');
$table->string('request_fingerprint')->default('');
$table->string('execution_mode')->default('agent');
$table->string('phase')->default('accepted');
$table->string('outcome')->nullable();
foreach (['available_at', 'lease_until', 'attempts', 'user_id',
'host_id', 'created_at', 'updated_at'] as $column) {
$table->integer($column)->default(0);
}
foreach (['dispatched_at', 'order_id', 'invoice_id', 'instance_id',
'snapshot_id', 'parent_job_id', 'dispatch_started_at',
'agent_accepted_at', 'receipt_deadline_at', 'terminal_at',
'effect_resolved_at', 'effect_resolved_by', 'read_at',
'vpc_batch_id', 'vpc_repair_root_job_id'] as $column) {
$table->integer($column)->nullable();
}
$table->integer('public_network_transition_id')->nullable();
$table->integer('dispatch_attempts')->default(0);
$table->integer('dispatch_deadline_at')->default(0);
$table->boolean('effect_may_have_applied')->nullable();
$table->boolean('safe_to_resubmit')->nullable();
$table->string('effect_resolution')->default('not_applicable');
$table->string('effect_resolution_reason')->default('');
$table->string('completion_event_id')->nullable();
$table->string('lease_token')->nullable();
$table->string('error_code')->nullable();
$table->boolean('finalization_incident')->default(0);
$table->string('finalization_error_code')->nullable();
$table->text('result_json')->nullable();
});
$schema->create('cloud_agent_completion_events',
static function (Blueprint $table): void {
$table->string('event_id')->primary();
$table->integer('host_id');
$table->string('operation_id');
$table->string('payload_sha256');
$table->string('outcome');
$table->boolean('effect_may_have_applied');
$table->boolean('safe_to_resubmit');
$table->string('error_code')->default('');
$table->text('result_json')->nullable();
$table->string('disposition')->default('received');
$table->integer('received_at');
$table->integer('applied_at')->nullable();
$table->unique(['host_id', 'operation_id']);
});
$schema->create('cloud_public_network_transitions', static function (
Blueprint $table
): void {
$table->increments('id');
$table->string('state');
$table->string('effect_resolution');
$table->string('last_error_code')->default('');
});
$schema->create('cloud_backup_restore_attempts', static function (
Blueprint $table
): void {
$table->increments('id');
$table->string('attempt_key')->unique();
$table->integer('job_id')->nullable();
$table->string('state');
$table->string('effect_resolution')->default('not_applicable');
$table->integer('active_attempt')->nullable();
$table->string('last_error_code')->default('');
$table->integer('updated_at')->default(0);
});
$schema->create('cloud_templates', static function (
Blueprint $table
): void {
$table->increments('id');
$table->string('catalog_key')->default('test-catalog');
$table->string('version_label')->default('test-version');
$table->boolean('is_default')->default(true);
$table->integer('active_job_id')->nullable();
$table->string('state')->default('active');
});
$schema->create('cloud_template_replicas', static function (
Blueprint $table
): void {
$table->integer('template_id');
$table->integer('job_id')->nullable();
$table->string('state')->default('pending');
});
$schema->create('cloud_template_artifacts', static function (
Blueprint $table
): void {
$table->integer('template_id')->primary();
$table->string('state');
$table->integer('delete_next_check_at')->default(0);
});
$schema->create('cloud_user_isos', static function (
Blueprint $table
): void {
$table->increments('id');
$table->integer('active_job_id')->nullable();
$table->string('state')->default('ready');
});
$schema->create('cloud_user_iso_replicas', static function (
Blueprint $table
): void {
$table->integer('user_iso_id');
$table->integer('job_id')->nullable();
$table->string('state')->default('pending');
});
$schema->create('cloud_mail_outbox', static function (Blueprint $table): void {
$table->increments('id');
$table->string('status');
$table->integer('available_at');
$table->integer('lease_until');
$table->integer('invoice_id')->default(0);
$table->string('kind')->default('renewal_due');
$table->integer('attempts')->default(0);
$table->string('lease_token')->nullable();
$table->string('last_error')->default('');
$table->integer('updated_at')->default(0);
$table->integer('sent_at')->nullable();
});
$schema->create('cloud_instance_backups', static function (
Blueprint $table
): void {
$table->increments('id');
$table->string('purpose');
$table->string('source')->default('manual');
$table->integer('subscription_id')->nullable();
$table->integer('user_id')->default(0);
$table->string('state');
$table->integer('updated_at');
});
$schema->create('cloud_backup_subscriptions', static function (
Blueprint $table
): void {
$table->increments('id');
$table->boolean('enabled')->default(false);
$table->integer('next_run_at')->default(0);
$table->integer('retention_count')->default(2);
});
$schema->create('cloud_backup_period_entitlements', static function (
Blueprint $table
): void {
$table->increments('id');
$table->integer('subscription_id');
$table->integer('ends_at');
});
$schema->create('cloud_deployment_snapshots', static function (
Blueprint $table
): void {
$table->increments('id');
$table->integer('backup_id');
$table->string('state');
});
$schema->create('cloud_instances', static function (Blueprint $table): void {
$table->increments('id');
foreach (['status', 'service_status', 'uuid', 'renewal_price'] as $column) {
$table->string($column)->nullable();
}
foreach (['host_id', 'user_id', 'expires_at', 'renewal_days'] as $column) {
$table->integer($column)->nullable();
}
});
$schema->create('cloud_vpc_ports', static function (
Blueprint $table
): void {
$table->increments('id');
$table->integer('instance_id');
$table->integer('active_claim')->nullable();
$table->string('state');
});
$schema->create('cloud_invoices', static function (Blueprint $table): void {
$table->increments('id');
$table->string('purpose');
$table->string('status');
foreach (['due_at', 'instance_id', 'renews_expires_at'] as $column) {
$table->integer($column)->nullable();
}
$table->integer('user_id')->default(0);
$table->integer('created_at')->default(0);
});
$schema->create(
'cloud_service_compensations',
static function (Blueprint $table): void {
$table->increments('id');
$table->integer('instance_count');
$table->integer('applied_count')->default(0);
$table->integer('deferred_count')->default(0);
$table->integer('rejected_count')->default(0);
$table->integer('attention_count')->default(0);
$table->string('state');
$table->string('last_error_code')->default('');
$table->integer('terminal_at')->nullable();
$table->integer('updated_at');
}
);
$schema->create(
'cloud_service_compensation_items',
static function (Blueprint $table): void {
$table->increments('id');
$table->string('item_key');
$table->integer('batch_id');
$table->integer('instance_id');
$table->string('state');
$table->integer('attempts')->default(0);
$table->integer('available_at');
$table->string('lease_token')->nullable();
$table->integer('lease_until')->default(0);
$table->integer('audit_job_id')->nullable();
$table->integer('before_expires_at')->nullable();
$table->integer('after_expires_at')->nullable();
$table->integer('applied_at')->nullable();
$table->integer('terminal_at')->nullable();
$table->string('last_error_code')->default('');
$table->string('last_error')->default('');
$table->integer('updated_at');
}
);
foreach (['normal', 'policy', 'vpc', 'recovery', 'lifecycle'] as $kind) {
$commandDirectory = $testRoot . '/command-' . $kind;
$heartbeat = CloudRuntimeHeartbeat::start($kind, true, $commandDirectory, $clock);
$status = $kind === 'lifecycle' ? CloudLifecycle::run(['--once'], $heartbeat)
: CloudWorker::run($kind === 'policy'
? ['--once', '--policy-only']
: ($kind === 'vpc'
? ['--once', '--vpc-batch-only']
: ($kind === 'recovery'
? ['--once', '--recovery-only'] : ['--once'])), $heartbeat);
$record = $read($commandDirectory)[0];
if ($kind === 'lifecycle') {
$assert($status === 0 && $record['state'] === 'stopped'
&& $record['last_success'] === null
&& $record['last_error'] === 'traffic_sync_failed'
&& $record['unresolved_error'] === true
&& $record['mode'] === 'once',
'Lifecycle --once hid its caught traffic-sync failure: '
. json_encode($record, JSON_UNESCAPED_SLASHES));
} else {
$assert($status === 0 && $record['state'] === 'stopped'
&& $record['last_success'] === $now
&& $record['last_error'] === null
&& $record['mode'] === 'once',
'Worker --once did not finish as a successful stopped observation.');
}
}
$workerCommand = file_get_contents(dirname(__DIR__) . '/app/Command/CloudWorker.php');
$lifecycleCommand = file_get_contents(dirname(__DIR__) . '/app/Command/CloudLifecycle.php');
$assert(!str_contains($workerCommand, 'CloudEngineReadiness')
&& !str_contains($lifecycleCommand, 'CloudEngineReadiness')
&& !str_contains($workerCommand, 'engine_readiness')
&& !str_contains($lifecycleCommand, 'engine_readiness'),
'A retired global engine-readiness gate still blocks per-host Agent work.');
$workerUnit = file_get_contents(dirname(__DIR__, 2)
. '/deploy/virt-cloud-recovery-worker.service');
$notificationUnit = file_get_contents(dirname(__DIR__, 2)
. '/deploy/virt-cloud-notification-worker.service');
$xcat = file_get_contents(dirname(__DIR__) . '/app/Command/XCat.php');
$assert(str_contains($workerCommand, "'--recovery-only'")
&& str_contains($workerCommand, 'CloudAutomaticOperationRecoveryService')
&& str_contains($workerCommand,
'!$policyOnly && !$vpcBatchOnly && !$recoveryOnly')
&& str_contains($workerCommand, '&& !$notificationOnly')
&& str_contains($xcat, '--recovery-only')
&& str_contains($workerUnit, 'cloudWorker --recovery-only'),
'The dedicated recovery worker is missing, sends mail, or is not deployable.');
$assert(str_contains($workerCommand, "'--notification-only'")
&& str_contains($workerCommand,
'CloudNotificationDeliveryService')
&& str_contains($xcat, '--notification-only')
&& str_contains($notificationUnit,
'cloudWorker --notification-only')
&& str_contains($notificationUnit,
'customer notification worker'),
'The isolated customer notification worker is not deployable.');
Capsule::table('cloud_instances')->insert(['id' => 10, 'status' => 'active',
'service_status' => 'active', 'expires_at' => time() + 3600,
'renewal_price' => 'invalid-amount', 'renewal_days' => 30]);
$assert(CloudInvoiceService::issueDue() === 0
&& CloudInvoiceService::observationErrors('issue_due') === ['invoice_generation_failed']
&& Capsule::table('cloud_invoices')->count() === 0,
'Per-invoice generation failure was hidden or changed invoice outcomes.');
Capsule::table('cloud_instances')->delete();
$assert(CloudInvoiceService::issueDue() === 0
&& CloudInvoiceService::observationErrors('issue_due') === [],
'Invoice observation failures carried into an unrelated subsequent invocation.');
$schema->create('user', static function (Blueprint $table): void {
$table->increments('id');
$table->string('email');
});
Capsule::table('user')->insert(['id' => 1, 'email' => 'invalid-email']);
Capsule::table('cloud_instances')->insert(['id' => 10, 'user_id' => 1,
'status' => 'active', 'service_status' => 'active']);
Capsule::table('cloud_invoices')->insert(['id' => 10, 'user_id' => 1, 'instance_id' => 10,
'purpose' => 'renewal', 'status' => 'open', 'due_at' => time() + 3600]);
Capsule::table('cloud_mail_outbox')->insert(['id' => 10, 'invoice_id' => 10,
'status' => 'pending', 'available_at' => 0, 'lease_until' => 0]);
$assert(CloudInvoiceService::deliverReminders(1) === 0
&& CloudInvoiceService::observationErrors('notifications') === ['notification_failed']
&& Capsule::table('cloud_mail_outbox')->where('id', 10)->value('status') === 'failed'
&& Capsule::table('cloud_mail_outbox')->where('id', 10)->value('attempts') === 1,
'Non-throwing notification failure was hidden or left eligible for retry.');
$assert(CloudInvoiceService::deliverReminders(1) === 0
&& CloudInvoiceService::observationErrors('notifications') === [],
'Notification observation failures carried into the next idle invocation.');
Capsule::table('cloud_mail_outbox')->where('id', 10)->update([
'status' => 'pending', 'attempts' => 0, 'available_at' => 0,
]);
Capsule::table('cloud_instances')->delete();
$assert(CloudInvoiceService::deliverReminders(1) === 0
&& CloudInvoiceService::observationErrors('notifications') === []
&& Capsule::table('cloud_mail_outbox')->where('id', 10)->value('status') === 'cancelled',
'Expected cancellation was incorrectly reported as a delivery failure.');
Capsule::table('cloud_mail_outbox')->delete();
$schema->drop('user');
$schema->drop('cloud_invoices');
$partialDirectory = $testRoot . '/partial';
$partial = CloudRuntimeHeartbeat::start('lifecycle', false, $partialDirectory, $clock);
$lifecycle = new CloudLifecycleService($partial);
$assert($lifecycle->tick() === 0
&& $lifecycle->observationErrors() === ['provision_expiry_failed',
'invoice_generation_failed', 'traffic_sync_failed',
'traffic_pool_accounting_failed']
&& $checks($partialDirectory)['lifecycle']['status'] === 'warning'
&& $checks($partialDirectory)['lifecycle']['last_success_at'] === null,
'Lifecycle swallowed sub-stage failures and falsely reported normal operation: '
. json_encode($lifecycle->observationErrors(), JSON_UNESCAPED_SLASHES));
$schema->drop('cloud_instances');
$schema->create('cloud_instances', static function (Blueprint $table): void {
$table->increments('id');
$table->string('status');
$table->string('service_status');
});
Capsule::table('cloud_instances')->insert(['id' => 1, 'status' => 'active', 'service_status' => 'active']);
$assert($lifecycle->tick() === 0
&& $lifecycle->observationErrors() === ['provision_expiry_failed', 'invoice_generation_failed',
'traffic_sync_failed', 'traffic_pool_accounting_failed',
'instance_reconcile_failed']
&& $checks($partialDirectory)['lifecycle']['status'] === 'warning',
'All internally caught lifecycle stages failed without an unhealthy observation: '
. json_encode($lifecycle->observationErrors(), JSON_UNESCAPED_SLASHES));
$partialOnceDirectory = $testRoot . '/partial-once';
$partialOnce = CloudRuntimeHeartbeat::start('lifecycle', true, $partialOnceDirectory, $clock);
$assert(CloudLifecycle::run(['--once'], $partialOnce) === 0
&& $read($partialOnceDirectory)[0]['last_success'] === null
&& $read($partialOnceDirectory)[0]['last_error'] === 'provision_expiry_failed'
&& $read($partialOnceDirectory)[0]['state'] === 'stopped',
'Partial failure changed legacy CLI exit semantics or was marked successful.');
Capsule::table('cloud_jobs')->insert(['kind' => 'action',
'execution_mode' => 'agent', 'phase' => 'accepted',
'status' => 'queued', 'user_id' => 1, 'host_id' => 1,
'sealed_payload' => 'deliberately-invalid-cipher',
'operation_id' => str_repeat('a', 32),
'request_idempotency_key' => str_repeat('a', 32),
'request_fingerprint' => str_repeat('b', 64),
'dispatch_deadline_at' => time() + 300,
'available_at' => 0, 'lease_until' => 0]);
$worker = new CloudWorkerService();
$assert($worker->tick() === true && in_array('worker_tick_failed', $worker->observationErrors(), true)
&& Capsule::table('cloud_jobs')->value('phase') === 'terminal'
&& Capsule::table('cloud_jobs')->value('outcome') === 'failed'
&& Capsule::table('cloud_jobs')->value('status') === 'failed',
'Worker exception did not become one terminal failed receipt.');
$assert($worker->tick() === false && $worker->observationErrors() === [],
'Worker observation errors were not reset for the next idle cycle.');
echo "Cloud runtime private atomic files, bounded snapshots, long-call freshness, multi-worker identity, --once CLI and SQLite partial failures passed.\n";
} finally {
ini_set('error_log', $oldLog === false ? '' : $oldLog);
$removeTree($testRoot);
}