You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
491 lines
26 KiB
PHP
491 lines
26 KiB
PHP
<?php
|
|
|
|
// Real private files and SQLite only: no deployment bootstrap, services or Agent calls.
|
|
require dirname(__DIR__) . '/vendor/autoload.php';
|
|
|
|
use App\Command\CloudLifecycle;
|
|
use App\Command\CloudWorker;
|
|
use App\Services\Cloud\CloudLifecycleService;
|
|
use App\Services\Cloud\CloudInvoiceService;
|
|
use App\Services\Cloud\CloudRuntimeHeartbeat;
|
|
use App\Services\Cloud\CloudWorkerService;
|
|
use Illuminate\Database\Capsule\Manager as Capsule;
|
|
use Illuminate\Database\Schema\Blueprint;
|
|
|
|
$assert = static function ($condition, string $message): void {
|
|
if (!$condition) {
|
|
throw new RuntimeException($message);
|
|
}
|
|
};
|
|
$testRoot = sys_get_temp_dir() . '/cloud-heartbeat-test-' . bin2hex(random_bytes(8));
|
|
mkdir($testRoot, 0700);
|
|
$oldLog = ini_get('error_log');
|
|
ini_set('error_log', $testRoot . '/expected-errors.log');
|
|
$now = time();
|
|
$clock = static function () use (&$now): int { return $now; };
|
|
$checks = static function (string $directory) use (&$now): array {
|
|
return array_column(CloudRuntimeHeartbeat::snapshot($directory, $now), null, 'key');
|
|
};
|
|
$files = static function (string $directory): array {
|
|
return glob($directory . '/*.json') ?: [];
|
|
};
|
|
$read = static function (string $directory) use ($files): array {
|
|
return array_map(static fn ($file) => json_decode(file_get_contents($file), true), $files($directory));
|
|
};
|
|
$removeTree = static function (string $directory) use (&$removeTree): void {
|
|
foreach (new FilesystemIterator($directory, FilesystemIterator::SKIP_DOTS) as $entry) {
|
|
if ($entry->isDir() && !$entry->isLink()) {
|
|
$removeTree($entry->getPathname());
|
|
} else {
|
|
unlink($entry->getPathname());
|
|
}
|
|
}
|
|
rmdir($directory);
|
|
};
|
|
|
|
try {
|
|
$missing = $testRoot . '/not-created';
|
|
$empty = $checks($missing);
|
|
$assert(array_keys($empty) === ['worker', 'policy', 'recovery', 'lifecycle']
|
|
&& array_unique(array_column($empty, 'status')) === ['unknown'] && !file_exists($missing),
|
|
'Missing snapshot must be unknown, cover all kinds, and never create runtime files.');
|
|
|
|
$directory = $testRoot . '/workers';
|
|
$normal = CloudRuntimeHeartbeat::start('normal', false, $directory, $clock);
|
|
$policy = CloudRuntimeHeartbeat::start('policy', false, $directory, $clock);
|
|
$policy2 = CloudRuntimeHeartbeat::start('policy', false, $directory, $clock);
|
|
$recovery = CloudRuntimeHeartbeat::start('recovery', false, $directory, $clock);
|
|
$assert(count($files($directory)) === 4, 'Concurrent worker identities overwrite one another.');
|
|
$assert(is_readable($directory) && is_writable($directory),
|
|
'Runtime directory is not readable and writable.');
|
|
foreach ($files($directory) as $path) {
|
|
$assert(is_file($path) && is_readable($path) && is_writable($path),
|
|
'Heartbeat file is not readable and writable.');
|
|
}
|
|
$assert($checks($directory)['worker']['status'] === 'busy'
|
|
&& $checks($directory)['policy']['active_count'] === 2
|
|
&& $checks($directory)['recovery']['active_count'] === 1,
|
|
'Starting workers or multiple policy reporters were not represented.');
|
|
|
|
$normal->busy('worker_tick');
|
|
$assert(str_contains($checks($directory)['worker']['message'], '任务消费或核对'),
|
|
'Worker dispatch was not exposed as its bounded runtime phase.');
|
|
$policy->busy('worker_tick');
|
|
$policy2->stopped();
|
|
$now += 121;
|
|
$assert($checks($directory)['worker']['status'] === 'busy'
|
|
&& $checks($directory)['policy']['status'] === 'busy',
|
|
'Normal long calls or policy 120-second calls were incorrectly declared stale.');
|
|
$now += 100;
|
|
$assert($checks($directory)['policy']['status'] === 'warning'
|
|
&& $checks($directory)['policy']['active_count'] === 0,
|
|
'Expired policy observations must request investigation without claiming a live process.');
|
|
$now += 700;
|
|
$assert($checks($directory)['worker']['status'] === 'busy',
|
|
'A normal long PHP-to-Agent request was incorrectly declared stopped.');
|
|
$normal->busy('notification');
|
|
$now += 300;
|
|
$assert(str_contains($checks($directory)['worker']['message'], '邮件通知'),
|
|
'SMTP did not receive its own observation stage and time window.');
|
|
$normal->success();
|
|
$assert($checks($directory)['worker']['status'] === 'ok'
|
|
&& $checks($directory)['worker']['last_success_at'] === $now,
|
|
'Completed cycle did not record its real success time.');
|
|
$successfulAt = $now;
|
|
$now++;
|
|
$normal->error('secret-password-token-must-not-leak');
|
|
$encoded = json_encode($read($directory), JSON_THROW_ON_ERROR);
|
|
$assert(!str_contains($encoded, 'secret-password-token') && str_contains($encoded, 'runtime_error')
|
|
&& $checks($directory)['worker']['status'] === 'warning'
|
|
&& $checks($directory)['worker']['last_success_at'] === $successfulAt,
|
|
'Error was exposed verbatim, hidden, or incorrectly advanced success time.');
|
|
$normal->busy('worker_tick');
|
|
$assert($checks($directory)['worker']['status'] === 'warning',
|
|
'Starting a new pass erased the previous unresolved failure.');
|
|
$normal->success();
|
|
$assert($checks($directory)['worker']['status'] === 'ok', 'Recovery was not observable.');
|
|
foreach (['traffic_collection_unconfigured', 'traffic_log_schema_missing',
|
|
'traffic_agent_unavailable', 'traffic_metric_unavailable',
|
|
'traffic_collection_failed', 'traffic_retention_failed'] as $collectorError) {
|
|
$normal->error($collectorError);
|
|
$record = $read($directory)[0];
|
|
$assert($record['last_error'] === $collectorError
|
|
&& $record['unresolved_error'] === true,
|
|
'Stable traffic collector observation was collapsed to runtime_error: '
|
|
. $collectorError);
|
|
$normal->success();
|
|
}
|
|
$normal->error('worker_tick_failed');
|
|
$normal->busy('notification');
|
|
$assert($checks($directory)['worker']['status'] === 'warning',
|
|
'Same-second success/error/busy events lost their unresolved failure.');
|
|
$normal->success();
|
|
$assert($checks($directory)['worker']['status'] === 'ok',
|
|
'Only a subsequent completed cycle should clear the explicit unresolved failure.');
|
|
$now += 91;
|
|
$assert($checks($directory)['worker']['status'] === 'warning', 'Idle stale records appeared current.');
|
|
$normal->stopped();
|
|
$assert($checks($directory)['worker']['active_count'] === 0
|
|
&& str_contains($checks($directory)['worker']['message'], '退出'),
|
|
'Stopped daemon still appeared active.');
|
|
$normal2 = CloudRuntimeHeartbeat::start('normal', false, $directory, $clock);
|
|
$normal2->success();
|
|
$assert($checks($directory)['worker']['status'] === 'ok'
|
|
&& $checks($directory)['worker']['active_count'] === 1,
|
|
'Old process records shadowed a fresh replacement process.');
|
|
$public = json_encode($checks($directory), JSON_THROW_ON_ERROR);
|
|
foreach (['pid', 'token', 'filename', $directory, 'runtime_error'] as $private) {
|
|
$assert(!str_contains($public, $private), 'Public checks leaked internal telemetry data.');
|
|
}
|
|
$before = array_map('file_get_contents', $files($directory));
|
|
$checks($directory);
|
|
$assert($before === array_map('file_get_contents', $files($directory)), 'Snapshot mutated heartbeat files.');
|
|
$atomicDirectory = $testRoot . '/atomic';
|
|
$atomic = CloudRuntimeHeartbeat::start('normal', false, $atomicDirectory, $clock);
|
|
$atomicPath = $files($atomicDirectory)[0];
|
|
$atomicBefore = file_get_contents($atomicPath);
|
|
file_put_contents($atomicPath . '.tmp', 'pre-existing object');
|
|
$atomic->success();
|
|
$assert(file_get_contents($atomicPath) === $atomicBefore
|
|
&& file_get_contents($atomicPath . '.tmp') === 'pre-existing object',
|
|
'Blocked atomic write corrupted the prior snapshot or removed another temporary file.');
|
|
|
|
$onceDirectory = $testRoot . '/once';
|
|
$once = CloudRuntimeHeartbeat::start('lifecycle', true, $onceDirectory, $clock);
|
|
$once->busy('lifecycle_tick');
|
|
$once->success();
|
|
$once->stopped();
|
|
$assert($checks($onceDirectory)['lifecycle']['status'] === 'unknown'
|
|
&& $checks($onceDirectory)['lifecycle']['active_count'] === 0
|
|
&& $read($onceDirectory)[0]['state'] === 'stopped',
|
|
'--once history falsely claimed a running daemon.');
|
|
|
|
$permissive = $testRoot . '/permissive';
|
|
mkdir($permissive, 0755);
|
|
chmod($permissive, 0755);
|
|
$permissiveHeartbeat = CloudRuntimeHeartbeat::start('normal', false, $permissive, $clock);
|
|
$permissiveFiles = $files($permissive);
|
|
$assert(count($permissiveFiles) === 1, 'Permissive runtime directory blocked startup.');
|
|
$permissiveHeartbeat->success();
|
|
chmod($permissiveFiles[0], 0644);
|
|
$assert($checks($permissive)['worker']['status'] === 'ok',
|
|
'Permissive heartbeat file mode blocked an otherwise valid observation.');
|
|
$symlinkDirectory = $testRoot . '/directory-link';
|
|
if (@symlink($directory, $symlinkDirectory)) {
|
|
$count = count($files($directory));
|
|
CloudRuntimeHeartbeat::start('normal', false, $symlinkDirectory, $clock)->success();
|
|
$assert(count($files($directory)) === $count
|
|
&& $checks($symlinkDirectory)['worker']['status'] === 'unknown',
|
|
'Symlink runtime directory was followed.');
|
|
}
|
|
$blocked = $testRoot . '/blocked';
|
|
file_put_contents($blocked, 'not a directory');
|
|
CloudRuntimeHeartbeat::start('normal', false, $blocked . '/child', $clock)->success();
|
|
CloudRuntimeHeartbeat::start('unsupported', false, $testRoot . '/invalid-kind', $clock)->success();
|
|
$assert(!file_exists($testRoot . '/invalid-kind'), 'Invalid worker kind created filesystem state.');
|
|
|
|
$corruptDirectory = $testRoot . '/corrupt';
|
|
$corrupt = CloudRuntimeHeartbeat::start('normal', false, $corruptDirectory, $clock);
|
|
$corrupt->success();
|
|
$good = file_get_contents($files($corruptDirectory)[0]);
|
|
$badFile = $corruptDirectory . '/normal-999-' . str_repeat('a', 24) . '.json';
|
|
foreach (['{truncated', str_repeat('x', 4097), json_encode(array_replace(json_decode($good, true),
|
|
['last_seen' => $now + 600, 'last_success' => $now + 600]))] as $bad) {
|
|
file_put_contents($badFile, $bad);
|
|
chmod($badFile, 0600);
|
|
$assert($checks($corruptDirectory)['worker']['status'] === 'warning',
|
|
'Corrupt, oversized or future-dated record falsely produced healthy status.');
|
|
}
|
|
unlink($badFile);
|
|
$target = $testRoot . '/private-target';
|
|
file_put_contents($target, $good);
|
|
chmod($target, 0600);
|
|
if (@symlink($target, $badFile)) {
|
|
$assert($checks($corruptDirectory)['worker']['status'] === 'warning'
|
|
&& file_get_contents($target) === $good, 'Snapshot accepted or modified a symlink heartbeat.');
|
|
unlink($badFile);
|
|
}
|
|
$busyRecord = json_decode($good, true);
|
|
$busyRecord['state'] = 'busy';
|
|
$busyRecord['busy_since'] = null;
|
|
file_put_contents($badFile, json_encode($busyRecord));
|
|
chmod($badFile, 0600);
|
|
$assert($checks($corruptDirectory)['worker']['status'] === 'warning', 'Malformed busy record was accepted.');
|
|
unlink($badFile);
|
|
|
|
$cappedDirectory = $testRoot . '/capped';
|
|
$capped = CloudRuntimeHeartbeat::start('normal', false, $cappedDirectory, $clock);
|
|
$capped->success();
|
|
for ($index = 0; $index < 513; $index++) {
|
|
file_put_contents($cappedDirectory . '/unrelated-' . $index, 'untouched');
|
|
}
|
|
$assert($checks($cappedDirectory)['worker']['status'] === 'warning',
|
|
'A truncated directory scan falsely returned complete healthy status.');
|
|
|
|
$cleanupDirectory = $testRoot . '/cleanup';
|
|
$old = CloudRuntimeHeartbeat::start('normal', false, $cleanupDirectory, $clock);
|
|
$oldPath = $files($cleanupDirectory)[0];
|
|
touch($oldPath, $now - 604801);
|
|
$unrelated = $cleanupDirectory . '/keep-me.json';
|
|
file_put_contents($unrelated, 'unrelated old file');
|
|
touch($unrelated, $now - 604801);
|
|
CloudRuntimeHeartbeat::start('normal', false, $cleanupDirectory, $clock);
|
|
$assert(!file_exists($oldPath) && file_get_contents($unrelated) === 'unrelated old file',
|
|
'Startup retention retained old telemetry or removed unrelated files.');
|
|
|
|
// Real CLI --once paths and real lifecycle internal catches, using empty SQLite schemas.
|
|
$capsule = new Capsule();
|
|
$capsule->addConnection(['driver' => 'sqlite', 'database' => ':memory:'], 'default');
|
|
$capsule->addConnection(['driver' => 'sqlite', 'database' => ':memory:'], 'log');
|
|
$capsule->setAsGlobal();
|
|
$capsule->bootEloquent();
|
|
$schema = $capsule->schema('default');
|
|
$schema->create('cloud_jobs', static function (Blueprint $table): void {
|
|
$table->increments('id');
|
|
foreach (['kind', 'status', 'sealed_payload', 'operation_id'] as $column) {
|
|
$table->string($column);
|
|
}
|
|
$table->string('request_idempotency_key')->default('');
|
|
$table->string('request_fingerprint')->default('');
|
|
$table->string('execution_mode')->default('agent');
|
|
$table->string('phase')->default('accepted');
|
|
$table->string('outcome')->nullable();
|
|
foreach (['available_at', 'lease_until', 'attempts', 'user_id',
|
|
'host_id', 'created_at', 'updated_at'] as $column) {
|
|
$table->integer($column)->default(0);
|
|
}
|
|
foreach (['dispatched_at', 'order_id', 'invoice_id', 'instance_id',
|
|
'snapshot_id', 'parent_job_id', 'dispatch_started_at',
|
|
'agent_accepted_at', 'receipt_deadline_at', 'terminal_at',
|
|
'effect_resolved_at', 'effect_resolved_by', 'read_at'] as $column) {
|
|
$table->integer($column)->nullable();
|
|
}
|
|
$table->integer('dispatch_attempts')->default(0);
|
|
$table->integer('dispatch_deadline_at')->default(0);
|
|
$table->boolean('effect_may_have_applied')->nullable();
|
|
$table->boolean('safe_to_resubmit')->nullable();
|
|
$table->string('effect_resolution')->default('not_applicable');
|
|
$table->string('effect_resolution_reason')->default('');
|
|
$table->string('completion_event_id')->nullable();
|
|
$table->string('lease_token')->nullable();
|
|
$table->string('error_code')->nullable();
|
|
$table->boolean('finalization_incident')->default(0);
|
|
$table->string('finalization_error_code')->nullable();
|
|
$table->text('result_json')->nullable();
|
|
});
|
|
$schema->create('cloud_agent_completion_events',
|
|
static function (Blueprint $table): void {
|
|
$table->string('event_id')->primary();
|
|
$table->integer('host_id');
|
|
$table->string('operation_id');
|
|
$table->string('payload_sha256');
|
|
$table->string('outcome');
|
|
$table->boolean('effect_may_have_applied');
|
|
$table->boolean('safe_to_resubmit');
|
|
$table->string('error_code')->default('');
|
|
$table->text('result_json')->nullable();
|
|
$table->string('disposition')->default('received');
|
|
$table->integer('received_at');
|
|
$table->integer('applied_at')->nullable();
|
|
$table->unique(['host_id', 'operation_id']);
|
|
});
|
|
$schema->create('cloud_templates', static function (
|
|
Blueprint $table
|
|
): void {
|
|
$table->increments('id');
|
|
$table->integer('active_job_id')->nullable();
|
|
$table->string('state')->default('active');
|
|
});
|
|
$schema->create('cloud_template_replicas', static function (
|
|
Blueprint $table
|
|
): void {
|
|
$table->integer('template_id');
|
|
$table->integer('job_id')->nullable();
|
|
$table->string('state')->default('pending');
|
|
});
|
|
$schema->create('cloud_user_isos', static function (
|
|
Blueprint $table
|
|
): void {
|
|
$table->increments('id');
|
|
$table->integer('active_job_id')->nullable();
|
|
$table->string('state')->default('ready');
|
|
});
|
|
$schema->create('cloud_user_iso_replicas', static function (
|
|
Blueprint $table
|
|
): void {
|
|
$table->integer('user_iso_id');
|
|
$table->integer('job_id')->nullable();
|
|
$table->string('state')->default('pending');
|
|
});
|
|
$schema->create('cloud_mail_outbox', static function (Blueprint $table): void {
|
|
$table->increments('id');
|
|
$table->string('status');
|
|
$table->integer('available_at');
|
|
$table->integer('lease_until');
|
|
$table->integer('invoice_id')->default(0);
|
|
$table->string('kind')->default('renewal_due');
|
|
$table->integer('attempts')->default(0);
|
|
$table->string('lease_token')->nullable();
|
|
$table->string('last_error')->default('');
|
|
$table->integer('updated_at')->default(0);
|
|
$table->integer('sent_at')->nullable();
|
|
});
|
|
$schema->create('cloud_instances', static function (Blueprint $table): void {
|
|
$table->increments('id');
|
|
foreach (['status', 'service_status', 'uuid', 'renewal_price'] as $column) {
|
|
$table->string($column)->nullable();
|
|
}
|
|
foreach (['host_id', 'user_id', 'expires_at', 'renewal_days'] as $column) {
|
|
$table->integer($column)->nullable();
|
|
}
|
|
});
|
|
$schema->create('cloud_invoices', static function (Blueprint $table): void {
|
|
$table->increments('id');
|
|
$table->string('purpose');
|
|
$table->string('status');
|
|
foreach (['due_at', 'instance_id', 'renews_expires_at'] as $column) {
|
|
$table->integer($column)->nullable();
|
|
}
|
|
$table->integer('user_id')->default(0);
|
|
$table->integer('created_at')->default(0);
|
|
});
|
|
foreach (['normal', 'policy', 'recovery', 'lifecycle'] as $kind) {
|
|
$commandDirectory = $testRoot . '/command-' . $kind;
|
|
$heartbeat = CloudRuntimeHeartbeat::start($kind, true, $commandDirectory, $clock);
|
|
$status = $kind === 'lifecycle' ? CloudLifecycle::run(['--once'], $heartbeat)
|
|
: CloudWorker::run($kind === 'policy'
|
|
? ['--once', '--policy-only']
|
|
: ($kind === 'recovery'
|
|
? ['--once', '--recovery-only'] : ['--once']), $heartbeat);
|
|
$record = $read($commandDirectory)[0];
|
|
if ($kind === 'lifecycle') {
|
|
$assert($status === 0 && $record['state'] === 'stopped'
|
|
&& $record['last_success'] === null
|
|
&& $record['last_error'] === 'traffic_sync_failed'
|
|
&& $record['unresolved_error'] === true
|
|
&& $record['mode'] === 'once',
|
|
'Lifecycle --once hid its caught traffic-sync failure: '
|
|
. json_encode($record, JSON_UNESCAPED_SLASHES));
|
|
} else {
|
|
$assert($status === 0 && $record['state'] === 'stopped'
|
|
&& $record['last_success'] === $now
|
|
&& $record['last_error'] === null
|
|
&& $record['mode'] === 'once',
|
|
'Worker --once did not finish as a successful stopped observation.');
|
|
}
|
|
}
|
|
|
|
$workerCommand = file_get_contents(dirname(__DIR__) . '/app/Command/CloudWorker.php');
|
|
$lifecycleCommand = file_get_contents(dirname(__DIR__) . '/app/Command/CloudLifecycle.php');
|
|
$assert(!str_contains($workerCommand, 'CloudEngineReadiness')
|
|
&& !str_contains($lifecycleCommand, 'CloudEngineReadiness')
|
|
&& !str_contains($workerCommand, 'engine_readiness')
|
|
&& !str_contains($lifecycleCommand, 'engine_readiness'),
|
|
'A retired global engine-readiness gate still blocks per-host Agent work.');
|
|
$workerUnit = file_get_contents(dirname(__DIR__, 2)
|
|
. '/deploy/virt-cloud-recovery-worker.service');
|
|
$xcat = file_get_contents(dirname(__DIR__) . '/app/Command/XCat.php');
|
|
$assert(str_contains($workerCommand, "'--recovery-only'")
|
|
&& str_contains($workerCommand, 'CloudAutomaticOperationRecoveryService')
|
|
&& str_contains($workerCommand, '!$policyOnly && !$recoveryOnly')
|
|
&& str_contains($xcat, '--recovery-only')
|
|
&& str_contains($workerUnit, 'cloudWorker --recovery-only'),
|
|
'The dedicated recovery worker is missing, sends mail, or is not deployable.');
|
|
|
|
Capsule::table('cloud_instances')->insert(['id' => 10, 'status' => 'active',
|
|
'service_status' => 'active', 'expires_at' => time() + 3600,
|
|
'renewal_price' => 'invalid-amount', 'renewal_days' => 30]);
|
|
$assert(CloudInvoiceService::issueDue() === 0
|
|
&& CloudInvoiceService::observationErrors('issue_due') === ['invoice_generation_failed']
|
|
&& Capsule::table('cloud_invoices')->count() === 0,
|
|
'Per-invoice generation failure was hidden or changed invoice outcomes.');
|
|
Capsule::table('cloud_instances')->delete();
|
|
$assert(CloudInvoiceService::issueDue() === 0
|
|
&& CloudInvoiceService::observationErrors('issue_due') === [],
|
|
'Invoice observation failures carried into an unrelated subsequent invocation.');
|
|
$schema->create('user', static function (Blueprint $table): void {
|
|
$table->increments('id');
|
|
$table->string('email');
|
|
});
|
|
Capsule::table('user')->insert(['id' => 1, 'email' => 'invalid-email']);
|
|
Capsule::table('cloud_instances')->insert(['id' => 10, 'user_id' => 1,
|
|
'status' => 'active', 'service_status' => 'active']);
|
|
Capsule::table('cloud_invoices')->insert(['id' => 10, 'user_id' => 1, 'instance_id' => 10,
|
|
'purpose' => 'renewal', 'status' => 'open', 'due_at' => time() + 3600]);
|
|
Capsule::table('cloud_mail_outbox')->insert(['id' => 10, 'invoice_id' => 10,
|
|
'status' => 'pending', 'available_at' => 0, 'lease_until' => 0]);
|
|
$assert(CloudInvoiceService::deliverReminders(1) === 0
|
|
&& CloudInvoiceService::observationErrors('notifications') === ['notification_failed']
|
|
&& Capsule::table('cloud_mail_outbox')->where('id', 10)->value('status') === 'failed'
|
|
&& Capsule::table('cloud_mail_outbox')->where('id', 10)->value('attempts') === 1,
|
|
'Non-throwing notification failure was hidden or left eligible for retry.');
|
|
$assert(CloudInvoiceService::deliverReminders(1) === 0
|
|
&& CloudInvoiceService::observationErrors('notifications') === [],
|
|
'Notification observation failures carried into the next idle invocation.');
|
|
Capsule::table('cloud_mail_outbox')->where('id', 10)->update([
|
|
'status' => 'pending', 'attempts' => 0, 'available_at' => 0,
|
|
]);
|
|
Capsule::table('cloud_instances')->delete();
|
|
$assert(CloudInvoiceService::deliverReminders(1) === 0
|
|
&& CloudInvoiceService::observationErrors('notifications') === []
|
|
&& Capsule::table('cloud_mail_outbox')->where('id', 10)->value('status') === 'cancelled',
|
|
'Expected cancellation was incorrectly reported as a delivery failure.');
|
|
Capsule::table('cloud_mail_outbox')->delete();
|
|
$schema->drop('user');
|
|
|
|
$schema->drop('cloud_invoices');
|
|
$partialDirectory = $testRoot . '/partial';
|
|
$partial = CloudRuntimeHeartbeat::start('lifecycle', false, $partialDirectory, $clock);
|
|
$lifecycle = new CloudLifecycleService($partial);
|
|
$assert($lifecycle->tick() === 0
|
|
&& $lifecycle->observationErrors() === ['provision_expiry_failed',
|
|
'invoice_generation_failed', 'traffic_sync_failed']
|
|
&& $checks($partialDirectory)['lifecycle']['status'] === 'warning'
|
|
&& $checks($partialDirectory)['lifecycle']['last_success_at'] === null,
|
|
'Lifecycle swallowed sub-stage failures and falsely reported normal operation: '
|
|
. json_encode($lifecycle->observationErrors(), JSON_UNESCAPED_SLASHES));
|
|
$schema->drop('cloud_instances');
|
|
$schema->create('cloud_instances', static function (Blueprint $table): void {
|
|
$table->increments('id');
|
|
$table->string('status');
|
|
$table->string('service_status');
|
|
});
|
|
Capsule::table('cloud_instances')->insert(['id' => 1, 'status' => 'active', 'service_status' => 'active']);
|
|
$assert($lifecycle->tick() === 0
|
|
&& $lifecycle->observationErrors() === ['provision_expiry_failed', 'invoice_generation_failed',
|
|
'traffic_sync_failed', 'instance_reconcile_failed']
|
|
&& $checks($partialDirectory)['lifecycle']['status'] === 'warning',
|
|
'All internally caught lifecycle stages failed without an unhealthy observation: '
|
|
. json_encode($lifecycle->observationErrors(), JSON_UNESCAPED_SLASHES));
|
|
$partialOnceDirectory = $testRoot . '/partial-once';
|
|
$partialOnce = CloudRuntimeHeartbeat::start('lifecycle', true, $partialOnceDirectory, $clock);
|
|
$assert(CloudLifecycle::run(['--once'], $partialOnce) === 0
|
|
&& $read($partialOnceDirectory)[0]['last_success'] === null
|
|
&& $read($partialOnceDirectory)[0]['last_error'] === 'provision_expiry_failed'
|
|
&& $read($partialOnceDirectory)[0]['state'] === 'stopped',
|
|
'Partial failure changed legacy CLI exit semantics or was marked successful.');
|
|
|
|
Capsule::table('cloud_jobs')->insert(['kind' => 'action',
|
|
'execution_mode' => 'agent', 'phase' => 'accepted',
|
|
'status' => 'queued', 'user_id' => 1, 'host_id' => 1,
|
|
'sealed_payload' => 'deliberately-invalid-cipher',
|
|
'operation_id' => str_repeat('a', 32),
|
|
'request_idempotency_key' => str_repeat('a', 32),
|
|
'request_fingerprint' => str_repeat('b', 64),
|
|
'dispatch_deadline_at' => time() + 300,
|
|
'available_at' => 0, 'lease_until' => 0]);
|
|
$worker = new CloudWorkerService();
|
|
$assert($worker->tick() === true && in_array('worker_tick_failed', $worker->observationErrors(), true)
|
|
&& Capsule::table('cloud_jobs')->value('phase') === 'terminal'
|
|
&& Capsule::table('cloud_jobs')->value('outcome') === 'failed'
|
|
&& Capsule::table('cloud_jobs')->value('status') === 'failed',
|
|
'Worker exception did not become one terminal failed receipt.');
|
|
$assert($worker->tick() === false && $worker->observationErrors() === [],
|
|
'Worker observation errors were not reset for the next idle cycle.');
|
|
|
|
echo "Cloud runtime private atomic files, bounded snapshots, long-call freshness, multi-worker identity, --once CLI and SQLite partial failures passed.\n";
|
|
} finally {
|
|
ini_set('error_log', $oldLog === false ? '' : $oldLog);
|
|
$removeTree($testRoot);
|
|
}
|