You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
virt/web/tests/CloudPowerReceiptWorkflowTe...

407 lines
22 KiB
PHTML

<?php
namespace App\Services {
final class Config
{
public static function get($name, $default = null) { return $default; }
}
}
namespace App\Services\Cloud {
/** Test-only keyring; never used by the application. */
final class CloudSecrets
{
private static array $values = [];
public static function encrypt(array $value): string
{
$key = bin2hex(random_bytes(16));
self::$values[$key] = $value;
return $key;
}
public static function decrypt(string $key): array { return self::$values[$key]; }
}
/** Simulate a healthy power API and unavailable guest/private readback. */
final class CloudEngineClient
{
public array $calls = [];
public array $response = [];
public $afterCommand = null;
public function host(int $host, string $method, string $resource,
array $actor, ?array $body, string $operation): array
{
$this->calls[] = [$host, $method, $resource, $body];
if (!str_ends_with($resource, '/actions')
&& !str_ends_with($resource, '/settings') && $method !== 'DELETE') {
throw new CloudError('Private network/QGA unavailable', 503);
}
if ($this->afterCommand !== null) { ($this->afterCommand)(); }
return $this->response + ['operation_id' => $operation];
}
}
}
namespace {
require dirname(__DIR__) . '/vendor/autoload.php';
use App\Models\CloudInstance;
use App\Models\CloudInstanceRuntime;
use App\Models\CloudJob;
use App\Models\CloudVpcMember;
use App\Models\CloudVpcOperation;
use App\Services\Cloud\CloudEngineClient;
use App\Services\Cloud\CloudError;
use App\Services\Cloud\CloudInstanceRuntimeService;
use App\Services\Cloud\CloudJobService;
use App\Services\Cloud\CloudOperationErrorCatalog;
use App\Services\Cloud\CloudSecrets;
use App\Services\Cloud\CloudVpcService;
use App\Services\Cloud\CloudVpcPowerWorkflow;
use App\Services\Cloud\CloudWorkerService;
use Illuminate\Database\Capsule\Manager as Capsule;
use Illuminate\Database\Schema\Blueprint;
$assert = static function (bool $condition, string $message): void {
if (!$condition) { throw new RuntimeException($message); }
};
$capsule = new Capsule();
$capsule->addConnection(['driver' => 'sqlite', 'database' => ':memory:'], 'default');
$capsule->setAsGlobal();
$capsule->bootEloquent();
$schema = $capsule->schema();
$schema->create('user', static function (Blueprint $t): void {
$t->increments('id'); $t->integer('enable')->default(1);
$t->boolean('is_admin')->default(false);
});
$schema->create('cloud_instances', static function (Blueprint $t): void {
$t->increments('id'); $t->integer('host_id'); $t->integer('user_id');
$t->integer('order_id')->nullable(); $t->string('uuid'); $t->string('name');
$t->string('status')->default('active');
$t->string('service_status')->default('exempt');
$t->string('network_status')->default('active');
$t->integer('expires_at')->nullable(); $t->text('spec_json')->default('{}');
$t->integer('updated_at')->default(0);
});
$schema->create('cloud_jobs', static function (Blueprint $t): void {
$t->increments('id'); $t->string('operation_id')->unique();
$t->integer('order_id')->nullable(); $t->integer('instance_id')->nullable();
$t->integer('snapshot_id')->nullable(); $t->integer('user_id'); $t->integer('host_id');
$t->string('kind'); $t->string('status'); $t->text('payload_cipher');
$t->integer('attempts')->default(0); $t->integer('dispatched_at')->nullable();
$t->string('lease_token')->nullable(); $t->integer('lease_until')->default(0);
$t->integer('available_at'); $t->integer('created_at'); $t->integer('updated_at');
$t->string('error_code')->nullable(); $t->text('result_json')->nullable();
});
$schema->create('cloud_instance_runtime', static function (Blueprint $t): void {
$t->integer('instance_id')->primary(); $t->integer('host_id'); $t->string('uuid');
$t->string('power_state'); $t->string('sync_state'); $t->string('last_error_code');
foreach (['last_success_at', 'last_attempt_at', 'revision', 'created_at', 'updated_at'] as $key) {
$t->integer($key);
}
});
$schema->create('cloud_instance_snapshots', static function (Blueprint $t): void {
$t->increments('id'); $t->integer('instance_id'); $t->string('state');
$t->string('last_operation_id')->nullable(); $t->integer('deleted_at')->nullable();
$t->integer('updated_at')->default(0);
});
$schema->create('cloud_vpcs', static function (Blueprint $t): void {
$t->increments('id'); $t->string('vpc_key'); $t->integer('desired_revision');
$t->string('desired_digest'); $t->string('state'); $t->string('sync_state');
$t->string('last_operation_id')->nullable();
});
$schema->create('cloud_vpc_operations', static function (Blueprint $t): void {
$t->increments('id'); $t->integer('vpc_id'); $t->integer('user_id');
$t->string('operation_id'); $t->string('kind'); $t->string('status');
$t->integer('desired_revision'); $t->text('power_workflow_json')->nullable();
$t->unsignedBigInteger('power_instance_id')->nullable()->index();
$t->string('lease_token')->nullable(); $t->integer('lease_until')->default(0);
$t->string('error_code')->nullable(); $t->integer('updated_at')->default(0);
});
$schema->create('cloud_vpc_host_deployments', static function (Blueprint $t): void {
$t->increments('id'); $t->integer('vpc_id'); $t->integer('host_id'); $t->string('state');
});
foreach (['cloud_vpc_members', 'cloud_vpc_private_ips'] as $table) {
$schema->create($table, static function (Blueprint $t) use ($table): void {
$t->increments('id'); $t->integer('vpc_id'); $t->integer('subnet_id');
$t->integer('instance_id'); $t->string('private_ip'); $t->string('state');
$t->integer('desired_revision'); $t->string('desired_digest');
$t->integer('observed_revision')->default(0); $t->string('observed_digest')->nullable();
$t->string('last_error_code')->nullable(); $t->integer('updated_at')->default(0);
if ($table === 'cloud_vpc_members') {
$t->string('member_key'); $t->integer('host_id'); $t->integer('private_ip_id');
$t->string('last_operation_id')->nullable();
} else {
$t->integer('member_id');
}
});
}
$uuid = '11111111-1111-4111-8111-111111111110';
Capsule::table('user')->insert(['id' => 1]);
Capsule::table('cloud_instances')->insert([
'id' => 10, 'host_id' => 7, 'user_id' => 1, 'uuid' => $uuid, 'name' => 'Power VM',
]);
Capsule::table('cloud_vpcs')->insert([
'id' => 1, 'vpc_key' => str_repeat('a', 32), 'desired_revision' => 1,
'desired_digest' => str_repeat('d', 64), 'state' => 'active', 'sync_state' => 'failed',
]);
$member = ['id' => 1, 'vpc_id' => 1, 'subnet_id' => 1, 'instance_id' => 10,
'private_ip' => '10.0.0.1', 'state' => 'uncertain',
'desired_revision' => 1, 'desired_digest' => str_repeat('d', 64)];
Capsule::table('cloud_vpc_members')->insert($member + [
'member_key' => str_repeat('b', 32), 'host_id' => 7, 'private_ip_id' => 1,
]);
Capsule::table('cloud_vpc_private_ips')->insert($member + ['member_id' => 1]);
$actor = ['user_id' => 1, 'is_admin' => false, 'is_super_admin' => false,
'can_manage' => false, 'session_id' => ''];
$engine = new CloudEngineClient();
$worker = new CloudWorkerService($engine);
$enqueue = static fn (string $action): CloudJob => CloudJobService::enqueue(
CloudInstance::find(10), $actor, 'action', 'POST', '/actions', ['action' => $action]
);
$expectation = CloudVpcService::instancePrivateExpectation(10);
$assert($expectation['required'] === true && $expectation['reason_code'] === null,
'Fixture must have a valid attached private target despite failed readiness.');
// Each command can be followed immediately by another command even when
// the guest is broken and every private-readback endpoint would fail.
foreach (['shutdown', 'poweroff', 'start', 'reboot', 'reset'] as $action) {
$instance = CloudInstance::find(10);
$instance->network_status = $action === 'start' ? 'blocked_traffic' : 'active';
$instance->save();
$beforeState = $action === 'start' ? 'stopped' : 'running';
CloudInstanceRuntimeService::markObserved(10, 7, $uuid, $beforeState);
$job = $enqueue($action);
$engine->response = ['status' => 'succeeded', 'result' => [
'uuid' => $uuid, 'action' => $action, 'command_accepted' => true,
]];
$count = count($engine->calls);
$assert($worker->tick(), 'Worker did not claim the power request.');
$done = CloudJob::find($job->id);
$receipt = json_decode($done->result_json, true);
$assert($done->status === 'succeeded' && $done->lease_token === null
&& (int) $done->lease_until === 0 && count($engine->calls) === $count + 1,
'Confirmed ' . $action . ' waited for or dispatched private reconciliation.');
$assert(($receipt['command_accepted'] ?? null) === true
&& ($receipt['action'] ?? null) === $action && !isset($receipt['state']),
'Power receipt was discarded or fabricated an observed power state.');
$assert(CloudInstanceRuntime::find(10)->power_state
=== ($action === 'shutdown' ? 'shutdown' : $beforeState),
'Receipt-only command fabricated a running/stopped observation.');
$assert(CloudInstance::find(10)->network_status === $instance->network_status
&& Capsule::table('cloud_vpc_members')->value('state') === 'uncertain'
&& Capsule::table('cloud_vpcs')->value('sync_state') === 'failed',
'Ordinary power command changed independent network policy/readiness.');
}
// Exercise the real admission path: a VPC graceful-shutdown timeout must
// permit force-off and durably revoke the old automatic restart intent.
$vpcOperation = new CloudVpcOperation();
$vpcOperation->operation_id = str_repeat('c', 32);
$vpcOperation->vpc_id = 1;
$vpcOperation->user_id = 1;
$vpcOperation->kind = 'member_attach';
$vpcOperation->status = 'paused';
$vpcOperation->desired_revision = 1;
$vpcOperation->save();
CloudVpcPowerWorkflow::initialize($vpcOperation,
CloudInstance::find(10), CloudVpcMember::find(1));
$powerWorkflow = CloudVpcPowerWorkflow::state($vpcOperation);
$powerWorkflow['phase'] = 'paused';
$powerWorkflow['paused_from'] = 'waiting_stopped';
$powerWorkflow['original_running'] = true;
$powerWorkflow['error_code'] = 'vpc_power_shutdown_timeout';
$vpcOperation->power_workflow_json = json_encode($powerWorkflow);
$vpcOperation->save();
Capsule::table('cloud_vpcs')->where('id', 1)->update([
'last_operation_id' => $vpcOperation->operation_id,
]);
try {
CloudJobService::enqueue(CloudInstance::find(10), $actor,
'instance_settings', 'POST', '/settings', [
'hostname' => 'during-vpc', 'confirm_restart' => true,
]);
throw new RuntimeException('Settings mutation bypassed unfinished VPC maintenance.');
} catch (CloudError $error) {
$assert($error->httpStatus === 409, 'Unexpected VPC admission failure.');
}
$manualPower = $enqueue('poweroff');
$assert($manualPower->status === 'queued'
&& CloudVpcPowerWorkflow::state($vpcOperation->fresh())['resume_cancelled'] === true,
'VPC timeout blocked force-off or did not revoke automatic restart.');
$engine->response = ['status' => 'succeeded', 'result' => [
'uuid' => $uuid, 'action' => 'poweroff', 'command_accepted' => true,
]];
$worker->tick();
$assert($manualPower->fresh()->status === 'succeeded',
'Force-off after VPC timeout acquired a private-readiness dependency.');
// End this isolated fixture; production completion requires host proof.
$powerWorkflow = CloudVpcPowerWorkflow::state($vpcOperation->fresh());
$powerWorkflow['phase'] = 'completed';
$vpcOperation->power_workflow_json = json_encode($powerWorkflow);
$vpcOperation->save();
// A changing private target does not invalidate an already accepted power
// command in finish()'s transaction either.
$job = $enqueue('shutdown');
$engine->response = ['status' => 'succeeded', 'result' => [
'uuid' => $uuid, 'action' => 'shutdown', 'command_accepted' => true,
]];
$engine->afterCommand = static function (): void {
foreach (['cloud_vpcs', 'cloud_vpc_members', 'cloud_vpc_private_ips'] as $table) {
Capsule::table($table)->update(['desired_revision' => 2]);
}
};
$worker->tick();
$assert(CloudJob::find($job->id)->status === 'succeeded',
'Private revision change retroactively blocked confirmed power receipt.');
$engine->afterCommand = null;
$next = $enqueue('poweroff');
$assert($next->status === 'queued', 'Accepted shutdown still blocks force poweroff.');
// Isolate the following rejection cases; no production cancellation code is used.
$next->status = 'cancelled'; $next->save();
// An engine-side preflight/admission failure is terminal even though PHP
// sent HTTP and the VM has an unavailable private network. No guest write
// happened, so it must not dispatch VPC reconciliation or block force-off.
foreach (['action', 'instance_settings'] as $kind) {
$job = $kind === 'action' ? $enqueue('reboot')
: CloudJobService::enqueue(CloudInstance::find(10), $actor,
'instance_settings', 'POST', '/settings', [
'hostname' => 'preflight-test', 'confirm_restart' => true,
]);
$engine->response = ['status' => 'failed',
'execution_outcome' => 'not_dispatched', 'mutation_started' => false,
'safe_to_refund' => false, 'error_code' => 'execution_result_uncertain'];
$count = count($engine->calls);
$worker->tick();
$failed = CloudJob::find($job->id);
$assert($failed->status === 'failed' && $failed->lease_token === null
&& (int) $failed->lease_until === 0
&& count($engine->calls) === $count + 1,
'Proven preflight failure acquired a private-state dependency for ' . $kind . '.');
$followup = $enqueue('poweroff');
$engine->response = ['status' => 'succeeded', 'result' => [
'uuid' => $uuid, 'action' => 'poweroff', 'command_accepted' => true,
]];
$worker->tick();
$assert(CloudJob::find($followup->id)->status === 'succeeded',
'Preflight failure blocked the next force-off request for ' . $kind . '.');
}
// User deletion has a separate instance admission latch. Undo it only
// for this operation's explicit no-write receipt, never a legacy or
// contradictory outcome; preserve the snapshot catalog in all failures.
foreach ([[], ['execution_outcome' => null], ['mutation_started' => true],
['operation_id' => str_repeat('f', 32)], ['executor_active' => true],
['vm_delete_committed' => true], ['result' => ['cleanup_pending' => true]]]
as $index => $change) {
$id = 11 + $index;
$targetUuid = sprintf('22222222-2222-4222-8222-%012d', $id);
Capsule::table('cloud_instances')->insert(['id' => $id, 'host_id' => 7,
'user_id' => 1, 'uuid' => $targetUuid, 'name' => 'Delete preflight']);
Capsule::table('cloud_instance_snapshots')->insert([
'instance_id' => $id, 'state' => 'ready']);
$delete = CloudJobService::enqueue(CloudInstance::find($id), $actor,
'delete', 'DELETE', '?delete_disks=true');
$assert(CloudInstance::find($id)->status === 'deleting',
'Delete request did not establish its admission latch.');
$engine->response = array_replace(['status' => 'failed',
'execution_outcome' => 'not_dispatched', 'mutation_started' => false,
'safe_to_refund' => false, 'error_code' => 'execution_result_uncertain'], $change);
$worker->tick();
$assert(CloudJob::find($delete->id)->status === 'failed'
&& CloudInstance::find($id)->status === ($index === 0 ? 'active' : 'delete_failed')
&& Capsule::table('cloud_instance_snapshots')->where('instance_id', $id)
->value('state') === 'ready',
'User-delete latch ignored missing/conflicting execution proof.');
if ($index === 0) {
$next = CloudJobService::enqueue(CloudInstance::find($id), $actor,
'action', 'POST', '/actions', ['action' => 'poweroff']);
$next->status = 'cancelled'; $next->save();
} else {
try {
CloudJobService::enqueue(CloudInstance::find($id), $actor,
'action', 'POST', '/actions', ['action' => 'poweroff']);
throw new RuntimeException('Unproven failed delete admitted another write.');
} catch (CloudError $error) {
$assert($error->httpStatus === 409, 'Unproven delete had unexpected admission.');
}
}
}
// Expiry cleanup owns a separate service-policy latch. A no-write failure
// must not restore an expired instance to an active service.
Capsule::table('cloud_instances')->insert(['id' => 30, 'host_id' => 7,
'user_id' => 1, 'uuid' => '22222222-2222-4222-8222-000000000030',
'name' => 'Expiry preflight', 'status' => 'deleting',
'service_status' => 'purge_deleting']);
$expiry = CloudJobService::enqueueExpiryDelete(CloudInstance::find(30));
$engine->response = ['status' => 'failed', 'execution_outcome' => 'not_dispatched',
'mutation_started' => false, 'safe_to_refund' => false];
$worker->tick();
$assert(CloudJob::find($expiry->id)->status === 'failed'
&& CloudInstance::find(30)->status === 'delete_failed'
&& CloudInstance::find(30)->service_status === 'purge_failed',
'Preflight failure reopened an expired service instead of preserving its policy latch.');
$gate = new ReflectionMethod(CloudWorkerService::class, 'applyPrivateLifecycleGate');
$confirmed = new ReflectionMethod(CloudWorkerService::class, 'confirmedPowerReceipt');
$job = $enqueue('shutdown');
$payload = CloudSecrets::decrypt($job->payload_cipher);
$success = ['operation_id' => $job->operation_id, 'status' => 'succeeded', 'result' => [
'uuid' => $uuid, 'action' => 'shutdown', 'command_accepted' => true,
]];
$assert($confirmed->invoke(null, $job, $success, $payload), 'Valid receipt not recognized.');
$invalid = [
array_replace_recursive($success, ['result' => ['uuid' => '22222222-2222-4222-8222-222222222222']]),
array_replace_recursive($success, ['result' => ['action' => 'poweroff']]),
array_replace_recursive($success, ['result' => ['command_accepted' => 'true']]),
array_replace_recursive($success, ['result' => ['command_accepted' => false]]),
array_replace($success, ['operation_id' => str_repeat('f', 32)]),
array_replace($success, ['status' => 'uncertain']),
array_replace($success, ['status' => 'running']),
array_replace($success, ['status' => 'failed']),
];
foreach ($invalid as $operation) {
$assert(!$confirmed->invoke(null, $job, $operation, $payload),
'Mismatched, unconfirmed or nonterminal receipt bypassed the private gate.');
}
foreach ([['method' => 'GET'], ['resource' => 'vms/' . $uuid . '/reinstall'],
['body' => ['action' => 'delete']]] as $change) {
$assert(!$confirmed->invoke(null, $job, $success, array_replace($payload, $change)),
'Receipt bypass was not bound to the immutable power request.');
}
$safe = CloudOperationErrorCatalog::workerResult([
'command_accepted' => 'true', 'action' => 'arbitrary-host-message',
]);
$assert($safe === [], 'Unvalidated power receipt fields reached stored/user results.');
// A lost power response is still uncertain, and the next write is rejected.
$engine->response = ['status' => 'uncertain', 'mutation_started' => true,
'execution_outcome' => 'unknown',
'error_code' => 'libvirt_execution_uncertain'];
$worker->tick();
$assert(CloudJob::find($job->id)->status === 'uncertain',
'A truly uncertain write was upgraded to success.');
try {
$enqueue('poweroff');
throw new RuntimeException('Unknown write allowed a new power mutation.');
} catch (CloudError $error) {
$assert($error->httpStatus === 409, 'Unknown write had an unexpected admission result.');
}
// Unavailable private proof still gates destructive and policy mutations.
// Invoke the production gate directly to avoid manufacturing their finance
// ledgers; these are not ordinary action jobs even if a result carries a receipt.
foreach (['delete', 'expiry_delete', 'reinstall', 'network_change',
'network_suspend', 'network_resume', 'service_suspend', 'service_resume'] as $kind) {
$other = clone $job;
$other->kind = $kind;
$count = count($engine->calls);
$result = $gate->invoke($worker, $other, $success, $payload, $actor);
$assert($result['status'] === 'uncertain' && count($engine->calls) > $count,
'Power receipt weakened private readback for ' . $kind . '.');
}
echo "Power command receipt workflow tests passed.\n";
}