calls[] = [$host, $method, $resource, $body]; if (!str_ends_with($resource, '/actions') && !str_ends_with($resource, '/settings') && $method !== 'DELETE') { throw new CloudError('Private network/QGA unavailable', 503); } if ($this->afterCommand !== null) { ($this->afterCommand)(); } return $this->response + ['operation_id' => $operation]; } } } namespace { require dirname(__DIR__) . '/vendor/autoload.php'; use App\Models\CloudInstance; use App\Models\CloudInstanceRuntime; use App\Models\CloudJob; use App\Models\CloudVpcMember; use App\Models\CloudVpcOperation; use App\Services\Cloud\CloudEngineClient; use App\Services\Cloud\CloudError; use App\Services\Cloud\CloudInstanceRuntimeService; use App\Services\Cloud\CloudJobService; use App\Services\Cloud\CloudOperationErrorCatalog; use App\Services\Cloud\CloudSecrets; use App\Services\Cloud\CloudVpcService; use App\Services\Cloud\CloudVpcPowerWorkflow; use App\Services\Cloud\CloudWorkerService; use Illuminate\Database\Capsule\Manager as Capsule; use Illuminate\Database\Schema\Blueprint; $assert = static function (bool $condition, string $message): void { if (!$condition) { throw new RuntimeException($message); } }; $capsule = new Capsule(); $capsule->addConnection(['driver' => 'sqlite', 'database' => ':memory:'], 'default'); $capsule->setAsGlobal(); $capsule->bootEloquent(); $schema = $capsule->schema(); $schema->create('user', static function (Blueprint $t): void { $t->increments('id'); $t->integer('enable')->default(1); $t->boolean('is_admin')->default(false); }); $schema->create('cloud_instances', static function (Blueprint $t): void { $t->increments('id'); $t->integer('host_id'); $t->integer('user_id'); $t->integer('order_id')->nullable(); $t->string('uuid'); $t->string('name'); $t->string('status')->default('active'); $t->string('service_status')->default('exempt'); $t->string('network_status')->default('active'); $t->integer('expires_at')->nullable(); $t->text('spec_json')->default('{}'); $t->integer('updated_at')->default(0); }); $schema->create('cloud_jobs', static function (Blueprint $t): void { $t->increments('id'); $t->string('operation_id')->unique(); $t->integer('order_id')->nullable(); $t->integer('instance_id')->nullable(); $t->integer('snapshot_id')->nullable(); $t->integer('user_id'); $t->integer('host_id'); $t->string('kind'); $t->string('status'); $t->text('payload_cipher'); $t->integer('attempts')->default(0); $t->integer('dispatched_at')->nullable(); $t->string('lease_token')->nullable(); $t->integer('lease_until')->default(0); $t->integer('available_at'); $t->integer('created_at'); $t->integer('updated_at'); $t->string('error_code')->nullable(); $t->text('result_json')->nullable(); }); $schema->create('cloud_instance_runtime', static function (Blueprint $t): void { $t->integer('instance_id')->primary(); $t->integer('host_id'); $t->string('uuid'); $t->string('power_state'); $t->string('sync_state'); $t->string('last_error_code'); foreach (['last_success_at', 'last_attempt_at', 'revision', 'created_at', 'updated_at'] as $key) { $t->integer($key); } }); $schema->create('cloud_instance_snapshots', static function (Blueprint $t): void { $t->increments('id'); $t->integer('instance_id'); $t->string('state'); $t->string('last_operation_id')->nullable(); $t->integer('deleted_at')->nullable(); $t->integer('updated_at')->default(0); }); $schema->create('cloud_vpcs', static function (Blueprint $t): void { $t->increments('id'); $t->string('vpc_key'); $t->integer('desired_revision'); $t->string('desired_digest'); $t->string('state'); $t->string('sync_state'); $t->string('last_operation_id')->nullable(); }); $schema->create('cloud_vpc_operations', static function (Blueprint $t): void { $t->increments('id'); $t->integer('vpc_id'); $t->integer('user_id'); $t->string('operation_id'); $t->string('kind'); $t->string('status'); $t->integer('desired_revision'); $t->text('power_workflow_json')->nullable(); $t->unsignedBigInteger('power_instance_id')->nullable()->index(); $t->string('lease_token')->nullable(); $t->integer('lease_until')->default(0); $t->string('error_code')->nullable(); $t->integer('updated_at')->default(0); }); $schema->create('cloud_vpc_host_deployments', static function (Blueprint $t): void { $t->increments('id'); $t->integer('vpc_id'); $t->integer('host_id'); $t->string('state'); }); foreach (['cloud_vpc_members', 'cloud_vpc_private_ips'] as $table) { $schema->create($table, static function (Blueprint $t) use ($table): void { $t->increments('id'); $t->integer('vpc_id'); $t->integer('subnet_id'); $t->integer('instance_id'); $t->string('private_ip'); $t->string('state'); $t->integer('desired_revision'); $t->string('desired_digest'); $t->integer('observed_revision')->default(0); $t->string('observed_digest')->nullable(); $t->string('last_error_code')->nullable(); $t->integer('updated_at')->default(0); if ($table === 'cloud_vpc_members') { $t->string('member_key'); $t->integer('host_id'); $t->integer('private_ip_id'); $t->string('last_operation_id')->nullable(); } else { $t->integer('member_id'); } }); } $uuid = '11111111-1111-4111-8111-111111111110'; Capsule::table('user')->insert(['id' => 1]); Capsule::table('cloud_instances')->insert([ 'id' => 10, 'host_id' => 7, 'user_id' => 1, 'uuid' => $uuid, 'name' => 'Power VM', ]); Capsule::table('cloud_vpcs')->insert([ 'id' => 1, 'vpc_key' => str_repeat('a', 32), 'desired_revision' => 1, 'desired_digest' => str_repeat('d', 64), 'state' => 'active', 'sync_state' => 'failed', ]); $member = ['id' => 1, 'vpc_id' => 1, 'subnet_id' => 1, 'instance_id' => 10, 'private_ip' => '10.0.0.1', 'state' => 'uncertain', 'desired_revision' => 1, 'desired_digest' => str_repeat('d', 64)]; Capsule::table('cloud_vpc_members')->insert($member + [ 'member_key' => str_repeat('b', 32), 'host_id' => 7, 'private_ip_id' => 1, ]); Capsule::table('cloud_vpc_private_ips')->insert($member + ['member_id' => 1]); $actor = ['user_id' => 1, 'is_admin' => false, 'is_super_admin' => false, 'can_manage' => false, 'session_id' => '']; $engine = new CloudEngineClient(); $worker = new CloudWorkerService($engine); $enqueue = static fn (string $action): CloudJob => CloudJobService::enqueue( CloudInstance::find(10), $actor, 'action', 'POST', '/actions', ['action' => $action] ); $expectation = CloudVpcService::instancePrivateExpectation(10); $assert($expectation['required'] === true && $expectation['reason_code'] === null, 'Fixture must have a valid attached private target despite failed readiness.'); // Each command can be followed immediately by another command even when // the guest is broken and every private-readback endpoint would fail. foreach (['shutdown', 'poweroff', 'start', 'reboot', 'reset'] as $action) { $instance = CloudInstance::find(10); $instance->network_status = $action === 'start' ? 'blocked_traffic' : 'active'; $instance->save(); $beforeState = $action === 'start' ? 'stopped' : 'running'; CloudInstanceRuntimeService::markObserved(10, 7, $uuid, $beforeState); $job = $enqueue($action); $engine->response = ['status' => 'succeeded', 'result' => [ 'uuid' => $uuid, 'action' => $action, 'command_accepted' => true, ]]; $count = count($engine->calls); $assert($worker->tick(), 'Worker did not claim the power request.'); $done = CloudJob::find($job->id); $receipt = json_decode($done->result_json, true); $assert($done->status === 'succeeded' && $done->lease_token === null && (int) $done->lease_until === 0 && count($engine->calls) === $count + 1, 'Confirmed ' . $action . ' waited for or dispatched private reconciliation.'); $assert(($receipt['command_accepted'] ?? null) === true && ($receipt['action'] ?? null) === $action && !isset($receipt['state']), 'Power receipt was discarded or fabricated an observed power state.'); $assert(CloudInstanceRuntime::find(10)->power_state === ($action === 'shutdown' ? 'shutdown' : $beforeState), 'Receipt-only command fabricated a running/stopped observation.'); $assert(CloudInstance::find(10)->network_status === $instance->network_status && Capsule::table('cloud_vpc_members')->value('state') === 'uncertain' && Capsule::table('cloud_vpcs')->value('sync_state') === 'failed', 'Ordinary power command changed independent network policy/readiness.'); } // Exercise the real admission path: a VPC graceful-shutdown timeout must // permit force-off and durably revoke the old automatic restart intent. $vpcOperation = new CloudVpcOperation(); $vpcOperation->operation_id = str_repeat('c', 32); $vpcOperation->vpc_id = 1; $vpcOperation->user_id = 1; $vpcOperation->kind = 'member_attach'; $vpcOperation->status = 'paused'; $vpcOperation->desired_revision = 1; $vpcOperation->save(); CloudVpcPowerWorkflow::initialize($vpcOperation, CloudInstance::find(10), CloudVpcMember::find(1)); $powerWorkflow = CloudVpcPowerWorkflow::state($vpcOperation); $powerWorkflow['phase'] = 'paused'; $powerWorkflow['paused_from'] = 'waiting_stopped'; $powerWorkflow['original_running'] = true; $powerWorkflow['error_code'] = 'vpc_power_shutdown_timeout'; $vpcOperation->power_workflow_json = json_encode($powerWorkflow); $vpcOperation->save(); Capsule::table('cloud_vpcs')->where('id', 1)->update([ 'last_operation_id' => $vpcOperation->operation_id, ]); try { CloudJobService::enqueue(CloudInstance::find(10), $actor, 'instance_settings', 'POST', '/settings', [ 'hostname' => 'during-vpc', 'confirm_restart' => true, ]); throw new RuntimeException('Settings mutation bypassed unfinished VPC maintenance.'); } catch (CloudError $error) { $assert($error->httpStatus === 409, 'Unexpected VPC admission failure.'); } $manualPower = $enqueue('poweroff'); $assert($manualPower->status === 'queued' && CloudVpcPowerWorkflow::state($vpcOperation->fresh())['resume_cancelled'] === true, 'VPC timeout blocked force-off or did not revoke automatic restart.'); $engine->response = ['status' => 'succeeded', 'result' => [ 'uuid' => $uuid, 'action' => 'poweroff', 'command_accepted' => true, ]]; $worker->tick(); $assert($manualPower->fresh()->status === 'succeeded', 'Force-off after VPC timeout acquired a private-readiness dependency.'); // End this isolated fixture; production completion requires host proof. $powerWorkflow = CloudVpcPowerWorkflow::state($vpcOperation->fresh()); $powerWorkflow['phase'] = 'completed'; $vpcOperation->power_workflow_json = json_encode($powerWorkflow); $vpcOperation->save(); // A changing private target does not invalidate an already accepted power // command in finish()'s transaction either. $job = $enqueue('shutdown'); $engine->response = ['status' => 'succeeded', 'result' => [ 'uuid' => $uuid, 'action' => 'shutdown', 'command_accepted' => true, ]]; $engine->afterCommand = static function (): void { foreach (['cloud_vpcs', 'cloud_vpc_members', 'cloud_vpc_private_ips'] as $table) { Capsule::table($table)->update(['desired_revision' => 2]); } }; $worker->tick(); $assert(CloudJob::find($job->id)->status === 'succeeded', 'Private revision change retroactively blocked confirmed power receipt.'); $engine->afterCommand = null; $next = $enqueue('poweroff'); $assert($next->status === 'queued', 'Accepted shutdown still blocks force poweroff.'); // Isolate the following rejection cases; no production cancellation code is used. $next->status = 'cancelled'; $next->save(); // An engine-side preflight/admission failure is terminal even though PHP // sent HTTP and the VM has an unavailable private network. No guest write // happened, so it must not dispatch VPC reconciliation or block force-off. foreach (['action', 'instance_settings'] as $kind) { $job = $kind === 'action' ? $enqueue('reboot') : CloudJobService::enqueue(CloudInstance::find(10), $actor, 'instance_settings', 'POST', '/settings', [ 'hostname' => 'preflight-test', 'confirm_restart' => true, ]); $engine->response = ['status' => 'failed', 'execution_outcome' => 'not_dispatched', 'mutation_started' => false, 'safe_to_refund' => false, 'error_code' => 'execution_result_uncertain']; $count = count($engine->calls); $worker->tick(); $failed = CloudJob::find($job->id); $assert($failed->status === 'failed' && $failed->lease_token === null && (int) $failed->lease_until === 0 && count($engine->calls) === $count + 1, 'Proven preflight failure acquired a private-state dependency for ' . $kind . '.'); $followup = $enqueue('poweroff'); $engine->response = ['status' => 'succeeded', 'result' => [ 'uuid' => $uuid, 'action' => 'poweroff', 'command_accepted' => true, ]]; $worker->tick(); $assert(CloudJob::find($followup->id)->status === 'succeeded', 'Preflight failure blocked the next force-off request for ' . $kind . '.'); } // User deletion has a separate instance admission latch. Undo it only // for this operation's explicit no-write receipt, never a legacy or // contradictory outcome; preserve the snapshot catalog in all failures. foreach ([[], ['execution_outcome' => null], ['mutation_started' => true], ['operation_id' => str_repeat('f', 32)], ['executor_active' => true], ['vm_delete_committed' => true], ['result' => ['cleanup_pending' => true]]] as $index => $change) { $id = 11 + $index; $targetUuid = sprintf('22222222-2222-4222-8222-%012d', $id); Capsule::table('cloud_instances')->insert(['id' => $id, 'host_id' => 7, 'user_id' => 1, 'uuid' => $targetUuid, 'name' => 'Delete preflight']); Capsule::table('cloud_instance_snapshots')->insert([ 'instance_id' => $id, 'state' => 'ready']); $delete = CloudJobService::enqueue(CloudInstance::find($id), $actor, 'delete', 'DELETE', '?delete_disks=true'); $assert(CloudInstance::find($id)->status === 'deleting', 'Delete request did not establish its admission latch.'); $engine->response = array_replace(['status' => 'failed', 'execution_outcome' => 'not_dispatched', 'mutation_started' => false, 'safe_to_refund' => false, 'error_code' => 'execution_result_uncertain'], $change); $worker->tick(); $assert(CloudJob::find($delete->id)->status === 'failed' && CloudInstance::find($id)->status === ($index === 0 ? 'active' : 'delete_failed') && Capsule::table('cloud_instance_snapshots')->where('instance_id', $id) ->value('state') === 'ready', 'User-delete latch ignored missing/conflicting execution proof.'); if ($index === 0) { $next = CloudJobService::enqueue(CloudInstance::find($id), $actor, 'action', 'POST', '/actions', ['action' => 'poweroff']); $next->status = 'cancelled'; $next->save(); } else { try { CloudJobService::enqueue(CloudInstance::find($id), $actor, 'action', 'POST', '/actions', ['action' => 'poweroff']); throw new RuntimeException('Unproven failed delete admitted another write.'); } catch (CloudError $error) { $assert($error->httpStatus === 409, 'Unproven delete had unexpected admission.'); } } } // Expiry cleanup owns a separate service-policy latch. A no-write failure // must not restore an expired instance to an active service. Capsule::table('cloud_instances')->insert(['id' => 30, 'host_id' => 7, 'user_id' => 1, 'uuid' => '22222222-2222-4222-8222-000000000030', 'name' => 'Expiry preflight', 'status' => 'deleting', 'service_status' => 'purge_deleting']); $expiry = CloudJobService::enqueueExpiryDelete(CloudInstance::find(30)); $engine->response = ['status' => 'failed', 'execution_outcome' => 'not_dispatched', 'mutation_started' => false, 'safe_to_refund' => false]; $worker->tick(); $assert(CloudJob::find($expiry->id)->status === 'failed' && CloudInstance::find(30)->status === 'delete_failed' && CloudInstance::find(30)->service_status === 'purge_failed', 'Preflight failure reopened an expired service instead of preserving its policy latch.'); $gate = new ReflectionMethod(CloudWorkerService::class, 'applyPrivateLifecycleGate'); $confirmed = new ReflectionMethod(CloudWorkerService::class, 'confirmedPowerReceipt'); $job = $enqueue('shutdown'); $payload = CloudSecrets::decrypt($job->payload_cipher); $success = ['operation_id' => $job->operation_id, 'status' => 'succeeded', 'result' => [ 'uuid' => $uuid, 'action' => 'shutdown', 'command_accepted' => true, ]]; $assert($confirmed->invoke(null, $job, $success, $payload), 'Valid receipt not recognized.'); $invalid = [ array_replace_recursive($success, ['result' => ['uuid' => '22222222-2222-4222-8222-222222222222']]), array_replace_recursive($success, ['result' => ['action' => 'poweroff']]), array_replace_recursive($success, ['result' => ['command_accepted' => 'true']]), array_replace_recursive($success, ['result' => ['command_accepted' => false]]), array_replace($success, ['operation_id' => str_repeat('f', 32)]), array_replace($success, ['status' => 'uncertain']), array_replace($success, ['status' => 'running']), array_replace($success, ['status' => 'failed']), ]; foreach ($invalid as $operation) { $assert(!$confirmed->invoke(null, $job, $operation, $payload), 'Mismatched, unconfirmed or nonterminal receipt bypassed the private gate.'); } foreach ([['method' => 'GET'], ['resource' => 'vms/' . $uuid . '/reinstall'], ['body' => ['action' => 'delete']]] as $change) { $assert(!$confirmed->invoke(null, $job, $success, array_replace($payload, $change)), 'Receipt bypass was not bound to the immutable power request.'); } $safe = CloudOperationErrorCatalog::workerResult([ 'command_accepted' => 'true', 'action' => 'arbitrary-host-message', ]); $assert($safe === [], 'Unvalidated power receipt fields reached stored/user results.'); // A lost power response is still uncertain, and the next write is rejected. $engine->response = ['status' => 'uncertain', 'mutation_started' => true, 'execution_outcome' => 'unknown', 'error_code' => 'libvirt_execution_uncertain']; $worker->tick(); $assert(CloudJob::find($job->id)->status === 'uncertain', 'A truly uncertain write was upgraded to success.'); try { $enqueue('poweroff'); throw new RuntimeException('Unknown write allowed a new power mutation.'); } catch (CloudError $error) { $assert($error->httpStatus === 409, 'Unknown write had an unexpected admission result.'); } // Unavailable private proof still gates destructive and policy mutations. // Invoke the production gate directly to avoid manufacturing their finance // ledgers; these are not ordinary action jobs even if a result carries a receipt. foreach (['delete', 'expiry_delete', 'reinstall', 'network_change', 'network_suspend', 'network_resume', 'service_suspend', 'service_resume'] as $kind) { $other = clone $job; $other->kind = $kind; $count = count($engine->calls); $result = $gate->invoke($worker, $other, $success, $payload, $actor); $assert($result['status'] === 'uncertain' && count($engine->calls) > $count, 'Power receipt weakened private readback for ' . $kind . '.'); } echo "Power command receipt workflow tests passed.\n"; }