20, 'cloud_engine_url' => 'http://127.0.0.1:8000', 'super_admin' => '99']; public static function get($name, $default = null) { return self::$values[$name] ?? $default; } } final class Auth { public static $user; public static function getUser() { return self::$user; } } } namespace App\Services\Cloud { final class CloudSecrets { private static array $memory = []; public static function key(string $purpose): string { return str_repeat('k', 32); } public static function encrypt(array $payload): string { $id = bin2hex(random_bytes(16)); self::$memory[$id] = $payload; return 'test-only:' . $id; } public static function decrypt(string $cipher): array { return self::$memory[substr($cipher, 10)]; } } } namespace { require dirname(__DIR__) . '/vendor/autoload.php'; use Illuminate\Database\Capsule\Manager as Capsule; use Illuminate\Database\Schema\Blueprint; use App\Models\User; use App\Models\Shop; use App\Models\CloudHost; use App\Models\CloudInstance; use App\Models\CloudInvoice; use App\Models\CloudInvoicePayment; use App\Models\CloudMailOutbox; use App\Models\CloudOrder; use App\Models\CloudJob; use App\Services\Cloud\CloudInvoiceService; use App\Services\Cloud\CloudInvoicePaymentService; use App\Services\Cloud\CloudOrderService; use App\Services\Cloud\CloudWorkerService; use App\Services\Cloud\CloudEngineClient; use App\Services\Cloud\CloudAccess; use App\Services\Cloud\CloudJobService; use App\Services\Cloud\CloudInstanceDiscovery; use App\Services\Cloud\CloudError; use App\Services\Cloud\CloudMoney; use GuzzleHttp\Client; use GuzzleHttp\Handler\MockHandler; use GuzzleHttp\HandlerStack; use GuzzleHttp\Middleware; use GuzzleHttp\Psr7\Response; use GuzzleHttp\Psr7\Request; use GuzzleHttp\Exception\ConnectException; $assert = function ($condition, $message) { if (!$condition) { throw new RuntimeException($message); } }; $capsule = new Capsule(); $capsule->addConnection(['driver' => 'sqlite', 'database' => ':memory:'], 'default'); $capsule->setAsGlobal(); $capsule->bootEloquent(); $schema = $capsule->schema('default'); $schema->create('user', function (Blueprint $t) { $t->increments('id'); $t->string('money'); $t->integer('enable')->default(1); $t->integer('is_admin')->default(0); $t->integer('restrict_balance_purchase')->default(0); }); $schema->create('shop', function (Blueprint $t) { $t->increments('id'); $t->integer('host_id'); $t->string('name'); $t->string('price'); foreach (['cpu_cores', 'memory_mib', 'bandwidth_mbps', 'disk_gib', 'validity_days', 'traffic_quota_bytes', 'status', 'stock', 'sort'] as $v) { $t->integer($v)->default(0); } }); $schema->create('cloud_hosts', function (Blueprint $t) { $t->increments('id'); $t->string('name'); $t->string('region')->default(''); $t->integer('enabled'); $t->string('default_network'); }); $schema->create('cloud_orders', function (Blueprint $t) { $t->increments('id'); foreach (['user_id', 'plan_id', 'host_id'] as $v) { $t->integer($v); } $t->string('idempotency_key'); $t->string('request_hash'); $t->string('amount'); $t->string('status'); $t->text('spec_json'); $t->text('payload_cipher')->nullable(); $t->integer('instance_id')->nullable(); $t->integer('refunded_at')->nullable(); $t->integer('stock_reserved_at')->nullable(); $t->integer('stock_released_at')->nullable(); $t->integer('created_at'); $t->integer('updated_at'); $t->unique(['user_id', 'idempotency_key']); }); $schema->create('cloud_jobs', function (Blueprint $t) { $t->increments('id'); $t->string('operation_id')->unique(); foreach (['order_id', 'instance_id'] as $v) { $t->integer($v)->nullable(); } foreach (['user_id', 'host_id'] as $v) { $t->integer($v); } $t->string('kind'); $t->string('status'); $t->text('payload_cipher'); $t->integer('attempts')->default(0); $t->integer('dispatched_at')->nullable(); $t->string('lease_token')->nullable(); $t->integer('lease_until')->default(0); $t->integer('available_at'); $t->string('error_code')->nullable(); $t->text('result_json')->nullable(); $t->integer('created_at'); $t->integer('updated_at'); $t->unique('order_id'); }); $schema->create('cloud_instances', function (Blueprint $t) { $t->increments('id'); foreach (['host_id', 'user_id'] as $v) { $t->integer($v); } foreach (['order_id', 'plan_id', 'current_period_id', 'expires_at', 'suspended_at', 'network_suspended_at'] as $v) { $t->integer($v)->nullable(); } foreach (['renewal_days', 'renewal_traffic_bytes', 'traffic_quota_bytes', 'traffic_download_bytes', 'traffic_upload_bytes', 'lifetime_download_bytes', 'lifetime_upload_bytes', 'traffic_synced_at'] as $v) { $t->integer($v)->default(0); } $t->string('renewal_price')->nullable(); $t->string('uuid'); $t->string('name'); $t->string('status'); $t->string('service_status')->default('exempt'); $t->string('network_status')->default('active'); $t->text('spec_json'); $t->text('credential_cipher')->nullable(); $t->integer('created_at'); $t->integer('updated_at'); $t->unique('order_id'); $t->unique(['host_id', 'uuid']); }); $schema->create('cloud_service_periods', function (Blueprint $t) { $t->increments('id'); foreach (['instance_id', 'user_id'] as $v) { $t->integer($v); } foreach (['plan_id', 'order_id', 'renewal_id', 'download_base_bytes', 'upload_base_bytes', 'activated_at'] as $v) { $t->integer($v)->nullable(); } foreach (['starts_at', 'ends_at', 'traffic_quota_bytes', 'download_used_bytes', 'upload_used_bytes', 'created_at', 'updated_at'] as $v) { $t->integer($v)->default(0); } $t->string('status'); $t->unique('order_id'); $t->unique('renewal_id'); }); $schema->create('cloud_invoices', function (Blueprint $t) { $t->increments('id'); $t->string('invoice_no')->unique(); $t->string('purpose')->default('renewal'); $t->integer('instance_id')->nullable(); $t->integer('order_id')->nullable()->unique(); $t->integer('user_id'); $t->integer('plan_id')->nullable(); $t->integer('renewal_id')->nullable(); $t->integer('renews_expires_at')->nullable(); $t->integer('due_at'); $t->string('amount'); $t->integer('validity_days'); $t->integer('traffic_quota_bytes'); $t->string('status'); $t->string('paid_via')->nullable(); $t->string('payment_reference')->nullable(); $t->integer('paid_at')->nullable(); $t->integer('created_at'); $t->integer('updated_at'); }); $schema->create('cloud_invoice_payments', function (Blueprint $t) { $t->increments('id'); $t->integer('invoice_id'); $t->integer('user_id'); $t->string('provider'); $t->string('provider_order_no'); $t->string('idempotency_key'); $t->string('amount'); $t->string('status'); $t->integer('created_at'); $t->integer('updated_at'); $t->unique(['provider', 'provider_order_no']); $t->unique(['user_id', 'idempotency_key']); }); $schema->create('cloud_mail_outbox', function (Blueprint $t) { $t->increments('id'); $t->integer('invoice_id'); $t->string('kind'); $t->string('status'); $t->integer('attempts'); $t->integer('available_at'); $t->string('lease_token')->nullable(); $t->integer('lease_until'); $t->integer('sent_at')->nullable(); $t->string('last_error'); $t->integer('created_at'); $t->integer('updated_at'); $t->unique(['invoice_id', 'kind']); }); $schema->create('cloud_ssh_keys', function (Blueprint $t) { $t->increments('id'); $t->integer('user_id'); $t->string('name'); $t->text('public_key'); $t->string('fingerprint'); $t->integer('created_at'); }); $user = new User(); $user->id = 1; $user->money = '0.00'; $user->enable = 1; $user->is_admin = false; $user->save(); $user->isLogin = true; \App\Services\Auth::$user = $user; $host = new CloudHost(); $host->name = 'host'; $host->enabled = true; $host->default_network = 'bridge:br0'; $host->save(); $plan = new Shop(); $plan->host_id = $host->id; $plan->name = 'plan'; $plan->price = '9.99'; $plan->cpu_cores = 2; $plan->memory_mib = 2048; $plan->bandwidth_mbps = 100; $plan->disk_gib = 20; $plan->validity_days = 30; $plan->traffic_quota_bytes = 1024 * 1024 * 1024; $plan->status = 1; $plan->stock = 10; $plan->save(); $actor = ['user_id' => 1, 'is_admin' => false, 'is_super_admin' => false, 'can_manage' => false, 'session_id' => '']; $wrongOperationClient = new CloudEngineClient(new Client(['handler' => HandlerStack::create(new MockHandler([ new Response(422, [], '{"operation_id":"another-operation","status":"failed","safe_to_refund":true}') ]))])); try { $wrongOperationClient->request('POST', '/internal/v1/hosts/1/vms', $actor, [], 'expected-operation'); throw new RuntimeException('Mismatched operation response was trusted.'); } catch (CloudError $expected) { $assert($expected->httpStatus === 502, 'Mismatched operation response exposed an unexpected status.'); } $input = ['plan_id' => 1, 'source' => 'cloud', 'name' => 'vm-one', 'template_id' => 'alma10-1234abcd-0123456789abcdef', 'idempotency_key' => 'test-request-key-0001']; $first = CloudOrderService::purchase($input, $actor); $same = CloudOrderService::purchase($input, $actor); $openingInvoice = CloudInvoice::where('order_id', $first->id)->first(); $openingDisplay = $openingInvoice === null ? [] : CloudInvoiceService::display($openingInvoice); $connection = Capsule::connection('default'); $connection->flushQueryLog(); $connection->enableQueryLog(); $openingSummaries = $openingInvoice === null ? [] : CloudInvoiceService::listSummaries(array_fill(0, 25, $openingInvoice)); $summaryQueryCount = count($connection->getQueryLog()); $connection->disableQueryLog(); $assert($first->id === $same->id && (string) $first->status === 'awaiting_payment' && $openingInvoice !== null && (string) $openingInvoice->purpose === 'provision' && $openingInvoice->instance_id === null && ($openingDisplay['purpose'] ?? null) === 'provision' && ($openingDisplay['title'] ?? null) === '实例开通' && ($openingDisplay['type_label'] ?? null) === '实例开通' && ($openingDisplay['cpu_cores'] ?? null) === 2 && ($openingDisplay['memory_mib'] ?? null) === 2048 && !array_key_exists('invoice_no', $openingDisplay) && ($openingDisplay['order_status'] ?? null) === 'awaiting_payment' && User::find(1)->money === '0.00' && CloudJob::count() === 0 && Shop::find(1)->stock === 9, 'Idempotent deployment request charged money, queued work, lost its opening invoice, or reserved stock twice.'); $assert(count($openingSummaries) === 25 && $summaryQueryCount <= 3 && array_keys($openingSummaries[0] ?? []) === [ 'id', 'title', 'type_label', 'purpose', 'amount', 'status', 'created_at', 'order_status', 'fulfillment_status', 'job_status', ] && ($openingSummaries[0]['order_status'] ?? null) === 'awaiting_payment', 'Invoice list summaries are not compact or use per-invoice association queries.'); try { CloudOrderService::purchase($input + ['vcpu' => 64], $actor); throw new RuntimeException('Resource bypass accepted.'); } catch (CloudError $e) {} try { CloudOrderService::purchase(array_replace($input, ['name' => 'changed']), $actor); throw new RuntimeException('Idempotency conflict accepted.'); } catch (CloudError $e) { $assert($e->httpStatus === 409, 'Wrong idempotency response.'); } try { CloudInvoiceService::payBalance($openingInvoice, array_replace($actor, ['user_id' => 2]), 'provision-cross-tenant-0001'); throw new RuntimeException('Another tenant paid an opening invoice.'); } catch (CloudError $e) { $assert($e->httpStatus === 403, 'Opening invoice ownership failure leaked another status.'); } $assert(User::find(1)->money === '0.00' && CloudOrder::find($first->id)->status === 'awaiting_payment' && CloudJob::count() === 0, 'Cross-tenant opening-invoice access changed financial or queue state.'); try { CloudInvoiceService::payBalance( $openingInvoice, $actor, 'provision-insufficient-balance-0001'); throw new RuntimeException('Opening invoice was paid without sufficient balance.'); } catch (CloudError $e) { $assert($e->httpStatus === 409, 'Insufficient opening-invoice balance returned the wrong status.'); } $assert(CloudOrder::find($first->id)->status === 'awaiting_payment' && CloudInvoice::find($openingInvoice->id)->status === 'open' && CloudJob::count() === 0, 'Failed opening-invoice payment changed invoice, order, or queue state.'); $user = User::find(1); $user->money = '100.00'; $user->save(); $paidOpening = CloudInvoiceService::payBalance( $openingInvoice, $actor, 'provision-balance-test-0001'); $paidOpeningAgain = CloudInvoiceService::payBalance( $paidOpening->fresh(), $actor, 'provision-balance-test-0001'); $assert((string) $paidOpeningAgain->status === 'paid' && User::find(1)->money === '90.01' && CloudOrder::find($first->id)->status === 'queued' && CloudJob::where('order_id', $first->id)->count() === 1 && CloudOrder::find($first->id)->payload_cipher === null, 'Opening invoice payment was not an atomic, exactly-once transition into the provisioning queue.'); $history = []; $mock = new MockHandler([new Response(200, [], json_encode([ 'status' => 'succeeded', 'uuid' => '11111111-1111-4111-8111-111111111111', 'name' => 'vm-one', 'vcpu' => 2, 'memory_mib' => 2048, 'disks' => [[ 'capacity_gib' => 20, 'source' => '/home/kvm/vm-one.qcow2', 'target' => 'vda', ]], 'autostart' => true, 'managed' => true, 'public_ip_cidr' => '203.0.113.18/28', 'os_source' => 'cloud_template', 'os_name' => 'AlmaLinux 10', 'login_name' => 'root', 'console_port' => 5908, 'generated_password' => 'must-not-persist', ]))]); $stack = HandlerStack::create($mock); $stack->push(Middleware::history($history)); $worker = new CloudWorkerService(new CloudEngineClient(new Client(['handler' => $stack]))); $worker->tick(); $openingDisplay = CloudInvoiceService::display($openingInvoice->fresh()); $openingJobDisplay = CloudJobService::display(CloudJob::where('order_id', $first->id)->first()); $assert(CloudOrder::find($first->id)->status === 'succeeded' && CloudInstance::count() === 1 && \App\Models\CloudServicePeriod::count() === 1 && CloudInvoice::find($openingInvoice->id)->instance_id !== null && CloudMailOutbox::where('invoice_id', $openingInvoice->id) ->where('kind', 'provision_succeeded')->count() === 1 && ($openingDisplay['order_status'] ?? null) === 'succeeded' && ($openingDisplay['instance_id'] ?? null) !== null && ($openingDisplay['job_status'] ?? null) === 'succeeded', 'Successful provision did not create its first service period or update the opening bill.'); $assert(!CloudInvoiceService::queueProvisionSucceeded( CloudInvoice::find($openingInvoice->id), time()) && CloudMailOutbox::where('invoice_id', $openingInvoice->id) ->where('kind', 'provision_succeeded')->count() === 1, 'Successful provisioning notification was not idempotent.'); $assert(($openingJobDisplay['attempts'] ?? 0) >= 1 && ($openingJobDisplay['created_at'] ?? 0) > 0 && ($openingJobDisplay['updated_at'] ?? 0) > 0, 'User-visible creation progress lacks safe timing or attempt metadata.'); $posted = json_decode((string) $history[0]['request']->getBody(), true); $signedActor = json_decode(base64_decode(strtr($history[0]['request']->getHeaderLine('X-Cloud-Actor'), '-_', '+/')), true); $assert($posted['vcpu'] === 2 && $posted['bandwidth_mbps'] === 100 && $posted['network'] === 'bridge:br0' && $signedActor['can_provision'] === true && $signedActor['can_manage'] === true, 'Plan or provisioning authority lost.'); $assert(!str_contains(CloudJob::first()->result_json, 'must-not-persist'), 'Plain password saved in result.'); $terminalPayload = \App\Services\Cloud\CloudSecrets::decrypt( CloudJob::first()->payload_cipher ); $assert(!array_key_exists('credential', $terminalPayload) && !array_key_exists('root_password', $terminalPayload['body'] ?? []), 'Terminal provisioning job retained a recoverable root password.'); $instance = CloudInstance::first(); $instanceSpec = json_decode((string) $instance->spec_json, true, flags: JSON_THROW_ON_ERROR); $assert(($instanceSpec['public_ip_cidr'] ?? null) === '203.0.113.18/28' && ($instanceSpec['os_source'] ?? null) === 'cloud_template' && ($instanceSpec['os_name'] ?? null) === 'AlmaLinux 10' && ($instanceSpec['login_name'] ?? null) === 'root' && ($instanceSpec['managed'] ?? null) === true && ($instanceSpec['engine_metadata_version'] ?? null) === 1 && ($instanceSpec['disks'] ?? null) === [['capacity_gib' => 20]] && !isset($instanceSpec['console_port']) && !isset($instanceSpec['disks'][0]['source']), 'Successful provisioning did not persist the safe engine metadata snapshot.'); // One active instance plus one unpaid reservation must exhaust a limit of // two. Expiry releases finite stock and the encrypted credential exactly // once without ever entering the worker queue or touching the wallet. \App\Services\Config::$values['cloud_max_instances_per_user'] = 2; $abandoned = CloudOrderService::purchase(array_replace($input, [ 'name' => 'vm-abandoned', 'idempotency_key' => 'test-request-key-abandoned', ]), $actor); try { CloudOrderService::purchase(array_replace($input, [ 'name' => 'vm-over-limit', 'idempotency_key' => 'test-request-key-over-limit', ]), $actor); throw new RuntimeException('Unpaid provisioning reservation bypassed the instance limit.'); } catch (CloudError $e) { $assert($e->httpStatus === 409, 'Reserved instance limit returned the wrong status.'); } $abandonedInvoice = CloudInvoice::where('order_id', $abandoned->id)->first(); $abandonedInvoice->due_at = time() - 1; $abandonedInvoice->save(); $assert(CloudInvoiceService::expireProvision() === 1 && CloudInvoiceService::expireProvision() === 0 && CloudOrder::find($abandoned->id)->status === 'expired' && CloudOrder::find($abandoned->id)->payload_cipher === null && CloudOrder::find($abandoned->id)->stock_released_at !== null && CloudJob::where('order_id', $abandoned->id)->count() === 0 && User::find(1)->money === '90.01' && Shop::find(1)->stock === 9, 'Expired opening invoice queued work, changed money, retained credentials, or released stock more than once.'); \App\Services\Config::$values['cloud_max_instances_per_user'] = 20; $assert(CloudAccess::instance((string) $instance->id, $actor)->id === $instance->id, 'Owner denied own VM.'); $other = array_replace($actor, ['user_id' => 2]); try { CloudAccess::instance((string) $instance->id, $other); throw new RuntimeException('Tenant isolation failed.'); } catch (CloudError $e) { $assert($e->httpStatus === 404, 'Nonowner information leaked.'); } $admin = array_replace($other, ['is_admin' => true]); $assert(CloudAccess::instance((string) $instance->id, $admin)->id === $instance->id, 'Admin readonly denied.'); try { CloudAccess::instance((string) $instance->id, $admin, true); throw new RuntimeException('Regular admin wrote other owner.'); } catch (CloudError $e) {} try { CloudJobService::enqueue($instance, array_replace($admin, ['can_manage' => true]), 'action', 'POST', '/actions', ['action' => 'reboot']); throw new RuntimeException('Scoped can_manage bypassed cross-owner queue authorization.'); } catch (CloudError $e) {} try { CloudAccess::manage($actor + ['can_provision' => true]); throw new RuntimeException('Provision actor globally privileged.'); } catch (CloudError $e) {} $failed = CloudOrderService::purchase(array_replace($input, ['name' => 'vm-two', 'idempotency_key' => 'test-request-key-0002']), $actor); $assert(User::find(1)->money === '90.01' && CloudJob::where('order_id', $failed->id)->count() === 0, 'A second unpaid deployment charged or entered the worker queue.'); CloudInvoiceService::payBalance(CloudInvoice::where('order_id', $failed->id)->first(), $actor, 'provision-balance-test-0002'); $failedJob = CloudJob::where('order_id', $failed->id)->first(); $mock->append(new Response(422, [], json_encode(['operation_id' => $failedJob->operation_id, 'status' => 'failed', 'safe_to_refund' => true, 'error' => '虚拟机参数校验失败']))); $worker->tick(); $worker->tick(); $failedDisplay = CloudOrderService::display(CloudOrder::find($failed->id)); $failedInvoiceDisplay = CloudInvoiceService::display( CloudInvoice::where('order_id', $failed->id)->first()); $assert(User::find(1)->money === '90.01' && CloudOrder::find($failed->id)->refunded_at !== null && Shop::find(1)->stock === 9, 'Structured non-2xx operation failure was not refunded exactly once.'); $assert($failedDisplay['job_status'] === 'failed' && $failedDisplay['attempts'] >= 1 && str_contains($failedDisplay['error'], '虚拟机参数校验失败') && ($failedInvoiceDisplay['status'] ?? null) === 'refunded' && ($failedInvoiceDisplay['order_status'] ?? null) === 'failed' && str_contains((string) ($failedInvoiceDisplay['error'] ?? ''), '虚拟机参数校验失败') && CloudMailOutbox::where('kind', 'provision_succeeded')->count() === 1, 'Safe job diagnostics are missing from the unified invoice display.'); $uncertain = CloudOrderService::purchase(array_replace($input, ['name' => 'vm-three', 'idempotency_key' => 'test-request-key-0003']), $actor); $assert(User::find(1)->money === '90.01' && CloudJob::where('order_id', $uncertain->id)->count() === 0, 'An unpaid ambiguous-case deployment charged or entered the worker queue.'); CloudInvoiceService::payBalance(CloudInvoice::where('order_id', $uncertain->id)->first(), $actor, 'provision-balance-test-0003'); $mock->append(new ConnectException('simulated timeout', new Request('POST', 'http://127.0.0.1:8000'))); $worker->tick(); $uncertainJob = CloudJob::where('order_id', $uncertain->id)->first(); $uncertainInvoiceDisplay = CloudInvoiceService::display( CloudInvoice::where('order_id', $uncertain->id)->first()); $assert($uncertainJob->status === 'uncertain' && CloudOrder::find($uncertain->id)->refunded_at === null && User::find(1)->money === '80.02' && ($uncertainInvoiceDisplay['order_status'] ?? null) === 'uncertain' && CloudMailOutbox::where('kind', 'provision_succeeded')->count() === 1, 'Timeout incorrectly refunded or disappeared from unified billing.'); $uncertainJob->available_at = 0; $uncertainJob->save(); $mock->append(new Response(404, [], '{}')); $worker->tick(); $last = end($history)['request']; $assert($last->getMethod() === 'GET' && str_contains($last->getUri()->getPath(), '/operations/') && User::find(1)->money === '80.02', 'Ambiguous timeout recreated or refunded on unknown operation.'); $uncertainJob->refresh(); $uncertainJob->available_at = 0; $uncertainJob->save(); $mock->append(new Response(200, [], '{"status":"succeeded","result":{"uuid":"33333333-3333-4333-8333-333333333333"}}')); $worker->tick(); $assert(CloudInstance::count() === 2 && CloudOrder::find($uncertain->id)->status === 'succeeded' && User::find(1)->money === '80.02' && CloudMailOutbox::where('kind', 'provision_succeeded')->count() === 2, 'Reconciliation did not recover exactly once or queue its success email.'); \App\Services\Config::$values['cloud_max_instances_per_user'] = 2; try { CloudOrderService::purchase(array_replace($input, ['name' => 'vm-four', 'idempotency_key' => 'test-request-key-0004']), $actor); throw new RuntimeException('Quota bypass.'); } catch (CloudError $e) { $assert($e->httpStatus === 409, 'Quota response incorrect.'); } $assert(User::find(1)->money === '80.02' && CloudOrder::count() === 4, 'Rejected order altered finance.'); $actionJob = CloudJobService::enqueue($instance, $actor, 'action', 'POST', '/actions', ['action' => 'reboot']); $mock->append(new Response(200, [], '{"status":"succeeded","uuid":"11111111-1111-4111-8111-111111111111","state":"running"}')); $worker->tick(); $last = end($history)['request']; $signedActor = json_decode(base64_decode(strtr($last->getHeaderLine('X-Cloud-Actor'), '-_', '+/')), true); $assert(CloudJob::find($actionJob->id)->status === 'succeeded' && $signedActor['can_manage'] === true && !isset($signedActor['can_provision']), 'Power task delegated provision authority or did not complete.'); // A policy request whose first POST was lost before the engine operation // record existed is safe to replay with the same operation ID. A stale // suspend target must instead be skipped after the authoritative 404. $instance->network_status = 'blocking'; $instance->save(); $policy = new CloudJob(); $policy->operation_id = bin2hex(random_bytes(16)); $policy->instance_id = $instance->id; $policy->user_id = $instance->user_id; $policy->host_id = $instance->host_id; $policy->kind = 'network_suspend'; $policy->status = 'uncertain'; $policy->payload_cipher = \App\Services\Cloud\CloudSecrets::encrypt([ 'method' => 'POST', 'resource' => 'vms/' . $instance->uuid . '/network-suspend', 'body' => [], 'owner_id' => (int) $instance->user_id, 'system_policy' => true, ]); $policy->attempts = 1; $policy->dispatched_at = time() - 30; $policy->lease_until = 0; $policy->available_at = 0; $policy->created_at = $policy->updated_at = time() - 30; $policy->save(); $beforePolicyRequests = count($history); $mock->append(new Response(404, [], '{}')); $mock->append(new Response(200, [], '{"status":"succeeded","result":{}}')); $worker->tick(); $policyRequests = array_slice($history, $beforePolicyRequests); $assert(count($policyRequests) === 2 && $policyRequests[0]['request']->getMethod() === 'GET' && $policyRequests[0]['request']->getUri()->getQuery() === 'reconcile=false' && $policyRequests[1]['request']->getMethod() === 'POST' && CloudJob::find($policy->id)->status === 'succeeded' && $instance->fresh()->network_status === 'blocked_traffic', 'Missing target-state operation was not replayed safely with the same desired state.'); $instance->network_status = 'active'; $instance->save(); $stalePolicy = $policy->replicate(); $stalePolicy->operation_id = bin2hex(random_bytes(16)); $stalePolicy->status = 'uncertain'; $stalePolicy->result_json = null; $stalePolicy->error_code = null; $stalePolicy->lease_token = null; $stalePolicy->lease_until = 0; $stalePolicy->available_at = 0; $stalePolicy->created_at = $stalePolicy->updated_at = time() - 30; $stalePolicy->save(); $beforeStaleRequests = count($history); $mock->append(new Response(404, [], '{}')); $worker->tick(); $assert(count($history) === $beforeStaleRequests + 1 && end($history)['request']->getUri()->getQuery() === 'reconcile=false' && CloudJob::find($stalePolicy->id)->status === 'succeeded' && $instance->fresh()->network_status === 'active', 'A stale policy suspend was replayed after the desired state changed.'); // Desired state may reverse while the old engine request is still inside // libvirt. The PHP job must remain nonterminal until that executor exits; // otherwise lifecycle could dispatch the opposite request concurrently. $instance->network_status = 'restore_pending'; $instance->save(); $activePolicy = $policy->replicate(); $activePolicy->operation_id = bin2hex(random_bytes(16)); $activePolicy->status = 'uncertain'; $activePolicy->result_json = null; $activePolicy->error_code = null; $activePolicy->lease_token = null; $activePolicy->lease_until = 0; $activePolicy->available_at = 0; $activePolicy->created_at = $activePolicy->updated_at = time() - 30; $activePolicy->save(); $beforeActiveRequests = count($history); $mock->append(new Response(200, [], json_encode([ 'status' => 'uncertain', 'executor_active' => true, ]))); $worker->tick(); $activePolicy->refresh(); $assert(count($history) === $beforeActiveRequests + 1 && end($history)['request']->getUri()->getQuery() === 'reconcile=false' && $activePolicy->status === 'uncertain' && $activePolicy->available_at > time() && $instance->fresh()->network_status === 'restore_pending', 'An active stale policy executor was closed before libvirt finished.'); $activePolicy->available_at = 0; $activePolicy->save(); $mock->append(new Response(200, [], json_encode([ 'status' => 'succeeded', 'executor_active' => false, 'result' => ['network_suspended' => true], ]))); $worker->tick(); $assert(CloudJob::find($activePolicy->id)->status === 'succeeded' && $instance->fresh()->network_status === 'restore_pending', 'A completed stale policy did not preserve the newer desired state.'); // After an engine restart executor_active is false, but an existing // nonterminal operation may have only partly changed libvirt. Reconcile // that old target before the pending reverse target is allowed to run. $crashedPolicy = $policy->replicate(); $crashedPolicy->operation_id = bin2hex(random_bytes(16)); $crashedPolicy->status = 'uncertain'; $crashedPolicy->result_json = null; $crashedPolicy->error_code = null; $crashedPolicy->lease_token = null; $crashedPolicy->lease_until = 0; $crashedPolicy->available_at = 0; $crashedPolicy->created_at = $crashedPolicy->updated_at = time() - 30; $crashedPolicy->save(); $beforeCrashRequests = count($history); $mock->append(new Response(200, [], json_encode([ 'status' => 'uncertain', 'executor_active' => false, ]))); $mock->append(new Response(200, [], json_encode([ 'status' => 'succeeded', 'executor_active' => false, 'result' => ['network_suspended' => true], ]))); $worker->tick(); $crashRequests = array_slice($history, $beforeCrashRequests); $assert(count($crashRequests) === 2 && $crashRequests[0]['request']->getMethod() === 'GET' && $crashRequests[0]['request']->getUri()->getQuery() === 'reconcile=false' && $crashRequests[1]['request']->getMethod() === 'POST' && CloudJob::find($crashedPolicy->id)->status === 'succeeded' && $instance->fresh()->network_status === 'restore_pending', 'A crash-interrupted stale target was skipped instead of converged before reversal.'); // The reverse-target assertion above intentionally leaves the fixture in // a lifecycle transition. Restore a neutral state before testing an // unrelated tenant mutation. $instance->network_status = 'active'; $instance->save(); // A non-policy mutation whose authoritative engine operation has vanished // cannot be replayed safely and must become terminal instead of blocking // expiry enforcement forever. $missingAction = CloudJobService::enqueue($instance, $actor, 'action', 'POST', '/actions', ['action' => 'reboot']); $missingAction->status = 'uncertain'; $missingAction->dispatched_at = time() - 30; $missingAction->available_at = 0; $missingAction->save(); $mock->append(new Response(404, [], '{}')); $worker->tick(); $assert(CloudJob::find($missingAction->id)->status === 'failed', 'A vanished non-policy engine operation remained permanently uncertain.'); // Expiry networking, graceful purge shutdown, and irreversible deletion // deliberately use separate states. Only the four target-state policy // jobs may converge; the final disk deletion is a one-shot mutation. $makeLifecycleInstance = function (string $uuid, string $name, string $serviceStatus, string $networkStatus) use ($instance) { $copy = $instance->replicate(); $copy->order_id = null; $copy->current_period_id = null; $copy->uuid = $uuid; $copy->name = $name; $copy->status = 'active'; $copy->service_status = $serviceStatus; $copy->network_status = $networkStatus; $copy->suspended_at = null; $copy->network_suspended_at = null; $copy->credential_cipher = \App\Services\Cloud\CloudSecrets::encrypt([ 'password' => 'LifecycleStrongPassword!123', 'login_name' => 'root', ]); $copy->created_at = $copy->updated_at = time(); $copy->save(); return $copy; }; $queuePolicy = function (CloudInstance $target, string $kind, string $suffix, ?string $reason = null) { $job = new CloudJob(); $job->operation_id = bin2hex(random_bytes(16)); $job->instance_id = $target->id; $job->user_id = $target->user_id; $job->host_id = $target->host_id; $job->kind = $kind; $job->status = 'queued'; $job->payload_cipher = \App\Services\Cloud\CloudSecrets::encrypt([ 'method' => 'POST', 'resource' => 'vms/' . $target->uuid . '/' . $suffix, 'body' => [], 'owner_id' => (int) $target->user_id, 'system_policy' => true, 'suspend_reason' => $reason, ]); $job->available_at = $job->created_at = $job->updated_at = time(); $job->save(); return $job; }; $expiredNetwork = $makeLifecycleInstance( '55555555-5555-4555-8555-555555555555', 'expired-network-vm', 'expired_grace', 'blocking_expired'); $expiredNetworkJob = $queuePolicy( $expiredNetwork, 'network_suspend', 'network-suspend', 'expired'); $mock->append(new Response(200, [], '{"status":"succeeded","result":{"network_suspended":true}}')); $worker->tick(); $assert(CloudJob::find($expiredNetworkJob->id)->status === 'succeeded' && $expiredNetwork->fresh()->network_status === 'blocked_expired', 'Expiry network suspension was recorded as a traffic quota block.'); // Online upgrade retires the old "stop during grace" target. A queued // job must be skipped without contacting the engine. If a previously // dispatched executor may already have stopped a running guest, preserve // that fact for a later network-first safe resume, but never replay stop. $legacyQueued = $makeLifecycleInstance( '56565656-5656-4656-8656-565656565656', 'legacy-queued-stop-vm', 'suspending_expired', 'active'); $legacyQueued->expires_at = time() - 1; $legacyQueued->save(); $legacyQueuedJob = $queuePolicy( $legacyQueued, 'service_suspend', 'service-suspend', 'expired'); $beforeLegacyQueuedRequests = count($history); $worker->tick(); $legacyQueued->refresh(); $assert(count($history) === $beforeLegacyQueuedRequests && CloudJob::find($legacyQueuedJob->id)->status === 'succeeded' && $legacyQueued->service_status === 'expired_grace' && $legacyQueued->suspended_at === null, 'A queued legacy expiry shutdown was sent after the network-only upgrade.'); $legacyUncertain = $makeLifecycleInstance( '57575757-5757-4757-8757-575757575757', 'legacy-uncertain-stop-vm', 'expired_grace', 'blocking_expired'); $legacyUncertain->expires_at = time() - 1; $legacyUncertain->save(); $legacyUncertainJob = $queuePolicy( $legacyUncertain, 'service_suspend', 'service-suspend', 'expired'); $legacyUncertainJob->status = 'uncertain'; $legacyUncertainJob->attempts = 1; $legacyUncertainJob->dispatched_at = time() - 30; $legacyUncertainJob->available_at = 0; $legacyUncertainJob->save(); $beforeLegacyUncertainRequests = count($history); $mock->append(new Response(200, [], json_encode([ 'status' => 'uncertain', 'executor_active' => false, 'service_was_running_hint' => true, ]))); $worker->tick(); $legacyUncertainRequests = array_slice($history, $beforeLegacyUncertainRequests); $legacyUncertain->refresh(); $assert(count($legacyUncertainRequests) === 1 && $legacyUncertainRequests[0]['request']->getMethod() === 'GET' && $legacyUncertainRequests[0]['request']->getUri()->getQuery() === 'reconcile=false' && CloudJob::find($legacyUncertainJob->id)->status === 'succeeded' && $legacyUncertain->service_status === 'resume_pending' && $legacyUncertain->suspended_at !== null, 'A crash-interrupted legacy expiry shutdown was replayed or lost its safe-resume marker.'); $purgeStopped = $makeLifecycleInstance( '66666666-6666-4666-8666-666666666666', 'purge-stop-success-vm', 'purge_stopping', 'blocked_expired'); $purgeStopped->status = 'deleting'; $purgeStopped->save(); $purgeStopJob = $queuePolicy( $purgeStopped, 'service_suspend', 'service-suspend', 'purge'); $mock->append(new Response(200, [], '{"status":"succeeded","result":{"service_was_running":true}}')); $worker->tick(); $purgeStopped->refresh(); $assert(CloudJob::find($purgeStopJob->id)->status === 'succeeded' && $purgeStopped->status === 'deleting' && $purgeStopped->service_status === 'purge_deleting' && $purgeStopped->suspended_at === null, 'Successful expiry shutdown did not advance to disk deletion safely.'); $purgeStopFailed = $makeLifecycleInstance( '77777777-7777-4777-8777-777777777777', 'purge-stop-failed-vm', 'purge_stopping', 'blocked_expired'); $purgeStopFailed->status = 'deleting'; $purgeStopFailed->save(); $purgeStopFailedJob = $queuePolicy( $purgeStopFailed, 'service_suspend', 'service-suspend', 'purge'); $mock->append(new Response(409, [], json_encode([ 'operation_id' => $purgeStopFailedJob->operation_id, 'status' => 'failed', 'safe_to_refund' => false, 'error' => 'shutdown rejected', ]))); $worker->tick(); $purgeStopFailed->refresh(); $assert(CloudJob::find($purgeStopFailedJob->id)->status === 'failed' && $purgeStopFailed->status === 'deleting' && $purgeStopFailed->service_status === 'purge_stop_failed', 'Failed expiry shutdown incorrectly advanced or failed the disk purge.'); $expiryDeleted = $makeLifecycleInstance( '88888888-8888-4888-8888-888888888888', 'expiry-delete-success-vm', 'purge_deleting', 'blocked_expired'); $expiryDeleted->status = 'deleting'; $expiryDeleted->save(); $olderBacklog = CloudJobService::enqueue($instance, $actor, 'action', 'POST', '/actions', ['action' => 'reboot']); $expiryDeleteJob = CloudJobService::enqueueExpiryDelete($expiryDeleted); $beforeExpiryDeleteRequests = count($history); $user->enable = 0; $user->save(); $mock->append(new Response(200, [], '{"status":"succeeded","result":{"deleted":true}}')); $worker->tick(); $user->enable = 1; $user->save(); $expiryDeleteRequests = array_slice($history, $beforeExpiryDeleteRequests); $expiryDeleteActor = count($expiryDeleteRequests) === 1 ? json_decode(base64_decode(strtr( $expiryDeleteRequests[0]['request']->getHeaderLine('X-Cloud-Actor'), '-_', '+/' )), true) : []; $expiryDeleted->refresh(); $assert(count($expiryDeleteRequests) === 1 && $expiryDeleteRequests[0]['request']->getMethod() === 'DELETE' && $expiryDeleteRequests[0]['request']->getUri()->getQuery() === 'delete_disks=true' && (string) $expiryDeleteRequests[0]['request']->getBody() === '{}' && ($expiryDeleteActor['system_cleanup'] ?? false) === true && $expiryDeleteActor['is_admin'] === false && CloudJob::find($expiryDeleteJob->id)->status === 'succeeded' && $expiryDeleted->status === 'deleted' && $expiryDeleted->credential_cipher === null, 'Expiry cleanup did not delete disks once or clear the saved credential.'); $olderBacklog->status = 'failed'; $olderBacklog->save(); $forgedCleanup = $makeLifecycleInstance( '89898989-8989-4898-8898-898989898989', 'forged-expiry-delete-vm', 'purge_deleting', 'blocked_expired'); $forgedCleanup->status = 'deleting'; $forgedCleanup->save(); $forgedCleanupJob = CloudJobService::enqueueExpiryDelete($forgedCleanup); $forgedPayload = \App\Services\Cloud\CloudSecrets::decrypt( $forgedCleanupJob->payload_cipher ); $forgedPayload['resource'] = 'vms/' . $instance->uuid . '?delete_disks=true'; $forgedCleanupJob->payload_cipher = \App\Services\Cloud\CloudSecrets::encrypt( $forgedPayload ); $forgedCleanupJob->save(); $beforeForgedCleanupRequests = count($history); $worker->tick(); $forgedCleanup->refresh(); $assert(count($history) === $beforeForgedCleanupRequests && CloudJob::find($forgedCleanupJob->id)->status === 'failed' && $forgedCleanup->status === 'delete_failed' && $forgedCleanup->service_status === 'purge_failed', 'A forged cleanup payload crossed its bound instance or reached the engine.'); $expiryDeleteFailed = $makeLifecycleInstance( '99999999-9999-4999-8999-999999999999', 'expiry-delete-failed-vm', 'purge_deleting', 'blocked_expired'); $expiryDeleteFailed->status = 'deleting'; $expiryDeleteFailed->save(); $expiryDeleteFailedJob = CloudJobService::enqueueExpiryDelete($expiryDeleteFailed); $mock->append(new Response(409, [], json_encode([ 'operation_id' => $expiryDeleteFailedJob->operation_id, 'status' => 'failed', 'safe_to_refund' => false, 'error' => 'volume delete rejected before undefine', ]))); $worker->tick(); $expiryDeleteFailed->refresh(); $assert(CloudJob::find($expiryDeleteFailedJob->id)->status === 'failed' && $expiryDeleteFailed->status === 'delete_failed' && $expiryDeleteFailed->service_status === 'purge_failed' && $expiryDeleteFailed->credential_cipher !== null, 'Explicit expiry deletion failure did not enter the manual-review state.'); $expiryDeleteUncertain = $makeLifecycleInstance( 'aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa', 'expiry-delete-uncertain-vm', 'purge_deleting', 'blocked_expired'); $expiryDeleteUncertain->status = 'deleting'; $expiryDeleteUncertain->save(); $expiryDeleteUncertainJob = CloudJobService::enqueueExpiryDelete( $expiryDeleteUncertain); $expiryDeleteUncertainJob->status = 'uncertain'; $expiryDeleteUncertainJob->attempts = 1; $expiryDeleteUncertainJob->dispatched_at = time() - 30; $expiryDeleteUncertainJob->available_at = 0; $expiryDeleteUncertainJob->save(); $beforeMissingDeleteCheck = count($history); $mock->append(new Response(404, [], '{}')); $worker->tick(); $missingDeleteRequests = array_slice($history, $beforeMissingDeleteCheck); $expiryDeleteUncertainJob->refresh(); $expiryDeleteUncertain->refresh(); $assert(count($missingDeleteRequests) === 1 && $missingDeleteRequests[0]['request']->getMethod() === 'GET' && $missingDeleteRequests[0]['request']->getUri()->getQuery() === 'reconcile=false' && $expiryDeleteUncertainJob->status === 'uncertain' && $expiryDeleteUncertain->status === 'deleting' && $expiryDeleteUncertain->service_status === 'purge_deleting', 'Missing expiry operation was replayed or closed without manual review.'); $expiryDeleteUncertainJob->available_at = 0; $expiryDeleteUncertainJob->save(); $beforeNonterminalDeleteCheck = count($history); $mock->append(new Response(200, [], '{"status":"running","executor_active":false}')); $worker->tick(); $nonterminalDeleteRequests = array_slice($history, $beforeNonterminalDeleteCheck); $expiryDeleteUncertainJob->refresh(); $expiryDeleteUncertain->refresh(); $assert(count($nonterminalDeleteRequests) === 1 && $nonterminalDeleteRequests[0]['request']->getMethod() === 'GET' && $nonterminalDeleteRequests[0]['request']->getUri()->getQuery() === 'reconcile=false' && $expiryDeleteUncertainJob->status === 'uncertain' && $expiryDeleteUncertain->status === 'deleting' && $expiryDeleteUncertain->service_status === 'purge_deleting', 'Nonterminal expiry deletion was replayed or marked failed automatically.'); $reinstall = CloudJobService::enqueue($instance, $actor, 'reinstall', 'POST', '/reinstall', ['template_id' => $input['template_id'], 'confirmation_name' => $instance->name, 'root_password' => 'NewStrongPassword!123'], ['password' => 'NewStrongPassword!123', 'login_name' => 'root']); $oldCredential = $instance->credential_cipher; $mock->append(new Response(202, [], '{"status":"pending","job_id":"remote-reinstall-id"}')); $worker->tick(); $assert(CloudJob::find($reinstall->id)->status === 'running' && $instance->fresh()->credential_cipher === $oldCredential, 'Pending reinstall replaced password prematurely.'); CloudJob::where('id', $reinstall->id)->update(['available_at' => 0]); $mock->append(new Response(200, [], '{"status":"succeeded","result":{"job_id":"remote-reinstall-id","generated_password":"never-public"}}')); $worker->tick(); $reinstalled = $instance->fresh(); $reinstalledSpec = json_decode((string) $reinstalled->spec_json, true, flags: JSON_THROW_ON_ERROR); $assert(CloudJob::find($reinstall->id)->status === 'succeeded' && $reinstalled->credential_cipher !== $oldCredential && !str_contains(CloudJob::find($reinstall->id)->result_json, 'never-public') && ($reinstalledSpec['template_id'] ?? null) === $input['template_id'] && ($reinstalledSpec['public_ip_cidr'] ?? null) === '203.0.113.18/28' && !isset($reinstalledSpec['os_source'], $reinstalledSpec['os_name'], $reinstalledSpec['install_iso'], $reinstalledSpec['login_name'], $reinstalledSpec['engine_metadata_version']), 'Completed reinstall password, sanitization or metadata invalidation failed.'); $delete = CloudJobService::enqueue($instance, $actor, 'delete', 'DELETE', '?delete_disks=true', []); $mock->append(new Response(200, [], '{"status":"succeeded","uuid":"11111111-1111-4111-8111-111111111111"}')); $worker->tick(); $last = end($history)['request']; $assert((string) $last->getBody() === '{}' && $last->getUri()->getQuery() === 'delete_disks=true' && $instance->fresh()->status === 'deleted' && $instance->fresh()->credential_cipher === null, 'Delete payload/query or credential removal failed.'); $manualUuid = '44444444-4444-4444-8444-444444444444'; $inventory = ['total_count' => 1, 'claimed_count' => 1, 'untracked_managed_count' => 0, 'items' => [[ 'uuid' => $manualUuid, 'owner_user_id' => 99, 'name' => 'manual-vm', 'state' => 'running', 'vcpu' => 4, 'memory_mib' => 4096, 'disks' => [['capacity_gib' => 40]], 'managed' => false, ]]]; $sync = CloudInstanceDiscovery::synchronize($host, $inventory); $imported = CloudInstance::where('uuid', $manualUuid)->first(); $assert($sync['scanned'] === 1 && $sync['imported'] === 1 && $imported !== null && (int) $imported->user_id === 99 && $imported->order_id === null && $imported->credential_cipher === null, 'Discovered VM was not assigned to the configuring administrator without a fake order.'); $repeat = CloudInstanceDiscovery::synchronize($host, $inventory); $assert($repeat['imported'] === 0 && $repeat['existing'] === 1 && CloudInstance::where('uuid', $manualUuid)->count() === 1, 'Repeated VM discovery duplicated or rewrote the imported instance.'); // Provider callbacks credit the wallet before the invoice bridge runs. // That credit must be consumed once, queue one create job, and remain // idempotent when the payment provider retries its callback. \App\Services\Config::$values['cloud_max_instances_per_user'] = 20; $providerOrder = CloudOrderService::purchase(array_replace($input, [ 'name' => 'vm-provider-payment', 'idempotency_key' => 'test-request-key-provider-payment', ]), $actor); $providerInvoice = CloudInvoice::where('order_id', $providerOrder->id)->first(); $providerPayment = new CloudInvoicePayment(); $providerPayment->invoice_id = $providerInvoice->id; $providerPayment->user_id = 1; $providerPayment->provider = 'paytaro'; $providerPayment->provider_order_no = 'PT-PROVISION-0001'; $providerPayment->idempotency_key = 'provision-provider-payment-0001'; $providerPayment->amount = '9.99'; $providerPayment->status = 'pending'; $providerPayment->created_at = $providerPayment->updated_at = time(); $providerPayment->save(); $balanceBeforeProviderCredit = CloudMoney::cents((string) User::find(1)->money); $user = User::find(1); $user->money = CloudMoney::decimal($balanceBeforeProviderCredit + 999); $user->save(); $providerOutcome = CloudInvoicePaymentService::settleForProvider( 'paytaro', 'PT-PROVISION-0001'); $providerOutcomeAgain = CloudInvoicePaymentService::settleForProvider( 'paytaro', 'PT-PROVISION-0001'); $assert($providerOutcome === 'paid_invoice' && $providerOutcomeAgain === 'paid_invoice' && CloudMoney::cents((string) User::find(1)->money) === $balanceBeforeProviderCredit && CloudInvoice::find($providerInvoice->id)->status === 'paid' && CloudOrder::find($providerOrder->id)->status === 'queued' && CloudJob::where('order_id', $providerOrder->id)->count() === 1, 'Provider callback did not settle an opening invoice exactly once.'); echo "Cloud SQLite purchase, provisioning, refund, ambiguity, quota and tenant checks passed (crypto double only).\n"; }