'99', 'baseUrl' => 'https://panel.example.test', 'appName' => 'Virt Cloud Test', ]; public static function get($name, $default = null) { return self::$values[$name] ?? $default; } } final class Auth { public static $user; public static function getUser() { return self::$user; } } final class Mail { public static array $sent = []; public static function send(...$arguments): bool { self::$sent[] = $arguments; return true; } } } namespace App\Services\Cloud { final class CloudSecrets { private static array $memory = []; public static function key(string $purpose): string { return str_repeat('k', 32); } public static function encrypt(array $payload, string $purpose): string { $id = bin2hex(random_bytes(16)); self::$memory[$id] = $payload; return 'test-only:' . $id; } public static function decrypt(string $cipher, string $purpose): array { return self::$memory[substr($cipher, 10)]; } } /** Isolated read-only Agent transport for create recovery checks. */ final class FinancialCreateRecoveryAgentClient { public array $requests = []; public array $responses = []; public function systemRecoveryRequest( string $method, string $target, string $operationId, ?array $payload = null ): array { $this->requests[] = compact( 'method', 'target', 'operationId', 'payload' ); if ($this->responses === []) { throw new \RuntimeException( 'Unexpected create recovery Agent request.' ); } $response = array_shift($this->responses); if ($response instanceof \Throwable) { throw $response; } if (is_callable($response)) { $response = $response(end($this->requests)); } if (!is_array($response)) { throw new \RuntimeException( 'Invalid create recovery Agent response.' ); } return $response; } } final class CloudAgentSettingsService { public static ?FinancialCreateRecoveryAgentClient $client = null; public static int $hostId = 0; public static function clientForConfiguration( int $hostId, float $timeoutSeconds = 120.0 ): FinancialCreateRecoveryAgentClient { if ($hostId !== self::$hostId || $timeoutSeconds !== 30.0 || self::$client === null) { throw new \RuntimeException( 'Unexpected create recovery client request.' ); } return self::$client; } public static function testedNetworkSelectors( \App\Models\CloudHostAgent $setting ): array { return $setting->testedNetworkSelectors(); } public static function publicNetworkReady( \App\Models\CloudHostAgent $setting, string $selector, int $verifiedRevision ): bool { if (!$setting->isReady() || $verifiedRevision !== (int) $setting->revision || $verifiedRevision !== (int) $setting->tested_revision || !in_array($selector, self::testedNetworkSelectors($setting), true)) { return false; } $capabilities = json_decode( (string) $setting->public_network_capabilities_json, true ); return is_array($capabilities) && ($capabilities['contract'] ?? null) === 1 && ($capabilities['dual_stack_required'] ?? null) === true && ($capabilities['project_owned_filter'] ?? null) === true && ($capabilities['ndp_guard'] ?? null) === true; } } } namespace { require dirname(__DIR__) . '/vendor/autoload.php'; use Illuminate\Database\Capsule\Manager as Capsule; use Illuminate\Database\Schema\Blueprint; use App\Models\User; use App\Models\Shop; use App\Models\CloudHost; use App\Models\CloudHostCapacityPolicy; use App\Models\CloudHostNetworkAttachment; use App\Models\CloudHostResourcePoolMembership; use App\Models\CloudBackupSubscription; use App\Models\CloudDeploymentSnapshot; use App\Models\CloudInstanceBackup; use App\Models\CloudBackupRestore; use App\Models\CloudInstance; use App\Models\CloudInvoice; use App\Models\CloudInvoicePayment; use App\Models\CloudMailOutbox; use App\Models\CloudNetworkLine; use App\Models\CloudOffer; use App\Models\CloudOrder; use App\Models\CloudOrderItem; use App\Models\CloudJob; use App\Models\CloudPublicIp; use App\Models\CloudPublicIpPool; use App\Models\CloudRegion; use App\Models\CloudResourcePool; use App\Models\CloudResourceReservation; use App\Models\CloudServicePeriod; use App\Models\CloudTemplate; use App\Services\Cloud\CloudInvoiceService; use App\Services\Cloud\CloudInvoicePaymentService; use App\Services\Cloud\CloudAdminOperationsService; use App\Services\Cloud\CloudAutomaticOperationRecoveryService; use App\Services\Cloud\CloudOrderService; use App\Services\Cloud\CloudWorkerService; use App\Services\Cloud\CloudAgentClient; use App\Services\Cloud\CloudAccess; use App\Services\Cloud\CloudJobService; use App\Services\Cloud\CloudIpChangeBillingService; use App\Services\Cloud\CloudProvisioningBillingConfig; use App\Services\Cloud\CloudLifecycleService; use App\Services\Cloud\CloudOperationCompletionService; use App\Services\Cloud\CloudOperationRecoveryIdentity; use App\Services\Cloud\CloudOperationRecoveryService; use App\Services\Cloud\CloudRenewalService; use App\Services\Cloud\CloudError; use App\Services\Cloud\CloudMoney; use App\Services\Cloud\CloudPlatformTestNetworkPolicy; use App\Services\Cloud\CloudPublicNetworkTransitionCommand; use App\Services\Cloud\CloudPublicNetworkTransitionWorkflow; use App\Services\Cloud\CloudDeploymentPageService; use App\Services\Cloud\CloudTemplateCatalogService; use App\Services\Cloud\CloudAgentSettingsService; use App\Services\Cloud\CloudAgentError; use App\Services\Cloud\CloudCreateFailureRecoveryService; use App\Services\Cloud\FinancialCreateRecoveryAgentClient; use GuzzleHttp\Client; use GuzzleHttp\Handler\MockHandler; use GuzzleHttp\HandlerStack; use GuzzleHttp\Middleware; use GuzzleHttp\Psr7\Response; use GuzzleHttp\Psr7\Request; use GuzzleHttp\Exception\ConnectException; $assert = function ($condition, $message) { if (!$condition) { throw new RuntimeException($message); } }; $bundleState = static function (CloudResourceReservation $reservation, string $claimState, string $ipState): bool { $claims = Capsule::table('cloud_resource_reservation_ips') ->where('reservation_id', (int) $reservation->id) ->where('state', $claimState)->get(); return $claims->count() === 2 && $claims->pluck('address_family')->sort()->values()->all() === [4, 6] && CloudPublicIp::whereIn('id', $claims->pluck('public_ip_id')) ->where('state', $ipState)->count() === 2; }; /** Strictly unwrap one local PHP-to-Gateway request into its Agent request. */ $agentRequest = static function (array $historyEntry) use ($assert): array { $outer = $historyEntry['request'] ?? null; $assert($outer instanceof \Psr\Http\Message\RequestInterface, 'Gateway history entry is missing its outer request.'); $uri = $outer->getUri(); $assert($outer->getMethod() === 'POST' && $uri->getScheme() === 'http' && $uri->getHost() === '127.0.0.1' && $uri->getPort() === 8001 && $uri->getPath() === '/internal/agent/request' && $uri->getQuery() === '' && $uri->getFragment() === '', 'Agent request did not use the fixed local Gateway endpoint.'); $envelope = json_decode((string) $outer->getBody(), true, 32, JSON_THROW_ON_ERROR); $assert(is_array($envelope) && !array_is_list($envelope), 'Gateway request envelope is not a JSON object.'); $actualKeys = array_keys($envelope); $expectedKeys = [ 'agent_id', 'type', 'id', 'method', 'request_target', 'body', 'headers', ]; sort($actualKeys); sort($expectedKeys); $assert($actualKeys === $expectedKeys && ($envelope['agent_id'] ?? null) === '1' && ($envelope['type'] ?? null) === 'request' && is_string($envelope['id'] ?? null) && preg_match('/^[a-f0-9]{32}$/D', $envelope['id']) === 1 && is_string($envelope['method'] ?? null) && in_array($envelope['method'], ['GET', 'POST', 'PUT', 'PATCH', 'DELETE'], true) && is_string($envelope['request_target'] ?? null) && preg_match('#^/v1(?:/|$)#D', $envelope['request_target']) === 1 && is_string($envelope['body'] ?? null) && is_array($envelope['headers'] ?? null) && !array_is_list($envelope['headers']), 'Gateway request envelope has an invalid shape.'); $headers = $envelope['headers']; $actualHeaderKeys = array_keys($headers); $expectedHeaderKeys = [ 'X-Virt-Timestamp', 'X-Virt-Nonce', 'X-Virt-Content-SHA256', 'X-Virt-Actor', 'X-Virt-Signature', ]; sort($actualHeaderKeys); sort($expectedHeaderKeys); $assert($actualHeaderKeys === $expectedHeaderKeys && preg_match('/^[0-9]{1,20}$/D', (string) ($headers['X-Virt-Timestamp'] ?? '')) === 1 && preg_match('/^[a-f0-9]{32}$/D', (string) ($headers['X-Virt-Nonce'] ?? '')) === 1 && ($headers['X-Virt-Content-SHA256'] ?? null) === hash('sha256', $envelope['body']) && preg_match('/^[A-Za-z0-9_-]+$/D', (string) ($headers['X-Virt-Actor'] ?? '')) === 1 && preg_match('/^[a-f0-9]{64}$/D', (string) ($headers['X-Virt-Signature'] ?? '')) === 1, 'Gateway envelope lost the signed Agent request headers.'); if ($envelope['body'] !== '') { $innerBody = json_decode($envelope['body'], false, 64, JSON_THROW_ON_ERROR); $assert($innerBody instanceof \stdClass, 'Agent request body is not a JSON object.'); } $encodedActor = $headers['X-Virt-Actor']; $padding = str_repeat('=', (4 - strlen($encodedActor) % 4) % 4); $decodedActor = base64_decode( strtr($encodedActor, '-_', '+/') . $padding, true ); $actorPayload = is_string($decodedActor) ? json_decode($decodedActor, true, 16, JSON_THROW_ON_ERROR) : null; $assert(is_array($actorPayload) && !array_is_list($actorPayload) && is_int($actorPayload['user_id'] ?? null) && is_bool($actorPayload['is_admin'] ?? null) && is_bool($actorPayload['is_super_admin'] ?? null) && is_bool($actorPayload['can_manage'] ?? null) && (!isset($actorPayload['operation_id']) || (is_string($actorPayload['operation_id']) && preg_match('/^[A-Za-z0-9_-]{8,80}$/D', $actorPayload['operation_id']) === 1)), 'Gateway envelope contains an invalid signed Agent actor.'); return $envelope; }; $capsule = new Capsule(); $capsule->addConnection(['driver' => 'sqlite', 'database' => ':memory:'], 'default'); $capsule->setAsGlobal(); $capsule->bootEloquent(); $schema = $capsule->schema('default'); $schema->create('user', function (Blueprint $t) { $t->increments('id'); $t->string('user_name')->default(''); $t->string('email')->default(''); $t->string('money'); $t->integer('enable')->default(1); $t->integer('is_admin')->default(0); }); $schema->create('configurations', function (Blueprint $t) { $t->increments('id'); $t->string('name')->unique(); $t->text('value'); }); $schema->create('shop', function (Blueprint $t) { $t->increments('id'); $t->string('name'); foreach (['cpu_cores', 'memory_mib', 'bandwidth_mbps', 'disk_gib', 'validity_days', 'traffic_quota_bytes', 'status', 'sort'] as $v) { $t->integer($v)->default(0); } }); $schema->create('cloud_regions', function (Blueprint $t) { $t->increments('id'); $t->string('code'); $t->string('name'); $t->string('country_code'); $t->string('status'); $t->integer('sort')->default(0); $t->integer('created_at'); $t->integer('updated_at'); }); $schema->create('cloud_network_lines', function (Blueprint $t) { $t->increments('id'); $t->integer('region_id'); $t->integer('traffic_pool_group_id')->nullable(); $t->string('code'); $t->string('name'); $t->integer('is_default')->default(0); $t->string('status'); $t->integer('sort')->default(0); $t->integer('created_at'); $t->integer('updated_at'); }); $schema->create('cloud_resource_pools', function (Blueprint $t) { $t->increments('id'); $t->integer('line_id'); $t->string('code'); $t->string('name'); $t->string('status'); $t->string('scheduling_policy')->default('balanced'); $t->integer('sort')->default(0); $t->integer('created_at'); $t->integer('updated_at'); }); $schema->create('cloud_hosts', function (Blueprint $t) { $t->increments('id'); $t->string('name'); $t->string('region')->default(''); $t->string('country_code')->default(''); $t->integer('enabled'); $t->integer('resource_pool_id')->nullable(); $t->string('scheduler_state')->default('disabled'); $t->string('default_network'); $t->integer('created_at')->default(0); $t->integer('updated_at')->default(0); }); $schema->create('cloud_host_agents', function (Blueprint $t) { $t->integer('host_id')->primary(); $t->text('sealed_api_secret'); $t->integer('enabled')->default(0); $t->integer('revision')->default(1); $t->integer('tested_revision')->default(0); $t->string('last_test_status')->default('never'); $t->string('last_error_code')->default(''); $t->string('last_agent_version')->default(''); $t->text('network_selectors_json')->default('[]'); $t->text('public_network_capabilities_json')->default('{}'); $t->integer('last_tested_at')->default(0); $t->integer('created_at'); $t->integer('updated_at'); }); $schema->create('cloud_host_storage_configurations', function (Blueprint $t) { $t->integer('host_id')->primary(); $t->integer('revision')->default(0); $t->string('state')->default('applying'); }); $schema->create('cloud_host_resource_pool_memberships', function (Blueprint $t) { $t->increments('id'); $t->integer('host_id'); $t->integer('resource_pool_id'); $t->string('scheduler_state')->default('disabled'); $t->integer('priority')->default(0); $t->integer('weight')->default(100); $t->integer('revision')->default(1); $t->integer('created_at'); $t->integer('updated_at'); $t->unique(['host_id', 'resource_pool_id']); }); $schema->create('cloud_host_network_attachments', function (Blueprint $t) { $t->increments('id'); $t->string('attachment_key'); $t->integer('host_id'); $t->integer('line_id'); $t->string('name'); $t->string('network_mode')->default('selector'); $t->string('network_selector'); $t->integer('vlan_id')->nullable(); $t->integer('routing_table')->nullable(); $t->string('state')->default('active'); $t->integer('priority')->default(0); $t->integer('config_revision')->default(1); $t->integer('verified_agent_revision')->default(0); $t->string('verification_state')->default('never'); $t->integer('last_verified_at')->default(0); $t->string('verification_error_code')->default(''); $t->integer('desired_revision')->default(1); $t->integer('observed_revision')->default(0); $t->string('sync_state')->default('never'); $t->string('last_error_code')->default(''); $t->integer('created_at'); $t->integer('updated_at'); }); $schema->create('cloud_public_ip_pools', function (Blueprint $t) { $t->increments('id'); $t->string('pool_key'); $t->integer('host_id'); $t->integer('network_attachment_id'); $t->string('name'); $t->integer('address_family'); $t->string('cidr'); $t->string('gateway'); $t->boolean('gateway_on_link')->default(false); $t->integer('guest_prefix_length'); $t->text('dns_servers_json'); $t->text('reserved_ips_json'); $t->string('allocation_start')->nullable(); $t->string('allocation_end')->nullable(); $t->string('allocation_cursor')->nullable(); $t->integer('max_allocations')->default(0); $t->string('state')->default('active'); $t->integer('priority')->default(0); $t->integer('desired_revision')->default(1); $t->integer('observed_revision')->default(0); $t->string('sync_state')->default('never'); $t->string('last_error_code')->default(''); $t->integer('created_at'); $t->integer('updated_at'); }); $schema->create('cloud_host_capacity_policies', function (Blueprint $t) { $t->integer('host_id')->primary(); foreach (['configured_cpu_cores', 'configured_memory_mib', 'configured_disk_gib', 'safety_cpu_cores', 'safety_memory_mib', 'safety_disk_gib', 'max_instances', 'created_at', 'updated_at'] as $v) { $t->integer($v)->default(0); } }); $schema->create('cloud_public_ips', function (Blueprint $t) { $t->increments('id'); $t->integer('host_id'); $t->integer('public_ip_pool_id'); $t->integer('address_family'); $t->string('address'); $t->binary('address_binary'); $t->integer('prefix_length'); $t->string('state'); $t->integer('reservation_id')->nullable(); $t->integer('reservation_ip_claim_id')->nullable(); $t->integer('instance_id')->nullable(); $t->string('operation_id')->nullable(); $t->integer('created_at'); $t->integer('updated_at'); $t->unique(['address_family', 'address_binary']); }); $schema->create('cloud_offers', function (Blueprint $t) { $t->increments('id'); $t->integer('plan_id'); $t->integer('resource_pool_id'); $t->string('code')->unique(); $t->string('name'); $t->string('price'); $t->string('backup_price')->default('0.00'); $t->string('currency')->default('USD'); $t->integer('price_version')->default(1); $t->integer('revision')->default(1); $t->integer('status')->default(1); $t->integer('stock')->default(-1); $t->integer('sort')->default(0); $t->integer('created_at'); $t->integer('updated_at'); $t->unique(['plan_id', 'resource_pool_id']); }); $schema->create('cloud_templates', function (Blueprint $t) { $t->string('id')->primary(); $t->string('catalog_key')->default('test-catalog'); $t->string('version_label')->default('test-version'); $t->boolean('is_default')->default(true); $t->string('name'); $t->string('os_family'); $t->string('os_version'); $t->integer('image_bytes'); $t->string('sha256'); $t->text('startup_script'); $t->integer('revision'); $t->string('generation_token'); $t->string('state'); $t->integer('active_job_id')->nullable(); $t->integer('created_at'); $t->integer('updated_at'); }); $schema->create('cloud_user_isos', function (Blueprint $t) { $t->string('id')->primary(); $t->integer('active_job_id')->nullable(); }); $schema->create('cloud_template_replicas', function (Blueprint $t) { $t->integer('host_id'); $t->string('template_id'); $t->integer('job_id')->nullable(); $t->integer('revision'); $t->string('generation_token'); $t->string('state')->default('pending'); $t->integer('image_bytes')->default(0); $t->string('sha256')->default(''); $t->string('operation_id')->default(''); $t->string('error_code')->default(''); $t->integer('created_at'); $t->integer('updated_at'); $t->primary(['host_id', 'template_id']); }); $schema->create('cloud_platform_test_policy_state', function (Blueprint $t) { $t->integer('id')->primary(); $t->integer('revision')->default(0); $t->integer('updated_by')->nullable(); $t->integer('updated_at')->default(0); }); $schema->create('cloud_platform_test_policy_rules', function (Blueprint $t) { $t->increments('id'); $t->string('rule_key', 32)->unique(); $t->string('service', 120); $t->string('protocol', 8); $t->integer('port_from'); $t->integer('port_to'); $t->string('direction', 8); $t->string('reason', 500); $t->integer('created_by'); $t->integer('updated_by'); $t->integer('deleted_by')->nullable(); $t->integer('deleted_at')->nullable(); $t->integer('created_at'); $t->integer('updated_at'); }); $schema->create('cloud_orders', function (Blueprint $t) { $t->increments('id'); foreach (['user_id', 'plan_id', 'offer_id', 'resource_pool_id'] as $v) { $t->integer($v); } $t->integer('host_id')->nullable(); $t->integer('reservation_id')->nullable()->unique(); $t->integer('deployment_snapshot_id')->nullable(); $t->string('idempotency_key'); $t->string('request_hash'); $t->string('amount'); $t->string('currency')->default('USD'); $t->string('status'); $t->text('spec_json'); $t->text('sealed_payload')->nullable(); $t->integer('instance_id')->nullable(); $t->integer('refunded_at')->nullable(); $t->integer('stock_reserved_at')->nullable(); $t->integer('stock_released_at')->nullable(); $t->integer('created_at'); $t->integer('updated_at'); $t->unique(['user_id', 'idempotency_key']); }); $schema->create('cloud_order_items', function (Blueprint $t) { $t->increments('id'); $t->integer('order_id'); $t->string('code'); $t->string('label'); $t->string('amount'); $t->string('currency')->default('USD'); $t->text('snapshot_json'); $t->integer('created_at'); $t->unique(['order_id', 'code']); }); $schema->create('cloud_resource_reservations', function (Blueprint $t) { $t->increments('id'); $t->string('reservation_key')->unique(); $t->string('operation_id')->unique(); $t->integer('order_id')->nullable(); $t->integer('backup_restore_id')->nullable(); $t->integer('offer_id')->nullable(); foreach (['resource_pool_id', 'host_id', 'user_id'] as $v) { $t->integer($v); } $t->integer('instance_id')->nullable(); $t->integer('network_attachment_id')->nullable(); $t->integer('public_ip_pool_id')->nullable(); $t->integer('public_ip_id')->nullable(); foreach (['cpu_cores', 'memory_mib', 'disk_gib'] as $v) { $t->integer($v); } $t->string('state'); $t->integer('expires_at')->nullable(); $t->integer('released_at')->nullable(); $t->string('release_reason')->default(''); $t->integer('created_at'); $t->integer('updated_at'); $t->unique('order_id'); $t->unique('backup_restore_id'); }); $schema->create('cloud_resource_reservation_ips', function (Blueprint $t) { $t->increments('id'); $t->integer('reservation_id'); $t->integer('address_family'); $t->integer('claim_generation')->default(1); $t->integer('active_claim')->nullable()->default(1); $t->integer('network_attachment_id'); $t->integer('public_ip_pool_id'); $t->integer('public_ip_id'); $t->string('operation_id'); $t->string('state'); $t->integer('created_at'); $t->integer('updated_at'); $t->integer('released_at')->nullable(); $t->string('release_reason')->default(''); $t->unique(['reservation_id', 'address_family', 'active_claim']); $t->unique(['reservation_id', 'claim_generation', 'address_family']); }); $schema->create('cloud_public_network_transitions', function (Blueprint $t) { $t->increments('id'); $t->string('transition_key')->unique(); $t->string('operation_id')->unique(); $t->string('kind'); $t->integer('instance_id'); $t->integer('host_id'); $t->integer('reservation_id'); $t->integer('invoice_id')->nullable()->unique(); $t->integer('pool_replacement_batch_item_id')->nullable()->unique(); $t->integer('requested_by_user_id'); $t->integer('source_network_attachment_id'); $t->integer('target_network_attachment_id'); $t->string('source_claim_set_sha256'); $t->string('target_claim_set_sha256'); $t->text('source_public_network_json'); $t->string('source_public_network_sha256'); $t->text('target_public_network_json'); $t->string('target_public_network_sha256'); $t->string('state'); $t->string('effect_resolution'); $t->integer('active_transition')->nullable(); $t->string('last_error_code'); $t->integer('effect_resolved_at')->nullable(); $t->integer('terminal_at')->nullable(); $t->integer('created_at'); $t->integer('updated_at'); $t->unique(['instance_id', 'active_transition']); $t->unique(['reservation_id', 'active_transition']); }); $schema->create('cloud_public_network_transition_ips', function (Blueprint $t) { $t->increments('id'); $t->integer('transition_id'); $t->integer('address_family'); $t->integer('source_claim_id'); $t->integer('target_claim_id')->nullable()->unique(); $t->boolean('changed'); $t->integer('created_at'); $t->integer('updated_at'); $t->unique(['transition_id', 'address_family']); }); $schema->create('cloud_backup_restores', function (Blueprint $t) { $t->increments('id'); $t->string('restore_key')->unique(); $t->integer('backup_id'); $t->integer('target_host_id'); $t->integer('target_instance_id')->nullable(); $t->integer('reservation_id')->nullable()->unique(); $t->integer('job_id')->nullable(); $t->string('operation_id')->unique(); $t->string('new_uuid'); $t->string('new_name'); $t->string('state'); $t->string('error_code')->default(''); $t->integer('ready_at')->nullable(); $t->integer('created_at'); $t->integer('updated_at'); }); $schema->create('cloud_backup_recoveries', function (Blueprint $t) { $t->increments('id'); $t->integer('instance_id'); $t->string('state'); }); $schema->create('cloud_instance_migrations', function (Blueprint $t) { $t->increments('id'); $t->integer('instance_id'); $t->integer('restore_id'); $t->integer('source_host_id'); $t->integer('source_reservation_id'); $t->integer('target_host_id'); $t->integer('target_reservation_id'); $t->string('state'); }); $schema->create('cloud_instance_backups', function (Blueprint $t) { $t->increments('id'); $t->string('backup_key')->unique(); $t->integer('source_instance_id'); $t->integer('source_host_id'); $t->integer('user_id'); $t->string('source_uuid'); $t->string('name'); $t->string('purpose')->default('portable'); $t->string('source')->default('manual'); $t->integer('subscription_id')->nullable(); $t->integer('scheduled_for')->nullable(); $t->string('state')->default('creating'); $t->string('consistency'); $t->string('object_key')->unique(); $t->integer('object_size_bytes')->default(0); $t->integer('virtual_size_bytes')->default(0); $t->string('sha256')->default(''); $t->text('spec_json'); $t->text('sealed_storage_runtime'); $t->text('sealed_credential')->nullable(); $t->integer('storage_revision'); $t->integer('active_job_id')->nullable(); $t->string('last_operation_id')->nullable(); $t->string('error_code')->default(''); $t->integer('ready_at')->nullable(); $t->integer('created_at'); $t->integer('updated_at'); $t->unique(['subscription_id', 'scheduled_for']); }); $schema->create('cloud_deployment_snapshots', function (Blueprint $t) { $t->increments('id'); $t->string('snapshot_key')->unique(); $t->integer('user_id'); $t->integer('backup_id')->unique(); $t->integer('source_instance_id'); $t->string('name'); $t->string('state')->default('ready'); $t->string('idempotency_key'); $t->string('request_hash'); $t->string('error_code')->default(''); $t->integer('ready_at'); $t->integer('deleted_at')->nullable(); $t->integer('created_at'); $t->integer('updated_at'); $t->unique(['user_id', 'idempotency_key']); }); $schema->create('cloud_jobs', function (Blueprint $t) { $t->increments('id'); $t->string('operation_id')->unique(); $t->string('request_idempotency_key'); $t->string('request_fingerprint'); foreach ([ 'parent_job_id', 'supersedes_job_id', 'order_id', 'invoice_id', 'instance_id', 'snapshot_id', 'backup_id', 'backup_restore_id', 'vpc_repair_root_job_id', 'public_network_transition_id', ] as $v) { $t->integer($v)->nullable(); } $t->integer('user_id'); $t->integer('host_id')->nullable(); $t->string('kind'); $t->string('execution_mode'); $t->string('phase'); $t->string('outcome')->nullable(); $t->string('status'); $t->text('sealed_payload'); $t->integer('attempts'); $t->integer('dispatch_attempts'); $t->integer('dispatch_deadline_at'); $t->integer('dispatch_started_at')->nullable(); $t->integer('agent_accepted_at')->nullable(); $t->integer('receipt_deadline_at')->nullable(); $t->integer('terminal_at')->nullable(); $t->boolean('effect_may_have_applied')->nullable(); $t->boolean('safe_to_resubmit')->nullable(); $t->string('effect_resolution'); $t->integer('effect_resolved_at')->nullable(); $t->integer('effect_resolved_by')->nullable(); $t->text('effect_resolution_reason')->nullable(); $t->string('completion_event_id')->nullable(); $t->integer('read_at')->nullable(); $t->integer('dispatched_at')->nullable(); $t->string('lease_token')->nullable(); $t->integer('lease_until')->default(0); $t->integer('available_at'); $t->string('error_code')->nullable(); $t->boolean('finalization_incident')->default(0); $t->string('finalization_error_code')->nullable(); $t->text('result_json')->nullable(); $t->integer('created_at'); $t->integer('updated_at'); $t->unique('order_id'); }); $schema->create('cloud_agent_completion_events', function (Blueprint $t) { $t->string('event_id')->primary(); $t->integer('host_id'); $t->string('operation_id'); $t->string('payload_sha256'); $t->string('outcome'); $t->boolean('effect_may_have_applied'); $t->boolean('safe_to_resubmit'); $t->string('error_code'); $t->text('result_json')->nullable(); $t->string('disposition'); $t->integer('received_at'); $t->integer('applied_at')->nullable(); $t->unique(['host_id', 'operation_id']); }); $schema->create('cloud_instances', function (Blueprint $t) { $t->increments('id'); foreach (['host_id', 'user_id'] as $v) { $t->integer($v); } foreach (['order_id', 'plan_id', 'offer_id', 'resource_pool_id', 'reservation_id', 'current_period_id', 'expires_at', 'suspended_at', 'network_suspended_at'] as $v) { $t->integer($v)->nullable(); } foreach (['renewal_days', 'renewal_traffic_bytes', 'traffic_quota_bytes', 'traffic_download_bytes', 'traffic_upload_bytes', 'lifetime_download_bytes', 'lifetime_upload_bytes', 'traffic_synced_at'] as $v) { $t->integer($v)->default(0); } $t->string('renewal_price')->nullable(); $t->string('uuid'); $t->string('name'); $t->string('status'); $t->string('service_status')->default('exempt'); $t->string('network_status')->default('active'); $t->text('spec_json'); $t->text('sealed_credential')->nullable(); $t->integer('created_at'); $t->integer('updated_at'); $t->unique('order_id'); $t->unique(['host_id', 'uuid']); }); $schema->create('cloud_instance_network_policy_states', function (Blueprint $t) { $t->integer('instance_id')->primary(); $t->integer('user_id'); $t->boolean('desired_suspended')->default(false); $t->boolean('applied_suspended')->default(false); $t->integer('revision')->default(1); $t->string('state')->default('applied'); $t->string('operation_id')->nullable()->unique(); $t->integer('created_at'); $t->integer('updated_at'); }); $schema->create('cloud_instance_network_restrictions', function (Blueprint $t) { $t->integer('instance_id'); $t->integer('user_id'); $t->string('reason_code'); $t->boolean('desired_active')->default(false); $t->integer('revision')->default(1); $t->string('source_type'); $t->string('source_id'); $t->integer('created_at'); $t->integer('updated_at'); $t->primary(['instance_id', 'reason_code']); }); $schema->create('cloud_vpc_ports', function (Blueprint $t) { $t->increments('id'); $t->integer('vpc_id'); $t->integer('instance_id'); $t->integer('user_id'); $t->integer('active_claim')->nullable(); $t->string('state'); }); $schema->create('cloud_instance_snapshots', function (Blueprint $t) { $t->increments('id'); $t->string('snapshot_key')->unique(); foreach (['instance_id', 'user_id', 'host_id'] as $v) { $t->integer($v); } $t->string('name'); $t->string('state'); $t->string('consistency'); $t->integer('size_bytes')->nullable(); $t->string('last_operation_id')->nullable(); foreach (['ready_at', 'last_restored_at', 'deleted_at'] as $v) { $t->integer($v)->nullable(); } $t->integer('created_at'); $t->integer('updated_at'); }); $schema->create('cloud_renewals', function (Blueprint $t) { $t->increments('id'); $t->integer('instance_id'); $t->integer('user_id'); $t->integer('plan_id')->nullable(); $t->integer('period_id')->nullable(); $t->string('idempotency_key'); $t->string('request_hash'); $t->string('amount'); $t->integer('validity_days'); $t->integer('traffic_bytes_added'); $t->integer('previous_expires_at'); $t->integer('new_expires_at'); $t->integer('created_at'); $t->unique(['user_id', 'idempotency_key']); }); $schema->create('cloud_service_periods', function (Blueprint $t) { $t->increments('id'); foreach (['instance_id', 'user_id'] as $v) { $t->integer($v); } foreach (['plan_id', 'order_id', 'renewal_id', 'traffic_pool_group_id', 'download_base_bytes', 'upload_base_bytes', 'activated_at'] as $v) { $t->integer($v)->nullable(); } $t->string('traffic_quota_mode')->default('finite'); foreach (['starts_at', 'ends_at', 'traffic_quota_bytes', 'download_used_bytes', 'upload_used_bytes', 'created_at', 'updated_at'] as $v) { $t->integer($v)->default(0); } $t->string('status'); $t->unique('order_id'); $t->unique('renewal_id'); }); $schema->create('cloud_invoices', function (Blueprint $t) { $t->increments('id'); $t->string('invoice_no')->unique(); $t->string('purpose')->default('renewal'); $t->integer('instance_id')->nullable(); $t->integer('order_id')->nullable()->unique(); $t->integer('user_id'); $t->integer('plan_id')->nullable(); $t->integer('offer_id')->nullable(); $t->integer('renewal_id')->nullable(); $t->integer('renews_expires_at')->nullable(); $t->integer('due_at'); $t->string('amount'); $t->string('currency')->default('USD'); $t->integer('validity_days'); $t->integer('traffic_quota_bytes'); $t->string('status'); $t->string('paid_via')->nullable(); $t->string('payment_reference')->nullable(); $t->integer('paid_at')->nullable(); $t->integer('created_at'); $t->integer('updated_at'); }); $schema->create('cloud_invoice_items', function (Blueprint $t) { $t->increments('id'); $t->integer('invoice_id'); $t->integer('order_item_id')->nullable(); $t->string('code'); $t->string('label'); $t->string('amount'); $t->string('currency')->default('USD'); $t->text('snapshot_json'); $t->integer('created_at'); $t->unique(['invoice_id', 'code']); }); $schema->create('cloud_backup_subscriptions', function (Blueprint $t) { $t->increments('id'); $t->integer('instance_id')->unique(); $t->integer('user_id'); $t->integer('offer_id'); $t->boolean('enabled')->default(true); $t->boolean('renew_next_period')->default(true); $t->string('frequency')->default('daily'); $t->integer('weekday')->default(0); $t->integer('retention_count')->default(2); $t->string('timezone')->default('UTC'); $t->string('price'); $t->string('currency')->default('USD'); $t->integer('price_version'); $t->integer('state_revision')->default(1); $t->integer('next_run_at'); $t->integer('last_scheduled_at')->nullable(); $t->integer('last_backup_id')->nullable(); $t->integer('created_at'); $t->integer('updated_at'); }); $schema->create('cloud_backup_period_entitlements', function (Blueprint $t) { $t->increments('id'); $t->integer('subscription_id'); $t->integer('instance_id'); $t->integer('service_period_id')->unique(); $t->integer('user_id'); $t->integer('offer_id'); $t->integer('invoice_id'); $t->integer('starts_at'); $t->integer('ends_at'); $t->string('amount'); $t->string('full_period_price'); $t->string('currency')->default('USD'); $t->integer('price_version'); $t->string('source'); $t->integer('paid_at'); $t->integer('created_at'); $t->integer('updated_at'); }); $schema->create('cloud_invoice_payments', function (Blueprint $t) { $t->increments('id'); $t->integer('invoice_id'); $t->integer('user_id'); $t->string('provider'); $t->string('provider_order_no'); $t->string('idempotency_key'); $t->string('amount'); $t->string('status'); $t->integer('created_at'); $t->integer('updated_at'); $t->unique(['provider', 'provider_order_no']); $t->unique(['user_id', 'idempotency_key']); }); $schema->create('cloud_mail_outbox', function (Blueprint $t) { $t->increments('id'); $t->integer('invoice_id'); $t->string('kind'); $t->string('status'); $t->integer('attempts'); $t->integer('available_at'); $t->string('lease_token')->nullable(); $t->integer('lease_until'); $t->integer('sent_at')->nullable(); $t->string('last_error'); $t->integer('created_at'); $t->integer('updated_at'); $t->unique(['invoice_id', 'kind']); }); $schema->create('cloud_ssh_keys', function (Blueprint $t) { $t->increments('id'); $t->integer('user_id'); $t->string('name'); $t->text('public_key'); $t->string('fingerprint'); $t->integer('created_at'); }); $capsule->connection('default')->table('configurations')->insert([ ['name' => 'cloud_network_lines_enabled', 'value' => 'false'], [ 'name' => CloudProvisioningBillingConfig::PAYMENT_WINDOW_MINUTES_KEY, 'value' => '60', ], ]); $capsule->connection('default')->table( 'cloud_platform_test_policy_state' )->insert(['id' => 1, 'revision' => 0, 'updated_by' => null, 'updated_at' => 0]); $user = new User(); $user->id = 1; $user->user_name = 'financial-owner'; $user->email = 'owner@example.test'; $user->money = '0.00'; $user->enable = 1; $user->is_admin = false; $user->save(); $user->isLogin = true; \App\Services\Auth::$user = $user; $now = time(); $region = new CloudRegion(); $region->code = 'test-region'; $region->name = 'Test Region'; $region->country_code = 'US'; $region->status = 'active'; $region->sort = 0; $region->created_at = $region->updated_at = $now; $region->save(); $line = new CloudNetworkLine(); $line->region_id = $region->id; $line->code = 'default'; $line->name = 'Default'; $line->is_default = true; $line->status = 'active'; $line->sort = 0; $line->created_at = $line->updated_at = $now; $line->save(); $pool = new CloudResourcePool(); $pool->line_id = $line->id; $pool->code = 'general'; $pool->name = 'General'; $pool->status = 'active'; $pool->scheduling_policy = 'balanced'; $pool->sort = 0; $pool->created_at = $pool->updated_at = $now; $pool->save(); $host = new CloudHost(); $host->name = 'host'; $host->enabled = true; $host->resource_pool_id = $pool->id; $host->scheduler_state = 'active'; $host->default_network = 'bridge:br0'; $host->created_at = $host->updated_at = $now; $host->save(); $capsule->connection('default')->table('cloud_host_agents')->insert([ 'host_id' => $host->id, 'sealed_api_secret' => 'test-only-agent-secret', 'enabled' => 1, 'revision' => 1, 'tested_revision' => 1, 'last_test_status' => 'healthy', 'last_error_code' => '', 'last_agent_version' => 'test', 'network_selectors_json' => '["bridge:br0","bridge:br1"]', 'public_network_capabilities_json' => json_encode([ 'contract' => 1, 'dual_stack_required' => true, 'project_owned_filter' => true, 'ndp_guard' => true, ], JSON_THROW_ON_ERROR), 'last_tested_at' => $now, 'created_at' => $now, 'updated_at' => $now, ]); $capsule->connection('default')->table( 'cloud_host_storage_configurations' )->insert([ 'host_id' => $host->id, 'revision' => 1, 'state' => 'applied', ]); $membership = new CloudHostResourcePoolMembership(); $membership->host_id = $host->id; $membership->resource_pool_id = $pool->id; $membership->scheduler_state = 'active'; $membership->priority = 0; $membership->weight = 100; $membership->created_at = $membership->updated_at = $now; $membership->save(); $attachment = new CloudHostNetworkAttachment(); $attachment->attachment_key = str_repeat('a', 32); $attachment->host_id = $host->id; $attachment->line_id = $line->id; $attachment->name = 'Primary selector'; $attachment->network_mode = 'selector'; $attachment->network_selector = 'bridge:br0'; $attachment->state = 'active'; $attachment->priority = 0; $attachment->config_revision = 1; $attachment->verified_agent_revision = 1; $attachment->verification_state = 'verified'; $attachment->last_verified_at = $now; $attachment->verification_error_code = ''; $attachment->desired_revision = 1; $attachment->observed_revision = 1; $attachment->sync_state = 'applied'; $attachment->last_error_code = ''; $attachment->created_at = $attachment->updated_at = $now; $attachment->save(); $targetAttachment = new CloudHostNetworkAttachment(); $targetAttachment->attachment_key = str_repeat('b', 32); $targetAttachment->host_id = $host->id; $targetAttachment->line_id = $line->id; $targetAttachment->name = 'Secondary selector'; $targetAttachment->network_mode = 'selector'; $targetAttachment->network_selector = 'bridge:br1'; $targetAttachment->state = 'active'; $targetAttachment->priority = 10; $targetAttachment->config_revision = 1; $targetAttachment->verified_agent_revision = 1; $targetAttachment->verification_state = 'verified'; $targetAttachment->last_verified_at = $now; $targetAttachment->verification_error_code = ''; $targetAttachment->desired_revision = 1; $targetAttachment->observed_revision = 0; $targetAttachment->sync_state = 'never'; $targetAttachment->last_error_code = ''; $targetAttachment->created_at = $targetAttachment->updated_at = $now; $targetAttachment->save(); $makePool = static function (string $key, int $attachmentId, string $name, int $family, string $cidr, string $gateway, int $prefix, array $dns, int $priority = 0, ?string $allocationStart = null, ?string $allocationEnd = null, int $maxAllocations = 0, bool $gatewayOnLink = false ) use ($host, $now): CloudPublicIpPool { $publicPool = new CloudPublicIpPool(); $publicPool->pool_key = $key; $publicPool->host_id = $host->id; $publicPool->network_attachment_id = $attachmentId; $publicPool->name = $name; $publicPool->address_family = $family; $publicPool->cidr = $cidr; $publicPool->gateway = $gateway; $publicPool->gateway_on_link = $gatewayOnLink; $publicPool->guest_prefix_length = $prefix; $publicPool->dns_servers_json = json_encode($dns, JSON_THROW_ON_ERROR); $publicPool->reserved_ips_json = '[]'; $publicPool->allocation_start = $allocationStart; $publicPool->allocation_end = $allocationEnd; $publicPool->allocation_cursor = null; $publicPool->max_allocations = $maxAllocations; $publicPool->state = 'active'; $publicPool->priority = $priority; $publicPool->desired_revision = 1; $publicPool->observed_revision = 1; $publicPool->sync_state = 'applied'; $publicPool->last_error_code = ''; $publicPool->created_at = $publicPool->updated_at = $now; $publicPool->save(); return $publicPool; }; $sourcePoolKey = str_repeat('1', 32); $sourceIpv6PoolKey = str_repeat('3', 32); $targetPoolKey = str_repeat('2', 32); $targetIpv6PoolKey = str_repeat('4', 32); $sourcePublicPool = $makePool( $sourcePoolKey, (int) $attachment->id, 'Primary IPv4 pool', 4, '8.8.8.0/24', '8.8.8.1', 24, ['1.1.1.1'] ); $sourceIpv6Pool = $makePool( $sourceIpv6PoolKey, (int) $attachment->id, 'Primary IPv6 pool', 6, '2404:c140:156:3a::/64', '2404:c140:156::1', 64, ['2606:4700:4700::1111'], 0, '2404:c140:156:3a::10', '2404:c140:156:3a::ff', 32, true ); $targetPublicPool = $makePool( $targetPoolKey, (int) $targetAttachment->id, 'Secondary IPv4 pool', 4, '9.9.9.0/24', '9.9.9.1', 24, ['1.0.0.1'], 10 ); $targetIpv6Pool = $makePool( $targetIpv6PoolKey, (int) $targetAttachment->id, 'Secondary IPv6 pool', 6, '2606:4700:4700:100::/64', '2606:4700:4700:100::1', 64, ['2606:4700:4700::1001'], 10, '2606:4700:4700:100::10', '2606:4700:4700:100::ff', 32 ); $policy = new CloudHostCapacityPolicy(); $policy->host_id = $host->id; $policy->configured_cpu_cores = 64; $policy->configured_memory_mib = 131072; $policy->configured_disk_gib = 4096; $policy->safety_cpu_cores = 2; $policy->safety_memory_mib = 2048; $policy->safety_disk_gib = 64; $policy->max_instances = 100; $policy->created_at = $policy->updated_at = $now; $policy->save(); foreach (['8.8.8.10', '8.8.8.11'] as $address) { $ip = new CloudPublicIp(); $ip->host_id = $host->id; $ip->public_ip_pool_id = $sourcePublicPool->id; $ip->address_family = 4; $ip->address = $address; $ip->address_binary = inet_pton($address); $ip->prefix_length = 24; $ip->state = 'available'; $ip->created_at = $ip->updated_at = $now; $ip->save(); } foreach (['9.9.9.10', '9.9.9.11'] as $address) { $ip = new CloudPublicIp(); $ip->host_id = $host->id; $ip->public_ip_pool_id = $targetPublicPool->id; $ip->address_family = 4; $ip->address = $address; $ip->address_binary = inet_pton($address); $ip->prefix_length = 24; $ip->state = 'available'; $ip->created_at = $ip->updated_at = $now; $ip->save(); } $plan = new Shop(); $plan->name = 'plan'; $plan->cpu_cores = 2; $plan->memory_mib = 2048; $plan->bandwidth_mbps = 100; $plan->disk_gib = 20; $plan->validity_days = 30; $plan->traffic_quota_bytes = 1024 * 1024 * 1024; $plan->status = 1; $plan->sort = 0; $plan->save(); $soldOutPlan = new Shop(); $soldOutPlan->name = 'sold-out-plan'; $soldOutPlan->cpu_cores = 2; $soldOutPlan->memory_mib = 2048; $soldOutPlan->bandwidth_mbps = 100; $soldOutPlan->disk_gib = 20; $soldOutPlan->validity_days = 30; $soldOutPlan->traffic_quota_bytes = 1024 * 1024 * 1024; $soldOutPlan->status = 1; $soldOutPlan->sort = 0; $soldOutPlan->save(); $offer = new CloudOffer(); $offer->plan_id = $plan->id; $offer->resource_pool_id = $pool->id; $offer->code = 'test-offer'; $offer->name = 'Test Offer'; $offer->price = '9.99'; $offer->backup_price = '2.50'; $offer->currency = 'USD'; $offer->price_version = 1; $offer->status = 1; $offer->stock = 10; $offer->sort = 0; $offer->created_at = $offer->updated_at = $now; $offer->save(); $soldOutOffer = new CloudOffer(); $soldOutOffer->plan_id = $soldOutPlan->id; $soldOutOffer->resource_pool_id = $pool->id; $soldOutOffer->code = 'sold-out-offer'; $soldOutOffer->name = 'Sold Out Offer'; $soldOutOffer->price = '8.99'; $soldOutOffer->backup_price = '0.00'; $soldOutOffer->currency = 'USD'; $soldOutOffer->price_version = 1; $soldOutOffer->status = 1; $soldOutOffer->stock = 0; $soldOutOffer->sort = 0; $soldOutOffer->created_at = $soldOutOffer->updated_at = $now; $soldOutOffer->save(); $template = new CloudTemplate(); $template->id = 'alma10-1234abcd-0123456789abcdef'; $template->name = 'AlmaLinux 10'; $template->os_family = 'AlmaLinux'; $template->os_version = '10'; $template->image_bytes = 1024; $template->sha256 = str_repeat('a', 64); $template->startup_script = ''; $template->revision = 1; $template->generation_token = str_repeat('b', 32); $template->state = 'active'; $template->created_at = $template->updated_at = $now; $template->save(); $reinstallTemplate = new CloudTemplate(); $reinstallTemplate->id = 'debian13-1234abcd-0123456789abcdef'; $reinstallTemplate->name = 'Debian 13'; $reinstallTemplate->os_family = 'Debian'; $reinstallTemplate->os_version = '13'; $reinstallTemplate->image_bytes = 2048; $reinstallTemplate->sha256 = str_repeat('c', 64); $reinstallTemplate->startup_script = ''; $reinstallTemplate->revision = 1; $reinstallTemplate->generation_token = str_repeat('d', 32); $reinstallTemplate->state = 'active'; $reinstallTemplate->created_at = $reinstallTemplate->updated_at = $now; $reinstallTemplate->save(); $capsule->connection('default')->table('cloud_template_replicas')->insert([ 'host_id' => $host->id, 'template_id' => $template->id, 'revision' => $template->revision, 'generation_token' => $template->generation_token, 'state' => 'active', 'image_bytes' => $template->image_bytes, 'sha256' => $template->sha256, 'operation_id' => 'fixture-template-import', 'error_code' => '', 'created_at' => $now, 'updated_at' => $now, ]); $capsule->connection('default')->table('cloud_template_replicas')->insert([ 'host_id' => $host->id, 'template_id' => $reinstallTemplate->id, 'revision' => $reinstallTemplate->revision, 'generation_token' => $reinstallTemplate->generation_token, 'state' => 'active', 'image_bytes' => $reinstallTemplate->image_bytes, 'sha256' => $reinstallTemplate->sha256, 'operation_id' => 'fixture-reinstall-template-import', 'error_code' => '', 'created_at' => $now, 'updated_at' => $now, ]); $actor = ['user_id' => 1, 'is_admin' => false, 'is_super_admin' => false, 'can_manage' => false, 'session_id' => '']; $input = ['offer_id' => 1, 'source' => 'cloud', 'name' => 'vm-one', 'template_id' => 'alma10-1234abcd-0123456789abcdef', 'idempotency_key' => 'test-request-key-0001']; $catalogConnection = Capsule::connection('default'); $catalogConnection->flushQueryLog(); $catalogConnection->enableQueryLog(); $catalog = CloudDeploymentPageService::saleCatalog(); $catalogQueryCount = count($catalogConnection->getQueryLog()); $catalogConnection->disableQueryLog(); $assert(count($catalog['offers'] ?? []) === 2 && ($catalog['offers'][0]['sellable'] ?? null) === true && ($catalog['offers'][1]['sellable'] ?? null) === false && $catalogQueryCount <= 10, 'Catalog availability hid a sold-out offer, misclassified capacity, or queried per offer.'); $host->default_network = 'public'; $host->save(); $catalogWithoutLegacyDefault = CloudDeploymentPageService::saleCatalog(); $assert(($catalogWithoutLegacyDefault['offers'][0]['sellable'] ?? null) === true, 'A retired host default network still controlled sale availability.'); $attachment->network_selector = 'public'; $attachment->save(); $targetAttachment->network_selector = 'public'; $targetAttachment->save(); $unverifiedSelectorCatalog = CloudDeploymentPageService::saleCatalog(); $assert(($unverifiedSelectorCatalog['offers'][0]['sellable'] ?? null) === false, 'An attachment outside current Agent evidence remained sellable.'); $attachment->network_selector = 'bridge:br0'; $attachment->verified_agent_revision = 0; $attachment->save(); $targetAttachment->network_selector = 'bridge:br1'; $targetAttachment->verified_agent_revision = 0; $targetAttachment->save(); $staleEvidenceCatalog = CloudDeploymentPageService::saleCatalog(); $assert(($staleEvidenceCatalog['offers'][0]['sellable'] ?? null) === false, 'A stale attachment verification revision remained sellable.'); $host->default_network = 'bridge:br0'; $host->save(); $attachment->network_selector = 'bridge:br0'; $attachment->verified_agent_revision = 1; $attachment->save(); $targetAttachment->network_selector = 'bridge:br1'; $targetAttachment->verified_agent_revision = 1; $targetAttachment->save(); try { CloudOrderService::purchase($input + ['network' => 'bridge:attacker'], $actor); throw new RuntimeException('Purchase accepted a user-selected host network.'); } catch (CloudError $error) { $assert($error->httpStatus === 422, 'User network injection returned the wrong status.'); } $membership->scheduler_state = 'disabled'; $membership->updated_at = time(); $membership->save(); $membershipUnavailable = CloudDeploymentPageService::saleCatalog(); $assert(($membershipUnavailable['offers'][0]['sellable'] ?? null) === false, 'Legacy host scheduling fields bypassed the canonical pool membership.'); $membership->scheduler_state = 'active'; $membership->updated_at = time(); $membership->save(); // Network application state is informational; panel capacity and the // concrete IP ledger decide sellability. A failed engine apply leaves no // ledger rows for new addresses, so placement still fails closed. CloudPublicIp::query()->update(['state' => 'reserved', 'updated_at' => time()]); $noAddressCatalog = CloudDeploymentPageService::saleCatalog(); $assert(($noAddressCatalog['offers'][0]['sellable'] ?? null) === false, 'A pool without an available public IP remained sellable.'); try { CloudOrderService::purchase(array_replace($input, [ 'name' => 'vm-without-ip', 'idempotency_key' => 'test-unavailable-address-0001', ]), $actor); throw new RuntimeException('Missing public IP created an order.'); } catch (CloudError $e) { $assert($e->httpStatus === 409, 'Missing public IP returned the wrong status.'); } $assert(CloudOrder::count() === 0 && CloudInvoice::count() === 0 && CloudOffer::find(1)->stock === 10, 'Address admission failure created finance state or consumed stock.'); CloudPublicIp::query()->update(['state' => 'available', 'updated_at' => time()]); $first = CloudOrderService::purchase($input, $actor); $same = CloudOrderService::purchase($input, $actor); $firstSpec = json_decode( (string) $first->spec_json, true, 64, JSON_THROW_ON_ERROR ); $openingInvoice = CloudInvoice::where('order_id', $first->id)->first(); $openingDisplay = $openingInvoice === null ? [] : CloudInvoiceService::display($openingInvoice); $connection = Capsule::connection('default'); $connection->flushQueryLog(); $connection->enableQueryLog(); $openingSummaries = $openingInvoice === null ? [] : CloudInvoiceService::listSummaries(array_fill(0, 25, $openingInvoice)); $summaryQueryCount = count($connection->getQueryLog()); $connection->disableQueryLog(); $assert($first->id === $same->id && (string) $first->status === 'awaiting_payment' && $openingInvoice !== null && (string) $openingInvoice->purpose === 'provision' && $openingInvoice->instance_id === null && (int) $openingInvoice->due_at - (int) $openingInvoice->created_at === 3600 && ($openingDisplay['purpose'] ?? null) === 'provision' && ($openingDisplay['title'] ?? null) === '实例开通' && ($openingDisplay['type_label'] ?? null) === '实例开通' && ($openingDisplay['cpu_cores'] ?? null) === 2 && ($openingDisplay['memory_mib'] ?? null) === 2048 && !array_key_exists('invoice_no', $openingDisplay) && ($openingDisplay['order_status'] ?? null) === 'awaiting_payment' && ($firstSpec['stock_mode'] ?? null) === 'finite' && array_key_exists('traffic_pool_group_id', $firstSpec) && $firstSpec['traffic_pool_group_id'] === null && ($firstSpec['traffic_quota_mode'] ?? null) === 'finite' && ($firstSpec['automatic_backup'] ?? null) === false && ($firstSpec['instance_price'] ?? null) === '9.99' && ($firstSpec['backup_price'] ?? null) === '0.00' && count($openingDisplay['items'] ?? []) === 1 && ($openingDisplay['items'][0]['code'] ?? null) === 'instance' && CloudOrderItem::where('order_id', $first->id)->count() === 1 && $first->stock_reserved_at !== null && $first->stock_released_at === null && User::find(1)->money === '0.00' && CloudJob::count() === 0 && CloudOffer::find(1)->stock === 9, 'Idempotent deployment request charged money, queued work, lost its opening invoice, or reserved stock twice.'); $assert(count($openingSummaries) === 25 && $summaryQueryCount <= 3 && array_keys($openingSummaries[0] ?? []) === [ 'id', 'title', 'type_label', 'purpose', 'amount', 'currency', 'status', 'display_status', 'available_action', 'payable', 'renews_expires_at', 'due_at', 'payment_deadline_at', 'created_at', 'order_status', 'fulfillment_status', 'task_id', 'job_status', 'instance_id', 'instance_name', ] && ($openingSummaries[0]['order_status'] ?? null) === 'awaiting_payment', 'Invoice list summaries are not compact or use per-invoice association queries.'); try { CloudOrderService::purchase($input + ['vcpu' => 64], $actor); throw new RuntimeException('Resource bypass accepted.'); } catch (CloudError $e) {} try { CloudOrderService::purchase(array_replace($input, ['name' => 'changed']), $actor); throw new RuntimeException('Idempotency conflict accepted.'); } catch (CloudError $e) { $assert($e->httpStatus === 409, 'Wrong idempotency response.'); } try { CloudInvoiceService::payBalance($openingInvoice, array_replace($actor, ['user_id' => 2]), 'provision-cross-tenant-0001'); throw new RuntimeException('Another tenant paid an opening invoice.'); } catch (CloudError $e) { $assert($e->httpStatus === 403, 'Opening invoice ownership failure leaked another status.'); } $assert(User::find(1)->money === '0.00' && CloudOrder::find($first->id)->status === 'awaiting_payment' && CloudJob::count() === 0, 'Cross-tenant opening-invoice access changed financial or queue state.'); try { CloudInvoiceService::payBalance( $openingInvoice, $actor, 'provision-insufficient-balance-0001'); throw new RuntimeException('Opening invoice was paid without sufficient balance.'); } catch (CloudError $e) { $assert($e->httpStatus === 409, 'Insufficient opening-invoice balance returned the wrong status.'); } $assert(CloudOrder::find($first->id)->status === 'awaiting_payment' && CloudInvoice::find($openingInvoice->id)->status === 'open' && CloudJob::count() === 0, 'Failed opening-invoice payment changed invoice, order, or queue state.'); $user = User::find(1); $user->money = '100.00'; $user->save(); $attachment->verified_agent_revision = 0; $attachment->save(); $targetAttachment->verified_agent_revision = 0; $targetAttachment->save(); try { CloudInvoiceService::payBalance( $openingInvoice, $actor, 'provision-stale-network-evidence-0001' ); throw new RuntimeException( 'Payment admitted stale attachment verification evidence.' ); } catch (CloudError $error) { $assert($error->httpStatus === 409, 'Stale attachment evidence returned the wrong payment status.'); } $assert(User::find(1)->money === '100.00' && CloudInvoice::find($openingInvoice->id)->status === 'open' && CloudOrder::find($first->id)->status === 'awaiting_payment' && CloudResourceReservation::count() === 0 && CloudJob::count() === 0, 'Failed network admission retained money, a reservation, or queued work.'); $attachment->verified_agent_revision = 1; $attachment->save(); $targetAttachment->verified_agent_revision = 1; $targetAttachment->save(); // Capacity must remain tied to the immutable order snapshot even if an // administrator changes the base plan while the invoice is still open. $changedPlan = Shop::find($plan->id); $changedPlan->cpu_cores = 8; $changedPlan->memory_mib = 8192; $changedPlan->disk_gib = 80; $changedPlan->save(); $policyManager = [ 'user_id' => 99, 'is_admin' => true, 'is_super_admin' => true, 'can_manage' => true, ]; $testCatalog = CloudPlatformTestNetworkPolicy::create([ 'expected_revision' => 0, 'service' => 'Provision snapshot test', 'protocol' => 'tcp', 'port' => '8080', 'direction' => 'ingress', 'reason' => 'Verify queued creates retain deleted catalog rules.', ], $policyManager); $testRuleKey = (string) $testCatalog['rules'][0]['id']; $host->default_network = 'public'; $host->save(); $paidOpening = CloudInvoiceService::payBalance( $openingInvoice, $actor, 'provision-balance-test-0001'); $paidOpeningAgain = CloudInvoiceService::payBalance( $paidOpening->fresh(), $actor, 'provision-balance-test-0001'); $host->default_network = 'bridge:br0'; $host->save(); $openingOrder = CloudOrder::find($first->id); $openingReservation = CloudResourceReservation::where('order_id', $first->id)->first(); $openingJob = CloudJob::where('order_id', $first->id)->first(); $queuedCreate = \App\Services\Cloud\CloudSecrets::decrypt( (string) $openingJob->sealed_payload, 'job-payload' ); $queuedPolicy = $queuedCreate['body']['platform_network_policy_snapshot'] ?? null; $orderPolicy = json_decode( (string) $openingOrder->spec_json, true, flags: JSON_THROW_ON_ERROR )['platform_network_policy_snapshot'] ?? null; $openingSpec = json_decode( (string) $openingOrder->spec_json, true, flags: JSON_THROW_ON_ERROR ); CloudPlatformTestNetworkPolicy::delete( $testRuleKey, ['expected_revision' => 1], $policyManager ); $currentPolicy = CloudPlatformTestNetworkPolicy::snapshotForProvisioning(); $sealedJobAfterDelete = CloudJob::where('order_id', $first->id)->first(); $sealedCreateAfterDelete = \App\Services\Cloud\CloudSecrets::decrypt( (string) $sealedJobAfterDelete->sealed_payload, 'job-payload' ); $queuedPolicyAfterDelete = $sealedCreateAfterDelete['body'] ['platform_network_policy_snapshot'] ?? null; $orderPolicyAfterDelete = json_decode( (string) CloudOrder::find($first->id)->spec_json, true, flags: JSON_THROW_ON_ERROR )['platform_network_policy_snapshot'] ?? null; $openingClaims = Capsule::table('cloud_resource_reservation_ips') ->where('reservation_id', (int) ($openingReservation->id ?? 0)) ->orderBy('address_family')->get(); $assert((string) $paidOpeningAgain->status === 'paid' && User::find(1)->money === '90.01' && $openingOrder->status === 'queued' && (int) $openingOrder->host_id === (int) $host->id && $openingReservation !== null && $openingReservation->state === 'held' && (int) $openingReservation->cpu_cores === 2 && (int) $openingReservation->memory_mib === 2048 && (int) $openingReservation->disk_gib === 20 && $openingReservation->network_attachment_id === null && $openingReservation->public_ip_pool_id === null && $openingReservation->public_ip_id === null && $openingClaims->count() === 2 && $openingClaims->pluck('address_family')->all() === [4, 6] && $openingClaims->every( static fn ($claim): bool => (string) $claim->state === 'reserved' ) && (string) $openingJob->operation_id === (string) $openingReservation->operation_id && CloudPublicIp::whereIn('id', $openingClaims->pluck('public_ip_id')) ->where('state', 'reserved')->count() === 2 && CloudJob::where('order_id', $first->id)->count() === 1 && is_array($queuedPolicy) && $queuedPolicy === $orderPolicy && ($queuedPolicy['catalog_revision'] ?? null) === 1 && ($queuedPolicy['rules'][9]['id'] ?? null) === 'test-' . $testRuleKey && $queuedPolicyAfterDelete === $queuedPolicy && $orderPolicyAfterDelete === $queuedPolicy && ($currentPolicy['catalog_revision'] ?? null) === 2 && count($currentPolicy['rules']) === 9 && ($openingSpec['template_id'] ?? null) === $template->id && ($openingSpec['os_source'] ?? null) === 'cloud_template' && ($openingSpec['os_name'] ?? null) === 'AlmaLinux 10' && ($openingSpec['login_name'] ?? null) === 'root' && ($openingSpec['public_network']['ipv4']['address_cidr'] ?? null) === '8.8.8.10/24' && ($openingSpec['public_network']['ipv6']['address_cidr'] ?? null) === '2404:c140:156:3a::10/64' && !array_key_exists('managed', $openingSpec) && CloudPlatformTestNetworkPolicy::assertSnapshot($queuedPolicy) === $queuedPolicy && $openingOrder->sealed_payload === null, 'Opening payment did not atomically freeze an immutable per-instance platform policy.'); $changedPlan->cpu_cores = 2; $changedPlan->memory_mib = 2048; $changedPlan->disk_gib = 20; $changedPlan->save(); $history = []; $openingIdentity = CloudOperationRecoveryIdentity::expected( $openingJob, true ); $createdVm = [ 'uuid' => $openingIdentity['resource_id'], 'name' => '1:vm-111111111111', 'vcpu' => 2, 'memory_mib' => 2048, 'disk_gib' => 20, 'disks' => [[ 'device' => 'disk', 'format' => 'qcow2', ]], 'autostart' => true, 'public_network' => $openingSpec['public_network'], 'console_port' => 5908, 'generated_password' => 'must-not-persist', ]; $mismatchedNetwork = $openingSpec['public_network']; $mismatchedNetwork['ipv4']['address_cidr'] = '8.8.8.11/24'; $mismatchedVm = array_replace($createdVm, [ 'public_network' => $mismatchedNetwork, ]); $mock = new MockHandler([ new Response(200, [], json_encode([ 'operation_id' => $openingJob->operation_id, 'status' => 'succeeded', 'result' => $createdVm, ...$openingIdentity, ])), ]); $stack = HandlerStack::create($mock); $stack->push(Middleware::history($history)); $worker = new CloudWorkerService(new CloudAgentClient( '1', 'test-secret-which-is-at-least-32-bytes', new Client(['handler' => $stack]), 20.0, 'http://127.0.0.1:8001/internal/agent/request', 'test-gateway-secret-at-least-32-bytes' )); $worker->tick(); $openingJob->refresh(); $openingReservation->refresh(); $assert(count($history) === 1 && $openingJob->phase === 'terminal' && $openingJob->outcome === 'succeeded' && $openingJob->dispatch_attempts === 1, 'Provisioning did not converge after exactly one Agent mutation dispatch.'); $openingDisplay = CloudInvoiceService::display($openingInvoice->fresh()); $openingJobDisplay = CloudJobService::display(CloudJob::where('order_id', $first->id)->first()); $assert(CloudOrder::find($first->id)->status === 'succeeded' && CloudInstance::count() === 1 && \App\Models\CloudServicePeriod::count() === 1 && \App\Models\CloudServicePeriod::first()->traffic_pool_group_id === null && (string) \App\Models\CloudServicePeriod::first()->traffic_quota_mode === 'finite' && CloudResourceReservation::where('order_id', $first->id)->first()->state === 'committed' && CloudPublicIp::whereIn('id', $openingClaims->pluck('public_ip_id')) ->where('state', 'assigned')->whereNotNull('instance_id') ->count() === 2 && CloudInvoice::find($openingInvoice->id)->instance_id !== null && CloudMailOutbox::where('invoice_id', $openingInvoice->id) ->where('kind', 'provision_succeeded')->count() === 1 && ($openingDisplay['order_status'] ?? null) === 'succeeded' && ($openingDisplay['instance_id'] ?? null) !== null && ($openingDisplay['job_status'] ?? null) === 'succeeded', 'Successful provision did not create its first service period or update the opening bill.'); $assert(!CloudInvoiceService::queueProvisionSucceeded( CloudInvoice::find($openingInvoice->id), time()) && CloudMailOutbox::where('invoice_id', $openingInvoice->id) ->where('kind', 'provision_succeeded')->count() === 1, 'Successful provisioning notification was not idempotent.'); $assert(($openingJobDisplay['completed'] ?? null) === true && ($openingJobDisplay['status'] ?? null) === 'succeeded' && ($openingJobDisplay['status_url'] ?? null) === '/api/v1/tasks/' . $openingJob->id && ($openingJobDisplay['created_at'] ?? 0) > 0 && ($openingJobDisplay['terminal_at'] ?? 0) > 0 && !array_key_exists('attempts', $openingJobDisplay), 'User-visible creation receipt lacks a strict terminal result or leaks retry metadata.'); $openingAgentRequest = $agentRequest($history[0]); $posted = json_decode($openingAgentRequest['body'], true); $signedActor = json_decode(base64_decode(strtr( $openingAgentRequest['headers']['X-Virt-Actor'], '-_', '+/' )), true); $assert($posted['vcpu'] === 2 && $posted['bandwidth_mbps'] === 100 && !array_key_exists('network', $posted) && ($posted['public_network'] ?? null) === $openingSpec['public_network'] && ($posted['platform_network_policy_snapshot'] ?? null) === $queuedPolicy && $signedActor['can_manage'] === true && ($signedActor['operation_id'] ?? null) === $openingJob->operation_id && !isset($signedActor['can_provision']), 'Plan snapshot or typed Agent provisioning authority was lost.'); $assert(!str_contains(CloudJob::first()->result_json, 'must-not-persist'), 'Plain password saved in result.'); $terminalPayload = \App\Services\Cloud\CloudSecrets::decrypt( CloudJob::first()->sealed_payload, 'job-payload' ); $assert(!array_key_exists('credential', $terminalPayload) && !array_key_exists('root_password', $terminalPayload['body'] ?? []), 'Terminal provisioning job retained a recoverable root password.'); $instance = CloudInstance::first(); $instanceSpec = json_decode((string) $instance->spec_json, true, flags: JSON_THROW_ON_ERROR); $assert($instance->name === 'vm-one' && ($instanceSpec['public_network'] ?? null) === $openingSpec['public_network'] && ($instanceSpec['os_source'] ?? null) === 'cloud_template' && ($instanceSpec['os_name'] ?? null) === 'AlmaLinux 10' && ($instanceSpec['login_name'] ?? null) === 'root' && ($instanceSpec['managed'] ?? null) === true && ($instanceSpec['agent_metadata_version'] ?? null) === 1 && ($instanceSpec['disk_gib'] ?? null) === 20 && !isset($instanceSpec['disks']) && !isset($instanceSpec['console_port']) && !isset($instanceSpec['disks'][0]['source']), 'Successful provisioning did not persist the safe Agent metadata snapshot.'); \App\Services\Cloud\CloudPlacementService::markUncertain($openingOrder->fresh()); $assert(CloudResourceReservation::find($openingReservation->id)->state === 'committed' && CloudPublicIp::whereIn('id', $openingClaims->pluck('public_ip_id')) ->where('state', 'assigned')->count() === 2, 'A late uncertainty marker rolled a committed reservation or assigned IP backward.'); // An unpaid provisioning reservation expires without entering the worker // queue, changing the wallet, or retaining its encrypted credential. $abandoned = CloudOrderService::purchase(array_replace($input, [ 'name' => 'vm-abandoned', 'idempotency_key' => 'test-request-key-abandoned', 'automatic_backup' => true, ]), $actor); $abandonedInvoice = CloudInvoice::where('order_id', $abandoned->id)->first(); $abandonedSpec = json_decode((string) $abandoned->spec_json, true, flags: JSON_THROW_ON_ERROR); $abandonedDisplay = CloudInvoiceService::display($abandonedInvoice); $assert((string) $abandoned->amount === '12.49' && ($abandonedSpec['automatic_backup'] ?? null) === true && ($abandonedSpec['instance_price'] ?? null) === '9.99' && ($abandonedSpec['backup_price'] ?? null) === '2.50' && array_column($abandonedDisplay['items'] ?? [], 'code') === ['instance', 'automatic_backup'], 'Automatic backup was not frozen as a separate opening-bill item.'); $abandonedInvoice->due_at = time() - 1; $abandonedInvoice->save(); $assert(CloudInvoiceService::expireProvision() === 1 && CloudInvoiceService::expireProvision() === 0 && CloudOrder::find($abandoned->id)->status === 'expired' && CloudOrder::find($abandoned->id)->sealed_payload === null && CloudOrder::find($abandoned->id)->stock_released_at !== null && CloudJob::where('order_id', $abandoned->id)->count() === 0 && User::find(1)->money === '90.01' && CloudOffer::find(1)->stock === 9, 'Expired opening invoice queued work, changed money, retained credentials, or released stock more than once.'); $assert(CloudAccess::instance((string) $instance->id, $actor)->id === $instance->id, 'Owner denied own VM.'); $other = array_replace($actor, ['user_id' => 2]); try { CloudAccess::instance((string) $instance->id, $other); throw new RuntimeException('Tenant isolation failed.'); } catch (CloudError $e) { $assert($e->httpStatus === 404, 'Nonowner information leaked.'); } $admin = array_replace($other, ['is_admin' => true]); $assert(CloudAccess::instance((string) $instance->id, $admin)->id === $instance->id, 'Admin readonly denied.'); try { CloudAccess::instance((string) $instance->id, $admin, true); throw new RuntimeException('Regular admin wrote other owner.'); } catch (CloudError $e) {} try { CloudJobService::enqueue($instance, array_replace($admin, ['can_manage' => true]), 'action', 'POST', '/actions', ['action' => 'reboot']); throw new RuntimeException('Scoped can_manage bypassed cross-owner queue authorization.'); } catch (CloudError $e) {} try { CloudAccess::manage($actor + ['can_provision' => true]); throw new RuntimeException('Provision actor globally privileged.'); } catch (CloudError $e) {} // A backup-origin deployment must cross the same paid create ledger, but // dispatch a signed restore-backup request and retain the exact backup // identity until a deterministic Agent rejection safely refunds it. $originalSourceSpec = (string) $instance->spec_json; $sourceBackupSpec = json_decode( $originalSourceSpec, true, 64, JSON_THROW_ON_ERROR ); $sourceBackupSpec['automatic_backup'] = true; $sourceBackupSpec['backup_price'] = '2.50'; $instance->spec_json = json_encode( $sourceBackupSpec, JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES ); $instance->save(); $subscription = new CloudBackupSubscription(); $subscription->instance_id = (int) $instance->id; $subscription->user_id = 1; $subscription->offer_id = (int) $offer->id; $subscription->enabled = true; $subscription->renew_next_period = true; $subscription->frequency = 'daily'; $subscription->weekday = 0; $subscription->retention_count = 2; $subscription->timezone = 'UTC'; $subscription->price = '2.50'; $subscription->currency = 'USD'; $subscription->price_version = 1; $subscription->next_run_at = time() + 86400; $subscription->created_at = $subscription->updated_at = time(); $subscription->save(); $capsule->table('cloud_backup_period_entitlements')->insert([ 'subscription_id' => (int) $subscription->id, 'instance_id' => (int) $instance->id, 'service_period_id' => (int) $instance->current_period_id, 'user_id' => 1, 'offer_id' => (int) $offer->id, 'invoice_id' => (int) CloudInvoice::where( 'order_id', (int) $instance->order_id )->value('id'), 'starts_at' => time() - 60, 'ends_at' => time() + 86400, 'amount' => '2.50', 'full_period_price' => '2.50', 'currency' => 'USD', 'price_version' => 1, 'source' => 'provision', 'paid_at' => time() - 60, 'created_at' => time() - 60, 'updated_at' => time() - 60, ]); $sourceBackup = new CloudInstanceBackup(); $sourceBackup->backup_key = str_repeat('c', 32); $sourceBackup->source_instance_id = (int) $instance->id; $sourceBackup->source_host_id = (int) $host->id; $sourceBackup->user_id = 1; $sourceBackup->source_uuid = strtolower((string) $instance->uuid); $sourceBackup->name = 'automatic-backup-fixture'; $sourceBackup->purpose = 'portable'; $sourceBackup->source = 'automatic'; $sourceBackup->subscription_id = (int) $subscription->id; $sourceBackup->scheduled_for = time() - 60; $sourceBackup->state = 'ready'; $sourceBackup->consistency = 'filesystem'; $sourceBackup->object_key = 'virt-cloud/backups/' . $sourceBackup->backup_key . '/system.qcow2'; $sourceBackup->object_size_bytes = 1073741824; $sourceBackup->virtual_size_bytes = 10 * 1073741824; $sourceBackup->sha256 = str_repeat('d', 64); $sourceBackup->spec_json = json_encode( $sourceBackupSpec, JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES ); $sourceBackup->sealed_storage_runtime = \App\Services\Cloud\CloudSecrets::encrypt([ 'config' => [ 'endpoint' => 'https://objects.example.test', 'region' => 'us-east-1', 'bucket' => 'virt-test', 'path_style' => true, ], 'credentials' => [ 'access_key_id' => 'test-access-key', 'secret_access_key' => 'test-secret-key', ], 'revision' => 1, ], 'object-storage-backup-runtime'); $sourceBackup->sealed_credential = $instance->sealed_credential; $sourceBackup->storage_revision = 1; $sourceBackup->ready_at = time(); $sourceBackup->created_at = $sourceBackup->updated_at = time(); $sourceBackup->save(); $orderCountBeforeDirectBackupRequest = CloudOrder::count(); try { CloudOrderService::purchase([ 'offer_id' => (int) $offer->id, 'source' => 'backup', 'backup_id' => (int) $sourceBackup->id, 'name' => 'forbidden-backup-source', 'automatic_backup' => false, 'idempotency_key' => 'test-direct-backup-source-0001', ], $actor); throw new RuntimeException( 'A cloud backup was accepted as a public deployment source.' ); } catch (CloudError $error) { $assert(str_contains($error->getMessage(), '云快照'), 'Direct backup deployment did not direct the user to cloud snapshots.'); } $assert(CloudOrder::count() === $orderCountBeforeDirectBackupRequest, 'Rejected direct backup deployment created finance state.'); // The guest system disk is authoritative. A restored source may no longer // have a trustworthy panel credential envelope and must still be usable // as a cloud snapshot deployment source. $sourceBackup->sealed_credential = null; $sourceBackup->updated_at = time(); $sourceBackup->save(); $deploymentSnapshot = new CloudDeploymentSnapshot(); $deploymentSnapshot->snapshot_key = str_repeat('e', 32); $deploymentSnapshot->user_id = 1; $deploymentSnapshot->backup_id = (int) $sourceBackup->id; $deploymentSnapshot->source_instance_id = (int) $instance->id; $deploymentSnapshot->name = 'deployment-snapshot-fixture'; $deploymentSnapshot->state = 'ready'; $deploymentSnapshot->idempotency_key = 'snapshot-promotion-test-0001'; $deploymentSnapshot->request_hash = str_repeat('f', 64); $deploymentSnapshot->error_code = ''; $deploymentSnapshot->ready_at = time(); $deploymentSnapshot->created_at = $deploymentSnapshot->updated_at = time(); $deploymentSnapshot->save(); $snapshotOrderCount = CloudOrder::count(); $snapshotInvoiceCount = CloudInvoice::count(); try { CloudOrderService::purchase([ 'offer_id' => (int) $offer->id, 'source' => 'deployment_snapshot', 'cloud_snapshot_key' => $deploymentSnapshot->snapshot_key, 'name' => 'vm-two', 'automatic_backup' => false, 'idempotency_key' => 'test-request-key-0002', ], $actor); throw new RuntimeException( 'Cloud snapshot provisioning crossed the C7 compatibility gate.' ); } catch (CloudError $error) { $assert($error->httpStatus === 503 && str_contains($error->getMessage(), '升级双栈网络'), 'Cloud snapshot provisioning returned the wrong C7 gate.'); } $assert(CloudOrder::count() === $snapshotOrderCount && CloudInvoice::count() === $snapshotInvoiceCount, 'The C7 cloud-snapshot gate created financial state.'); // A failed local TCP connection is not an ambiguous Agent dispatch. // Exercise the full worker/financial path, not only client classification. foreach ([7 => 'refused', 28 => 'connect-timeout'] as $curlError => $failure) { $gatewayFailure = CloudOrderService::purchase(array_replace($input, [ 'name' => 'vm-gateway-' . $failure, 'idempotency_key' => 'gateway-before-send-' . $failure, ]), $actor); CloudInvoiceService::payBalance( CloudInvoice::where('order_id', $gatewayFailure->id)->first(), $actor, 'gateway-balance-' . $failure ); $mock->append(new ConnectException('simulated local connection failure', new Request('POST', 'http://127.0.0.1:8001/internal/agent/request'), null, [ 'errno' => $curlError, 'url' => 'http://127.0.0.1:8001/internal/agent/request', 'request_size' => 0, 'size_upload' => 0.0, 'size_download' => 0.0, 'header_size' => 0, 'http_code' => 0, 'redirect_count' => 0, 'connect_time' => 0.0, 'pretransfer_time' => 0.0, 'starttransfer_time' => 0.0, ])); $beforeGatewayFailure = count($history); $assert($worker->tick(), 'Gateway failure task was not processed.'); $gatewayFailureJob = CloudJob::where('order_id', $gatewayFailure->id)->first(); $gatewayFailureReservation = CloudResourceReservation::where( 'order_id', $gatewayFailure->id )->first(); $assert($gatewayFailureJob->phase === 'terminal' && $gatewayFailureJob->outcome === 'failed' && $gatewayFailureJob->effect_resolution === 'not_applied' && !(bool) $gatewayFailureJob->effect_may_have_applied && !(bool) $gatewayFailureJob->finalization_incident && CloudOrder::find($gatewayFailure->id)->refunded_at !== null && $gatewayFailureReservation->state === 'released' && $bundleState( $gatewayFailureReservation, 'released', 'available' ) && User::find(1)->money === '90.01' && CloudOffer::find(1)->stock === 9, 'Proven pre-send Gateway failure retained unknown effects, funds or placement.'); $worker->tick(); $assert(count($history) === $beforeGatewayFailure + 1 && User::find(1)->money === '90.01' && CloudOffer::find(1)->stock === 9, 'Pre-send Gateway failure replayed its mutation or refunded twice.'); } $uncertain = CloudOrderService::purchase(array_replace($input, ['name' => 'vm-three', 'idempotency_key' => 'test-request-key-0003']), $actor); $assert(User::find(1)->money === '90.01' && CloudJob::where('order_id', $uncertain->id)->count() === 0, 'An unpaid ambiguous-case deployment charged or entered the worker queue.'); CloudInvoiceService::payBalance(CloudInvoice::where('order_id', $uncertain->id)->first(), $actor, 'provision-balance-test-0003'); $mock->append(new ConnectException('simulated timeout', new Request('POST', 'http://127.0.0.1:8000'))); $worker->tick(); $uncertainJob = CloudJob::where('order_id', $uncertain->id)->first(); $uncertainInvoiceDisplay = CloudInvoiceService::display( CloudInvoice::where('order_id', $uncertain->id)->first()); $uncertainReservation = CloudResourceReservation::where( 'order_id', $uncertain->id )->first(); $assert($uncertainJob->phase === 'terminal' && $uncertainJob->outcome === 'failed' && $uncertainJob->status === 'failed' && (int) $uncertainJob->dispatch_attempts === 1 && (bool) $uncertainJob->effect_may_have_applied === true && (bool) $uncertainJob->safe_to_resubmit === false && $uncertainJob->effect_resolution === 'unknown' && CloudOrder::find($uncertain->id)->status === 'failed' && CloudOrder::find($uncertain->id)->refunded_at === null && $uncertainReservation->state === 'uncertain' && $bundleState($uncertainReservation, 'uncertain', 'uncertain') && User::find(1)->money === '80.02' && ($uncertainInvoiceDisplay['order_status'] ?? null) === 'failed' && CloudMailOutbox::where('kind', 'provision_succeeded')->count() === 1, 'A dispatched create timeout did not become a terminal unknown-effect failure.'); $beforeFailedCreateRetry = count($history); $worker->tick(); $uncertainJob->refresh(); $assert(count($history) === $beforeFailedCreateRetry && $uncertainJob->phase === 'terminal' && $uncertainJob->outcome === 'failed' && (int) $uncertainJob->dispatch_attempts === 1 && CloudInstance::count() === 1 && CloudResourceReservation::find($uncertainReservation->id)->state === 'uncertain' && User::find(1)->money === '80.02' && CloudMailOutbox::where('kind', 'provision_succeeded')->count() === 1, 'A terminal create failure was automatically polled, reconciled, or mutation-retried.'); // Simulate the crash window where the Agent already committed its // not-applied fence receipt but PHP died before settling local ledgers. A // later fresh inspection may consume the older matching receipt, while // the immutable public task remains terminal failed. $resolutionReason = '已通过新的只读检查确认虚拟机及面板管理资源均不存在,执行原子退款。'; $inspectionNow = time(); $freshInspection = [ 'operation_id' => (string) $uncertainJob->operation_id, 'checked_at' => $inspectionNow, 'outcome' => 'failed', 'summary_code' => 'operation_failed', 'error_code' => 'creation_absent', ]; $persistedFenceReceipt = [ 'operation_id' => (string) $uncertainJob->operation_id, 'resolution_id' => str_repeat('a', 32), 'resolution' => 'not_applied', 'inspection_checked_at' => $inspectionNow - 60, 'reason_sha256' => hash('sha256', $resolutionReason), 'resolved_at' => $inspectionNow - 50, 'resolved_by' => 7, 'replayed' => false, ]; $resolvedCreate = CloudAdminOperationsService::resolveEffect([ 'user_id' => 99, 'is_admin' => true, 'is_super_admin' => true, 'can_manage' => true, ], (string) $uncertainJob->id, [ 'resolution' => 'not_applied', 'reason' => $resolutionReason, ], $freshInspection, $persistedFenceReceipt); $uncertainJob->refresh(); $assert($uncertainJob->phase === 'terminal' && $uncertainJob->outcome === 'failed' && $uncertainJob->status === 'failed' && $uncertainJob->effect_resolution === 'not_applied' && (int) $uncertainJob->effect_resolved_by === 99 && ($resolvedCreate['job']['completed'] ?? false) === true && ($resolvedCreate['job']['status'] ?? null) === 'failed' && ($resolvedCreate['job']['can_resolve_effect'] ?? true) === false && CloudOrder::find($uncertain->id)->refunded_at !== null && CloudInvoice::where('order_id', $uncertain->id) ->value('status') === 'refunded' && CloudResourceReservation::find($uncertainReservation->id) ->state === 'released' && $bundleState( CloudResourceReservation::find($uncertainReservation->id), 'released', 'available' ) && User::find(1)->money === '90.01' && CloudOffer::find(1)->stock === 9, 'A prior Agent fence commit was not recovered into one atomic local create settlement.'); // A completion may be durably accepted before PHP finishes the create // ledgers. Keep the encrypted request escrow while the effect is unknown, // then let the exact same authenticated receipt atomically materialize the // instance and ACK its event after the local ledger fault is repaired. $finiteStockBeforeUnlimitedCreate = (int) CloudOffer::findOrFail(1)->stock; $unlimitedOffer = CloudOffer::findOrFail(1); $offerCreatedAtBeforeUnlimitedCreate = (int) $unlimitedOffer->created_at; $unlimitedOffer->stock = -1; $unlimitedOffer->created_at = time() - 20; $unlimitedOffer->updated_at = time() - 10; $unlimitedOffer->save(); $lateCreate = CloudOrderService::purchase(array_replace($input, [ 'name' => 'vm-late-create', 'idempotency_key' => 'test-request-key-late-create', ]), $actor); $lateCreateSpec = json_decode( (string) $lateCreate->spec_json, true, 64, JSON_THROW_ON_ERROR ); $assert(($lateCreateSpec['stock_mode'] ?? null) === 'unlimited' && $lateCreate->stock_reserved_at === null && $lateCreate->stock_released_at === null && (int) CloudOffer::findOrFail(1)->stock === -1, 'An unlimited-stock purchase consumed or fabricated finite stock.'); $lateInvoice = CloudInvoice::where('order_id', $lateCreate->id)->first(); CloudInvoiceService::payBalance( $lateInvoice, $actor, 'provision-balance-late-create' ); $lateCreateJob = CloudJob::where('order_id', $lateCreate->id)->first(); $lateCreateIdentity = CloudOperationRecoveryIdentity::expected( $lateCreateJob, true ); $lateCreatePayload = \App\Services\Cloud\CloudSecrets::decrypt( (string) $lateCreateJob->sealed_payload, 'job-payload' ); $mock->append(new Response(202, [], json_encode([ 'operation_id' => $lateCreateJob->operation_id, 'status' => 'reserved', ...$lateCreateIdentity, 'replayed' => false, 'retryable' => false, ]))); $worker->tick(); $lateCreateJob->refresh(); $assert((string) $lateCreateJob->phase === 'awaiting_receipt', 'Late-create fixture did not cross the Agent dispatch boundary.'); $lateOrder = CloudOrder::find($lateCreate->id); $lateOrderSpec = json_decode( (string) $lateOrder->spec_json, true, 64, JSON_THROW_ON_ERROR ); $damagedLateOrderSpec = $lateOrderSpec; unset($damagedLateOrderSpec['validity_days']); $lateOrder->spec_json = json_encode( $damagedLateOrderSpec, JSON_THROW_ON_ERROR ); $lateOrder->save(); $lateReceiptId = 'create-late-success-event-0001'; $lateCreatedVm = array_replace($createdVm, [ 'uuid' => $lateCreateIdentity['resource_id'], 'name' => '1:vm-late-create', 'public_network' => $lateCreatePayload['body']['public_network'], ]); unset($lateCreatedVm['generated_password']); $lateReceipt = [ 'type' => 'completion', 'id' => $lateReceiptId, 'agent_id' => (string) $lateCreateJob->host_id, 'operation_id' => (string) $lateCreateJob->operation_id, ...$lateCreateIdentity, 'status' => 'succeeded', 'effect_may_have_applied' => false, 'safe_to_resubmit' => false, 'error_code' => '', 'result' => $lateCreatedVm, ]; $firstLateReceipt = CloudOperationCompletionService::accept($lateReceipt); $lateCreateJob->refresh(); $lateEscrow = \App\Services\Cloud\CloudSecrets::decrypt( (string) $lateCreateJob->sealed_payload, 'job-payload' ); $lateEvent = \App\Models\CloudAgentCompletionEvent::where( 'event_id', $lateReceiptId )->first(); $assert($firstLateReceipt === [ 'accepted' => true, 'disposition' => 'received', ] && (string) $lateCreateJob->phase === 'terminal' && (string) $lateCreateJob->outcome === 'failed' && (string) $lateCreateJob->effect_resolution === 'unknown' && ($lateEscrow['credential']['password'] ?? null) === ($lateEscrow['body']['root_password'] ?? null) && (string) $lateEvent->disposition === 'received' && CloudInstance::where('order_id', $lateCreate->id)->count() === 0, 'A failed local create finalizer lost its exact receipt or encrypted recovery escrow.'); $lateWalletBeforeManualResolution = (string) User::find(1)->money; $lateReservationBeforeManualResolution = CloudResourceReservation::where( 'order_id', $lateCreate->id )->first(); try { \App\Services\Cloud\CloudCreateFailureFinalizer::resolveNotApplied( (int) $lateCreateJob->id, (string) $lateCreateJob->operation_id, 99, 'contradictory-manual-absence', [ 'operation_id' => (string) $lateCreateJob->operation_id, 'outcome' => 'failed', 'summary_code' => 'operation_failed', 'error_code' => 'creation_absent', ], time() ); throw new RuntimeException( 'Manual no-effect resolution ignored a durable create success.' ); } catch (CloudError $error) { $assert($error->httpStatus === 409, 'Create success/manual absence conflict returned wrong status.'); } $assert((string) User::find(1)->money === $lateWalletBeforeManualResolution && (string) CloudOrder::find($lateCreate->id)->status === 'failed' && (string) CloudInvoice::where('order_id', $lateCreate->id) ->value('status') === 'paid' && (string) CloudResourceReservation::find( $lateReservationBeforeManualResolution->id )->state === (string) $lateReservationBeforeManualResolution->state, 'Conflicting manual absence resolution refunded or released a create.'); $legacyLateOrderSpec = $lateOrderSpec; unset($legacyLateOrderSpec['stock_mode']); $lateOrder->spec_json = json_encode( $legacyLateOrderSpec, JSON_THROW_ON_ERROR ); $lateOrder->save(); $makeLateCreateRetryable = static function () use ( $lateCreateJob ): void { Capsule::table('cloud_jobs')->where( 'id', (int) $lateCreateJob->id )->update([ 'available_at' => 0, 'lease_token' => null, 'lease_until' => 0, ]); }; $assertLateCreateStillPending = static function ( string $message ) use ($assert, $lateCreate, $lateReceiptId): void { $assert((string) \App\Models\CloudAgentCompletionEvent::where( 'event_id', $lateReceiptId )->value('disposition') === 'received' && CloudInstance::where('order_id', $lateCreate->id)->count() === 0, $message); }; $finiteMarkerlessSpec = $legacyLateOrderSpec; $finiteMarkerlessSpec['stock_mode'] = 'finite'; $lateOrder->spec_json = json_encode( $finiteMarkerlessSpec, JSON_THROW_ON_ERROR ); $lateOrder->save(); $finiteGuardWallet = (string) User::findOrFail(1)->money; $finiteGuardReservation = CloudResourceReservation::where( 'order_id', $lateCreate->id )->firstOrFail(); $makeLateCreateRetryable(); $assert((new CloudAutomaticOperationRecoveryService())->tick(), 'Markerless finite-stock create was not retried safely.'); $assertLateCreateStillPending( 'A markerless finite-stock order allowed late create materialisation or ACK.' ); $assert(CloudOrder::findOrFail($lateCreate->id)->refunded_at === null && (string) CloudInvoice::findOrFail($lateInvoice->id)->status === 'paid' && (string) CloudResourceReservation::findOrFail( $finiteGuardReservation->id )->state === (string) $finiteGuardReservation->state && $bundleState( CloudResourceReservation::findOrFail( $finiteGuardReservation->id ), 'uncertain', 'uncertain' ) && (string) User::findOrFail(1)->money === $finiteGuardWallet && (int) CloudOffer::findOrFail(1)->stock === -1, 'A markerless finite-stock order released or refunded fenced ledgers.'); $unknownStockModeSpec = $legacyLateOrderSpec; $unknownStockModeSpec['stock_mode'] = 'unknown'; $lateOrder->spec_json = json_encode( $unknownStockModeSpec, JSON_THROW_ON_ERROR ); $lateOrder->save(); $makeLateCreateRetryable(); $assert((new CloudAutomaticOperationRecoveryService())->tick(), 'Unknown stock-mode create was not retried safely.'); $assertLateCreateStillPending( 'An unknown stock mode allowed late create materialisation or ACK.' ); $nullStockModeSpec = $legacyLateOrderSpec; $nullStockModeSpec['stock_mode'] = null; $lateOrder->spec_json = json_encode( $nullStockModeSpec, JSON_THROW_ON_ERROR ); $lateOrder->save(); $makeLateCreateRetryable(); $assert((new CloudAutomaticOperationRecoveryService())->tick(), 'Null stock-mode create was not retried safely.'); $assertLateCreateStillPending( 'A null stock mode allowed late create materialisation or ACK.' ); $lateOrder->spec_json = json_encode( $legacyLateOrderSpec, JSON_THROW_ON_ERROR ); $lateOrder->save(); $unlimitedOffer->updated_at = (int) $lateOrder->created_at; $unlimitedOffer->save(); $makeLateCreateRetryable(); $assert((new CloudAutomaticOperationRecoveryService())->tick(), 'Ambiguous legacy unlimited-stock create was not retried safely.'); $assertLateCreateStillPending( 'A changed legacy offer allowed late create materialisation or ACK.' ); $unlimitedOffer->updated_at = (int) $lateOrder->created_at - 1; $unlimitedOffer->save(); $lateInvoice->purpose = 'renewal'; $lateInvoice->save(); $makeLateCreateRetryable(); $assert((new CloudAutomaticOperationRecoveryService())->tick(), 'Wrong-purpose create invoice was not retried safely.'); $assertLateCreateStillPending( 'A wrong-purpose invoice allowed late create materialisation or ACK.' ); $lateInvoice->purpose = 'provision'; $lateInvoice->status = 'refunded'; $lateInvoice->save(); $makeLateCreateRetryable(); $assert((new CloudAutomaticOperationRecoveryService())->tick(), 'Refunded create invoice was not retried safely.'); $assertLateCreateStillPending( 'A refunded invoice allowed late create materialisation or ACK.' ); $lateInvoice->status = 'paid'; $lateInvoice->save(); $lateInvoiceRow = (array) Capsule::table('cloud_invoices') ->where('id', (int) $lateInvoice->id)->first(); Capsule::table('cloud_invoices')->where( 'id', (int) $lateInvoice->id )->delete(); $makeLateCreateRetryable(); $assert((new CloudAutomaticOperationRecoveryService())->tick(), 'Missing create invoice was not retried safely.'); $assertLateCreateStillPending( 'A missing invoice allowed late create materialisation or ACK.' ); Capsule::table('cloud_invoices')->insert($lateInvoiceRow); $lateInvoice = CloudInvoice::findOrFail((int) $lateInvoice->id); $makeLateCreateRetryable(); $lateRecoveryTick = (new CloudAutomaticOperationRecoveryService())->tick(); // The Agent may already have stopped retransmitting after the panel // durably accepted the event. The recovery worker must consume that same // stored receipt; a later duplicate is only an idempotent ACK check. $secondLateReceipt = CloudOperationCompletionService::accept($lateReceipt); $lateCreateJob->refresh(); $lateOrder->refresh(); $lateInvoice->refresh(); $lateInstance = CloudInstance::where('order_id', $lateCreate->id)->first(); $lateReservation = CloudResourceReservation::where( 'order_id', $lateCreate->id )->first(); $latePeriod = \App\Models\CloudServicePeriod::where( 'order_id', $lateCreate->id )->first(); $lateCredential = $lateInstance === null ? null : \App\Services\Cloud\CloudSecrets::decrypt( (string) $lateInstance->sealed_credential, 'instance-credential' ); $lateEvent->refresh(); $lateFinalPayload = \App\Services\Cloud\CloudSecrets::decrypt( (string) $lateCreateJob->sealed_payload, 'job-payload' ); $lateInvoiceDisplay = CloudInvoiceService::display($lateInvoice); $lateTaskDisplay = CloudJobService::display($lateCreateJob); $assert($lateRecoveryTick && $secondLateReceipt === [ 'accepted' => true, 'disposition' => 'applied', ] && $lateInstance !== null && (string) $lateCreateJob->phase === 'terminal' && (string) $lateCreateJob->outcome === 'failed' && (string) $lateCreateJob->status === 'failed' && (string) $lateCreateJob->effect_resolution === 'applied' && (string) $lateCreateJob->effect_resolution_reason === 'late_completion_receipt:' . $lateReceiptId && (string) $lateEvent->disposition === 'applied' && $lateEvent->applied_at !== null && (string) $lateOrder->status === 'succeeded' && (int) $lateOrder->instance_id === (int) $lateInstance->id && (int) $lateInvoice->instance_id === (int) $lateInstance->id && (string) $lateReservation->state === 'committed' && $latePeriod !== null && (string) $latePeriod->status === 'active' && (int) $latePeriod->instance_id === (int) $lateInstance->id && (int) $lateInstance->current_period_id === (int) $latePeriod->id && $lateCredential === $lateCreatePayload['credential'] && !array_key_exists('credential', $lateFinalPayload) && !array_key_exists('root_password', $lateFinalPayload['body'] ?? []) && ($lateInvoiceDisplay['job_status'] ?? null) === 'succeeded' && ($lateInvoiceDisplay['fulfillment_status'] ?? null) === 'succeeded' && ($lateTaskDisplay['status'] ?? null) === 'failed' && ($lateTaskDisplay['effect_resolution'] ?? null) === 'applied', 'The exact late create receipt did not atomically converge PHP ledgers and public fulfillment state.'); $assert(CloudInvoiceService::deliverReminders(20) >= 1 && (string) CloudMailOutbox::where( 'invoice_id', $lateInvoice->id )->where('kind', 'provision_succeeded')->value('status') === 'sent', 'Recovered create success was cancelled instead of delivering its provision notice.'); $assert($lateOrder->stock_reserved_at === null && $lateOrder->stock_released_at === null && (int) CloudOffer::findOrFail(1)->stock === -1, 'Unlimited-stock create recovery mutated finite inventory.'); $unlimitedOffer->stock = $finiteStockBeforeUnlimitedCreate; $unlimitedOffer->created_at = $offerCreatedAtBeforeUnlimitedCreate; $unlimitedOffer->updated_at = time(); $unlimitedOffer->save(); $actionJob = CloudJobService::enqueue($instance, $actor, 'action', 'POST', '/actions', ['action' => 'reboot']); $mock->append(new Response(200, [], json_encode([ 'operation_id' => $actionJob->operation_id, 'status' => 'succeeded', ...CloudOperationRecoveryIdentity::expected($actionJob, true), 'result' => [ 'uuid' => (string) $instance->uuid, 'state' => 'running', ], ]))); $worker->tick(); $last = $agentRequest(end($history)); $signedActor = json_decode(base64_decode(strtr( $last['headers']['X-Virt-Actor'], '-_', '+/' )), true); $assert(CloudJob::find($actionJob->id)->status === 'succeeded' && $signedActor['can_manage'] === true && ($signedActor['operation_id'] ?? null) === $actionJob->operation_id, 'Power task lost its typed Agent authority or did not complete.'); // Keep the panel/audit vocabulary stable while translating the force-off // command to the Go Agent's strict PowerAction wire value. $poweroffJob = CloudJobService::enqueue( $instance, $actor, 'action', 'POST', '/actions', ['action' => 'poweroff'] ); $storedPoweroff = \App\Services\Cloud\CloudSecrets::decrypt( (string) $poweroffJob->sealed_payload, 'job-payload' ); $mock->append(new Response(200, [], json_encode([ 'operation_id' => $poweroffJob->operation_id, 'status' => 'succeeded', ...CloudOperationRecoveryIdentity::expected($poweroffJob, true), 'result' => [ 'uuid' => (string) $instance->uuid, 'state' => 'stopped', ], ]))); $worker->tick(); $poweroffRequest = $agentRequest(end($history)); $poweroffBody = json_decode( (string) ($poweroffRequest['body'] ?? ''), true, 8, JSON_THROW_ON_ERROR ); $assert(CloudJob::find($poweroffJob->id)->status === 'succeeded' && ($storedPoweroff['body']['action'] ?? null) === 'poweroff' && $poweroffBody === ['action' => 'destroy'], 'Panel poweroff was not translated exactly once at the Agent wire boundary.'); // Keep one bounded Agent cause for administrator diagnosis while the // public job contract continues to expose only the panel-owned error. $diagnosticJob = CloudJobService::enqueue( $instance, $actor, 'action', 'POST', '/actions', ['action' => 'reboot'] ); $mock->append(new Response(200, [], json_encode([ 'operation_id' => $diagnosticJob->operation_id, 'status' => 'failed', ...CloudOperationRecoveryIdentity::expected($diagnosticJob), 'cause_code' => 'domain_start_failed', 'error' => [ 'code' => 'libvirt_error', 'message' => 'private path /srv/libvirt/domain.xml', ], 'effect_may_have_applied' => false, 'safe_to_resubmit' => true, 'result' => ['state' => 'stopped'], ]))); $worker->tick(); $diagnosticRow = CloudJob::find($diagnosticJob->id); $diagnosticResult = json_decode((string) $diagnosticRow->result_json, true, flags: JSON_THROW_ON_ERROR); $publicDiagnostic = CloudJobService::display($diagnosticRow); $assert($diagnosticRow->status === 'failed' && $diagnosticRow->error_code === 'operation_failed' && ($diagnosticResult['cause_code'] ?? null) === 'domain_start_failed' && !isset($publicDiagnostic['result']['cause_code']) && !str_contains(json_encode($publicDiagnostic, JSON_UNESCAPED_UNICODE), '/srv/libvirt'), 'Stable Agent cause was not persisted privately or leaked into the public job DTO.'); // A strict two-hop admission rejection proves that the mutation never // reached the Agent journal. It is terminal and may permit an explicit // future request, but the worker never re-dispatches this task. $requeueJob = CloudJobService::enqueue( $instance, $actor, 'action', 'POST', '/actions', ['action' => 'reboot'] ); $beforeRequeue = count($history); $mock->append(new Response(503, [], json_encode(['error' => [ 'code' => 'agent_busy', 'message' => 'Agent control request was not dispatched', 'retryable' => false, 'dispatch_state' => 'not_dispatched', ]]))); $worker->tick(); $requeueJob->refresh(); $assert($requeueJob->phase === 'terminal' && $requeueJob->outcome === 'failed' && $requeueJob->status === 'failed' && (int) $requeueJob->dispatch_attempts === 1 && $requeueJob->dispatched_at !== null && $requeueJob->lease_token === null && (bool) $requeueJob->effect_may_have_applied === false && (bool) $requeueJob->safe_to_resubmit === true && $requeueJob->effect_resolution === 'not_applied', 'Definitive pre-dispatch backpressure did not become a safe terminal failure.'); $worker->tick(); $requeueRequests = array_slice($history, $beforeRequeue); $assert(CloudJob::find($requeueJob->id)->status === 'failed' && count($requeueRequests) === 1 && $agentRequest($requeueRequests[0])['method'] === 'POST', 'Safe pre-dispatch failure was automatically retried.'); // A strict DTO rejection happens before the Agent operation journal for // every mutation kind, not only provisioning. It is terminal validation // failure and must not strand a non-create task in uncertain/404 polling. $invalidActionJob = CloudJobService::enqueue( $instance, $actor, 'action', 'POST', '/actions', ['action' => 'reboot'] ); $beforeInvalidAction = count($history); $mock->append(new Response(422, [], json_encode(['error' => [ 'code' => 'invalid_json', 'message' => 'request contains an unknown field', 'retryable' => false, 'dispatch_state' => 'not_dispatched', ]]))); $worker->tick(); $invalidActionJob->refresh(); $assert($invalidActionJob->status === 'failed' && $invalidActionJob->dispatched_at !== null && count($history) === $beforeInvalidAction + 1, 'Definitive non-create validation failure became permanently uncertain.'); // A gateway secret mismatch is a terminal configuration/authentication // rejection before WSS dispatch. It must not strand the task in uncertain // polling or loop forever in the queue. $gatewayAuthJob = CloudJobService::enqueue( $instance, $actor, 'action', 'POST', '/actions', ['action' => 'reboot'] ); $beforeGatewayAuth = count($history); $mock->append(new Response(403, [], json_encode(['error' => [ 'code' => 'gateway_forbidden', 'message' => 'Agent gateway request is forbidden', 'retryable' => false, 'dispatch_state' => 'not_dispatched', ]]))); $worker->tick(); $gatewayAuthJob->refresh(); $assert($gatewayAuthJob->status === 'failed' && $gatewayAuthJob->dispatched_at !== null && $gatewayAuthJob->lease_token === null, 'Gateway authentication failure did not terminate before journal dispatch.'); $gatewayAuthRequests = array_slice($history, $beforeGatewayAuth); $assert(count($gatewayAuthRequests) === 1 && $agentRequest($gatewayAuthRequests[0])['method'] === 'POST', 'Gateway authentication failure was replayed after a terminal response.'); // A journal-backed error envelope for another operation must never finish // this job, even when the foreign record itself is terminal. $crossBoundJob = CloudJobService::enqueue( $instance, $actor, 'action', 'POST', '/actions', ['action' => 'reset'] ); $mock->append(new Response(409, [], json_encode([ 'error' => [ 'code' => 'operation_payload_mismatch', 'retryable' => false, 'operation' => [ 'operation_id' => $actionJob->operation_id, 'status' => 'failed', 'error' => ['code' => 'operation_payload_mismatch'], ], ], ]))); $worker->tick(); $crossBoundJob->refresh(); $assert($crossBoundJob->phase === 'terminal' && $crossBoundJob->outcome === 'failed' && $crossBoundJob->status === 'failed' && (bool) $crossBoundJob->effect_may_have_applied === true && (bool) $crossBoundJob->safe_to_resubmit === false && $crossBoundJob->effect_resolution === 'unknown' && $instance->fresh()->status === 'active', 'A foreign durable error envelope advanced the current job.'); $crossBoundJob->effect_resolution = 'not_applied'; $crossBoundJob->effect_resolved_at = time(); $crossBoundJob->effect_resolved_by = 99; $crossBoundJob->effect_resolution_reason = 'test fixture fence resolution'; $crossBoundJob->save(); // Clean-break protocol: a legacy success shortcut without operation_id is // ambiguous and cannot be used to advance any mutation. $legacyShortcutJob = CloudJobService::enqueue( $instance, $actor, 'action', 'POST', '/actions', ['action' => 'reboot'] ); $mock->append(new Response(200, [], json_encode([ 'status' => 'succeeded', 'uuid' => (string) $instance->uuid, 'state' => 'running', ]))); $worker->tick(); $legacyShortcutJob->refresh(); $assert($legacyShortcutJob->phase === 'terminal' && $legacyShortcutJob->outcome === 'failed' && $legacyShortcutJob->status === 'failed' && $legacyShortcutJob->effect_resolution === 'unknown', 'A mutation response without operation_id was accepted as success.'); $legacyShortcutJob->effect_resolution = 'not_applied'; $legacyShortcutJob->effect_resolved_at = time(); $legacyShortcutJob->effect_resolved_by = 99; $legacyShortcutJob->effect_resolution_reason = 'test fixture fence resolution'; $legacyShortcutJob->save(); // Terminal tasks are never polled through the retired Agent operation // lookup/reconcile loop. Every Agent request issued above was the single // mutation dispatch for its own durable task. $lookupRequests = array_filter($history, static function (array $entry) use ($agentRequest): bool { return $agentRequest($entry)['method'] === 'GET'; }); $assert($lookupRequests === [], 'A terminal task triggered retired Agent lookup/reconcile traffic.'); $instance->network_status = 'active'; $instance->save(); // Expiry networking, graceful purge shutdown, and irreversible deletion // deliberately use separate states. Only the four target-state policy // jobs may converge; the final disk deletion is a one-shot mutation. $makeLifecycleInstance = function (string $uuid, string $name, string $serviceStatus, string $networkStatus) use ($instance) { $copy = $instance->replicate(); $copy->order_id = null; $copy->reservation_id = null; $copy->current_period_id = null; $copy->uuid = $uuid; $copy->name = $name; $copy->status = 'active'; $copy->service_status = $serviceStatus; $copy->network_status = $networkStatus; $copy->suspended_at = null; $copy->network_suspended_at = null; $copy->sealed_credential = \App\Services\Cloud\CloudSecrets::encrypt([ 'password' => 'LifecycleStrongPassword!123', 'login_name' => 'root', ], 'instance-credential'); $copy->created_at = $copy->updated_at = time(); $copy->save(); return $copy; }; $queuePolicy = function (CloudInstance $target, string $kind, string $suffix, ?string $reason = null) { $job = new CloudJob(); $job->operation_id = bin2hex(random_bytes(16)); $job->instance_id = $target->id; $job->user_id = $target->user_id; $job->host_id = $target->host_id; $job->kind = $kind; $job->execution_mode = 'agent'; $job->status = 'queued'; $job->sealed_payload = \App\Services\Cloud\CloudSecrets::encrypt([ 'method' => 'POST', 'request_target' => '/v1/vms/' . $target->uuid . '/' . $suffix, 'body' => $reason === null ? [] : ['reason' => $reason], 'owner_id' => (int) $target->user_id, 'system_policy' => true, 'suspend_reason' => $reason, ], 'job-payload'); $job->available_at = $job->created_at = $job->updated_at = time(); $job->save(); return $job; }; $adminUser = new User(); $adminUser->id = 99; $adminUser->money = '0.00'; $adminUser->enable = 1; $adminUser->is_admin = true; $adminUser->save(); $adminActor = ['user_id' => 99, 'is_admin' => true, 'is_super_admin' => true, 'can_manage' => true, 'session_id' => '']; $expiredNetwork = $makeLifecycleInstance( '55555555-5555-4555-8555-555555555555', 'expired-network-vm', 'expired_grace', 'blocking_expired'); $expiredNetworkJob = $queuePolicy( $expiredNetwork, 'network_suspend', 'network/suspend', 'expired'); $mock->append(new Response(200, [], json_encode([ 'operation_id' => $expiredNetworkJob->operation_id, 'status' => 'succeeded', ...CloudOperationRecoveryIdentity::expected( $expiredNetworkJob, true ), 'result' => ['network_suspended' => true], ]))); $worker->tick(); $assert(CloudJob::find($expiredNetworkJob->id)->status === 'succeeded' && $expiredNetwork->fresh()->network_status === 'blocked_expired', 'Expiry network suspension was recorded as a traffic quota block.'); $purgeStopped = $makeLifecycleInstance( '66666666-6666-4666-8666-666666666666', 'purge-stop-success-vm', 'purge_stopping', 'blocked_expired'); $purgeStopped->status = 'deleting'; $purgeStopped->save(); $purgeStopJob = $queuePolicy( $purgeStopped, 'service_suspend', 'service/suspend', 'purge'); $mock->append(new Response(200, [], json_encode([ 'operation_id' => $purgeStopJob->operation_id, 'status' => 'succeeded', ...CloudOperationRecoveryIdentity::expected($purgeStopJob, true), 'result' => ['service_was_running' => true], ]))); $worker->tick(); $purgeStopped->refresh(); $assert(CloudJob::find($purgeStopJob->id)->status === 'succeeded' && $purgeStopped->status === 'deleting' && $purgeStopped->service_status === 'purge_deleting' && $purgeStopped->suspended_at === null, 'Successful expiry shutdown did not advance to disk deletion safely.'); $purgeStopFailed = $makeLifecycleInstance( '77777777-7777-4777-8777-777777777777', 'purge-stop-failed-vm', 'purge_stopping', 'blocked_expired'); $purgeStopFailed->status = 'deleting'; $purgeStopFailed->save(); $purgeStopFailedJob = $queuePolicy( $purgeStopFailed, 'service_suspend', 'service/suspend', 'purge'); $mock->append(new Response(409, [], json_encode([ 'operation_id' => $purgeStopFailedJob->operation_id, 'status' => 'failed', ...CloudOperationRecoveryIdentity::expected($purgeStopFailedJob), 'error' => ['code' => 'conflict', 'message' => 'shutdown rejected'], 'effect_may_have_applied' => false, 'safe_to_resubmit' => true, 'replayed' => false, 'retryable' => false, ]))); $worker->tick(); $purgeStopFailed->refresh(); $assert(CloudJob::find($purgeStopFailedJob->id)->status === 'failed' && $purgeStopFailed->status === 'deleting' && $purgeStopFailed->service_status === 'purge_stop_failed', 'Failed expiry shutdown incorrectly advanced or failed the disk purge.'); $purgeForceRequired = $makeLifecycleInstance( '76767676-7676-4676-8676-767676767676', 'purge-force-required-vm', 'purge_stopping', 'blocked_expired'); $purgeForceRequired->status = 'deleting'; $purgeForceRequired->save(); $purgeForcePrior = $queuePolicy( $purgeForceRequired, 'service_suspend', 'service/suspend', 'purge'); $mock->append(new Response(409, [], json_encode([ 'operation_id' => $purgeForcePrior->operation_id, 'status' => 'failed', ...CloudOperationRecoveryIdentity::expected($purgeForcePrior), 'error' => ['code' => 'force_required'], 'replayed' => false, 'retryable' => false, ]))); $worker->tick(); $purgeForceRequired->refresh(); $purgeForcePrior->refresh(); $assert($purgeForcePrior->phase === 'terminal' && $purgeForcePrior->outcome === 'failed' && $purgeForcePrior->status === 'failed' && $purgeForcePrior->error_code === 'force_required' && $purgeForceRequired->status === 'deleting' && $purgeForceRequired->service_status === 'purge_force_required', 'Expiry graceful shutdown crossed the explicit force boundary.'); // A force-required graceful shutdown is already a terminal failed task. // Any later destructive force action must be a separately audited task; // inspection never rewrites the prior terminal receipt. Capsule::table('cloud_backup_subscriptions')->insert([ 'instance_id' => (int) $purgeForceRequired->id, 'user_id' => (int) $purgeForceRequired->user_id, 'offer_id' => (int) $purgeForceRequired->offer_id, 'enabled' => 1, 'renew_next_period' => 1, 'frequency' => 'daily', 'weekday' => 0, 'retention_count' => 2, 'timezone' => 'UTC', 'price' => '2.50', 'currency' => 'USD', 'price_version' => 1, 'next_run_at' => time() + 3600, 'created_at' => time(), 'updated_at' => time(), ]); $purgeForceJob = CloudJobService::enqueueForceDelete( $purgeForcePrior, $adminActor ); $mock->append(new Response(200, [], json_encode([ 'operation_id' => $purgeForceJob->operation_id, 'status' => 'succeeded', ...CloudOperationRecoveryIdentity::expected($purgeForceJob, true), 'result' => [ 'deleted' => true, 'managed_artifacts_absent' => true, 'forced' => true, ], ]))); $worker->tick(); $purgeForcedRequest = $agentRequest(end($history)); $purgeForcedBody = json_decode($purgeForcedRequest['body'], true); $assert($purgeForceJob->fresh()->status === 'succeeded' && $purgeForcePrior->fresh()->status === 'failed' && $purgeForcePrior->fresh()->error_code === 'force_required' && $purgeForceRequired->fresh()->status === 'deleted' && (int) Capsule::table('cloud_backup_subscriptions') ->where('instance_id', (int) $purgeForceRequired->id) ->value('enabled') === 0 && (int) Capsule::table('cloud_backup_subscriptions') ->where('instance_id', (int) $purgeForceRequired->id) ->value('renew_next_period') === 0 && $purgeForcedRequest['method'] === 'POST' && str_ends_with($purgeForcedRequest['request_target'], '/force-delete') && ($purgeForcedBody['prior_operation_id'] ?? null) === $purgeForcePrior->operation_id && ($purgeForcedBody['ack_data_loss'] ?? null) === true, 'Expiry force-required state did not reach the separately audited force route.'); $expiryDeleted = $makeLifecycleInstance( '88888888-8888-4888-8888-888888888888', 'expiry-delete-success-vm', 'purge_deleting', 'blocked_expired'); $expiryDeleted->status = 'deleting'; $expiryDeleted->save(); $olderBacklog = CloudJobService::enqueue($instance, $actor, 'action', 'POST', '/actions', ['action' => 'reboot']); $expiryDeleteJob = CloudJobService::enqueueExpiryDelete($expiryDeleted); $beforeExpiryDeleteRequests = count($history); $user->enable = 0; $user->save(); $mock->append(new Response(200, [], json_encode([ 'operation_id' => $expiryDeleteJob->operation_id, 'status' => 'succeeded', ...CloudOperationRecoveryIdentity::expected($expiryDeleteJob, true), 'result' => [ 'deleted' => true, 'managed_artifacts_absent' => true, 'forced' => true, ], ]))); $worker->tick(); $user->enable = 1; $user->save(); $expiryDeleteRequests = array_slice($history, $beforeExpiryDeleteRequests); $expiryDeleteRequest = count($expiryDeleteRequests) === 1 ? $agentRequest($expiryDeleteRequests[0]) : []; $expiryDeleteActor = count($expiryDeleteRequests) === 1 ? json_decode(base64_decode(strtr( $expiryDeleteRequest['headers']['X-Virt-Actor'], '-_', '+/' )), true) : []; $expiryDeleted->refresh(); $assert(count($expiryDeleteRequests) === 1 && $expiryDeleteRequest['method'] === 'DELETE' && !str_contains($expiryDeleteRequest['request_target'], '?') && json_decode($expiryDeleteRequest['body'], true) === ['remove_managed_disks' => true] && ($expiryDeleteActor['can_manage'] ?? false) === true && ($expiryDeleteActor['operation_id'] ?? null) === $expiryDeleteJob->operation_id && !array_key_exists('system_cleanup', $expiryDeleteActor) && $expiryDeleteActor['is_admin'] === false && CloudJob::find($expiryDeleteJob->id)->phase === 'terminal' && CloudJob::find($expiryDeleteJob->id)->outcome === 'failed' && CloudJob::find($expiryDeleteJob->id)->status === 'failed' && (bool) CloudJob::find($expiryDeleteJob->id) ->effect_may_have_applied === true && (bool) CloudJob::find($expiryDeleteJob->id) ->safe_to_resubmit === false && $expiryDeleted->status === 'delete_failed' && $expiryDeleted->service_status === 'purge_failed' && $expiryDeleted->sealed_credential !== null, 'Expiry cleanup accepted force evidence on the graceful route or released panel state.'); $olderBacklog->phase = 'terminal'; $olderBacklog->outcome = 'failed'; $olderBacklog->status = 'failed'; $olderBacklog->terminal_at = time(); $olderBacklog->effect_may_have_applied = 0; $olderBacklog->safe_to_resubmit = 1; $olderBacklog->effect_resolution = 'not_applied'; $olderBacklog->available_at = 0; $olderBacklog->save(); $beforeExpiryDeleteRetry = count($history); $worker->tick(); $assert(count($history) === $beforeExpiryDeleteRetry && $expiryDeleteJob->fresh()->status === 'failed' && (int) $expiryDeleteJob->fresh()->dispatch_attempts === 1, 'Terminal expiry deletion was automatically looked up or mutation-retried.'); $forgedCleanup = $makeLifecycleInstance( '89898989-8989-4898-8898-898989898989', 'forged-expiry-delete-vm', 'purge_deleting', 'blocked_expired'); $forgedCleanup->status = 'deleting'; $forgedCleanup->save(); $forgedCleanupJob = CloudJobService::enqueueExpiryDelete($forgedCleanup); $forgedPayload = \App\Services\Cloud\CloudSecrets::decrypt( $forgedCleanupJob->sealed_payload, 'job-payload' ); $forgedPayload['request_target'] = '/v1/vms/' . $instance->uuid; $forgedCleanupJob->sealed_payload = \App\Services\Cloud\CloudSecrets::encrypt( $forgedPayload, 'job-payload' ); $forgedCleanupJob->save(); $beforeForgedCleanupRequests = count($history); $worker->tick(); $forgedCleanup->refresh(); $assert(count($history) === $beforeForgedCleanupRequests && CloudJob::find($forgedCleanupJob->id)->status === 'failed' && $forgedCleanup->status === 'delete_failed' && $forgedCleanup->service_status === 'purge_failed', 'A forged cleanup payload crossed its bound instance or reached the Agent.'); $expiryDeleteFailed = $makeLifecycleInstance( '99999999-9999-4999-8999-999999999999', 'expiry-delete-failed-vm', 'purge_deleting', 'blocked_expired'); $expiryDeleteFailed->status = 'deleting'; $expiryDeleteFailed->save(); $expiryDeleteFailedJob = CloudJobService::enqueueExpiryDelete($expiryDeleteFailed); $mock->append(new Response(409, [], json_encode([ 'operation_id' => $expiryDeleteFailedJob->operation_id, 'status' => 'failed', ...CloudOperationRecoveryIdentity::expected($expiryDeleteFailedJob), 'error' => [ 'code' => 'conflict', 'message' => 'volume delete rejected before undefine', ], 'effect_may_have_applied' => false, 'safe_to_resubmit' => true, 'replayed' => false, 'retryable' => false, ]))); $worker->tick(); $expiryDeleteFailed->refresh(); $assert(CloudJob::find($expiryDeleteFailedJob->id)->status === 'failed' && $expiryDeleteFailed->status === 'delete_failed' && $expiryDeleteFailed->service_status === 'purge_failed' && $expiryDeleteFailed->sealed_credential !== null, 'Explicit expiry deletion failure did not enter the manual-review state.'); $beforeFailedExpiryRetry = count($history); $worker->tick(); $expiryDeleteFailedJob->refresh(); $assert(count($history) === $beforeFailedExpiryRetry && $expiryDeleteFailedJob->phase === 'terminal' && $expiryDeleteFailedJob->outcome === 'failed' && (int) $expiryDeleteFailedJob->dispatch_attempts === 1, 'Terminal expiry failure was automatically looked up, reconciled, or mutation-retried.'); $reinstall = CloudJobService::enqueue($instance, $actor, 'reinstall', 'POST', '/reinstall', ['template_id' => $reinstallTemplate->id, 'confirmation_name' => $instance->name, 'root_password' => 'NewStrongPassword!123'], ['password' => 'NewStrongPassword!123', 'login_name' => 'root', 'auth_mode' => 'password']); $oldCredential = $instance->sealed_credential; $queuedReinstall = \App\Services\Cloud\CloudSecrets::decrypt( (string) $reinstall->sealed_payload, 'job-payload' ); $assert(($queuedReinstall['template_metadata']['template_id'] ?? null) === $reinstallTemplate->id && ($queuedReinstall['template_metadata']['os_source'] ?? null) === 'cloud_template' && ($queuedReinstall['template_metadata']['os_name'] ?? null) === 'Debian 13' && ($queuedReinstall['template_metadata']['login_name'] ?? null) === 'root' && ($queuedReinstall['template_metadata']['debian_12_network_fix'] ?? null) === false && !array_key_exists('os_source', $queuedReinstall['body'] ?? []) && !array_key_exists('os_name', $queuedReinstall['body'] ?? []) && !array_key_exists('login_name', $queuedReinstall['body'] ?? []), 'Reinstall did not keep PHP template metadata outside the strict Agent DTO.'); $mock->append(new Response(202, [], json_encode([ 'operation_id' => $reinstall->operation_id, 'status' => 'reserved', ...CloudOperationRecoveryIdentity::expected($reinstall), 'replayed' => false, 'retryable' => false, ]))); $worker->tick(); $reinstallRequest = $agentRequest(end($history)); $reinstallBody = json_decode( (string) ($reinstallRequest['body'] ?? ''), true, 16, JSON_THROW_ON_ERROR ); $reinstallKeys = array_keys($reinstallBody); sort($reinstallKeys); $expectedReinstallKeys = [ 'root_password', 'template_generation_token', 'template_id', 'template_revision', ]; sort($expectedReinstallKeys); $assert(CloudJob::find($reinstall->id)->status === 'running' && $instance->fresh()->sealed_credential === $oldCredential, 'Pending reinstall replaced password prematurely.'); $assert($reinstallKeys === $expectedReinstallKeys && !array_key_exists('confirmation_name', $reinstallBody), 'Browser reinstall confirmation leaked into the strict Agent DTO.'); $beforeReinstallReceipt = count($history); $reinstallReceipt = CloudOperationCompletionService::accept([ 'type' => 'completion', 'id' => bin2hex(random_bytes(16)), 'agent_id' => (string) $instance->host_id, 'operation_id' => $reinstall->operation_id, ...CloudOperationRecoveryIdentity::expected($reinstall, true), 'status' => 'succeeded', 'effect_may_have_applied' => false, 'safe_to_resubmit' => false, 'error_code' => '', 'result' => [ 'state' => 'running', ], ]); $reinstalled = $instance->fresh(); $reinstalledSpec = json_decode((string) $reinstalled->spec_json, true, flags: JSON_THROW_ON_ERROR); $assert($reinstallReceipt === ['accepted' => true, 'disposition' => 'applied'] && count($history) === $beforeReinstallReceipt && CloudJob::find($reinstall->id)->status === 'succeeded' && $reinstalled->sealed_credential !== $oldCredential && ($reinstalledSpec['template_id'] ?? null) === $reinstallTemplate->id && ($reinstalledSpec['public_network'] ?? null) === $openingSpec['public_network'] && ($reinstalledSpec['os_source'] ?? null) === 'cloud_template' && ($reinstalledSpec['os_name'] ?? null) === 'Debian 13' && ($reinstalledSpec['login_name'] ?? null) === 'root' && ($reinstalledSpec['managed'] ?? null) === true && !isset($reinstalledSpec['install_iso'], $reinstalledSpec['agent_metadata_version']), 'Completed reinstall lost its trusted template metadata.'); // A replacement guest may be running while old-volume cleanup remains // unresolved. The worker must keep the job uncertain but still persist // the new credential and template metadata; raw host warnings stay out of // result_json. $partialReinstall = CloudJobService::enqueue($instance->fresh(), $actor, 'reinstall', 'POST', '/reinstall', [ 'template_id' => $input['template_id'], 'confirmation_name' => $instance->name, 'root_password' => 'PartialStrongPassword!123', ], ['password' => 'PartialStrongPassword!123', 'login_name' => 'root', 'auth_mode' => 'password']); $mock->append(new Response(202, [], json_encode([ 'operation_id' => $partialReinstall->operation_id, 'status' => 'reserved', ...CloudOperationRecoveryIdentity::expected($partialReinstall), 'replayed' => false, 'retryable' => false, ]))); $worker->tick(); $beforePartialReceipt = count($history); $partialReceipt = CloudOperationCompletionService::accept([ 'type' => 'completion', 'id' => bin2hex(random_bytes(16)), 'agent_id' => (string) $instance->host_id, 'operation_id' => $partialReinstall->operation_id, ...CloudOperationRecoveryIdentity::expected( $partialReinstall, true ), 'status' => 'succeeded', 'effect_may_have_applied' => false, 'safe_to_resubmit' => false, 'error_code' => '', 'result' => [ 'reinstall_committed' => true, 'cleanup_pending' => true, 'vm' => ['uuid' => (string) $instance->uuid, 'name' => $instance->name, 'state' => 'running'], ], ]); $partialJob = CloudJob::find($partialReinstall->id); $assert($partialReceipt === ['accepted' => true, 'disposition' => 'received'] && count($history) === $beforePartialReceipt && $partialJob->phase === 'terminal' && $partialJob->outcome === 'failed' && $partialJob->status === 'failed' && (int) $partialJob->dispatch_attempts === 1 && (bool) $partialJob->effect_may_have_applied === true && (bool) $partialJob->safe_to_resubmit === false && $instance->fresh()->sealed_credential === $reinstalled->sealed_credential, 'Cleanup-pending reinstall was acknowledged or did not become a terminal fenced failure.'); $beforePartialRetry = count($history); $worker->tick(); $assert(count($history) === $beforePartialRetry && CloudJob::find($partialReinstall->id)->status === 'failed', 'Cleanup-pending reinstall was automatically looked up or retried.'); $partialJob->effect_resolution = 'applied'; $partialJob->effect_resolved_at = time(); $partialJob->effect_resolved_by = 99; $partialJob->effect_resolution_reason = 'test fixture confirms replacement guest committed'; $partialJob->save(); // A transition-owned network change crosses the complete Worker path // exactly once and publishes only the exact typed public_network result. $networkTargetIp = new CloudPublicIp(); $networkTargetIp->host_id = (int) $host->id; $networkTargetIp->public_ip_pool_id = (int) $sourcePublicPool->id; $networkTargetIp->address_family = 4; $networkTargetIp->address = '8.8.8.200'; $networkTargetIp->address_binary = inet_pton('8.8.8.200'); $networkTargetIp->prefix_length = 24; $networkTargetIp->state = 'available'; $networkTargetIp->created_at = $networkTargetIp->updated_at = time(); $networkTargetIp->save(); $networkDispatch = CloudPublicNetworkTransitionWorkflow::begin( CloudPublicNetworkTransitionCommand::adminExact( (int) $instance->id, 99, 'financial-network-transition-0001', [4 => [ 'pool_key' => $sourcePoolKey, 'address_cidr' => '8.8.8.200/24', ]] ) ); $networkJob = CloudJob::find($networkDispatch->jobId()); $networkPayload = \App\Services\Cloud\CloudSecrets::decrypt( (string) $networkJob->sealed_payload, 'job-payload' ); $networkIdentity = CloudOperationRecoveryIdentity::expectedFromPayload( $networkJob, $networkPayload, true ); $networkTarget = $networkDispatch->target()->toArray(); $mock->append(new Response(200, [], json_encode([ 'operation_id' => (string) $networkJob->operation_id, 'status' => 'succeeded', ...$networkIdentity, 'result' => [ 'uuid' => (string) $instance->uuid, 'network_changed' => true, 'public_network' => $networkTarget, ], ], JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES))); $beforeNetworkDispatch = count($history); $assert($worker->tick(), 'Transition-owned network task was not consumed by the Worker.'); $networkRequest = $agentRequest($history[$beforeNetworkDispatch]); $networkBody = json_decode( (string) $networkRequest['body'], true, 32, JSON_THROW_ON_ERROR ); $networkSpec = json_decode( (string) $instance->fresh()->spec_json, true, 64, JSON_THROW_ON_ERROR ); $assert(count($history) === $beforeNetworkDispatch + 1 && ($networkBody['public_network'] ?? null) === $networkTarget && CloudJob::find($networkJob->id)->phase === 'terminal' && CloudJob::find($networkJob->id)->outcome === 'succeeded' && (int) CloudJob::find($networkJob->id)->dispatch_attempts === 1 && Capsule::table('cloud_public_network_transitions') ->where('id', $networkDispatch->transitionId()) ->value('state') === 'succeeded' && ($networkSpec['public_network'] ?? null) === $networkTarget, 'Worker did not dispatch/finalize the exact transition projection.'); $afterNetworkDispatch = count($history); $worker->tick(); $assert(count($history) === $afterNetworkDispatch && (int) CloudJob::find($networkJob->id)->dispatch_attempts === 1, 'A terminal transition-owned network task was replayed.'); // A legacy scalar task that lacks transition ownership fails before the // Agent. Use an isolated instance so its durable incident cannot affect // the canonical instance's later lifecycle tests. $legacyNetworkInstance = $makeLifecycleInstance( '45454545-4545-4545-8545-454545454545', 'legacy-network-gate-vm', 'active', 'active' ); $legacyWorkerJob = new CloudJob(); $legacyWorkerJob->operation_id = bin2hex(random_bytes(16)); $legacyWorkerJob->request_idempotency_key = 'legacy-network-worker-gate-0001'; $legacyWorkerJob->request_fingerprint = hash( 'sha256', 'legacy-network-worker-gate-0001' ); $legacyWorkerJob->instance_id = (int) $legacyNetworkInstance->id; $legacyWorkerJob->user_id = (int) $legacyNetworkInstance->user_id; $legacyWorkerJob->host_id = (int) $legacyNetworkInstance->host_id; $legacyWorkerJob->kind = 'network_change'; $legacyWorkerJob->execution_mode = 'agent'; $legacyWorkerJob->status = 'queued'; $legacyWorkerJob->public_network_transition_id = null; $legacyWorkerJob->sealed_payload = \App\Services\Cloud\CloudSecrets::encrypt([ 'method' => 'POST', 'request_target' => '/v1/vms/' . rawurlencode((string) $legacyNetworkInstance->uuid) . '/network', 'body' => [ 'public_ip_cidr' => '9.9.9.10/24', 'confirm_restart' => true, ], 'credential' => null, 'owner_id' => (int) $legacyNetworkInstance->user_id, 'can_manage' => false, ], 'job-payload'); $legacyWorkerJob->available_at = $legacyWorkerJob->created_at = $legacyWorkerJob->updated_at = time(); $legacyWorkerJob->save(); $beforeLegacyWorker = count($history); $assert($worker->tick(), 'Legacy network task was not consumed by the local fail-closed path.'); $legacyWorkerJob->refresh(); $assert(count($history) === $beforeLegacyWorker && (int) $legacyWorkerJob->dispatch_attempts === 0 && $legacyWorkerJob->phase === 'terminal' && $legacyWorkerJob->outcome === 'failed', 'Legacy network task reached the Agent or remained claimable.'); $worker->tick(); $assert(count($history) === $beforeLegacyWorker && (int) $legacyWorkerJob->fresh()->dispatch_attempts === 0, 'Legacy network task was replayed after local rejection.'); // The ordinary job enqueue path is permanently closed to legacy scalar // network changes. Canonical transitions have their own durable workflow // and are tested in the dedicated begin/finalization behavior suites. $networkInput = ['public_ip_cidr' => '9.9.9.10/24', 'public_pool_key' => $targetPoolKey, 'confirm_restart' => true]; $beforeLegacyNetworkJobs = CloudJob::count(); try { CloudJobService::enqueue( $instance->fresh(), $actor, 'network_change', 'POST', '/network', $networkInput ); throw new RuntimeException( 'Legacy scalar network writer bypassed the transition gate.' ); } catch (CloudError $error) { $assert($error->httpStatus === 409 && CloudJob::count() === $beforeLegacyNetworkJobs, 'Transition gate created a network job before rejecting it.'); } // Stage 5 keeps one executable legacy-gate assertion above; paid charge, // idempotency, not-applied refund and unknown-effect recovery are covered // by the canonical transition Workflow and Finalization behavior suites. // Deletion consumes the unchanged exact dual-stack authority and never // reconstructs it from retained singular reservation columns. $canonicalClaimSet = \App\Services\Cloud\CloudPublicNetworkAllocationService::currentClaimSet( (int) $openingReservation->id, 'committed', (int) $instance->id ); $canonicalClaimIds = [ $canonicalClaimSet->claimId(4), $canonicalClaimSet->claimId(6), ]; $canonicalPublicIpIds = [ $canonicalClaimSet->publicIpId(4), $canonicalClaimSet->publicIpId(6), ]; $interruptedDelete = CloudJobService::enqueue($instance, $actor, 'delete', 'DELETE', '', ['remove_managed_disks' => true]); $mock->append(new Response(422, [], json_encode([ 'operation_id' => $interruptedDelete->operation_id, 'status' => 'failed', ...CloudOperationRecoveryIdentity::expected($interruptedDelete), 'error' => ['code' => 'graceful_shutdown_unconfirmed'], 'effect_may_have_applied' => false, 'safe_to_resubmit' => true, 'replayed' => false, 'retryable' => false, ]))); $worker->tick(); $interruptedDelete->refresh(); $assert($interruptedDelete->status === 'failed' && $interruptedDelete->error_code === 'graceful_shutdown_unconfirmed' && $instance->fresh()->status === 'active', 'An unconfirmed graceful delete did not close safely for a new normal retry.'); $delete = CloudJobService::enqueue($instance->fresh(), $actor, 'delete', 'DELETE', '', ['remove_managed_disks' => true]); $assert($delete->operation_id !== $interruptedDelete->operation_id, 'A graceful retry reused the interrupted delete operation identifier.'); $mock->append(new Response(409, [], json_encode([ 'operation_id' => $delete->operation_id, 'status' => 'failed', ...CloudOperationRecoveryIdentity::expected($delete), 'error' => ['code' => 'force_required'], 'replayed' => false, 'retryable' => false, ]))); $worker->tick(); $graceful = $agentRequest(end($history)); $delete->refresh(); $assert(json_decode($graceful['body'], true) === ['remove_managed_disks' => true] && !str_contains($graceful['request_target'], '?') && $delete->phase === 'terminal' && $delete->outcome === 'failed' && $delete->status === 'failed' && $delete->error_code === 'force_required' && $instance->fresh()->status === 'active', 'Graceful delete did not stop at the explicit force-required boundary.'); $failedForceDelete = CloudJobService::enqueueForceDelete( $delete, $adminActor ); $mock->append(new Response(422, [], json_encode([ 'operation_id' => $failedForceDelete->operation_id, 'status' => 'failed', ...CloudOperationRecoveryIdentity::expected($failedForceDelete), 'error' => ['code' => 'invalid_request'], 'effect_may_have_applied' => false, 'safe_to_resubmit' => true, 'replayed' => false, 'retryable' => false, ]))); $worker->tick(); $delete->refresh(); $failedForceDelete->refresh(); $assert($failedForceDelete->status === 'failed' && $delete->status === 'failed' && $delete->error_code === 'force_required' && $instance->fresh()->status === 'active', 'A definitely failed force attempt did not safely reopen the explicit force gate.'); $forceDelete = CloudJobService::enqueueForceDelete( $delete, $adminActor ); $mock->append(new Response(200, [], json_encode([ 'operation_id' => $forceDelete->operation_id, 'status' => 'succeeded', ...CloudOperationRecoveryIdentity::expected($forceDelete, true), 'result' => [ 'uuid' => (string) $instance->uuid, 'deleted' => true, 'managed_artifacts_absent' => true, 'forced' => true, ], ]))); $worker->tick(); $forced = $agentRequest(end($history)); $forcedBody = json_decode($forced['body'], true); $retainedDeleteReservation = CloudResourceReservation::where( 'order_id', $first->id )->first(); $assert($forced['method'] === 'POST' && str_ends_with($forced['request_target'], '/force-delete') && $forceDelete->operation_id !== $failedForceDelete->operation_id && ($forcedBody['operation_id'] ?? null) === $forceDelete->operation_id && ($forcedBody['prior_operation_id'] ?? null) === $delete->operation_id && ($forcedBody['ack_data_loss'] ?? null) === true && ($forcedBody['remove_managed_disks'] ?? null) === true && $forceDelete->fresh()->status === 'succeeded' && $delete->fresh()->status === 'failed' && $delete->fresh()->error_code === 'force_required' && $instance->fresh()->status === 'deleted' && $instance->fresh()->sealed_credential === null && $retainedDeleteReservation->state === 'released' && CloudPublicIp::whereIn('id', $canonicalPublicIpIds) ->where('state', 'available') ->whereNull('reservation_id') ->whereNull('reservation_ip_claim_id') ->whereNull('instance_id') ->whereNull('operation_id')->count() === 2 && Capsule::table('cloud_resource_reservation_ips') ->whereIn('id', $canonicalClaimIds) ->where('state', 'released') ->whereNull('active_claim')->count() === 2, 'Force-delete identity, acknowledgement, cleanup, or immutable prior receipt failed.'); $beforeForceReplay = count($history); $worker->tick(); $assert(count($history) === $beforeForceReplay && $forceDelete->fresh()->phase === 'terminal' && (int) $forceDelete->fresh()->dispatch_attempts === 1, 'Successful force-delete task was automatically re-dispatched.'); // Independent backup restores have no purchase order. Their committed // capacity and fresh IP must still be released after a proven deletion. $restoreOperationId = str_repeat('e', 32); $restoredUuid = 'eeeeeeee-eeee-4eee-8eee-eeeeeeeeeeee'; $backupRestore = new CloudBackupRestore(); $backupRestore->restore_key = str_repeat('f', 32); $backupRestore->backup_id = 9001; $backupRestore->target_host_id = (int) $host->id; $backupRestore->target_instance_id = null; $backupRestore->reservation_id = null; $backupRestore->job_id = null; $backupRestore->operation_id = $restoreOperationId; $backupRestore->new_uuid = $restoredUuid; $backupRestore->new_name = 'restored-delete-vm'; $backupRestore->state = 'ready'; $backupRestore->error_code = ''; $backupRestore->ready_at = time(); $backupRestore->created_at = $backupRestore->updated_at = time(); $backupRestore->save(); $restoredIp = new CloudPublicIp(); $restoredIp->host_id = (int) $host->id; $restoredIp->public_ip_pool_id = (int) $targetPublicPool->id; $restoredIp->address_family = 4; $restoredIp->address = '9.9.9.12'; $restoredIp->address_binary = inet_pton('9.9.9.12'); $restoredIp->prefix_length = 24; $restoredIp->state = 'assigned'; $restoredIp->reservation_id = null; $restoredIp->instance_id = null; $restoredIp->operation_id = null; $restoredIp->created_at = $restoredIp->updated_at = time(); $restoredIp->save(); $restoredReservation = new CloudResourceReservation(); $restoredReservation->reservation_key = str_repeat('d', 32); $restoredReservation->operation_id = $restoreOperationId; $restoredReservation->order_id = null; $restoredReservation->backup_restore_id = (int) $backupRestore->id; $restoredReservation->offer_id = null; $restoredReservation->resource_pool_id = (int) $pool->id; $restoredReservation->host_id = (int) $host->id; $restoredReservation->user_id = 1; $restoredReservation->instance_id = null; $restoredReservation->network_attachment_id = (int) $targetAttachment->id; $restoredReservation->public_ip_pool_id = (int) $targetPublicPool->id; $restoredReservation->public_ip_id = (int) $restoredIp->id; $restoredReservation->cpu_cores = 2; $restoredReservation->memory_mib = 2048; $restoredReservation->disk_gib = 20; $restoredReservation->state = 'committed'; $restoredReservation->expires_at = null; $restoredReservation->released_at = null; $restoredReservation->release_reason = ''; $restoredReservation->created_at = $restoredReservation->updated_at = time(); $restoredReservation->save(); $restoredInstance = $instance->replicate(); $restoredInstance->uuid = $restoredUuid; $restoredInstance->host_id = (int) $host->id; $restoredInstance->user_id = 1; $restoredInstance->order_id = null; $restoredInstance->plan_id = null; $restoredInstance->offer_id = null; $restoredInstance->resource_pool_id = (int) $pool->id; $restoredInstance->reservation_id = (int) $restoredReservation->id; $restoredInstance->current_period_id = null; $restoredInstance->name = 'restored-delete-vm'; $restoredInstance->status = 'active'; $restoredInstance->service_status = 'exempt'; $restoredInstance->network_status = 'active'; $restoredInstance->expires_at = null; $restoredInstance->spec_json = json_encode([ 'managed' => true, 'vcpu' => 2, 'memory_mib' => 2048, 'disk_gib' => 20, 'public_ip_cidr' => '9.9.9.12/24', ], JSON_THROW_ON_ERROR); $restoredInstance->sealed_credential = 'sealed-restored-credential'; $restoredInstance->created_at = $restoredInstance->updated_at = time(); $restoredInstance->save(); $restoredReservation->instance_id = (int) $restoredInstance->id; $restoredReservation->save(); $restoredIp->reservation_id = (int) $restoredReservation->id; $restoredIp->instance_id = (int) $restoredInstance->id; $restoredIp->operation_id = $restoreOperationId; $restoredIp->save(); $backupRestore->target_instance_id = (int) $restoredInstance->id; $backupRestore->reservation_id = (int) $restoredReservation->id; $backupRestore->save(); $beforeRestoredDeleteJobs = CloudJob::count(); $beforeRestoredDeleteRequests = count($history); try { CloudJobService::enqueue( $restoredInstance, $adminActor, 'delete', 'DELETE', '', ['remove_managed_disks' => true] ); throw new RuntimeException( 'A restored instance bypassed the dual-stack transition gate.' ); } catch (CloudError $error) { $assert($error->httpStatus === 409, 'Restored-instance delete gate returned the wrong status.'); } $restoredInstance->refresh(); $restoredReservation->refresh(); $restoredIp->refresh(); $assert(CloudJob::count() === $beforeRestoredDeleteJobs && count($history) === $beforeRestoredDeleteRequests && $restoredInstance->status === 'active' && $restoredReservation->state === 'committed' && $restoredIp->state === 'assigned' && (int) $restoredIp->reservation_id === (int) $restoredReservation->id && (int) $restoredIp->instance_id === (int) $restoredInstance->id, 'Restored-instance delete gate created work or changed its old ledger.'); $restoredInstance->status = 'deleting'; $restoredInstance->service_status = 'purge_deleting'; $restoredInstance->save(); try { CloudJobService::enqueueExpiryDelete($restoredInstance); throw new RuntimeException( 'A restored instance bypassed the expiry-delete transition gate.' ); } catch (CloudError $error) { $assert($error->httpStatus === 409, 'Restored-instance expiry-delete gate returned the wrong status.'); } $assert(CloudJob::count() === $beforeRestoredDeleteJobs && count($history) === $beforeRestoredDeleteRequests, 'Restored-instance expiry-delete gate created durable work.'); // This clean-break fixture intentionally models the retired singular // restore shape. Remove it explicitly after proving the gate so it cannot // poison later active-pair placement scans in this long scenario. $restoredInstance->delete(); $restoredIp->delete(); $restoredReservation->delete(); $backupRestore->delete(); // Provider callbacks credit the wallet before the invoice bridge runs. // That credit must be consumed once, queue one create job, and remain // idempotent when the payment provider retries its callback. $providerOrder = CloudOrderService::purchase(array_replace($input, [ 'name' => 'vm-provider-payment', 'idempotency_key' => 'test-request-key-provider-payment', ]), $actor); $providerInvoice = CloudInvoice::where('order_id', $providerOrder->id)->first(); $providerPayment = new CloudInvoicePayment(); $providerPayment->invoice_id = $providerInvoice->id; $providerPayment->user_id = 1; $providerPayment->provider = 'paytaro'; $providerPayment->provider_order_no = 'PT-PROVISION-0001'; $providerPayment->idempotency_key = 'provision-provider-payment-0001'; $providerPayment->amount = '9.99'; $providerPayment->status = 'pending'; $providerPayment->created_at = $providerPayment->updated_at = time(); $providerPayment->save(); $balanceBeforeProviderCredit = CloudMoney::cents((string) User::find(1)->money); $user = User::find(1); $user->money = CloudMoney::decimal($balanceBeforeProviderCredit + 999); $user->save(); $providerOutcome = CloudInvoicePaymentService::settleForProvider( 'paytaro', 'PT-PROVISION-0001'); $providerOutcomeAgain = CloudInvoicePaymentService::settleForProvider( 'paytaro', 'PT-PROVISION-0001'); $providerQueuedCreate = \App\Services\Cloud\CloudSecrets::decrypt( (string) CloudJob::where('order_id', $providerOrder->id) ->first()->sealed_payload, 'job-payload' ); $providerPolicy = $providerQueuedCreate['body'] ['platform_network_policy_snapshot'] ?? null; $assert($providerOutcome === 'paid_invoice' && $providerOutcomeAgain === 'paid_invoice' && CloudMoney::cents((string) User::find(1)->money) === $balanceBeforeProviderCredit && CloudInvoice::find($providerInvoice->id)->status === 'paid' && CloudOrder::find($providerOrder->id)->status === 'queued' && CloudJob::where('order_id', $providerOrder->id)->count() === 1 && ($providerPolicy['catalog_revision'] ?? null) === 2 && count($providerPolicy['rules'] ?? []) === 9, 'Provider callback did not settle an opening invoice exactly once.'); // A blank optional label is normalized before hashing, but generated only // for a genuinely new order. Replaying the same request with an explicit // empty field must therefore return the original generated display name. $automaticInput = array_replace($input, [ 'idempotency_key' => 'test-request-key-automatic-name', ]); unset($automaticInput['name']); $automatic = CloudOrderService::purchase($automaticInput, $actor); $automaticReplay = CloudOrderService::purchase( $automaticInput + ['name' => ''], $actor ); $automaticSpec = json_decode((string) $automatic->spec_json, true, flags: JSON_THROW_ON_ERROR); $assert($automatic->id === $automaticReplay->id && preg_match('/^vm-[abcdefghjkmnpqrstuvwxyz23456789]{6}$/D', (string) ($automaticSpec['name'] ?? '')) === 1, 'Blank instance names were not generated securely or idempotently.'); // Repeated provider settlement reuses the one accepted provisioning task. // It never creates a recovery generation or an automatic retry. $providerJob = CloudJob::where('order_id', $providerOrder->id)->first(); $providerOrderAgain = CloudOrderService::queuePaidProvision( CloudInvoice::find($providerInvoice->id), CloudOrder::find($providerOrder->id) ); $assert($providerJob !== null && $providerJob->phase === 'accepted' && $providerJob->outcome === null && $providerJob->status === 'queued' && (int) $providerJob->dispatch_attempts === 0 && (string) $providerOrderAgain->status === 'queued' && CloudJob::where('order_id', $providerOrder->id)->count() === 1, 'A repeated paid callback duplicated or retried its durable create task.'); // A definitively not-applied lifecycle failure starts a new audited // generation after backoff. The failed job and its operation ID remain // immutable; unknown, applied, or sticky failures remain fail-closed. $lifecycle = new CloudLifecycleService(); $reconcile = (new ReflectionClass(CloudLifecycleService::class)) ->getMethod('reconcile'); $oldPurgeOperation = (string) $purgeStopFailedJob->operation_id; $purgeCount = CloudJob::where('instance_id', $purgeStopFailed->id) ->where('kind', 'service_suspend')->count(); $assert($reconcile->invoke($lifecycle, (int) $purgeStopFailed->id) === false && CloudJob::where('instance_id', $purgeStopFailed->id) ->where('kind', 'service_suspend')->count() === $purgeCount, 'Lifecycle policy failure ignored the mandatory retry backoff.'); Capsule::table('cloud_jobs')->where('id', $purgeStopFailedJob->id) ->update(['terminal_at' => time() - 310, 'updated_at' => time() - 310]); $assert($reconcile->invoke($lifecycle, (int) $purgeStopFailed->id) === true, 'A definitively not-applied purge stop did not create a new generation: ' . json_encode($purgeStopFailedJob->fresh()->toArray(), JSON_THROW_ON_ERROR)); $purgeRetry = CloudJob::where('instance_id', $purgeStopFailed->id) ->where('kind', 'service_suspend')->orderByDesc('id')->first(); $assert($purgeRetry !== null && (int) $purgeRetry->id !== (int) $purgeStopFailedJob->id && (string) $purgeRetry->operation_id !== $oldPurgeOperation && (string) $purgeStopFailedJob->fresh()->operation_id === $oldPurgeOperation && (string) $purgeStopFailedJob->fresh()->phase === 'terminal' && (string) $purgeStopFailed->fresh()->service_status === 'purge_stopping', 'Purge retry replayed or rewrote the failed generation.'); $oldDeleteOperation = (string) $expiryDeleteFailedJob->operation_id; Capsule::table('cloud_jobs')->where('id', $expiryDeleteFailedJob->id) ->update(['terminal_at' => time() - 310, 'updated_at' => time() - 310]); $assert($reconcile->invoke($lifecycle, (int) $expiryDeleteFailed->id) === true, 'A definitively not-applied expiry delete did not create a new generation.'); $deleteRetry = CloudJob::where('instance_id', $expiryDeleteFailed->id) ->where('kind', 'expiry_delete')->orderByDesc('id')->first(); $assert($deleteRetry !== null && (int) $deleteRetry->id !== (int) $expiryDeleteFailedJob->id && (string) $deleteRetry->operation_id !== $oldDeleteOperation && (string) $expiryDeleteFailedJob->fresh()->operation_id === $oldDeleteOperation && (string) $expiryDeleteFailedJob->fresh()->phase === 'terminal' && (string) $expiryDeleteFailed->fresh()->status === 'deleting' && (string) $expiryDeleteFailed->fresh()->service_status === 'purge_deleting', 'Expiry-delete retry replayed or rewrote the failed generation.'); $finalizeUndispatchedFailure = static function ( CloudJob $job ): CloudJob { $past = time() - 310; $job->attempts = 0; $job->dispatch_attempts = 0; $job->dispatch_started_at = null; $job->agent_accepted_at = null; $job->dispatched_at = null; $job->receipt_deadline_at = null; $job->completion_event_id = null; $job->phase = 'terminal'; $job->outcome = 'failed'; $job->status = 'failed'; $job->terminal_at = $past; $job->effect_may_have_applied = 0; $job->safe_to_resubmit = 1; $job->effect_resolution = 'not_applied'; $job->error_code = 'dispatch_not_sent'; $job->finalization_incident = 0; $job->finalization_error_code = null; $job->updated_at = $past; $job->save(); return $job; }; $undispatchedDelete = $makeLifecycleInstance( '90909090-9090-4090-8090-909090909090', 'undispatched-delete-vm', 'purge_deleting', 'blocked_expired' ); $undispatchedDelete->status = 'deleting'; $undispatchedDelete->save(); $oldUndispatchedDelete = $finalizeUndispatchedFailure( CloudJobService::enqueueExpiryDelete($undispatchedDelete) ); $undispatchedDelete->status = 'delete_failed'; $undispatchedDelete->service_status = 'purge_failed'; $undispatchedDelete->save(); $assert($reconcile->invoke( $lifecycle, (int) $undispatchedDelete->id ) === true, 'A generation-zero expiry failure did not create a safe new generation.'); $undispatchedDeleteRetry = CloudJob::where( 'instance_id', $undispatchedDelete->id )->where('kind', 'expiry_delete')->orderByDesc('id')->first(); $assert($undispatchedDeleteRetry !== null && (int) $undispatchedDeleteRetry->id !== (int) $oldUndispatchedDelete->id && (string) $undispatchedDeleteRetry->operation_id !== (string) $oldUndispatchedDelete->operation_id && (int) $oldUndispatchedDelete->fresh()->dispatch_attempts === 0, 'Generation-zero expiry retry replayed or rewrote the prior operation.'); $undispatchedPurgeStop = $makeLifecycleInstance( '97979797-9797-4797-8797-979797979797', 'undispatched-purge-stop-vm', 'purge_stopping', 'blocked_expired' ); $undispatchedPurgeStop->status = 'deleting'; $undispatchedPurgeStop->save(); $oldUndispatchedPurgeStop = $finalizeUndispatchedFailure($queuePolicy( $undispatchedPurgeStop, 'service_suspend', 'service/suspend', 'purge' )); $undispatchedPurgeStop->service_status = 'purge_stop_failed'; $undispatchedPurgeStop->save(); $assert($reconcile->invoke( $lifecycle, (int) $undispatchedPurgeStop->id ) === true, 'A generation-zero purge stop did not create a safe new generation.'); $undispatchedPurgeStopRetry = CloudJob::where( 'instance_id', $undispatchedPurgeStop->id )->where('kind', 'service_suspend')->orderByDesc('id')->first(); $assert($undispatchedPurgeStopRetry !== null && (int) $undispatchedPurgeStopRetry->id !== (int) $oldUndispatchedPurgeStop->id && (string) $undispatchedPurgeStopRetry->operation_id !== (string) $oldUndispatchedPurgeStop->operation_id && (int) $oldUndispatchedPurgeStop->fresh()->dispatch_attempts === 0 && (string) $undispatchedPurgeStop->fresh()->service_status === 'purge_stopping', 'Generation-zero purge-stop retry replayed or rewrote the prior operation.'); $preparePolicyFailure = static function ( CloudInstance $target, string $kind, string $suffix, ?string $reason ) use ($queuePolicy): CloudJob { $job = $queuePolicy($target, $kind, $suffix, $reason); $identity = CloudOperationRecoveryIdentity::expected($job); $past = time() - 310; $job->request_fingerprint = $identity['request_fingerprint']; $job->attempts = 1; $job->dispatch_attempts = 1; $job->dispatch_started_at = $past - 30; $job->agent_accepted_at = $past - 20; $job->dispatched_at = $past - 20; $job->receipt_deadline_at = $past - 10; $job->phase = 'terminal'; $job->outcome = 'failed'; $job->status = 'failed'; $job->terminal_at = $past; $job->effect_may_have_applied = 0; $job->safe_to_resubmit = 1; $job->effect_resolution = 'not_applied'; $job->error_code = 'operation_failed'; $job->finalization_incident = 0; $job->finalization_error_code = null; $job->updated_at = $past; $job->save(); return $job; }; $setPolicyResolution = static function ( CloudJob $job, string $resolution, bool $incident = false ): CloudJob { if ($resolution === 'applied') { $job->outcome = 'succeeded'; $job->status = 'succeeded'; $job->effect_may_have_applied = 0; $job->safe_to_resubmit = 0; $job->error_code = null; } elseif ($resolution === 'unknown') { $job->outcome = 'failed'; $job->status = 'failed'; $job->effect_may_have_applied = 1; $job->safe_to_resubmit = 0; $job->error_code = 'operation_failed_effect_unknown'; } $job->effect_resolution = $resolution; $job->finalization_incident = $incident ? 1 : 0; $job->finalization_error_code = $incident ? 'recovery_ledger_conflict' : null; $job->save(); return $job; }; $insertScheduledPeriod = static function ( CloudInstance $target ): int { $now = time(); return (int) Capsule::table('cloud_service_periods')->insertGetId([ 'instance_id' => $target->id, 'user_id' => $target->user_id, 'plan_id' => $target->plan_id, 'starts_at' => $now - 10, 'ends_at' => $now + 3600, 'traffic_quota_bytes' => 1000, 'download_base_bytes' => null, 'upload_base_bytes' => null, 'download_used_bytes' => 0, 'upload_used_bytes' => 0, 'status' => 'scheduled', 'created_at' => $now - 100, 'updated_at' => $now - 100, ]); }; // Activating a scheduled paid period may update the billing ledger, but // it must not overwrite a policy projection whose Agent conclusion is // still unknown, contradictory, or sticky. $scheduledSuspend = $makeLifecycleInstance( '80808080-8080-4080-8080-808080808080', 'scheduled-suspend-conflict-vm', 'suspend_failed', 'active' ); $scheduledSuspend->expires_at = time() + 3600; $scheduledSuspend->current_period_id = null; $scheduledSuspend->save(); $scheduledSuspendJob = $setPolicyResolution($preparePolicyFailure( $scheduledSuspend, 'service_suspend', 'service/suspend', 'purge' ), 'unknown'); $scheduledSuspendPeriod = $insertScheduledPeriod($scheduledSuspend); $reconcile->invoke($lifecycle, (int) $scheduledSuspend->id); $assert((int) $scheduledSuspend->fresh()->current_period_id === $scheduledSuspendPeriod && (string) CloudServicePeriod::find($scheduledSuspendPeriod)->status === 'active' && (string) $scheduledSuspend->fresh()->service_status === 'suspend_failed' && (string) $scheduledSuspendJob->fresh()->effect_resolution === 'unknown', 'Scheduled activation falsely cleared an unknown service suspension.'); $scheduledBlock = $makeLifecycleInstance( '81818181-8181-4181-8181-818181818181', 'scheduled-block-conflict-vm', 'expired_grace', 'block_failed' ); $scheduledBlock->expires_at = time() + 3600; $scheduledBlock->current_period_id = null; $scheduledBlock->save(); $scheduledBlockJob = $setPolicyResolution($preparePolicyFailure( $scheduledBlock, 'network_suspend', 'network/suspend', 'expired' ), 'applied'); $scheduledBlockPeriod = $insertScheduledPeriod($scheduledBlock); $reconcile->invoke($lifecycle, (int) $scheduledBlock->id); $assert((int) $scheduledBlock->fresh()->current_period_id === $scheduledBlockPeriod && (string) $scheduledBlock->fresh()->network_status === 'block_failed' && (bool) $scheduledBlockJob->fresh()->finalization_incident && (string) $scheduledBlockJob->fresh()->finalization_error_code === 'recovery_instance_state_conflict', 'Scheduled activation hid an applied suspension/local-state conflict.'); $scheduledRestore = $makeLifecycleInstance( '82828282-8282-4282-8282-828282828282', 'scheduled-restore-conflict-vm', 'expired_grace', 'restore_failed' ); $scheduledRestore->expires_at = time() + 3600; $scheduledRestore->current_period_id = null; $scheduledRestore->save(); $scheduledRestoreJob = $setPolicyResolution($preparePolicyFailure( $scheduledRestore, 'network_resume', 'network/resume', null ), 'not_applied', true); $scheduledRestorePeriod = $insertScheduledPeriod($scheduledRestore); $reconcile->invoke($lifecycle, (int) $scheduledRestore->id); $assert((int) $scheduledRestore->fresh()->current_period_id === $scheduledRestorePeriod && (string) $scheduledRestore->fresh()->network_status === 'restore_failed' && (bool) $scheduledRestoreJob->fresh()->finalization_incident, 'Scheduled activation falsely cleared a sticky network restore failure.'); $makeImmediatelyRenewable = static function ( CloudInstance $target ): void { $past = time() - 60; $target->current_period_id = null; $target->expires_at = $past; $target->renewal_days = 30; $target->renewal_price = '1.00'; $target->renewal_traffic_bytes = 1000; $target->traffic_synced_at = time() + 10; $target->save(); Capsule::table('cloud_service_periods')->insert([ 'instance_id' => $target->id, 'user_id' => $target->user_id, 'plan_id' => $target->plan_id, 'starts_at' => $past - 86400, 'ends_at' => $past, 'traffic_quota_bytes' => 1000, 'download_base_bytes' => 0, 'upload_base_bytes' => 0, 'download_used_bytes' => 0, 'upload_used_bytes' => 0, 'status' => 'finished', 'created_at' => $past - 86400, 'updated_at' => $past, ]); $issueOne = new ReflectionMethod( CloudInvoiceService::class, 'issueOne' ); if ($issueOne->invoke(null, (int) $target->id, time()) !== true) { throw new RuntimeException( 'Failed to create the renewal invoice fixture.' ); } }; $user->money = '1000.00'; $user->save(); $renewUnknownSuspend = $makeLifecycleInstance( '83838383-8383-4383-8383-838383838383', 'renew-unknown-suspend-vm', 'suspend_failed', 'active' ); $makeImmediatelyRenewable($renewUnknownSuspend); $setPolicyResolution($preparePolicyFailure( $renewUnknownSuspend, 'service_suspend', 'service/suspend', 'purge' ), 'unknown'); $unknownRenewal = CloudRenewalService::renew( $renewUnknownSuspend, ['idempotency_key' => 'renew-unknown-suspend-0001'], $actor ); $assert((string) CloudServicePeriod::find( $unknownRenewal->period_id )->status === 'active' && (string) $renewUnknownSuspend->fresh()->service_status === 'suspend_failed', 'Immediate renewal falsely cleared an unknown service suspension.'); $renewAppliedBlock = $makeLifecycleInstance( '84848484-8484-4484-8484-848484848484', 'renew-applied-block-vm', 'expired_grace', 'block_failed' ); $makeImmediatelyRenewable($renewAppliedBlock); $setPolicyResolution($preparePolicyFailure( $renewAppliedBlock, 'network_suspend', 'network/suspend', 'expired' ), 'applied'); CloudRenewalService::renew( $renewAppliedBlock, ['idempotency_key' => 'renew-applied-block-0001'], $actor ); $assert((string) $renewAppliedBlock->fresh()->network_status === 'block_failed', 'Immediate renewal hid an applied network suspension conflict.'); $renewStickyRestore = $makeLifecycleInstance( '85858585-8585-4585-8585-858585858585', 'renew-sticky-restore-vm', 'expired_grace', 'restore_failed' ); $makeImmediatelyRenewable($renewStickyRestore); $setPolicyResolution($preparePolicyFailure( $renewStickyRestore, 'network_resume', 'network/resume', null ), 'not_applied', true); CloudRenewalService::renew( $renewStickyRestore, ['idempotency_key' => 'renew-sticky-restore-0001'], $actor ); $assert((string) $renewStickyRestore->fresh()->network_status === 'restore_failed', 'Immediate renewal cleared a sticky network restore failure.'); $renewSafeSuspend = $makeLifecycleInstance( '86868686-8686-4686-8686-868686868686', 'renew-safe-suspend-vm', 'suspend_failed', 'active' ); $makeImmediatelyRenewable($renewSafeSuspend); $safeSuspendJob = $preparePolicyFailure( $renewSafeSuspend, 'service_suspend', 'service/suspend', 'purge' ); CloudRenewalService::renew( $renewSafeSuspend, ['idempotency_key' => 'renew-safe-suspend-0001'], $actor ); $assert((string) $renewSafeSuspend->fresh()->service_status === 'suspend_failed', 'Immediate renewal bypassed lifecycle verification of a safe failure.'); $reconcile->invoke($lifecycle, (int) $renewSafeSuspend->id); $assert((string) $renewSafeSuspend->fresh()->service_status === 'active' && (string) $safeSuspendJob->fresh()->phase === 'terminal', 'Lifecycle did not clear a canonical not-applied failure after renewal.'); $trafficRetryInstance = $makeLifecycleInstance( '91919191-9191-4191-8191-919191919191', 'traffic-retry-vm', 'active', 'block_failed' ); $trafficRetryInstance->expires_at = time() + 86400; $trafficRetryInstance->lifetime_download_bytes = 200; $trafficRetryInstance->lifetime_upload_bytes = 0; $trafficRetryInstance->traffic_synced_at = time(); $trafficRetryInstance->save(); $trafficPeriodId = Capsule::table('cloud_service_periods')->insertGetId([ 'instance_id' => $trafficRetryInstance->id, 'user_id' => $trafficRetryInstance->user_id, 'starts_at' => time() - 3600, 'ends_at' => time() + 3600, 'traffic_quota_bytes' => 100, 'download_base_bytes' => 0, 'upload_base_bytes' => 0, 'download_used_bytes' => 200, 'upload_used_bytes' => 0, 'status' => 'exhausted', 'created_at' => time() - 3600, 'updated_at' => time() - 310, ]); $trafficRetryInstance->current_period_id = $trafficPeriodId; $trafficRetryInstance->save(); $oldTrafficJob = $preparePolicyFailure( $trafficRetryInstance, 'network_suspend', 'network/suspend', 'traffic' ); $assert($reconcile->invoke( $lifecycle, (int) $trafficRetryInstance->id ) === true, 'A still-required traffic block did not create a new generation.'); $trafficRetry = CloudJob::where('instance_id', $trafficRetryInstance->id) ->where('kind', 'network_suspend')->orderByDesc('id')->first(); $assert($trafficRetry !== null && (int) $trafficRetry->id !== (int) $oldTrafficJob->id && (string) $trafficRetry->operation_id !== (string) $oldTrafficJob->operation_id && (string) $oldTrafficJob->fresh()->phase === 'terminal' && (string) $trafficRetryInstance->fresh()->network_status === 'blocking', 'Traffic-block retry mutated or replayed its prior generation.'); $undispatchedTraffic = $makeLifecycleInstance( '96969696-9696-4696-8696-969696969696', 'undispatched-traffic-vm', 'active', 'block_failed' ); $undispatchedTraffic->expires_at = time() + 86400; $undispatchedTraffic->lifetime_download_bytes = 200; $undispatchedTraffic->lifetime_upload_bytes = 0; $undispatchedTraffic->traffic_synced_at = time(); $undispatchedTraffic->save(); $undispatchedTrafficPeriod = Capsule::table( 'cloud_service_periods' )->insertGetId([ 'instance_id' => $undispatchedTraffic->id, 'user_id' => $undispatchedTraffic->user_id, 'starts_at' => time() - 3600, 'ends_at' => time() + 3600, 'traffic_quota_bytes' => 100, 'download_base_bytes' => 0, 'upload_base_bytes' => 0, 'download_used_bytes' => 200, 'upload_used_bytes' => 0, 'status' => 'exhausted', 'created_at' => time() - 3600, 'updated_at' => time() - 310, ]); $undispatchedTraffic->current_period_id = $undispatchedTrafficPeriod; $undispatchedTraffic->save(); $oldUndispatchedTraffic = $finalizeUndispatchedFailure($queuePolicy( $undispatchedTraffic, 'network_suspend', 'network/suspend', 'traffic' )); $assert($reconcile->invoke( $lifecycle, (int) $undispatchedTraffic->id ) === true, 'A generation-zero traffic failure did not create a safe new generation.'); $undispatchedTrafficRetry = CloudJob::where( 'instance_id', $undispatchedTraffic->id )->where('kind', 'network_suspend')->orderByDesc('id')->first(); $assert($undispatchedTrafficRetry !== null && (int) $undispatchedTrafficRetry->id !== (int) $oldUndispatchedTraffic->id && (string) $undispatchedTrafficRetry->operation_id !== (string) $oldUndispatchedTraffic->operation_id && (int) $oldUndispatchedTraffic->fresh()->dispatch_attempts === 0, 'Generation-zero traffic retry replayed or rewrote the prior operation.'); $resumeRetryInstance = $makeLifecycleInstance( '92929292-9292-4292-8292-929292929292', 'resume-retry-vm', 'active', 'restore_failed' ); $resumeRetryInstance->expires_at = time() + 86400; $resumeRetryInstance->lifetime_download_bytes = 50; $resumeRetryInstance->lifetime_upload_bytes = 0; $resumeRetryInstance->traffic_synced_at = time(); $resumeRetryInstance->save(); $resumePeriodId = Capsule::table('cloud_service_periods')->insertGetId([ 'instance_id' => $resumeRetryInstance->id, 'user_id' => $resumeRetryInstance->user_id, 'starts_at' => time() - 3600, 'ends_at' => time() + 3600, 'traffic_quota_bytes' => 1000, 'download_base_bytes' => 0, 'upload_base_bytes' => 0, 'download_used_bytes' => 50, 'upload_used_bytes' => 0, 'status' => 'active', 'created_at' => time() - 3600, 'updated_at' => time() - 310, ]); $resumeRetryInstance->current_period_id = $resumePeriodId; $resumeRetryInstance->save(); $oldResumeJob = $preparePolicyFailure( $resumeRetryInstance, 'network_resume', 'network/resume', null ); $assert($reconcile->invoke( $lifecycle, (int) $resumeRetryInstance->id ) === true, 'A still-required network restore did not create a new generation.'); $resumeRetry = CloudJob::where('instance_id', $resumeRetryInstance->id) ->where('kind', 'network_resume')->orderByDesc('id')->first(); $assert($resumeRetry !== null && (int) $resumeRetry->id !== (int) $oldResumeJob->id && (string) $resumeRetry->operation_id !== (string) $oldResumeJob->operation_id && (string) $oldResumeJob->fresh()->phase === 'terminal' && (string) $resumeRetryInstance->fresh()->network_status === 'restoring', 'Network-restore retry mutated or replayed its prior generation.'); $attachActivePeriod = static function (CloudInstance $target): void { $target->expires_at = time() + 86400; $target->lifetime_download_bytes = 50; $target->lifetime_upload_bytes = 0; $target->traffic_synced_at = time(); $target->save(); $periodId = Capsule::table('cloud_service_periods')->insertGetId([ 'instance_id' => $target->id, 'user_id' => $target->user_id, 'starts_at' => time() - 3600, 'ends_at' => time() + 3600, 'traffic_quota_bytes' => 1000, 'download_base_bytes' => 0, 'upload_base_bytes' => 0, 'download_used_bytes' => 50, 'upload_used_bytes' => 0, 'status' => 'active', 'created_at' => time() - 3600, 'updated_at' => time() - 310, ]); $target->current_period_id = $periodId; $target->save(); }; $undispatchedResume = $makeLifecycleInstance( '98989898-9898-4898-8898-989898989898', 'undispatched-resume-vm', 'active', 'restore_failed' ); $attachActivePeriod($undispatchedResume); $oldUndispatchedResume = $finalizeUndispatchedFailure($queuePolicy( $undispatchedResume, 'network_resume', 'network/resume' )); $assert($reconcile->invoke( $lifecycle, (int) $undispatchedResume->id ) === true, 'A generation-zero network restore did not create a safe new generation.'); $undispatchedResumeRetry = CloudJob::where( 'instance_id', $undispatchedResume->id )->where('kind', 'network_resume')->orderByDesc('id')->first(); $assert($undispatchedResumeRetry !== null && (int) $undispatchedResumeRetry->id !== (int) $oldUndispatchedResume->id && (string) $undispatchedResumeRetry->operation_id !== (string) $oldUndispatchedResume->operation_id && (int) $oldUndispatchedResume->fresh()->dispatch_attempts === 0 && (string) $undispatchedResume->fresh()->network_status === 'restoring', 'Generation-zero network-restore retry replayed or rewrote the prior operation.'); $obsoleteSuspend = $makeLifecycleInstance( '94949494-9494-4494-8494-949494949494', 'obsolete-suspend-vm', 'suspend_failed', 'active' ); $attachActivePeriod($obsoleteSuspend); $oldObsoleteSuspend = $preparePolicyFailure( $obsoleteSuspend, 'service_suspend', 'service/suspend', 'purge' ); $obsoleteCount = CloudJob::where( 'instance_id', $obsoleteSuspend->id )->count(); $assert($reconcile->invoke( $lifecycle, (int) $obsoleteSuspend->id ) === false && (string) $obsoleteSuspend->fresh()->service_status === 'active' && CloudJob::where('instance_id', $obsoleteSuspend->id)->count() === $obsoleteCount && (string) $oldObsoleteSuspend->fresh()->phase === 'terminal', 'A no-longer-required, definitively not-applied suspend was not safely cleared.'); $uncertainSuspend = $makeLifecycleInstance( '95959595-9595-4595-8595-959595959595', 'uncertain-suspend-vm', 'suspend_failed', 'active' ); $attachActivePeriod($uncertainSuspend); $oldUncertainSuspend = $preparePolicyFailure( $uncertainSuspend, 'service_suspend', 'service/suspend', 'purge' ); $oldUncertainSuspend->effect_resolution = 'unknown'; $oldUncertainSuspend->effect_may_have_applied = 1; $oldUncertainSuspend->safe_to_resubmit = 0; $oldUncertainSuspend->save(); $assert($reconcile->invoke( $lifecycle, (int) $uncertainSuspend->id ) === false && (string) $uncertainSuspend->fresh()->service_status === 'suspend_failed' && (string) $oldUncertainSuspend->fresh()->effect_resolution === 'unknown', 'An unknown-effect suspend failure was falsely unlocked by a paid period.'); foreach ([ ['unknown', 1, 0, false], ['applied', 0, 0, false], ['not_applied', 0, 1, true], ] as $index => [$resolution, $effect, $safe, $incident]) { $guarded = $makeLifecycleInstance( sprintf('93939393-9393-4393-8393-%012d', $index + 1), 'guarded-lifecycle-' . $index, 'purge_deleting', 'blocked_expired' ); $guarded->status = 'deleting'; $guarded->save(); $guardedJob = CloudJobService::enqueueExpiryDelete($guarded); $identity = CloudOperationRecoveryIdentity::expected($guardedJob); $past = time() - 310; $guardedJob->request_fingerprint = $identity['request_fingerprint']; $guardedJob->attempts = 1; $guardedJob->dispatch_attempts = 1; $guardedJob->dispatch_started_at = $past - 30; $guardedJob->agent_accepted_at = $past - 20; $guardedJob->dispatched_at = $past - 20; $guardedJob->receipt_deadline_at = $past - 10; $guardedJob->phase = 'terminal'; $guardedJob->terminal_at = $past; $guardedJob->error_code = 'operation_failed'; $guardedJob->updated_at = $past; $guardedJob->effect_resolution = $resolution; $guardedJob->effect_may_have_applied = $effect; $guardedJob->safe_to_resubmit = $safe; $guardedJob->outcome = $resolution === 'applied' ? 'succeeded' : 'failed'; $guardedJob->status = $guardedJob->outcome; $guardedJob->finalization_incident = $incident ? 1 : 0; $guardedJob->finalization_error_code = $incident ? 'recovery_ledger_conflict' : null; $guardedJob->save(); $guarded->status = 'delete_failed'; $guarded->service_status = 'purge_failed'; $guarded->save(); $beforeGuarded = CloudJob::where('instance_id', $guarded->id)->count(); $assert($reconcile->invoke($lifecycle, (int) $guarded->id) === false && CloudJob::where('instance_id', $guarded->id)->count() === $beforeGuarded && (string) $guardedJob->fresh()->operation_id === (string) $guardedJob->operation_id, 'Unknown, applied, or sticky lifecycle evidence created a new mutation generation.'); } // A create which crossed the Agent boundary may time out locally before // the authenticated terminal failure reaches PHP. Exact Agent proof that // both the guest and every managed artifact are absent must finish the // original paid order locally; it must never dispatch a second create. foreach (['8.8.8.20', '8.8.8.21', '8.8.8.22'] as $address) { if (!CloudPublicIp::where('address', $address)->exists()) { $ip = new CloudPublicIp(); $ip->host_id = $host->id; $ip->public_ip_pool_id = $sourcePublicPool->id; $ip->address_family = 4; $ip->address = $address; $ip->address_binary = inet_pton($address); $ip->prefix_length = 24; $ip->state = 'available'; $ip->created_at = $ip->updated_at = time(); $ip->save(); } } $lateFailureOrder = CloudOrderService::purchase(array_replace($input, [ 'name' => 'vm-late-failure', 'idempotency_key' => 'test-request-key-late-failure', ]), $actor); $lateFailureInvoice = CloudInvoice::where( 'order_id', $lateFailureOrder->id )->firstOrFail(); CloudInvoiceService::payBalance( $lateFailureInvoice, $actor, 'provision-balance-late-failure' ); $lateFailureJob = CloudJob::where( 'order_id', $lateFailureOrder->id )->firstOrFail(); // Earlier lifecycle tests intentionally leave separately audited retries // queued. Park those unrelated rows so this tracer observes exactly one // create dispatch rather than depending on suite-wide queue order. Capsule::table('cloud_jobs') ->where('id', '<>', (int) $lateFailureJob->id) ->where('phase', '<>', 'terminal') ->update([ 'available_at' => time() + 3600, 'dispatch_deadline_at' => time() + 3600, 'receipt_deadline_at' => time() + 3600, ]); Capsule::table('cloud_jobs')->where( 'id', (int) $lateFailureJob->id )->update(['available_at' => 0]); $lateFailureJob->refresh(); $lateFailureIdentity = CloudOperationRecoveryIdentity::expected( $lateFailureJob, false ); $mock->append(new Response(202, [], json_encode([ 'operation_id' => $lateFailureJob->operation_id, 'status' => 'reserved', ...$lateFailureIdentity, 'replayed' => false, 'retryable' => false, ], JSON_THROW_ON_ERROR))); $beforeLateFailureDispatch = count($history); $assert($worker->tick(), 'Late-failure create did not cross the Agent dispatch boundary.'); $lateFailureJob->refresh(); $assert((string) $lateFailureJob->phase === 'awaiting_receipt' && (int) $lateFailureJob->dispatch_attempts === 1 && count($history) === $beforeLateFailureDispatch + 1, 'Late-failure create did not persist its single dispatch generation.'); Capsule::table('cloud_jobs')->where( 'id', (int) $lateFailureJob->id )->update([ 'receipt_deadline_at' => time() - 1, 'lease_token' => null, 'lease_until' => 0, ]); $assert(CloudWorkerService::expireDueTaskForAdmin( (int) $lateFailureJob->id ) === 1, 'Late-failure create did not reach its local deadline.'); $lateFailureJob->refresh(); $lateFailureOrder->refresh(); $lateFailureInvoice->refresh(); $lateFailureReservation = CloudResourceReservation::where( 'order_id', $lateFailureOrder->id )->firstOrFail(); $walletWhileFenced = (string) User::findOrFail(1)->money; $stockWhileFenced = (int) CloudOffer::findOrFail(1)->stock; $assert((string) $lateFailureJob->phase === 'terminal' && (string) $lateFailureJob->outcome === 'failed' && (string) $lateFailureJob->effect_resolution === 'unknown' && (string) $lateFailureOrder->status === 'failed' && $lateFailureOrder->refunded_at === null && (string) $lateFailureInvoice->status === 'paid' && (string) $lateFailureReservation->state === 'uncertain' && $bundleState( $lateFailureReservation, 'uncertain', 'uncertain' ), 'Timed-out dispatched create did not retain its financial fence.'); $lateFailureEventId = 'create-late-failure-event-0001'; $lateFailureReceipt = [ 'type' => 'completion', 'id' => $lateFailureEventId, 'agent_id' => (string) $lateFailureJob->host_id, 'operation_id' => (string) $lateFailureJob->operation_id, ...$lateFailureIdentity, 'status' => 'failed', 'effect_may_have_applied' => false, 'safe_to_resubmit' => false, 'error_code' => 'creation_absent', 'result' => [ 'uuid' => $lateFailureIdentity['resource_id'], 'creation_absent' => true, 'managed_artifacts_absent' => true, ], ]; $lateFailureAccepted = CloudOperationCompletionService::accept( $lateFailureReceipt ); $lateFailureJob->refresh(); $lateFailureOrder->refresh(); $lateFailureInvoice->refresh(); $lateFailureReservation->refresh(); $lateFailureEvent = \App\Models\CloudAgentCompletionEvent::where( 'event_id', $lateFailureEventId )->firstOrFail(); $assert($lateFailureAccepted === [ 'accepted' => true, 'disposition' => 'applied', ] && (string) $lateFailureEvent->disposition === 'applied' && $lateFailureEvent->applied_at !== null && (string) $lateFailureJob->effect_resolution === 'not_applied' && !(bool) $lateFailureJob->finalization_incident && (string) $lateFailureOrder->status === 'failed' && $lateFailureOrder->refunded_at !== null && (string) $lateFailureInvoice->status === 'refunded' && (string) $lateFailureReservation->state === 'released' && $bundleState( $lateFailureReservation, 'released', 'available' ) && (string) User::findOrFail(1)->money === CloudMoney::decimal( CloudMoney::cents($walletWhileFenced) + 999 ) && (int) CloudOffer::findOrFail(1)->stock === $stockWhileFenced + 1 && count($history) === $beforeLateFailureDispatch + 1, 'Exact late create failure did not atomically refund, release, ACK, ' . 'or replayed the original mutation.'); $lateFailureWalletAfter = (string) User::findOrFail(1)->money; $lateFailureStockAfter = (int) CloudOffer::findOrFail(1)->stock; $lateFailureRefundedAt = (int) $lateFailureOrder->refunded_at; $lateFailureJobUpdatedAt = (int) $lateFailureJob->updated_at; $lateFailureAppliedAt = (int) $lateFailureEvent->applied_at; $lateFailureDuplicate = CloudOperationCompletionService::accept( $lateFailureReceipt ); $assert($lateFailureDuplicate === [ 'accepted' => true, 'disposition' => 'applied', ] && (string) User::findOrFail(1)->money === $lateFailureWalletAfter && (int) CloudOffer::findOrFail(1)->stock === $lateFailureStockAfter && (int) CloudOrder::findOrFail( (int) $lateFailureOrder->id )->refunded_at === $lateFailureRefundedAt && (int) CloudJob::findOrFail( (int) $lateFailureJob->id )->updated_at === $lateFailureJobUpdatedAt && (int) \App\Models\CloudAgentCompletionEvent::findOrFail( $lateFailureEventId )->applied_at === $lateFailureAppliedAt && count($history) === $beforeLateFailureDispatch + 1, 'Duplicate late create failure refunded, released, ACKed, or dispatched ' . 'more than once.'); /** * Create one paid order, cross the Agent boundary exactly once, and force * the panel-side receipt deadline to the terminal unknown-effect state. * Later recovery assertions can then focus on finance and evidence rather * than reproducing dispatch bookkeeping by hand. * * @return array */ $makeTimedOutCreate = static function ( string $suffix ) use ( $input, $actor, $mock, $worker, &$history, $assert, $bundleState ): array { $order = CloudOrderService::purchase(array_replace($input, [ 'name' => 'vm-create-recovery-' . $suffix, 'idempotency_key' => 'test-create-recovery-' . $suffix, ]), $actor); $invoice = CloudInvoice::where('order_id', $order->id) ->firstOrFail(); CloudInvoiceService::payBalance( $invoice, $actor, 'provision-create-recovery-' . $suffix ); $job = CloudJob::where('order_id', $order->id)->firstOrFail(); $future = time() + 3600; Capsule::table('cloud_jobs') ->where('id', '<>', (int) $job->id) ->update([ 'available_at' => $future, 'lease_until' => $future, ]); Capsule::table('cloud_jobs')->where('id', (int) $job->id) ->update(['available_at' => 0, 'lease_until' => 0]); $job->refresh(); $identity = CloudOperationRecoveryIdentity::expected($job, false); $mock->append(new Response(202, [], json_encode([ 'operation_id' => $job->operation_id, 'status' => 'reserved', ...$identity, 'replayed' => false, 'retryable' => false, ], JSON_THROW_ON_ERROR))); $historyBefore = count($history); $assert($worker->tick(), 'Create recovery fixture did not dispatch: ' . $suffix); $job->refresh(); $assert((string) $job->phase === 'awaiting_receipt' && (int) $job->dispatch_attempts === 1 && count($history) === $historyBefore + 1, 'Create recovery fixture did not persist one dispatch: ' . $suffix); Capsule::table('cloud_jobs')->where('id', (int) $job->id) ->update([ 'receipt_deadline_at' => time() - 1, 'lease_token' => null, 'lease_until' => 0, ]); $assert(CloudWorkerService::expireDueTaskForAdmin((int) $job->id) === 1, 'Create recovery fixture did not expire: ' . $suffix); $job->refresh(); $order->refresh(); $invoice->refresh(); $reservation = CloudResourceReservation::where( 'order_id', $order->id )->firstOrFail(); $assert((string) $job->phase === 'terminal' && (string) $job->outcome === 'failed' && (string) $job->effect_resolution === 'unknown' && (string) $order->status === 'failed' && $order->refunded_at === null && (string) $invoice->status === 'paid' && (string) $reservation->state === 'uncertain' && $bundleState($reservation, 'uncertain', 'uncertain'), 'Create recovery fixture did not retain its safety fence: ' . $suffix); return compact( 'order', 'invoice', 'job', 'identity', 'reservation', 'historyBefore' ) + [ 'walletBefore' => (string) User::findOrFail(1)->money, 'stockBefore' => (int) CloudOffer::findOrFail(1)->stock, ]; }; $makeRecoveryReady = static function (CloudJob $job): void { $future = time() + 3600; Capsule::table('cloud_jobs') ->where('id', '<>', (int) $job->id) ->update([ 'available_at' => $future, 'lease_until' => $future, ]); Capsule::table('cloud_jobs')->where('id', (int) $job->id) ->update([ 'available_at' => 0, 'lease_token' => null, 'lease_until' => 0, ]); $job->refresh(); }; $inspectionFor = static function ( CloudJob $job, array $identity, bool $fenced = false ): array { return [ 'operation_id' => (string) $job->operation_id, ...$identity, 'operation_status' => 'failed', 'checked_at' => time(), 'outcome' => 'failed', 'summary_code' => 'operation_failed', 'error_code' => 'creation_absent', 'effect_resolution' => 'not_applied', 'fence_state' => $fenced ? 'active' : 'not_required', 'fence_resolution_required' => $fenced, 'effect_projection' => [ 'creation_absent' => true, 'managed_artifacts_absent' => true, ], 'abort_job' => ['available' => false], 'cleanup' => ['available' => false], ]; }; $assertCreateSettled = static function ( array $case, string $message ) use ($assert, $bundleState, &$history): void { $case['job']->refresh(); $case['order']->refresh(); $case['invoice']->refresh(); $case['reservation']->refresh(); $assert((string) $case['job']->effect_resolution === 'not_applied' && !(bool) $case['job']->finalization_incident && $case['job']->lease_token === null && (string) $case['order']->status === 'failed' && $case['order']->refunded_at !== null && (string) $case['invoice']->status === 'refunded' && (string) $case['reservation']->state === 'released' && $bundleState( $case['reservation'], 'released', 'available' ) && (string) User::findOrFail(1)->money === CloudMoney::decimal( CloudMoney::cents($case['walletBefore']) + 999 ) && (int) CloudOffer::findOrFail(1)->stock === $case['stockBefore'] + 1 && count($history) === $case['historyBefore'] + 1, $message); }; $failedCreateReceipt = static function ( array $case, string $eventId ): array { return [ 'type' => 'completion', 'id' => $eventId, 'agent_id' => (string) $case['job']->host_id, 'operation_id' => (string) $case['job']->operation_id, ...$case['identity'], 'status' => 'failed', 'effect_may_have_applied' => false, 'safe_to_resubmit' => false, 'error_code' => 'creation_absent', 'result' => [ 'uuid' => $case['identity']['resource_id'], 'creation_absent' => true, 'managed_artifacts_absent' => true, ], ]; }; // Immutable host absence evidence must wait for a temporarily inconsistent // local invoice. Keeping the event received allows the exact same receipt // to finish atomically after the PHP ledger is repaired. $invoiceGuardCase = $makeTimedOutCreate('receipt-invoice-guard'); $invoiceGuardCase['invoice']->purpose = 'renewal'; $invoiceGuardCase['invoice']->save(); $invoiceGuardReceipt = $failedCreateReceipt( $invoiceGuardCase, 'create-invoice-guard-failure-event-0001' ); $invoiceGuardPending = CloudOperationCompletionService::accept( $invoiceGuardReceipt ); $invoiceGuardCase['job']->refresh(); $invoiceGuardCase['order']->refresh(); $invoiceGuardCase['reservation']->refresh(); $invoiceGuardEvent = \App\Models\CloudAgentCompletionEvent::findOrFail( $invoiceGuardReceipt['id'] ); $assert($invoiceGuardPending === [ 'accepted' => true, 'disposition' => 'received', ] && (string) $invoiceGuardEvent->disposition === 'received' && $invoiceGuardEvent->applied_at === null && (string) $invoiceGuardCase['job']->effect_resolution === 'unknown' && $invoiceGuardCase['order']->refunded_at === null && (string) $invoiceGuardCase['reservation']->state === 'uncertain' && $bundleState( $invoiceGuardCase['reservation'], 'uncertain', 'uncertain' ) && (string) User::findOrFail(1)->money === $invoiceGuardCase['walletBefore'] && (int) CloudOffer::findOrFail(1)->stock === $invoiceGuardCase['stockBefore'], 'Exact create failure bypassed an inconsistent local invoice: ' . json_encode([ 'accepted' => $invoiceGuardPending, 'event_disposition' => $invoiceGuardEvent->disposition, 'event_applied_at' => $invoiceGuardEvent->applied_at, 'job_resolution' => $invoiceGuardCase['job']->effect_resolution, 'refunded_at' => $invoiceGuardCase['order']->refunded_at, 'reservation' => $invoiceGuardCase['reservation']->state, 'bundle_uncertain' => $bundleState( $invoiceGuardCase['reservation'], 'uncertain', 'uncertain' ), 'wallet_before' => $invoiceGuardCase['walletBefore'], 'wallet_after' => User::findOrFail(1)->money, 'stock_before' => $invoiceGuardCase['stockBefore'], 'stock_after' => CloudOffer::findOrFail(1)->stock, ], JSON_THROW_ON_ERROR)); $invoiceGuardCase['invoice']->purpose = 'provision'; $invoiceGuardCase['invoice']->save(); $assert(CloudOperationCompletionService::accept($invoiceGuardReceipt) === ['accepted' => true, 'disposition' => 'applied'], 'Repaired local invoice did not resume the same create receipt.'); $assertCreateSettled( $invoiceGuardCase, 'Repaired local invoice did not atomically settle create finance.' ); // The specialised create finalizer must validate the complete dispatched // envelope itself. Generic recovery policy deliberately does not own // create, so neither a damaged attempt count nor a missing receipt // deadline may reach the refund path. foreach ([ 'attempts' => 2, 'receipt_deadline_at' => null, ] as $damagedField => $damagedValue) { $suffix = 'receipt-envelope-' . str_replace('_', '-', $damagedField); $envelopeCase = $makeTimedOutCreate($suffix); $eventId = 'create-envelope-' . str_replace('_', '-', $damagedField) . '-event-0001'; $envelopeReceipt = $failedCreateReceipt($envelopeCase, $eventId); $originalValue = $envelopeCase['job']->{$damagedField}; Capsule::table('cloud_jobs')->where( 'id', (int) $envelopeCase['job']->id )->update([$damagedField => $damagedValue]); $envelopePending = CloudOperationCompletionService::accept( $envelopeReceipt ); $envelopeCase['job']->refresh(); $envelopeCase['order']->refresh(); $envelopeCase['invoice']->refresh(); $envelopeCase['reservation']->refresh(); $envelopeEvent = \App\Models\CloudAgentCompletionEvent::findOrFail( $eventId ); $assert($envelopePending === [ 'accepted' => true, 'disposition' => 'received', ] && (string) $envelopeEvent->disposition === 'received' && $envelopeEvent->applied_at === null && (string) $envelopeCase['job']->effect_resolution === 'unknown' && $envelopeCase['order']->refunded_at === null && (string) $envelopeCase['invoice']->status === 'paid' && (string) $envelopeCase['reservation']->state === 'uncertain' && $bundleState( $envelopeCase['reservation'], 'uncertain', 'uncertain' ) && (string) User::findOrFail(1)->money === $envelopeCase['walletBefore'] && (int) CloudOffer::findOrFail(1)->stock === $envelopeCase['stockBefore'], 'Damaged create dispatch envelope released finance: ' . $damagedField); Capsule::table('cloud_jobs')->where( 'id', (int) $envelopeCase['job']->id )->update([$damagedField => $originalValue]); $assert(CloudOperationCompletionService::accept($envelopeReceipt) === ['accepted' => true, 'disposition' => 'applied'], 'Repaired create dispatch envelope did not resume receipt: ' . $damagedField); $assertCreateSettled( $envelopeCase, 'Repaired dispatch envelope did not settle create finance: ' . $damagedField ); } // A fresh read-only Journal failure with complete absence projection can // finish a timed-out create even when no completion event ever arrived. $inspectionCase = $makeTimedOutCreate('inspection-no-receipt'); $makeRecoveryReady($inspectionCase['job']); $inspectionClient = new FinancialCreateRecoveryAgentClient(); $inspectionClient->responses[] = $inspectionFor( $inspectionCase['job'], $inspectionCase['identity'] ); CloudAgentSettingsService::$hostId = (int) $inspectionCase['job']->host_id; CloudAgentSettingsService::$client = $inspectionClient; $inspectionRecovery = new CloudAutomaticOperationRecoveryService(); $assert($inspectionRecovery->tick(), 'No-receipt create was not claimed for read-only recovery.'); $assert(count($inspectionClient->requests) === 1 && $inspectionClient->requests[0]['method'] === 'GET' && str_ends_with( $inspectionClient->requests[0]['target'], '/inspect' ) && $inspectionClient->requests[0]['payload'] === null, 'No-receipt create recovery used something other than one read-only ' . 'inspection.'); $assertCreateSettled( $inspectionCase, 'Read-only create absence evidence did not atomically settle finance.' ); // Once inspection has settled the ledger, the first later failure receipt // is an idempotent ACK of the late delivery. It must not re-enter the received queue, // rewrite the job incident, or credit/release anything twice. $inspectionCase['job']->refresh(); $inspectionJobUpdatedAt = (int) $inspectionCase['job']->updated_at; $inspectionWalletAfter = (string) User::findOrFail(1)->money; $inspectionStockAfter = (int) CloudOffer::findOrFail(1)->stock; $inspectionLateReceipt = [ 'type' => 'completion', 'id' => 'create-inspection-late-failure-event-0001', 'agent_id' => (string) $inspectionCase['job']->host_id, 'operation_id' => (string) $inspectionCase['job']->operation_id, ...$inspectionCase['identity'], 'status' => 'failed', 'effect_may_have_applied' => false, 'safe_to_resubmit' => false, 'error_code' => 'creation_absent', 'result' => [ 'uuid' => $inspectionCase['identity']['resource_id'], 'creation_absent' => true, 'managed_artifacts_absent' => true, ], ]; $inspectionLateAccepted = CloudOperationCompletionService::accept( $inspectionLateReceipt ); $inspectionLateDuplicate = CloudOperationCompletionService::accept( $inspectionLateReceipt ); $inspectionCase['job']->refresh(); $inspectionLateEvent = \App\Models\CloudAgentCompletionEvent::findOrFail( $inspectionLateReceipt['id'] ); $assert($inspectionLateAccepted === [ 'accepted' => true, 'disposition' => 'applied', ] && $inspectionLateDuplicate === $inspectionLateAccepted && (string) $inspectionLateEvent->disposition === 'applied' && $inspectionLateEvent->applied_at !== null && (int) $inspectionCase['job']->updated_at === $inspectionJobUpdatedAt && !(bool) $inspectionCase['job']->finalization_incident && (string) User::findOrFail(1)->money === $inspectionWalletAfter && (int) CloudOffer::findOrFail(1)->stock === $inspectionStockAfter, 'A failure receipt arriving after inspection settlement was requeued ' . 'or changed the financial ledger: ' . json_encode([ 'first' => $inspectionLateAccepted, 'duplicate' => $inspectionLateDuplicate, 'event_disposition' => $inspectionLateEvent->disposition, 'event_applied_at' => $inspectionLateEvent->applied_at, 'job_updated_at_before' => $inspectionJobUpdatedAt, 'job_updated_at_after' => $inspectionCase['job']->updated_at, 'job_incident' => $inspectionCase['job']->finalization_incident, 'wallet_before' => $inspectionWalletAfter, 'wallet_after' => User::findOrFail(1)->money, 'stock_before' => $inspectionStockAfter, 'stock_after' => CloudOffer::findOrFail(1)->stock, ], JSON_THROW_ON_ERROR)); // An incomplete failed receipt is retained as a late audit event, then a // fresh read-only inspection supplies the missing managed-artifact proof. $incompleteCase = $makeTimedOutCreate('incomplete-receipt'); $incompleteReceipt = [ 'type' => 'completion', 'id' => 'create-incomplete-failure-event-0001', 'agent_id' => (string) $incompleteCase['job']->host_id, 'operation_id' => (string) $incompleteCase['job']->operation_id, ...$incompleteCase['identity'], 'status' => 'failed', 'effect_may_have_applied' => false, 'safe_to_resubmit' => false, 'error_code' => 'creation_absent', 'result' => [ 'uuid' => $incompleteCase['identity']['resource_id'], 'creation_absent' => true, ], ]; $assert(CloudOperationCompletionService::accept($incompleteReceipt) === ['accepted' => true, 'disposition' => 'late'], 'Incomplete create absence receipt was not ACKed as late.'); $makeRecoveryReady($incompleteCase['job']); $incompleteClient = new FinancialCreateRecoveryAgentClient(); $incompleteClient->responses[] = $inspectionFor( $incompleteCase['job'], $incompleteCase['identity'] ); CloudAgentSettingsService::$client = $incompleteClient; $assert((new CloudAutomaticOperationRecoveryService())->tick(), 'Incomplete receipt create did not enter fresh inspection recovery.'); $assertCreateSettled( $incompleteCase, 'Fresh inspection did not finish an incomplete failed receipt create.' ); // If PHP fails after durably preparing a no-fence conclusion, a later // tick must finish locally even while the Agent is offline. $noFenceRetry = $makeTimedOutCreate('nofence-db-retry'); $noFenceRetry['invoice']->purpose = 'renewal'; $noFenceRetry['invoice']->save(); $makeRecoveryReady($noFenceRetry['job']); $noFenceClient = new FinancialCreateRecoveryAgentClient(); $noFenceClient->responses[] = $inspectionFor( $noFenceRetry['job'], $noFenceRetry['identity'] ); CloudAgentSettingsService::$client = $noFenceClient; $assert((new CloudAutomaticOperationRecoveryService())->tick(), 'No-fence DB-failure create was not claimed.'); $noFenceRetry['job']->refresh(); $noFenceRetry['order']->refresh(); $noFenceRetry['invoice']->refresh(); $assert(preg_match( '/^prepared:create_not_applied:v1:[a-f0-9]{64}$/D', (string) $noFenceRetry['job']->effect_resolution_reason ) === 1 && (string) $noFenceRetry['job']->effect_resolution === 'unknown' && $noFenceRetry['order']->refunded_at === null && (string) $noFenceRetry['invoice']->status === 'paid' && count($noFenceClient->requests) === 1, 'No-fence financial failure did not retain its durable prepare.'); $noFenceRetry['invoice']->purpose = 'provision'; $noFenceRetry['invoice']->save(); $makeRecoveryReady($noFenceRetry['job']); // Deliberately leave the response queue empty: any Agent call now fails. $assert((new CloudAutomaticOperationRecoveryService())->tick(), 'Prepared no-fence create was not retried locally.'); $assert(count($noFenceClient->requests) === 1, 'Prepared no-fence retry contacted an offline Agent.'); $assertCreateSettled( $noFenceRetry, 'Prepared no-fence create did not settle after the DB fault cleared.' ); // If the Agent fence write committed but the same PHP transaction later // failed, recovery reads that immutable verdict once and never POSTs it a // second time. $fenceRetry = $makeTimedOutCreate('fence-db-retry'); $fenceRetry['invoice']->purpose = 'renewal'; $fenceRetry['invoice']->save(); $makeRecoveryReady($fenceRetry['job']); $fenceClient = new FinancialCreateRecoveryAgentClient(); $fenceClient->responses[] = $inspectionFor( $fenceRetry['job'], $fenceRetry['identity'], true ); $fenceClient->responses[] = new CloudAgentError( 'no committed fence', 404, 'not_found', false, 404 ); $committedFence = null; $fenceClient->responses[] = static function ( array $request ) use (&$committedFence, $fenceRetry): array { $payload = $request['payload']; $committedFence = [ 'operation_id' => (string) $fenceRetry['job']->operation_id, 'resolution_id' => (string) $payload['resolution_id'], 'resolution' => (string) $payload['resolution'], 'inspection_checked_at' => (int) $payload['inspection_checked_at'], 'reason_sha256' => (string) $payload['reason_sha256'], 'resolved_at' => time(), 'resolved_by' => 0, 'replayed' => false, ]; return $committedFence; }; CloudAgentSettingsService::$client = $fenceClient; $assert((new CloudAutomaticOperationRecoveryService())->tick(), 'Active-fence DB-failure create was not claimed.'); $fenceRetry['job']->refresh(); $assert(is_array($committedFence) && count($fenceClient->requests) === 3 && $fenceClient->requests[0]['method'] === 'GET' && str_ends_with($fenceClient->requests[0]['target'], '/inspect') && $fenceClient->requests[1]['method'] === 'GET' && str_ends_with( $fenceClient->requests[1]['target'], '/fence-resolution' ) && $fenceClient->requests[2]['method'] === 'POST' && str_ends_with( $fenceClient->requests[2]['target'], '/resolve-fence' ) && preg_match( '/^prepared:create_not_applied:v1:[a-f0-9]{64}$/D', (string) $fenceRetry['job']->effect_resolution_reason ) === 1 && (string) $fenceRetry['job']->effect_resolution === 'unknown', 'Fence commit plus PHP failure did not preserve resumable evidence.'); $fenceRetry['invoice']->purpose = 'provision'; $fenceRetry['invoice']->save(); $makeRecoveryReady($fenceRetry['job']); $fenceClient->responses[] = $committedFence; $assert((new CloudAutomaticOperationRecoveryService())->tick(), 'Prepared active-fence create was not retried.'); $assert(count($fenceClient->requests) === 4 && $fenceClient->requests[3]['method'] === 'GET' && str_ends_with( $fenceClient->requests[3]['target'], '/fence-resolution' ), 'Prepared active-fence retry wrote or re-inspected Agent state.'); $assertCreateSettled( $fenceRetry, 'Prepared active-fence create did not settle after DB recovery.' ); // A fence write can fail before the Agent commits it. The next tick first // proves that no prior verdict exists, then obtains a fresh inspection. // checked_at is freshness metadata, not part of the durable conclusion: // the same facts must produce the same marker and fence reason. $stableCase = $makeTimedOutCreate('stable-inspection-marker'); $makeRecoveryReady($stableCase['job']); $stableClient = new FinancialCreateRecoveryAgentClient(); $stableFirstInspection = $inspectionFor( $stableCase['job'], $stableCase['identity'], true ); $stableClient->responses = [ $stableFirstInspection, new CloudAgentError('no committed fence', 404, 'not_found', false, 404), new CloudAgentError('fence write unavailable', 503, 'unavailable', true, 503), new CloudAgentError('no committed fence', 404, 'not_found', false, 404), ]; CloudAgentSettingsService::$client = $stableClient; $assert((new CloudAutomaticOperationRecoveryService())->tick(), 'Stable-marker first recovery attempt was not claimed.'); $stableCase['job']->refresh(); $stableMarker = (string) $stableCase['job']->effect_resolution_reason; $assert(preg_match( '/^prepared:create_not_applied:v1:[a-f0-9]{64}$/D', $stableMarker ) === 1 && (string) $stableCase['job']->effect_resolution === 'unknown' && count($stableClient->requests) === 4, 'Failed fence write did not retain a stable prepared create marker.'); $makeRecoveryReady($stableCase['job']); $stableSecondInspection = $stableFirstInspection; $stableSecondInspection['checked_at'] = (int) $stableFirstInspection['checked_at'] + 1; $stableCommittedFence = null; $stableClient->responses = [ new CloudAgentError('no committed fence', 404, 'not_found', false, 404), $stableSecondInspection, new CloudAgentError('no committed fence', 404, 'not_found', false, 404), static function (array $request) use ( &$stableCommittedFence, $stableCase ): array { $payload = $request['payload']; $stableCommittedFence = [ 'operation_id' => (string) $stableCase['job']->operation_id, 'resolution_id' => (string) $payload['resolution_id'], 'resolution' => (string) $payload['resolution'], 'inspection_checked_at' => (int) $payload['inspection_checked_at'], 'reason_sha256' => (string) $payload['reason_sha256'], 'resolved_at' => time(), 'resolved_by' => 0, 'replayed' => false, ]; return $stableCommittedFence; }, ]; $assert((new CloudAutomaticOperationRecoveryService())->tick(), 'Stable-marker create did not resume from fresh inspection.'); $stableExpectedReason = '自动恢复已确认原开通操作未创建虚拟机或面板管理资源;operation_id=' . (string) $stableCase['job']->operation_id . ';evidence=' . substr($stableMarker, -64); $assert(is_array($stableCommittedFence) && count($stableClient->requests) === 8 && $stableClient->requests[4]['method'] === 'GET' && str_ends_with( $stableClient->requests[4]['target'], '/fence-resolution' ) && $stableClient->requests[5]['method'] === 'GET' && str_ends_with($stableClient->requests[5]['target'], '/inspect') && $stableClient->requests[7]['method'] === 'POST' && (string) $stableClient->requests[7]['payload']['reason_sha256'] === hash('sha256', $stableExpectedReason), 'Fresh equivalent inspection changed the prepared fence reason or ' . 'replayed the original create.'); $assertCreateSettled( $stableCase, 'Stable-marker retry did not finish the create financial ledger.' ); // Partial/residual host state and an identity mismatch are both // contradictions, not refund evidence. They remain fenced and retryable. $conflictCase = $makeTimedOutCreate('inspection-conflicts'); $makeRecoveryReady($conflictCase['job']); $conflictClient = new FinancialCreateRecoveryAgentClient(); $partialInspection = $inspectionFor( $conflictCase['job'], $conflictCase['identity'], true ); $partialInspection['outcome'] = 'uncertain'; $partialInspection['summary_code'] = 'evidence_incomplete'; $partialInspection['effect_resolution'] = 'partial'; $partialInspection['error_code'] = 'cleanup_required'; $partialInspection['cleanup'] = [ 'available' => true, 'plan_id' => str_repeat('a', 64), 'steps' => ['delete_managed_volumes'], ]; unset($partialInspection['effect_projection']); $conflictClient->responses[] = $partialInspection; CloudAgentSettingsService::$client = $conflictClient; $assert((new CloudAutomaticOperationRecoveryService())->tick(), 'Partial create evidence was not safely deferred.'); $conflictCase['job']->refresh(); $conflictCase['order']->refresh(); $conflictCase['invoice']->refresh(); $assert((string) $conflictCase['job']->effect_resolution === 'unknown' && (bool) $conflictCase['job']->finalization_incident && $conflictCase['order']->refunded_at === null && (string) $conflictCase['invoice']->status === 'paid' && (string) User::findOrFail(1)->money === $conflictCase['walletBefore'] && (int) CloudOffer::findOrFail(1)->stock === $conflictCase['stockBefore'], 'Partial create evidence released a financial/resource fence.'); $makeRecoveryReady($conflictCase['job']); $wrongIdentity = $inspectionFor( $conflictCase['job'], $conflictCase['identity'] ); $wrongIdentity['resource_id'] = 'wrong-create-resource'; $conflictClient->responses[] = $wrongIdentity; $assert((new CloudAutomaticOperationRecoveryService())->tick(), 'Create identity contradiction was not safely deferred.'); $conflictCase['job']->refresh(); $assert((string) $conflictCase['job']->effect_resolution === 'unknown' && (bool) $conflictCase['job']->finalization_incident && $conflictCase['order']->fresh()->refunded_at === null && (string) User::findOrFail(1)->money === $conflictCase['walletBefore'] && (int) CloudOffer::findOrFail(1)->stock === $conflictCase['stockBefore'], 'Create identity contradiction released a financial/resource fence.'); // A caller cannot reuse an automatic claim after its lease has expired. $leaseToken = bin2hex(random_bytes(16)); Capsule::table('cloud_jobs')->where('id', (int) $conflictCase['job']->id) ->update([ 'available_at' => 0, 'lease_token' => $leaseToken, 'lease_until' => time() + 180, ]); $staleClaim = CloudJob::findOrFail((int) $conflictCase['job']->id); Capsule::table('cloud_jobs')->where('id', (int) $conflictCase['job']->id) ->update(['lease_until' => time() - 1]); $staleRequestCount = count($conflictClient->requests); $staleRejected = false; try { CloudCreateFailureRecoveryService::recover($staleClaim); } catch (CloudError $error) { $staleRejected = true; } $assert($staleRejected && count($conflictClient->requests) === $staleRequestCount && (string) CloudJob::findOrFail( (int) $conflictCase['job']->id )->effect_resolution === 'unknown' && $conflictCase['order']->fresh()->refunded_at === null, 'Expired create recovery lease contacted Agent or released finance.'); // Real address exhaustion on the first scored host is a scheduler miss, // not a corrupt-order failure. The failed host must leave neither its // temporary reservation nor a partial address claim before placement // continues with the next host. $attachment->state = 'active'; $attachment->verification_state = 'verified'; $attachment->verified_agent_revision = 1; $attachment->last_verified_at = time(); $attachment->save(); $sourcePublicPool->state = 'active'; $sourcePublicPool->sync_state = 'applied'; $sourcePublicPool->desired_revision = 1; $sourcePublicPool->observed_revision = 1; $sourcePublicPool->save(); $sourceIpv6Pool->state = 'active'; $sourceIpv6Pool->sync_state = 'applied'; $sourceIpv6Pool->desired_revision = 1; $sourceIpv6Pool->observed_revision = 1; $sourceIpv6Pool->allocation_start = '2404:c140:156:3a::10'; $sourceIpv6Pool->allocation_end = '2404:c140:156:3a::10'; $sourceIpv6Pool->allocation_cursor = null; $sourceIpv6Pool->max_allocations = 1; $sourceIpv6Pool->save(); Capsule::table('cloud_public_ip_pools') ->where('id', (int) $sourceIpv6Pool->id) ->update(['allocation_cursor' => null]); CloudPublicIp::where('public_ip_pool_id', $sourceIpv6Pool->id)->update([ 'state' => 'disabled', 'reservation_id' => null, 'reservation_ip_claim_id' => null, 'instance_id' => null, 'operation_id' => null, 'updated_at' => time(), ]); // Earlier scalar network-change scenarios deliberately reuse and then // disable concrete IPv4 rows. New provisioning now validates the whole // pool ledger, so return those test-only rows to the real disabled shape // before exercising host-capacity fallback. CloudPublicIp::where('state', 'disabled')->update([ 'reservation_id' => null, 'reservation_ip_claim_id' => null, 'instance_id' => null, 'operation_id' => null, 'updated_at' => time(), ]); if (!CloudPublicIp::where('address_family', 6) ->where('address_binary', inet_pton('2404:c140:156:3a::10')) ->exists()) { $blockedIpv6 = new CloudPublicIp(); $blockedIpv6->host_id = (int) $host->id; $blockedIpv6->public_ip_pool_id = (int) $sourceIpv6Pool->id; $blockedIpv6->address_family = 6; $blockedIpv6->address = '2404:c140:156:3a::10'; $blockedIpv6->address_binary = inet_pton( '2404:c140:156:3a::10' ); $blockedIpv6->prefix_length = 64; $blockedIpv6->state = 'disabled'; $blockedIpv6->created_at = $blockedIpv6->updated_at = time(); $blockedIpv6->save(); } if (!CloudPublicIp::where('address_family', 4) ->where('address_binary', inet_pton('8.8.8.23'))->exists()) { $fallbackProbeIpv4 = new CloudPublicIp(); $fallbackProbeIpv4->host_id = (int) $host->id; $fallbackProbeIpv4->public_ip_pool_id = (int) $sourcePublicPool->id; $fallbackProbeIpv4->address_family = 4; $fallbackProbeIpv4->address = '8.8.8.23'; $fallbackProbeIpv4->address_binary = inet_pton('8.8.8.23'); $fallbackProbeIpv4->prefix_length = 24; $fallbackProbeIpv4->state = 'available'; $fallbackProbeIpv4->created_at = $fallbackProbeIpv4->updated_at = time(); $fallbackProbeIpv4->save(); } $targetAttachment->state = 'draining'; $targetAttachment->save(); $firstHostPolicy = CloudHostCapacityPolicy::findOrFail((int) $host->id); $firstHostPolicy->configured_cpu_cores = 1000000; $firstHostPolicy->configured_memory_mib = 1000000000; $firstHostPolicy->configured_disk_gib = 1000000000; $firstHostPolicy->safety_cpu_cores = 0; $firstHostPolicy->safety_memory_mib = 0; $firstHostPolicy->safety_disk_gib = 0; $firstHostPolicy->max_instances = 1000000; $firstHostPolicy->save(); $fallbackHost = new CloudHost(); $fallbackHost->name = 'fallback-host'; $fallbackHost->enabled = true; $fallbackHost->resource_pool_id = $pool->id; $fallbackHost->scheduler_state = 'active'; $fallbackHost->default_network = 'bridge:br0'; $fallbackHost->created_at = $fallbackHost->updated_at = time(); $fallbackHost->save(); Capsule::table('cloud_host_agents')->insert([ 'host_id' => $fallbackHost->id, 'sealed_api_secret' => 'fallback-test-only-agent-secret', 'enabled' => 1, 'revision' => 1, 'tested_revision' => 1, 'last_test_status' => 'healthy', 'last_error_code' => '', 'last_agent_version' => 'test', 'network_selectors_json' => '["bridge:br0"]', 'public_network_capabilities_json' => json_encode([ 'contract' => 1, 'dual_stack_required' => true, 'project_owned_filter' => true, 'ndp_guard' => true, ], JSON_THROW_ON_ERROR), 'last_tested_at' => time(), 'created_at' => time(), 'updated_at' => time(), ]); Capsule::table('cloud_host_storage_configurations')->insert([ 'host_id' => $fallbackHost->id, 'revision' => 1, 'state' => 'applied', ]); $fallbackMembership = new CloudHostResourcePoolMembership(); $fallbackMembership->host_id = $fallbackHost->id; $fallbackMembership->resource_pool_id = $pool->id; $fallbackMembership->scheduler_state = 'active'; $fallbackMembership->priority = 0; $fallbackMembership->weight = 100; $fallbackMembership->created_at = $fallbackMembership->updated_at = time(); $fallbackMembership->save(); $fallbackAttachment = new CloudHostNetworkAttachment(); $fallbackAttachment->attachment_key = str_repeat('9', 32); $fallbackAttachment->host_id = $fallbackHost->id; $fallbackAttachment->line_id = $line->id; $fallbackAttachment->name = 'Fallback dual stack'; $fallbackAttachment->network_mode = 'selector'; $fallbackAttachment->network_selector = 'bridge:br0'; $fallbackAttachment->state = 'active'; $fallbackAttachment->priority = 0; $fallbackAttachment->config_revision = 1; $fallbackAttachment->verified_agent_revision = 1; $fallbackAttachment->verification_state = 'verified'; $fallbackAttachment->last_verified_at = time(); $fallbackAttachment->verification_error_code = ''; $fallbackAttachment->desired_revision = 1; $fallbackAttachment->observed_revision = 1; $fallbackAttachment->sync_state = 'applied'; $fallbackAttachment->last_error_code = ''; $fallbackAttachment->created_at = $fallbackAttachment->updated_at = time(); $fallbackAttachment->save(); $fallbackIpv4Pool = new CloudPublicIpPool(); $fallbackIpv4Pool->pool_key = str_repeat('8', 32); $fallbackIpv4Pool->host_id = $fallbackHost->id; $fallbackIpv4Pool->network_attachment_id = $fallbackAttachment->id; $fallbackIpv4Pool->name = 'Fallback IPv4'; $fallbackIpv4Pool->address_family = 4; $fallbackIpv4Pool->cidr = '11.0.0.0/24'; $fallbackIpv4Pool->gateway = '11.0.0.1'; $fallbackIpv4Pool->gateway_on_link = false; $fallbackIpv4Pool->guest_prefix_length = 24; $fallbackIpv4Pool->dns_servers_json = '["1.1.1.1"]'; $fallbackIpv4Pool->reserved_ips_json = '[]'; $fallbackIpv4Pool->allocation_start = null; $fallbackIpv4Pool->allocation_end = null; $fallbackIpv4Pool->allocation_cursor = null; $fallbackIpv4Pool->max_allocations = 0; $fallbackIpv4Pool->state = 'active'; $fallbackIpv4Pool->priority = 0; $fallbackIpv4Pool->desired_revision = 1; $fallbackIpv4Pool->observed_revision = 1; $fallbackIpv4Pool->sync_state = 'applied'; $fallbackIpv4Pool->last_error_code = ''; $fallbackIpv4Pool->created_at = $fallbackIpv4Pool->updated_at = time(); $fallbackIpv4Pool->save(); $fallbackIpv6Pool = new CloudPublicIpPool(); $fallbackIpv6Pool->pool_key = str_repeat('7', 32); $fallbackIpv6Pool->host_id = $fallbackHost->id; $fallbackIpv6Pool->network_attachment_id = $fallbackAttachment->id; $fallbackIpv6Pool->name = 'Fallback IPv6'; $fallbackIpv6Pool->address_family = 6; $fallbackIpv6Pool->cidr = '2606:4700:4700:200::/64'; $fallbackIpv6Pool->gateway = '2606:4700:4700:200::1'; $fallbackIpv6Pool->gateway_on_link = false; $fallbackIpv6Pool->guest_prefix_length = 64; $fallbackIpv6Pool->dns_servers_json = '["2606:4700:4700::1111"]'; $fallbackIpv6Pool->reserved_ips_json = '[]'; $fallbackIpv6Pool->allocation_start = '2606:4700:4700:200::10'; $fallbackIpv6Pool->allocation_end = '2606:4700:4700:200::20'; $fallbackIpv6Pool->allocation_cursor = null; $fallbackIpv6Pool->max_allocations = 4; $fallbackIpv6Pool->state = 'active'; $fallbackIpv6Pool->priority = 0; $fallbackIpv6Pool->desired_revision = 1; $fallbackIpv6Pool->observed_revision = 1; $fallbackIpv6Pool->sync_state = 'applied'; $fallbackIpv6Pool->last_error_code = ''; $fallbackIpv6Pool->created_at = $fallbackIpv6Pool->updated_at = time(); $fallbackIpv6Pool->save(); $fallbackIpv4 = new CloudPublicIp(); $fallbackIpv4->host_id = $fallbackHost->id; $fallbackIpv4->public_ip_pool_id = $fallbackIpv4Pool->id; $fallbackIpv4->address_family = 4; $fallbackIpv4->address = '11.0.0.10'; $fallbackIpv4->address_binary = inet_pton('11.0.0.10'); $fallbackIpv4->prefix_length = 24; $fallbackIpv4->state = 'available'; $fallbackIpv4->created_at = $fallbackIpv4->updated_at = time(); $fallbackIpv4->save(); $fallbackPolicy = new CloudHostCapacityPolicy(); $fallbackPolicy->host_id = $fallbackHost->id; $fallbackPolicy->configured_cpu_cores = 4; $fallbackPolicy->configured_memory_mib = 4096; $fallbackPolicy->configured_disk_gib = 40; $fallbackPolicy->safety_cpu_cores = 0; $fallbackPolicy->safety_memory_mib = 0; $fallbackPolicy->safety_disk_gib = 0; $fallbackPolicy->max_instances = 2; $fallbackPolicy->created_at = $fallbackPolicy->updated_at = time(); $fallbackPolicy->save(); $fallbackScoreOrder = new CloudOrder(); $fallbackScoreOrder->user_id = 1; $fallbackScoreOrder->plan_id = $plan->id; $fallbackScoreOrder->offer_id = $offer->id; $fallbackScoreOrder->resource_pool_id = $pool->id; $fallbackScoreOrder->host_id = $fallbackHost->id; $fallbackScoreOrder->reservation_id = null; $fallbackScoreOrder->deployment_snapshot_id = null; $fallbackScoreOrder->idempotency_key = 'fallback-score-fixture'; $fallbackScoreOrder->request_hash = str_repeat('4', 64); $fallbackScoreOrder->amount = '0.00'; $fallbackScoreOrder->currency = 'USD'; $fallbackScoreOrder->status = 'succeeded'; $fallbackScoreOrder->spec_json = '{}'; $fallbackScoreOrder->sealed_payload = null; $fallbackScoreOrder->instance_id = 999999; $fallbackScoreOrder->refunded_at = null; $fallbackScoreOrder->stock_reserved_at = null; $fallbackScoreOrder->stock_released_at = null; $fallbackScoreOrder->created_at = $fallbackScoreOrder->updated_at = time(); $fallbackScoreOrder->save(); $fallbackScoreReservation = new CloudResourceReservation(); $fallbackScoreReservation->reservation_key = str_repeat('6', 32); $fallbackScoreReservation->operation_id = str_repeat('5', 32); $fallbackScoreReservation->order_id = $fallbackScoreOrder->id; $fallbackScoreReservation->backup_restore_id = null; $fallbackScoreReservation->offer_id = $offer->id; $fallbackScoreReservation->resource_pool_id = $pool->id; $fallbackScoreReservation->host_id = $fallbackHost->id; $fallbackScoreReservation->user_id = 1; $fallbackScoreReservation->instance_id = 999999; $fallbackScoreReservation->network_attachment_id = null; $fallbackScoreReservation->public_ip_pool_id = null; $fallbackScoreReservation->public_ip_id = null; $fallbackScoreReservation->cpu_cores = 1; $fallbackScoreReservation->memory_mib = 512; $fallbackScoreReservation->disk_gib = 1; $fallbackScoreReservation->state = 'committed'; $fallbackScoreReservation->expires_at = null; $fallbackScoreReservation->released_at = null; $fallbackScoreReservation->release_reason = ''; $fallbackScoreReservation->created_at = $fallbackScoreReservation->updated_at = time(); $fallbackScoreReservation->save(); $fallbackScoreOrder->reservation_id = $fallbackScoreReservation->id; $fallbackScoreOrder->save(); Capsule::table('cloud_template_replicas')->insert([ 'host_id' => $fallbackHost->id, 'template_id' => $template->id, 'revision' => $template->revision, 'generation_token' => $template->generation_token, 'state' => 'active', 'image_bytes' => $template->image_bytes, 'sha256' => $template->sha256, 'operation_id' => 'fixture-fallback-template-import', 'error_code' => '', 'created_at' => time(), 'updated_at' => time(), ]); $offerForFallback = CloudOffer::findOrFail((int) $offer->id); $offerForFallback->stock = -1; $offerForFallback->save(); $fallbackUser = User::findOrFail(1); $fallbackUser->money = '100.00'; $fallbackUser->save(); $beforeFallbackReservationId = (int) ( CloudResourceReservation::max('id') ?? 0 ); $fallbackOrder = CloudOrderService::purchase(array_replace($input, [ 'name' => 'vm-host-capacity-fallback', 'idempotency_key' => 'test-host-capacity-fallback', ]), $actor); CloudInvoiceService::payBalance( CloudInvoice::where('order_id', $fallbackOrder->id)->firstOrFail(), $actor, 'provision-host-capacity-fallback' ); $fallbackReservation = CloudResourceReservation::where( 'order_id', $fallbackOrder->id )->firstOrFail(); $assert((int) $fallbackOrder->fresh()->host_id === (int) $fallbackHost->id && (int) $fallbackReservation->host_id === (int) $fallbackHost->id && (int) $fallbackReservation->id >= $beforeFallbackReservationId + 2 && Capsule::table('cloud_resource_reservation_ips') ->where('reservation_id', (int) $fallbackReservation->id) ->count() === 2 && CloudResourceReservation::where('order_id', $fallbackOrder->id) ->where('host_id', $host->id)->count() === 0 && Capsule::table('cloud_resource_reservation_ips as claim') ->leftJoin('cloud_resource_reservations as reservation', 'reservation.id', '=', 'claim.reservation_id') ->where('claim.reservation_id', '>', $beforeFallbackReservationId) ->whereNull('reservation.id')->count() === 0, 'Capacity fallback did not continue to host two or left host-one ledger residue.'); echo "Cloud SQLite purchase, provisioning, refund, ambiguity, quota and tenant checks passed (crypto double only).\n"; }