You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
virt/web/tests/CloudFinancialWorkflowTest.php

5503 lines
270 KiB
PHP

This file contains ambiguous Unicode characters!

This file contains ambiguous Unicode characters that may be confused with others in your current locale. If your use case is intentional and legitimate, you can safely ignore this warning. Use the Escape button to highlight these characters.

<?php
// Financial/state-machine tests only. The isolated keyring double is NOT encryption and is never loaded by production.
namespace App\Services {
final class Config {
public static array $values = [
'super_admin' => '99',
'baseUrl' => 'https://panel.example.test',
'appName' => 'Virt Cloud Test',
];
public static function get($name, $default = null) { return self::$values[$name] ?? $default; }
}
final class Auth { public static $user; public static function getUser() { return self::$user; } }
final class Mail {
public static array $sent = [];
public static function send(...$arguments): bool {
self::$sent[] = $arguments;
return true;
}
}
}
namespace App\Services\Cloud {
final class CloudSecrets {
private static array $memory = [];
public static function key(string $purpose): string { return str_repeat('k', 32); }
public static function encrypt(array $payload, string $purpose): string { $id = bin2hex(random_bytes(16)); self::$memory[$id] = $payload; return 'test-only:' . $id; }
public static function decrypt(string $cipher, string $purpose): array { return self::$memory[substr($cipher, 10)]; }
}
/** Isolated read-only Agent transport for create recovery checks. */
final class FinancialCreateRecoveryAgentClient
{
public array $requests = [];
public array $responses = [];
public function systemRecoveryRequest(
string $method,
string $target,
string $operationId,
?array $payload = null
): array {
$this->requests[] = compact(
'method', 'target', 'operationId', 'payload'
);
if ($this->responses === []) {
throw new \RuntimeException(
'Unexpected create recovery Agent request.'
);
}
$response = array_shift($this->responses);
if ($response instanceof \Throwable) {
throw $response;
}
if (is_callable($response)) {
$response = $response(end($this->requests));
}
if (!is_array($response)) {
throw new \RuntimeException(
'Invalid create recovery Agent response.'
);
}
return $response;
}
}
final class CloudAgentSettingsService
{
public static ?FinancialCreateRecoveryAgentClient $client = null;
public static int $hostId = 0;
public static function clientForConfiguration(
int $hostId,
float $timeoutSeconds = 120.0
): FinancialCreateRecoveryAgentClient {
if ($hostId !== self::$hostId || $timeoutSeconds !== 30.0
|| self::$client === null) {
throw new \RuntimeException(
'Unexpected create recovery client request.'
);
}
return self::$client;
}
public static function testedNetworkSelectors(
\App\Models\CloudHostAgent $setting
): array {
return $setting->testedNetworkSelectors();
}
public static function publicNetworkReady(
\App\Models\CloudHostAgent $setting,
string $selector,
int $verifiedRevision
): bool {
if (!$setting->isReady()
|| $verifiedRevision !== (int) $setting->revision
|| $verifiedRevision !== (int) $setting->tested_revision
|| !in_array($selector,
self::testedNetworkSelectors($setting), true)) {
return false;
}
$capabilities = json_decode(
(string) $setting->public_network_capabilities_json,
true
);
return is_array($capabilities)
&& ($capabilities['contract'] ?? null) === 1
&& ($capabilities['dual_stack_required'] ?? null) === true
&& ($capabilities['project_owned_filter'] ?? null) === true
&& ($capabilities['ndp_guard'] ?? null) === true;
}
}
}
namespace {
require dirname(__DIR__) . '/vendor/autoload.php';
use Illuminate\Database\Capsule\Manager as Capsule;
use Illuminate\Database\Schema\Blueprint;
use App\Models\User;
use App\Models\Shop;
use App\Models\CloudHost;
use App\Models\CloudHostCapacityPolicy;
use App\Models\CloudHostNetworkAttachment;
use App\Models\CloudHostResourcePoolMembership;
use App\Models\CloudBackupSubscription;
use App\Models\CloudDeploymentSnapshot;
use App\Models\CloudInstanceBackup;
use App\Models\CloudBackupRestore;
use App\Models\CloudInstance;
use App\Models\CloudInvoice;
use App\Models\CloudInvoicePayment;
use App\Models\CloudMailOutbox;
use App\Models\CloudNetworkLine;
use App\Models\CloudOffer;
use App\Models\CloudOrder;
use App\Models\CloudOrderItem;
use App\Models\CloudJob;
use App\Models\CloudPublicIp;
use App\Models\CloudPublicIpPool;
use App\Models\CloudRegion;
use App\Models\CloudResourcePool;
use App\Models\CloudResourceReservation;
use App\Models\CloudServicePeriod;
use App\Models\CloudTemplate;
use App\Services\Cloud\CloudInvoiceService;
use App\Services\Cloud\CloudInvoicePaymentService;
use App\Services\Cloud\CloudAdminOperationsService;
use App\Services\Cloud\CloudAutomaticOperationRecoveryService;
use App\Services\Cloud\CloudOrderService;
use App\Services\Cloud\CloudWorkerService;
use App\Services\Cloud\CloudAgentClient;
use App\Services\Cloud\CloudAccess;
use App\Services\Cloud\CloudJobService;
use App\Services\Cloud\CloudIpChangeBillingService;
use App\Services\Cloud\CloudProvisioningBillingConfig;
use App\Services\Cloud\CloudLifecycleService;
use App\Services\Cloud\CloudOperationCompletionService;
use App\Services\Cloud\CloudOperationRecoveryIdentity;
use App\Services\Cloud\CloudOperationRecoveryService;
use App\Services\Cloud\CloudRenewalService;
use App\Services\Cloud\CloudError;
use App\Services\Cloud\CloudMoney;
use App\Services\Cloud\CloudPlatformTestNetworkPolicy;
use App\Services\Cloud\CloudPlacementService;
use App\Services\Cloud\CloudPublicNetworkAllocationService;
use App\Services\Cloud\CloudPublicNetworkTransitionCommand;
use App\Services\Cloud\CloudPublicNetworkTransitionWorkflow;
use App\Services\Cloud\CloudDeploymentPageService;
use App\Services\Cloud\CloudTemplateCatalogService;
use App\Services\Cloud\CloudAgentSettingsService;
use App\Services\Cloud\CloudAgentError;
use App\Services\Cloud\CloudCreateFailureRecoveryService;
use App\Services\Cloud\FinancialCreateRecoveryAgentClient;
use GuzzleHttp\Client;
use GuzzleHttp\Handler\MockHandler;
use GuzzleHttp\HandlerStack;
use GuzzleHttp\Middleware;
use GuzzleHttp\Psr7\Response;
use GuzzleHttp\Psr7\Request;
use GuzzleHttp\Exception\ConnectException;
$assert = function ($condition, $message) { if (!$condition) { throw new RuntimeException($message); } };
$bundleState = static function (CloudResourceReservation $reservation,
string $claimState, string $ipState): bool {
$claims = Capsule::table('cloud_resource_reservation_ips')
->where('reservation_id', (int) $reservation->id)
->where('state', $claimState)->get();
return $claims->count() === 2
&& $claims->pluck('address_family')->sort()->values()->all()
=== [4, 6]
&& CloudPublicIp::whereIn('id', $claims->pluck('public_ip_id'))
->where('state', $ipState)->count() === 2;
};
/** Strictly unwrap one local PHP-to-Gateway request into its Agent request. */
$agentRequest = static function (array $historyEntry) use ($assert): array {
$outer = $historyEntry['request'] ?? null;
$assert($outer instanceof \Psr\Http\Message\RequestInterface,
'Gateway history entry is missing its outer request.');
$uri = $outer->getUri();
$assert($outer->getMethod() === 'POST'
&& $uri->getScheme() === 'http'
&& $uri->getHost() === '127.0.0.1'
&& $uri->getPort() === 8001
&& $uri->getPath() === '/internal/agent/request'
&& $uri->getQuery() === ''
&& $uri->getFragment() === '',
'Agent request did not use the fixed local Gateway endpoint.');
$envelope = json_decode((string) $outer->getBody(), true, 32,
JSON_THROW_ON_ERROR);
$assert(is_array($envelope) && !array_is_list($envelope),
'Gateway request envelope is not a JSON object.');
$actualKeys = array_keys($envelope);
$expectedKeys = [
'agent_id', 'type', 'id', 'method', 'request_target', 'body',
'headers',
];
sort($actualKeys);
sort($expectedKeys);
$assert($actualKeys === $expectedKeys
&& ($envelope['agent_id'] ?? null) === '1'
&& ($envelope['type'] ?? null) === 'request'
&& is_string($envelope['id'] ?? null)
&& preg_match('/^[a-f0-9]{32}$/D', $envelope['id']) === 1
&& is_string($envelope['method'] ?? null)
&& in_array($envelope['method'],
['GET', 'POST', 'PUT', 'PATCH', 'DELETE'], true)
&& is_string($envelope['request_target'] ?? null)
&& preg_match('#^/v1(?:/|$)#D', $envelope['request_target']) === 1
&& is_string($envelope['body'] ?? null)
&& is_array($envelope['headers'] ?? null)
&& !array_is_list($envelope['headers']),
'Gateway request envelope has an invalid shape.');
$headers = $envelope['headers'];
$actualHeaderKeys = array_keys($headers);
$expectedHeaderKeys = [
'X-Virt-Timestamp', 'X-Virt-Nonce', 'X-Virt-Content-SHA256',
'X-Virt-Actor', 'X-Virt-Signature',
];
sort($actualHeaderKeys);
sort($expectedHeaderKeys);
$assert($actualHeaderKeys === $expectedHeaderKeys
&& preg_match('/^[0-9]{1,20}$/D',
(string) ($headers['X-Virt-Timestamp'] ?? '')) === 1
&& preg_match('/^[a-f0-9]{32}$/D',
(string) ($headers['X-Virt-Nonce'] ?? '')) === 1
&& ($headers['X-Virt-Content-SHA256'] ?? null)
=== hash('sha256', $envelope['body'])
&& preg_match('/^[A-Za-z0-9_-]+$/D',
(string) ($headers['X-Virt-Actor'] ?? '')) === 1
&& preg_match('/^[a-f0-9]{64}$/D',
(string) ($headers['X-Virt-Signature'] ?? '')) === 1,
'Gateway envelope lost the signed Agent request headers.');
if ($envelope['body'] !== '') {
$innerBody = json_decode($envelope['body'], false, 64,
JSON_THROW_ON_ERROR);
$assert($innerBody instanceof \stdClass,
'Agent request body is not a JSON object.');
}
$encodedActor = $headers['X-Virt-Actor'];
$padding = str_repeat('=', (4 - strlen($encodedActor) % 4) % 4);
$decodedActor = base64_decode(
strtr($encodedActor, '-_', '+/') . $padding, true
);
$actorPayload = is_string($decodedActor)
? json_decode($decodedActor, true, 16, JSON_THROW_ON_ERROR) : null;
$assert(is_array($actorPayload) && !array_is_list($actorPayload)
&& is_int($actorPayload['user_id'] ?? null)
&& is_bool($actorPayload['is_admin'] ?? null)
&& is_bool($actorPayload['is_super_admin'] ?? null)
&& is_bool($actorPayload['can_manage'] ?? null)
&& (!isset($actorPayload['operation_id'])
|| (is_string($actorPayload['operation_id'])
&& preg_match('/^[A-Za-z0-9_-]{8,80}$/D',
$actorPayload['operation_id']) === 1)),
'Gateway envelope contains an invalid signed Agent actor.');
return $envelope;
};
$capsule = new Capsule();
$capsule->addConnection(['driver' => 'sqlite', 'database' => ':memory:'], 'default');
$capsule->setAsGlobal(); $capsule->bootEloquent();
$schema = $capsule->schema('default');
$schema->create('user', function (Blueprint $t) { $t->increments('id'); $t->string('user_name')->default(''); $t->string('email')->default(''); $t->string('money'); $t->integer('enable')->default(1); $t->integer('is_admin')->default(0); });
$schema->create('configurations', function (Blueprint $t) { $t->increments('id'); $t->string('name')->unique(); $t->text('value'); });
$schema->create('shop', function (Blueprint $t) { $t->increments('id'); $t->string('name'); foreach (['cpu_cores', 'memory_mib', 'bandwidth_mbps', 'disk_gib', 'validity_days', 'traffic_quota_bytes', 'status', 'sort'] as $v) { $t->integer($v)->default(0); } });
$schema->create('cloud_regions', function (Blueprint $t) { $t->increments('id'); $t->string('code'); $t->string('name'); $t->string('country_code'); $t->string('status'); $t->integer('sort')->default(0); $t->integer('created_at'); $t->integer('updated_at'); });
$schema->create('cloud_network_lines', function (Blueprint $t) { $t->increments('id'); $t->integer('region_id'); $t->integer('traffic_pool_group_id')->nullable(); $t->string('code'); $t->string('name'); $t->integer('is_default')->default(0); $t->string('status'); $t->integer('sort')->default(0); $t->integer('created_at'); $t->integer('updated_at'); });
$schema->create('cloud_resource_pools', function (Blueprint $t) { $t->increments('id'); $t->integer('line_id'); $t->string('code'); $t->string('name'); $t->string('status'); $t->string('scheduling_policy')->default('balanced'); $t->integer('sort')->default(0); $t->integer('created_at'); $t->integer('updated_at'); });
$schema->create('cloud_hosts', function (Blueprint $t) { $t->increments('id'); $t->string('name'); $t->string('region')->default(''); $t->string('country_code')->default(''); $t->integer('enabled'); $t->integer('resource_pool_id')->nullable(); $t->string('scheduler_state')->default('disabled'); $t->string('default_network'); $t->integer('created_at')->default(0); $t->integer('updated_at')->default(0); });
$schema->create('cloud_host_agents', function (Blueprint $t) { $t->integer('host_id')->primary(); $t->text('sealed_api_secret'); $t->integer('enabled')->default(0); $t->integer('revision')->default(1); $t->integer('tested_revision')->default(0); $t->string('last_test_status')->default('never'); $t->string('last_error_code')->default(''); $t->string('last_agent_version')->default(''); $t->text('network_selectors_json')->default('[]'); $t->text('public_network_capabilities_json')->default('{}'); $t->text('portable_restore_contract_json')->default('{}'); $t->string('portable_restore_contract_sha256', 64)->default(''); $t->integer('last_tested_at')->default(0); $t->integer('created_at'); $t->integer('updated_at'); });
$schema->create('cloud_host_storage_configurations', function (Blueprint $t) { $t->integer('host_id')->primary(); $t->integer('revision')->default(0); $t->string('state')->default('applying'); });
$schema->create('cloud_host_resource_pool_memberships', function (Blueprint $t) { $t->increments('id'); $t->integer('host_id'); $t->integer('resource_pool_id'); $t->string('scheduler_state')->default('disabled'); $t->integer('priority')->default(0); $t->integer('weight')->default(100); $t->integer('revision')->default(1); $t->integer('created_at'); $t->integer('updated_at'); $t->unique(['host_id', 'resource_pool_id']); });
$schema->create('cloud_host_network_attachments', function (Blueprint $t) { $t->increments('id'); $t->string('attachment_key'); $t->integer('host_id'); $t->integer('line_id'); $t->string('name'); $t->string('network_mode')->default('selector'); $t->string('network_selector'); $t->integer('vlan_id')->nullable(); $t->integer('routing_table')->nullable(); $t->string('state')->default('active'); $t->integer('priority')->default(0); $t->integer('config_revision')->default(1); $t->integer('verified_agent_revision')->default(0); $t->string('verification_state')->default('never'); $t->integer('last_verified_at')->default(0); $t->string('verification_error_code')->default(''); $t->integer('desired_revision')->default(1); $t->integer('observed_revision')->default(0); $t->string('sync_state')->default('never'); $t->string('last_error_code')->default(''); $t->integer('created_at'); $t->integer('updated_at'); });
$schema->create('cloud_public_ip_pools', function (Blueprint $t) { $t->increments('id'); $t->string('pool_key'); $t->integer('host_id'); $t->integer('network_attachment_id'); $t->string('name'); $t->integer('address_family'); $t->string('cidr'); $t->string('gateway'); $t->boolean('gateway_on_link')->default(false); $t->integer('guest_prefix_length'); $t->text('dns_servers_json'); $t->text('reserved_ips_json'); $t->string('allocation_start')->nullable(); $t->string('allocation_end')->nullable(); $t->string('allocation_cursor')->nullable(); $t->integer('max_allocations')->default(0); $t->string('state')->default('active'); $t->integer('priority')->default(0); $t->integer('desired_revision')->default(1); $t->integer('observed_revision')->default(0); $t->string('sync_state')->default('never'); $t->string('last_error_code')->default(''); $t->integer('created_at'); $t->integer('updated_at'); });
$schema->create('cloud_host_capacity_policies', function (Blueprint $t) { $t->integer('host_id')->primary(); foreach (['configured_cpu_cores', 'configured_memory_mib', 'configured_disk_gib', 'safety_cpu_cores', 'safety_memory_mib', 'safety_disk_gib', 'max_instances', 'created_at', 'updated_at'] as $v) { $t->integer($v)->default(0); } });
$schema->create('cloud_public_ips', function (Blueprint $t) { $t->increments('id'); $t->integer('host_id'); $t->integer('public_ip_pool_id'); $t->integer('address_family'); $t->string('address'); $t->binary('address_binary'); $t->integer('prefix_length'); $t->string('state'); $t->integer('reservation_id')->nullable(); $t->integer('reservation_ip_claim_id')->nullable(); $t->integer('instance_id')->nullable(); $t->string('operation_id')->nullable(); $t->integer('created_at'); $t->integer('updated_at'); $t->unique(['address_family', 'address_binary']); });
$schema->create('cloud_offers', function (Blueprint $t) { $t->increments('id'); $t->integer('plan_id'); $t->integer('resource_pool_id'); $t->string('code')->unique(); $t->string('name'); $t->string('price'); $t->string('backup_price')->default('0.00'); $t->string('currency')->default('USD'); $t->integer('price_version')->default(1); $t->integer('revision')->default(1); $t->integer('status')->default(1); $t->integer('stock')->default(-1); $t->integer('sort')->default(0); $t->integer('created_at'); $t->integer('updated_at'); $t->unique(['plan_id', 'resource_pool_id']); });
$schema->create('cloud_templates', function (Blueprint $t) { $t->string('id')->primary(); $t->string('catalog_key')->default('test-catalog'); $t->string('version_label')->default('test-version'); $t->boolean('is_default')->default(true); $t->string('name'); $t->string('os_family'); $t->string('os_version'); $t->integer('image_bytes'); $t->string('sha256'); $t->text('startup_script'); $t->integer('revision'); $t->string('generation_token'); $t->string('state'); $t->integer('active_job_id')->nullable(); $t->integer('created_at'); $t->integer('updated_at'); });
$schema->create('cloud_user_isos', function (Blueprint $t) { $t->string('id')->primary(); $t->integer('active_job_id')->nullable(); });
$schema->create('cloud_template_replicas', function (Blueprint $t) { $t->integer('host_id'); $t->string('template_id'); $t->integer('job_id')->nullable(); $t->integer('revision'); $t->string('generation_token'); $t->string('state')->default('pending'); $t->integer('image_bytes')->default(0); $t->string('sha256')->default(''); $t->string('operation_id')->default(''); $t->string('error_code')->default(''); $t->integer('created_at'); $t->integer('updated_at'); $t->primary(['host_id', 'template_id']); });
$schema->create('cloud_platform_test_policy_state', function (Blueprint $t) { $t->integer('id')->primary(); $t->integer('revision')->default(0); $t->integer('updated_by')->nullable(); $t->integer('updated_at')->default(0); });
$schema->create('cloud_platform_test_policy_rules', function (Blueprint $t) { $t->increments('id'); $t->string('rule_key', 32)->unique(); $t->string('service', 120); $t->string('protocol', 8); $t->integer('port_from'); $t->integer('port_to'); $t->string('direction', 8); $t->string('reason', 500); $t->integer('created_by'); $t->integer('updated_by'); $t->integer('deleted_by')->nullable(); $t->integer('deleted_at')->nullable(); $t->integer('created_at'); $t->integer('updated_at'); });
$schema->create('cloud_orders', function (Blueprint $t) { $t->increments('id'); foreach (['user_id', 'plan_id', 'offer_id', 'resource_pool_id'] as $v) { $t->integer($v); } $t->integer('host_id')->nullable(); $t->integer('reservation_id')->nullable()->unique(); $t->integer('deployment_snapshot_id')->nullable(); $t->string('idempotency_key'); $t->string('request_hash'); $t->string('amount'); $t->string('currency')->default('USD'); $t->string('status'); $t->text('spec_json'); $t->text('sealed_payload')->nullable(); $t->integer('instance_id')->nullable(); $t->integer('refunded_at')->nullable(); $t->integer('stock_reserved_at')->nullable(); $t->integer('stock_released_at')->nullable(); $t->integer('created_at'); $t->integer('updated_at'); $t->unique(['user_id', 'idempotency_key']); });
$schema->create('cloud_order_items', function (Blueprint $t) { $t->increments('id'); $t->integer('order_id'); $t->string('code'); $t->string('label'); $t->string('amount'); $t->string('currency')->default('USD'); $t->text('snapshot_json'); $t->integer('created_at'); $t->unique(['order_id', 'code']); });
$schema->create('cloud_resource_reservations', function (Blueprint $t) { $t->increments('id'); $t->string('reservation_key')->unique(); $t->string('operation_id')->unique(); $t->integer('order_id')->nullable(); $t->integer('backup_restore_id')->nullable(); $t->string('backup_restore_attempt_key')->nullable(); $t->integer('migration_source_debt_id')->nullable(); $t->integer('offer_id')->nullable(); foreach (['resource_pool_id', 'host_id', 'user_id'] as $v) { $t->integer($v); } $t->integer('instance_id')->nullable(); $t->integer('network_attachment_id')->nullable(); $t->integer('public_ip_pool_id')->nullable(); $t->integer('public_ip_id')->nullable(); foreach (['cpu_cores', 'memory_mib', 'disk_gib'] as $v) { $t->integer($v); } $t->string('state'); $t->integer('expires_at')->nullable(); $t->integer('released_at')->nullable(); $t->string('release_reason')->default(''); $t->integer('created_at'); $t->integer('updated_at'); $t->unique('order_id'); $t->unique('backup_restore_id'); });
$schema->create('cloud_resource_reservation_ips', function (Blueprint $t) { $t->increments('id'); $t->integer('reservation_id'); $t->integer('address_family'); $t->integer('claim_generation')->default(1); $t->integer('active_claim')->nullable()->default(1); $t->integer('network_attachment_id'); $t->integer('public_ip_pool_id'); $t->integer('public_ip_id'); $t->string('operation_id'); $t->string('state'); $t->integer('created_at'); $t->integer('updated_at'); $t->integer('released_at')->nullable(); $t->string('release_reason')->default(''); $t->unique(['reservation_id', 'address_family', 'active_claim']); $t->unique(['reservation_id', 'claim_generation', 'address_family']); });
$schema->create('cloud_public_network_transitions', function (Blueprint $t) { $t->increments('id'); $t->string('transition_key')->unique(); $t->string('operation_id')->unique(); $t->string('kind'); $t->integer('instance_id'); $t->integer('host_id'); $t->integer('reservation_id'); $t->integer('invoice_id')->nullable()->unique(); $t->integer('pool_replacement_batch_item_id')->nullable()->unique(); $t->integer('requested_by_user_id'); $t->integer('source_network_attachment_id'); $t->integer('target_network_attachment_id'); $t->string('source_claim_set_sha256'); $t->string('target_claim_set_sha256'); $t->text('source_public_network_json'); $t->string('source_public_network_sha256'); $t->text('target_public_network_json'); $t->string('target_public_network_sha256'); $t->string('state'); $t->string('effect_resolution'); $t->integer('active_transition')->nullable(); $t->string('last_error_code'); $t->integer('effect_resolved_at')->nullable(); $t->integer('terminal_at')->nullable(); $t->integer('created_at'); $t->integer('updated_at'); $t->unique(['instance_id', 'active_transition']); $t->unique(['reservation_id', 'active_transition']); });
$schema->create('cloud_public_network_transition_ips', function (Blueprint $t) { $t->increments('id'); $t->integer('transition_id'); $t->integer('address_family'); $t->integer('source_claim_id'); $t->integer('target_claim_id')->nullable()->unique(); $t->boolean('changed'); $t->integer('created_at'); $t->integer('updated_at'); $t->unique(['transition_id', 'address_family']); });
$schema->create('cloud_backup_restores', function (Blueprint $t) { $t->increments('id'); $t->string('restore_key')->unique(); $t->integer('backup_id')->nullable(); $t->integer('target_host_id'); $t->integer('target_instance_id')->nullable(); $t->integer('reservation_id')->nullable()->unique(); $t->integer('job_id')->nullable(); $t->string('operation_id')->unique(); $t->string('new_uuid'); $t->string('new_name'); $t->string('state'); $t->string('error_code')->default(''); $t->integer('ready_at')->nullable(); $t->integer('created_at'); $t->integer('updated_at'); });
$schema->create('cloud_backup_recoveries', function (Blueprint $t) { $t->increments('id'); $t->integer('instance_id'); $t->string('state'); });
$schema->create('cloud_instance_migrations', function (Blueprint $t) { $t->increments('id'); $t->integer('instance_id'); $t->integer('restore_id'); $t->integer('source_host_id'); $t->integer('source_reservation_id'); $t->integer('target_host_id'); $t->integer('target_reservation_id'); $t->string('state'); });
$schema->create('cloud_instance_backups', function (Blueprint $t) { $t->increments('id'); $t->string('backup_key')->unique(); $t->integer('source_instance_id'); $t->integer('source_host_id'); $t->integer('user_id'); $t->string('source_uuid'); $t->string('name'); $t->string('purpose')->default('portable'); $t->string('source')->default('manual'); $t->integer('subscription_id')->nullable(); $t->integer('scheduled_for')->nullable(); $t->string('state')->default('creating'); $t->string('consistency'); $t->string('object_key')->unique(); $t->integer('object_size_bytes')->default(0); $t->integer('virtual_size_bytes')->default(0); $t->string('sha256')->default(''); $t->text('spec_json'); $t->text('sealed_storage_runtime'); $t->text('sealed_credential')->nullable(); $t->integer('storage_revision'); $t->text('source_portable_restore_contract_json')->default('{}'); $t->string('source_portable_restore_contract_sha256', 64)->default(''); $t->integer('active_job_id')->nullable(); $t->string('last_operation_id')->nullable(); $t->string('error_code')->default(''); $t->integer('ready_at')->nullable(); $t->integer('created_at'); $t->integer('updated_at'); $t->unique(['subscription_id', 'scheduled_for']); });
$schema->create('cloud_deployment_snapshots', function (Blueprint $t) { $t->increments('id'); $t->string('snapshot_key')->unique(); $t->integer('user_id'); $t->integer('backup_id')->unique(); $t->integer('source_instance_id'); $t->string('name'); $t->string('state')->default('ready'); $t->string('idempotency_key'); $t->string('request_hash'); $t->string('error_code')->default(''); $t->integer('ready_at'); $t->integer('deleted_at')->nullable(); $t->integer('created_at'); $t->integer('updated_at'); $t->unique(['user_id', 'idempotency_key']); });
$schema->create('cloud_jobs', function (Blueprint $t) {
$t->increments('id');
$t->string('operation_id')->unique();
$t->string('request_idempotency_key');
$t->string('request_fingerprint');
foreach ([
'parent_job_id', 'supersedes_job_id', 'order_id', 'invoice_id',
'instance_id', 'snapshot_id', 'backup_id', 'backup_restore_id',
'vpc_repair_root_job_id', 'public_network_transition_id',
] as $v) {
$t->integer($v)->nullable();
}
$t->integer('user_id');
$t->integer('host_id')->nullable();
$t->string('kind');
$t->string('execution_mode');
$t->string('phase');
$t->string('outcome')->nullable();
$t->string('status');
$t->text('sealed_payload');
$t->integer('attempts');
$t->integer('dispatch_attempts');
$t->integer('dispatch_deadline_at');
$t->integer('dispatch_started_at')->nullable();
$t->integer('agent_accepted_at')->nullable();
$t->integer('receipt_deadline_at')->nullable();
$t->integer('terminal_at')->nullable();
$t->boolean('effect_may_have_applied')->nullable();
$t->boolean('safe_to_resubmit')->nullable();
$t->string('effect_resolution');
$t->integer('effect_resolved_at')->nullable();
$t->integer('effect_resolved_by')->nullable();
$t->text('effect_resolution_reason')->nullable();
$t->string('completion_event_id')->nullable();
$t->integer('read_at')->nullable();
$t->integer('dispatched_at')->nullable();
$t->string('lease_token')->nullable();
$t->integer('lease_until')->default(0);
$t->integer('available_at');
$t->string('error_code')->nullable();
$t->boolean('finalization_incident')->default(0);
$t->string('finalization_error_code')->nullable();
$t->text('result_json')->nullable();
$t->integer('created_at');
$t->integer('updated_at');
$t->unique('order_id');
});
$schema->create('cloud_backup_restore_attempts',
static function (Blueprint $table): void {
$table->increments('id');
$table->integer('job_id');
$table->string('state');
$table->string('effect_resolution');
$table->integer('active_attempt')->nullable();
$table->string('last_error_code')->default('');
});
$schema->create('cloud_agent_completion_events', function (Blueprint $t) {
$t->string('event_id')->primary();
$t->integer('host_id');
$t->string('operation_id');
$t->string('payload_sha256');
$t->string('outcome');
$t->boolean('effect_may_have_applied');
$t->boolean('safe_to_resubmit');
$t->string('error_code');
$t->text('result_json')->nullable();
$t->string('disposition');
$t->integer('received_at');
$t->integer('applied_at')->nullable();
$t->unique(['host_id', 'operation_id']);
});
$schema->create('cloud_instances', function (Blueprint $t) { $t->increments('id'); foreach (['host_id', 'user_id'] as $v) { $t->integer($v); } foreach (['order_id', 'plan_id', 'offer_id', 'resource_pool_id', 'reservation_id', 'current_period_id', 'expires_at', 'suspended_at', 'network_suspended_at'] as $v) { $t->integer($v)->nullable(); } foreach (['renewal_days', 'renewal_traffic_bytes', 'traffic_quota_bytes', 'traffic_download_bytes', 'traffic_upload_bytes', 'lifetime_download_bytes', 'lifetime_upload_bytes', 'traffic_synced_at'] as $v) { $t->integer($v)->default(0); } $t->string('renewal_price')->nullable(); $t->string('uuid'); $t->string('name'); $t->string('status'); $t->string('service_status')->default('exempt'); $t->string('network_status')->default('active'); $t->text('spec_json'); $t->text('sealed_credential')->nullable(); $t->integer('created_at'); $t->integer('updated_at'); $t->unique('order_id'); $t->unique(['host_id', 'uuid']); });
$schema->create('cloud_instance_network_policy_states', function (Blueprint $t) { $t->integer('instance_id')->primary(); $t->integer('user_id'); $t->boolean('desired_suspended')->default(false); $t->boolean('applied_suspended')->default(false); $t->integer('revision')->default(1); $t->string('state')->default('applied'); $t->string('operation_id')->nullable()->unique(); $t->integer('created_at'); $t->integer('updated_at'); });
$schema->create('cloud_instance_network_restrictions', function (Blueprint $t) { $t->integer('instance_id'); $t->integer('user_id'); $t->string('reason_code'); $t->boolean('desired_active')->default(false); $t->integer('revision')->default(1); $t->string('source_type'); $t->string('source_id'); $t->integer('created_at'); $t->integer('updated_at'); $t->primary(['instance_id', 'reason_code']); });
$schema->create('cloud_vpc_ports', function (Blueprint $t) { $t->increments('id'); $t->integer('vpc_id'); $t->integer('instance_id'); $t->integer('user_id'); $t->integer('active_claim')->nullable(); $t->string('state'); });
$schema->create('cloud_instance_snapshots', function (Blueprint $t) { $t->increments('id'); $t->string('snapshot_key')->unique(); foreach (['instance_id', 'user_id', 'host_id'] as $v) { $t->integer($v); } $t->string('name'); $t->string('state'); $t->string('consistency'); $t->integer('size_bytes')->nullable(); $t->string('last_operation_id')->nullable(); foreach (['ready_at', 'last_restored_at', 'deleted_at'] as $v) { $t->integer($v)->nullable(); } $t->integer('created_at'); $t->integer('updated_at'); });
$schema->create('cloud_renewals', function (Blueprint $t) { $t->increments('id'); $t->integer('instance_id'); $t->integer('user_id'); $t->integer('plan_id')->nullable(); $t->integer('period_id')->nullable(); $t->string('idempotency_key'); $t->string('request_hash'); $t->string('amount'); $t->integer('validity_days'); $t->integer('traffic_bytes_added'); $t->integer('previous_expires_at'); $t->integer('new_expires_at'); $t->integer('created_at'); $t->unique(['user_id', 'idempotency_key']); });
$schema->create('cloud_service_periods', function (Blueprint $t) { $t->increments('id'); foreach (['instance_id', 'user_id'] as $v) { $t->integer($v); } foreach (['plan_id', 'order_id', 'renewal_id', 'traffic_pool_group_id', 'download_base_bytes', 'upload_base_bytes', 'activated_at'] as $v) { $t->integer($v)->nullable(); } $t->string('traffic_quota_mode')->default('finite'); foreach (['starts_at', 'ends_at', 'traffic_quota_bytes', 'download_used_bytes', 'upload_used_bytes', 'created_at', 'updated_at'] as $v) { $t->integer($v)->default(0); } $t->string('status'); $t->unique('order_id'); $t->unique('renewal_id'); });
$schema->create('cloud_invoices', function (Blueprint $t) { $t->increments('id'); $t->string('invoice_no')->unique(); $t->string('purpose')->default('renewal'); $t->integer('instance_id')->nullable(); $t->integer('order_id')->nullable()->unique(); $t->integer('user_id'); $t->integer('plan_id')->nullable(); $t->integer('offer_id')->nullable(); $t->integer('renewal_id')->nullable(); $t->integer('renews_expires_at')->nullable(); $t->integer('due_at'); $t->string('amount'); $t->string('currency')->default('USD'); $t->integer('validity_days'); $t->integer('traffic_quota_bytes'); $t->string('status'); $t->string('paid_via')->nullable(); $t->string('payment_reference')->nullable(); $t->integer('paid_at')->nullable(); $t->integer('created_at'); $t->integer('updated_at'); });
$schema->create('cloud_invoice_items', function (Blueprint $t) { $t->increments('id'); $t->integer('invoice_id'); $t->integer('order_item_id')->nullable(); $t->string('code'); $t->string('label'); $t->string('amount'); $t->string('currency')->default('USD'); $t->text('snapshot_json'); $t->integer('created_at'); $t->unique(['invoice_id', 'code']); });
$schema->create('cloud_backup_subscriptions', function (Blueprint $t) { $t->increments('id'); $t->integer('instance_id')->unique(); $t->integer('user_id'); $t->integer('offer_id'); $t->boolean('enabled')->default(true); $t->boolean('renew_next_period')->default(true); $t->string('frequency')->default('daily'); $t->integer('weekday')->default(0); $t->integer('retention_count')->default(2); $t->string('timezone')->default('UTC'); $t->string('price'); $t->string('currency')->default('USD'); $t->integer('price_version'); $t->integer('state_revision')->default(1); $t->integer('next_run_at'); $t->integer('last_scheduled_at')->nullable(); $t->integer('last_backup_id')->nullable(); $t->integer('created_at'); $t->integer('updated_at'); });
$schema->create('cloud_backup_period_entitlements', function (Blueprint $t) { $t->increments('id'); $t->integer('subscription_id'); $t->integer('instance_id'); $t->integer('service_period_id')->unique(); $t->integer('user_id'); $t->integer('offer_id'); $t->integer('invoice_id'); $t->integer('starts_at'); $t->integer('ends_at'); $t->string('amount'); $t->string('full_period_price'); $t->string('currency')->default('USD'); $t->integer('price_version'); $t->string('source'); $t->integer('paid_at'); $t->integer('created_at'); $t->integer('updated_at'); });
$schema->create('cloud_invoice_payments', function (Blueprint $t) { $t->increments('id'); $t->integer('invoice_id'); $t->integer('user_id'); $t->string('provider'); $t->string('provider_order_no'); $t->string('idempotency_key'); $t->string('amount'); $t->string('status'); $t->integer('created_at'); $t->integer('updated_at'); $t->unique(['provider', 'provider_order_no']); $t->unique(['user_id', 'idempotency_key']); });
$schema->create('cloud_mail_outbox', function (Blueprint $t) { $t->increments('id'); $t->integer('invoice_id'); $t->string('kind'); $t->string('status'); $t->integer('attempts'); $t->integer('available_at'); $t->string('lease_token')->nullable(); $t->integer('lease_until'); $t->integer('sent_at')->nullable(); $t->string('last_error'); $t->integer('created_at'); $t->integer('updated_at'); $t->unique(['invoice_id', 'kind']); });
$schema->create('cloud_ssh_keys', function (Blueprint $t) { $t->increments('id'); $t->integer('user_id'); $t->string('name'); $t->text('public_key'); $t->string('fingerprint'); $t->integer('created_at'); });
$capsule->connection('default')->table('configurations')->insert([
['name' => 'cloud_network_lines_enabled', 'value' => 'false'],
[
'name' => CloudProvisioningBillingConfig::PAYMENT_WINDOW_MINUTES_KEY,
'value' => '60',
],
]);
$capsule->connection('default')->table(
'cloud_platform_test_policy_state'
)->insert(['id' => 1, 'revision' => 0, 'updated_by' => null, 'updated_at' => 0]);
$user = new User(); $user->id = 1; $user->user_name = 'financial-owner'; $user->email = 'owner@example.test'; $user->money = '0.00'; $user->enable = 1; $user->is_admin = false; $user->save(); $user->isLogin = true;
\App\Services\Auth::$user = $user;
$now = time();
$portableRestoreCompatibility =
\App\Services\Cloud\CloudPortableRestoreCompatibility::supported();
$region = new CloudRegion(); $region->code = 'test-region'; $region->name = 'Test Region'; $region->country_code = 'US'; $region->status = 'active'; $region->sort = 0; $region->created_at = $region->updated_at = $now; $region->save();
$line = new CloudNetworkLine(); $line->region_id = $region->id; $line->code = 'default'; $line->name = 'Default'; $line->is_default = true; $line->status = 'active'; $line->sort = 0; $line->created_at = $line->updated_at = $now; $line->save();
$pool = new CloudResourcePool(); $pool->line_id = $line->id; $pool->code = 'general'; $pool->name = 'General'; $pool->status = 'active'; $pool->scheduling_policy = 'balanced'; $pool->sort = 0; $pool->created_at = $pool->updated_at = $now; $pool->save();
$host = new CloudHost(); $host->name = 'host'; $host->enabled = true; $host->resource_pool_id = $pool->id; $host->scheduler_state = 'active'; $host->default_network = 'bridge:br0'; $host->created_at = $host->updated_at = $now; $host->save();
$capsule->connection('default')->table('cloud_host_agents')->insert([
'host_id' => $host->id,
'sealed_api_secret' => 'test-only-agent-secret',
'enabled' => 1,
'revision' => 1,
'tested_revision' => 1,
'last_test_status' => 'healthy',
'last_error_code' => '',
'last_agent_version' => 'test',
'network_selectors_json' => '["bridge:br0","bridge:br1"]',
'public_network_capabilities_json' => json_encode([
'contract' => 1,
'dual_stack_required' => true,
'project_owned_filter' => true,
'ndp_guard' => true,
], JSON_THROW_ON_ERROR),
'portable_restore_contract_json' =>
$portableRestoreCompatibility->json(),
'portable_restore_contract_sha256' =>
$portableRestoreCompatibility->sha256(),
'last_tested_at' => $now,
'created_at' => $now,
'updated_at' => $now,
]);
$capsule->connection('default')->table(
'cloud_host_storage_configurations'
)->insert([
'host_id' => $host->id, 'revision' => 1,
'state' => 'applied',
]);
$membership = new CloudHostResourcePoolMembership(); $membership->host_id = $host->id; $membership->resource_pool_id = $pool->id; $membership->scheduler_state = 'active'; $membership->priority = 0; $membership->weight = 100; $membership->created_at = $membership->updated_at = $now; $membership->save();
$attachment = new CloudHostNetworkAttachment(); $attachment->attachment_key = str_repeat('a', 32); $attachment->host_id = $host->id; $attachment->line_id = $line->id; $attachment->name = 'Primary selector'; $attachment->network_mode = 'selector'; $attachment->network_selector = 'bridge:br0'; $attachment->state = 'active'; $attachment->priority = 0; $attachment->config_revision = 1; $attachment->verified_agent_revision = 1; $attachment->verification_state = 'verified'; $attachment->last_verified_at = $now; $attachment->verification_error_code = ''; $attachment->desired_revision = 1; $attachment->observed_revision = 1; $attachment->sync_state = 'applied'; $attachment->last_error_code = ''; $attachment->created_at = $attachment->updated_at = $now; $attachment->save();
$targetAttachment = new CloudHostNetworkAttachment(); $targetAttachment->attachment_key = str_repeat('b', 32); $targetAttachment->host_id = $host->id; $targetAttachment->line_id = $line->id; $targetAttachment->name = 'Secondary selector'; $targetAttachment->network_mode = 'selector'; $targetAttachment->network_selector = 'bridge:br1'; $targetAttachment->state = 'active'; $targetAttachment->priority = 10; $targetAttachment->config_revision = 1; $targetAttachment->verified_agent_revision = 1; $targetAttachment->verification_state = 'verified'; $targetAttachment->last_verified_at = $now; $targetAttachment->verification_error_code = ''; $targetAttachment->desired_revision = 1; $targetAttachment->observed_revision = 0; $targetAttachment->sync_state = 'never'; $targetAttachment->last_error_code = ''; $targetAttachment->created_at = $targetAttachment->updated_at = $now; $targetAttachment->save();
$makePool = static function (string $key, int $attachmentId,
string $name, int $family, string $cidr, string $gateway,
int $prefix, array $dns, int $priority = 0,
?string $allocationStart = null, ?string $allocationEnd = null,
int $maxAllocations = 0, bool $gatewayOnLink = false
) use ($host, $now): CloudPublicIpPool {
$publicPool = new CloudPublicIpPool();
$publicPool->pool_key = $key;
$publicPool->host_id = $host->id;
$publicPool->network_attachment_id = $attachmentId;
$publicPool->name = $name;
$publicPool->address_family = $family;
$publicPool->cidr = $cidr;
$publicPool->gateway = $gateway;
$publicPool->gateway_on_link = $gatewayOnLink;
$publicPool->guest_prefix_length = $prefix;
$publicPool->dns_servers_json = json_encode($dns,
JSON_THROW_ON_ERROR);
$publicPool->reserved_ips_json = '[]';
$publicPool->allocation_start = $allocationStart;
$publicPool->allocation_end = $allocationEnd;
$publicPool->allocation_cursor = null;
$publicPool->max_allocations = $maxAllocations;
$publicPool->state = 'active';
$publicPool->priority = $priority;
$publicPool->desired_revision = 1;
$publicPool->observed_revision = 1;
$publicPool->sync_state = 'applied';
$publicPool->last_error_code = '';
$publicPool->created_at = $publicPool->updated_at = $now;
$publicPool->save();
return $publicPool;
};
$sourcePoolKey = str_repeat('1', 32);
$sourceIpv6PoolKey = str_repeat('3', 32);
$targetPoolKey = str_repeat('2', 32);
$targetIpv6PoolKey = str_repeat('4', 32);
$sourcePublicPool = $makePool(
$sourcePoolKey, (int) $attachment->id, 'Primary IPv4 pool',
4, '8.8.8.0/24', '8.8.8.1', 24, ['1.1.1.1']
);
$sourceIpv6Pool = $makePool(
$sourceIpv6PoolKey, (int) $attachment->id, 'Primary IPv6 pool',
6, '2404:c140:156:3a::/64', '2404:c140:156::1', 64,
['2606:4700:4700::1111'], 0, '2404:c140:156:3a::10',
'2404:c140:156:3a::ff', 32, true
);
$targetPublicPool = $makePool(
$targetPoolKey, (int) $targetAttachment->id, 'Secondary IPv4 pool',
4, '9.9.9.0/24', '9.9.9.1', 24, ['1.0.0.1'], 10
);
$targetIpv6Pool = $makePool(
$targetIpv6PoolKey, (int) $targetAttachment->id,
'Secondary IPv6 pool', 6, '2606:4700:4700:100::/64',
'2606:4700:4700:100::1', 64, ['2606:4700:4700::1001'],
10, '2606:4700:4700:100::10',
'2606:4700:4700:100::ff', 32
);
$policy = new CloudHostCapacityPolicy(); $policy->host_id = $host->id; $policy->configured_cpu_cores = 64; $policy->configured_memory_mib = 131072; $policy->configured_disk_gib = 4096; $policy->safety_cpu_cores = 2; $policy->safety_memory_mib = 2048; $policy->safety_disk_gib = 64; $policy->max_instances = 100; $policy->created_at = $policy->updated_at = $now; $policy->save();
foreach (['8.8.8.10', '8.8.8.11'] as $address) {
$ip = new CloudPublicIp();
$ip->host_id = $host->id;
$ip->public_ip_pool_id = $sourcePublicPool->id;
$ip->address_family = 4;
$ip->address = $address;
$ip->address_binary = inet_pton($address);
$ip->prefix_length = 24;
$ip->state = 'available';
$ip->created_at = $ip->updated_at = $now;
$ip->save();
}
foreach (['9.9.9.10', '9.9.9.11'] as $address) {
$ip = new CloudPublicIp();
$ip->host_id = $host->id;
$ip->public_ip_pool_id = $targetPublicPool->id;
$ip->address_family = 4;
$ip->address = $address;
$ip->address_binary = inet_pton($address);
$ip->prefix_length = 24;
$ip->state = 'available';
$ip->created_at = $ip->updated_at = $now;
$ip->save();
}
$plan = new Shop(); $plan->name = 'plan'; $plan->cpu_cores = 2; $plan->memory_mib = 2048; $plan->bandwidth_mbps = 100; $plan->disk_gib = 20; $plan->validity_days = 30; $plan->traffic_quota_bytes = 1024 * 1024 * 1024; $plan->status = 1; $plan->sort = 0; $plan->save();
$soldOutPlan = new Shop(); $soldOutPlan->name = 'sold-out-plan'; $soldOutPlan->cpu_cores = 2; $soldOutPlan->memory_mib = 2048; $soldOutPlan->bandwidth_mbps = 100; $soldOutPlan->disk_gib = 20; $soldOutPlan->validity_days = 30; $soldOutPlan->traffic_quota_bytes = 1024 * 1024 * 1024; $soldOutPlan->status = 1; $soldOutPlan->sort = 0; $soldOutPlan->save();
$offer = new CloudOffer(); $offer->plan_id = $plan->id; $offer->resource_pool_id = $pool->id; $offer->code = 'test-offer'; $offer->name = 'Test Offer'; $offer->price = '9.99'; $offer->backup_price = '2.50'; $offer->currency = 'USD'; $offer->price_version = 1; $offer->status = 1; $offer->stock = 10; $offer->sort = 0; $offer->created_at = $offer->updated_at = $now; $offer->save();
$soldOutOffer = new CloudOffer(); $soldOutOffer->plan_id = $soldOutPlan->id; $soldOutOffer->resource_pool_id = $pool->id; $soldOutOffer->code = 'sold-out-offer'; $soldOutOffer->name = 'Sold Out Offer'; $soldOutOffer->price = '8.99'; $soldOutOffer->backup_price = '0.00'; $soldOutOffer->currency = 'USD'; $soldOutOffer->price_version = 1; $soldOutOffer->status = 1; $soldOutOffer->stock = 0; $soldOutOffer->sort = 0; $soldOutOffer->created_at = $soldOutOffer->updated_at = $now; $soldOutOffer->save();
$template = new CloudTemplate();
$template->id = 'alma10-1234abcd-0123456789abcdef';
$template->name = 'AlmaLinux 10';
$template->os_family = 'AlmaLinux';
$template->os_version = '10';
$template->image_bytes = 1024;
$template->sha256 = str_repeat('a', 64);
$template->startup_script = '';
$template->revision = 1;
$template->generation_token = str_repeat('b', 32);
$template->state = 'active';
$template->created_at = $template->updated_at = $now;
$template->save();
$reinstallTemplate = new CloudTemplate();
$reinstallTemplate->id = 'debian13-1234abcd-0123456789abcdef';
$reinstallTemplate->name = 'Debian 13';
$reinstallTemplate->os_family = 'Debian';
$reinstallTemplate->os_version = '13';
$reinstallTemplate->image_bytes = 2048;
$reinstallTemplate->sha256 = str_repeat('c', 64);
$reinstallTemplate->startup_script = '';
$reinstallTemplate->revision = 1;
$reinstallTemplate->generation_token = str_repeat('d', 32);
$reinstallTemplate->state = 'active';
$reinstallTemplate->created_at = $reinstallTemplate->updated_at = $now;
$reinstallTemplate->save();
$capsule->connection('default')->table('cloud_template_replicas')->insert([
'host_id' => $host->id,
'template_id' => $template->id,
'revision' => $template->revision,
'generation_token' => $template->generation_token,
'state' => 'active',
'image_bytes' => $template->image_bytes,
'sha256' => $template->sha256,
'operation_id' => 'fixture-template-import',
'error_code' => '',
'created_at' => $now,
'updated_at' => $now,
]);
$capsule->connection('default')->table('cloud_template_replicas')->insert([
'host_id' => $host->id,
'template_id' => $reinstallTemplate->id,
'revision' => $reinstallTemplate->revision,
'generation_token' => $reinstallTemplate->generation_token,
'state' => 'active',
'image_bytes' => $reinstallTemplate->image_bytes,
'sha256' => $reinstallTemplate->sha256,
'operation_id' => 'fixture-reinstall-template-import',
'error_code' => '',
'created_at' => $now,
'updated_at' => $now,
]);
$actor = ['user_id' => 1, 'is_admin' => false, 'is_super_admin' => false, 'can_manage' => false, 'session_id' => ''];
$input = ['offer_id' => 1, 'source' => 'cloud', 'name' => 'vm-one', 'template_id' => 'alma10-1234abcd-0123456789abcdef', 'idempotency_key' => 'test-request-key-0001'];
$catalogConnection = Capsule::connection('default');
$catalogConnection->flushQueryLog();
$catalogConnection->enableQueryLog();
$catalog = CloudDeploymentPageService::saleCatalog();
$catalogQueryCount = count($catalogConnection->getQueryLog());
$catalogConnection->disableQueryLog();
$assert(count($catalog['offers'] ?? []) === 2
&& ($catalog['offers'][0]['sellable'] ?? null) === true
&& ($catalog['offers'][1]['sellable'] ?? null) === false
&& $catalogQueryCount <= 10,
'Catalog availability hid a sold-out offer, misclassified capacity, or queried per offer.');
$host->default_network = 'public';
$host->save();
$catalogWithoutLegacyDefault = CloudDeploymentPageService::saleCatalog();
$assert(($catalogWithoutLegacyDefault['offers'][0]['sellable'] ?? null)
=== true,
'A retired host default network still controlled sale availability.');
$attachment->network_selector = 'public';
$attachment->save();
$targetAttachment->network_selector = 'public';
$targetAttachment->save();
$unverifiedSelectorCatalog = CloudDeploymentPageService::saleCatalog();
$assert(($unverifiedSelectorCatalog['offers'][0]['sellable'] ?? null)
=== false,
'An attachment outside current Agent evidence remained sellable.');
$attachment->network_selector = 'bridge:br0';
$attachment->verified_agent_revision = 0;
$attachment->save();
$targetAttachment->network_selector = 'bridge:br1';
$targetAttachment->verified_agent_revision = 0;
$targetAttachment->save();
$staleEvidenceCatalog = CloudDeploymentPageService::saleCatalog();
$assert(($staleEvidenceCatalog['offers'][0]['sellable'] ?? null) === false,
'A stale attachment verification revision remained sellable.');
$host->default_network = 'bridge:br0';
$host->save();
$attachment->network_selector = 'bridge:br0';
$attachment->verified_agent_revision = 1;
$attachment->save();
$targetAttachment->network_selector = 'bridge:br1';
$targetAttachment->verified_agent_revision = 1;
$targetAttachment->save();
try {
CloudOrderService::purchase($input + ['network' => 'bridge:attacker'], $actor);
throw new RuntimeException('Purchase accepted a user-selected host network.');
} catch (CloudError $error) {
$assert($error->httpStatus === 422,
'User network injection returned the wrong status.');
}
$membership->scheduler_state = 'disabled';
$membership->updated_at = time();
$membership->save();
$membershipUnavailable = CloudDeploymentPageService::saleCatalog();
$assert(($membershipUnavailable['offers'][0]['sellable'] ?? null) === false,
'Legacy host scheduling fields bypassed the canonical pool membership.');
$membership->scheduler_state = 'active';
$membership->updated_at = time();
$membership->save();
// Network application state is informational; panel capacity and the
// concrete IP ledger decide sellability. A failed engine apply leaves no
// ledger rows for new addresses, so placement still fails closed.
CloudPublicIp::query()->update(['state' => 'reserved', 'updated_at' => time()]);
$noAddressCatalog = CloudDeploymentPageService::saleCatalog();
$assert(($noAddressCatalog['offers'][0]['sellable'] ?? null) === false,
'A pool without an available public IP remained sellable.');
try {
CloudOrderService::purchase(array_replace($input, [
'name' => 'vm-without-ip',
'idempotency_key' => 'test-unavailable-address-0001',
]), $actor);
throw new RuntimeException('Missing public IP created an order.');
} catch (CloudError $e) {
$assert($e->httpStatus === 409,
'Missing public IP returned the wrong status.');
}
$assert(CloudOrder::count() === 0 && CloudInvoice::count() === 0
&& CloudOffer::find(1)->stock === 10,
'Address admission failure created finance state or consumed stock.');
CloudPublicIp::query()->update(['state' => 'available', 'updated_at' => time()]);
$migrationPlacementReservationCount = CloudResourceReservation::count();
$migrationPlacementClaimCount = Capsule::table(
'cloud_resource_reservation_ips'
)->count();
$migrationPlacementConnection = Capsule::connection('default');
$migrationPlacementConnection->beginTransaction();
try {
$migrationInstance = new CloudInstance();
$migrationInstance->host_id = (int) $host->id;
$migrationInstance->user_id = 1;
$migrationInstance->resource_pool_id = (int) $pool->id;
$migrationInstance->uuid =
'66666666-6666-4666-8666-666666666666';
$migrationInstance->name = 'migration-source-fixture';
$migrationInstance->status = 'running';
$migrationInstance->service_status = 'active';
$migrationInstance->network_status = 'active';
$migrationInstance->spec_json = json_encode([
'vcpu' => 2,
'memory_mib' => 2048,
'disk_gib' => 20,
], JSON_THROW_ON_ERROR);
$migrationInstance->created_at =
$migrationInstance->updated_at = time();
$migrationInstance->save();
$migrationRestore = new CloudBackupRestore();
$migrationRestore->restore_key = str_repeat('7', 32);
$migrationRestore->backup_id = null;
$migrationRestore->target_host_id = (int) $host->id;
$migrationRestore->target_instance_id = null;
$migrationRestore->reservation_id = null;
$migrationRestore->job_id = null;
$migrationRestore->operation_id = str_repeat('8', 32);
$migrationRestore->new_uuid =
'77777777-7777-4777-8777-777777777777';
$migrationRestore->new_name = 'migration-target-fixture';
$migrationRestore->state = 'reserved';
$migrationRestore->error_code = '';
$migrationRestore->ready_at = null;
$migrationRestore->created_at =
$migrationRestore->updated_at = time();
$migrationRestore->save();
$migrationPlacement = CloudPlacementService::
reserveMaintenanceMigrationTarget(
$migrationRestore,
1,
[
'cpu_cores' => 2,
'memory_mib' => 2048,
'disk_gib' => 20,
'bandwidth_mbps' => 100,
],
(int) $pool->id,
5
);
$migrationReservation = $migrationPlacement['reservation'];
$migrationNetwork = $migrationPlacement['public_network'];
$migrationRestore->reservation_id = (int) $migrationReservation->id;
$migrationRestore->updated_at = time();
$migrationRestore->save();
$migrationClaimSet = CloudPublicNetworkAllocationService::
currentClaimSet((int) $migrationReservation->id, 'held');
$preparedMigrationPlacement = CloudPlacementService::
preparedMaintenanceMigrationTarget(
$migrationRestore,
(int) $migrationInstance->id
);
$migrationIdentity = $preparedMigrationPlacement['identity'];
$migrationIdentityValue = json_decode(
$migrationIdentity->identityJson(),
true,
16,
JSON_THROW_ON_ERROR
);
$assert((string) $migrationReservation->state === 'held'
&& $migrationReservation->instance_id === null
&& $migrationReservation->network_attachment_id === null
&& $migrationReservation->public_ip_pool_id === null
&& $migrationReservation->public_ip_id === null
&& (int) $migrationReservation->disk_gib === 25
&& (int) $migrationPlacement['target_agent_revision'] === 1
&& $migrationNetwork->toArray()
=== $migrationClaimSet->publicNetwork()->toArray()
&& (int) $preparedMigrationPlacement['reservation']->id
=== (int) $migrationReservation->id
&& $preparedMigrationPlacement['public_network']->toArray()
=== $migrationNetwork->toArray()
&& (int) $preparedMigrationPlacement['target_agent_revision'] === 1
&& hash_equals(
$portableRestoreCompatibility->sha256(),
(string) $preparedMigrationPlacement[
'target_agent_contract_sha256'
]
)
&& (int) ($migrationIdentityValue['reservation_id'] ?? 0)
=== (int) $migrationReservation->id
&& (int) ($migrationIdentityValue['host_id'] ?? 0)
=== (int) $host->id
&& (int) ($migrationIdentityValue[
'network_attachment_id'
] ?? 0) === $migrationClaimSet->networkAttachmentId()
&& ($migrationIdentityValue['uuid'] ?? null)
=== strtolower((string) $migrationRestore->new_uuid)
&& hash_equals(
$migrationIdentity->claimSetSha256(),
$migrationClaimSet->digest()
)
&& hash_equals(
$migrationIdentity->publicNetworkSha256(),
$migrationNetwork->fingerprint()
)
&& $migrationClaimSet->publicNetwork()->address(4) !== ''
&& $migrationClaimSet->publicNetwork()->address(6) !== '',
'Maintenance migration target did not reserve one exact held dual-stack bundle.');
$ipv6MigrationClaim = Capsule::table(
'cloud_resource_reservation_ips'
)->where('reservation_id', (int) $migrationReservation->id)
->where('address_family', 6)->first();
$tamperedOperation = str_repeat('9', 32);
Capsule::table('cloud_resource_reservation_ips')
->where('id', (int) $ipv6MigrationClaim->id)->update([
'operation_id' => $tamperedOperation,
]);
Capsule::table('cloud_public_ips')
->where('id', (int) $ipv6MigrationClaim->public_ip_id)->update([
'operation_id' => $tamperedOperation,
]);
$rejectedMigrationTargetDrift = false;
try {
CloudPlacementService::preparedMaintenanceMigrationTarget(
$migrationRestore,
(int) $migrationInstance->id
);
} catch (CloudError $error) {
$rejectedMigrationTargetDrift = $error->httpStatus === 409;
}
$assert($rejectedMigrationTargetDrift,
'Maintenance migration accepted a drifted IPv6 target claim.');
} finally {
$migrationPlacementConnection->rollBack();
}
$assert(CloudResourceReservation::count()
=== $migrationPlacementReservationCount
&& Capsule::table('cloud_resource_reservation_ips')->count()
=== $migrationPlacementClaimCount,
'Rolled-back migration target placement leaked a reservation or address claim.');
$first = CloudOrderService::purchase($input, $actor);
$same = CloudOrderService::purchase($input, $actor);
$firstSpec = json_decode(
(string) $first->spec_json,
true,
64,
JSON_THROW_ON_ERROR
);
$openingInvoice = CloudInvoice::where('order_id', $first->id)->first();
$openingDisplay = $openingInvoice === null ? []
: CloudInvoiceService::display($openingInvoice);
$connection = Capsule::connection('default');
$connection->flushQueryLog();
$connection->enableQueryLog();
$openingSummaries = $openingInvoice === null ? []
: CloudInvoiceService::listSummaries(array_fill(0, 25, $openingInvoice));
$summaryQueryCount = count($connection->getQueryLog());
$connection->disableQueryLog();
$assert($first->id === $same->id
&& (string) $first->status === 'awaiting_payment'
&& $openingInvoice !== null
&& (string) $openingInvoice->purpose === 'provision'
&& $openingInvoice->instance_id === null
&& (int) $openingInvoice->due_at - (int) $openingInvoice->created_at === 3600
&& ($openingDisplay['purpose'] ?? null) === 'provision'
&& ($openingDisplay['title'] ?? null) === '实例开通'
&& ($openingDisplay['type_label'] ?? null) === '实例开通'
&& ($openingDisplay['cpu_cores'] ?? null) === 2
&& ($openingDisplay['memory_mib'] ?? null) === 2048
&& !array_key_exists('invoice_no', $openingDisplay)
&& ($openingDisplay['order_status'] ?? null) === 'awaiting_payment'
&& ($firstSpec['stock_mode'] ?? null) === 'finite'
&& array_key_exists('traffic_pool_group_id', $firstSpec)
&& $firstSpec['traffic_pool_group_id'] === null
&& ($firstSpec['traffic_quota_mode'] ?? null) === 'finite'
&& ($firstSpec['automatic_backup'] ?? null) === false
&& ($firstSpec['instance_price'] ?? null) === '9.99'
&& ($firstSpec['backup_price'] ?? null) === '0.00'
&& count($openingDisplay['items'] ?? []) === 1
&& ($openingDisplay['items'][0]['code'] ?? null) === 'instance'
&& CloudOrderItem::where('order_id', $first->id)->count() === 1
&& $first->stock_reserved_at !== null
&& $first->stock_released_at === null
&& User::find(1)->money === '0.00'
&& CloudJob::count() === 0
&& CloudOffer::find(1)->stock === 9,
'Idempotent deployment request charged money, queued work, lost its opening invoice, or reserved stock twice.');
$assert(count($openingSummaries) === 25
&& $summaryQueryCount <= 3
&& array_keys($openingSummaries[0] ?? []) === [
'id', 'title', 'type_label', 'purpose', 'amount', 'currency',
'status', 'display_status', 'available_action', 'payable',
'renews_expires_at', 'due_at', 'payment_deadline_at', 'created_at',
'order_status',
'fulfillment_status', 'task_id', 'job_status',
'instance_id', 'instance_name',
]
&& ($openingSummaries[0]['order_status'] ?? null) === 'awaiting_payment',
'Invoice list summaries are not compact or use per-invoice association queries.');
try { CloudOrderService::purchase($input + ['vcpu' => 64], $actor); throw new RuntimeException('Resource bypass accepted.'); } catch (CloudError $e) {}
try { CloudOrderService::purchase(array_replace($input, ['name' => 'changed']), $actor); throw new RuntimeException('Idempotency conflict accepted.'); } catch (CloudError $e) { $assert($e->httpStatus === 409, 'Wrong idempotency response.'); }
try {
CloudInvoiceService::payBalance($openingInvoice,
array_replace($actor, ['user_id' => 2]), 'provision-cross-tenant-0001');
throw new RuntimeException('Another tenant paid an opening invoice.');
} catch (CloudError $e) {
$assert($e->httpStatus === 403, 'Opening invoice ownership failure leaked another status.');
}
$assert(User::find(1)->money === '0.00'
&& CloudOrder::find($first->id)->status === 'awaiting_payment'
&& CloudJob::count() === 0,
'Cross-tenant opening-invoice access changed financial or queue state.');
try {
CloudInvoiceService::payBalance(
$openingInvoice, $actor, 'provision-insufficient-balance-0001');
throw new RuntimeException('Opening invoice was paid without sufficient balance.');
} catch (CloudError $e) {
$assert($e->httpStatus === 409, 'Insufficient opening-invoice balance returned the wrong status.');
}
$assert(CloudOrder::find($first->id)->status === 'awaiting_payment'
&& CloudInvoice::find($openingInvoice->id)->status === 'open'
&& CloudJob::count() === 0,
'Failed opening-invoice payment changed invoice, order, or queue state.');
$user = User::find(1); $user->money = '100.00'; $user->save();
$attachment->verified_agent_revision = 0;
$attachment->save();
$targetAttachment->verified_agent_revision = 0;
$targetAttachment->save();
try {
CloudInvoiceService::payBalance(
$openingInvoice,
$actor,
'provision-stale-network-evidence-0001'
);
throw new RuntimeException(
'Payment admitted stale attachment verification evidence.'
);
} catch (CloudError $error) {
$assert($error->httpStatus === 409,
'Stale attachment evidence returned the wrong payment status.');
}
$assert(User::find(1)->money === '100.00'
&& CloudInvoice::find($openingInvoice->id)->status === 'open'
&& CloudOrder::find($first->id)->status === 'awaiting_payment'
&& CloudResourceReservation::count() === 0
&& CloudJob::count() === 0,
'Failed network admission retained money, a reservation, or queued work.');
$attachment->verified_agent_revision = 1;
$attachment->save();
$targetAttachment->verified_agent_revision = 1;
$targetAttachment->save();
// Capacity must remain tied to the immutable order snapshot even if an
// administrator changes the base plan while the invoice is still open.
$changedPlan = Shop::find($plan->id);
$changedPlan->cpu_cores = 8;
$changedPlan->memory_mib = 8192;
$changedPlan->disk_gib = 80;
$changedPlan->save();
$policyManager = [
'user_id' => 99, 'is_admin' => true,
'is_super_admin' => true, 'can_manage' => true,
];
$testCatalog = CloudPlatformTestNetworkPolicy::create([
'expected_revision' => 0,
'service' => 'Provision snapshot test',
'protocol' => 'tcp',
'port' => '8080',
'direction' => 'ingress',
'reason' => 'Verify queued creates retain deleted catalog rules.',
], $policyManager);
$testRuleKey = (string) $testCatalog['rules'][0]['id'];
$host->default_network = 'public';
$host->save();
$paidOpening = CloudInvoiceService::payBalance(
$openingInvoice, $actor, 'provision-balance-test-0001');
$paidOpeningAgain = CloudInvoiceService::payBalance(
$paidOpening->fresh(), $actor, 'provision-balance-test-0001');
$host->default_network = 'bridge:br0';
$host->save();
$openingOrder = CloudOrder::find($first->id);
$openingReservation = CloudResourceReservation::where('order_id', $first->id)->first();
$openingJob = CloudJob::where('order_id', $first->id)->first();
$queuedCreate = \App\Services\Cloud\CloudSecrets::decrypt(
(string) $openingJob->sealed_payload,
'job-payload'
);
$queuedPolicy = $queuedCreate['body']['platform_network_policy_snapshot'] ?? null;
$orderPolicy = json_decode(
(string) $openingOrder->spec_json, true,
flags: JSON_THROW_ON_ERROR
)['platform_network_policy_snapshot'] ?? null;
$openingSpec = json_decode(
(string) $openingOrder->spec_json, true,
flags: JSON_THROW_ON_ERROR
);
CloudPlatformTestNetworkPolicy::delete(
$testRuleKey, ['expected_revision' => 1], $policyManager
);
$currentPolicy = CloudPlatformTestNetworkPolicy::snapshotForProvisioning();
$sealedJobAfterDelete = CloudJob::where('order_id', $first->id)->first();
$sealedCreateAfterDelete = \App\Services\Cloud\CloudSecrets::decrypt(
(string) $sealedJobAfterDelete->sealed_payload,
'job-payload'
);
$queuedPolicyAfterDelete = $sealedCreateAfterDelete['body']
['platform_network_policy_snapshot'] ?? null;
$orderPolicyAfterDelete = json_decode(
(string) CloudOrder::find($first->id)->spec_json, true,
flags: JSON_THROW_ON_ERROR
)['platform_network_policy_snapshot'] ?? null;
$openingClaims = Capsule::table('cloud_resource_reservation_ips')
->where('reservation_id', (int) ($openingReservation->id ?? 0))
->orderBy('address_family')->get();
$assert((string) $paidOpeningAgain->status === 'paid'
&& User::find(1)->money === '90.01'
&& $openingOrder->status === 'queued'
&& (int) $openingOrder->host_id === (int) $host->id
&& $openingReservation !== null
&& $openingReservation->state === 'held'
&& (int) $openingReservation->cpu_cores === 2
&& (int) $openingReservation->memory_mib === 2048
&& (int) $openingReservation->disk_gib === 20
&& $openingReservation->network_attachment_id === null
&& $openingReservation->public_ip_pool_id === null
&& $openingReservation->public_ip_id === null
&& $openingClaims->count() === 2
&& $openingClaims->pluck('address_family')->all() === [4, 6]
&& $openingClaims->every(
static fn ($claim): bool => (string) $claim->state === 'reserved'
)
&& (string) $openingJob->operation_id === (string) $openingReservation->operation_id
&& CloudPublicIp::whereIn('id', $openingClaims->pluck('public_ip_id'))
->where('state', 'reserved')->count() === 2
&& CloudJob::where('order_id', $first->id)->count() === 1
&& is_array($queuedPolicy)
&& $queuedPolicy === $orderPolicy
&& ($queuedPolicy['catalog_revision'] ?? null) === 1
&& ($queuedPolicy['rules'][9]['id'] ?? null) === 'test-' . $testRuleKey
&& $queuedPolicyAfterDelete === $queuedPolicy
&& $orderPolicyAfterDelete === $queuedPolicy
&& ($currentPolicy['catalog_revision'] ?? null) === 2
&& count($currentPolicy['rules']) === 9
&& ($openingSpec['template_id'] ?? null) === $template->id
&& ($openingSpec['os_source'] ?? null) === 'cloud_template'
&& ($openingSpec['os_name'] ?? null) === 'AlmaLinux 10'
&& ($openingSpec['login_name'] ?? null) === 'root'
&& ($openingSpec['public_network']['ipv4']['address_cidr'] ?? null)
=== '8.8.8.10/24'
&& ($openingSpec['public_network']['ipv6']['address_cidr'] ?? null)
=== '2404:c140:156:3a::10/64'
&& !array_key_exists('managed', $openingSpec)
&& CloudPlatformTestNetworkPolicy::assertSnapshot($queuedPolicy)
=== $queuedPolicy
&& $openingOrder->sealed_payload === null,
'Opening payment did not atomically freeze an immutable per-instance platform policy.');
$changedPlan->cpu_cores = 2;
$changedPlan->memory_mib = 2048;
$changedPlan->disk_gib = 20;
$changedPlan->save();
$history = [];
$openingIdentity = CloudOperationRecoveryIdentity::expected(
$openingJob,
true
);
$createdVm = [
'uuid' => $openingIdentity['resource_id'],
'name' => '1:vm-111111111111',
'vcpu' => 2,
'memory_mib' => 2048,
'disk_gib' => 20,
'disks' => [[
'device' => 'disk',
'format' => 'qcow2',
]],
'autostart' => true,
'public_network' => $openingSpec['public_network'],
'console_port' => 5908,
'generated_password' => 'must-not-persist',
];
$mismatchedNetwork = $openingSpec['public_network'];
$mismatchedNetwork['ipv4']['address_cidr'] = '8.8.8.11/24';
$mismatchedVm = array_replace($createdVm, [
'public_network' => $mismatchedNetwork,
]);
$mock = new MockHandler([
new Response(200, [], json_encode([
'operation_id' => $openingJob->operation_id,
'status' => 'succeeded', 'result' => $createdVm,
...$openingIdentity,
])),
]);
$stack = HandlerStack::create($mock); $stack->push(Middleware::history($history));
$worker = new CloudWorkerService(new CloudAgentClient(
'1',
'test-secret-which-is-at-least-32-bytes',
new Client(['handler' => $stack]),
20.0,
'http://127.0.0.1:8001/internal/agent/request',
'test-gateway-secret-at-least-32-bytes'
));
$worker->tick();
$openingJob->refresh();
$openingReservation->refresh();
$assert(count($history) === 1
&& $openingJob->phase === 'terminal'
&& $openingJob->outcome === 'succeeded'
&& $openingJob->dispatch_attempts === 1,
'Provisioning did not converge after exactly one Agent mutation dispatch.');
$openingDisplay = CloudInvoiceService::display($openingInvoice->fresh());
$openingJobDisplay = CloudJobService::display(CloudJob::where('order_id', $first->id)->first());
$assert(CloudOrder::find($first->id)->status === 'succeeded' && CloudInstance::count() === 1
&& \App\Models\CloudServicePeriod::count() === 1
&& \App\Models\CloudServicePeriod::first()->traffic_pool_group_id === null
&& (string) \App\Models\CloudServicePeriod::first()->traffic_quota_mode
=== 'finite'
&& CloudResourceReservation::where('order_id', $first->id)->first()->state === 'committed'
&& CloudPublicIp::whereIn('id', $openingClaims->pluck('public_ip_id'))
->where('state', 'assigned')->whereNotNull('instance_id')
->count() === 2
&& CloudInvoice::find($openingInvoice->id)->instance_id !== null
&& CloudMailOutbox::where('invoice_id', $openingInvoice->id)
->where('kind', 'provision_succeeded')->count() === 1
&& ($openingDisplay['order_status'] ?? null) === 'succeeded'
&& ($openingDisplay['instance_id'] ?? null) !== null
&& ($openingDisplay['job_status'] ?? null) === 'succeeded',
'Successful provision did not create its first service period or update the opening bill.');
$assert(!CloudInvoiceService::queueProvisionSucceeded(
CloudInvoice::find($openingInvoice->id), time())
&& CloudMailOutbox::where('invoice_id', $openingInvoice->id)
->where('kind', 'provision_succeeded')->count() === 1,
'Successful provisioning notification was not idempotent.');
$assert(($openingJobDisplay['completed'] ?? null) === true
&& ($openingJobDisplay['status'] ?? null) === 'succeeded'
&& ($openingJobDisplay['status_url'] ?? null)
=== '/api/v1/tasks/' . $openingJob->id
&& ($openingJobDisplay['created_at'] ?? 0) > 0
&& ($openingJobDisplay['terminal_at'] ?? 0) > 0
&& !array_key_exists('attempts', $openingJobDisplay),
'User-visible creation receipt lacks a strict terminal result or leaks retry metadata.');
$openingAgentRequest = $agentRequest($history[0]);
$posted = json_decode($openingAgentRequest['body'], true);
$signedActor = json_decode(base64_decode(strtr(
$openingAgentRequest['headers']['X-Virt-Actor'], '-_', '+/'
)), true);
$assert($posted['vcpu'] === 2 && $posted['bandwidth_mbps'] === 100
&& !array_key_exists('network', $posted)
&& ($posted['public_network'] ?? null)
=== $openingSpec['public_network']
&& ($posted['platform_network_policy_snapshot'] ?? null)
=== $queuedPolicy
&& $signedActor['can_manage'] === true
&& ($signedActor['operation_id'] ?? null) === $openingJob->operation_id
&& !isset($signedActor['can_provision']),
'Plan snapshot or typed Agent provisioning authority was lost.');
$assert(!str_contains(CloudJob::first()->result_json, 'must-not-persist'), 'Plain password saved in result.');
$terminalPayload = \App\Services\Cloud\CloudSecrets::decrypt(
CloudJob::first()->sealed_payload,
'job-payload'
);
$assert(!array_key_exists('credential', $terminalPayload)
&& !array_key_exists('root_password', $terminalPayload['body'] ?? []),
'Terminal provisioning job retained a recoverable root password.');
$instance = CloudInstance::first();
$instanceSpec = json_decode((string) $instance->spec_json, true,
flags: JSON_THROW_ON_ERROR);
$assert($instance->name === 'vm-one'
&& ($instanceSpec['public_network'] ?? null)
=== $openingSpec['public_network']
&& ($instanceSpec['os_source'] ?? null) === 'cloud_template'
&& ($instanceSpec['os_name'] ?? null) === 'AlmaLinux 10'
&& ($instanceSpec['login_name'] ?? null) === 'root'
&& ($instanceSpec['managed'] ?? null) === true
&& ($instanceSpec['agent_metadata_version'] ?? null) === 1
&& ($instanceSpec['disk_gib'] ?? null) === 20
&& !isset($instanceSpec['disks'])
&& !isset($instanceSpec['console_port'])
&& !isset($instanceSpec['disks'][0]['source']),
'Successful provisioning did not persist the safe Agent metadata snapshot.');
\App\Services\Cloud\CloudPlacementService::markUncertain($openingOrder->fresh());
$assert(CloudResourceReservation::find($openingReservation->id)->state === 'committed'
&& CloudPublicIp::whereIn('id', $openingClaims->pluck('public_ip_id'))
->where('state', 'assigned')->count() === 2,
'A late uncertainty marker rolled a committed reservation or assigned IP backward.');
// An unpaid provisioning reservation expires without entering the worker
// queue, changing the wallet, or retaining its encrypted credential.
$abandoned = CloudOrderService::purchase(array_replace($input, [
'name' => 'vm-abandoned', 'idempotency_key' => 'test-request-key-abandoned',
'automatic_backup' => true,
]), $actor);
$abandonedInvoice = CloudInvoice::where('order_id', $abandoned->id)->first();
$abandonedSpec = json_decode((string) $abandoned->spec_json, true,
flags: JSON_THROW_ON_ERROR);
$abandonedDisplay = CloudInvoiceService::display($abandonedInvoice);
$assert((string) $abandoned->amount === '12.49'
&& ($abandonedSpec['automatic_backup'] ?? null) === true
&& ($abandonedSpec['instance_price'] ?? null) === '9.99'
&& ($abandonedSpec['backup_price'] ?? null) === '2.50'
&& array_column($abandonedDisplay['items'] ?? [], 'code')
=== ['instance', 'automatic_backup'],
'Automatic backup was not frozen as a separate opening-bill item.');
$abandonedInvoice->due_at = time() - 1;
$abandonedInvoice->save();
$assert(CloudInvoiceService::expireProvision() === 1
&& CloudInvoiceService::expireProvision() === 0
&& CloudOrder::find($abandoned->id)->status === 'expired'
&& CloudOrder::find($abandoned->id)->sealed_payload === null
&& CloudOrder::find($abandoned->id)->stock_released_at !== null
&& CloudJob::where('order_id', $abandoned->id)->count() === 0
&& User::find(1)->money === '90.01'
&& CloudOffer::find(1)->stock === 9,
'Expired opening invoice queued work, changed money, retained credentials, or released stock more than once.');
$assert(CloudAccess::instance((string) $instance->id, $actor)->id === $instance->id, 'Owner denied own VM.');
$other = array_replace($actor, ['user_id' => 2]);
try { CloudAccess::instance((string) $instance->id, $other); throw new RuntimeException('Tenant isolation failed.'); } catch (CloudError $e) { $assert($e->httpStatus === 404, 'Nonowner information leaked.'); }
$admin = array_replace($other, ['is_admin' => true]);
$assert(CloudAccess::instance((string) $instance->id, $admin)->id === $instance->id, 'Admin readonly denied.');
try { CloudAccess::instance((string) $instance->id, $admin, true); throw new RuntimeException('Regular admin wrote other owner.'); } catch (CloudError $e) {}
try { CloudJobService::enqueue($instance, array_replace($admin, ['can_manage' => true]), 'action', 'POST', '/actions', ['action' => 'reboot']); throw new RuntimeException('Scoped can_manage bypassed cross-owner queue authorization.'); } catch (CloudError $e) {}
try { CloudAccess::manage($actor + ['can_provision' => true]); throw new RuntimeException('Provision actor globally privileged.'); } catch (CloudError $e) {}
// A backup-origin deployment must cross the same paid create ledger, but
// dispatch a signed restore-backup request and retain the exact backup
// identity until a deterministic Agent rejection safely refunds it.
$originalSourceSpec = (string) $instance->spec_json;
$sourceBackupSpec = json_decode(
$originalSourceSpec,
true,
64,
JSON_THROW_ON_ERROR
);
$sourceBackupSpec['automatic_backup'] = true;
$sourceBackupSpec['backup_price'] = '2.50';
$instance->spec_json = json_encode(
$sourceBackupSpec,
JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES
);
$instance->save();
$subscription = new CloudBackupSubscription();
$subscription->instance_id = (int) $instance->id;
$subscription->user_id = 1;
$subscription->offer_id = (int) $offer->id;
$subscription->enabled = true;
$subscription->renew_next_period = true;
$subscription->frequency = 'daily';
$subscription->weekday = 0;
$subscription->retention_count = 2;
$subscription->timezone = 'UTC';
$subscription->price = '2.50';
$subscription->currency = 'USD';
$subscription->price_version = 1;
$subscription->next_run_at = time() + 86400;
$subscription->created_at = $subscription->updated_at = time();
$subscription->save();
$capsule->table('cloud_backup_period_entitlements')->insert([
'subscription_id' => (int) $subscription->id,
'instance_id' => (int) $instance->id,
'service_period_id' => (int) $instance->current_period_id,
'user_id' => 1,
'offer_id' => (int) $offer->id,
'invoice_id' => (int) CloudInvoice::where(
'order_id', (int) $instance->order_id
)->value('id'),
'starts_at' => time() - 60,
'ends_at' => time() + 86400,
'amount' => '2.50',
'full_period_price' => '2.50',
'currency' => 'USD',
'price_version' => 1,
'source' => 'provision',
'paid_at' => time() - 60,
'created_at' => time() - 60,
'updated_at' => time() - 60,
]);
$sourceBackup = new CloudInstanceBackup();
$sourceBackup->backup_key = str_repeat('c', 32);
$sourceBackup->source_instance_id = (int) $instance->id;
$sourceBackup->source_host_id = (int) $host->id;
$sourceBackup->user_id = 1;
$sourceBackup->source_uuid = strtolower((string) $instance->uuid);
$sourceBackup->name = 'automatic-backup-fixture';
$sourceBackup->purpose = 'portable';
$sourceBackup->source = 'automatic';
$sourceBackup->subscription_id = (int) $subscription->id;
$sourceBackup->scheduled_for = time() - 60;
$sourceBackup->state = 'ready';
$sourceBackup->consistency = 'filesystem';
$sourceBackup->object_key = 'virt-cloud/backups/'
. $sourceBackup->backup_key . '/system.qcow2';
$sourceBackup->object_size_bytes = 1073741824;
$sourceBackup->virtual_size_bytes = 10 * 1073741824;
$sourceBackup->sha256 = str_repeat('d', 64);
$sourceBackup->spec_json = json_encode(
$sourceBackupSpec,
JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES
);
$sourceBackup->sealed_storage_runtime =
\App\Services\Cloud\CloudSecrets::encrypt([
'config' => [
'endpoint' => 'https://objects.example.test',
'region' => 'us-east-1',
'bucket' => 'virt-test',
'path_style' => true,
],
'credentials' => [
'access_key_id' => 'test-access-key',
'secret_access_key' => 'test-secret-key',
],
'revision' => 1,
], 'object-storage-backup-runtime');
$sourceBackup->sealed_credential = $instance->sealed_credential;
$sourceBackup->storage_revision = 1;
$sourceBackup->source_portable_restore_contract_json =
$portableRestoreCompatibility->json();
$sourceBackup->source_portable_restore_contract_sha256 =
$portableRestoreCompatibility->sha256();
$sourceBackup->ready_at = time();
$sourceBackup->created_at = $sourceBackup->updated_at = time();
$sourceBackup->save();
$orderCountBeforeDirectBackupRequest = CloudOrder::count();
try {
CloudOrderService::purchase([
'offer_id' => (int) $offer->id,
'source' => 'backup',
'backup_id' => (int) $sourceBackup->id,
'name' => 'forbidden-backup-source',
'automatic_backup' => false,
'idempotency_key' => 'test-direct-backup-source-0001',
], $actor);
throw new RuntimeException(
'A cloud backup was accepted as a public deployment source.'
);
} catch (CloudError $error) {
$assert(str_contains($error->getMessage(), '云快照'),
'Direct backup deployment did not direct the user to cloud snapshots.');
}
$assert(CloudOrder::count() === $orderCountBeforeDirectBackupRequest,
'Rejected direct backup deployment created finance state.');
// The guest system disk is authoritative. A restored source may no longer
// have a trustworthy panel credential envelope and must still be usable
// as a cloud snapshot deployment source.
$sourceBackup->sealed_credential = null;
$sourceBackup->updated_at = time();
$sourceBackup->save();
$deploymentSnapshot = new CloudDeploymentSnapshot();
$deploymentSnapshot->snapshot_key = str_repeat('e', 32);
$deploymentSnapshot->user_id = 1;
$deploymentSnapshot->backup_id = (int) $sourceBackup->id;
$deploymentSnapshot->source_instance_id = (int) $instance->id;
$deploymentSnapshot->name = 'deployment-snapshot-fixture';
$deploymentSnapshot->state = 'ready';
$deploymentSnapshot->idempotency_key =
'snapshot-promotion-test-0001';
$deploymentSnapshot->request_hash = str_repeat('f', 64);
$deploymentSnapshot->error_code = '';
$deploymentSnapshot->ready_at = time();
$deploymentSnapshot->created_at =
$deploymentSnapshot->updated_at = time();
$deploymentSnapshot->save();
$contractGateOrderCount = CloudOrder::count();
$contractGateInvoiceCount = CloudInvoice::count();
$sourceBackup->source_portable_restore_contract_json = '{}';
$sourceBackup->source_portable_restore_contract_sha256 = '';
$sourceBackup->save();
try {
CloudOrderService::purchase([
'offer_id' => (int) $offer->id,
'source' => 'deployment_snapshot',
'cloud_snapshot_key' => $deploymentSnapshot->snapshot_key,
'name' => 'vm-incompatible-backup',
'automatic_backup' => false,
'idempotency_key' => 'snapshot-source-contract-missing-0001',
], $actor);
throw new RuntimeException(
'Cloud snapshot without source compatibility evidence was sold.'
);
} catch (CloudError $error) {
$assert($error->httpStatus === 409,
'Missing source compatibility evidence returned the wrong status.');
}
$sourceBackup->source_portable_restore_contract_json =
$portableRestoreCompatibility->json();
$sourceBackup->source_portable_restore_contract_sha256 =
$portableRestoreCompatibility->sha256();
$sourceBackup->save();
$assert(CloudOrder::count() === $contractGateOrderCount
&& CloudInvoice::count() === $contractGateInvoiceCount,
'Rejected incompatible cloud snapshot created finance state.');
$snapshotOrderCount = CloudOrder::count();
$snapshotInvoiceCount = CloudInvoice::count();
$snapshotJobCount = CloudJob::count();
$snapshotReservationCount = CloudResourceReservation::count();
$snapshotBalance = (string) User::findOrFail(1)->money;
$snapshotStock = (int) CloudOffer::findOrFail($offer->id)->stock;
$connection = Capsule::connection('default');
$connection->beginTransaction();
try {
$snapshotOrder = CloudOrderService::purchase([
'offer_id' => (int) $offer->id,
'source' => 'deployment_snapshot',
'cloud_snapshot_key' => $deploymentSnapshot->snapshot_key,
'name' => 'vm-two',
'automatic_backup' => false,
'idempotency_key' => 'test-request-key-0002',
], $actor);
$snapshotReplay = CloudOrderService::purchase([
'offer_id' => (int) $offer->id,
'source' => 'deployment_snapshot',
'cloud_snapshot_key' => $deploymentSnapshot->snapshot_key,
'name' => 'vm-two',
'automatic_backup' => false,
'idempotency_key' => 'test-request-key-0002',
], $actor);
$snapshotInvoice = CloudInvoice::where(
'order_id', (int) $snapshotOrder->id
)->firstOrFail();
$snapshotOrderSpec = json_decode(
(string) $snapshotOrder->spec_json,
true,
64,
JSON_THROW_ON_ERROR
);
$assert((int) $snapshotReplay->id === (int) $snapshotOrder->id
&& (string) $snapshotOrder->status === 'awaiting_payment'
&& (int) $snapshotOrder->deployment_snapshot_id
=== (int) $deploymentSnapshot->id
&& (int) ($snapshotOrderSpec['backing_backup_id'] ?? 0)
=== (int) $sourceBackup->id
&& ($snapshotOrderSpec['cloud_snapshot_key'] ?? null)
=== (string) $deploymentSnapshot->snapshot_key
&& CloudOrder::count() === $snapshotOrderCount + 1
&& CloudInvoice::count() === $snapshotInvoiceCount + 1
&& CloudJob::count() === $snapshotJobCount
&& CloudResourceReservation::count()
=== $snapshotReservationCount
&& (string) User::findOrFail(1)->money === $snapshotBalance,
'Unpaid cloud-snapshot purchase was not idempotent or reserved runtime resources.');
Capsule::table('cloud_host_agents')->where(
'host_id', (int) $host->id
)->update([
'portable_restore_contract_json' => '{}',
'portable_restore_contract_sha256' => '',
'updated_at' => time(),
]);
try {
CloudInvoiceService::payBalance(
$snapshotInvoice,
$actor,
'snapshot-incompatible-target-0002'
);
throw new RuntimeException(
'Cloud snapshot payment selected an incompatible target Agent.'
);
} catch (CloudError $error) {
$assert($error->httpStatus === 409,
'Incompatible target Agent returned the wrong status.');
}
$snapshotOrder->refresh();
$snapshotInvoice->refresh();
$assert((string) $snapshotOrder->status === 'awaiting_payment'
&& (string) $snapshotInvoice->status === 'open'
&& CloudJob::count() === $snapshotJobCount
&& CloudResourceReservation::count()
=== $snapshotReservationCount
&& (string) User::findOrFail(1)->money === $snapshotBalance,
'Rejected target compatibility changed money, queue, or placement state.');
Capsule::table('cloud_host_agents')->where(
'host_id', (int) $host->id
)->update([
'portable_restore_contract_json' =>
$portableRestoreCompatibility->json(),
'portable_restore_contract_sha256' =>
$portableRestoreCompatibility->sha256(),
'updated_at' => time(),
]);
// Payment must retain the prepaid order's 20 GiB target even if an
// administrator edits the mutable plan before the invoice is paid.
$snapshotChangedPlan = Shop::findOrFail((int) $plan->id);
$snapshotChangedPlan->disk_gib = 5;
$snapshotChangedPlan->save();
CloudInvoiceService::payBalance(
$snapshotInvoice,
$actor,
'snapshot-provision-balance-0002'
);
$snapshotOrder->refresh();
$snapshotInvoice->refresh();
$snapshotJob = CloudJob::where(
'order_id', (int) $snapshotOrder->id
)->firstOrFail();
$snapshotReservation = CloudResourceReservation::where(
'order_id', (int) $snapshotOrder->id
)->firstOrFail();
$snapshotClaims = Capsule::table(
'cloud_resource_reservation_ips'
)->where('reservation_id', (int) $snapshotReservation->id)
->orderBy('address_family')->get();
$snapshotPayload = \App\Services\Cloud\CloudSecrets::decrypt(
(string) $snapshotJob->sealed_payload,
'job-payload'
);
$snapshotQueuedSpec = json_decode(
(string) $snapshotOrder->spec_json,
true,
64,
JSON_THROW_ON_ERROR
);
$snapshotNetwork = $snapshotQueuedSpec['public_network'] ?? null;
$snapshotRequestTarget = $snapshotPayload['request_target'] ?? null;
$assert((string) $snapshotOrder->status === 'queued'
&& (string) $snapshotInvoice->status === 'paid'
&& $snapshotOrder->sealed_payload === null
&& (string) $snapshotJob->kind === 'create'
&& (int) $snapshotJob->backup_id === (int) $sourceBackup->id
&& is_string($snapshotRequestTarget)
&& preg_match(
'#^/v1/vms/[a-f0-9-]{36}/restore-backup$#D',
$snapshotRequestTarget
) === 1
&& ($snapshotPayload['body']['backup_key'] ?? null)
=== (string) $sourceBackup->backup_key
&& ($snapshotPayload['body']['source_sha256'] ?? null)
=== (string) $sourceBackup->sha256
&& ($snapshotPayload['body']['disk_gib'] ?? null) === 20
&& ($snapshotQueuedSpec['target_agent_revision'] ?? null) === 1
&& ($snapshotQueuedSpec['target_agent_contract_sha256'] ?? null)
=== $portableRestoreCompatibility->sha256()
&& ($snapshotPayload['backup_metadata'][
'target_agent_revision'
] ?? null) === 1
&& ($snapshotPayload['backup_metadata'][
'target_agent_contract_sha256'
] ?? null) === $portableRestoreCompatibility->sha256()
&& (int) $snapshotReservation->disk_gib === 20
&& is_string($snapshotPayload['body']['source_url'] ?? null)
&& str_starts_with(
$snapshotPayload['body']['source_url'],
'https://objects.example.test/'
)
&& !array_key_exists('credential', $snapshotPayload)
&& is_array($snapshotNetwork)
&& ($snapshotPayload['body']['public_network'] ?? null)
=== $snapshotNetwork
&& ($snapshotNetwork['ipv4']['address_cidr'] ?? null)
=== '8.8.8.11/24'
&& ($snapshotNetwork['ipv6']['address_cidr'] ?? null)
=== '2404:c140:156:3a::11/64'
&& $snapshotClaims->count() === 2
&& $snapshotClaims->pluck('address_family')->all() === [4, 6]
&& $snapshotClaims->every(
static fn ($claim): bool =>
(string) $claim->state === 'reserved'
),
'Paid cloud-snapshot provisioning did not freeze one exact dual-stack restore task.');
$snapshotIdentity = CloudOperationRecoveryIdentity::expected(
$snapshotJob,
true
);
// Finalization must use the immutable target proof selected while the
// payment transaction held the Agent row lock. A later administrator
// edit may make the live Agent temporarily unready, but it cannot
// rewrite the identity of an already dispatched restore.
Capsule::table('cloud_host_agents')->where(
'host_id', (int) $host->id
)->update([
'revision' => 2,
'tested_revision' => 0,
'last_test_status' => 'never',
'portable_restore_contract_json' => '{}',
'portable_restore_contract_sha256' => '',
'updated_at' => time(),
]);
$snapshotCreatedVm = array_replace($createdVm, [
'uuid' => $snapshotIdentity['resource_id'],
'name' => '1:vm-snapshot-restore',
'public_network' => $snapshotNetwork,
'backup_key' => (string) $sourceBackup->backup_key,
'sha256' => (string) $sourceBackup->sha256,
]);
unset($snapshotCreatedVm['generated_password']);
$historyBeforeSnapshot = count($history);
$mock->append(new Response(200, [], json_encode([
'operation_id' => (string) $snapshotJob->operation_id,
'status' => 'succeeded',
'result' => $snapshotCreatedVm,
...$snapshotIdentity,
], JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES)));
$assert($worker->tick(),
'Paid cloud-snapshot provisioning task was not processed.');
$snapshotJob->refresh();
$snapshotOrder->refresh();
$snapshotReservation->refresh();
$snapshotInstance = CloudInstance::where(
'order_id', (int) $snapshotOrder->id
)->first();
$snapshotInstanceSpec = $snapshotInstance === null ? null
: json_decode(
(string) $snapshotInstance->spec_json,
true,
64,
JSON_THROW_ON_ERROR
);
$snapshotAgentRequest = $agentRequest(
$history[$historyBeforeSnapshot]
);
$snapshotPosted = json_decode(
$snapshotAgentRequest['body'],
true,
64,
JSON_THROW_ON_ERROR
);
$assert((string) $snapshotJob->phase === 'terminal'
&& (string) $snapshotJob->outcome === 'succeeded'
&& (int) $snapshotJob->dispatch_attempts === 1
&& (int) $snapshotJob->receipt_deadline_at
- (int) $snapshotJob->dispatched_at
=== CloudJobService::receiptDeadlineSeconds('backup_restore')
&& (string) $snapshotOrder->status === 'succeeded'
&& $snapshotInstance !== null
&& $snapshotInstance->sealed_credential === null
&& ($snapshotInstanceSpec['public_network'] ?? null)
=== $snapshotNetwork
&& ($snapshotInstanceSpec['restored_from_backup'] ?? null)
=== (string) $sourceBackup->backup_key
&& !array_key_exists('guest_hostname',
$snapshotInstanceSpec ?? [])
&& (string) $snapshotReservation->state === 'committed'
&& $bundleState(
$snapshotReservation,
'committed',
'assigned'
)
&& ($snapshotAgentRequest['request_target'] ?? null)
=== $snapshotRequestTarget
&& ($snapshotPosted['hostname'] ?? null) === ''
&& ($snapshotPosted['root_password'] ?? null) === ''
&& ($snapshotPosted['guest_ssh_public_key'] ?? null) === ''
&& ($snapshotPosted['public_network'] ?? null)
=== $snapshotNetwork,
'Cloud-snapshot Worker delivery replaced guest identity or failed to commit the exact dual-stack ledger.');
} finally {
$connection->rollBack();
}
$assert(CloudOrder::count() === $snapshotOrderCount
&& CloudInvoice::count() === $snapshotInvoiceCount
&& CloudJob::count() === $snapshotJobCount
&& CloudResourceReservation::count() === $snapshotReservationCount
&& (string) User::findOrFail(1)->money === $snapshotBalance
&& (int) CloudOffer::findOrFail($offer->id)->stock === $snapshotStock,
'Rolled-back cloud-snapshot fixture leaked finance or placement state.');
$queueSnapshotProvision = static function (
string $name,
string $requestKey,
string $paymentKey
) use ($actor, $offer, $deploymentSnapshot): array {
$order = CloudOrderService::purchase([
'offer_id' => (int) $offer->id,
'source' => 'deployment_snapshot',
'cloud_snapshot_key' => $deploymentSnapshot->snapshot_key,
'name' => $name,
'automatic_backup' => false,
'idempotency_key' => $requestKey,
], $actor);
CloudInvoiceService::payBalance(
CloudInvoice::where('order_id', (int) $order->id)->firstOrFail(),
$actor,
$paymentKey
);
return [
$order,
CloudJob::where('order_id', (int) $order->id)->firstOrFail(),
CloudResourceReservation::where(
'order_id', (int) $order->id
)->firstOrFail(),
];
};
$connection->beginTransaction();
try {
[$snapshotPreSendOrder, $snapshotPreSendJob,
$snapshotPreSendReservation] = $queueSnapshotProvision(
'vm-snapshot-presend',
'snapshot-presend-request-0001',
'snapshot-presend-payment-0001'
);
$mock->append(new ConnectException(
'simulated snapshot local connection failure',
new Request(
'POST',
'http://127.0.0.1:8001/internal/agent/request'
),
null,
[
'errno' => 7,
'url' =>
'http://127.0.0.1:8001/internal/agent/request',
'request_size' => 0,
'size_upload' => 0.0,
'size_download' => 0.0,
'header_size' => 0,
'http_code' => 0,
'redirect_count' => 0,
'connect_time' => 0.0,
'pretransfer_time' => 0.0,
'starttransfer_time' => 0.0,
]
));
$snapshotPreSendHistory = count($history);
$assert($worker->tick(),
'Cloud-snapshot pre-send failure was not processed.');
$snapshotPreSendJob->refresh();
$snapshotPreSendOrder->refresh();
$snapshotPreSendReservation->refresh();
$assert((string) $snapshotPreSendJob->phase === 'terminal'
&& (string) $snapshotPreSendJob->outcome === 'failed'
&& (string) $snapshotPreSendJob->effect_resolution
=== 'not_applied'
&& !(bool) $snapshotPreSendJob->effect_may_have_applied
&& $snapshotPreSendOrder->refunded_at !== null
&& (string) $snapshotPreSendReservation->state === 'released'
&& $bundleState(
$snapshotPreSendReservation,
'released',
'available'
)
&& (string) User::findOrFail(1)->money === $snapshotBalance
&& (int) CloudOffer::findOrFail($offer->id)->stock
=== $snapshotStock,
'Cloud-snapshot pre-send failure retained money or either address family.');
$worker->tick();
$assert(count($history) === $snapshotPreSendHistory + 1,
'Cloud-snapshot pre-send failure replayed its mutation.');
} finally {
$connection->rollBack();
}
$connection->beginTransaction();
try {
[$snapshotUnknownOrder, $snapshotUnknownJob,
$snapshotUnknownReservation] = $queueSnapshotProvision(
'vm-snapshot-unknown',
'snapshot-unknown-request-0001',
'snapshot-unknown-payment-0001'
);
$mock->append(new ConnectException(
'simulated snapshot dispatched timeout',
new Request('POST', 'http://127.0.0.1:8000')
));
$snapshotUnknownHistory = count($history);
$assert($worker->tick(),
'Cloud-snapshot unknown-effect failure was not processed.');
$snapshotUnknownJob->refresh();
$snapshotUnknownOrder->refresh();
$snapshotUnknownReservation->refresh();
$assert((string) $snapshotUnknownJob->phase === 'terminal'
&& (string) $snapshotUnknownJob->outcome === 'failed'
&& (string) $snapshotUnknownJob->effect_resolution === 'unknown'
&& (bool) $snapshotUnknownJob->effect_may_have_applied
&& !(bool) $snapshotUnknownJob->safe_to_resubmit
&& $snapshotUnknownOrder->refunded_at === null
&& (string) $snapshotUnknownReservation->state === 'uncertain'
&& $bundleState(
$snapshotUnknownReservation,
'uncertain',
'uncertain'
)
&& (string) User::findOrFail(1)->money === '80.02'
&& (int) CloudOffer::findOrFail($offer->id)->stock === 8,
'Cloud-snapshot dispatched timeout refunded or released an unknown dual-stack effect.');
$worker->tick();
$assert(count($history) === $snapshotUnknownHistory + 1,
'Cloud-snapshot unknown effect replayed its mutation.');
} finally {
$connection->rollBack();
}
$assert(CloudOrder::count() === $snapshotOrderCount
&& CloudInvoice::count() === $snapshotInvoiceCount
&& CloudJob::count() === $snapshotJobCount
&& CloudResourceReservation::count() === $snapshotReservationCount
&& (string) User::findOrFail(1)->money === $snapshotBalance
&& (int) CloudOffer::findOrFail($offer->id)->stock === $snapshotStock,
'Cloud-snapshot failure fixtures leaked finance or placement state.');
// A failed local TCP connection is not an ambiguous Agent dispatch.
// Exercise the full worker/financial path, not only client classification.
foreach ([7 => 'refused', 28 => 'connect-timeout'] as $curlError => $failure) {
$gatewayFailure = CloudOrderService::purchase(array_replace($input, [
'name' => 'vm-gateway-' . $failure,
'idempotency_key' => 'gateway-before-send-' . $failure,
]), $actor);
CloudInvoiceService::payBalance(
CloudInvoice::where('order_id', $gatewayFailure->id)->first(),
$actor, 'gateway-balance-' . $failure
);
$mock->append(new ConnectException('simulated local connection failure',
new Request('POST', 'http://127.0.0.1:8001/internal/agent/request'),
null, [
'errno' => $curlError,
'url' => 'http://127.0.0.1:8001/internal/agent/request',
'request_size' => 0, 'size_upload' => 0.0,
'size_download' => 0.0, 'header_size' => 0,
'http_code' => 0, 'redirect_count' => 0,
'connect_time' => 0.0, 'pretransfer_time' => 0.0,
'starttransfer_time' => 0.0,
]));
$beforeGatewayFailure = count($history);
$assert($worker->tick(), 'Gateway failure task was not processed.');
$gatewayFailureJob = CloudJob::where('order_id', $gatewayFailure->id)->first();
$gatewayFailureReservation = CloudResourceReservation::where(
'order_id', $gatewayFailure->id
)->first();
$assert($gatewayFailureJob->phase === 'terminal'
&& $gatewayFailureJob->outcome === 'failed'
&& $gatewayFailureJob->effect_resolution === 'not_applied'
&& !(bool) $gatewayFailureJob->effect_may_have_applied
&& !(bool) $gatewayFailureJob->finalization_incident
&& CloudOrder::find($gatewayFailure->id)->refunded_at !== null
&& $gatewayFailureReservation->state === 'released'
&& $bundleState(
$gatewayFailureReservation,
'released',
'available'
)
&& User::find(1)->money === '90.01'
&& CloudOffer::find(1)->stock === 9,
'Proven pre-send Gateway failure retained unknown effects, funds or placement.');
$worker->tick();
$assert(count($history) === $beforeGatewayFailure + 1
&& User::find(1)->money === '90.01'
&& CloudOffer::find(1)->stock === 9,
'Pre-send Gateway failure replayed its mutation or refunded twice.');
}
$uncertain = CloudOrderService::purchase(array_replace($input, ['name' => 'vm-three', 'idempotency_key' => 'test-request-key-0003']), $actor);
$assert(User::find(1)->money === '90.01'
&& CloudJob::where('order_id', $uncertain->id)->count() === 0,
'An unpaid ambiguous-case deployment charged or entered the worker queue.');
CloudInvoiceService::payBalance(CloudInvoice::where('order_id', $uncertain->id)->first(),
$actor, 'provision-balance-test-0003');
$mock->append(new ConnectException('simulated timeout', new Request('POST', 'http://127.0.0.1:8000')));
$worker->tick();
$uncertainJob = CloudJob::where('order_id', $uncertain->id)->first();
$uncertainInvoiceDisplay = CloudInvoiceService::display(
CloudInvoice::where('order_id', $uncertain->id)->first());
$uncertainReservation = CloudResourceReservation::where(
'order_id', $uncertain->id
)->first();
$assert($uncertainJob->phase === 'terminal'
&& $uncertainJob->outcome === 'failed'
&& $uncertainJob->status === 'failed'
&& (int) $uncertainJob->dispatch_attempts === 1
&& (bool) $uncertainJob->effect_may_have_applied === true
&& (bool) $uncertainJob->safe_to_resubmit === false
&& $uncertainJob->effect_resolution === 'unknown'
&& CloudOrder::find($uncertain->id)->status === 'failed'
&& CloudOrder::find($uncertain->id)->refunded_at === null
&& $uncertainReservation->state === 'uncertain'
&& $bundleState($uncertainReservation, 'uncertain', 'uncertain')
&& User::find(1)->money === '80.02'
&& ($uncertainInvoiceDisplay['order_status'] ?? null) === 'failed'
&& CloudMailOutbox::where('kind', 'provision_succeeded')->count() === 1,
'A dispatched create timeout did not become a terminal unknown-effect failure.');
$beforeFailedCreateRetry = count($history);
$worker->tick();
$uncertainJob->refresh();
$assert(count($history) === $beforeFailedCreateRetry
&& $uncertainJob->phase === 'terminal'
&& $uncertainJob->outcome === 'failed'
&& (int) $uncertainJob->dispatch_attempts === 1
&& CloudInstance::count() === 1
&& CloudResourceReservation::find($uncertainReservation->id)->state === 'uncertain'
&& User::find(1)->money === '80.02'
&& CloudMailOutbox::where('kind', 'provision_succeeded')->count() === 1,
'A terminal create failure was automatically polled, reconciled, or mutation-retried.');
// Simulate the crash window where the Agent already committed its
// not-applied fence receipt but PHP died before settling local ledgers. A
// later fresh inspection may consume the older matching receipt, while
// the immutable public task remains terminal failed.
$resolutionReason = '已通过新的只读检查确认虚拟机及面板管理资源均不存在,执行原子退款。';
$inspectionNow = time();
$freshInspection = [
'operation_id' => (string) $uncertainJob->operation_id,
'checked_at' => $inspectionNow,
'outcome' => 'failed',
'summary_code' => 'operation_failed',
'error_code' => 'creation_absent',
];
$persistedFenceReceipt = [
'operation_id' => (string) $uncertainJob->operation_id,
'resolution_id' => str_repeat('a', 32),
'resolution' => 'not_applied',
'inspection_checked_at' => $inspectionNow - 60,
'reason_sha256' => hash('sha256', $resolutionReason),
'resolved_at' => $inspectionNow - 50,
'resolved_by' => 7,
'replayed' => false,
];
$resolvedCreate = CloudAdminOperationsService::resolveEffect([
'user_id' => 99, 'is_admin' => true,
'is_super_admin' => true, 'can_manage' => true,
], (string) $uncertainJob->id, [
'resolution' => 'not_applied', 'reason' => $resolutionReason,
], $freshInspection, $persistedFenceReceipt);
$uncertainJob->refresh();
$assert($uncertainJob->phase === 'terminal'
&& $uncertainJob->outcome === 'failed'
&& $uncertainJob->status === 'failed'
&& $uncertainJob->effect_resolution === 'not_applied'
&& (int) $uncertainJob->effect_resolved_by === 99
&& ($resolvedCreate['job']['completed'] ?? false) === true
&& ($resolvedCreate['job']['status'] ?? null) === 'failed'
&& ($resolvedCreate['job']['can_resolve_effect'] ?? true) === false
&& CloudOrder::find($uncertain->id)->refunded_at !== null
&& CloudInvoice::where('order_id', $uncertain->id)
->value('status') === 'refunded'
&& CloudResourceReservation::find($uncertainReservation->id)
->state === 'released'
&& $bundleState(
CloudResourceReservation::find($uncertainReservation->id),
'released',
'available'
)
&& User::find(1)->money === '90.01'
&& CloudOffer::find(1)->stock === 9,
'A prior Agent fence commit was not recovered into one atomic local create settlement.');
// A completion may be durably accepted before PHP finishes the create
// ledgers. Keep the encrypted request escrow while the effect is unknown,
// then let the exact same authenticated receipt atomically materialize the
// instance and ACK its event after the local ledger fault is repaired.
$finiteStockBeforeUnlimitedCreate = (int) CloudOffer::findOrFail(1)->stock;
$unlimitedOffer = CloudOffer::findOrFail(1);
$offerCreatedAtBeforeUnlimitedCreate = (int) $unlimitedOffer->created_at;
$unlimitedOffer->stock = -1;
$unlimitedOffer->created_at = time() - 20;
$unlimitedOffer->updated_at = time() - 10;
$unlimitedOffer->save();
$lateCreate = CloudOrderService::purchase(array_replace($input, [
'name' => 'vm-late-create',
'idempotency_key' => 'test-request-key-late-create',
]), $actor);
$lateCreateSpec = json_decode(
(string) $lateCreate->spec_json,
true,
64,
JSON_THROW_ON_ERROR
);
$assert(($lateCreateSpec['stock_mode'] ?? null) === 'unlimited'
&& $lateCreate->stock_reserved_at === null
&& $lateCreate->stock_released_at === null
&& (int) CloudOffer::findOrFail(1)->stock === -1,
'An unlimited-stock purchase consumed or fabricated finite stock.');
$lateInvoice = CloudInvoice::where('order_id', $lateCreate->id)->first();
CloudInvoiceService::payBalance(
$lateInvoice,
$actor,
'provision-balance-late-create'
);
$lateCreateJob = CloudJob::where('order_id', $lateCreate->id)->first();
$lateCreateIdentity = CloudOperationRecoveryIdentity::expected(
$lateCreateJob,
true
);
$lateCreatePayload = \App\Services\Cloud\CloudSecrets::decrypt(
(string) $lateCreateJob->sealed_payload,
'job-payload'
);
$mock->append(new Response(202, [], json_encode([
'operation_id' => $lateCreateJob->operation_id,
'status' => 'reserved',
...$lateCreateIdentity,
'replayed' => false,
'retryable' => false,
])));
$worker->tick();
$lateCreateJob->refresh();
$assert((string) $lateCreateJob->phase === 'awaiting_receipt',
'Late-create fixture did not cross the Agent dispatch boundary.');
$lateOrder = CloudOrder::find($lateCreate->id);
$lateOrderSpec = json_decode(
(string) $lateOrder->spec_json,
true,
64,
JSON_THROW_ON_ERROR
);
$damagedLateOrderSpec = $lateOrderSpec;
unset($damagedLateOrderSpec['validity_days']);
$lateOrder->spec_json = json_encode(
$damagedLateOrderSpec,
JSON_THROW_ON_ERROR
);
$lateOrder->save();
$lateReceiptId = 'create-late-success-event-0001';
$lateCreatedVm = array_replace($createdVm, [
'uuid' => $lateCreateIdentity['resource_id'],
'name' => '1:vm-late-create',
'public_network' => $lateCreatePayload['body']['public_network'],
]);
unset($lateCreatedVm['generated_password']);
$lateReceipt = [
'type' => 'completion',
'id' => $lateReceiptId,
'agent_id' => (string) $lateCreateJob->host_id,
'operation_id' => (string) $lateCreateJob->operation_id,
...$lateCreateIdentity,
'status' => 'succeeded',
'effect_may_have_applied' => false,
'safe_to_resubmit' => false,
'error_code' => '',
'result' => $lateCreatedVm,
];
$firstLateReceipt = CloudOperationCompletionService::accept($lateReceipt);
$lateCreateJob->refresh();
$lateEscrow = \App\Services\Cloud\CloudSecrets::decrypt(
(string) $lateCreateJob->sealed_payload,
'job-payload'
);
$lateEvent = \App\Models\CloudAgentCompletionEvent::where(
'event_id',
$lateReceiptId
)->first();
$assert($firstLateReceipt === [
'accepted' => true,
'disposition' => 'received',
]
&& (string) $lateCreateJob->phase === 'terminal'
&& (string) $lateCreateJob->outcome === 'failed'
&& (string) $lateCreateJob->effect_resolution === 'unknown'
&& ($lateEscrow['credential']['password'] ?? null)
=== ($lateEscrow['body']['root_password'] ?? null)
&& (string) $lateEvent->disposition === 'received'
&& CloudInstance::where('order_id', $lateCreate->id)->count() === 0,
'A failed local create finalizer lost its exact receipt or encrypted recovery escrow.');
$lateWalletBeforeManualResolution = (string) User::find(1)->money;
$lateReservationBeforeManualResolution = CloudResourceReservation::where(
'order_id',
$lateCreate->id
)->first();
try {
\App\Services\Cloud\CloudCreateFailureFinalizer::resolveNotApplied(
(int) $lateCreateJob->id,
(string) $lateCreateJob->operation_id,
99,
'contradictory-manual-absence',
[
'operation_id' => (string) $lateCreateJob->operation_id,
'outcome' => 'failed',
'summary_code' => 'operation_failed',
'error_code' => 'creation_absent',
],
time()
);
throw new RuntimeException(
'Manual no-effect resolution ignored a durable create success.'
);
} catch (CloudError $error) {
$assert($error->httpStatus === 409,
'Create success/manual absence conflict returned wrong status.');
}
$assert((string) User::find(1)->money
=== $lateWalletBeforeManualResolution
&& (string) CloudOrder::find($lateCreate->id)->status === 'failed'
&& (string) CloudInvoice::where('order_id', $lateCreate->id)
->value('status') === 'paid'
&& (string) CloudResourceReservation::find(
$lateReservationBeforeManualResolution->id
)->state === (string) $lateReservationBeforeManualResolution->state,
'Conflicting manual absence resolution refunded or released a create.');
$legacyLateOrderSpec = $lateOrderSpec;
unset($legacyLateOrderSpec['stock_mode']);
$lateOrder->spec_json = json_encode(
$legacyLateOrderSpec,
JSON_THROW_ON_ERROR
);
$lateOrder->save();
$makeLateCreateRetryable = static function () use (
$lateCreateJob
): void {
Capsule::table('cloud_jobs')->where(
'id',
(int) $lateCreateJob->id
)->update([
'available_at' => 0,
'lease_token' => null,
'lease_until' => 0,
]);
};
$assertLateCreateStillPending = static function (
string $message
) use ($assert, $lateCreate, $lateReceiptId): void {
$assert((string) \App\Models\CloudAgentCompletionEvent::where(
'event_id',
$lateReceiptId
)->value('disposition') === 'received'
&& CloudInstance::where('order_id', $lateCreate->id)->count() === 0,
$message);
};
$finiteMarkerlessSpec = $legacyLateOrderSpec;
$finiteMarkerlessSpec['stock_mode'] = 'finite';
$lateOrder->spec_json = json_encode(
$finiteMarkerlessSpec,
JSON_THROW_ON_ERROR
);
$lateOrder->save();
$finiteGuardWallet = (string) User::findOrFail(1)->money;
$finiteGuardReservation = CloudResourceReservation::where(
'order_id',
$lateCreate->id
)->firstOrFail();
$makeLateCreateRetryable();
$assert((new CloudAutomaticOperationRecoveryService())->tick(),
'Markerless finite-stock create was not retried safely.');
$assertLateCreateStillPending(
'A markerless finite-stock order allowed late create materialisation or ACK.'
);
$assert(CloudOrder::findOrFail($lateCreate->id)->refunded_at === null
&& (string) CloudInvoice::findOrFail($lateInvoice->id)->status === 'paid'
&& (string) CloudResourceReservation::findOrFail(
$finiteGuardReservation->id
)->state === (string) $finiteGuardReservation->state
&& $bundleState(
CloudResourceReservation::findOrFail(
$finiteGuardReservation->id
),
'uncertain',
'uncertain'
)
&& (string) User::findOrFail(1)->money === $finiteGuardWallet
&& (int) CloudOffer::findOrFail(1)->stock === -1,
'A markerless finite-stock order released or refunded fenced ledgers.');
$unknownStockModeSpec = $legacyLateOrderSpec;
$unknownStockModeSpec['stock_mode'] = 'unknown';
$lateOrder->spec_json = json_encode(
$unknownStockModeSpec,
JSON_THROW_ON_ERROR
);
$lateOrder->save();
$makeLateCreateRetryable();
$assert((new CloudAutomaticOperationRecoveryService())->tick(),
'Unknown stock-mode create was not retried safely.');
$assertLateCreateStillPending(
'An unknown stock mode allowed late create materialisation or ACK.'
);
$nullStockModeSpec = $legacyLateOrderSpec;
$nullStockModeSpec['stock_mode'] = null;
$lateOrder->spec_json = json_encode(
$nullStockModeSpec,
JSON_THROW_ON_ERROR
);
$lateOrder->save();
$makeLateCreateRetryable();
$assert((new CloudAutomaticOperationRecoveryService())->tick(),
'Null stock-mode create was not retried safely.');
$assertLateCreateStillPending(
'A null stock mode allowed late create materialisation or ACK.'
);
$lateOrder->spec_json = json_encode(
$legacyLateOrderSpec,
JSON_THROW_ON_ERROR
);
$lateOrder->save();
$unlimitedOffer->updated_at = (int) $lateOrder->created_at;
$unlimitedOffer->save();
$makeLateCreateRetryable();
$assert((new CloudAutomaticOperationRecoveryService())->tick(),
'Ambiguous legacy unlimited-stock create was not retried safely.');
$assertLateCreateStillPending(
'A changed legacy offer allowed late create materialisation or ACK.'
);
$unlimitedOffer->updated_at = (int) $lateOrder->created_at - 1;
$unlimitedOffer->save();
$lateInvoice->purpose = 'renewal';
$lateInvoice->save();
$makeLateCreateRetryable();
$assert((new CloudAutomaticOperationRecoveryService())->tick(),
'Wrong-purpose create invoice was not retried safely.');
$assertLateCreateStillPending(
'A wrong-purpose invoice allowed late create materialisation or ACK.'
);
$lateInvoice->purpose = 'provision';
$lateInvoice->status = 'refunded';
$lateInvoice->save();
$makeLateCreateRetryable();
$assert((new CloudAutomaticOperationRecoveryService())->tick(),
'Refunded create invoice was not retried safely.');
$assertLateCreateStillPending(
'A refunded invoice allowed late create materialisation or ACK.'
);
$lateInvoice->status = 'paid';
$lateInvoice->save();
$lateInvoiceRow = (array) Capsule::table('cloud_invoices')
->where('id', (int) $lateInvoice->id)->first();
Capsule::table('cloud_invoices')->where(
'id',
(int) $lateInvoice->id
)->delete();
$makeLateCreateRetryable();
$assert((new CloudAutomaticOperationRecoveryService())->tick(),
'Missing create invoice was not retried safely.');
$assertLateCreateStillPending(
'A missing invoice allowed late create materialisation or ACK.'
);
Capsule::table('cloud_invoices')->insert($lateInvoiceRow);
$lateInvoice = CloudInvoice::findOrFail((int) $lateInvoice->id);
$makeLateCreateRetryable();
$lateRecoveryTick = (new CloudAutomaticOperationRecoveryService())->tick();
// The Agent may already have stopped retransmitting after the panel
// durably accepted the event. The recovery worker must consume that same
// stored receipt; a later duplicate is only an idempotent ACK check.
$secondLateReceipt = CloudOperationCompletionService::accept($lateReceipt);
$lateCreateJob->refresh();
$lateOrder->refresh();
$lateInvoice->refresh();
$lateInstance = CloudInstance::where('order_id', $lateCreate->id)->first();
$lateReservation = CloudResourceReservation::where(
'order_id',
$lateCreate->id
)->first();
$latePeriod = \App\Models\CloudServicePeriod::where(
'order_id',
$lateCreate->id
)->first();
$lateCredential = $lateInstance === null ? null
: \App\Services\Cloud\CloudSecrets::decrypt(
(string) $lateInstance->sealed_credential,
'instance-credential'
);
$lateEvent->refresh();
$lateFinalPayload = \App\Services\Cloud\CloudSecrets::decrypt(
(string) $lateCreateJob->sealed_payload,
'job-payload'
);
$lateInvoiceDisplay = CloudInvoiceService::display($lateInvoice);
$lateTaskDisplay = CloudJobService::display($lateCreateJob);
$assert($lateRecoveryTick
&& $secondLateReceipt === [
'accepted' => true,
'disposition' => 'applied',
]
&& $lateInstance !== null
&& (string) $lateCreateJob->phase === 'terminal'
&& (string) $lateCreateJob->outcome === 'failed'
&& (string) $lateCreateJob->status === 'failed'
&& (string) $lateCreateJob->effect_resolution === 'applied'
&& (string) $lateCreateJob->effect_resolution_reason
=== 'late_completion_receipt:' . $lateReceiptId
&& (string) $lateEvent->disposition === 'applied'
&& $lateEvent->applied_at !== null
&& (string) $lateOrder->status === 'succeeded'
&& (int) $lateOrder->instance_id === (int) $lateInstance->id
&& (int) $lateInvoice->instance_id === (int) $lateInstance->id
&& (string) $lateReservation->state === 'committed'
&& $latePeriod !== null
&& (string) $latePeriod->status === 'active'
&& (int) $latePeriod->instance_id === (int) $lateInstance->id
&& (int) $lateInstance->current_period_id === (int) $latePeriod->id
&& $lateCredential === $lateCreatePayload['credential']
&& !array_key_exists('credential', $lateFinalPayload)
&& !array_key_exists('root_password', $lateFinalPayload['body'] ?? [])
&& ($lateInvoiceDisplay['job_status'] ?? null) === 'succeeded'
&& ($lateInvoiceDisplay['fulfillment_status'] ?? null) === 'succeeded'
&& ($lateTaskDisplay['status'] ?? null) === 'failed'
&& ($lateTaskDisplay['effect_resolution'] ?? null) === 'applied',
'The exact late create receipt did not atomically converge PHP ledgers and public fulfillment state.');
$assert(CloudInvoiceService::deliverReminders(20) >= 1
&& (string) CloudMailOutbox::where(
'invoice_id',
$lateInvoice->id
)->where('kind', 'provision_succeeded')->value('status') === 'sent',
'Recovered create success was cancelled instead of delivering its provision notice.');
$assert($lateOrder->stock_reserved_at === null
&& $lateOrder->stock_released_at === null
&& (int) CloudOffer::findOrFail(1)->stock === -1,
'Unlimited-stock create recovery mutated finite inventory.');
$unlimitedOffer->stock = $finiteStockBeforeUnlimitedCreate;
$unlimitedOffer->created_at = $offerCreatedAtBeforeUnlimitedCreate;
$unlimitedOffer->updated_at = time();
$unlimitedOffer->save();
$actionJob = CloudJobService::enqueue($instance, $actor, 'action', 'POST', '/actions', ['action' => 'reboot']);
$mock->append(new Response(200, [], json_encode([
'operation_id' => $actionJob->operation_id,
'status' => 'succeeded',
...CloudOperationRecoveryIdentity::expected($actionJob, true),
'result' => [
'uuid' => (string) $instance->uuid,
'state' => 'running',
],
]))); $worker->tick();
$last = $agentRequest(end($history));
$signedActor = json_decode(base64_decode(strtr(
$last['headers']['X-Virt-Actor'], '-_', '+/'
)), true);
$assert(CloudJob::find($actionJob->id)->status === 'succeeded'
&& $signedActor['can_manage'] === true
&& ($signedActor['operation_id'] ?? null) === $actionJob->operation_id,
'Power task lost its typed Agent authority or did not complete.');
// Keep the panel/audit vocabulary stable while translating the force-off
// command to the Go Agent's strict PowerAction wire value.
$poweroffJob = CloudJobService::enqueue(
$instance, $actor, 'action', 'POST', '/actions', ['action' => 'poweroff']
);
$storedPoweroff = \App\Services\Cloud\CloudSecrets::decrypt(
(string) $poweroffJob->sealed_payload, 'job-payload'
);
$mock->append(new Response(200, [], json_encode([
'operation_id' => $poweroffJob->operation_id,
'status' => 'succeeded',
...CloudOperationRecoveryIdentity::expected($poweroffJob, true),
'result' => [
'uuid' => (string) $instance->uuid,
'state' => 'stopped',
],
])));
$worker->tick();
$poweroffRequest = $agentRequest(end($history));
$poweroffBody = json_decode(
(string) ($poweroffRequest['body'] ?? ''), true, 8,
JSON_THROW_ON_ERROR
);
$assert(CloudJob::find($poweroffJob->id)->status === 'succeeded'
&& ($storedPoweroff['body']['action'] ?? null) === 'poweroff'
&& $poweroffBody === ['action' => 'destroy'],
'Panel poweroff was not translated exactly once at the Agent wire boundary.');
// Keep one bounded Agent cause for administrator diagnosis while the
// public job contract continues to expose only the panel-owned error.
$diagnosticJob = CloudJobService::enqueue(
$instance, $actor, 'action', 'POST', '/actions', ['action' => 'reboot']
);
$mock->append(new Response(200, [], json_encode([
'operation_id' => $diagnosticJob->operation_id,
'status' => 'failed',
...CloudOperationRecoveryIdentity::expected($diagnosticJob),
'cause_code' => 'domain_start_failed',
'error' => [
'code' => 'libvirt_error',
'message' => 'private path /srv/libvirt/domain.xml',
],
'effect_may_have_applied' => false,
'safe_to_resubmit' => true,
'result' => ['state' => 'stopped'],
])));
$worker->tick();
$diagnosticRow = CloudJob::find($diagnosticJob->id);
$diagnosticResult = json_decode((string) $diagnosticRow->result_json,
true, flags: JSON_THROW_ON_ERROR);
$publicDiagnostic = CloudJobService::display($diagnosticRow);
$assert($diagnosticRow->status === 'failed'
&& $diagnosticRow->error_code === 'operation_failed'
&& ($diagnosticResult['cause_code'] ?? null) === 'domain_start_failed'
&& !isset($publicDiagnostic['result']['cause_code'])
&& !str_contains(json_encode($publicDiagnostic,
JSON_UNESCAPED_UNICODE), '/srv/libvirt'),
'Stable Agent cause was not persisted privately or leaked into the public job DTO.');
// A strict two-hop admission rejection proves that the mutation never
// reached the Agent journal. It is terminal and may permit an explicit
// future request, but the worker never re-dispatches this task.
$requeueJob = CloudJobService::enqueue(
$instance, $actor, 'action', 'POST', '/actions', ['action' => 'reboot']
);
$beforeRequeue = count($history);
$mock->append(new Response(503, [], json_encode(['error' => [
'code' => 'agent_busy',
'message' => 'Agent control request was not dispatched',
'retryable' => false,
'dispatch_state' => 'not_dispatched',
]])));
$worker->tick();
$requeueJob->refresh();
$assert($requeueJob->phase === 'terminal'
&& $requeueJob->outcome === 'failed'
&& $requeueJob->status === 'failed'
&& (int) $requeueJob->dispatch_attempts === 1
&& $requeueJob->dispatched_at !== null
&& $requeueJob->lease_token === null
&& (bool) $requeueJob->effect_may_have_applied === false
&& (bool) $requeueJob->safe_to_resubmit === true
&& $requeueJob->effect_resolution === 'not_applied',
'Definitive pre-dispatch backpressure did not become a safe terminal failure.');
$worker->tick();
$requeueRequests = array_slice($history, $beforeRequeue);
$assert(CloudJob::find($requeueJob->id)->status === 'failed'
&& count($requeueRequests) === 1
&& $agentRequest($requeueRequests[0])['method'] === 'POST',
'Safe pre-dispatch failure was automatically retried.');
// A strict DTO rejection happens before the Agent operation journal for
// every mutation kind, not only provisioning. It is terminal validation
// failure and must not strand a non-create task in uncertain/404 polling.
$invalidActionJob = CloudJobService::enqueue(
$instance, $actor, 'action', 'POST', '/actions', ['action' => 'reboot']
);
$beforeInvalidAction = count($history);
$mock->append(new Response(422, [], json_encode(['error' => [
'code' => 'invalid_json',
'message' => 'request contains an unknown field',
'retryable' => false,
'dispatch_state' => 'not_dispatched',
]])));
$worker->tick();
$invalidActionJob->refresh();
$assert($invalidActionJob->status === 'failed'
&& $invalidActionJob->dispatched_at !== null
&& count($history) === $beforeInvalidAction + 1,
'Definitive non-create validation failure became permanently uncertain.');
// A gateway secret mismatch is a terminal configuration/authentication
// rejection before WSS dispatch. It must not strand the task in uncertain
// polling or loop forever in the queue.
$gatewayAuthJob = CloudJobService::enqueue(
$instance, $actor, 'action', 'POST', '/actions', ['action' => 'reboot']
);
$beforeGatewayAuth = count($history);
$mock->append(new Response(403, [], json_encode(['error' => [
'code' => 'gateway_forbidden',
'message' => 'Agent gateway request is forbidden',
'retryable' => false,
'dispatch_state' => 'not_dispatched',
]])));
$worker->tick();
$gatewayAuthJob->refresh();
$assert($gatewayAuthJob->status === 'failed'
&& $gatewayAuthJob->dispatched_at !== null
&& $gatewayAuthJob->lease_token === null,
'Gateway authentication failure did not terminate before journal dispatch.');
$gatewayAuthRequests = array_slice($history, $beforeGatewayAuth);
$assert(count($gatewayAuthRequests) === 1
&& $agentRequest($gatewayAuthRequests[0])['method'] === 'POST',
'Gateway authentication failure was replayed after a terminal response.');
// A journal-backed error envelope for another operation must never finish
// this job, even when the foreign record itself is terminal.
$crossBoundJob = CloudJobService::enqueue(
$instance, $actor, 'action', 'POST', '/actions', ['action' => 'reset']
);
$mock->append(new Response(409, [], json_encode([
'error' => [
'code' => 'operation_payload_mismatch',
'retryable' => false,
'operation' => [
'operation_id' => $actionJob->operation_id,
'status' => 'failed',
'error' => ['code' => 'operation_payload_mismatch'],
],
],
])));
$worker->tick();
$crossBoundJob->refresh();
$assert($crossBoundJob->phase === 'terminal'
&& $crossBoundJob->outcome === 'failed'
&& $crossBoundJob->status === 'failed'
&& (bool) $crossBoundJob->effect_may_have_applied === true
&& (bool) $crossBoundJob->safe_to_resubmit === false
&& $crossBoundJob->effect_resolution === 'unknown'
&& $instance->fresh()->status === 'active',
'A foreign durable error envelope advanced the current job.');
$crossBoundJob->effect_resolution = 'not_applied';
$crossBoundJob->effect_resolved_at = time();
$crossBoundJob->effect_resolved_by = 99;
$crossBoundJob->effect_resolution_reason = 'test fixture fence resolution';
$crossBoundJob->save();
// Clean-break protocol: a legacy success shortcut without operation_id is
// ambiguous and cannot be used to advance any mutation.
$legacyShortcutJob = CloudJobService::enqueue(
$instance, $actor, 'action', 'POST', '/actions', ['action' => 'reboot']
);
$mock->append(new Response(200, [], json_encode([
'status' => 'succeeded',
'uuid' => (string) $instance->uuid,
'state' => 'running',
])));
$worker->tick();
$legacyShortcutJob->refresh();
$assert($legacyShortcutJob->phase === 'terminal'
&& $legacyShortcutJob->outcome === 'failed'
&& $legacyShortcutJob->status === 'failed'
&& $legacyShortcutJob->effect_resolution === 'unknown',
'A mutation response without operation_id was accepted as success.');
$legacyShortcutJob->effect_resolution = 'not_applied';
$legacyShortcutJob->effect_resolved_at = time();
$legacyShortcutJob->effect_resolved_by = 99;
$legacyShortcutJob->effect_resolution_reason = 'test fixture fence resolution';
$legacyShortcutJob->save();
// Terminal tasks are never polled through the retired Agent operation
// lookup/reconcile loop. Every Agent request issued above was the single
// mutation dispatch for its own durable task.
$lookupRequests = array_filter($history, static function (array $entry)
use ($agentRequest): bool {
return $agentRequest($entry)['method'] === 'GET';
});
$assert($lookupRequests === [],
'A terminal task triggered retired Agent lookup/reconcile traffic.');
$instance->network_status = 'active';
$instance->save();
// Expiry networking, graceful purge shutdown, and irreversible deletion
// deliberately use separate states. Only the four target-state policy
// jobs may converge; the final disk deletion is a one-shot mutation.
$makeLifecycleInstance = function (string $uuid, string $name,
string $serviceStatus, string $networkStatus) use ($instance) {
$copy = $instance->replicate();
$copy->order_id = null;
$copy->reservation_id = null;
$copy->current_period_id = null;
$copy->uuid = $uuid;
$copy->name = $name;
$copy->status = 'active';
$copy->service_status = $serviceStatus;
$copy->network_status = $networkStatus;
$copy->suspended_at = null;
$copy->network_suspended_at = null;
$copy->sealed_credential = \App\Services\Cloud\CloudSecrets::encrypt([
'password' => 'LifecycleStrongPassword!123', 'login_name' => 'root',
], 'instance-credential');
$copy->created_at = $copy->updated_at = time();
$copy->save();
return $copy;
};
$queuePolicy = function (CloudInstance $target, string $kind,
string $suffix, ?string $reason = null) {
$job = new CloudJob();
$job->operation_id = bin2hex(random_bytes(16));
$job->instance_id = $target->id;
$job->user_id = $target->user_id;
$job->host_id = $target->host_id;
$job->kind = $kind;
$job->execution_mode = 'agent';
$job->status = 'queued';
$job->sealed_payload = \App\Services\Cloud\CloudSecrets::encrypt([
'method' => 'POST',
'request_target' => '/v1/vms/' . $target->uuid . '/' . $suffix,
'body' => $reason === null ? [] : ['reason' => $reason],
'owner_id' => (int) $target->user_id,
'system_policy' => true, 'suspend_reason' => $reason,
], 'job-payload');
$job->available_at = $job->created_at = $job->updated_at = time();
$job->save();
return $job;
};
$adminUser = new User();
$adminUser->id = 99; $adminUser->money = '0.00'; $adminUser->enable = 1;
$adminUser->is_admin = true; $adminUser->save();
$adminActor = ['user_id' => 99, 'is_admin' => true,
'is_super_admin' => true, 'can_manage' => true, 'session_id' => ''];
$expiredNetwork = $makeLifecycleInstance(
'55555555-5555-4555-8555-555555555555', 'expired-network-vm',
'expired_grace', 'blocking_expired');
$expiredNetworkJob = $queuePolicy(
$expiredNetwork, 'network_suspend', 'network/suspend', 'expired');
$mock->append(new Response(200, [], json_encode([
'operation_id' => $expiredNetworkJob->operation_id,
'status' => 'succeeded',
...CloudOperationRecoveryIdentity::expected(
$expiredNetworkJob,
true
),
'result' => ['network_suspended' => true],
])));
$worker->tick();
$assert(CloudJob::find($expiredNetworkJob->id)->status === 'succeeded'
&& $expiredNetwork->fresh()->network_status === 'blocked_expired',
'Expiry network suspension was recorded as a traffic quota block.');
$purgeStopped = $makeLifecycleInstance(
'66666666-6666-4666-8666-666666666666', 'purge-stop-success-vm',
'purge_stopping', 'blocked_expired');
$purgeStopped->status = 'deleting';
$purgeStopped->save();
$purgeStopJob = $queuePolicy(
$purgeStopped, 'service_suspend', 'service/suspend', 'purge');
$mock->append(new Response(200, [], json_encode([
'operation_id' => $purgeStopJob->operation_id,
'status' => 'succeeded',
...CloudOperationRecoveryIdentity::expected($purgeStopJob, true),
'result' => ['service_was_running' => true],
])));
$worker->tick();
$purgeStopped->refresh();
$assert(CloudJob::find($purgeStopJob->id)->status === 'succeeded'
&& $purgeStopped->status === 'deleting'
&& $purgeStopped->service_status === 'purge_deleting'
&& $purgeStopped->suspended_at === null,
'Successful expiry shutdown did not advance to disk deletion safely.');
$purgeStopFailed = $makeLifecycleInstance(
'77777777-7777-4777-8777-777777777777', 'purge-stop-failed-vm',
'purge_stopping', 'blocked_expired');
$purgeStopFailed->status = 'deleting';
$purgeStopFailed->save();
$purgeStopFailedJob = $queuePolicy(
$purgeStopFailed, 'service_suspend', 'service/suspend', 'purge');
$mock->append(new Response(409, [], json_encode([
'operation_id' => $purgeStopFailedJob->operation_id,
'status' => 'failed',
...CloudOperationRecoveryIdentity::expected($purgeStopFailedJob),
'error' => ['code' => 'conflict', 'message' => 'shutdown rejected'],
'effect_may_have_applied' => false,
'safe_to_resubmit' => true,
'replayed' => false,
'retryable' => false,
])));
$worker->tick();
$purgeStopFailed->refresh();
$assert(CloudJob::find($purgeStopFailedJob->id)->status === 'failed'
&& $purgeStopFailed->status === 'deleting'
&& $purgeStopFailed->service_status === 'purge_stop_failed',
'Failed expiry shutdown incorrectly advanced or failed the disk purge.');
$purgeForceRequired = $makeLifecycleInstance(
'76767676-7676-4676-8676-767676767676',
'purge-force-required-vm', 'purge_stopping', 'blocked_expired');
$purgeForceRequired->status = 'deleting';
$purgeForceRequired->save();
$purgeForcePrior = $queuePolicy(
$purgeForceRequired, 'service_suspend', 'service/suspend', 'purge');
$mock->append(new Response(409, [], json_encode([
'operation_id' => $purgeForcePrior->operation_id,
'status' => 'failed',
...CloudOperationRecoveryIdentity::expected($purgeForcePrior),
'error' => ['code' => 'force_required'],
'replayed' => false,
'retryable' => false,
])));
$worker->tick();
$purgeForceRequired->refresh();
$purgeForcePrior->refresh();
$assert($purgeForcePrior->phase === 'terminal'
&& $purgeForcePrior->outcome === 'failed'
&& $purgeForcePrior->status === 'failed'
&& $purgeForcePrior->error_code === 'force_required'
&& $purgeForceRequired->status === 'deleting'
&& $purgeForceRequired->service_status === 'purge_force_required',
'Expiry graceful shutdown crossed the explicit force boundary.');
// A force-required graceful shutdown is already a terminal failed task.
// Any later destructive force action must be a separately audited task;
// inspection never rewrites the prior terminal receipt.
Capsule::table('cloud_backup_subscriptions')->insert([
'instance_id' => (int) $purgeForceRequired->id,
'user_id' => (int) $purgeForceRequired->user_id,
'offer_id' => (int) $purgeForceRequired->offer_id,
'enabled' => 1,
'renew_next_period' => 1,
'frequency' => 'daily',
'weekday' => 0,
'retention_count' => 2,
'timezone' => 'UTC',
'price' => '2.50',
'currency' => 'USD',
'price_version' => 1,
'next_run_at' => time() + 3600,
'created_at' => time(),
'updated_at' => time(),
]);
$purgeForceJob = CloudJobService::enqueueForceDelete(
$purgeForcePrior, $adminActor
);
$mock->append(new Response(200, [], json_encode([
'operation_id' => $purgeForceJob->operation_id,
'status' => 'succeeded',
...CloudOperationRecoveryIdentity::expected($purgeForceJob, true),
'result' => [
'deleted' => true, 'managed_artifacts_absent' => true,
'forced' => true,
],
])));
$worker->tick();
$purgeForcedRequest = $agentRequest(end($history));
$purgeForcedBody = json_decode($purgeForcedRequest['body'], true);
$assert($purgeForceJob->fresh()->status === 'succeeded'
&& $purgeForcePrior->fresh()->status === 'failed'
&& $purgeForcePrior->fresh()->error_code === 'force_required'
&& $purgeForceRequired->fresh()->status === 'deleted'
&& (int) Capsule::table('cloud_backup_subscriptions')
->where('instance_id', (int) $purgeForceRequired->id)
->value('enabled') === 0
&& (int) Capsule::table('cloud_backup_subscriptions')
->where('instance_id', (int) $purgeForceRequired->id)
->value('renew_next_period') === 0
&& $purgeForcedRequest['method'] === 'POST'
&& str_ends_with($purgeForcedRequest['request_target'], '/force-delete')
&& ($purgeForcedBody['prior_operation_id'] ?? null)
=== $purgeForcePrior->operation_id
&& ($purgeForcedBody['ack_data_loss'] ?? null) === true,
'Expiry force-required state did not reach the separately audited force route.');
$expiryDeleted = $makeLifecycleInstance(
'88888888-8888-4888-8888-888888888888', 'expiry-delete-success-vm',
'purge_deleting', 'blocked_expired');
$expiryDeleted->status = 'deleting';
$expiryDeleted->save();
$olderBacklog = CloudJobService::enqueue($instance, $actor, 'action', 'POST',
'/actions', ['action' => 'reboot']);
$expiryDeleteJob = CloudJobService::enqueueExpiryDelete($expiryDeleted);
$beforeExpiryDeleteRequests = count($history);
$user->enable = 0;
$user->save();
$mock->append(new Response(200, [], json_encode([
'operation_id' => $expiryDeleteJob->operation_id,
'status' => 'succeeded',
...CloudOperationRecoveryIdentity::expected($expiryDeleteJob, true),
'result' => [
'deleted' => true,
'managed_artifacts_absent' => true,
'forced' => true,
],
])));
$worker->tick();
$user->enable = 1;
$user->save();
$expiryDeleteRequests = array_slice($history, $beforeExpiryDeleteRequests);
$expiryDeleteRequest = count($expiryDeleteRequests) === 1
? $agentRequest($expiryDeleteRequests[0]) : [];
$expiryDeleteActor = count($expiryDeleteRequests) === 1
? json_decode(base64_decode(strtr(
$expiryDeleteRequest['headers']['X-Virt-Actor'],
'-_', '+/'
)), true) : [];
$expiryDeleted->refresh();
$assert(count($expiryDeleteRequests) === 1
&& $expiryDeleteRequest['method'] === 'DELETE'
&& !str_contains($expiryDeleteRequest['request_target'], '?')
&& json_decode($expiryDeleteRequest['body'], true)
=== ['remove_managed_disks' => true]
&& ($expiryDeleteActor['can_manage'] ?? false) === true
&& ($expiryDeleteActor['operation_id'] ?? null)
=== $expiryDeleteJob->operation_id
&& !array_key_exists('system_cleanup', $expiryDeleteActor)
&& $expiryDeleteActor['is_admin'] === false
&& CloudJob::find($expiryDeleteJob->id)->phase === 'terminal'
&& CloudJob::find($expiryDeleteJob->id)->outcome === 'failed'
&& CloudJob::find($expiryDeleteJob->id)->status === 'failed'
&& (bool) CloudJob::find($expiryDeleteJob->id)
->effect_may_have_applied === true
&& (bool) CloudJob::find($expiryDeleteJob->id)
->safe_to_resubmit === false
&& $expiryDeleted->status === 'delete_failed'
&& $expiryDeleted->service_status === 'purge_failed'
&& $expiryDeleted->sealed_credential !== null,
'Expiry cleanup accepted force evidence on the graceful route or released panel state.');
$olderBacklog->phase = 'terminal';
$olderBacklog->outcome = 'failed';
$olderBacklog->status = 'failed';
$olderBacklog->terminal_at = time();
$olderBacklog->effect_may_have_applied = 0;
$olderBacklog->safe_to_resubmit = 1;
$olderBacklog->effect_resolution = 'not_applied';
$olderBacklog->available_at = 0;
$olderBacklog->save();
$beforeExpiryDeleteRetry = count($history);
$worker->tick();
$assert(count($history) === $beforeExpiryDeleteRetry
&& $expiryDeleteJob->fresh()->status === 'failed'
&& (int) $expiryDeleteJob->fresh()->dispatch_attempts === 1,
'Terminal expiry deletion was automatically looked up or mutation-retried.');
$forgedCleanup = $makeLifecycleInstance(
'89898989-8989-4898-8898-898989898989', 'forged-expiry-delete-vm',
'purge_deleting', 'blocked_expired');
$forgedCleanup->status = 'deleting';
$forgedCleanup->save();
$forgedCleanupJob = CloudJobService::enqueueExpiryDelete($forgedCleanup);
$forgedPayload = \App\Services\Cloud\CloudSecrets::decrypt(
$forgedCleanupJob->sealed_payload,
'job-payload'
);
$forgedPayload['request_target'] = '/v1/vms/' . $instance->uuid;
$forgedCleanupJob->sealed_payload = \App\Services\Cloud\CloudSecrets::encrypt(
$forgedPayload,
'job-payload'
);
$forgedCleanupJob->save();
$beforeForgedCleanupRequests = count($history);
$worker->tick();
$forgedCleanup->refresh();
$assert(count($history) === $beforeForgedCleanupRequests
&& CloudJob::find($forgedCleanupJob->id)->status === 'failed'
&& $forgedCleanup->status === 'delete_failed'
&& $forgedCleanup->service_status === 'purge_failed',
'A forged cleanup payload crossed its bound instance or reached the Agent.');
$expiryDeleteFailed = $makeLifecycleInstance(
'99999999-9999-4999-8999-999999999999', 'expiry-delete-failed-vm',
'purge_deleting', 'blocked_expired');
$expiryDeleteFailed->status = 'deleting';
$expiryDeleteFailed->save();
$expiryDeleteFailedJob = CloudJobService::enqueueExpiryDelete($expiryDeleteFailed);
$mock->append(new Response(409, [], json_encode([
'operation_id' => $expiryDeleteFailedJob->operation_id,
'status' => 'failed',
...CloudOperationRecoveryIdentity::expected($expiryDeleteFailedJob),
'error' => [
'code' => 'conflict',
'message' => 'volume delete rejected before undefine',
],
'effect_may_have_applied' => false,
'safe_to_resubmit' => true,
'replayed' => false,
'retryable' => false,
])));
$worker->tick();
$expiryDeleteFailed->refresh();
$assert(CloudJob::find($expiryDeleteFailedJob->id)->status === 'failed'
&& $expiryDeleteFailed->status === 'delete_failed'
&& $expiryDeleteFailed->service_status === 'purge_failed'
&& $expiryDeleteFailed->sealed_credential !== null,
'Explicit expiry deletion failure did not enter the manual-review state.');
$beforeFailedExpiryRetry = count($history);
$worker->tick();
$expiryDeleteFailedJob->refresh();
$assert(count($history) === $beforeFailedExpiryRetry
&& $expiryDeleteFailedJob->phase === 'terminal'
&& $expiryDeleteFailedJob->outcome === 'failed'
&& (int) $expiryDeleteFailedJob->dispatch_attempts === 1,
'Terminal expiry failure was automatically looked up, reconciled, or mutation-retried.');
$reinstall = CloudJobService::enqueue($instance, $actor, 'reinstall', 'POST', '/reinstall', ['template_id' => $reinstallTemplate->id, 'confirmation_name' => $instance->name, 'root_password' => 'NewStrongPassword!123'], ['password' => 'NewStrongPassword!123', 'login_name' => 'root', 'auth_mode' => 'password']);
$oldCredential = $instance->sealed_credential;
$queuedReinstall = \App\Services\Cloud\CloudSecrets::decrypt(
(string) $reinstall->sealed_payload, 'job-payload'
);
$assert(($queuedReinstall['template_metadata']['template_id'] ?? null)
=== $reinstallTemplate->id
&& ($queuedReinstall['template_metadata']['os_source'] ?? null)
=== 'cloud_template'
&& ($queuedReinstall['template_metadata']['os_name'] ?? null)
=== 'Debian 13'
&& ($queuedReinstall['template_metadata']['login_name'] ?? null)
=== 'root'
&& ($queuedReinstall['template_metadata']['debian_12_network_fix']
?? null) === false
&& !array_key_exists('os_source', $queuedReinstall['body'] ?? [])
&& !array_key_exists('os_name', $queuedReinstall['body'] ?? [])
&& !array_key_exists('login_name', $queuedReinstall['body'] ?? []),
'Reinstall did not keep PHP template metadata outside the strict Agent DTO.');
$mock->append(new Response(202, [], json_encode([
'operation_id' => $reinstall->operation_id,
'status' => 'reserved',
...CloudOperationRecoveryIdentity::expected($reinstall),
'replayed' => false,
'retryable' => false,
]))); $worker->tick();
$reinstallRequest = $agentRequest(end($history));
$reinstallBody = json_decode(
(string) ($reinstallRequest['body'] ?? ''), true, 16,
JSON_THROW_ON_ERROR
);
$reinstallKeys = array_keys($reinstallBody);
sort($reinstallKeys);
$expectedReinstallKeys = [
'root_password', 'template_generation_token', 'template_id',
'template_revision',
];
sort($expectedReinstallKeys);
$assert(CloudJob::find($reinstall->id)->status === 'running' && $instance->fresh()->sealed_credential === $oldCredential, 'Pending reinstall replaced password prematurely.');
$assert($reinstallKeys === $expectedReinstallKeys
&& !array_key_exists('confirmation_name', $reinstallBody),
'Browser reinstall confirmation leaked into the strict Agent DTO.');
$beforeReinstallReceipt = count($history);
$reinstallReceipt = CloudOperationCompletionService::accept([
'type' => 'completion',
'id' => bin2hex(random_bytes(16)),
'agent_id' => (string) $instance->host_id,
'operation_id' => $reinstall->operation_id,
...CloudOperationRecoveryIdentity::expected($reinstall, true),
'status' => 'succeeded',
'effect_may_have_applied' => false,
'safe_to_resubmit' => false,
'error_code' => '',
'result' => [
'state' => 'running',
],
]);
$reinstalled = $instance->fresh();
$reinstalledSpec = json_decode((string) $reinstalled->spec_json, true,
flags: JSON_THROW_ON_ERROR);
$assert($reinstallReceipt === ['accepted' => true,
'disposition' => 'applied']
&& count($history) === $beforeReinstallReceipt
&& CloudJob::find($reinstall->id)->status === 'succeeded'
&& $reinstalled->sealed_credential !== $oldCredential
&& ($reinstalledSpec['template_id'] ?? null) === $reinstallTemplate->id
&& ($reinstalledSpec['public_network'] ?? null)
=== $openingSpec['public_network']
&& ($reinstalledSpec['os_source'] ?? null) === 'cloud_template'
&& ($reinstalledSpec['os_name'] ?? null) === 'Debian 13'
&& ($reinstalledSpec['login_name'] ?? null) === 'root'
&& ($reinstalledSpec['managed'] ?? null) === true
&& !isset($reinstalledSpec['install_iso'],
$reinstalledSpec['agent_metadata_version']),
'Completed reinstall lost its trusted template metadata.');
// A replacement guest may be running while old-volume cleanup remains
// unresolved. The worker must keep the job uncertain but still persist
// the new credential and template metadata; raw host warnings stay out of
// result_json.
$partialReinstall = CloudJobService::enqueue($instance->fresh(), $actor,
'reinstall', 'POST', '/reinstall', [
'template_id' => $input['template_id'],
'confirmation_name' => $instance->name,
'root_password' => 'PartialStrongPassword!123',
], ['password' => 'PartialStrongPassword!123', 'login_name' => 'root',
'auth_mode' => 'password']);
$mock->append(new Response(202, [], json_encode([
'operation_id' => $partialReinstall->operation_id,
'status' => 'reserved',
...CloudOperationRecoveryIdentity::expected($partialReinstall),
'replayed' => false,
'retryable' => false,
])));
$worker->tick();
$beforePartialReceipt = count($history);
$partialReceipt = CloudOperationCompletionService::accept([
'type' => 'completion',
'id' => bin2hex(random_bytes(16)),
'agent_id' => (string) $instance->host_id,
'operation_id' => $partialReinstall->operation_id,
...CloudOperationRecoveryIdentity::expected(
$partialReinstall,
true
),
'status' => 'succeeded',
'effect_may_have_applied' => false,
'safe_to_resubmit' => false,
'error_code' => '',
'result' => [
'reinstall_committed' => true,
'cleanup_pending' => true,
'vm' => ['uuid' => (string) $instance->uuid,
'name' => $instance->name, 'state' => 'running'],
],
]);
$partialJob = CloudJob::find($partialReinstall->id);
$assert($partialReceipt === ['accepted' => true,
'disposition' => 'received']
&& count($history) === $beforePartialReceipt
&& $partialJob->phase === 'terminal'
&& $partialJob->outcome === 'failed'
&& $partialJob->status === 'failed'
&& (int) $partialJob->dispatch_attempts === 1
&& (bool) $partialJob->effect_may_have_applied === true
&& (bool) $partialJob->safe_to_resubmit === false
&& $instance->fresh()->sealed_credential
=== $reinstalled->sealed_credential,
'Cleanup-pending reinstall was acknowledged or did not become a terminal fenced failure.');
$beforePartialRetry = count($history);
$worker->tick();
$assert(count($history) === $beforePartialRetry
&& CloudJob::find($partialReinstall->id)->status === 'failed',
'Cleanup-pending reinstall was automatically looked up or retried.');
$partialJob->effect_resolution = 'applied';
$partialJob->effect_resolved_at = time();
$partialJob->effect_resolved_by = 99;
$partialJob->effect_resolution_reason =
'test fixture confirms replacement guest committed';
$partialJob->save();
// A transition-owned network change crosses the complete Worker path
// exactly once and publishes only the exact typed public_network result.
$networkTargetIp = new CloudPublicIp();
$networkTargetIp->host_id = (int) $host->id;
$networkTargetIp->public_ip_pool_id = (int) $sourcePublicPool->id;
$networkTargetIp->address_family = 4;
$networkTargetIp->address = '8.8.8.200';
$networkTargetIp->address_binary = inet_pton('8.8.8.200');
$networkTargetIp->prefix_length = 24;
$networkTargetIp->state = 'available';
$networkTargetIp->created_at = $networkTargetIp->updated_at = time();
$networkTargetIp->save();
$networkDispatch = CloudPublicNetworkTransitionWorkflow::begin(
CloudPublicNetworkTransitionCommand::adminExact(
(int) $instance->id,
99,
'financial-network-transition-0001',
[4 => [
'pool_key' => $sourcePoolKey,
'address_cidr' => '8.8.8.200/24',
]]
)
);
$networkJob = CloudJob::find($networkDispatch->jobId());
$networkPayload = \App\Services\Cloud\CloudSecrets::decrypt(
(string) $networkJob->sealed_payload,
'job-payload'
);
$networkIdentity = CloudOperationRecoveryIdentity::expectedFromPayload(
$networkJob,
$networkPayload,
true
);
$networkTarget = $networkDispatch->target()->toArray();
$mock->append(new Response(200, [], json_encode([
'operation_id' => (string) $networkJob->operation_id,
'status' => 'succeeded',
...$networkIdentity,
'result' => [
'uuid' => (string) $instance->uuid,
'network_changed' => true,
'public_network' => $networkTarget,
],
], JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES)));
$beforeNetworkDispatch = count($history);
$assert($worker->tick(),
'Transition-owned network task was not consumed by the Worker.');
$networkRequest = $agentRequest($history[$beforeNetworkDispatch]);
$networkBody = json_decode(
(string) $networkRequest['body'],
true,
32,
JSON_THROW_ON_ERROR
);
$networkSpec = json_decode(
(string) $instance->fresh()->spec_json,
true,
64,
JSON_THROW_ON_ERROR
);
$assert(count($history) === $beforeNetworkDispatch + 1
&& ($networkBody['public_network'] ?? null) === $networkTarget
&& CloudJob::find($networkJob->id)->phase === 'terminal'
&& CloudJob::find($networkJob->id)->outcome === 'succeeded'
&& (int) CloudJob::find($networkJob->id)->dispatch_attempts === 1
&& Capsule::table('cloud_public_network_transitions')
->where('id', $networkDispatch->transitionId())
->value('state') === 'succeeded'
&& ($networkSpec['public_network'] ?? null) === $networkTarget,
'Worker did not dispatch/finalize the exact transition projection.');
$afterNetworkDispatch = count($history);
$worker->tick();
$assert(count($history) === $afterNetworkDispatch
&& (int) CloudJob::find($networkJob->id)->dispatch_attempts === 1,
'A terminal transition-owned network task was replayed.');
// A legacy scalar task that lacks transition ownership fails before the
// Agent. Use an isolated instance so its durable incident cannot affect
// the canonical instance's later lifecycle tests.
$legacyNetworkInstance = $makeLifecycleInstance(
'45454545-4545-4545-8545-454545454545',
'legacy-network-gate-vm',
'active',
'active'
);
$legacyWorkerJob = new CloudJob();
$legacyWorkerJob->operation_id = bin2hex(random_bytes(16));
$legacyWorkerJob->request_idempotency_key =
'legacy-network-worker-gate-0001';
$legacyWorkerJob->request_fingerprint = hash(
'sha256',
'legacy-network-worker-gate-0001'
);
$legacyWorkerJob->instance_id = (int) $legacyNetworkInstance->id;
$legacyWorkerJob->user_id = (int) $legacyNetworkInstance->user_id;
$legacyWorkerJob->host_id = (int) $legacyNetworkInstance->host_id;
$legacyWorkerJob->kind = 'network_change';
$legacyWorkerJob->execution_mode = 'agent';
$legacyWorkerJob->status = 'queued';
$legacyWorkerJob->public_network_transition_id = null;
$legacyWorkerJob->sealed_payload =
\App\Services\Cloud\CloudSecrets::encrypt([
'method' => 'POST',
'request_target' => '/v1/vms/'
. rawurlencode((string) $legacyNetworkInstance->uuid)
. '/network',
'body' => [
'public_ip_cidr' => '9.9.9.10/24',
'confirm_restart' => true,
],
'credential' => null,
'owner_id' => (int) $legacyNetworkInstance->user_id,
'can_manage' => false,
], 'job-payload');
$legacyWorkerJob->available_at = $legacyWorkerJob->created_at =
$legacyWorkerJob->updated_at = time();
$legacyWorkerJob->save();
$beforeLegacyWorker = count($history);
$assert($worker->tick(),
'Legacy network task was not consumed by the local fail-closed path.');
$legacyWorkerJob->refresh();
$assert(count($history) === $beforeLegacyWorker
&& (int) $legacyWorkerJob->dispatch_attempts === 0
&& $legacyWorkerJob->phase === 'terminal'
&& $legacyWorkerJob->outcome === 'failed',
'Legacy network task reached the Agent or remained claimable.');
$worker->tick();
$assert(count($history) === $beforeLegacyWorker
&& (int) $legacyWorkerJob->fresh()->dispatch_attempts === 0,
'Legacy network task was replayed after local rejection.');
// A stale migration row that still carries the retired scalar public-IP
// request shape must fail before the accepted-to-dispatching CAS. The
// immutable migration/instance/reservation ledgers are the only authority
// for a PeerMigration receive request.
$legacyMigrationJob = new CloudJob();
$legacyMigrationJob->operation_id = bin2hex(random_bytes(16));
$legacyMigrationJob->request_idempotency_key =
'legacy-migration-worker-gate-0001';
$legacyMigrationJob->request_fingerprint = hash(
'sha256',
'legacy-migration-worker-gate-0001'
);
$legacyMigrationJob->instance_id = (int) $legacyNetworkInstance->id;
$legacyMigrationJob->user_id = (int) $legacyNetworkInstance->user_id;
$legacyMigrationJob->host_id = (int) $legacyNetworkInstance->host_id;
$legacyMigrationJob->kind = 'instance_migration_transfer';
$legacyMigrationJob->execution_mode = 'agent';
$legacyMigrationJob->status = 'queued';
$legacyMigrationStoredIdentity = [
'version' => 1,
'action' => 'peer-migration-receive',
'resource_id' => 'migration:987654',
'request_fingerprint' => str_repeat('a', 64),
];
$legacyMigrationJob->sealed_payload =
\App\Services\Cloud\CloudSecrets::encrypt([
\App\Services\Cloud\CloudOperationRecoveryIdentity::payloadKey()
=> $legacyMigrationStoredIdentity,
'_operation_recovery' => [
'version' => 1,
'kind' => 'instance_migration_transfer',
'prepared_marker' => 'prepared:test:legacy-migration',
'operation' => [
'operation_id' =>
(string) $legacyMigrationJob->operation_id,
'action' => $legacyMigrationStoredIdentity['action'],
'resource_id' =>
$legacyMigrationStoredIdentity['resource_id'],
'request_fingerprint' =>
$legacyMigrationStoredIdentity[
'request_fingerprint'
],
],
],
'method' => 'POST',
'request_target' => '/v1/vms/'
. rawurlencode((string) $legacyNetworkInstance->uuid)
. '/peer-migration/receive',
'body' => [
'operation_id' => (string) $legacyMigrationJob->operation_id,
'public_ip_cidr' => '9.9.9.10/24',
'gateway' => '9.9.9.1',
],
'maintenance_migration_id' => 987654,
'owner_id' => (int) $legacyNetworkInstance->user_id,
'can_manage' => false,
], 'job-payload');
$legacyMigrationJob->available_at = $legacyMigrationJob->created_at =
$legacyMigrationJob->updated_at = time();
$legacyMigrationJob->save();
$beforeLegacyMigration = count($history);
$assert($worker->tick(),
'Legacy migration task was not consumed by local admission.');
$legacyMigrationJob->refresh();
$assert(count($history) === $beforeLegacyMigration
&& (int) $legacyMigrationJob->dispatch_attempts === 0
&& $legacyMigrationJob->phase === 'terminal'
&& $legacyMigrationJob->outcome === 'failed',
'Legacy migration task reached the Agent or remained claimable.');
$worker->tick();
$assert(count($history) === $beforeLegacyMigration
&& (int) $legacyMigrationJob->fresh()->dispatch_attempts === 0,
'Legacy migration task was replayed after local rejection.');
// The ordinary job enqueue path is permanently closed to legacy scalar
// network changes. Canonical transitions have their own durable workflow
// and are tested in the dedicated begin/finalization behavior suites.
$networkInput = ['public_ip_cidr' => '9.9.9.10/24',
'public_pool_key' => $targetPoolKey, 'confirm_restart' => true];
$beforeLegacyNetworkJobs = CloudJob::count();
try {
CloudJobService::enqueue(
$instance->fresh(),
$actor,
'network_change',
'POST',
'/network',
$networkInput
);
throw new RuntimeException(
'Legacy scalar network writer bypassed the transition gate.'
);
} catch (CloudError $error) {
$assert($error->httpStatus === 409
&& CloudJob::count() === $beforeLegacyNetworkJobs,
'Transition gate created a network job before rejecting it.');
}
// Stage 5 keeps one executable legacy-gate assertion above; paid charge,
// idempotency, not-applied refund and unknown-effect recovery are covered
// by the canonical transition Workflow and Finalization behavior suites.
// Deletion consumes the unchanged exact dual-stack authority and never
// reconstructs it from retained singular reservation columns.
$canonicalClaimSet =
\App\Services\Cloud\CloudPublicNetworkAllocationService::currentClaimSet(
(int) $openingReservation->id,
'committed',
(int) $instance->id
);
$canonicalClaimIds = [
$canonicalClaimSet->claimId(4),
$canonicalClaimSet->claimId(6),
];
$canonicalPublicIpIds = [
$canonicalClaimSet->publicIpId(4),
$canonicalClaimSet->publicIpId(6),
];
$interruptedDelete = CloudJobService::enqueue($instance, $actor,
'delete', 'DELETE', '',
['remove_managed_disks' => true]);
$mock->append(new Response(422, [], json_encode([
'operation_id' => $interruptedDelete->operation_id,
'status' => 'failed',
...CloudOperationRecoveryIdentity::expected($interruptedDelete),
'error' => ['code' => 'graceful_shutdown_unconfirmed'],
'effect_may_have_applied' => false,
'safe_to_resubmit' => true,
'replayed' => false,
'retryable' => false,
])));
$worker->tick();
$interruptedDelete->refresh();
$assert($interruptedDelete->status === 'failed'
&& $interruptedDelete->error_code === 'graceful_shutdown_unconfirmed'
&& $instance->fresh()->status === 'active',
'An unconfirmed graceful delete did not close safely for a new normal retry.');
$delete = CloudJobService::enqueue($instance->fresh(), $actor,
'delete', 'DELETE', '',
['remove_managed_disks' => true]);
$assert($delete->operation_id !== $interruptedDelete->operation_id,
'A graceful retry reused the interrupted delete operation identifier.');
$mock->append(new Response(409, [], json_encode([
'operation_id' => $delete->operation_id,
'status' => 'failed',
...CloudOperationRecoveryIdentity::expected($delete),
'error' => ['code' => 'force_required'],
'replayed' => false,
'retryable' => false,
])));
$worker->tick();
$graceful = $agentRequest(end($history));
$delete->refresh();
$assert(json_decode($graceful['body'], true)
=== ['remove_managed_disks' => true]
&& !str_contains($graceful['request_target'], '?')
&& $delete->phase === 'terminal'
&& $delete->outcome === 'failed'
&& $delete->status === 'failed'
&& $delete->error_code === 'force_required'
&& $instance->fresh()->status === 'active',
'Graceful delete did not stop at the explicit force-required boundary.');
$failedForceDelete = CloudJobService::enqueueForceDelete(
$delete,
$adminActor
);
$mock->append(new Response(422, [], json_encode([
'operation_id' => $failedForceDelete->operation_id,
'status' => 'failed',
...CloudOperationRecoveryIdentity::expected($failedForceDelete),
'error' => ['code' => 'invalid_request'],
'effect_may_have_applied' => false,
'safe_to_resubmit' => true,
'replayed' => false,
'retryable' => false,
])));
$worker->tick();
$delete->refresh();
$failedForceDelete->refresh();
$assert($failedForceDelete->status === 'failed'
&& $delete->status === 'failed'
&& $delete->error_code === 'force_required'
&& $instance->fresh()->status === 'active',
'A definitely failed force attempt did not safely reopen the explicit force gate.');
$forceDelete = CloudJobService::enqueueForceDelete(
$delete,
$adminActor
);
$mock->append(new Response(200, [], json_encode([
'operation_id' => $forceDelete->operation_id,
'status' => 'succeeded',
...CloudOperationRecoveryIdentity::expected($forceDelete, true),
'result' => [
'uuid' => (string) $instance->uuid,
'deleted' => true,
'managed_artifacts_absent' => true,
'forced' => true,
],
])));
$worker->tick();
$forced = $agentRequest(end($history));
$forcedBody = json_decode($forced['body'], true);
$retainedDeleteReservation = CloudResourceReservation::where(
'order_id', $first->id
)->first();
$assert($forced['method'] === 'POST'
&& str_ends_with($forced['request_target'], '/force-delete')
&& $forceDelete->operation_id !== $failedForceDelete->operation_id
&& ($forcedBody['operation_id'] ?? null) === $forceDelete->operation_id
&& ($forcedBody['prior_operation_id'] ?? null) === $delete->operation_id
&& ($forcedBody['ack_data_loss'] ?? null) === true
&& ($forcedBody['remove_managed_disks'] ?? null) === true
&& $forceDelete->fresh()->status === 'succeeded'
&& $delete->fresh()->status === 'failed'
&& $delete->fresh()->error_code === 'force_required'
&& $instance->fresh()->status === 'deleted'
&& $instance->fresh()->sealed_credential === null
&& $retainedDeleteReservation->state === 'released'
&& CloudPublicIp::whereIn('id', $canonicalPublicIpIds)
->where('state', 'available')
->whereNull('reservation_id')
->whereNull('reservation_ip_claim_id')
->whereNull('instance_id')
->whereNull('operation_id')->count() === 2
&& Capsule::table('cloud_resource_reservation_ips')
->whereIn('id', $canonicalClaimIds)
->where('state', 'released')
->whereNull('active_claim')->count() === 2,
'Force-delete identity, acknowledgement, cleanup, or immutable prior receipt failed.');
$beforeForceReplay = count($history);
$worker->tick();
$assert(count($history) === $beforeForceReplay
&& $forceDelete->fresh()->phase === 'terminal'
&& (int) $forceDelete->fresh()->dispatch_attempts === 1,
'Successful force-delete task was automatically re-dispatched.');
// Independent backup restores have no purchase order. Their committed
// capacity and fresh IP must still be released after a proven deletion.
$restoreOperationId = str_repeat('e', 32);
$restoredUuid = 'eeeeeeee-eeee-4eee-8eee-eeeeeeeeeeee';
$backupRestore = new CloudBackupRestore();
$backupRestore->restore_key = str_repeat('f', 32);
$backupRestore->backup_id = 9001;
$backupRestore->target_host_id = (int) $host->id;
$backupRestore->target_instance_id = null;
$backupRestore->reservation_id = null;
$backupRestore->job_id = null;
$backupRestore->operation_id = $restoreOperationId;
$backupRestore->new_uuid = $restoredUuid;
$backupRestore->new_name = 'restored-delete-vm';
$backupRestore->state = 'ready';
$backupRestore->error_code = '';
$backupRestore->ready_at = time();
$backupRestore->created_at = $backupRestore->updated_at = time();
$backupRestore->save();
$restoredIp = new CloudPublicIp();
$restoredIp->host_id = (int) $host->id;
$restoredIp->public_ip_pool_id = (int) $targetPublicPool->id;
$restoredIp->address_family = 4;
$restoredIp->address = '9.9.9.12';
$restoredIp->address_binary = inet_pton('9.9.9.12');
$restoredIp->prefix_length = 24;
$restoredIp->state = 'assigned';
$restoredIp->reservation_id = null;
$restoredIp->instance_id = null;
$restoredIp->operation_id = null;
$restoredIp->created_at = $restoredIp->updated_at = time();
$restoredIp->save();
$restoredReservation = new CloudResourceReservation();
$restoredReservation->reservation_key = str_repeat('d', 32);
$restoredReservation->operation_id = $restoreOperationId;
$restoredReservation->order_id = null;
$restoredReservation->backup_restore_id = (int) $backupRestore->id;
$restoredReservation->offer_id = null;
$restoredReservation->resource_pool_id = (int) $pool->id;
$restoredReservation->host_id = (int) $host->id;
$restoredReservation->user_id = 1;
$restoredReservation->instance_id = null;
$restoredReservation->network_attachment_id =
(int) $targetAttachment->id;
$restoredReservation->public_ip_pool_id = (int) $targetPublicPool->id;
$restoredReservation->public_ip_id = (int) $restoredIp->id;
$restoredReservation->cpu_cores = 2;
$restoredReservation->memory_mib = 2048;
$restoredReservation->disk_gib = 20;
$restoredReservation->state = 'committed';
$restoredReservation->expires_at = null;
$restoredReservation->released_at = null;
$restoredReservation->release_reason = '';
$restoredReservation->created_at = $restoredReservation->updated_at = time();
$restoredReservation->save();
$restoredInstance = $instance->replicate();
$restoredInstance->uuid = $restoredUuid;
$restoredInstance->host_id = (int) $host->id;
$restoredInstance->user_id = 1;
$restoredInstance->order_id = null;
$restoredInstance->plan_id = null;
$restoredInstance->offer_id = null;
$restoredInstance->resource_pool_id = (int) $pool->id;
$restoredInstance->reservation_id = (int) $restoredReservation->id;
$restoredInstance->current_period_id = null;
$restoredInstance->name = 'restored-delete-vm';
$restoredInstance->status = 'active';
$restoredInstance->service_status = 'exempt';
$restoredInstance->network_status = 'active';
$restoredInstance->expires_at = null;
$restoredInstance->spec_json = json_encode([
'managed' => true,
'vcpu' => 2,
'memory_mib' => 2048,
'disk_gib' => 20,
'public_ip_cidr' => '9.9.9.12/24',
], JSON_THROW_ON_ERROR);
$restoredInstance->sealed_credential = 'sealed-restored-credential';
$restoredInstance->created_at = $restoredInstance->updated_at = time();
$restoredInstance->save();
$restoredReservation->instance_id = (int) $restoredInstance->id;
$restoredReservation->save();
$restoredIp->reservation_id = (int) $restoredReservation->id;
$restoredIp->instance_id = (int) $restoredInstance->id;
$restoredIp->operation_id = $restoreOperationId;
$restoredIp->save();
$backupRestore->target_instance_id = (int) $restoredInstance->id;
$backupRestore->reservation_id = (int) $restoredReservation->id;
$backupRestore->save();
$beforeRestoredDeleteJobs = CloudJob::count();
$beforeRestoredDeleteRequests = count($history);
try {
CloudJobService::enqueue(
$restoredInstance,
$adminActor,
'delete',
'DELETE',
'',
['remove_managed_disks' => true]
);
throw new RuntimeException(
'A restored instance bypassed the dual-stack transition gate.'
);
} catch (CloudError $error) {
$assert($error->httpStatus === 409,
'Restored-instance delete gate returned the wrong status.');
}
$restoredInstance->refresh();
$restoredReservation->refresh();
$restoredIp->refresh();
$assert(CloudJob::count() === $beforeRestoredDeleteJobs
&& count($history) === $beforeRestoredDeleteRequests
&& $restoredInstance->status === 'active'
&& $restoredReservation->state === 'committed'
&& $restoredIp->state === 'assigned'
&& (int) $restoredIp->reservation_id
=== (int) $restoredReservation->id
&& (int) $restoredIp->instance_id === (int) $restoredInstance->id,
'Restored-instance delete gate created work or changed its old ledger.');
$restoredInstance->status = 'deleting';
$restoredInstance->service_status = 'purge_deleting';
$restoredInstance->save();
try {
CloudJobService::enqueueExpiryDelete($restoredInstance);
throw new RuntimeException(
'A restored instance bypassed the expiry-delete transition gate.'
);
} catch (CloudError $error) {
$assert($error->httpStatus === 409,
'Restored-instance expiry-delete gate returned the wrong status.');
}
$assert(CloudJob::count() === $beforeRestoredDeleteJobs
&& count($history) === $beforeRestoredDeleteRequests,
'Restored-instance expiry-delete gate created durable work.');
// This clean-break fixture intentionally models the retired singular
// restore shape. Remove it explicitly after proving the gate so it cannot
// poison later active-pair placement scans in this long scenario.
$restoredInstance->delete();
$restoredIp->delete();
$restoredReservation->delete();
$backupRestore->delete();
// Provider callbacks credit the wallet before the invoice bridge runs.
// That credit must be consumed once, queue one create job, and remain
// idempotent when the payment provider retries its callback.
$providerOrder = CloudOrderService::purchase(array_replace($input, [
'name' => 'vm-provider-payment',
'idempotency_key' => 'test-request-key-provider-payment',
]), $actor);
$providerInvoice = CloudInvoice::where('order_id', $providerOrder->id)->first();
$providerPayment = new CloudInvoicePayment();
$providerPayment->invoice_id = $providerInvoice->id;
$providerPayment->user_id = 1;
$providerPayment->provider = 'paytaro';
$providerPayment->provider_order_no = 'PT-PROVISION-0001';
$providerPayment->idempotency_key = 'provision-provider-payment-0001';
$providerPayment->amount = '9.99';
$providerPayment->status = 'pending';
$providerPayment->created_at = $providerPayment->updated_at = time();
$providerPayment->save();
$balanceBeforeProviderCredit = CloudMoney::cents((string) User::find(1)->money);
$user = User::find(1);
$user->money = CloudMoney::decimal($balanceBeforeProviderCredit + 999);
$user->save();
$providerOutcome = CloudInvoicePaymentService::settleForProvider(
'paytaro', 'PT-PROVISION-0001');
$providerOutcomeAgain = CloudInvoicePaymentService::settleForProvider(
'paytaro', 'PT-PROVISION-0001');
$providerQueuedCreate = \App\Services\Cloud\CloudSecrets::decrypt(
(string) CloudJob::where('order_id', $providerOrder->id)
->first()->sealed_payload,
'job-payload'
);
$providerPolicy = $providerQueuedCreate['body']
['platform_network_policy_snapshot'] ?? null;
$assert($providerOutcome === 'paid_invoice'
&& $providerOutcomeAgain === 'paid_invoice'
&& CloudMoney::cents((string) User::find(1)->money) === $balanceBeforeProviderCredit
&& CloudInvoice::find($providerInvoice->id)->status === 'paid'
&& CloudOrder::find($providerOrder->id)->status === 'queued'
&& CloudJob::where('order_id', $providerOrder->id)->count() === 1
&& ($providerPolicy['catalog_revision'] ?? null) === 2
&& count($providerPolicy['rules'] ?? []) === 9,
'Provider callback did not settle an opening invoice exactly once.');
// A blank optional label is normalized before hashing, but generated only
// for a genuinely new order. Replaying the same request with an explicit
// empty field must therefore return the original generated display name.
$automaticInput = array_replace($input, [
'idempotency_key' => 'test-request-key-automatic-name',
]);
unset($automaticInput['name']);
$automatic = CloudOrderService::purchase($automaticInput, $actor);
$automaticReplay = CloudOrderService::purchase(
$automaticInput + ['name' => ''], $actor
);
$automaticSpec = json_decode((string) $automatic->spec_json, true,
flags: JSON_THROW_ON_ERROR);
$assert($automatic->id === $automaticReplay->id
&& preg_match('/^vm-[abcdefghjkmnpqrstuvwxyz23456789]{6}$/D',
(string) ($automaticSpec['name'] ?? '')) === 1,
'Blank instance names were not generated securely or idempotently.');
// Repeated provider settlement reuses the one accepted provisioning task.
// It never creates a recovery generation or an automatic retry.
$providerJob = CloudJob::where('order_id', $providerOrder->id)->first();
$providerOrderAgain = CloudOrderService::queuePaidProvision(
CloudInvoice::find($providerInvoice->id),
CloudOrder::find($providerOrder->id)
);
$assert($providerJob !== null
&& $providerJob->phase === 'accepted'
&& $providerJob->outcome === null
&& $providerJob->status === 'queued'
&& (int) $providerJob->dispatch_attempts === 0
&& (string) $providerOrderAgain->status === 'queued'
&& CloudJob::where('order_id', $providerOrder->id)->count() === 1,
'A repeated paid callback duplicated or retried its durable create task.');
// A definitively not-applied lifecycle failure starts a new audited
// generation after backoff. The failed job and its operation ID remain
// immutable; unknown, applied, or sticky failures remain fail-closed.
$lifecycle = new CloudLifecycleService();
$reconcile = (new ReflectionClass(CloudLifecycleService::class))
->getMethod('reconcile');
$oldPurgeOperation = (string) $purgeStopFailedJob->operation_id;
$purgeCount = CloudJob::where('instance_id', $purgeStopFailed->id)
->where('kind', 'service_suspend')->count();
$assert($reconcile->invoke($lifecycle, (int) $purgeStopFailed->id) === false
&& CloudJob::where('instance_id', $purgeStopFailed->id)
->where('kind', 'service_suspend')->count() === $purgeCount,
'Lifecycle policy failure ignored the mandatory retry backoff.');
Capsule::table('cloud_jobs')->where('id', $purgeStopFailedJob->id)
->update(['terminal_at' => time() - 310, 'updated_at' => time() - 310]);
$assert($reconcile->invoke($lifecycle, (int) $purgeStopFailed->id) === true,
'A definitively not-applied purge stop did not create a new generation: '
. json_encode($purgeStopFailedJob->fresh()->toArray(),
JSON_THROW_ON_ERROR));
$purgeRetry = CloudJob::where('instance_id', $purgeStopFailed->id)
->where('kind', 'service_suspend')->orderByDesc('id')->first();
$assert($purgeRetry !== null
&& (int) $purgeRetry->id !== (int) $purgeStopFailedJob->id
&& (string) $purgeRetry->operation_id !== $oldPurgeOperation
&& (string) $purgeStopFailedJob->fresh()->operation_id
=== $oldPurgeOperation
&& (string) $purgeStopFailedJob->fresh()->phase === 'terminal'
&& (string) $purgeStopFailed->fresh()->service_status
=== 'purge_stopping',
'Purge retry replayed or rewrote the failed generation.');
$oldDeleteOperation = (string) $expiryDeleteFailedJob->operation_id;
Capsule::table('cloud_jobs')->where('id', $expiryDeleteFailedJob->id)
->update(['terminal_at' => time() - 310, 'updated_at' => time() - 310]);
$assert($reconcile->invoke($lifecycle, (int) $expiryDeleteFailed->id) === true,
'A definitively not-applied expiry delete did not create a new generation.');
$deleteRetry = CloudJob::where('instance_id', $expiryDeleteFailed->id)
->where('kind', 'expiry_delete')->orderByDesc('id')->first();
$assert($deleteRetry !== null
&& (int) $deleteRetry->id !== (int) $expiryDeleteFailedJob->id
&& (string) $deleteRetry->operation_id !== $oldDeleteOperation
&& (string) $expiryDeleteFailedJob->fresh()->operation_id
=== $oldDeleteOperation
&& (string) $expiryDeleteFailedJob->fresh()->phase === 'terminal'
&& (string) $expiryDeleteFailed->fresh()->status === 'deleting'
&& (string) $expiryDeleteFailed->fresh()->service_status
=== 'purge_deleting',
'Expiry-delete retry replayed or rewrote the failed generation.');
$finalizeUndispatchedFailure = static function (
CloudJob $job
): CloudJob {
$past = time() - 310;
$job->attempts = 0;
$job->dispatch_attempts = 0;
$job->dispatch_started_at = null;
$job->agent_accepted_at = null;
$job->dispatched_at = null;
$job->receipt_deadline_at = null;
$job->completion_event_id = null;
$job->phase = 'terminal';
$job->outcome = 'failed';
$job->status = 'failed';
$job->terminal_at = $past;
$job->effect_may_have_applied = 0;
$job->safe_to_resubmit = 1;
$job->effect_resolution = 'not_applied';
$job->error_code = 'dispatch_not_sent';
$job->finalization_incident = 0;
$job->finalization_error_code = null;
$job->updated_at = $past;
$job->save();
return $job;
};
$undispatchedDelete = $makeLifecycleInstance(
'90909090-9090-4090-8090-909090909090',
'undispatched-delete-vm',
'purge_deleting',
'blocked_expired'
);
$undispatchedDelete->status = 'deleting';
$undispatchedDelete->save();
$oldUndispatchedDelete = $finalizeUndispatchedFailure(
CloudJobService::enqueueExpiryDelete($undispatchedDelete)
);
$undispatchedDelete->status = 'delete_failed';
$undispatchedDelete->service_status = 'purge_failed';
$undispatchedDelete->save();
$assert($reconcile->invoke(
$lifecycle,
(int) $undispatchedDelete->id
) === true, 'A generation-zero expiry failure did not create a safe new generation.');
$undispatchedDeleteRetry = CloudJob::where(
'instance_id',
$undispatchedDelete->id
)->where('kind', 'expiry_delete')->orderByDesc('id')->first();
$assert($undispatchedDeleteRetry !== null
&& (int) $undispatchedDeleteRetry->id
!== (int) $oldUndispatchedDelete->id
&& (string) $undispatchedDeleteRetry->operation_id
!== (string) $oldUndispatchedDelete->operation_id
&& (int) $oldUndispatchedDelete->fresh()->dispatch_attempts === 0,
'Generation-zero expiry retry replayed or rewrote the prior operation.');
$undispatchedPurgeStop = $makeLifecycleInstance(
'97979797-9797-4797-8797-979797979797',
'undispatched-purge-stop-vm',
'purge_stopping',
'blocked_expired'
);
$undispatchedPurgeStop->status = 'deleting';
$undispatchedPurgeStop->save();
$oldUndispatchedPurgeStop = $finalizeUndispatchedFailure($queuePolicy(
$undispatchedPurgeStop,
'service_suspend',
'service/suspend',
'purge'
));
$undispatchedPurgeStop->service_status = 'purge_stop_failed';
$undispatchedPurgeStop->save();
$assert($reconcile->invoke(
$lifecycle,
(int) $undispatchedPurgeStop->id
) === true, 'A generation-zero purge stop did not create a safe new generation.');
$undispatchedPurgeStopRetry = CloudJob::where(
'instance_id',
$undispatchedPurgeStop->id
)->where('kind', 'service_suspend')->orderByDesc('id')->first();
$assert($undispatchedPurgeStopRetry !== null
&& (int) $undispatchedPurgeStopRetry->id
!== (int) $oldUndispatchedPurgeStop->id
&& (string) $undispatchedPurgeStopRetry->operation_id
!== (string) $oldUndispatchedPurgeStop->operation_id
&& (int) $oldUndispatchedPurgeStop->fresh()->dispatch_attempts === 0
&& (string) $undispatchedPurgeStop->fresh()->service_status
=== 'purge_stopping',
'Generation-zero purge-stop retry replayed or rewrote the prior operation.');
$preparePolicyFailure = static function (
CloudInstance $target,
string $kind,
string $suffix,
?string $reason
) use ($queuePolicy): CloudJob {
$job = $queuePolicy($target, $kind, $suffix, $reason);
$identity = CloudOperationRecoveryIdentity::expected($job);
$past = time() - 310;
$job->request_fingerprint = $identity['request_fingerprint'];
$job->attempts = 1;
$job->dispatch_attempts = 1;
$job->dispatch_started_at = $past - 30;
$job->agent_accepted_at = $past - 20;
$job->dispatched_at = $past - 20;
$job->receipt_deadline_at = $past - 10;
$job->phase = 'terminal';
$job->outcome = 'failed';
$job->status = 'failed';
$job->terminal_at = $past;
$job->effect_may_have_applied = 0;
$job->safe_to_resubmit = 1;
$job->effect_resolution = 'not_applied';
$job->error_code = 'operation_failed';
$job->finalization_incident = 0;
$job->finalization_error_code = null;
$job->updated_at = $past;
$job->save();
return $job;
};
$setPolicyResolution = static function (
CloudJob $job,
string $resolution,
bool $incident = false
): CloudJob {
if ($resolution === 'applied') {
$job->outcome = 'succeeded';
$job->status = 'succeeded';
$job->effect_may_have_applied = 0;
$job->safe_to_resubmit = 0;
$job->error_code = null;
} elseif ($resolution === 'unknown') {
$job->outcome = 'failed';
$job->status = 'failed';
$job->effect_may_have_applied = 1;
$job->safe_to_resubmit = 0;
$job->error_code = 'operation_failed_effect_unknown';
}
$job->effect_resolution = $resolution;
$job->finalization_incident = $incident ? 1 : 0;
$job->finalization_error_code = $incident
? 'recovery_ledger_conflict' : null;
$job->save();
return $job;
};
$insertScheduledPeriod = static function (
CloudInstance $target
): int {
$now = time();
return (int) Capsule::table('cloud_service_periods')->insertGetId([
'instance_id' => $target->id,
'user_id' => $target->user_id,
'plan_id' => $target->plan_id,
'starts_at' => $now - 10,
'ends_at' => $now + 3600,
'traffic_quota_bytes' => 1000,
'download_base_bytes' => null,
'upload_base_bytes' => null,
'download_used_bytes' => 0,
'upload_used_bytes' => 0,
'status' => 'scheduled',
'created_at' => $now - 100,
'updated_at' => $now - 100,
]);
};
// Activating a scheduled paid period may update the billing ledger, but
// it must not overwrite a policy projection whose Agent conclusion is
// still unknown, contradictory, or sticky.
$scheduledSuspend = $makeLifecycleInstance(
'80808080-8080-4080-8080-808080808080',
'scheduled-suspend-conflict-vm',
'suspend_failed',
'active'
);
$scheduledSuspend->expires_at = time() + 3600;
$scheduledSuspend->current_period_id = null;
$scheduledSuspend->save();
$scheduledSuspendJob = $setPolicyResolution($preparePolicyFailure(
$scheduledSuspend,
'service_suspend',
'service/suspend',
'purge'
), 'unknown');
$scheduledSuspendPeriod = $insertScheduledPeriod($scheduledSuspend);
$reconcile->invoke($lifecycle, (int) $scheduledSuspend->id);
$assert((int) $scheduledSuspend->fresh()->current_period_id
=== $scheduledSuspendPeriod
&& (string) CloudServicePeriod::find($scheduledSuspendPeriod)->status
=== 'active'
&& (string) $scheduledSuspend->fresh()->service_status
=== 'suspend_failed'
&& (string) $scheduledSuspendJob->fresh()->effect_resolution
=== 'unknown',
'Scheduled activation falsely cleared an unknown service suspension.');
$scheduledBlock = $makeLifecycleInstance(
'81818181-8181-4181-8181-818181818181',
'scheduled-block-conflict-vm',
'expired_grace',
'block_failed'
);
$scheduledBlock->expires_at = time() + 3600;
$scheduledBlock->current_period_id = null;
$scheduledBlock->save();
$scheduledBlockJob = $setPolicyResolution($preparePolicyFailure(
$scheduledBlock,
'network_suspend',
'network/suspend',
'expired'
), 'applied');
$scheduledBlockPeriod = $insertScheduledPeriod($scheduledBlock);
$reconcile->invoke($lifecycle, (int) $scheduledBlock->id);
$assert((int) $scheduledBlock->fresh()->current_period_id
=== $scheduledBlockPeriod
&& (string) $scheduledBlock->fresh()->network_status
=== 'block_failed'
&& (bool) $scheduledBlockJob->fresh()->finalization_incident
&& (string) $scheduledBlockJob->fresh()->finalization_error_code
=== 'recovery_instance_state_conflict',
'Scheduled activation hid an applied suspension/local-state conflict.');
$scheduledRestore = $makeLifecycleInstance(
'82828282-8282-4282-8282-828282828282',
'scheduled-restore-conflict-vm',
'expired_grace',
'restore_failed'
);
$scheduledRestore->expires_at = time() + 3600;
$scheduledRestore->current_period_id = null;
$scheduledRestore->save();
$scheduledRestoreJob = $setPolicyResolution($preparePolicyFailure(
$scheduledRestore,
'network_resume',
'network/resume',
null
), 'not_applied', true);
$scheduledRestorePeriod = $insertScheduledPeriod($scheduledRestore);
$reconcile->invoke($lifecycle, (int) $scheduledRestore->id);
$assert((int) $scheduledRestore->fresh()->current_period_id
=== $scheduledRestorePeriod
&& (string) $scheduledRestore->fresh()->network_status
=== 'restore_failed'
&& (bool) $scheduledRestoreJob->fresh()->finalization_incident,
'Scheduled activation falsely cleared a sticky network restore failure.');
$makeImmediatelyRenewable = static function (
CloudInstance $target
): void {
$past = time() - 60;
$target->current_period_id = null;
$target->expires_at = $past;
$target->renewal_days = 30;
$target->renewal_price = '1.00';
$target->renewal_traffic_bytes = 1000;
$target->traffic_synced_at = time() + 10;
$target->save();
Capsule::table('cloud_service_periods')->insert([
'instance_id' => $target->id,
'user_id' => $target->user_id,
'plan_id' => $target->plan_id,
'starts_at' => $past - 86400,
'ends_at' => $past,
'traffic_quota_bytes' => 1000,
'download_base_bytes' => 0,
'upload_base_bytes' => 0,
'download_used_bytes' => 0,
'upload_used_bytes' => 0,
'status' => 'finished',
'created_at' => $past - 86400,
'updated_at' => $past,
]);
$issueOne = new ReflectionMethod(
CloudInvoiceService::class,
'issueOne'
);
if ($issueOne->invoke(null, (int) $target->id, time()) !== true) {
throw new RuntimeException(
'Failed to create the renewal invoice fixture.'
);
}
};
$user->money = '1000.00';
$user->save();
$renewUnknownSuspend = $makeLifecycleInstance(
'83838383-8383-4383-8383-838383838383',
'renew-unknown-suspend-vm',
'suspend_failed',
'active'
);
$makeImmediatelyRenewable($renewUnknownSuspend);
$setPolicyResolution($preparePolicyFailure(
$renewUnknownSuspend,
'service_suspend',
'service/suspend',
'purge'
), 'unknown');
$unknownRenewal = CloudRenewalService::renew(
$renewUnknownSuspend,
['idempotency_key' => 'renew-unknown-suspend-0001'],
$actor
);
$assert((string) CloudServicePeriod::find(
$unknownRenewal->period_id
)->status === 'active'
&& (string) $renewUnknownSuspend->fresh()->service_status
=== 'suspend_failed',
'Immediate renewal falsely cleared an unknown service suspension.');
$renewAppliedBlock = $makeLifecycleInstance(
'84848484-8484-4484-8484-848484848484',
'renew-applied-block-vm',
'expired_grace',
'block_failed'
);
$makeImmediatelyRenewable($renewAppliedBlock);
$setPolicyResolution($preparePolicyFailure(
$renewAppliedBlock,
'network_suspend',
'network/suspend',
'expired'
), 'applied');
CloudRenewalService::renew(
$renewAppliedBlock,
['idempotency_key' => 'renew-applied-block-0001'],
$actor
);
$assert((string) $renewAppliedBlock->fresh()->network_status
=== 'block_failed',
'Immediate renewal hid an applied network suspension conflict.');
$renewStickyRestore = $makeLifecycleInstance(
'85858585-8585-4585-8585-858585858585',
'renew-sticky-restore-vm',
'expired_grace',
'restore_failed'
);
$makeImmediatelyRenewable($renewStickyRestore);
$setPolicyResolution($preparePolicyFailure(
$renewStickyRestore,
'network_resume',
'network/resume',
null
), 'not_applied', true);
CloudRenewalService::renew(
$renewStickyRestore,
['idempotency_key' => 'renew-sticky-restore-0001'],
$actor
);
$assert((string) $renewStickyRestore->fresh()->network_status
=== 'restore_failed',
'Immediate renewal cleared a sticky network restore failure.');
$renewSafeSuspend = $makeLifecycleInstance(
'86868686-8686-4686-8686-868686868686',
'renew-safe-suspend-vm',
'suspend_failed',
'active'
);
$makeImmediatelyRenewable($renewSafeSuspend);
$safeSuspendJob = $preparePolicyFailure(
$renewSafeSuspend,
'service_suspend',
'service/suspend',
'purge'
);
CloudRenewalService::renew(
$renewSafeSuspend,
['idempotency_key' => 'renew-safe-suspend-0001'],
$actor
);
$assert((string) $renewSafeSuspend->fresh()->service_status
=== 'suspend_failed',
'Immediate renewal bypassed lifecycle verification of a safe failure.');
$reconcile->invoke($lifecycle, (int) $renewSafeSuspend->id);
$assert((string) $renewSafeSuspend->fresh()->service_status === 'active'
&& (string) $safeSuspendJob->fresh()->phase === 'terminal',
'Lifecycle did not clear a canonical not-applied failure after renewal.');
$trafficRetryInstance = $makeLifecycleInstance(
'91919191-9191-4191-8191-919191919191',
'traffic-retry-vm',
'active',
'block_failed'
);
$trafficRetryInstance->expires_at = time() + 86400;
$trafficRetryInstance->lifetime_download_bytes = 200;
$trafficRetryInstance->lifetime_upload_bytes = 0;
$trafficRetryInstance->traffic_synced_at = time();
$trafficRetryInstance->save();
$trafficPeriodId = Capsule::table('cloud_service_periods')->insertGetId([
'instance_id' => $trafficRetryInstance->id,
'user_id' => $trafficRetryInstance->user_id,
'starts_at' => time() - 3600,
'ends_at' => time() + 3600,
'traffic_quota_bytes' => 100,
'download_base_bytes' => 0,
'upload_base_bytes' => 0,
'download_used_bytes' => 200,
'upload_used_bytes' => 0,
'status' => 'exhausted',
'created_at' => time() - 3600,
'updated_at' => time() - 310,
]);
$trafficRetryInstance->current_period_id = $trafficPeriodId;
$trafficRetryInstance->save();
$oldTrafficJob = $preparePolicyFailure(
$trafficRetryInstance,
'network_suspend',
'network/suspend',
'traffic'
);
$assert($reconcile->invoke(
$lifecycle,
(int) $trafficRetryInstance->id
) === true, 'A still-required traffic block did not create a new generation.');
$trafficRetry = CloudJob::where('instance_id', $trafficRetryInstance->id)
->where('kind', 'network_suspend')->orderByDesc('id')->first();
$assert($trafficRetry !== null
&& (int) $trafficRetry->id !== (int) $oldTrafficJob->id
&& (string) $trafficRetry->operation_id
!== (string) $oldTrafficJob->operation_id
&& (string) $oldTrafficJob->fresh()->phase === 'terminal'
&& (string) $trafficRetryInstance->fresh()->network_status
=== 'blocking',
'Traffic-block retry mutated or replayed its prior generation.');
$undispatchedTraffic = $makeLifecycleInstance(
'96969696-9696-4696-8696-969696969696',
'undispatched-traffic-vm',
'active',
'block_failed'
);
$undispatchedTraffic->expires_at = time() + 86400;
$undispatchedTraffic->lifetime_download_bytes = 200;
$undispatchedTraffic->lifetime_upload_bytes = 0;
$undispatchedTraffic->traffic_synced_at = time();
$undispatchedTraffic->save();
$undispatchedTrafficPeriod = Capsule::table(
'cloud_service_periods'
)->insertGetId([
'instance_id' => $undispatchedTraffic->id,
'user_id' => $undispatchedTraffic->user_id,
'starts_at' => time() - 3600,
'ends_at' => time() + 3600,
'traffic_quota_bytes' => 100,
'download_base_bytes' => 0,
'upload_base_bytes' => 0,
'download_used_bytes' => 200,
'upload_used_bytes' => 0,
'status' => 'exhausted',
'created_at' => time() - 3600,
'updated_at' => time() - 310,
]);
$undispatchedTraffic->current_period_id = $undispatchedTrafficPeriod;
$undispatchedTraffic->save();
$oldUndispatchedTraffic = $finalizeUndispatchedFailure($queuePolicy(
$undispatchedTraffic,
'network_suspend',
'network/suspend',
'traffic'
));
$assert($reconcile->invoke(
$lifecycle,
(int) $undispatchedTraffic->id
) === true, 'A generation-zero traffic failure did not create a safe new generation.');
$undispatchedTrafficRetry = CloudJob::where(
'instance_id',
$undispatchedTraffic->id
)->where('kind', 'network_suspend')->orderByDesc('id')->first();
$assert($undispatchedTrafficRetry !== null
&& (int) $undispatchedTrafficRetry->id
!== (int) $oldUndispatchedTraffic->id
&& (string) $undispatchedTrafficRetry->operation_id
!== (string) $oldUndispatchedTraffic->operation_id
&& (int) $oldUndispatchedTraffic->fresh()->dispatch_attempts === 0,
'Generation-zero traffic retry replayed or rewrote the prior operation.');
$resumeRetryInstance = $makeLifecycleInstance(
'92929292-9292-4292-8292-929292929292',
'resume-retry-vm',
'active',
'restore_failed'
);
$resumeRetryInstance->expires_at = time() + 86400;
$resumeRetryInstance->lifetime_download_bytes = 50;
$resumeRetryInstance->lifetime_upload_bytes = 0;
$resumeRetryInstance->traffic_synced_at = time();
$resumeRetryInstance->save();
$resumePeriodId = Capsule::table('cloud_service_periods')->insertGetId([
'instance_id' => $resumeRetryInstance->id,
'user_id' => $resumeRetryInstance->user_id,
'starts_at' => time() - 3600,
'ends_at' => time() + 3600,
'traffic_quota_bytes' => 1000,
'download_base_bytes' => 0,
'upload_base_bytes' => 0,
'download_used_bytes' => 50,
'upload_used_bytes' => 0,
'status' => 'active',
'created_at' => time() - 3600,
'updated_at' => time() - 310,
]);
$resumeRetryInstance->current_period_id = $resumePeriodId;
$resumeRetryInstance->save();
$oldResumeJob = $preparePolicyFailure(
$resumeRetryInstance,
'network_resume',
'network/resume',
null
);
$assert($reconcile->invoke(
$lifecycle,
(int) $resumeRetryInstance->id
) === true, 'A still-required network restore did not create a new generation.');
$resumeRetry = CloudJob::where('instance_id', $resumeRetryInstance->id)
->where('kind', 'network_resume')->orderByDesc('id')->first();
$assert($resumeRetry !== null
&& (int) $resumeRetry->id !== (int) $oldResumeJob->id
&& (string) $resumeRetry->operation_id
!== (string) $oldResumeJob->operation_id
&& (string) $oldResumeJob->fresh()->phase === 'terminal'
&& (string) $resumeRetryInstance->fresh()->network_status
=== 'restoring',
'Network-restore retry mutated or replayed its prior generation.');
$attachActivePeriod = static function (CloudInstance $target): void {
$target->expires_at = time() + 86400;
$target->lifetime_download_bytes = 50;
$target->lifetime_upload_bytes = 0;
$target->traffic_synced_at = time();
$target->save();
$periodId = Capsule::table('cloud_service_periods')->insertGetId([
'instance_id' => $target->id,
'user_id' => $target->user_id,
'starts_at' => time() - 3600,
'ends_at' => time() + 3600,
'traffic_quota_bytes' => 1000,
'download_base_bytes' => 0,
'upload_base_bytes' => 0,
'download_used_bytes' => 50,
'upload_used_bytes' => 0,
'status' => 'active',
'created_at' => time() - 3600,
'updated_at' => time() - 310,
]);
$target->current_period_id = $periodId;
$target->save();
};
$undispatchedResume = $makeLifecycleInstance(
'98989898-9898-4898-8898-989898989898',
'undispatched-resume-vm',
'active',
'restore_failed'
);
$attachActivePeriod($undispatchedResume);
$oldUndispatchedResume = $finalizeUndispatchedFailure($queuePolicy(
$undispatchedResume,
'network_resume',
'network/resume'
));
$assert($reconcile->invoke(
$lifecycle,
(int) $undispatchedResume->id
) === true, 'A generation-zero network restore did not create a safe new generation.');
$undispatchedResumeRetry = CloudJob::where(
'instance_id',
$undispatchedResume->id
)->where('kind', 'network_resume')->orderByDesc('id')->first();
$assert($undispatchedResumeRetry !== null
&& (int) $undispatchedResumeRetry->id
!== (int) $oldUndispatchedResume->id
&& (string) $undispatchedResumeRetry->operation_id
!== (string) $oldUndispatchedResume->operation_id
&& (int) $oldUndispatchedResume->fresh()->dispatch_attempts === 0
&& (string) $undispatchedResume->fresh()->network_status
=== 'restoring',
'Generation-zero network-restore retry replayed or rewrote the prior operation.');
$obsoleteSuspend = $makeLifecycleInstance(
'94949494-9494-4494-8494-949494949494',
'obsolete-suspend-vm',
'suspend_failed',
'active'
);
$attachActivePeriod($obsoleteSuspend);
$oldObsoleteSuspend = $preparePolicyFailure(
$obsoleteSuspend,
'service_suspend',
'service/suspend',
'purge'
);
$obsoleteCount = CloudJob::where(
'instance_id',
$obsoleteSuspend->id
)->count();
$assert($reconcile->invoke(
$lifecycle,
(int) $obsoleteSuspend->id
) === false
&& (string) $obsoleteSuspend->fresh()->service_status === 'active'
&& CloudJob::where('instance_id', $obsoleteSuspend->id)->count()
=== $obsoleteCount
&& (string) $oldObsoleteSuspend->fresh()->phase === 'terminal',
'A no-longer-required, definitively not-applied suspend was not safely cleared.');
$uncertainSuspend = $makeLifecycleInstance(
'95959595-9595-4595-8595-959595959595',
'uncertain-suspend-vm',
'suspend_failed',
'active'
);
$attachActivePeriod($uncertainSuspend);
$oldUncertainSuspend = $preparePolicyFailure(
$uncertainSuspend,
'service_suspend',
'service/suspend',
'purge'
);
$oldUncertainSuspend->effect_resolution = 'unknown';
$oldUncertainSuspend->effect_may_have_applied = 1;
$oldUncertainSuspend->safe_to_resubmit = 0;
$oldUncertainSuspend->save();
$assert($reconcile->invoke(
$lifecycle,
(int) $uncertainSuspend->id
) === false
&& (string) $uncertainSuspend->fresh()->service_status
=== 'suspend_failed'
&& (string) $oldUncertainSuspend->fresh()->effect_resolution
=== 'unknown',
'An unknown-effect suspend failure was falsely unlocked by a paid period.');
foreach ([
['unknown', 1, 0, false],
['applied', 0, 0, false],
['not_applied', 0, 1, true],
] as $index => [$resolution, $effect, $safe, $incident]) {
$guarded = $makeLifecycleInstance(
sprintf('93939393-9393-4393-8393-%012d', $index + 1),
'guarded-lifecycle-' . $index,
'purge_deleting',
'blocked_expired'
);
$guarded->status = 'deleting';
$guarded->save();
$guardedJob = CloudJobService::enqueueExpiryDelete($guarded);
$identity = CloudOperationRecoveryIdentity::expected($guardedJob);
$past = time() - 310;
$guardedJob->request_fingerprint = $identity['request_fingerprint'];
$guardedJob->attempts = 1;
$guardedJob->dispatch_attempts = 1;
$guardedJob->dispatch_started_at = $past - 30;
$guardedJob->agent_accepted_at = $past - 20;
$guardedJob->dispatched_at = $past - 20;
$guardedJob->receipt_deadline_at = $past - 10;
$guardedJob->phase = 'terminal';
$guardedJob->terminal_at = $past;
$guardedJob->error_code = 'operation_failed';
$guardedJob->updated_at = $past;
$guardedJob->effect_resolution = $resolution;
$guardedJob->effect_may_have_applied = $effect;
$guardedJob->safe_to_resubmit = $safe;
$guardedJob->outcome = $resolution === 'applied'
? 'succeeded' : 'failed';
$guardedJob->status = $guardedJob->outcome;
$guardedJob->finalization_incident = $incident ? 1 : 0;
$guardedJob->finalization_error_code = $incident
? 'recovery_ledger_conflict' : null;
$guardedJob->save();
$guarded->status = 'delete_failed';
$guarded->service_status = 'purge_failed';
$guarded->save();
$beforeGuarded = CloudJob::where('instance_id', $guarded->id)->count();
$assert($reconcile->invoke($lifecycle, (int) $guarded->id) === false
&& CloudJob::where('instance_id', $guarded->id)->count()
=== $beforeGuarded
&& (string) $guardedJob->fresh()->operation_id
=== (string) $guardedJob->operation_id,
'Unknown, applied, or sticky lifecycle evidence created a new mutation generation.');
}
// A create which crossed the Agent boundary may time out locally before
// the authenticated terminal failure reaches PHP. Exact Agent proof that
// both the guest and every managed artifact are absent must finish the
// original paid order locally; it must never dispatch a second create.
foreach (['8.8.8.20', '8.8.8.21', '8.8.8.22'] as $address) {
if (!CloudPublicIp::where('address', $address)->exists()) {
$ip = new CloudPublicIp();
$ip->host_id = $host->id;
$ip->public_ip_pool_id = $sourcePublicPool->id;
$ip->address_family = 4;
$ip->address = $address;
$ip->address_binary = inet_pton($address);
$ip->prefix_length = 24;
$ip->state = 'available';
$ip->created_at = $ip->updated_at = time();
$ip->save();
}
}
$lateFailureOrder = CloudOrderService::purchase(array_replace($input, [
'name' => 'vm-late-failure',
'idempotency_key' => 'test-request-key-late-failure',
]), $actor);
$lateFailureInvoice = CloudInvoice::where(
'order_id',
$lateFailureOrder->id
)->firstOrFail();
CloudInvoiceService::payBalance(
$lateFailureInvoice,
$actor,
'provision-balance-late-failure'
);
$lateFailureJob = CloudJob::where(
'order_id',
$lateFailureOrder->id
)->firstOrFail();
// Earlier lifecycle tests intentionally leave separately audited retries
// queued. Park those unrelated rows so this tracer observes exactly one
// create dispatch rather than depending on suite-wide queue order.
Capsule::table('cloud_jobs')
->where('id', '<>', (int) $lateFailureJob->id)
->where('phase', '<>', 'terminal')
->update([
'available_at' => time() + 3600,
'dispatch_deadline_at' => time() + 3600,
'receipt_deadline_at' => time() + 3600,
]);
Capsule::table('cloud_jobs')->where(
'id',
(int) $lateFailureJob->id
)->update(['available_at' => 0]);
$lateFailureJob->refresh();
$lateFailureIdentity = CloudOperationRecoveryIdentity::expected(
$lateFailureJob,
false
);
$mock->append(new Response(202, [], json_encode([
'operation_id' => $lateFailureJob->operation_id,
'status' => 'reserved',
...$lateFailureIdentity,
'replayed' => false,
'retryable' => false,
], JSON_THROW_ON_ERROR)));
$beforeLateFailureDispatch = count($history);
$assert($worker->tick(),
'Late-failure create did not cross the Agent dispatch boundary.');
$lateFailureJob->refresh();
$assert((string) $lateFailureJob->phase === 'awaiting_receipt'
&& (int) $lateFailureJob->dispatch_attempts === 1
&& count($history) === $beforeLateFailureDispatch + 1,
'Late-failure create did not persist its single dispatch generation.');
Capsule::table('cloud_jobs')->where(
'id',
(int) $lateFailureJob->id
)->update([
'receipt_deadline_at' => time() - 1,
'lease_token' => null,
'lease_until' => 0,
]);
$assert(CloudWorkerService::expireDueTaskForAdmin(
(int) $lateFailureJob->id
) === 1, 'Late-failure create did not reach its local deadline.');
$lateFailureJob->refresh();
$lateFailureOrder->refresh();
$lateFailureInvoice->refresh();
$lateFailureReservation = CloudResourceReservation::where(
'order_id',
$lateFailureOrder->id
)->firstOrFail();
$walletWhileFenced = (string) User::findOrFail(1)->money;
$stockWhileFenced = (int) CloudOffer::findOrFail(1)->stock;
$assert((string) $lateFailureJob->phase === 'terminal'
&& (string) $lateFailureJob->outcome === 'failed'
&& (string) $lateFailureJob->effect_resolution === 'unknown'
&& (string) $lateFailureOrder->status === 'failed'
&& $lateFailureOrder->refunded_at === null
&& (string) $lateFailureInvoice->status === 'paid'
&& (string) $lateFailureReservation->state === 'uncertain'
&& $bundleState(
$lateFailureReservation,
'uncertain',
'uncertain'
),
'Timed-out dispatched create did not retain its financial fence.');
$lateFailureEventId = 'create-late-failure-event-0001';
$lateFailureReceipt = [
'type' => 'completion',
'id' => $lateFailureEventId,
'agent_id' => (string) $lateFailureJob->host_id,
'operation_id' => (string) $lateFailureJob->operation_id,
...$lateFailureIdentity,
'status' => 'failed',
'effect_may_have_applied' => false,
'safe_to_resubmit' => false,
'error_code' => 'creation_absent',
'result' => [
'uuid' => $lateFailureIdentity['resource_id'],
'creation_absent' => true,
'managed_artifacts_absent' => true,
],
];
$lateFailureAccepted = CloudOperationCompletionService::accept(
$lateFailureReceipt
);
$lateFailureJob->refresh();
$lateFailureOrder->refresh();
$lateFailureInvoice->refresh();
$lateFailureReservation->refresh();
$lateFailureEvent = \App\Models\CloudAgentCompletionEvent::where(
'event_id',
$lateFailureEventId
)->firstOrFail();
$assert($lateFailureAccepted === [
'accepted' => true,
'disposition' => 'applied',
]
&& (string) $lateFailureEvent->disposition === 'applied'
&& $lateFailureEvent->applied_at !== null
&& (string) $lateFailureJob->effect_resolution === 'not_applied'
&& !(bool) $lateFailureJob->finalization_incident
&& (string) $lateFailureOrder->status === 'failed'
&& $lateFailureOrder->refunded_at !== null
&& (string) $lateFailureInvoice->status === 'refunded'
&& (string) $lateFailureReservation->state === 'released'
&& $bundleState(
$lateFailureReservation,
'released',
'available'
)
&& (string) User::findOrFail(1)->money
=== CloudMoney::decimal(
CloudMoney::cents($walletWhileFenced) + 999
)
&& (int) CloudOffer::findOrFail(1)->stock
=== $stockWhileFenced + 1
&& count($history) === $beforeLateFailureDispatch + 1,
'Exact late create failure did not atomically refund, release, ACK, '
. 'or replayed the original mutation.');
$lateFailureWalletAfter = (string) User::findOrFail(1)->money;
$lateFailureStockAfter = (int) CloudOffer::findOrFail(1)->stock;
$lateFailureRefundedAt = (int) $lateFailureOrder->refunded_at;
$lateFailureJobUpdatedAt = (int) $lateFailureJob->updated_at;
$lateFailureAppliedAt = (int) $lateFailureEvent->applied_at;
$lateFailureDuplicate = CloudOperationCompletionService::accept(
$lateFailureReceipt
);
$assert($lateFailureDuplicate === [
'accepted' => true,
'disposition' => 'applied',
]
&& (string) User::findOrFail(1)->money
=== $lateFailureWalletAfter
&& (int) CloudOffer::findOrFail(1)->stock
=== $lateFailureStockAfter
&& (int) CloudOrder::findOrFail(
(int) $lateFailureOrder->id
)->refunded_at === $lateFailureRefundedAt
&& (int) CloudJob::findOrFail(
(int) $lateFailureJob->id
)->updated_at === $lateFailureJobUpdatedAt
&& (int) \App\Models\CloudAgentCompletionEvent::findOrFail(
$lateFailureEventId
)->applied_at === $lateFailureAppliedAt
&& count($history) === $beforeLateFailureDispatch + 1,
'Duplicate late create failure refunded, released, ACKed, or dispatched '
. 'more than once.');
/**
* Create one paid order, cross the Agent boundary exactly once, and force
* the panel-side receipt deadline to the terminal unknown-effect state.
* Later recovery assertions can then focus on finance and evidence rather
* than reproducing dispatch bookkeeping by hand.
*
* @return array<string,mixed>
*/
$makeTimedOutCreate = static function (
string $suffix
) use (
$input,
$actor,
$mock,
$worker,
&$history,
$assert,
$bundleState
): array {
$order = CloudOrderService::purchase(array_replace($input, [
'name' => 'vm-create-recovery-' . $suffix,
'idempotency_key' => 'test-create-recovery-' . $suffix,
]), $actor);
$invoice = CloudInvoice::where('order_id', $order->id)
->firstOrFail();
CloudInvoiceService::payBalance(
$invoice,
$actor,
'provision-create-recovery-' . $suffix
);
$job = CloudJob::where('order_id', $order->id)->firstOrFail();
$future = time() + 3600;
Capsule::table('cloud_jobs')
->where('id', '<>', (int) $job->id)
->update([
'available_at' => $future,
'lease_until' => $future,
]);
Capsule::table('cloud_jobs')->where('id', (int) $job->id)
->update(['available_at' => 0, 'lease_until' => 0]);
$job->refresh();
$identity = CloudOperationRecoveryIdentity::expected($job, false);
$mock->append(new Response(202, [], json_encode([
'operation_id' => $job->operation_id,
'status' => 'reserved',
...$identity,
'replayed' => false,
'retryable' => false,
], JSON_THROW_ON_ERROR)));
$historyBefore = count($history);
$assert($worker->tick(),
'Create recovery fixture did not dispatch: ' . $suffix);
$job->refresh();
$assert((string) $job->phase === 'awaiting_receipt'
&& (int) $job->dispatch_attempts === 1
&& count($history) === $historyBefore + 1,
'Create recovery fixture did not persist one dispatch: '
. $suffix);
Capsule::table('cloud_jobs')->where('id', (int) $job->id)
->update([
'receipt_deadline_at' => time() - 1,
'lease_token' => null,
'lease_until' => 0,
]);
$assert(CloudWorkerService::expireDueTaskForAdmin((int) $job->id)
=== 1, 'Create recovery fixture did not expire: ' . $suffix);
$job->refresh();
$order->refresh();
$invoice->refresh();
$reservation = CloudResourceReservation::where(
'order_id',
$order->id
)->firstOrFail();
$assert((string) $job->phase === 'terminal'
&& (string) $job->outcome === 'failed'
&& (string) $job->effect_resolution === 'unknown'
&& (string) $order->status === 'failed'
&& $order->refunded_at === null
&& (string) $invoice->status === 'paid'
&& (string) $reservation->state === 'uncertain'
&& $bundleState($reservation, 'uncertain', 'uncertain'),
'Create recovery fixture did not retain its safety fence: '
. $suffix);
return compact(
'order', 'invoice', 'job', 'identity', 'reservation',
'historyBefore'
) + [
'walletBefore' => (string) User::findOrFail(1)->money,
'stockBefore' => (int) CloudOffer::findOrFail(1)->stock,
];
};
$makeRecoveryReady = static function (CloudJob $job): void {
$future = time() + 3600;
Capsule::table('cloud_jobs')
->where('id', '<>', (int) $job->id)
->update([
'available_at' => $future,
'lease_until' => $future,
]);
Capsule::table('cloud_jobs')->where('id', (int) $job->id)
->update([
'available_at' => 0,
'lease_token' => null,
'lease_until' => 0,
]);
$job->refresh();
};
$inspectionFor = static function (
CloudJob $job,
array $identity,
bool $fenced = false
): array {
return [
'operation_id' => (string) $job->operation_id,
...$identity,
'operation_status' => 'failed',
'checked_at' => time(),
'outcome' => 'failed',
'summary_code' => 'operation_failed',
'error_code' => 'creation_absent',
'effect_resolution' => 'not_applied',
'fence_state' => $fenced ? 'active' : 'not_required',
'fence_resolution_required' => $fenced,
'effect_projection' => [
'creation_absent' => true,
'managed_artifacts_absent' => true,
],
'abort_job' => ['available' => false],
'cleanup' => ['available' => false],
];
};
$assertCreateSettled = static function (
array $case,
string $message
) use ($assert, $bundleState, &$history): void {
$case['job']->refresh();
$case['order']->refresh();
$case['invoice']->refresh();
$case['reservation']->refresh();
$assert((string) $case['job']->effect_resolution === 'not_applied'
&& !(bool) $case['job']->finalization_incident
&& $case['job']->lease_token === null
&& (string) $case['order']->status === 'failed'
&& $case['order']->refunded_at !== null
&& (string) $case['invoice']->status === 'refunded'
&& (string) $case['reservation']->state === 'released'
&& $bundleState(
$case['reservation'],
'released',
'available'
)
&& (string) User::findOrFail(1)->money
=== CloudMoney::decimal(
CloudMoney::cents($case['walletBefore']) + 999
)
&& (int) CloudOffer::findOrFail(1)->stock
=== $case['stockBefore'] + 1
&& count($history) === $case['historyBefore'] + 1,
$message);
};
$failedCreateReceipt = static function (
array $case,
string $eventId
): array {
return [
'type' => 'completion',
'id' => $eventId,
'agent_id' => (string) $case['job']->host_id,
'operation_id' => (string) $case['job']->operation_id,
...$case['identity'],
'status' => 'failed',
'effect_may_have_applied' => false,
'safe_to_resubmit' => false,
'error_code' => 'creation_absent',
'result' => [
'uuid' => $case['identity']['resource_id'],
'creation_absent' => true,
'managed_artifacts_absent' => true,
],
];
};
// Immutable host absence evidence must wait for a temporarily inconsistent
// local invoice. Keeping the event received allows the exact same receipt
// to finish atomically after the PHP ledger is repaired.
$invoiceGuardCase = $makeTimedOutCreate('receipt-invoice-guard');
$invoiceGuardCase['invoice']->purpose = 'renewal';
$invoiceGuardCase['invoice']->save();
$invoiceGuardReceipt = $failedCreateReceipt(
$invoiceGuardCase,
'create-invoice-guard-failure-event-0001'
);
$invoiceGuardPending = CloudOperationCompletionService::accept(
$invoiceGuardReceipt
);
$invoiceGuardCase['job']->refresh();
$invoiceGuardCase['order']->refresh();
$invoiceGuardCase['reservation']->refresh();
$invoiceGuardEvent = \App\Models\CloudAgentCompletionEvent::findOrFail(
$invoiceGuardReceipt['id']
);
$assert($invoiceGuardPending === [
'accepted' => true,
'disposition' => 'received',
]
&& (string) $invoiceGuardEvent->disposition === 'received'
&& $invoiceGuardEvent->applied_at === null
&& (string) $invoiceGuardCase['job']->effect_resolution === 'unknown'
&& $invoiceGuardCase['order']->refunded_at === null
&& (string) $invoiceGuardCase['reservation']->state === 'uncertain'
&& $bundleState(
$invoiceGuardCase['reservation'],
'uncertain',
'uncertain'
)
&& (string) User::findOrFail(1)->money
=== $invoiceGuardCase['walletBefore']
&& (int) CloudOffer::findOrFail(1)->stock
=== $invoiceGuardCase['stockBefore'],
'Exact create failure bypassed an inconsistent local invoice: '
. json_encode([
'accepted' => $invoiceGuardPending,
'event_disposition' => $invoiceGuardEvent->disposition,
'event_applied_at' => $invoiceGuardEvent->applied_at,
'job_resolution' =>
$invoiceGuardCase['job']->effect_resolution,
'refunded_at' => $invoiceGuardCase['order']->refunded_at,
'reservation' => $invoiceGuardCase['reservation']->state,
'bundle_uncertain' => $bundleState(
$invoiceGuardCase['reservation'],
'uncertain',
'uncertain'
),
'wallet_before' => $invoiceGuardCase['walletBefore'],
'wallet_after' => User::findOrFail(1)->money,
'stock_before' => $invoiceGuardCase['stockBefore'],
'stock_after' => CloudOffer::findOrFail(1)->stock,
], JSON_THROW_ON_ERROR));
$invoiceGuardCase['invoice']->purpose = 'provision';
$invoiceGuardCase['invoice']->save();
$assert(CloudOperationCompletionService::accept($invoiceGuardReceipt)
=== ['accepted' => true, 'disposition' => 'applied'],
'Repaired local invoice did not resume the same create receipt.');
$assertCreateSettled(
$invoiceGuardCase,
'Repaired local invoice did not atomically settle create finance.'
);
// The specialised create finalizer must validate the complete dispatched
// envelope itself. Generic recovery policy deliberately does not own
// create, so neither a damaged attempt count nor a missing receipt
// deadline may reach the refund path.
foreach ([
'attempts' => 2,
'receipt_deadline_at' => null,
] as $damagedField => $damagedValue) {
$suffix = 'receipt-envelope-' . str_replace('_', '-', $damagedField);
$envelopeCase = $makeTimedOutCreate($suffix);
$eventId = 'create-envelope-' . str_replace('_', '-', $damagedField)
. '-event-0001';
$envelopeReceipt = $failedCreateReceipt($envelopeCase, $eventId);
$originalValue = $envelopeCase['job']->{$damagedField};
Capsule::table('cloud_jobs')->where(
'id',
(int) $envelopeCase['job']->id
)->update([$damagedField => $damagedValue]);
$envelopePending = CloudOperationCompletionService::accept(
$envelopeReceipt
);
$envelopeCase['job']->refresh();
$envelopeCase['order']->refresh();
$envelopeCase['invoice']->refresh();
$envelopeCase['reservation']->refresh();
$envelopeEvent = \App\Models\CloudAgentCompletionEvent::findOrFail(
$eventId
);
$assert($envelopePending === [
'accepted' => true,
'disposition' => 'received',
]
&& (string) $envelopeEvent->disposition === 'received'
&& $envelopeEvent->applied_at === null
&& (string) $envelopeCase['job']->effect_resolution === 'unknown'
&& $envelopeCase['order']->refunded_at === null
&& (string) $envelopeCase['invoice']->status === 'paid'
&& (string) $envelopeCase['reservation']->state === 'uncertain'
&& $bundleState(
$envelopeCase['reservation'],
'uncertain',
'uncertain'
)
&& (string) User::findOrFail(1)->money
=== $envelopeCase['walletBefore']
&& (int) CloudOffer::findOrFail(1)->stock
=== $envelopeCase['stockBefore'],
'Damaged create dispatch envelope released finance: '
. $damagedField);
Capsule::table('cloud_jobs')->where(
'id',
(int) $envelopeCase['job']->id
)->update([$damagedField => $originalValue]);
$assert(CloudOperationCompletionService::accept($envelopeReceipt)
=== ['accepted' => true, 'disposition' => 'applied'],
'Repaired create dispatch envelope did not resume receipt: '
. $damagedField);
$assertCreateSettled(
$envelopeCase,
'Repaired dispatch envelope did not settle create finance: '
. $damagedField
);
}
// A fresh read-only Journal failure with complete absence projection can
// finish a timed-out create even when no completion event ever arrived.
$inspectionCase = $makeTimedOutCreate('inspection-no-receipt');
$makeRecoveryReady($inspectionCase['job']);
$inspectionClient = new FinancialCreateRecoveryAgentClient();
$inspectionClient->responses[] = $inspectionFor(
$inspectionCase['job'],
$inspectionCase['identity']
);
CloudAgentSettingsService::$hostId =
(int) $inspectionCase['job']->host_id;
CloudAgentSettingsService::$client = $inspectionClient;
$inspectionRecovery = new CloudAutomaticOperationRecoveryService();
$assert($inspectionRecovery->tick(),
'No-receipt create was not claimed for read-only recovery.');
$assert(count($inspectionClient->requests) === 1
&& $inspectionClient->requests[0]['method'] === 'GET'
&& str_ends_with(
$inspectionClient->requests[0]['target'],
'/inspect'
)
&& $inspectionClient->requests[0]['payload'] === null,
'No-receipt create recovery used something other than one read-only '
. 'inspection.');
$assertCreateSettled(
$inspectionCase,
'Read-only create absence evidence did not atomically settle finance.'
);
// Once inspection has settled the ledger, the first later failure receipt
// is an idempotent ACK of the late delivery. It must not re-enter the received queue,
// rewrite the job incident, or credit/release anything twice.
$inspectionCase['job']->refresh();
$inspectionJobUpdatedAt = (int) $inspectionCase['job']->updated_at;
$inspectionWalletAfter = (string) User::findOrFail(1)->money;
$inspectionStockAfter = (int) CloudOffer::findOrFail(1)->stock;
$inspectionLateReceipt = [
'type' => 'completion',
'id' => 'create-inspection-late-failure-event-0001',
'agent_id' => (string) $inspectionCase['job']->host_id,
'operation_id' => (string) $inspectionCase['job']->operation_id,
...$inspectionCase['identity'],
'status' => 'failed',
'effect_may_have_applied' => false,
'safe_to_resubmit' => false,
'error_code' => 'creation_absent',
'result' => [
'uuid' => $inspectionCase['identity']['resource_id'],
'creation_absent' => true,
'managed_artifacts_absent' => true,
],
];
$inspectionLateAccepted = CloudOperationCompletionService::accept(
$inspectionLateReceipt
);
$inspectionLateDuplicate = CloudOperationCompletionService::accept(
$inspectionLateReceipt
);
$inspectionCase['job']->refresh();
$inspectionLateEvent = \App\Models\CloudAgentCompletionEvent::findOrFail(
$inspectionLateReceipt['id']
);
$assert($inspectionLateAccepted === [
'accepted' => true,
'disposition' => 'applied',
]
&& $inspectionLateDuplicate === $inspectionLateAccepted
&& (string) $inspectionLateEvent->disposition === 'applied'
&& $inspectionLateEvent->applied_at !== null
&& (int) $inspectionCase['job']->updated_at
=== $inspectionJobUpdatedAt
&& !(bool) $inspectionCase['job']->finalization_incident
&& (string) User::findOrFail(1)->money === $inspectionWalletAfter
&& (int) CloudOffer::findOrFail(1)->stock === $inspectionStockAfter,
'A failure receipt arriving after inspection settlement was requeued '
. 'or changed the financial ledger: ' . json_encode([
'first' => $inspectionLateAccepted,
'duplicate' => $inspectionLateDuplicate,
'event_disposition' => $inspectionLateEvent->disposition,
'event_applied_at' => $inspectionLateEvent->applied_at,
'job_updated_at_before' => $inspectionJobUpdatedAt,
'job_updated_at_after' => $inspectionCase['job']->updated_at,
'job_incident' => $inspectionCase['job']->finalization_incident,
'wallet_before' => $inspectionWalletAfter,
'wallet_after' => User::findOrFail(1)->money,
'stock_before' => $inspectionStockAfter,
'stock_after' => CloudOffer::findOrFail(1)->stock,
], JSON_THROW_ON_ERROR));
// An incomplete failed receipt is retained as a late audit event, then a
// fresh read-only inspection supplies the missing managed-artifact proof.
$incompleteCase = $makeTimedOutCreate('incomplete-receipt');
$incompleteReceipt = [
'type' => 'completion',
'id' => 'create-incomplete-failure-event-0001',
'agent_id' => (string) $incompleteCase['job']->host_id,
'operation_id' => (string) $incompleteCase['job']->operation_id,
...$incompleteCase['identity'],
'status' => 'failed',
'effect_may_have_applied' => false,
'safe_to_resubmit' => false,
'error_code' => 'creation_absent',
'result' => [
'uuid' => $incompleteCase['identity']['resource_id'],
'creation_absent' => true,
],
];
$assert(CloudOperationCompletionService::accept($incompleteReceipt)
=== ['accepted' => true, 'disposition' => 'late'],
'Incomplete create absence receipt was not ACKed as late.');
$makeRecoveryReady($incompleteCase['job']);
$incompleteClient = new FinancialCreateRecoveryAgentClient();
$incompleteClient->responses[] = $inspectionFor(
$incompleteCase['job'],
$incompleteCase['identity']
);
CloudAgentSettingsService::$client = $incompleteClient;
$assert((new CloudAutomaticOperationRecoveryService())->tick(),
'Incomplete receipt create did not enter fresh inspection recovery.');
$assertCreateSettled(
$incompleteCase,
'Fresh inspection did not finish an incomplete failed receipt create.'
);
// If PHP fails after durably preparing a no-fence conclusion, a later
// tick must finish locally even while the Agent is offline.
$noFenceRetry = $makeTimedOutCreate('nofence-db-retry');
$noFenceRetry['invoice']->purpose = 'renewal';
$noFenceRetry['invoice']->save();
$makeRecoveryReady($noFenceRetry['job']);
$noFenceClient = new FinancialCreateRecoveryAgentClient();
$noFenceClient->responses[] = $inspectionFor(
$noFenceRetry['job'],
$noFenceRetry['identity']
);
CloudAgentSettingsService::$client = $noFenceClient;
$assert((new CloudAutomaticOperationRecoveryService())->tick(),
'No-fence DB-failure create was not claimed.');
$noFenceRetry['job']->refresh();
$noFenceRetry['order']->refresh();
$noFenceRetry['invoice']->refresh();
$assert(preg_match(
'/^prepared:create_not_applied:v1:[a-f0-9]{64}$/D',
(string) $noFenceRetry['job']->effect_resolution_reason
) === 1
&& (string) $noFenceRetry['job']->effect_resolution === 'unknown'
&& $noFenceRetry['order']->refunded_at === null
&& (string) $noFenceRetry['invoice']->status === 'paid'
&& count($noFenceClient->requests) === 1,
'No-fence financial failure did not retain its durable prepare.');
$noFenceRetry['invoice']->purpose = 'provision';
$noFenceRetry['invoice']->save();
$makeRecoveryReady($noFenceRetry['job']);
// Deliberately leave the response queue empty: any Agent call now fails.
$assert((new CloudAutomaticOperationRecoveryService())->tick(),
'Prepared no-fence create was not retried locally.');
$assert(count($noFenceClient->requests) === 1,
'Prepared no-fence retry contacted an offline Agent.');
$assertCreateSettled(
$noFenceRetry,
'Prepared no-fence create did not settle after the DB fault cleared.'
);
// If the Agent fence write committed but the same PHP transaction later
// failed, recovery reads that immutable verdict once and never POSTs it a
// second time.
$fenceRetry = $makeTimedOutCreate('fence-db-retry');
$fenceRetry['invoice']->purpose = 'renewal';
$fenceRetry['invoice']->save();
$makeRecoveryReady($fenceRetry['job']);
$fenceClient = new FinancialCreateRecoveryAgentClient();
$fenceClient->responses[] = $inspectionFor(
$fenceRetry['job'],
$fenceRetry['identity'],
true
);
$fenceClient->responses[] = new CloudAgentError(
'no committed fence',
404,
'not_found',
false,
404
);
$committedFence = null;
$fenceClient->responses[] = static function (
array $request
) use (&$committedFence, $fenceRetry): array {
$payload = $request['payload'];
$committedFence = [
'operation_id' => (string) $fenceRetry['job']->operation_id,
'resolution_id' => (string) $payload['resolution_id'],
'resolution' => (string) $payload['resolution'],
'inspection_checked_at' =>
(int) $payload['inspection_checked_at'],
'reason_sha256' => (string) $payload['reason_sha256'],
'resolved_at' => time(),
'resolved_by' => 0,
'replayed' => false,
];
return $committedFence;
};
CloudAgentSettingsService::$client = $fenceClient;
$assert((new CloudAutomaticOperationRecoveryService())->tick(),
'Active-fence DB-failure create was not claimed.');
$fenceRetry['job']->refresh();
$assert(is_array($committedFence)
&& count($fenceClient->requests) === 3
&& $fenceClient->requests[0]['method'] === 'GET'
&& str_ends_with($fenceClient->requests[0]['target'], '/inspect')
&& $fenceClient->requests[1]['method'] === 'GET'
&& str_ends_with(
$fenceClient->requests[1]['target'],
'/fence-resolution'
)
&& $fenceClient->requests[2]['method'] === 'POST'
&& str_ends_with(
$fenceClient->requests[2]['target'],
'/resolve-fence'
)
&& preg_match(
'/^prepared:create_not_applied:v1:[a-f0-9]{64}$/D',
(string) $fenceRetry['job']->effect_resolution_reason
) === 1
&& (string) $fenceRetry['job']->effect_resolution === 'unknown',
'Fence commit plus PHP failure did not preserve resumable evidence.');
$fenceRetry['invoice']->purpose = 'provision';
$fenceRetry['invoice']->save();
$makeRecoveryReady($fenceRetry['job']);
$fenceClient->responses[] = $committedFence;
$assert((new CloudAutomaticOperationRecoveryService())->tick(),
'Prepared active-fence create was not retried.');
$assert(count($fenceClient->requests) === 4
&& $fenceClient->requests[3]['method'] === 'GET'
&& str_ends_with(
$fenceClient->requests[3]['target'],
'/fence-resolution'
),
'Prepared active-fence retry wrote or re-inspected Agent state.');
$assertCreateSettled(
$fenceRetry,
'Prepared active-fence create did not settle after DB recovery.'
);
// A fence write can fail before the Agent commits it. The next tick first
// proves that no prior verdict exists, then obtains a fresh inspection.
// checked_at is freshness metadata, not part of the durable conclusion:
// the same facts must produce the same marker and fence reason.
$stableCase = $makeTimedOutCreate('stable-inspection-marker');
$makeRecoveryReady($stableCase['job']);
$stableClient = new FinancialCreateRecoveryAgentClient();
$stableFirstInspection = $inspectionFor(
$stableCase['job'],
$stableCase['identity'],
true
);
$stableClient->responses = [
$stableFirstInspection,
new CloudAgentError('no committed fence', 404, 'not_found', false, 404),
new CloudAgentError('fence write unavailable', 503, 'unavailable', true, 503),
new CloudAgentError('no committed fence', 404, 'not_found', false, 404),
];
CloudAgentSettingsService::$client = $stableClient;
$assert((new CloudAutomaticOperationRecoveryService())->tick(),
'Stable-marker first recovery attempt was not claimed.');
$stableCase['job']->refresh();
$stableMarker = (string) $stableCase['job']->effect_resolution_reason;
$assert(preg_match(
'/^prepared:create_not_applied:v1:[a-f0-9]{64}$/D',
$stableMarker
) === 1
&& (string) $stableCase['job']->effect_resolution === 'unknown'
&& count($stableClient->requests) === 4,
'Failed fence write did not retain a stable prepared create marker.');
$makeRecoveryReady($stableCase['job']);
$stableSecondInspection = $stableFirstInspection;
$stableSecondInspection['checked_at'] =
(int) $stableFirstInspection['checked_at'] + 1;
$stableCommittedFence = null;
$stableClient->responses = [
new CloudAgentError('no committed fence', 404, 'not_found', false, 404),
$stableSecondInspection,
new CloudAgentError('no committed fence', 404, 'not_found', false, 404),
static function (array $request) use (
&$stableCommittedFence,
$stableCase
): array {
$payload = $request['payload'];
$stableCommittedFence = [
'operation_id' => (string) $stableCase['job']->operation_id,
'resolution_id' => (string) $payload['resolution_id'],
'resolution' => (string) $payload['resolution'],
'inspection_checked_at' =>
(int) $payload['inspection_checked_at'],
'reason_sha256' => (string) $payload['reason_sha256'],
'resolved_at' => time(),
'resolved_by' => 0,
'replayed' => false,
];
return $stableCommittedFence;
},
];
$assert((new CloudAutomaticOperationRecoveryService())->tick(),
'Stable-marker create did not resume from fresh inspection.');
$stableExpectedReason =
'自动恢复已确认原开通操作未创建虚拟机或面板管理资源;operation_id='
. (string) $stableCase['job']->operation_id . ';evidence='
. substr($stableMarker, -64);
$assert(is_array($stableCommittedFence)
&& count($stableClient->requests) === 8
&& $stableClient->requests[4]['method'] === 'GET'
&& str_ends_with(
$stableClient->requests[4]['target'],
'/fence-resolution'
)
&& $stableClient->requests[5]['method'] === 'GET'
&& str_ends_with($stableClient->requests[5]['target'], '/inspect')
&& $stableClient->requests[7]['method'] === 'POST'
&& (string) $stableClient->requests[7]['payload']['reason_sha256']
=== hash('sha256', $stableExpectedReason),
'Fresh equivalent inspection changed the prepared fence reason or '
. 'replayed the original create.');
$assertCreateSettled(
$stableCase,
'Stable-marker retry did not finish the create financial ledger.'
);
// Partial/residual host state and an identity mismatch are both
// contradictions, not refund evidence. They remain fenced and retryable.
$conflictCase = $makeTimedOutCreate('inspection-conflicts');
$makeRecoveryReady($conflictCase['job']);
$conflictClient = new FinancialCreateRecoveryAgentClient();
$partialInspection = $inspectionFor(
$conflictCase['job'],
$conflictCase['identity'],
true
);
$partialInspection['outcome'] = 'uncertain';
$partialInspection['summary_code'] = 'evidence_incomplete';
$partialInspection['effect_resolution'] = 'partial';
$partialInspection['error_code'] = 'cleanup_required';
$partialInspection['cleanup'] = [
'available' => true,
'plan_id' => str_repeat('a', 64),
'steps' => ['delete_managed_volumes'],
];
unset($partialInspection['effect_projection']);
$conflictClient->responses[] = $partialInspection;
CloudAgentSettingsService::$client = $conflictClient;
$assert((new CloudAutomaticOperationRecoveryService())->tick(),
'Partial create evidence was not safely deferred.');
$conflictCase['job']->refresh();
$conflictCase['order']->refresh();
$conflictCase['invoice']->refresh();
$assert((string) $conflictCase['job']->effect_resolution === 'unknown'
&& (bool) $conflictCase['job']->finalization_incident
&& $conflictCase['order']->refunded_at === null
&& (string) $conflictCase['invoice']->status === 'paid'
&& (string) User::findOrFail(1)->money
=== $conflictCase['walletBefore']
&& (int) CloudOffer::findOrFail(1)->stock
=== $conflictCase['stockBefore'],
'Partial create evidence released a financial/resource fence.');
$makeRecoveryReady($conflictCase['job']);
$wrongIdentity = $inspectionFor(
$conflictCase['job'],
$conflictCase['identity']
);
$wrongIdentity['resource_id'] = 'wrong-create-resource';
$conflictClient->responses[] = $wrongIdentity;
$assert((new CloudAutomaticOperationRecoveryService())->tick(),
'Create identity contradiction was not safely deferred.');
$conflictCase['job']->refresh();
$assert((string) $conflictCase['job']->effect_resolution === 'unknown'
&& (bool) $conflictCase['job']->finalization_incident
&& $conflictCase['order']->fresh()->refunded_at === null
&& (string) User::findOrFail(1)->money
=== $conflictCase['walletBefore']
&& (int) CloudOffer::findOrFail(1)->stock
=== $conflictCase['stockBefore'],
'Create identity contradiction released a financial/resource fence.');
// A caller cannot reuse an automatic claim after its lease has expired.
$leaseToken = bin2hex(random_bytes(16));
Capsule::table('cloud_jobs')->where('id', (int) $conflictCase['job']->id)
->update([
'available_at' => 0,
'lease_token' => $leaseToken,
'lease_until' => time() + 180,
]);
$staleClaim = CloudJob::findOrFail((int) $conflictCase['job']->id);
Capsule::table('cloud_jobs')->where('id', (int) $conflictCase['job']->id)
->update(['lease_until' => time() - 1]);
$staleRequestCount = count($conflictClient->requests);
$staleRejected = false;
try {
CloudCreateFailureRecoveryService::recover($staleClaim);
} catch (CloudError $error) {
$staleRejected = true;
}
$assert($staleRejected
&& count($conflictClient->requests) === $staleRequestCount
&& (string) CloudJob::findOrFail(
(int) $conflictCase['job']->id
)->effect_resolution === 'unknown'
&& $conflictCase['order']->fresh()->refunded_at === null,
'Expired create recovery lease contacted Agent or released finance.');
// Real address exhaustion on the first scored host is a scheduler miss,
// not a corrupt-order failure. The failed host must leave neither its
// temporary reservation nor a partial address claim before placement
// continues with the next host.
$attachment->state = 'active';
$attachment->verification_state = 'verified';
$attachment->verified_agent_revision = 1;
$attachment->last_verified_at = time();
$attachment->save();
$sourcePublicPool->state = 'active';
$sourcePublicPool->sync_state = 'applied';
$sourcePublicPool->desired_revision = 1;
$sourcePublicPool->observed_revision = 1;
$sourcePublicPool->save();
$sourceIpv6Pool->state = 'active';
$sourceIpv6Pool->sync_state = 'applied';
$sourceIpv6Pool->desired_revision = 1;
$sourceIpv6Pool->observed_revision = 1;
$sourceIpv6Pool->allocation_start = '2404:c140:156:3a::10';
$sourceIpv6Pool->allocation_end = '2404:c140:156:3a::10';
$sourceIpv6Pool->allocation_cursor = null;
$sourceIpv6Pool->max_allocations = 1;
$sourceIpv6Pool->save();
Capsule::table('cloud_public_ip_pools')
->where('id', (int) $sourceIpv6Pool->id)
->update(['allocation_cursor' => null]);
CloudPublicIp::where('public_ip_pool_id', $sourceIpv6Pool->id)->update([
'state' => 'disabled',
'reservation_id' => null,
'reservation_ip_claim_id' => null,
'instance_id' => null,
'operation_id' => null,
'updated_at' => time(),
]);
// Earlier scalar network-change scenarios deliberately reuse and then
// disable concrete IPv4 rows. New provisioning now validates the whole
// pool ledger, so return those test-only rows to the real disabled shape
// before exercising host-capacity fallback.
CloudPublicIp::where('state', 'disabled')->update([
'reservation_id' => null,
'reservation_ip_claim_id' => null,
'instance_id' => null,
'operation_id' => null,
'updated_at' => time(),
]);
if (!CloudPublicIp::where('address_family', 6)
->where('address_binary', inet_pton('2404:c140:156:3a::10'))
->exists()) {
$blockedIpv6 = new CloudPublicIp();
$blockedIpv6->host_id = (int) $host->id;
$blockedIpv6->public_ip_pool_id = (int) $sourceIpv6Pool->id;
$blockedIpv6->address_family = 6;
$blockedIpv6->address = '2404:c140:156:3a::10';
$blockedIpv6->address_binary = inet_pton(
'2404:c140:156:3a::10'
);
$blockedIpv6->prefix_length = 64;
$blockedIpv6->state = 'disabled';
$blockedIpv6->created_at = $blockedIpv6->updated_at = time();
$blockedIpv6->save();
}
if (!CloudPublicIp::where('address_family', 4)
->where('address_binary', inet_pton('8.8.8.23'))->exists()) {
$fallbackProbeIpv4 = new CloudPublicIp();
$fallbackProbeIpv4->host_id = (int) $host->id;
$fallbackProbeIpv4->public_ip_pool_id =
(int) $sourcePublicPool->id;
$fallbackProbeIpv4->address_family = 4;
$fallbackProbeIpv4->address = '8.8.8.23';
$fallbackProbeIpv4->address_binary = inet_pton('8.8.8.23');
$fallbackProbeIpv4->prefix_length = 24;
$fallbackProbeIpv4->state = 'available';
$fallbackProbeIpv4->created_at =
$fallbackProbeIpv4->updated_at = time();
$fallbackProbeIpv4->save();
}
$targetAttachment->state = 'draining';
$targetAttachment->save();
$firstHostPolicy = CloudHostCapacityPolicy::findOrFail((int) $host->id);
$firstHostPolicy->configured_cpu_cores = 1000000;
$firstHostPolicy->configured_memory_mib = 1000000000;
$firstHostPolicy->configured_disk_gib = 1000000000;
$firstHostPolicy->safety_cpu_cores = 0;
$firstHostPolicy->safety_memory_mib = 0;
$firstHostPolicy->safety_disk_gib = 0;
$firstHostPolicy->max_instances = 1000000;
$firstHostPolicy->save();
$fallbackHost = new CloudHost();
$fallbackHost->name = 'fallback-host';
$fallbackHost->enabled = true;
$fallbackHost->resource_pool_id = $pool->id;
$fallbackHost->scheduler_state = 'active';
$fallbackHost->default_network = 'bridge:br0';
$fallbackHost->created_at = $fallbackHost->updated_at = time();
$fallbackHost->save();
Capsule::table('cloud_host_agents')->insert([
'host_id' => $fallbackHost->id,
'sealed_api_secret' => 'fallback-test-only-agent-secret',
'enabled' => 1,
'revision' => 1,
'tested_revision' => 1,
'last_test_status' => 'healthy',
'last_error_code' => '',
'last_agent_version' => 'test',
'network_selectors_json' => '["bridge:br0"]',
'public_network_capabilities_json' => json_encode([
'contract' => 1,
'dual_stack_required' => true,
'project_owned_filter' => true,
'ndp_guard' => true,
], JSON_THROW_ON_ERROR),
'last_tested_at' => time(),
'created_at' => time(),
'updated_at' => time(),
]);
Capsule::table('cloud_host_storage_configurations')->insert([
'host_id' => $fallbackHost->id,
'revision' => 1,
'state' => 'applied',
]);
$fallbackMembership = new CloudHostResourcePoolMembership();
$fallbackMembership->host_id = $fallbackHost->id;
$fallbackMembership->resource_pool_id = $pool->id;
$fallbackMembership->scheduler_state = 'active';
$fallbackMembership->priority = 0;
$fallbackMembership->weight = 100;
$fallbackMembership->created_at =
$fallbackMembership->updated_at = time();
$fallbackMembership->save();
$fallbackAttachment = new CloudHostNetworkAttachment();
$fallbackAttachment->attachment_key = str_repeat('9', 32);
$fallbackAttachment->host_id = $fallbackHost->id;
$fallbackAttachment->line_id = $line->id;
$fallbackAttachment->name = 'Fallback dual stack';
$fallbackAttachment->network_mode = 'selector';
$fallbackAttachment->network_selector = 'bridge:br0';
$fallbackAttachment->state = 'active';
$fallbackAttachment->priority = 0;
$fallbackAttachment->config_revision = 1;
$fallbackAttachment->verified_agent_revision = 1;
$fallbackAttachment->verification_state = 'verified';
$fallbackAttachment->last_verified_at = time();
$fallbackAttachment->verification_error_code = '';
$fallbackAttachment->desired_revision = 1;
$fallbackAttachment->observed_revision = 1;
$fallbackAttachment->sync_state = 'applied';
$fallbackAttachment->last_error_code = '';
$fallbackAttachment->created_at =
$fallbackAttachment->updated_at = time();
$fallbackAttachment->save();
$fallbackIpv4Pool = new CloudPublicIpPool();
$fallbackIpv4Pool->pool_key = str_repeat('8', 32);
$fallbackIpv4Pool->host_id = $fallbackHost->id;
$fallbackIpv4Pool->network_attachment_id = $fallbackAttachment->id;
$fallbackIpv4Pool->name = 'Fallback IPv4';
$fallbackIpv4Pool->address_family = 4;
$fallbackIpv4Pool->cidr = '11.0.0.0/24';
$fallbackIpv4Pool->gateway = '11.0.0.1';
$fallbackIpv4Pool->gateway_on_link = false;
$fallbackIpv4Pool->guest_prefix_length = 24;
$fallbackIpv4Pool->dns_servers_json = '["1.1.1.1"]';
$fallbackIpv4Pool->reserved_ips_json = '[]';
$fallbackIpv4Pool->allocation_start = null;
$fallbackIpv4Pool->allocation_end = null;
$fallbackIpv4Pool->allocation_cursor = null;
$fallbackIpv4Pool->max_allocations = 0;
$fallbackIpv4Pool->state = 'active';
$fallbackIpv4Pool->priority = 0;
$fallbackIpv4Pool->desired_revision = 1;
$fallbackIpv4Pool->observed_revision = 1;
$fallbackIpv4Pool->sync_state = 'applied';
$fallbackIpv4Pool->last_error_code = '';
$fallbackIpv4Pool->created_at =
$fallbackIpv4Pool->updated_at = time();
$fallbackIpv4Pool->save();
$fallbackIpv6Pool = new CloudPublicIpPool();
$fallbackIpv6Pool->pool_key = str_repeat('7', 32);
$fallbackIpv6Pool->host_id = $fallbackHost->id;
$fallbackIpv6Pool->network_attachment_id = $fallbackAttachment->id;
$fallbackIpv6Pool->name = 'Fallback IPv6';
$fallbackIpv6Pool->address_family = 6;
$fallbackIpv6Pool->cidr = '2606:4700:4700:200::/64';
$fallbackIpv6Pool->gateway = '2606:4700:4700:200::1';
$fallbackIpv6Pool->gateway_on_link = false;
$fallbackIpv6Pool->guest_prefix_length = 64;
$fallbackIpv6Pool->dns_servers_json = '["2606:4700:4700::1111"]';
$fallbackIpv6Pool->reserved_ips_json = '[]';
$fallbackIpv6Pool->allocation_start = '2606:4700:4700:200::10';
$fallbackIpv6Pool->allocation_end = '2606:4700:4700:200::20';
$fallbackIpv6Pool->allocation_cursor = null;
$fallbackIpv6Pool->max_allocations = 4;
$fallbackIpv6Pool->state = 'active';
$fallbackIpv6Pool->priority = 0;
$fallbackIpv6Pool->desired_revision = 1;
$fallbackIpv6Pool->observed_revision = 1;
$fallbackIpv6Pool->sync_state = 'applied';
$fallbackIpv6Pool->last_error_code = '';
$fallbackIpv6Pool->created_at =
$fallbackIpv6Pool->updated_at = time();
$fallbackIpv6Pool->save();
$fallbackIpv4 = new CloudPublicIp();
$fallbackIpv4->host_id = $fallbackHost->id;
$fallbackIpv4->public_ip_pool_id = $fallbackIpv4Pool->id;
$fallbackIpv4->address_family = 4;
$fallbackIpv4->address = '11.0.0.10';
$fallbackIpv4->address_binary = inet_pton('11.0.0.10');
$fallbackIpv4->prefix_length = 24;
$fallbackIpv4->state = 'available';
$fallbackIpv4->created_at = $fallbackIpv4->updated_at = time();
$fallbackIpv4->save();
$fallbackPolicy = new CloudHostCapacityPolicy();
$fallbackPolicy->host_id = $fallbackHost->id;
$fallbackPolicy->configured_cpu_cores = 4;
$fallbackPolicy->configured_memory_mib = 4096;
$fallbackPolicy->configured_disk_gib = 40;
$fallbackPolicy->safety_cpu_cores = 0;
$fallbackPolicy->safety_memory_mib = 0;
$fallbackPolicy->safety_disk_gib = 0;
$fallbackPolicy->max_instances = 2;
$fallbackPolicy->created_at = $fallbackPolicy->updated_at = time();
$fallbackPolicy->save();
$fallbackScoreOrder = new CloudOrder();
$fallbackScoreOrder->user_id = 1;
$fallbackScoreOrder->plan_id = $plan->id;
$fallbackScoreOrder->offer_id = $offer->id;
$fallbackScoreOrder->resource_pool_id = $pool->id;
$fallbackScoreOrder->host_id = $fallbackHost->id;
$fallbackScoreOrder->reservation_id = null;
$fallbackScoreOrder->deployment_snapshot_id = null;
$fallbackScoreOrder->idempotency_key = 'fallback-score-fixture';
$fallbackScoreOrder->request_hash = str_repeat('4', 64);
$fallbackScoreOrder->amount = '0.00';
$fallbackScoreOrder->currency = 'USD';
$fallbackScoreOrder->status = 'succeeded';
$fallbackScoreOrder->spec_json = '{}';
$fallbackScoreOrder->sealed_payload = null;
$fallbackScoreOrder->instance_id = 999999;
$fallbackScoreOrder->refunded_at = null;
$fallbackScoreOrder->stock_reserved_at = null;
$fallbackScoreOrder->stock_released_at = null;
$fallbackScoreOrder->created_at =
$fallbackScoreOrder->updated_at = time();
$fallbackScoreOrder->save();
$fallbackScoreReservation = new CloudResourceReservation();
$fallbackScoreReservation->reservation_key = str_repeat('6', 32);
$fallbackScoreReservation->operation_id = str_repeat('5', 32);
$fallbackScoreReservation->order_id = $fallbackScoreOrder->id;
$fallbackScoreReservation->backup_restore_id = null;
$fallbackScoreReservation->offer_id = $offer->id;
$fallbackScoreReservation->resource_pool_id = $pool->id;
$fallbackScoreReservation->host_id = $fallbackHost->id;
$fallbackScoreReservation->user_id = 1;
$fallbackScoreReservation->instance_id = 999999;
$fallbackScoreReservation->network_attachment_id = null;
$fallbackScoreReservation->public_ip_pool_id = null;
$fallbackScoreReservation->public_ip_id = null;
$fallbackScoreReservation->cpu_cores = 1;
$fallbackScoreReservation->memory_mib = 512;
$fallbackScoreReservation->disk_gib = 1;
$fallbackScoreReservation->state = 'committed';
$fallbackScoreReservation->expires_at = null;
$fallbackScoreReservation->released_at = null;
$fallbackScoreReservation->release_reason = '';
$fallbackScoreReservation->created_at =
$fallbackScoreReservation->updated_at = time();
$fallbackScoreReservation->save();
$fallbackScoreOrder->reservation_id = $fallbackScoreReservation->id;
$fallbackScoreOrder->save();
Capsule::table('cloud_template_replicas')->insert([
'host_id' => $fallbackHost->id,
'template_id' => $template->id,
'revision' => $template->revision,
'generation_token' => $template->generation_token,
'state' => 'active',
'image_bytes' => $template->image_bytes,
'sha256' => $template->sha256,
'operation_id' => 'fixture-fallback-template-import',
'error_code' => '',
'created_at' => time(),
'updated_at' => time(),
]);
$offerForFallback = CloudOffer::findOrFail((int) $offer->id);
$offerForFallback->stock = -1;
$offerForFallback->save();
$fallbackUser = User::findOrFail(1);
$fallbackUser->money = '100.00';
$fallbackUser->save();
$beforeFallbackReservationId = (int) (
CloudResourceReservation::max('id') ?? 0
);
$fallbackOrder = CloudOrderService::purchase(array_replace($input, [
'name' => 'vm-host-capacity-fallback',
'idempotency_key' => 'test-host-capacity-fallback',
]), $actor);
CloudInvoiceService::payBalance(
CloudInvoice::where('order_id', $fallbackOrder->id)->firstOrFail(),
$actor,
'provision-host-capacity-fallback'
);
$fallbackReservation = CloudResourceReservation::where(
'order_id',
$fallbackOrder->id
)->firstOrFail();
$assert((int) $fallbackOrder->fresh()->host_id
=== (int) $fallbackHost->id
&& (int) $fallbackReservation->host_id === (int) $fallbackHost->id
&& (int) $fallbackReservation->id
>= $beforeFallbackReservationId + 2
&& Capsule::table('cloud_resource_reservation_ips')
->where('reservation_id', (int) $fallbackReservation->id)
->count() === 2
&& CloudResourceReservation::where('order_id', $fallbackOrder->id)
->where('host_id', $host->id)->count() === 0
&& Capsule::table('cloud_resource_reservation_ips as claim')
->leftJoin('cloud_resource_reservations as reservation',
'reservation.id', '=', 'claim.reservation_id')
->where('claim.reservation_id', '>', $beforeFallbackReservationId)
->whereNull('reservation.id')->count() === 0,
'Capacity fallback did not continue to host two or left host-one ledger residue.');
echo "Cloud SQLite purchase, provisioning, refund, ambiguity, quota and tenant checks passed (crypto double only).\n";
}