|
|
|
|
|
<?php
|
|
|
|
|
|
// Financial/state-machine tests only. The isolated keyring double is NOT encryption and is never loaded by production.
|
|
|
|
|
|
namespace App\Services {
|
|
|
|
|
|
final class Config {
|
|
|
|
|
|
public static array $values = [
|
|
|
|
|
|
'super_admin' => '99',
|
|
|
|
|
|
'baseUrl' => 'https://panel.example.test',
|
|
|
|
|
|
'appName' => 'Virt Cloud Test',
|
|
|
|
|
|
];
|
|
|
|
|
|
public static function get($name, $default = null) { return self::$values[$name] ?? $default; }
|
|
|
|
|
|
}
|
|
|
|
|
|
final class Auth { public static $user; public static function getUser() { return self::$user; } }
|
|
|
|
|
|
final class Mail {
|
|
|
|
|
|
public static array $sent = [];
|
|
|
|
|
|
public static function send(...$arguments): bool {
|
|
|
|
|
|
self::$sent[] = $arguments;
|
|
|
|
|
|
return true;
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
namespace App\Services\Cloud {
|
|
|
|
|
|
final class CloudSecrets {
|
|
|
|
|
|
private static array $memory = [];
|
|
|
|
|
|
public static function key(string $purpose): string { return str_repeat('k', 32); }
|
|
|
|
|
|
public static function encrypt(array $payload, string $purpose): string { $id = bin2hex(random_bytes(16)); self::$memory[$id] = $payload; return 'test-only:' . $id; }
|
|
|
|
|
|
public static function decrypt(string $cipher, string $purpose): array { return self::$memory[substr($cipher, 10)]; }
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
/** Isolated read-only Agent transport for create recovery checks. */
|
|
|
|
|
|
final class FinancialCreateRecoveryAgentClient
|
|
|
|
|
|
{
|
|
|
|
|
|
public array $requests = [];
|
|
|
|
|
|
public array $responses = [];
|
|
|
|
|
|
|
|
|
|
|
|
public function systemRecoveryRequest(
|
|
|
|
|
|
string $method,
|
|
|
|
|
|
string $target,
|
|
|
|
|
|
string $operationId,
|
|
|
|
|
|
?array $payload = null
|
|
|
|
|
|
): array {
|
|
|
|
|
|
$this->requests[] = compact(
|
|
|
|
|
|
'method', 'target', 'operationId', 'payload'
|
|
|
|
|
|
);
|
|
|
|
|
|
if ($this->responses === []) {
|
|
|
|
|
|
throw new \RuntimeException(
|
|
|
|
|
|
'Unexpected create recovery Agent request.'
|
|
|
|
|
|
);
|
|
|
|
|
|
}
|
|
|
|
|
|
$response = array_shift($this->responses);
|
|
|
|
|
|
if ($response instanceof \Throwable) {
|
|
|
|
|
|
throw $response;
|
|
|
|
|
|
}
|
|
|
|
|
|
if (is_callable($response)) {
|
|
|
|
|
|
$response = $response(end($this->requests));
|
|
|
|
|
|
}
|
|
|
|
|
|
if (!is_array($response)) {
|
|
|
|
|
|
throw new \RuntimeException(
|
|
|
|
|
|
'Invalid create recovery Agent response.'
|
|
|
|
|
|
);
|
|
|
|
|
|
}
|
|
|
|
|
|
return $response;
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
final class CloudAgentSettingsService
|
|
|
|
|
|
{
|
|
|
|
|
|
public static ?FinancialCreateRecoveryAgentClient $client = null;
|
|
|
|
|
|
public static int $hostId = 0;
|
|
|
|
|
|
|
|
|
|
|
|
public static function clientForConfiguration(
|
|
|
|
|
|
int $hostId,
|
|
|
|
|
|
float $timeoutSeconds = 120.0
|
|
|
|
|
|
): FinancialCreateRecoveryAgentClient {
|
|
|
|
|
|
if ($hostId !== self::$hostId || $timeoutSeconds !== 30.0
|
|
|
|
|
|
|| self::$client === null) {
|
|
|
|
|
|
throw new \RuntimeException(
|
|
|
|
|
|
'Unexpected create recovery client request.'
|
|
|
|
|
|
);
|
|
|
|
|
|
}
|
|
|
|
|
|
return self::$client;
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
public static function testedNetworkSelectors(
|
|
|
|
|
|
\App\Models\CloudHostAgent $setting
|
|
|
|
|
|
): array {
|
|
|
|
|
|
return $setting->testedNetworkSelectors();
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
public static function publicNetworkReady(
|
|
|
|
|
|
\App\Models\CloudHostAgent $setting,
|
|
|
|
|
|
string $selector,
|
|
|
|
|
|
int $verifiedRevision
|
|
|
|
|
|
): bool {
|
|
|
|
|
|
if (!$setting->isReady()
|
|
|
|
|
|
|| $verifiedRevision !== (int) $setting->revision
|
|
|
|
|
|
|| $verifiedRevision !== (int) $setting->tested_revision
|
|
|
|
|
|
|| !in_array($selector,
|
|
|
|
|
|
self::testedNetworkSelectors($setting), true)) {
|
|
|
|
|
|
return false;
|
|
|
|
|
|
}
|
|
|
|
|
|
$capabilities = json_decode(
|
|
|
|
|
|
(string) $setting->public_network_capabilities_json,
|
|
|
|
|
|
true
|
|
|
|
|
|
);
|
|
|
|
|
|
return is_array($capabilities)
|
|
|
|
|
|
&& ($capabilities['contract'] ?? null) === 1
|
|
|
|
|
|
&& ($capabilities['dual_stack_required'] ?? null) === true
|
|
|
|
|
|
&& ($capabilities['project_owned_filter'] ?? null) === true
|
|
|
|
|
|
&& ($capabilities['ndp_guard'] ?? null) === true;
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
namespace {
|
|
|
|
|
|
require dirname(__DIR__) . '/vendor/autoload.php';
|
|
|
|
|
|
use Illuminate\Database\Capsule\Manager as Capsule;
|
|
|
|
|
|
use Illuminate\Database\Schema\Blueprint;
|
|
|
|
|
|
use App\Models\User;
|
|
|
|
|
|
use App\Models\Shop;
|
|
|
|
|
|
use App\Models\CloudHost;
|
|
|
|
|
|
use App\Models\CloudHostCapacityPolicy;
|
|
|
|
|
|
use App\Models\CloudHostNetworkAttachment;
|
|
|
|
|
|
use App\Models\CloudHostResourcePoolMembership;
|
|
|
|
|
|
use App\Models\CloudBackupSubscription;
|
|
|
|
|
|
use App\Models\CloudDeploymentSnapshot;
|
|
|
|
|
|
use App\Models\CloudInstanceBackup;
|
|
|
|
|
|
use App\Models\CloudBackupRestore;
|
|
|
|
|
|
use App\Models\CloudInstance;
|
|
|
|
|
|
use App\Models\CloudInvoice;
|
|
|
|
|
|
use App\Models\CloudInvoicePayment;
|
|
|
|
|
|
use App\Models\CloudMailOutbox;
|
|
|
|
|
|
use App\Models\CloudNetworkLine;
|
|
|
|
|
|
use App\Models\CloudOffer;
|
|
|
|
|
|
use App\Models\CloudOrder;
|
|
|
|
|
|
use App\Models\CloudOrderItem;
|
|
|
|
|
|
use App\Models\CloudJob;
|
|
|
|
|
|
use App\Models\CloudPublicIp;
|
|
|
|
|
|
use App\Models\CloudPublicIpPool;
|
|
|
|
|
|
use App\Models\CloudRegion;
|
|
|
|
|
|
use App\Models\CloudResourcePool;
|
|
|
|
|
|
use App\Models\CloudResourceReservation;
|
|
|
|
|
|
use App\Models\CloudServicePeriod;
|
|
|
|
|
|
use App\Models\CloudTemplate;
|
|
|
|
|
|
use App\Services\Cloud\CloudInvoiceService;
|
|
|
|
|
|
use App\Services\Cloud\CloudInvoicePaymentService;
|
|
|
|
|
|
use App\Services\Cloud\CloudAdminOperationsService;
|
|
|
|
|
|
use App\Services\Cloud\CloudAutomaticOperationRecoveryService;
|
|
|
|
|
|
use App\Services\Cloud\CloudOrderService;
|
|
|
|
|
|
use App\Services\Cloud\CloudWorkerService;
|
|
|
|
|
|
use App\Services\Cloud\CloudAgentClient;
|
|
|
|
|
|
use App\Services\Cloud\CloudAccess;
|
|
|
|
|
|
use App\Services\Cloud\CloudJobService;
|
|
|
|
|
|
use App\Services\Cloud\CloudIpChangeBillingService;
|
|
|
|
|
|
use App\Services\Cloud\CloudProvisioningBillingConfig;
|
|
|
|
|
|
use App\Services\Cloud\CloudLifecycleService;
|
|
|
|
|
|
use App\Services\Cloud\CloudOperationCompletionService;
|
|
|
|
|
|
use App\Services\Cloud\CloudOperationRecoveryIdentity;
|
|
|
|
|
|
use App\Services\Cloud\CloudOperationRecoveryService;
|
|
|
|
|
|
use App\Services\Cloud\CloudRenewalService;
|
|
|
|
|
|
use App\Services\Cloud\CloudError;
|
|
|
|
|
|
use App\Services\Cloud\CloudMoney;
|
|
|
|
|
|
use App\Services\Cloud\CloudPlatformTestNetworkPolicy;
|
|
|
|
|
|
use App\Services\Cloud\CloudPlacementService;
|
|
|
|
|
|
use App\Services\Cloud\CloudPublicNetworkAllocationService;
|
|
|
|
|
|
use App\Services\Cloud\CloudPublicNetworkTransitionCommand;
|
|
|
|
|
|
use App\Services\Cloud\CloudPublicNetworkTransitionWorkflow;
|
|
|
|
|
|
use App\Services\Cloud\CloudDeploymentPageService;
|
|
|
|
|
|
use App\Services\Cloud\CloudTemplateCatalogService;
|
|
|
|
|
|
use App\Services\Cloud\CloudAgentSettingsService;
|
|
|
|
|
|
use App\Services\Cloud\CloudAgentError;
|
|
|
|
|
|
use App\Services\Cloud\CloudCreateFailureRecoveryService;
|
|
|
|
|
|
use App\Services\Cloud\FinancialCreateRecoveryAgentClient;
|
|
|
|
|
|
use GuzzleHttp\Client;
|
|
|
|
|
|
use GuzzleHttp\Handler\MockHandler;
|
|
|
|
|
|
use GuzzleHttp\HandlerStack;
|
|
|
|
|
|
use GuzzleHttp\Middleware;
|
|
|
|
|
|
use GuzzleHttp\Psr7\Response;
|
|
|
|
|
|
use GuzzleHttp\Psr7\Request;
|
|
|
|
|
|
use GuzzleHttp\Exception\ConnectException;
|
|
|
|
|
|
$assert = function ($condition, $message) { if (!$condition) { throw new RuntimeException($message); } };
|
|
|
|
|
|
$bundleState = static function (CloudResourceReservation $reservation,
|
|
|
|
|
|
string $claimState, string $ipState): bool {
|
|
|
|
|
|
$claims = Capsule::table('cloud_resource_reservation_ips')
|
|
|
|
|
|
->where('reservation_id', (int) $reservation->id)
|
|
|
|
|
|
->where('state', $claimState)->get();
|
|
|
|
|
|
return $claims->count() === 2
|
|
|
|
|
|
&& $claims->pluck('address_family')->sort()->values()->all()
|
|
|
|
|
|
=== [4, 6]
|
|
|
|
|
|
&& CloudPublicIp::whereIn('id', $claims->pluck('public_ip_id'))
|
|
|
|
|
|
->where('state', $ipState)->count() === 2;
|
|
|
|
|
|
};
|
|
|
|
|
|
/** Strictly unwrap one local PHP-to-Gateway request into its Agent request. */
|
|
|
|
|
|
$agentRequest = static function (array $historyEntry) use ($assert): array {
|
|
|
|
|
|
$outer = $historyEntry['request'] ?? null;
|
|
|
|
|
|
$assert($outer instanceof \Psr\Http\Message\RequestInterface,
|
|
|
|
|
|
'Gateway history entry is missing its outer request.');
|
|
|
|
|
|
$uri = $outer->getUri();
|
|
|
|
|
|
$assert($outer->getMethod() === 'POST'
|
|
|
|
|
|
&& $uri->getScheme() === 'http'
|
|
|
|
|
|
&& $uri->getHost() === '127.0.0.1'
|
|
|
|
|
|
&& $uri->getPort() === 8001
|
|
|
|
|
|
&& $uri->getPath() === '/internal/agent/request'
|
|
|
|
|
|
&& $uri->getQuery() === ''
|
|
|
|
|
|
&& $uri->getFragment() === '',
|
|
|
|
|
|
'Agent request did not use the fixed local Gateway endpoint.');
|
|
|
|
|
|
|
|
|
|
|
|
$envelope = json_decode((string) $outer->getBody(), true, 32,
|
|
|
|
|
|
JSON_THROW_ON_ERROR);
|
|
|
|
|
|
$assert(is_array($envelope) && !array_is_list($envelope),
|
|
|
|
|
|
'Gateway request envelope is not a JSON object.');
|
|
|
|
|
|
$actualKeys = array_keys($envelope);
|
|
|
|
|
|
$expectedKeys = [
|
|
|
|
|
|
'agent_id', 'type', 'id', 'method', 'request_target', 'body',
|
|
|
|
|
|
'headers',
|
|
|
|
|
|
];
|
|
|
|
|
|
sort($actualKeys);
|
|
|
|
|
|
sort($expectedKeys);
|
|
|
|
|
|
$assert($actualKeys === $expectedKeys
|
|
|
|
|
|
&& ($envelope['agent_id'] ?? null) === '1'
|
|
|
|
|
|
&& ($envelope['type'] ?? null) === 'request'
|
|
|
|
|
|
&& is_string($envelope['id'] ?? null)
|
|
|
|
|
|
&& preg_match('/^[a-f0-9]{32}$/D', $envelope['id']) === 1
|
|
|
|
|
|
&& is_string($envelope['method'] ?? null)
|
|
|
|
|
|
&& in_array($envelope['method'],
|
|
|
|
|
|
['GET', 'POST', 'PUT', 'PATCH', 'DELETE'], true)
|
|
|
|
|
|
&& is_string($envelope['request_target'] ?? null)
|
|
|
|
|
|
&& preg_match('#^/v1(?:/|$)#D', $envelope['request_target']) === 1
|
|
|
|
|
|
&& is_string($envelope['body'] ?? null)
|
|
|
|
|
|
&& is_array($envelope['headers'] ?? null)
|
|
|
|
|
|
&& !array_is_list($envelope['headers']),
|
|
|
|
|
|
'Gateway request envelope has an invalid shape.');
|
|
|
|
|
|
|
|
|
|
|
|
$headers = $envelope['headers'];
|
|
|
|
|
|
$actualHeaderKeys = array_keys($headers);
|
|
|
|
|
|
$expectedHeaderKeys = [
|
|
|
|
|
|
'X-Virt-Timestamp', 'X-Virt-Nonce', 'X-Virt-Content-SHA256',
|
|
|
|
|
|
'X-Virt-Actor', 'X-Virt-Signature',
|
|
|
|
|
|
];
|
|
|
|
|
|
sort($actualHeaderKeys);
|
|
|
|
|
|
sort($expectedHeaderKeys);
|
|
|
|
|
|
$assert($actualHeaderKeys === $expectedHeaderKeys
|
|
|
|
|
|
&& preg_match('/^[0-9]{1,20}$/D',
|
|
|
|
|
|
(string) ($headers['X-Virt-Timestamp'] ?? '')) === 1
|
|
|
|
|
|
&& preg_match('/^[a-f0-9]{32}$/D',
|
|
|
|
|
|
(string) ($headers['X-Virt-Nonce'] ?? '')) === 1
|
|
|
|
|
|
&& ($headers['X-Virt-Content-SHA256'] ?? null)
|
|
|
|
|
|
=== hash('sha256', $envelope['body'])
|
|
|
|
|
|
&& preg_match('/^[A-Za-z0-9_-]+$/D',
|
|
|
|
|
|
(string) ($headers['X-Virt-Actor'] ?? '')) === 1
|
|
|
|
|
|
&& preg_match('/^[a-f0-9]{64}$/D',
|
|
|
|
|
|
(string) ($headers['X-Virt-Signature'] ?? '')) === 1,
|
|
|
|
|
|
'Gateway envelope lost the signed Agent request headers.');
|
|
|
|
|
|
|
|
|
|
|
|
if ($envelope['body'] !== '') {
|
|
|
|
|
|
$innerBody = json_decode($envelope['body'], false, 64,
|
|
|
|
|
|
JSON_THROW_ON_ERROR);
|
|
|
|
|
|
$assert($innerBody instanceof \stdClass,
|
|
|
|
|
|
'Agent request body is not a JSON object.');
|
|
|
|
|
|
}
|
|
|
|
|
|
$encodedActor = $headers['X-Virt-Actor'];
|
|
|
|
|
|
$padding = str_repeat('=', (4 - strlen($encodedActor) % 4) % 4);
|
|
|
|
|
|
$decodedActor = base64_decode(
|
|
|
|
|
|
strtr($encodedActor, '-_', '+/') . $padding, true
|
|
|
|
|
|
);
|
|
|
|
|
|
$actorPayload = is_string($decodedActor)
|
|
|
|
|
|
? json_decode($decodedActor, true, 16, JSON_THROW_ON_ERROR) : null;
|
|
|
|
|
|
$assert(is_array($actorPayload) && !array_is_list($actorPayload)
|
|
|
|
|
|
&& is_int($actorPayload['user_id'] ?? null)
|
|
|
|
|
|
&& is_bool($actorPayload['is_admin'] ?? null)
|
|
|
|
|
|
&& is_bool($actorPayload['is_super_admin'] ?? null)
|
|
|
|
|
|
&& is_bool($actorPayload['can_manage'] ?? null)
|
|
|
|
|
|
&& (!isset($actorPayload['operation_id'])
|
|
|
|
|
|
|| (is_string($actorPayload['operation_id'])
|
|
|
|
|
|
&& preg_match('/^[A-Za-z0-9_-]{8,80}$/D',
|
|
|
|
|
|
$actorPayload['operation_id']) === 1)),
|
|
|
|
|
|
'Gateway envelope contains an invalid signed Agent actor.');
|
|
|
|
|
|
return $envelope;
|
|
|
|
|
|
};
|
|
|
|
|
|
$capsule = new Capsule();
|
|
|
|
|
|
$capsule->addConnection(['driver' => 'sqlite', 'database' => ':memory:'], 'default');
|
|
|
|
|
|
$capsule->setAsGlobal(); $capsule->bootEloquent();
|
|
|
|
|
|
$schema = $capsule->schema('default');
|
|
|
|
|
|
$schema->create('user', function (Blueprint $t) { $t->increments('id'); $t->string('user_name')->default(''); $t->string('email')->default(''); $t->string('money'); $t->integer('enable')->default(1); $t->integer('is_admin')->default(0); });
|
|
|
|
|
|
$schema->create('configurations', function (Blueprint $t) { $t->increments('id'); $t->string('name')->unique(); $t->text('value'); });
|
|
|
|
|
|
$schema->create('shop', function (Blueprint $t) { $t->increments('id'); $t->string('name'); foreach (['cpu_cores', 'memory_mib', 'bandwidth_mbps', 'disk_gib', 'validity_days', 'traffic_quota_bytes', 'status', 'sort'] as $v) { $t->integer($v)->default(0); } });
|
|
|
|
|
|
$schema->create('cloud_regions', function (Blueprint $t) { $t->increments('id'); $t->string('code'); $t->string('name'); $t->string('country_code'); $t->string('status'); $t->integer('sort')->default(0); $t->integer('created_at'); $t->integer('updated_at'); });
|
|
|
|
|
|
$schema->create('cloud_network_lines', function (Blueprint $t) { $t->increments('id'); $t->integer('region_id'); $t->integer('traffic_pool_group_id')->nullable(); $t->string('code'); $t->string('name'); $t->integer('is_default')->default(0); $t->string('status'); $t->integer('sort')->default(0); $t->integer('created_at'); $t->integer('updated_at'); });
|
|
|
|
|
|
$schema->create('cloud_resource_pools', function (Blueprint $t) { $t->increments('id'); $t->integer('line_id'); $t->string('code'); $t->string('name'); $t->string('status'); $t->string('scheduling_policy')->default('balanced'); $t->integer('sort')->default(0); $t->integer('created_at'); $t->integer('updated_at'); });
|
|
|
|
|
|
$schema->create('cloud_hosts', function (Blueprint $t) { $t->increments('id'); $t->string('name'); $t->string('region')->default(''); $t->string('country_code')->default(''); $t->integer('enabled'); $t->integer('resource_pool_id')->nullable(); $t->string('scheduler_state')->default('disabled'); $t->string('default_network'); $t->integer('created_at')->default(0); $t->integer('updated_at')->default(0); });
|
|
|
|
|
|
$schema->create('cloud_host_agents', function (Blueprint $t) { $t->integer('host_id')->primary(); $t->text('sealed_api_secret'); $t->integer('enabled')->default(0); $t->integer('revision')->default(1); $t->integer('tested_revision')->default(0); $t->string('last_test_status')->default('never'); $t->string('last_error_code')->default(''); $t->string('last_agent_version')->default(''); $t->text('network_selectors_json')->default('[]'); $t->text('public_network_capabilities_json')->default('{}'); $t->text('portable_restore_contract_json')->default('{}'); $t->string('portable_restore_contract_sha256', 64)->default(''); $t->integer('last_tested_at')->default(0); $t->integer('created_at'); $t->integer('updated_at'); });
|
|
|
|
|
|
$schema->create('cloud_host_storage_configurations', function (Blueprint $t) { $t->integer('host_id')->primary(); $t->integer('revision')->default(0); $t->string('state')->default('applying'); });
|
|
|
|
|
|
$schema->create('cloud_host_resource_pool_memberships', function (Blueprint $t) { $t->increments('id'); $t->integer('host_id'); $t->integer('resource_pool_id'); $t->string('scheduler_state')->default('disabled'); $t->integer('priority')->default(0); $t->integer('weight')->default(100); $t->integer('revision')->default(1); $t->integer('created_at'); $t->integer('updated_at'); $t->unique(['host_id', 'resource_pool_id']); });
|
|
|
|
|
|
$schema->create('cloud_host_network_attachments', function (Blueprint $t) { $t->increments('id'); $t->string('attachment_key'); $t->integer('host_id'); $t->integer('line_id'); $t->string('name'); $t->string('network_mode')->default('selector'); $t->string('network_selector'); $t->integer('vlan_id')->nullable(); $t->integer('routing_table')->nullable(); $t->string('state')->default('active'); $t->integer('priority')->default(0); $t->integer('config_revision')->default(1); $t->integer('verified_agent_revision')->default(0); $t->string('verification_state')->default('never'); $t->integer('last_verified_at')->default(0); $t->string('verification_error_code')->default(''); $t->integer('desired_revision')->default(1); $t->integer('observed_revision')->default(0); $t->string('sync_state')->default('never'); $t->string('last_error_code')->default(''); $t->integer('created_at'); $t->integer('updated_at'); });
|
|
|
|
|
|
$schema->create('cloud_public_ip_pools', function (Blueprint $t) { $t->increments('id'); $t->string('pool_key'); $t->integer('host_id'); $t->integer('network_attachment_id'); $t->string('name'); $t->integer('address_family'); $t->string('cidr'); $t->string('gateway'); $t->boolean('gateway_on_link')->default(false); $t->integer('guest_prefix_length'); $t->text('dns_servers_json'); $t->text('reserved_ips_json'); $t->string('allocation_start')->nullable(); $t->string('allocation_end')->nullable(); $t->string('allocation_cursor')->nullable(); $t->integer('max_allocations')->default(0); $t->string('state')->default('active'); $t->integer('priority')->default(0); $t->integer('desired_revision')->default(1); $t->integer('observed_revision')->default(0); $t->string('sync_state')->default('never'); $t->string('last_error_code')->default(''); $t->integer('created_at'); $t->integer('updated_at'); });
|
|
|
|
|
|
$schema->create('cloud_host_capacity_policies', function (Blueprint $t) { $t->integer('host_id')->primary(); foreach (['configured_cpu_cores', 'configured_memory_mib', 'configured_disk_gib', 'safety_cpu_cores', 'safety_memory_mib', 'safety_disk_gib', 'max_instances', 'created_at', 'updated_at'] as $v) { $t->integer($v)->default(0); } });
|
|
|
|
|
|
$schema->create('cloud_public_ips', function (Blueprint $t) { $t->increments('id'); $t->integer('host_id'); $t->integer('public_ip_pool_id'); $t->integer('address_family'); $t->string('address'); $t->binary('address_binary'); $t->integer('prefix_length'); $t->string('state'); $t->integer('reservation_id')->nullable(); $t->integer('reservation_ip_claim_id')->nullable(); $t->integer('instance_id')->nullable(); $t->string('operation_id')->nullable(); $t->integer('created_at'); $t->integer('updated_at'); $t->unique(['address_family', 'address_binary']); });
|
|
|
|
|
|
$schema->create('cloud_offers', function (Blueprint $t) { $t->increments('id'); $t->integer('plan_id'); $t->integer('resource_pool_id'); $t->string('code')->unique(); $t->string('name'); $t->string('price'); $t->string('backup_price')->default('0.00'); $t->string('currency')->default('USD'); $t->integer('price_version')->default(1); $t->integer('revision')->default(1); $t->integer('status')->default(1); $t->integer('stock')->default(-1); $t->integer('sort')->default(0); $t->integer('created_at'); $t->integer('updated_at'); $t->unique(['plan_id', 'resource_pool_id']); });
|
|
|
|
|
|
$schema->create('cloud_templates', function (Blueprint $t) { $t->string('id')->primary(); $t->string('catalog_key')->default('test-catalog'); $t->string('version_label')->default('test-version'); $t->boolean('is_default')->default(true); $t->string('name'); $t->string('os_family'); $t->string('os_version'); $t->integer('image_bytes'); $t->string('sha256'); $t->text('startup_script'); $t->integer('revision'); $t->string('generation_token'); $t->string('state'); $t->integer('active_job_id')->nullable(); $t->integer('created_at'); $t->integer('updated_at'); });
|
|
|
|
|
|
$schema->create('cloud_user_isos', function (Blueprint $t) { $t->string('id')->primary(); $t->integer('active_job_id')->nullable(); });
|
|
|
|
|
|
$schema->create('cloud_template_replicas', function (Blueprint $t) { $t->integer('host_id'); $t->string('template_id'); $t->integer('job_id')->nullable(); $t->integer('revision'); $t->string('generation_token'); $t->string('state')->default('pending'); $t->integer('image_bytes')->default(0); $t->string('sha256')->default(''); $t->string('operation_id')->default(''); $t->string('error_code')->default(''); $t->integer('created_at'); $t->integer('updated_at'); $t->primary(['host_id', 'template_id']); });
|
|
|
|
|
|
$schema->create('cloud_platform_test_policy_state', function (Blueprint $t) { $t->integer('id')->primary(); $t->integer('revision')->default(0); $t->integer('updated_by')->nullable(); $t->integer('updated_at')->default(0); });
|
|
|
|
|
|
$schema->create('cloud_platform_test_policy_rules', function (Blueprint $t) { $t->increments('id'); $t->string('rule_key', 32)->unique(); $t->string('service', 120); $t->string('protocol', 8); $t->integer('port_from'); $t->integer('port_to'); $t->string('direction', 8); $t->string('reason', 500); $t->integer('created_by'); $t->integer('updated_by'); $t->integer('deleted_by')->nullable(); $t->integer('deleted_at')->nullable(); $t->integer('created_at'); $t->integer('updated_at'); });
|
|
|
|
|
|
$schema->create('cloud_orders', function (Blueprint $t) { $t->increments('id'); foreach (['user_id', 'plan_id', 'offer_id', 'resource_pool_id'] as $v) { $t->integer($v); } $t->integer('host_id')->nullable(); $t->integer('reservation_id')->nullable()->unique(); $t->integer('deployment_snapshot_id')->nullable(); $t->string('idempotency_key'); $t->string('request_hash'); $t->string('amount'); $t->string('currency')->default('USD'); $t->string('status'); $t->text('spec_json'); $t->text('sealed_payload')->nullable(); $t->integer('instance_id')->nullable(); $t->integer('refunded_at')->nullable(); $t->integer('stock_reserved_at')->nullable(); $t->integer('stock_released_at')->nullable(); $t->integer('created_at'); $t->integer('updated_at'); $t->unique(['user_id', 'idempotency_key']); });
|
|
|
|
|
|
$schema->create('cloud_order_items', function (Blueprint $t) { $t->increments('id'); $t->integer('order_id'); $t->string('code'); $t->string('label'); $t->string('amount'); $t->string('currency')->default('USD'); $t->text('snapshot_json'); $t->integer('created_at'); $t->unique(['order_id', 'code']); });
|
|
|
|
|
|
$schema->create('cloud_resource_reservations', function (Blueprint $t) { $t->increments('id'); $t->string('reservation_key')->unique(); $t->string('operation_id')->unique(); $t->integer('order_id')->nullable(); $t->integer('backup_restore_id')->nullable(); $t->string('backup_restore_attempt_key')->nullable(); $t->integer('migration_source_debt_id')->nullable(); $t->integer('offer_id')->nullable(); foreach (['resource_pool_id', 'host_id', 'user_id'] as $v) { $t->integer($v); } $t->integer('instance_id')->nullable(); $t->integer('network_attachment_id')->nullable(); $t->integer('public_ip_pool_id')->nullable(); $t->integer('public_ip_id')->nullable(); foreach (['cpu_cores', 'memory_mib', 'disk_gib'] as $v) { $t->integer($v); } $t->string('state'); $t->integer('expires_at')->nullable(); $t->integer('released_at')->nullable(); $t->string('release_reason')->default(''); $t->integer('created_at'); $t->integer('updated_at'); $t->unique('order_id'); $t->unique('backup_restore_id'); });
|
|
|
|
|
|
$schema->create('cloud_resource_reservation_ips', function (Blueprint $t) { $t->increments('id'); $t->integer('reservation_id'); $t->integer('address_family'); $t->integer('claim_generation')->default(1); $t->integer('active_claim')->nullable()->default(1); $t->integer('network_attachment_id'); $t->integer('public_ip_pool_id'); $t->integer('public_ip_id'); $t->string('operation_id'); $t->string('state'); $t->integer('created_at'); $t->integer('updated_at'); $t->integer('released_at')->nullable(); $t->string('release_reason')->default(''); $t->unique(['reservation_id', 'address_family', 'active_claim']); $t->unique(['reservation_id', 'claim_generation', 'address_family']); });
|
|
|
|
|
|
$schema->create('cloud_public_network_transitions', function (Blueprint $t) { $t->increments('id'); $t->string('transition_key')->unique(); $t->string('operation_id')->unique(); $t->string('kind'); $t->integer('instance_id'); $t->integer('host_id'); $t->integer('reservation_id'); $t->integer('invoice_id')->nullable()->unique(); $t->integer('pool_replacement_batch_item_id')->nullable()->unique(); $t->integer('requested_by_user_id'); $t->integer('source_network_attachment_id'); $t->integer('target_network_attachment_id'); $t->string('source_claim_set_sha256'); $t->string('target_claim_set_sha256'); $t->text('source_public_network_json'); $t->string('source_public_network_sha256'); $t->text('target_public_network_json'); $t->string('target_public_network_sha256'); $t->string('state'); $t->string('effect_resolution'); $t->integer('active_transition')->nullable(); $t->string('last_error_code'); $t->integer('effect_resolved_at')->nullable(); $t->integer('terminal_at')->nullable(); $t->integer('created_at'); $t->integer('updated_at'); $t->unique(['instance_id', 'active_transition']); $t->unique(['reservation_id', 'active_transition']); });
|
|
|
|
|
|
$schema->create('cloud_public_network_transition_ips', function (Blueprint $t) { $t->increments('id'); $t->integer('transition_id'); $t->integer('address_family'); $t->integer('source_claim_id'); $t->integer('target_claim_id')->nullable()->unique(); $t->boolean('changed'); $t->integer('created_at'); $t->integer('updated_at'); $t->unique(['transition_id', 'address_family']); });
|
|
|
|
|
|
$schema->create('cloud_backup_restores', function (Blueprint $t) { $t->increments('id'); $t->string('restore_key')->unique(); $t->integer('backup_id')->nullable(); $t->integer('target_host_id'); $t->integer('target_instance_id')->nullable(); $t->integer('reservation_id')->nullable()->unique(); $t->integer('job_id')->nullable(); $t->string('operation_id')->unique(); $t->string('new_uuid'); $t->string('new_name'); $t->string('state'); $t->string('error_code')->default(''); $t->integer('ready_at')->nullable(); $t->integer('created_at'); $t->integer('updated_at'); });
|
|
|
|
|
|
$schema->create('cloud_backup_recoveries', function (Blueprint $t) { $t->increments('id'); $t->integer('instance_id'); $t->string('state'); });
|
|
|
|
|
|
$schema->create('cloud_instance_migrations', function (Blueprint $t) { $t->increments('id'); $t->integer('instance_id'); $t->integer('restore_id'); $t->integer('source_host_id'); $t->integer('source_reservation_id'); $t->integer('target_host_id'); $t->integer('target_reservation_id'); $t->string('state'); });
|
|
|
|
|
|
$schema->create('cloud_instance_backups', function (Blueprint $t) { $t->increments('id'); $t->string('backup_key')->unique(); $t->integer('storage_artifact_id')->nullable(); $t->integer('source_instance_id'); $t->integer('source_host_id'); $t->integer('user_id'); $t->string('source_uuid'); $t->string('name'); $t->string('purpose')->default('portable'); $t->string('source')->default('manual'); $t->integer('subscription_id')->nullable(); $t->integer('scheduled_for')->nullable(); $t->string('state')->default('creating'); $t->string('consistency'); $t->string('object_key')->unique(); $t->integer('object_size_bytes')->default(0); $t->integer('virtual_size_bytes')->default(0); $t->string('sha256')->default(''); $t->text('spec_json'); $t->text('sealed_storage_runtime'); $t->text('sealed_credential')->nullable(); $t->integer('storage_revision'); $t->text('source_portable_restore_contract_json')->default('{}'); $t->string('source_portable_restore_contract_sha256', 64)->default(''); $t->integer('active_job_id')->nullable(); $t->string('last_operation_id')->nullable(); $t->string('error_code')->default(''); $t->integer('ready_at')->nullable(); $t->integer('created_at'); $t->integer('updated_at'); $t->unique(['subscription_id', 'scheduled_for']); });
|
|
|
|
|
|
$schema->create('cloud_storage_backends', function (Blueprint $t) { $t->increments('id'); $t->string('backend_key'); $t->string('identity_sha256'); $t->string('name'); $t->string('driver'); $t->string('endpoint'); $t->string('region'); $t->string('bucket'); $t->string('prefix'); $t->boolean('path_style'); $t->string('failure_domain'); $t->text('sealed_credentials'); $t->integer('config_revision'); $t->integer('tested_revision'); $t->string('admin_state'); $t->string('health_state'); $t->integer('write_priority'); $t->integer('consecutive_failures')->default(0); $t->integer('consecutive_successes')->default(0); $t->string('last_error_code')->default(''); $t->integer('last_tested_at')->default(0); $t->integer('last_healthy_at')->default(0); $t->integer('next_probe_at')->default(0); $t->integer('created_at'); $t->integer('updated_at'); });
|
|
|
|
|
|
$schema->create('cloud_storage_artifacts', function (Blueprint $t) { $t->increments('id'); $t->string('artifact_key'); $t->string('kind'); $t->integer('maximum_size_bytes'); $t->integer('size_bytes'); $t->string('sha256', 64); $t->string('durability_policy'); $t->integer('desired_copies'); $t->integer('required_copies'); $t->string('state'); $t->string('last_operation_id')->nullable(); $t->integer('verified_at'); $t->integer('created_at'); $t->integer('updated_at'); });
|
|
|
|
|
|
$schema->create('cloud_storage_publications', function (Blueprint $t) { $t->increments('id'); $t->integer('artifact_id'); $t->integer('backup_id')->nullable(); $t->string('state'); $t->integer('created_at'); $t->integer('updated_at'); });
|
|
|
|
|
|
$schema->create('cloud_storage_replicas', function (Blueprint $t) { $t->increments('id'); $t->string('replica_key'); $t->integer('artifact_id'); $t->integer('backend_id'); $t->string('role'); $t->string('object_key'); $t->string('object_version_id')->default(''); $t->string('object_etag')->default(''); $t->integer('size_bytes'); $t->string('sha256', 64); $t->string('state'); $t->string('publish_operation_id')->nullable(); $t->string('delete_operation_id')->nullable(); $t->integer('delete_generation')->default(0); $t->integer('delete_cycle_attempts')->default(0); $t->integer('delete_presence_confirmations')->default(0); $t->integer('delete_last_checked_at')->default(0); $t->integer('delete_next_check_at')->default(0); $t->integer('delete_attention_at')->nullable(); $t->string('lease_token')->nullable(); $t->integer('lease_until')->default(0); $t->string('error_code')->default(''); $t->integer('verified_at'); $t->integer('created_at'); $t->integer('updated_at'); });
|
|
|
|
|
|
$schema->create('cloud_deployment_snapshots', function (Blueprint $t) { $t->increments('id'); $t->string('snapshot_key')->unique(); $t->integer('user_id'); $t->integer('backup_id')->unique(); $t->integer('source_instance_id'); $t->string('name'); $t->string('state')->default('ready'); $t->string('idempotency_key'); $t->string('request_hash'); $t->string('error_code')->default(''); $t->integer('ready_at'); $t->integer('deleted_at')->nullable(); $t->integer('created_at'); $t->integer('updated_at'); $t->unique(['user_id', 'idempotency_key']); });
|
|
|
|
|
|
$schema->create('cloud_jobs', function (Blueprint $t) {
|
|
|
|
|
|
$t->increments('id');
|
|
|
|
|
|
$t->string('operation_id')->unique();
|
|
|
|
|
|
$t->string('request_idempotency_key');
|
|
|
|
|
|
$t->string('request_fingerprint');
|
|
|
|
|
|
foreach ([
|
|
|
|
|
|
'parent_job_id', 'supersedes_job_id', 'order_id', 'invoice_id',
|
|
|
|
|
|
'instance_id', 'snapshot_id', 'backup_id', 'backup_restore_id',
|
|
|
|
|
|
'vpc_repair_root_job_id', 'public_network_transition_id',
|
|
|
|
|
|
] as $v) {
|
|
|
|
|
|
$t->integer($v)->nullable();
|
|
|
|
|
|
}
|
|
|
|
|
|
$t->integer('user_id');
|
|
|
|
|
|
$t->integer('host_id')->nullable();
|
|
|
|
|
|
$t->string('kind');
|
|
|
|
|
|
$t->string('execution_mode');
|
|
|
|
|
|
$t->string('phase');
|
|
|
|
|
|
$t->string('outcome')->nullable();
|
|
|
|
|
|
$t->string('status');
|
|
|
|
|
|
$t->text('sealed_payload');
|
|
|
|
|
|
$t->integer('attempts');
|
|
|
|
|
|
$t->integer('dispatch_attempts');
|
|
|
|
|
|
$t->integer('dispatch_deadline_at');
|
|
|
|
|
|
$t->integer('dispatch_started_at')->nullable();
|
|
|
|
|
|
$t->integer('agent_accepted_at')->nullable();
|
|
|
|
|
|
$t->integer('receipt_deadline_at')->nullable();
|
|
|
|
|
|
$t->integer('terminal_at')->nullable();
|
|
|
|
|
|
$t->boolean('effect_may_have_applied')->nullable();
|
|
|
|
|
|
$t->boolean('safe_to_resubmit')->nullable();
|
|
|
|
|
|
$t->string('effect_resolution');
|
|
|
|
|
|
$t->integer('effect_resolved_at')->nullable();
|
|
|
|
|
|
$t->integer('effect_resolved_by')->nullable();
|
|
|
|
|
|
$t->text('effect_resolution_reason')->nullable();
|
|
|
|
|
|
$t->string('completion_event_id')->nullable();
|
|
|
|
|
|
$t->integer('read_at')->nullable();
|
|
|
|
|
|
$t->integer('dispatched_at')->nullable();
|
|
|
|
|
|
$t->string('lease_token')->nullable();
|
|
|
|
|
|
$t->integer('lease_until')->default(0);
|
|
|
|
|
|
$t->integer('available_at');
|
|
|
|
|
|
$t->string('error_code')->nullable();
|
|
|
|
|
|
$t->boolean('finalization_incident')->default(0);
|
|
|
|
|
|
$t->string('finalization_error_code')->nullable();
|
|
|
|
|
|
$t->text('result_json')->nullable();
|
|
|
|
|
|
$t->integer('created_at');
|
|
|
|
|
|
$t->integer('updated_at');
|
|
|
|
|
|
$t->unique('order_id');
|
|
|
|
|
|
});
|
|
|
|
|
|
$schema->create('cloud_backup_restore_attempts',
|
|
|
|
|
|
static function (Blueprint $table): void {
|
|
|
|
|
|
$table->increments('id');
|
|
|
|
|
|
$table->integer('job_id');
|
|
|
|
|
|
$table->string('state');
|
|
|
|
|
|
$table->string('effect_resolution');
|
|
|
|
|
|
$table->integer('active_attempt')->nullable();
|
|
|
|
|
|
$table->string('last_error_code')->default('');
|
|
|
|
|
|
});
|
|
|
|
|
|
$schema->create('cloud_agent_completion_events', function (Blueprint $t) {
|
|
|
|
|
|
$t->string('event_id')->primary();
|
|
|
|
|
|
$t->integer('host_id');
|
|
|
|
|
|
$t->string('operation_id');
|
|
|
|
|
|
$t->string('payload_sha256');
|
|
|
|
|
|
$t->string('outcome');
|
|
|
|
|
|
$t->boolean('effect_may_have_applied');
|
|
|
|
|
|
$t->boolean('safe_to_resubmit');
|
|
|
|
|
|
$t->string('error_code');
|
|
|
|
|
|
$t->text('result_json')->nullable();
|
|
|
|
|
|
$t->string('disposition');
|
|
|
|
|
|
$t->integer('received_at');
|
|
|
|
|
|
$t->integer('applied_at')->nullable();
|
|
|
|
|
|
$t->unique(['host_id', 'operation_id']);
|
|
|
|
|
|
});
|
|
|
|
|
|
$schema->create('cloud_instances', function (Blueprint $t) { $t->increments('id'); foreach (['host_id', 'user_id'] as $v) { $t->integer($v); } foreach (['order_id', 'plan_id', 'offer_id', 'resource_pool_id', 'reservation_id', 'current_period_id', 'expires_at', 'suspended_at', 'network_suspended_at'] as $v) { $t->integer($v)->nullable(); } foreach (['renewal_days', 'renewal_traffic_bytes', 'traffic_quota_bytes', 'traffic_download_bytes', 'traffic_upload_bytes', 'lifetime_download_bytes', 'lifetime_upload_bytes', 'traffic_synced_at'] as $v) { $t->integer($v)->default(0); } $t->string('renewal_price')->nullable(); $t->string('uuid'); $t->string('name'); $t->string('status'); $t->string('service_status')->default('exempt'); $t->string('network_status')->default('active'); $t->text('spec_json'); $t->text('sealed_credential')->nullable(); $t->integer('created_at'); $t->integer('updated_at'); $t->unique('order_id'); $t->unique(['host_id', 'uuid']); });
|
|
|
|
|
|
$schema->create('cloud_instance_network_policy_states', function (Blueprint $t) { $t->integer('instance_id')->primary(); $t->integer('user_id'); $t->boolean('desired_suspended')->default(false); $t->boolean('applied_suspended')->default(false); $t->integer('revision')->default(1); $t->string('state')->default('applied'); $t->string('operation_id')->nullable()->unique(); $t->integer('created_at'); $t->integer('updated_at'); });
|
|
|
|
|
|
$schema->create('cloud_instance_network_restrictions', function (Blueprint $t) { $t->integer('instance_id'); $t->integer('user_id'); $t->string('reason_code'); $t->boolean('desired_active')->default(false); $t->integer('revision')->default(1); $t->string('source_type'); $t->string('source_id'); $t->integer('created_at'); $t->integer('updated_at'); $t->primary(['instance_id', 'reason_code']); });
|
|
|
|
|
|
$schema->create('cloud_vpc_ports', function (Blueprint $t) { $t->increments('id'); $t->integer('vpc_id'); $t->integer('instance_id'); $t->integer('user_id'); $t->integer('active_claim')->nullable(); $t->string('state'); });
|
|
|
|
|
|
$schema->create('cloud_instance_snapshots', function (Blueprint $t) { $t->increments('id'); $t->string('snapshot_key')->unique(); foreach (['instance_id', 'user_id', 'host_id'] as $v) { $t->integer($v); } $t->string('name'); $t->string('state'); $t->string('consistency'); $t->integer('size_bytes')->nullable(); $t->string('last_operation_id')->nullable(); foreach (['ready_at', 'last_restored_at', 'deleted_at'] as $v) { $t->integer($v)->nullable(); } $t->integer('created_at'); $t->integer('updated_at'); });
|
|
|
|
|
|
$schema->create('cloud_renewals', function (Blueprint $t) { $t->increments('id'); $t->integer('instance_id'); $t->integer('user_id'); $t->integer('plan_id')->nullable(); $t->integer('period_id')->nullable(); $t->string('idempotency_key'); $t->string('request_hash'); $t->string('amount'); $t->integer('validity_days'); $t->integer('traffic_bytes_added'); $t->integer('previous_expires_at'); $t->integer('new_expires_at'); $t->integer('created_at'); $t->unique(['user_id', 'idempotency_key']); });
|
|
|
|
|
|
$schema->create('cloud_service_periods', function (Blueprint $t) { $t->increments('id'); foreach (['instance_id', 'user_id'] as $v) { $t->integer($v); } foreach (['plan_id', 'order_id', 'renewal_id', 'traffic_pool_group_id', 'download_base_bytes', 'upload_base_bytes', 'activated_at'] as $v) { $t->integer($v)->nullable(); } $t->string('traffic_quota_mode')->default('finite'); foreach (['starts_at', 'ends_at', 'traffic_quota_bytes', 'download_used_bytes', 'upload_used_bytes', 'created_at', 'updated_at'] as $v) { $t->integer($v)->default(0); } $t->string('status'); $t->unique('order_id'); $t->unique('renewal_id'); });
|
|
|
|
|
|
$schema->create('cloud_invoices', function (Blueprint $t) { $t->increments('id'); $t->string('invoice_no')->unique(); $t->string('purpose')->default('renewal'); $t->integer('instance_id')->nullable(); $t->integer('order_id')->nullable()->unique(); $t->integer('user_id'); $t->integer('plan_id')->nullable(); $t->integer('offer_id')->nullable(); $t->integer('renewal_id')->nullable(); $t->integer('renews_expires_at')->nullable(); $t->integer('due_at'); $t->string('amount'); $t->string('currency')->default('USD'); $t->integer('validity_days'); $t->integer('traffic_quota_bytes'); $t->string('status'); $t->string('paid_via')->nullable(); $t->string('payment_reference')->nullable(); $t->integer('paid_at')->nullable(); $t->integer('created_at'); $t->integer('updated_at'); });
|
|
|
|
|
|
$schema->create('cloud_invoice_items', function (Blueprint $t) { $t->increments('id'); $t->integer('invoice_id'); $t->integer('order_item_id')->nullable(); $t->string('code'); $t->string('label'); $t->string('amount'); $t->string('currency')->default('USD'); $t->text('snapshot_json'); $t->integer('created_at'); $t->unique(['invoice_id', 'code']); });
|
|
|
|
|
|
$schema->create('cloud_backup_subscriptions', function (Blueprint $t) { $t->increments('id'); $t->integer('instance_id')->unique(); $t->integer('user_id'); $t->integer('offer_id'); $t->boolean('enabled')->default(true); $t->boolean('renew_next_period')->default(true); $t->string('frequency')->default('daily'); $t->integer('weekday')->default(0); $t->integer('retention_count')->default(2); $t->string('timezone')->default('UTC'); $t->string('price'); $t->string('currency')->default('USD'); $t->integer('price_version'); $t->integer('state_revision')->default(1); $t->integer('next_run_at'); $t->integer('last_scheduled_at')->nullable(); $t->integer('last_backup_id')->nullable(); $t->integer('created_at'); $t->integer('updated_at'); });
|
|
|
|
|
|
$schema->create('cloud_backup_period_entitlements', function (Blueprint $t) { $t->increments('id'); $t->integer('subscription_id'); $t->integer('instance_id'); $t->integer('service_period_id')->unique(); $t->integer('user_id'); $t->integer('offer_id'); $t->integer('invoice_id'); $t->integer('starts_at'); $t->integer('ends_at'); $t->string('amount'); $t->string('full_period_price'); $t->string('currency')->default('USD'); $t->integer('price_version'); $t->string('source'); $t->integer('paid_at'); $t->integer('created_at'); $t->integer('updated_at'); });
|
|
|
|
|
|
$schema->create('cloud_invoice_payments', function (Blueprint $t) { $t->increments('id'); $t->integer('invoice_id'); $t->integer('user_id'); $t->string('provider'); $t->string('provider_order_no'); $t->string('idempotency_key'); $t->string('amount'); $t->string('status'); $t->integer('created_at'); $t->integer('updated_at'); $t->unique(['provider', 'provider_order_no']); $t->unique(['user_id', 'idempotency_key']); });
|
|
|
|
|
|
$schema->create('cloud_mail_outbox', function (Blueprint $t) { $t->increments('id'); $t->integer('invoice_id'); $t->string('kind'); $t->string('status'); $t->integer('attempts'); $t->integer('available_at'); $t->string('lease_token')->nullable(); $t->integer('lease_until'); $t->integer('sent_at')->nullable(); $t->string('last_error'); $t->integer('created_at'); $t->integer('updated_at'); $t->unique(['invoice_id', 'kind']); });
|
|
|
|
|
|
$schema->create('cloud_ssh_keys', function (Blueprint $t) { $t->increments('id'); $t->integer('user_id'); $t->string('name'); $t->text('public_key'); $t->string('fingerprint'); $t->integer('created_at'); });
|
|
|
|
|
|
$capsule->connection('default')->table('configurations')->insert([
|
|
|
|
|
|
['name' => 'cloud_network_lines_enabled', 'value' => 'false'],
|
|
|
|
|
|
[
|
|
|
|
|
|
'name' => CloudProvisioningBillingConfig::PAYMENT_WINDOW_MINUTES_KEY,
|
|
|
|
|
|
'value' => '60',
|
|
|
|
|
|
],
|
|
|
|
|
|
]);
|
|
|
|
|
|
$capsule->connection('default')->table(
|
|
|
|
|
|
'cloud_platform_test_policy_state'
|
|
|
|
|
|
)->insert(['id' => 1, 'revision' => 0, 'updated_by' => null, 'updated_at' => 0]);
|
|
|
|
|
|
$user = new User(); $user->id = 1; $user->user_name = 'financial-owner'; $user->email = 'owner@example.test'; $user->money = '0.00'; $user->enable = 1; $user->is_admin = false; $user->save(); $user->isLogin = true;
|
|
|
|
|
|
\App\Services\Auth::$user = $user;
|
|
|
|
|
|
$now = time();
|
|
|
|
|
|
$portableRestoreCompatibility =
|
|
|
|
|
|
\App\Services\Cloud\CloudPortableRestoreCompatibility::supported();
|
|
|
|
|
|
$region = new CloudRegion(); $region->code = 'test-region'; $region->name = 'Test Region'; $region->country_code = 'US'; $region->status = 'active'; $region->sort = 0; $region->created_at = $region->updated_at = $now; $region->save();
|
|
|
|
|
|
$line = new CloudNetworkLine(); $line->region_id = $region->id; $line->code = 'default'; $line->name = 'Default'; $line->is_default = true; $line->status = 'active'; $line->sort = 0; $line->created_at = $line->updated_at = $now; $line->save();
|
|
|
|
|
|
$pool = new CloudResourcePool(); $pool->line_id = $line->id; $pool->code = 'general'; $pool->name = 'General'; $pool->status = 'active'; $pool->scheduling_policy = 'balanced'; $pool->sort = 0; $pool->created_at = $pool->updated_at = $now; $pool->save();
|
|
|
|
|
|
$host = new CloudHost(); $host->name = 'host'; $host->enabled = true; $host->resource_pool_id = $pool->id; $host->scheduler_state = 'active'; $host->default_network = 'bridge:br0'; $host->created_at = $host->updated_at = $now; $host->save();
|
|
|
|
|
|
$capsule->connection('default')->table('cloud_host_agents')->insert([
|
|
|
|
|
|
'host_id' => $host->id,
|
|
|
|
|
|
'sealed_api_secret' => 'test-only-agent-secret',
|
|
|
|
|
|
'enabled' => 1,
|
|
|
|
|
|
'revision' => 1,
|
|
|
|
|
|
'tested_revision' => 1,
|
|
|
|
|
|
'last_test_status' => 'healthy',
|
|
|
|
|
|
'last_error_code' => '',
|
|
|
|
|
|
'last_agent_version' => 'test',
|
|
|
|
|
|
'network_selectors_json' => '["bridge:br0","bridge:br1"]',
|
|
|
|
|
|
'public_network_capabilities_json' => json_encode([
|
|
|
|
|
|
'contract' => 1,
|
|
|
|
|
|
'dual_stack_required' => true,
|
|
|
|
|
|
'project_owned_filter' => true,
|
|
|
|
|
|
'ndp_guard' => true,
|
|
|
|
|
|
], JSON_THROW_ON_ERROR),
|
|
|
|
|
|
'portable_restore_contract_json' =>
|
|
|
|
|
|
$portableRestoreCompatibility->json(),
|
|
|
|
|
|
'portable_restore_contract_sha256' =>
|
|
|
|
|
|
$portableRestoreCompatibility->sha256(),
|
|
|
|
|
|
'last_tested_at' => $now,
|
|
|
|
|
|
'created_at' => $now,
|
|
|
|
|
|
'updated_at' => $now,
|
|
|
|
|
|
]);
|
|
|
|
|
|
$capsule->connection('default')->table(
|
|
|
|
|
|
'cloud_host_storage_configurations'
|
|
|
|
|
|
)->insert([
|
|
|
|
|
|
'host_id' => $host->id, 'revision' => 1,
|
|
|
|
|
|
'state' => 'applied',
|
|
|
|
|
|
]);
|
|
|
|
|
|
$membership = new CloudHostResourcePoolMembership(); $membership->host_id = $host->id; $membership->resource_pool_id = $pool->id; $membership->scheduler_state = 'active'; $membership->priority = 0; $membership->weight = 100; $membership->created_at = $membership->updated_at = $now; $membership->save();
|
|
|
|
|
|
$attachment = new CloudHostNetworkAttachment(); $attachment->attachment_key = str_repeat('a', 32); $attachment->host_id = $host->id; $attachment->line_id = $line->id; $attachment->name = 'Primary selector'; $attachment->network_mode = 'selector'; $attachment->network_selector = 'bridge:br0'; $attachment->state = 'active'; $attachment->priority = 0; $attachment->config_revision = 1; $attachment->verified_agent_revision = 1; $attachment->verification_state = 'verified'; $attachment->last_verified_at = $now; $attachment->verification_error_code = ''; $attachment->desired_revision = 1; $attachment->observed_revision = 1; $attachment->sync_state = 'applied'; $attachment->last_error_code = ''; $attachment->created_at = $attachment->updated_at = $now; $attachment->save();
|
|
|
|
|
|
$targetAttachment = new CloudHostNetworkAttachment(); $targetAttachment->attachment_key = str_repeat('b', 32); $targetAttachment->host_id = $host->id; $targetAttachment->line_id = $line->id; $targetAttachment->name = 'Secondary selector'; $targetAttachment->network_mode = 'selector'; $targetAttachment->network_selector = 'bridge:br1'; $targetAttachment->state = 'active'; $targetAttachment->priority = 10; $targetAttachment->config_revision = 1; $targetAttachment->verified_agent_revision = 1; $targetAttachment->verification_state = 'verified'; $targetAttachment->last_verified_at = $now; $targetAttachment->verification_error_code = ''; $targetAttachment->desired_revision = 1; $targetAttachment->observed_revision = 0; $targetAttachment->sync_state = 'never'; $targetAttachment->last_error_code = ''; $targetAttachment->created_at = $targetAttachment->updated_at = $now; $targetAttachment->save();
|
|
|
|
|
|
$makePool = static function (string $key, int $attachmentId,
|
|
|
|
|
|
string $name, int $family, string $cidr, string $gateway,
|
|
|
|
|
|
int $prefix, array $dns, int $priority = 0,
|
|
|
|
|
|
?string $allocationStart = null, ?string $allocationEnd = null,
|
|
|
|
|
|
int $maxAllocations = 0, bool $gatewayOnLink = false
|
|
|
|
|
|
) use ($host, $now): CloudPublicIpPool {
|
|
|
|
|
|
$publicPool = new CloudPublicIpPool();
|
|
|
|
|
|
$publicPool->pool_key = $key;
|
|
|
|
|
|
$publicPool->host_id = $host->id;
|
|
|
|
|
|
$publicPool->network_attachment_id = $attachmentId;
|
|
|
|
|
|
$publicPool->name = $name;
|
|
|
|
|
|
$publicPool->address_family = $family;
|
|
|
|
|
|
$publicPool->cidr = $cidr;
|
|
|
|
|
|
$publicPool->gateway = $gateway;
|
|
|
|
|
|
$publicPool->gateway_on_link = $gatewayOnLink;
|
|
|
|
|
|
$publicPool->guest_prefix_length = $prefix;
|
|
|
|
|
|
$publicPool->dns_servers_json = json_encode($dns,
|
|
|
|
|
|
JSON_THROW_ON_ERROR);
|
|
|
|
|
|
$publicPool->reserved_ips_json = '[]';
|
|
|
|
|
|
$publicPool->allocation_start = $allocationStart;
|
|
|
|
|
|
$publicPool->allocation_end = $allocationEnd;
|
|
|
|
|
|
$publicPool->allocation_cursor = null;
|
|
|
|
|
|
$publicPool->max_allocations = $maxAllocations;
|
|
|
|
|
|
$publicPool->state = 'active';
|
|
|
|
|
|
$publicPool->priority = $priority;
|
|
|
|
|
|
$publicPool->desired_revision = 1;
|
|
|
|
|
|
$publicPool->observed_revision = 1;
|
|
|
|
|
|
$publicPool->sync_state = 'applied';
|
|
|
|
|
|
$publicPool->last_error_code = '';
|
|
|
|
|
|
$publicPool->created_at = $publicPool->updated_at = $now;
|
|
|
|
|
|
$publicPool->save();
|
|
|
|
|
|
return $publicPool;
|
|
|
|
|
|
};
|
|
|
|
|
|
$sourcePoolKey = str_repeat('1', 32);
|
|
|
|
|
|
$sourceIpv6PoolKey = str_repeat('3', 32);
|
|
|
|
|
|
$targetPoolKey = str_repeat('2', 32);
|
|
|
|
|
|
$targetIpv6PoolKey = str_repeat('4', 32);
|
|
|
|
|
|
$sourcePublicPool = $makePool(
|
|
|
|
|
|
$sourcePoolKey, (int) $attachment->id, 'Primary IPv4 pool',
|
|
|
|
|
|
4, '8.8.8.0/24', '8.8.8.1', 24, ['1.1.1.1']
|
|
|
|
|
|
);
|
|
|
|
|
|
$sourceIpv6Pool = $makePool(
|
|
|
|
|
|
$sourceIpv6PoolKey, (int) $attachment->id, 'Primary IPv6 pool',
|
|
|
|
|
|
6, '2404:c140:156:3a::/64', '2404:c140:156::1', 64,
|
|
|
|
|
|
['2606:4700:4700::1111'], 0, '2404:c140:156:3a::10',
|
|
|
|
|
|
'2404:c140:156:3a::ff', 32, true
|
|
|
|
|
|
);
|
|
|
|
|
|
$targetPublicPool = $makePool(
|
|
|
|
|
|
$targetPoolKey, (int) $targetAttachment->id, 'Secondary IPv4 pool',
|
|
|
|
|
|
4, '9.9.9.0/24', '9.9.9.1', 24, ['1.0.0.1'], 10
|
|
|
|
|
|
);
|
|
|
|
|
|
$targetIpv6Pool = $makePool(
|
|
|
|
|
|
$targetIpv6PoolKey, (int) $targetAttachment->id,
|
|
|
|
|
|
'Secondary IPv6 pool', 6, '2606:4700:4700:100::/64',
|
|
|
|
|
|
'2606:4700:4700:100::1', 64, ['2606:4700:4700::1001'],
|
|
|
|
|
|
10, '2606:4700:4700:100::10',
|
|
|
|
|
|
'2606:4700:4700:100::ff', 32
|
|
|
|
|
|
);
|
|
|
|
|
|
$policy = new CloudHostCapacityPolicy(); $policy->host_id = $host->id; $policy->configured_cpu_cores = 64; $policy->configured_memory_mib = 131072; $policy->configured_disk_gib = 4096; $policy->safety_cpu_cores = 2; $policy->safety_memory_mib = 2048; $policy->safety_disk_gib = 64; $policy->max_instances = 100; $policy->created_at = $policy->updated_at = $now; $policy->save();
|
|
|
|
|
|
foreach (['8.8.8.10', '8.8.8.11'] as $address) {
|
|
|
|
|
|
$ip = new CloudPublicIp();
|
|
|
|
|
|
$ip->host_id = $host->id;
|
|
|
|
|
|
$ip->public_ip_pool_id = $sourcePublicPool->id;
|
|
|
|
|
|
$ip->address_family = 4;
|
|
|
|
|
|
$ip->address = $address;
|
|
|
|
|
|
$ip->address_binary = inet_pton($address);
|
|
|
|
|
|
$ip->prefix_length = 24;
|
|
|
|
|
|
$ip->state = 'available';
|
|
|
|
|
|
$ip->created_at = $ip->updated_at = $now;
|
|
|
|
|
|
$ip->save();
|
|
|
|
|
|
}
|
|
|
|
|
|
foreach (['9.9.9.10', '9.9.9.11'] as $address) {
|
|
|
|
|
|
$ip = new CloudPublicIp();
|
|
|
|
|
|
$ip->host_id = $host->id;
|
|
|
|
|
|
$ip->public_ip_pool_id = $targetPublicPool->id;
|
|
|
|
|
|
$ip->address_family = 4;
|
|
|
|
|
|
$ip->address = $address;
|
|
|
|
|
|
$ip->address_binary = inet_pton($address);
|
|
|
|
|
|
$ip->prefix_length = 24;
|
|
|
|
|
|
$ip->state = 'available';
|
|
|
|
|
|
$ip->created_at = $ip->updated_at = $now;
|
|
|
|
|
|
$ip->save();
|
|
|
|
|
|
}
|
|
|
|
|
|
$plan = new Shop(); $plan->name = 'plan'; $plan->cpu_cores = 2; $plan->memory_mib = 2048; $plan->bandwidth_mbps = 100; $plan->disk_gib = 20; $plan->validity_days = 30; $plan->traffic_quota_bytes = 1024 * 1024 * 1024; $plan->status = 1; $plan->sort = 0; $plan->save();
|
|
|
|
|
|
$soldOutPlan = new Shop(); $soldOutPlan->name = 'sold-out-plan'; $soldOutPlan->cpu_cores = 2; $soldOutPlan->memory_mib = 2048; $soldOutPlan->bandwidth_mbps = 100; $soldOutPlan->disk_gib = 20; $soldOutPlan->validity_days = 30; $soldOutPlan->traffic_quota_bytes = 1024 * 1024 * 1024; $soldOutPlan->status = 1; $soldOutPlan->sort = 0; $soldOutPlan->save();
|
|
|
|
|
|
$offer = new CloudOffer(); $offer->plan_id = $plan->id; $offer->resource_pool_id = $pool->id; $offer->code = 'test-offer'; $offer->name = 'Test Offer'; $offer->price = '9.99'; $offer->backup_price = '2.50'; $offer->currency = 'USD'; $offer->price_version = 1; $offer->status = 1; $offer->stock = 10; $offer->sort = 0; $offer->created_at = $offer->updated_at = $now; $offer->save();
|
|
|
|
|
|
$soldOutOffer = new CloudOffer(); $soldOutOffer->plan_id = $soldOutPlan->id; $soldOutOffer->resource_pool_id = $pool->id; $soldOutOffer->code = 'sold-out-offer'; $soldOutOffer->name = 'Sold Out Offer'; $soldOutOffer->price = '8.99'; $soldOutOffer->backup_price = '0.00'; $soldOutOffer->currency = 'USD'; $soldOutOffer->price_version = 1; $soldOutOffer->status = 1; $soldOutOffer->stock = 0; $soldOutOffer->sort = 0; $soldOutOffer->created_at = $soldOutOffer->updated_at = $now; $soldOutOffer->save();
|
|
|
|
|
|
$template = new CloudTemplate();
|
|
|
|
|
|
$template->id = 'alma10-1234abcd-0123456789abcdef';
|
|
|
|
|
|
$template->name = 'AlmaLinux 10';
|
|
|
|
|
|
$template->os_family = 'AlmaLinux';
|
|
|
|
|
|
$template->os_version = '10';
|
|
|
|
|
|
$template->image_bytes = 1024;
|
|
|
|
|
|
$template->sha256 = str_repeat('a', 64);
|
|
|
|
|
|
$template->startup_script = '';
|
|
|
|
|
|
$template->revision = 1;
|
|
|
|
|
|
$template->generation_token = str_repeat('b', 32);
|
|
|
|
|
|
$template->state = 'active';
|
|
|
|
|
|
$template->created_at = $template->updated_at = $now;
|
|
|
|
|
|
$template->save();
|
|
|
|
|
|
$reinstallTemplate = new CloudTemplate();
|
|
|
|
|
|
$reinstallTemplate->id = 'debian13-1234abcd-0123456789abcdef';
|
|
|
|
|
|
$reinstallTemplate->name = 'Debian 13';
|
|
|
|
|
|
$reinstallTemplate->os_family = 'Debian';
|
|
|
|
|
|
$reinstallTemplate->os_version = '13';
|
|
|
|
|
|
$reinstallTemplate->image_bytes = 2048;
|
|
|
|
|
|
$reinstallTemplate->sha256 = str_repeat('c', 64);
|
|
|
|
|
|
$reinstallTemplate->startup_script = '';
|
|
|
|
|
|
$reinstallTemplate->revision = 1;
|
|
|
|
|
|
$reinstallTemplate->generation_token = str_repeat('d', 32);
|
|
|
|
|
|
$reinstallTemplate->state = 'active';
|
|
|
|
|
|
$reinstallTemplate->created_at = $reinstallTemplate->updated_at = $now;
|
|
|
|
|
|
$reinstallTemplate->save();
|
|
|
|
|
|
$capsule->connection('default')->table('cloud_template_replicas')->insert([
|
|
|
|
|
|
'host_id' => $host->id,
|
|
|
|
|
|
'template_id' => $template->id,
|
|
|
|
|
|
'revision' => $template->revision,
|
|
|
|
|
|
'generation_token' => $template->generation_token,
|
|
|
|
|
|
'state' => 'active',
|
|
|
|
|
|
'image_bytes' => $template->image_bytes,
|
|
|
|
|
|
'sha256' => $template->sha256,
|
|
|
|
|
|
'operation_id' => 'fixture-template-import',
|
|
|
|
|
|
'error_code' => '',
|
|
|
|
|
|
'created_at' => $now,
|
|
|
|
|
|
'updated_at' => $now,
|
|
|
|
|
|
]);
|
|
|
|
|
|
$capsule->connection('default')->table('cloud_template_replicas')->insert([
|
|
|
|
|
|
'host_id' => $host->id,
|
|
|
|
|
|
'template_id' => $reinstallTemplate->id,
|
|
|
|
|
|
'revision' => $reinstallTemplate->revision,
|
|
|
|
|
|
'generation_token' => $reinstallTemplate->generation_token,
|
|
|
|
|
|
'state' => 'active',
|
|
|
|
|
|
'image_bytes' => $reinstallTemplate->image_bytes,
|
|
|
|
|
|
'sha256' => $reinstallTemplate->sha256,
|
|
|
|
|
|
'operation_id' => 'fixture-reinstall-template-import',
|
|
|
|
|
|
'error_code' => '',
|
|
|
|
|
|
'created_at' => $now,
|
|
|
|
|
|
'updated_at' => $now,
|
|
|
|
|
|
]);
|
|
|
|
|
|
$actor = ['user_id' => 1, 'is_admin' => false, 'is_super_admin' => false, 'can_manage' => false, 'session_id' => ''];
|
|
|
|
|
|
$input = ['offer_id' => 1, 'source' => 'cloud', 'name' => 'vm-one', 'template_id' => 'alma10-1234abcd-0123456789abcdef', 'idempotency_key' => 'test-request-key-0001'];
|
|
|
|
|
|
$catalogConnection = Capsule::connection('default');
|
|
|
|
|
|
$catalogConnection->flushQueryLog();
|
|
|
|
|
|
$catalogConnection->enableQueryLog();
|
|
|
|
|
|
$catalog = CloudDeploymentPageService::saleCatalog();
|
|
|
|
|
|
$catalogQueryCount = count($catalogConnection->getQueryLog());
|
|
|
|
|
|
$catalogConnection->disableQueryLog();
|
|
|
|
|
|
$assert(count($catalog['offers'] ?? []) === 2
|
|
|
|
|
|
&& ($catalog['offers'][0]['sellable'] ?? null) === true
|
|
|
|
|
|
&& ($catalog['offers'][1]['sellable'] ?? null) === false
|
|
|
|
|
|
&& $catalogQueryCount <= 10,
|
|
|
|
|
|
'Catalog availability hid a sold-out offer, misclassified capacity, or queried per offer.');
|
|
|
|
|
|
$host->default_network = 'public';
|
|
|
|
|
|
$host->save();
|
|
|
|
|
|
$catalogWithoutLegacyDefault = CloudDeploymentPageService::saleCatalog();
|
|
|
|
|
|
$assert(($catalogWithoutLegacyDefault['offers'][0]['sellable'] ?? null)
|
|
|
|
|
|
=== true,
|
|
|
|
|
|
'A retired host default network still controlled sale availability.');
|
|
|
|
|
|
$attachment->network_selector = 'public';
|
|
|
|
|
|
$attachment->save();
|
|
|
|
|
|
$targetAttachment->network_selector = 'public';
|
|
|
|
|
|
$targetAttachment->save();
|
|
|
|
|
|
$unverifiedSelectorCatalog = CloudDeploymentPageService::saleCatalog();
|
|
|
|
|
|
$assert(($unverifiedSelectorCatalog['offers'][0]['sellable'] ?? null)
|
|
|
|
|
|
=== false,
|
|
|
|
|
|
'An attachment outside current Agent evidence remained sellable.');
|
|
|
|
|
|
$attachment->network_selector = 'bridge:br0';
|
|
|
|
|
|
$attachment->verified_agent_revision = 0;
|
|
|
|
|
|
$attachment->save();
|
|
|
|
|
|
$targetAttachment->network_selector = 'bridge:br1';
|
|
|
|
|
|
$targetAttachment->verified_agent_revision = 0;
|
|
|
|
|
|
$targetAttachment->save();
|
|
|
|
|
|
$staleEvidenceCatalog = CloudDeploymentPageService::saleCatalog();
|
|
|
|
|
|
$assert(($staleEvidenceCatalog['offers'][0]['sellable'] ?? null) === false,
|
|
|
|
|
|
'A stale attachment verification revision remained sellable.');
|
|
|
|
|
|
$host->default_network = 'bridge:br0';
|
|
|
|
|
|
$host->save();
|
|
|
|
|
|
$attachment->network_selector = 'bridge:br0';
|
|
|
|
|
|
$attachment->verified_agent_revision = 1;
|
|
|
|
|
|
$attachment->save();
|
|
|
|
|
|
$targetAttachment->network_selector = 'bridge:br1';
|
|
|
|
|
|
$targetAttachment->verified_agent_revision = 1;
|
|
|
|
|
|
$targetAttachment->save();
|
|
|
|
|
|
try {
|
|
|
|
|
|
CloudOrderService::purchase($input + ['network' => 'bridge:attacker'], $actor);
|
|
|
|
|
|
throw new RuntimeException('Purchase accepted a user-selected host network.');
|
|
|
|
|
|
} catch (CloudError $error) {
|
|
|
|
|
|
$assert($error->httpStatus === 422,
|
|
|
|
|
|
'User network injection returned the wrong status.');
|
|
|
|
|
|
}
|
|
|
|
|
|
$membership->scheduler_state = 'disabled';
|
|
|
|
|
|
$membership->updated_at = time();
|
|
|
|
|
|
$membership->save();
|
|
|
|
|
|
$membershipUnavailable = CloudDeploymentPageService::saleCatalog();
|
|
|
|
|
|
$assert(($membershipUnavailable['offers'][0]['sellable'] ?? null) === false,
|
|
|
|
|
|
'Legacy host scheduling fields bypassed the canonical pool membership.');
|
|
|
|
|
|
$membership->scheduler_state = 'active';
|
|
|
|
|
|
$membership->updated_at = time();
|
|
|
|
|
|
$membership->save();
|
|
|
|
|
|
// Network application state is informational; panel capacity and the
|
|
|
|
|
|
// concrete IP ledger decide sellability. A failed engine apply leaves no
|
|
|
|
|
|
// ledger rows for new addresses, so placement still fails closed.
|
|
|
|
|
|
CloudPublicIp::query()->update(['state' => 'reserved', 'updated_at' => time()]);
|
|
|
|
|
|
$noAddressCatalog = CloudDeploymentPageService::saleCatalog();
|
|
|
|
|
|
$assert(($noAddressCatalog['offers'][0]['sellable'] ?? null) === false,
|
|
|
|
|
|
'A pool without an available public IP remained sellable.');
|
|
|
|
|
|
try {
|
|
|
|
|
|
CloudOrderService::purchase(array_replace($input, [
|
|
|
|
|
|
'name' => 'vm-without-ip',
|
|
|
|
|
|
'idempotency_key' => 'test-unavailable-address-0001',
|
|
|
|
|
|
]), $actor);
|
|
|
|
|
|
throw new RuntimeException('Missing public IP created an order.');
|
|
|
|
|
|
} catch (CloudError $e) {
|
|
|
|
|
|
$assert($e->httpStatus === 409,
|
|
|
|
|
|
'Missing public IP returned the wrong status.');
|
|
|
|
|
|
}
|
|
|
|
|
|
$assert(CloudOrder::count() === 0 && CloudInvoice::count() === 0
|
|
|
|
|
|
&& CloudOffer::find(1)->stock === 10,
|
|
|
|
|
|
'Address admission failure created finance state or consumed stock.');
|
|
|
|
|
|
CloudPublicIp::query()->update(['state' => 'available', 'updated_at' => time()]);
|
|
|
|
|
|
$migrationPlacementReservationCount = CloudResourceReservation::count();
|
|
|
|
|
|
$migrationPlacementClaimCount = Capsule::table(
|
|
|
|
|
|
'cloud_resource_reservation_ips'
|
|
|
|
|
|
)->count();
|
|
|
|
|
|
$migrationPlacementConnection = Capsule::connection('default');
|
|
|
|
|
|
$migrationPlacementConnection->beginTransaction();
|
|
|
|
|
|
try {
|
|
|
|
|
|
$migrationInstance = new CloudInstance();
|
|
|
|
|
|
$migrationInstance->host_id = (int) $host->id;
|
|
|
|
|
|
$migrationInstance->user_id = 1;
|
|
|
|
|
|
$migrationInstance->resource_pool_id = (int) $pool->id;
|
|
|
|
|
|
$migrationInstance->uuid =
|
|
|
|
|
|
'66666666-6666-4666-8666-666666666666';
|
|
|
|
|
|
$migrationInstance->name = 'migration-source-fixture';
|
|
|
|
|
|
$migrationInstance->status = 'running';
|
|
|
|
|
|
$migrationInstance->service_status = 'active';
|
|
|
|
|
|
$migrationInstance->network_status = 'active';
|
|
|
|
|
|
$migrationInstance->spec_json = json_encode([
|
|
|
|
|
|
'vcpu' => 2,
|
|
|
|
|
|
'memory_mib' => 2048,
|
|
|
|
|
|
'disk_gib' => 20,
|
|
|
|
|
|
], JSON_THROW_ON_ERROR);
|
|
|
|
|
|
$migrationInstance->created_at =
|
|
|
|
|
|
$migrationInstance->updated_at = time();
|
|
|
|
|
|
$migrationInstance->save();
|
|
|
|
|
|
$migrationRestore = new CloudBackupRestore();
|
|
|
|
|
|
$migrationRestore->restore_key = str_repeat('7', 32);
|
|
|
|
|
|
$migrationRestore->backup_id = null;
|
|
|
|
|
|
$migrationRestore->target_host_id = (int) $host->id;
|
|
|
|
|
|
$migrationRestore->target_instance_id = null;
|
|
|
|
|
|
$migrationRestore->reservation_id = null;
|
|
|
|
|
|
$migrationRestore->job_id = null;
|
|
|
|
|
|
$migrationRestore->operation_id = str_repeat('8', 32);
|
|
|
|
|
|
$migrationRestore->new_uuid =
|
|
|
|
|
|
'77777777-7777-4777-8777-777777777777';
|
|
|
|
|
|
$migrationRestore->new_name = 'migration-target-fixture';
|
|
|
|
|
|
$migrationRestore->state = 'reserved';
|
|
|
|
|
|
$migrationRestore->error_code = '';
|
|
|
|
|
|
$migrationRestore->ready_at = null;
|
|
|
|
|
|
$migrationRestore->created_at =
|
|
|
|
|
|
$migrationRestore->updated_at = time();
|
|
|
|
|
|
$migrationRestore->save();
|
|
|
|
|
|
$migrationPlacement = CloudPlacementService::
|
|
|
|
|
|
reserveMaintenanceMigrationTarget(
|
|
|
|
|
|
$migrationRestore,
|
|
|
|
|
|
1,
|
|
|
|
|
|
[
|
|
|
|
|
|
'cpu_cores' => 2,
|
|
|
|
|
|
'memory_mib' => 2048,
|
|
|
|
|
|
'disk_gib' => 20,
|
|
|
|
|
|
'bandwidth_mbps' => 100,
|
|
|
|
|
|
],
|
|
|
|
|
|
(int) $pool->id,
|
|
|
|
|
|
5
|
|
|
|
|
|
);
|
|
|
|
|
|
$migrationReservation = $migrationPlacement['reservation'];
|
|
|
|
|
|
$migrationNetwork = $migrationPlacement['public_network'];
|
|
|
|
|
|
$migrationRestore->reservation_id = (int) $migrationReservation->id;
|
|
|
|
|
|
$migrationRestore->updated_at = time();
|
|
|
|
|
|
$migrationRestore->save();
|
|
|
|
|
|
$migrationClaimSet = CloudPublicNetworkAllocationService::
|
|
|
|
|
|
currentClaimSet((int) $migrationReservation->id, 'held');
|
|
|
|
|
|
$preparedMigrationPlacement = CloudPlacementService::
|
|
|
|
|
|
preparedMaintenanceMigrationTarget(
|
|
|
|
|
|
$migrationRestore,
|
|
|
|
|
|
(int) $migrationInstance->id
|
|
|
|
|
|
);
|
|
|
|
|
|
$migrationIdentity = $preparedMigrationPlacement['identity'];
|
|
|
|
|
|
$migrationIdentityValue = json_decode(
|
|
|
|
|
|
$migrationIdentity->identityJson(),
|
|
|
|
|
|
true,
|
|
|
|
|
|
16,
|
|
|
|
|
|
JSON_THROW_ON_ERROR
|
|
|
|
|
|
);
|
|
|
|
|
|
$assert((string) $migrationReservation->state === 'held'
|
|
|
|
|
|
&& $migrationReservation->instance_id === null
|
|
|
|
|
|
&& $migrationReservation->network_attachment_id === null
|
|
|
|
|
|
&& $migrationReservation->public_ip_pool_id === null
|
|
|
|
|
|
&& $migrationReservation->public_ip_id === null
|
|
|
|
|
|
&& (int) $migrationReservation->disk_gib === 25
|
|
|
|
|
|
&& (int) $migrationPlacement['target_agent_revision'] === 1
|
|
|
|
|
|
&& $migrationNetwork->toArray()
|
|
|
|
|
|
=== $migrationClaimSet->publicNetwork()->toArray()
|
|
|
|
|
|
&& (int) $preparedMigrationPlacement['reservation']->id
|
|
|
|
|
|
=== (int) $migrationReservation->id
|
|
|
|
|
|
&& $preparedMigrationPlacement['public_network']->toArray()
|
|
|
|
|
|
=== $migrationNetwork->toArray()
|
|
|
|
|
|
&& (int) $preparedMigrationPlacement['target_agent_revision'] === 1
|
|
|
|
|
|
&& hash_equals(
|
|
|
|
|
|
$portableRestoreCompatibility->sha256(),
|
|
|
|
|
|
(string) $preparedMigrationPlacement[
|
|
|
|
|
|
'target_agent_contract_sha256'
|
|
|
|
|
|
]
|
|
|
|
|
|
)
|
|
|
|
|
|
&& (int) ($migrationIdentityValue['reservation_id'] ?? 0)
|
|
|
|
|
|
=== (int) $migrationReservation->id
|
|
|
|
|
|
&& (int) ($migrationIdentityValue['host_id'] ?? 0)
|
|
|
|
|
|
=== (int) $host->id
|
|
|
|
|
|
&& (int) ($migrationIdentityValue[
|
|
|
|
|
|
'network_attachment_id'
|
|
|
|
|
|
] ?? 0) === $migrationClaimSet->networkAttachmentId()
|
|
|
|
|
|
&& ($migrationIdentityValue['uuid'] ?? null)
|
|
|
|
|
|
=== strtolower((string) $migrationRestore->new_uuid)
|
|
|
|
|
|
&& hash_equals(
|
|
|
|
|
|
$migrationIdentity->claimSetSha256(),
|
|
|
|
|
|
$migrationClaimSet->digest()
|
|
|
|
|
|
)
|
|
|
|
|
|
&& hash_equals(
|
|
|
|
|
|
$migrationIdentity->publicNetworkSha256(),
|
|
|
|
|
|
$migrationNetwork->fingerprint()
|
|
|
|
|
|
)
|
|
|
|
|
|
&& $migrationClaimSet->publicNetwork()->address(4) !== ''
|
|
|
|
|
|
&& $migrationClaimSet->publicNetwork()->address(6) !== '',
|
|
|
|
|
|
'Maintenance migration target did not reserve one exact held dual-stack bundle.');
|
|
|
|
|
|
$ipv6MigrationClaim = Capsule::table(
|
|
|
|
|
|
'cloud_resource_reservation_ips'
|
|
|
|
|
|
)->where('reservation_id', (int) $migrationReservation->id)
|
|
|
|
|
|
->where('address_family', 6)->first();
|
|
|
|
|
|
$tamperedOperation = str_repeat('9', 32);
|
|
|
|
|
|
Capsule::table('cloud_resource_reservation_ips')
|
|
|
|
|
|
->where('id', (int) $ipv6MigrationClaim->id)->update([
|
|
|
|
|
|
'operation_id' => $tamperedOperation,
|
|
|
|
|
|
]);
|
|
|
|
|
|
Capsule::table('cloud_public_ips')
|
|
|
|
|
|
->where('id', (int) $ipv6MigrationClaim->public_ip_id)->update([
|
|
|
|
|
|
'operation_id' => $tamperedOperation,
|
|
|
|
|
|
]);
|
|
|
|
|
|
$rejectedMigrationTargetDrift = false;
|
|
|
|
|
|
try {
|
|
|
|
|
|
CloudPlacementService::preparedMaintenanceMigrationTarget(
|
|
|
|
|
|
$migrationRestore,
|
|
|
|
|
|
(int) $migrationInstance->id
|
|
|
|
|
|
);
|
|
|
|
|
|
} catch (CloudError $error) {
|
|
|
|
|
|
$rejectedMigrationTargetDrift = $error->httpStatus === 409;
|
|
|
|
|
|
}
|
|
|
|
|
|
$assert($rejectedMigrationTargetDrift,
|
|
|
|
|
|
'Maintenance migration accepted a drifted IPv6 target claim.');
|
|
|
|
|
|
} finally {
|
|
|
|
|
|
$migrationPlacementConnection->rollBack();
|
|
|
|
|
|
}
|
|
|
|
|
|
$assert(CloudResourceReservation::count()
|
|
|
|
|
|
=== $migrationPlacementReservationCount
|
|
|
|
|
|
&& Capsule::table('cloud_resource_reservation_ips')->count()
|
|
|
|
|
|
=== $migrationPlacementClaimCount,
|
|
|
|
|
|
'Rolled-back migration target placement leaked a reservation or address claim.');
|
|
|
|
|
|
$first = CloudOrderService::purchase($input, $actor);
|
|
|
|
|
|
$same = CloudOrderService::purchase($input, $actor);
|
|
|
|
|
|
$firstSpec = json_decode(
|
|
|
|
|
|
(string) $first->spec_json,
|
|
|
|
|
|
true,
|
|
|
|
|
|
64,
|
|
|
|
|
|
JSON_THROW_ON_ERROR
|
|
|
|
|
|
);
|
|
|
|
|
|
$openingInvoice = CloudInvoice::where('order_id', $first->id)->first();
|
|
|
|
|
|
$openingDisplay = $openingInvoice === null ? []
|
|
|
|
|
|
: CloudInvoiceService::display($openingInvoice);
|
|
|
|
|
|
$connection = Capsule::connection('default');
|
|
|
|
|
|
$connection->flushQueryLog();
|
|
|
|
|
|
$connection->enableQueryLog();
|
|
|
|
|
|
$openingSummaries = $openingInvoice === null ? []
|
|
|
|
|
|
: CloudInvoiceService::listSummaries(array_fill(0, 25, $openingInvoice));
|
|
|
|
|
|
$summaryQueryCount = count($connection->getQueryLog());
|
|
|
|
|
|
$connection->disableQueryLog();
|
|
|
|
|
|
$assert($first->id === $same->id
|
|
|
|
|
|
&& (string) $first->status === 'awaiting_payment'
|
|
|
|
|
|
&& $openingInvoice !== null
|
|
|
|
|
|
&& (string) $openingInvoice->purpose === 'provision'
|
|
|
|
|
|
&& $openingInvoice->instance_id === null
|
|
|
|
|
|
&& (int) $openingInvoice->due_at - (int) $openingInvoice->created_at === 3600
|
|
|
|
|
|
&& ($openingDisplay['purpose'] ?? null) === 'provision'
|
|
|
|
|
|
&& ($openingDisplay['title'] ?? null) === '实例开通'
|
|
|
|
|
|
&& ($openingDisplay['type_label'] ?? null) === '实例开通'
|
|
|
|
|
|
&& ($openingDisplay['cpu_cores'] ?? null) === 2
|
|
|
|
|
|
&& ($openingDisplay['memory_mib'] ?? null) === 2048
|
|
|
|
|
|
&& !array_key_exists('invoice_no', $openingDisplay)
|
|
|
|
|
|
&& ($openingDisplay['order_status'] ?? null) === 'awaiting_payment'
|
|
|
|
|
|
&& ($firstSpec['stock_mode'] ?? null) === 'finite'
|
|
|
|
|
|
&& array_key_exists('traffic_pool_group_id', $firstSpec)
|
|
|
|
|
|
&& $firstSpec['traffic_pool_group_id'] === null
|
|
|
|
|
|
&& ($firstSpec['traffic_quota_mode'] ?? null) === 'finite'
|
|
|
|
|
|
&& ($firstSpec['automatic_backup'] ?? null) === false
|
|
|
|
|
|
&& ($firstSpec['instance_price'] ?? null) === '9.99'
|
|
|
|
|
|
&& ($firstSpec['backup_price'] ?? null) === '0.00'
|
|
|
|
|
|
&& count($openingDisplay['items'] ?? []) === 1
|
|
|
|
|
|
&& ($openingDisplay['items'][0]['code'] ?? null) === 'instance'
|
|
|
|
|
|
&& CloudOrderItem::where('order_id', $first->id)->count() === 1
|
|
|
|
|
|
&& $first->stock_reserved_at !== null
|
|
|
|
|
|
&& $first->stock_released_at === null
|
|
|
|
|
|
&& User::find(1)->money === '0.00'
|
|
|
|
|
|
&& CloudJob::count() === 0
|
|
|
|
|
|
&& CloudOffer::find(1)->stock === 9,
|
|
|
|
|
|
'Idempotent deployment request charged money, queued work, lost its opening invoice, or reserved stock twice.');
|
|
|
|
|
|
$assert(count($openingSummaries) === 25
|
|
|
|
|
|
&& $summaryQueryCount <= 3
|
|
|
|
|
|
&& array_keys($openingSummaries[0] ?? []) === [
|
|
|
|
|
|
'id', 'title', 'type_label', 'purpose', 'amount', 'currency',
|
|
|
|
|
|
'status', 'display_status', 'available_action', 'payable',
|
|
|
|
|
|
'renews_expires_at', 'due_at', 'payment_deadline_at', 'created_at',
|
|
|
|
|
|
'order_status',
|
|
|
|
|
|
'fulfillment_status', 'task_id', 'job_status',
|
|
|
|
|
|
'instance_id', 'instance_name',
|
|
|
|
|
|
]
|
|
|
|
|
|
&& ($openingSummaries[0]['order_status'] ?? null) === 'awaiting_payment',
|
|
|
|
|
|
'Invoice list summaries are not compact or use per-invoice association queries.');
|
|
|
|
|
|
try { CloudOrderService::purchase($input + ['vcpu' => 64], $actor); throw new RuntimeException('Resource bypass accepted.'); } catch (CloudError $e) {}
|
|
|
|
|
|
try { CloudOrderService::purchase(array_replace($input, ['name' => 'changed']), $actor); throw new RuntimeException('Idempotency conflict accepted.'); } catch (CloudError $e) { $assert($e->httpStatus === 409, 'Wrong idempotency response.'); }
|
|
|
|
|
|
try {
|
|
|
|
|
|
CloudInvoiceService::payBalance($openingInvoice,
|
|
|
|
|
|
array_replace($actor, ['user_id' => 2]), 'provision-cross-tenant-0001');
|
|
|
|
|
|
throw new RuntimeException('Another tenant paid an opening invoice.');
|
|
|
|
|
|
} catch (CloudError $e) {
|
|
|
|
|
|
$assert($e->httpStatus === 403, 'Opening invoice ownership failure leaked another status.');
|
|
|
|
|
|
}
|
|
|
|
|
|
$assert(User::find(1)->money === '0.00'
|
|
|
|
|
|
&& CloudOrder::find($first->id)->status === 'awaiting_payment'
|
|
|
|
|
|
&& CloudJob::count() === 0,
|
|
|
|
|
|
'Cross-tenant opening-invoice access changed financial or queue state.');
|
|
|
|
|
|
try {
|
|
|
|
|
|
CloudInvoiceService::payBalance(
|
|
|
|
|
|
$openingInvoice, $actor, 'provision-insufficient-balance-0001');
|
|
|
|
|
|
throw new RuntimeException('Opening invoice was paid without sufficient balance.');
|
|
|
|
|
|
} catch (CloudError $e) {
|
|
|
|
|
|
$assert($e->httpStatus === 409, 'Insufficient opening-invoice balance returned the wrong status.');
|
|
|
|
|
|
}
|
|
|
|
|
|
$assert(CloudOrder::find($first->id)->status === 'awaiting_payment'
|
|
|
|
|
|
&& CloudInvoice::find($openingInvoice->id)->status === 'open'
|
|
|
|
|
|
&& CloudJob::count() === 0,
|
|
|
|
|
|
'Failed opening-invoice payment changed invoice, order, or queue state.');
|
|
|
|
|
|
$user = User::find(1); $user->money = '100.00'; $user->save();
|
|
|
|
|
|
$attachment->verified_agent_revision = 0;
|
|
|
|
|
|
$attachment->save();
|
|
|
|
|
|
$targetAttachment->verified_agent_revision = 0;
|
|
|
|
|
|
$targetAttachment->save();
|
|
|
|
|
|
try {
|
|
|
|
|
|
CloudInvoiceService::payBalance(
|
|
|
|
|
|
$openingInvoice,
|
|
|
|
|
|
$actor,
|
|
|
|
|
|
'provision-stale-network-evidence-0001'
|
|
|
|
|
|
);
|
|
|
|
|
|
throw new RuntimeException(
|
|
|
|
|
|
'Payment admitted stale attachment verification evidence.'
|
|
|
|
|
|
);
|
|
|
|
|
|
} catch (CloudError $error) {
|
|
|
|
|
|
$assert($error->httpStatus === 409,
|
|
|
|
|
|
'Stale attachment evidence returned the wrong payment status.');
|
|
|
|
|
|
}
|
|
|
|
|
|
$assert(User::find(1)->money === '100.00'
|
|
|
|
|
|
&& CloudInvoice::find($openingInvoice->id)->status === 'open'
|
|
|
|
|
|
&& CloudOrder::find($first->id)->status === 'awaiting_payment'
|
|
|
|
|
|
&& CloudResourceReservation::count() === 0
|
|
|
|
|
|
&& CloudJob::count() === 0,
|
|
|
|
|
|
'Failed network admission retained money, a reservation, or queued work.');
|
|
|
|
|
|
$attachment->verified_agent_revision = 1;
|
|
|
|
|
|
$attachment->save();
|
|
|
|
|
|
$targetAttachment->verified_agent_revision = 1;
|
|
|
|
|
|
$targetAttachment->save();
|
|
|
|
|
|
// Capacity must remain tied to the immutable order snapshot even if an
|
|
|
|
|
|
// administrator changes the base plan while the invoice is still open.
|
|
|
|
|
|
$changedPlan = Shop::find($plan->id);
|
|
|
|
|
|
$changedPlan->cpu_cores = 8;
|
|
|
|
|
|
$changedPlan->memory_mib = 8192;
|
|
|
|
|
|
$changedPlan->disk_gib = 80;
|
|
|
|
|
|
$changedPlan->save();
|
|
|
|
|
|
$policyManager = [
|
|
|
|
|
|
'user_id' => 99, 'is_admin' => true,
|
|
|
|
|
|
'is_super_admin' => true, 'can_manage' => true,
|
|
|
|
|
|
];
|
|
|
|
|
|
$testCatalog = CloudPlatformTestNetworkPolicy::create([
|
|
|
|
|
|
'expected_revision' => 0,
|
|
|
|
|
|
'service' => 'Provision snapshot test',
|
|
|
|
|
|
'protocol' => 'tcp',
|
|
|
|
|
|
'port' => '8080',
|
|
|
|
|
|
'direction' => 'ingress',
|
|
|
|
|
|
'reason' => 'Verify queued creates retain deleted catalog rules.',
|
|
|
|
|
|
], $policyManager);
|
|
|
|
|
|
$testRuleKey = (string) $testCatalog['rules'][0]['id'];
|
|
|
|
|
|
$host->default_network = 'public';
|
|
|
|
|
|
$host->save();
|
|
|
|
|
|
$paidOpening = CloudInvoiceService::payBalance(
|
|
|
|
|
|
$openingInvoice, $actor, 'provision-balance-test-0001');
|
|
|
|
|
|
$paidOpeningAgain = CloudInvoiceService::payBalance(
|
|
|
|
|
|
$paidOpening->fresh(), $actor, 'provision-balance-test-0001');
|
|
|
|
|
|
$host->default_network = 'bridge:br0';
|
|
|
|
|
|
$host->save();
|
|
|
|
|
|
$openingOrder = CloudOrder::find($first->id);
|
|
|
|
|
|
$openingReservation = CloudResourceReservation::where('order_id', $first->id)->first();
|
|
|
|
|
|
$openingJob = CloudJob::where('order_id', $first->id)->first();
|
|
|
|
|
|
$queuedCreate = \App\Services\Cloud\CloudSecrets::decrypt(
|
|
|
|
|
|
(string) $openingJob->sealed_payload,
|
|
|
|
|
|
'job-payload'
|
|
|
|
|
|
);
|
|
|
|
|
|
$queuedPolicy = $queuedCreate['body']['platform_network_policy_snapshot'] ?? null;
|
|
|
|
|
|
$orderPolicy = json_decode(
|
|
|
|
|
|
(string) $openingOrder->spec_json, true,
|
|
|
|
|
|
flags: JSON_THROW_ON_ERROR
|
|
|
|
|
|
)['platform_network_policy_snapshot'] ?? null;
|
|
|
|
|
|
$openingSpec = json_decode(
|
|
|
|
|
|
(string) $openingOrder->spec_json, true,
|
|
|
|
|
|
flags: JSON_THROW_ON_ERROR
|
|
|
|
|
|
);
|
|
|
|
|
|
CloudPlatformTestNetworkPolicy::delete(
|
|
|
|
|
|
$testRuleKey, ['expected_revision' => 1], $policyManager
|
|
|
|
|
|
);
|
|
|
|
|
|
$currentPolicy = CloudPlatformTestNetworkPolicy::snapshotForProvisioning();
|
|
|
|
|
|
$sealedJobAfterDelete = CloudJob::where('order_id', $first->id)->first();
|
|
|
|
|
|
$sealedCreateAfterDelete = \App\Services\Cloud\CloudSecrets::decrypt(
|
|
|
|
|
|
(string) $sealedJobAfterDelete->sealed_payload,
|
|
|
|
|
|
'job-payload'
|
|
|
|
|
|
);
|
|
|
|
|
|
$queuedPolicyAfterDelete = $sealedCreateAfterDelete['body']
|
|
|
|
|
|
['platform_network_policy_snapshot'] ?? null;
|
|
|
|
|
|
$orderPolicyAfterDelete = json_decode(
|
|
|
|
|
|
(string) CloudOrder::find($first->id)->spec_json, true,
|
|
|
|
|
|
flags: JSON_THROW_ON_ERROR
|
|
|
|
|
|
)['platform_network_policy_snapshot'] ?? null;
|
|
|
|
|
|
$openingClaims = Capsule::table('cloud_resource_reservation_ips')
|
|
|
|
|
|
->where('reservation_id', (int) ($openingReservation->id ?? 0))
|
|
|
|
|
|
->orderBy('address_family')->get();
|
|
|
|
|
|
$assert((string) $paidOpeningAgain->status === 'paid'
|
|
|
|
|
|
&& User::find(1)->money === '90.01'
|
|
|
|
|
|
&& $openingOrder->status === 'queued'
|
|
|
|
|
|
&& (int) $openingOrder->host_id === (int) $host->id
|
|
|
|
|
|
&& $openingReservation !== null
|
|
|
|
|
|
&& $openingReservation->state === 'held'
|
|
|
|
|
|
&& (int) $openingReservation->cpu_cores === 2
|
|
|
|
|
|
&& (int) $openingReservation->memory_mib === 2048
|
|
|
|
|
|
&& (int) $openingReservation->disk_gib === 20
|
|
|
|
|
|
&& $openingReservation->network_attachment_id === null
|
|
|
|
|
|
&& $openingReservation->public_ip_pool_id === null
|
|
|
|
|
|
&& $openingReservation->public_ip_id === null
|
|
|
|
|
|
&& $openingClaims->count() === 2
|
|
|
|
|
|
&& $openingClaims->pluck('address_family')->all() === [4, 6]
|
|
|
|
|
|
&& $openingClaims->every(
|
|
|
|
|
|
static fn ($claim): bool => (string) $claim->state === 'reserved'
|
|
|
|
|
|
)
|
|
|
|
|
|
&& (string) $openingJob->operation_id === (string) $openingReservation->operation_id
|
|
|
|
|
|
&& CloudPublicIp::whereIn('id', $openingClaims->pluck('public_ip_id'))
|
|
|
|
|
|
->where('state', 'reserved')->count() === 2
|
|
|
|
|
|
&& CloudJob::where('order_id', $first->id)->count() === 1
|
|
|
|
|
|
&& is_array($queuedPolicy)
|
|
|
|
|
|
&& $queuedPolicy === $orderPolicy
|
|
|
|
|
|
&& ($queuedPolicy['catalog_revision'] ?? null) === 1
|
|
|
|
|
|
&& ($queuedPolicy['rules'][9]['id'] ?? null) === 'test-' . $testRuleKey
|
|
|
|
|
|
&& $queuedPolicyAfterDelete === $queuedPolicy
|
|
|
|
|
|
&& $orderPolicyAfterDelete === $queuedPolicy
|
|
|
|
|
|
&& ($currentPolicy['catalog_revision'] ?? null) === 2
|
|
|
|
|
|
&& count($currentPolicy['rules']) === 9
|
|
|
|
|
|
&& ($openingSpec['template_id'] ?? null) === $template->id
|
|
|
|
|
|
&& ($openingSpec['os_source'] ?? null) === 'cloud_template'
|
|
|
|
|
|
&& ($openingSpec['os_name'] ?? null) === 'AlmaLinux 10'
|
|
|
|
|
|
&& ($openingSpec['login_name'] ?? null) === 'root'
|
|
|
|
|
|
&& ($openingSpec['public_network']['ipv4']['address_cidr'] ?? null)
|
|
|
|
|
|
=== '8.8.8.10/24'
|
|
|
|
|
|
&& ($openingSpec['public_network']['ipv6']['address_cidr'] ?? null)
|
|
|
|
|
|
=== '2404:c140:156:3a::10/64'
|
|
|
|
|
|
&& !array_key_exists('managed', $openingSpec)
|
|
|
|
|
|
&& CloudPlatformTestNetworkPolicy::assertSnapshot($queuedPolicy)
|
|
|
|
|
|
=== $queuedPolicy
|
|
|
|
|
|
&& $openingOrder->sealed_payload === null,
|
|
|
|
|
|
'Opening payment did not atomically freeze an immutable per-instance platform policy.');
|
|
|
|
|
|
$changedPlan->cpu_cores = 2;
|
|
|
|
|
|
$changedPlan->memory_mib = 2048;
|
|
|
|
|
|
$changedPlan->disk_gib = 20;
|
|
|
|
|
|
$changedPlan->save();
|
|
|
|
|
|
$history = [];
|
|
|
|
|
|
$openingIdentity = CloudOperationRecoveryIdentity::expected(
|
|
|
|
|
|
$openingJob,
|
|
|
|
|
|
true
|
|
|
|
|
|
);
|
|
|
|
|
|
$createdVm = [
|
|
|
|
|
|
'uuid' => $openingIdentity['resource_id'],
|
|
|
|
|
|
'name' => '1:vm-111111111111',
|
|
|
|
|
|
'vcpu' => 2,
|
|
|
|
|
|
'memory_mib' => 2048,
|
|
|
|
|
|
'disk_gib' => 20,
|
|
|
|
|
|
'disks' => [[
|
|
|
|
|
|
'device' => 'disk',
|
|
|
|
|
|
'format' => 'qcow2',
|
|
|
|
|
|
]],
|
|
|
|
|
|
'autostart' => true,
|
|
|
|
|
|
'public_network' => $openingSpec['public_network'],
|
|
|
|
|
|
'console_port' => 5908,
|
|
|
|
|
|
'generated_password' => 'must-not-persist',
|
|
|
|
|
|
];
|
|
|
|
|
|
$mismatchedNetwork = $openingSpec['public_network'];
|
|
|
|
|
|
$mismatchedNetwork['ipv4']['address_cidr'] = '8.8.8.11/24';
|
|
|
|
|
|
$mismatchedVm = array_replace($createdVm, [
|
|
|
|
|
|
'public_network' => $mismatchedNetwork,
|
|
|
|
|
|
]);
|
|
|
|
|
|
$mock = new MockHandler([
|
|
|
|
|
|
new Response(200, [], json_encode([
|
|
|
|
|
|
'operation_id' => $openingJob->operation_id,
|
|
|
|
|
|
'status' => 'succeeded', 'result' => $createdVm,
|
|
|
|
|
|
...$openingIdentity,
|
|
|
|
|
|
])),
|
|
|
|
|
|
]);
|
|
|
|
|
|
$stack = HandlerStack::create($mock); $stack->push(Middleware::history($history));
|
|
|
|
|
|
$worker = new CloudWorkerService(new CloudAgentClient(
|
|
|
|
|
|
'1',
|
|
|
|
|
|
'test-secret-which-is-at-least-32-bytes',
|
|
|
|
|
|
new Client(['handler' => $stack]),
|
|
|
|
|
|
20.0,
|
|
|
|
|
|
'http://127.0.0.1:8001/internal/agent/request',
|
|
|
|
|
|
'test-gateway-secret-at-least-32-bytes'
|
|
|
|
|
|
));
|
|
|
|
|
|
$worker->tick();
|
|
|
|
|
|
$openingJob->refresh();
|
|
|
|
|
|
$openingReservation->refresh();
|
|
|
|
|
|
$assert(count($history) === 1
|
|
|
|
|
|
&& $openingJob->phase === 'terminal'
|
|
|
|
|
|
&& $openingJob->outcome === 'succeeded'
|
|
|
|
|
|
&& $openingJob->dispatch_attempts === 1,
|
|
|
|
|
|
'Provisioning did not converge after exactly one Agent mutation dispatch.');
|
|
|
|
|
|
$openingDisplay = CloudInvoiceService::display($openingInvoice->fresh());
|
|
|
|
|
|
$openingJobDisplay = CloudJobService::display(CloudJob::where('order_id', $first->id)->first());
|
|
|
|
|
|
$assert(CloudOrder::find($first->id)->status === 'succeeded' && CloudInstance::count() === 1
|
|
|
|
|
|
&& \App\Models\CloudServicePeriod::count() === 1
|
|
|
|
|
|
&& \App\Models\CloudServicePeriod::first()->traffic_pool_group_id === null
|
|
|
|
|
|
&& (string) \App\Models\CloudServicePeriod::first()->traffic_quota_mode
|
|
|
|
|
|
=== 'finite'
|
|
|
|
|
|
&& CloudResourceReservation::where('order_id', $first->id)->first()->state === 'committed'
|
|
|
|
|
|
&& CloudPublicIp::whereIn('id', $openingClaims->pluck('public_ip_id'))
|
|
|
|
|
|
->where('state', 'assigned')->whereNotNull('instance_id')
|
|
|
|
|
|
->count() === 2
|
|
|
|
|
|
&& CloudInvoice::find($openingInvoice->id)->instance_id !== null
|
|
|
|
|
|
&& CloudMailOutbox::where('invoice_id', $openingInvoice->id)
|
|
|
|
|
|
->where('kind', 'provision_succeeded')->count() === 1
|
|
|
|
|
|
&& ($openingDisplay['order_status'] ?? null) === 'succeeded'
|
|
|
|
|
|
&& ($openingDisplay['instance_id'] ?? null) !== null
|
|
|
|
|
|
&& ($openingDisplay['job_status'] ?? null) === 'succeeded',
|
|
|
|
|
|
'Successful provision did not create its first service period or update the opening bill.');
|
|
|
|
|
|
$assert(!CloudInvoiceService::queueProvisionSucceeded(
|
|
|
|
|
|
CloudInvoice::find($openingInvoice->id), time())
|
|
|
|
|
|
&& CloudMailOutbox::where('invoice_id', $openingInvoice->id)
|
|
|
|
|
|
->where('kind', 'provision_succeeded')->count() === 1,
|
|
|
|
|
|
'Successful provisioning notification was not idempotent.');
|
|
|
|
|
|
$assert(($openingJobDisplay['completed'] ?? null) === true
|
|
|
|
|
|
&& ($openingJobDisplay['status'] ?? null) === 'succeeded'
|
|
|
|
|
|
&& ($openingJobDisplay['status_url'] ?? null)
|
|
|
|
|
|
=== '/api/v1/tasks/' . $openingJob->id
|
|
|
|
|
|
&& ($openingJobDisplay['created_at'] ?? 0) > 0
|
|
|
|
|
|
&& ($openingJobDisplay['terminal_at'] ?? 0) > 0
|
|
|
|
|
|
&& !array_key_exists('attempts', $openingJobDisplay),
|
|
|
|
|
|
'User-visible creation receipt lacks a strict terminal result or leaks retry metadata.');
|
|
|
|
|
|
$openingAgentRequest = $agentRequest($history[0]);
|
|
|
|
|
|
$posted = json_decode($openingAgentRequest['body'], true);
|
|
|
|
|
|
$signedActor = json_decode(base64_decode(strtr(
|
|
|
|
|
|
$openingAgentRequest['headers']['X-Virt-Actor'], '-_', '+/'
|
|
|
|
|
|
)), true);
|
|
|
|
|
|
$assert($posted['vcpu'] === 2 && $posted['bandwidth_mbps'] === 100
|
|
|
|
|
|
&& !array_key_exists('network', $posted)
|
|
|
|
|
|
&& ($posted['public_network'] ?? null)
|
|
|
|
|
|
=== $openingSpec['public_network']
|
|
|
|
|
|
&& ($posted['platform_network_policy_snapshot'] ?? null)
|
|
|
|
|
|
=== $queuedPolicy
|
|
|
|
|
|
&& $signedActor['can_manage'] === true
|
|
|
|
|
|
&& ($signedActor['operation_id'] ?? null) === $openingJob->operation_id
|
|
|
|
|
|
&& !isset($signedActor['can_provision']),
|
|
|
|
|
|
'Plan snapshot or typed Agent provisioning authority was lost.');
|
|
|
|
|
|
$assert(!str_contains(CloudJob::first()->result_json, 'must-not-persist'), 'Plain password saved in result.');
|
|
|
|
|
|
$terminalPayload = \App\Services\Cloud\CloudSecrets::decrypt(
|
|
|
|
|
|
CloudJob::first()->sealed_payload,
|
|
|
|
|
|
'job-payload'
|
|
|
|
|
|
);
|
|
|
|
|
|
$assert(!array_key_exists('credential', $terminalPayload)
|
|
|
|
|
|
&& !array_key_exists('root_password', $terminalPayload['body'] ?? []),
|
|
|
|
|
|
'Terminal provisioning job retained a recoverable root password.');
|
|
|
|
|
|
$instance = CloudInstance::first();
|
|
|
|
|
|
$instanceSpec = json_decode((string) $instance->spec_json, true,
|
|
|
|
|
|
flags: JSON_THROW_ON_ERROR);
|
|
|
|
|
|
$assert($instance->name === 'vm-one'
|
|
|
|
|
|
&& ($instanceSpec['public_network'] ?? null)
|
|
|
|
|
|
=== $openingSpec['public_network']
|
|
|
|
|
|
&& ($instanceSpec['os_source'] ?? null) === 'cloud_template'
|
|
|
|
|
|
&& ($instanceSpec['os_name'] ?? null) === 'AlmaLinux 10'
|
|
|
|
|
|
&& ($instanceSpec['login_name'] ?? null) === 'root'
|
|
|
|
|
|
&& ($instanceSpec['managed'] ?? null) === true
|
|
|
|
|
|
&& ($instanceSpec['agent_metadata_version'] ?? null) === 1
|
|
|
|
|
|
&& ($instanceSpec['disk_gib'] ?? null) === 20
|
|
|
|
|
|
&& !isset($instanceSpec['disks'])
|
|
|
|
|
|
&& !isset($instanceSpec['console_port'])
|
|
|
|
|
|
&& !isset($instanceSpec['disks'][0]['source']),
|
|
|
|
|
|
'Successful provisioning did not persist the safe Agent metadata snapshot.');
|
|
|
|
|
|
\App\Services\Cloud\CloudPlacementService::markUncertain($openingOrder->fresh());
|
|
|
|
|
|
$assert(CloudResourceReservation::find($openingReservation->id)->state === 'committed'
|
|
|
|
|
|
&& CloudPublicIp::whereIn('id', $openingClaims->pluck('public_ip_id'))
|
|
|
|
|
|
->where('state', 'assigned')->count() === 2,
|
|
|
|
|
|
'A late uncertainty marker rolled a committed reservation or assigned IP backward.');
|
|
|
|
|
|
|
|
|
|
|
|
// An unpaid provisioning reservation expires without entering the worker
|
|
|
|
|
|
// queue, changing the wallet, or retaining its encrypted credential.
|
|
|
|
|
|
$abandoned = CloudOrderService::purchase(array_replace($input, [
|
|
|
|
|
|
'name' => 'vm-abandoned', 'idempotency_key' => 'test-request-key-abandoned',
|
|
|
|
|
|
'automatic_backup' => true,
|
|
|
|
|
|
]), $actor);
|
|
|
|
|
|
$abandonedInvoice = CloudInvoice::where('order_id', $abandoned->id)->first();
|
|
|
|
|
|
$abandonedSpec = json_decode((string) $abandoned->spec_json, true,
|
|
|
|
|
|
flags: JSON_THROW_ON_ERROR);
|
|
|
|
|
|
$abandonedDisplay = CloudInvoiceService::display($abandonedInvoice);
|
|
|
|
|
|
$assert((string) $abandoned->amount === '12.49'
|
|
|
|
|
|
&& ($abandonedSpec['automatic_backup'] ?? null) === true
|
|
|
|
|
|
&& ($abandonedSpec['instance_price'] ?? null) === '9.99'
|
|
|
|
|
|
&& ($abandonedSpec['backup_price'] ?? null) === '2.50'
|
|
|
|
|
|
&& array_column($abandonedDisplay['items'] ?? [], 'code')
|
|
|
|
|
|
=== ['instance', 'automatic_backup'],
|
|
|
|
|
|
'Automatic backup was not frozen as a separate opening-bill item.');
|
|
|
|
|
|
$abandonedInvoice->due_at = time() - 1;
|
|
|
|
|
|
$abandonedInvoice->save();
|
|
|
|
|
|
$assert(CloudInvoiceService::expireProvision() === 1
|
|
|
|
|
|
&& CloudInvoiceService::expireProvision() === 0
|
|
|
|
|
|
&& CloudOrder::find($abandoned->id)->status === 'expired'
|
|
|
|
|
|
&& CloudOrder::find($abandoned->id)->sealed_payload === null
|
|
|
|
|
|
&& CloudOrder::find($abandoned->id)->stock_released_at !== null
|
|
|
|
|
|
&& CloudJob::where('order_id', $abandoned->id)->count() === 0
|
|
|
|
|
|
&& User::find(1)->money === '90.01'
|
|
|
|
|
|
&& CloudOffer::find(1)->stock === 9,
|
|
|
|
|
|
'Expired opening invoice queued work, changed money, retained credentials, or released stock more than once.');
|
|
|
|
|
|
$assert(CloudAccess::instance((string) $instance->id, $actor)->id === $instance->id, 'Owner denied own VM.');
|
|
|
|
|
|
$other = array_replace($actor, ['user_id' => 2]);
|
|
|
|
|
|
try { CloudAccess::instance((string) $instance->id, $other); throw new RuntimeException('Tenant isolation failed.'); } catch (CloudError $e) { $assert($e->httpStatus === 404, 'Nonowner information leaked.'); }
|
|
|
|
|
|
$admin = array_replace($other, ['is_admin' => true]);
|
|
|
|
|
|
$assert(CloudAccess::instance((string) $instance->id, $admin)->id === $instance->id, 'Admin readonly denied.');
|
|
|
|
|
|
try { CloudAccess::instance((string) $instance->id, $admin, true); throw new RuntimeException('Regular admin wrote other owner.'); } catch (CloudError $e) {}
|
|
|
|
|
|
try { CloudJobService::enqueue($instance, array_replace($admin, ['can_manage' => true]), 'action', 'POST', '/actions', ['action' => 'reboot']); throw new RuntimeException('Scoped can_manage bypassed cross-owner queue authorization.'); } catch (CloudError $e) {}
|
|
|
|
|
|
try { CloudAccess::manage($actor + ['can_provision' => true]); throw new RuntimeException('Provision actor globally privileged.'); } catch (CloudError $e) {}
|
|
|
|
|
|
// A backup-origin deployment must cross the same paid create ledger, but
|
|
|
|
|
|
// dispatch a signed restore-backup request and retain the exact backup
|
|
|
|
|
|
// identity until a deterministic Agent rejection safely refunds it.
|
|
|
|
|
|
$originalSourceSpec = (string) $instance->spec_json;
|
|
|
|
|
|
$sourceBackupSpec = json_decode(
|
|
|
|
|
|
$originalSourceSpec,
|
|
|
|
|
|
true,
|
|
|
|
|
|
64,
|
|
|
|
|
|
JSON_THROW_ON_ERROR
|
|
|
|
|
|
);
|
|
|
|
|
|
$sourceBackupSpec['automatic_backup'] = true;
|
|
|
|
|
|
$sourceBackupSpec['backup_price'] = '2.50';
|
|
|
|
|
|
$instance->spec_json = json_encode(
|
|
|
|
|
|
$sourceBackupSpec,
|
|
|
|
|
|
JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES
|
|
|
|
|
|
);
|
|
|
|
|
|
$instance->save();
|
|
|
|
|
|
$subscription = new CloudBackupSubscription();
|
|
|
|
|
|
$subscription->instance_id = (int) $instance->id;
|
|
|
|
|
|
$subscription->user_id = 1;
|
|
|
|
|
|
$subscription->offer_id = (int) $offer->id;
|
|
|
|
|
|
$subscription->enabled = true;
|
|
|
|
|
|
$subscription->renew_next_period = true;
|
|
|
|
|
|
$subscription->frequency = 'daily';
|
|
|
|
|
|
$subscription->weekday = 0;
|
|
|
|
|
|
$subscription->retention_count = 2;
|
|
|
|
|
|
$subscription->timezone = 'UTC';
|
|
|
|
|
|
$subscription->price = '2.50';
|
|
|
|
|
|
$subscription->currency = 'USD';
|
|
|
|
|
|
$subscription->price_version = 1;
|
|
|
|
|
|
$subscription->next_run_at = time() + 86400;
|
|
|
|
|
|
$subscription->created_at = $subscription->updated_at = time();
|
|
|
|
|
|
$subscription->save();
|
|
|
|
|
|
$capsule->table('cloud_backup_period_entitlements')->insert([
|
|
|
|
|
|
'subscription_id' => (int) $subscription->id,
|
|
|
|
|
|
'instance_id' => (int) $instance->id,
|
|
|
|
|
|
'service_period_id' => (int) $instance->current_period_id,
|
|
|
|
|
|
'user_id' => 1,
|
|
|
|
|
|
'offer_id' => (int) $offer->id,
|
|
|
|
|
|
'invoice_id' => (int) CloudInvoice::where(
|
|
|
|
|
|
'order_id', (int) $instance->order_id
|
|
|
|
|
|
)->value('id'),
|
|
|
|
|
|
'starts_at' => time() - 60,
|
|
|
|
|
|
'ends_at' => time() + 86400,
|
|
|
|
|
|
'amount' => '2.50',
|
|
|
|
|
|
'full_period_price' => '2.50',
|
|
|
|
|
|
'currency' => 'USD',
|
|
|
|
|
|
'price_version' => 1,
|
|
|
|
|
|
'source' => 'provision',
|
|
|
|
|
|
'paid_at' => time() - 60,
|
|
|
|
|
|
'created_at' => time() - 60,
|
|
|
|
|
|
'updated_at' => time() - 60,
|
|
|
|
|
|
]);
|
|
|
|
|
|
$sourceBackup = new CloudInstanceBackup();
|
|
|
|
|
|
$sourceBackup->backup_key = str_repeat('c', 32);
|
|
|
|
|
|
$sourceBackup->storage_artifact_id = 501;
|
|
|
|
|
|
$sourceBackup->source_instance_id = (int) $instance->id;
|
|
|
|
|
|
$sourceBackup->source_host_id = (int) $host->id;
|
|
|
|
|
|
$sourceBackup->user_id = 1;
|
|
|
|
|
|
$sourceBackup->source_uuid = strtolower((string) $instance->uuid);
|
|
|
|
|
|
$sourceBackup->name = 'automatic-backup-fixture';
|
|
|
|
|
|
$sourceBackup->purpose = 'portable';
|
|
|
|
|
|
$sourceBackup->source = 'automatic';
|
|
|
|
|
|
$sourceBackup->subscription_id = (int) $subscription->id;
|
|
|
|
|
|
$sourceBackup->scheduled_for = time() - 60;
|
|
|
|
|
|
$sourceBackup->state = 'ready';
|
|
|
|
|
|
$sourceBackup->consistency = 'filesystem';
|
|
|
|
|
|
$sourceBackup->object_key = 'virt-cloud/backups/'
|
|
|
|
|
|
. $sourceBackup->backup_key . '/system.qcow2';
|
|
|
|
|
|
$sourceBackup->object_size_bytes = 1073741824;
|
|
|
|
|
|
$sourceBackup->virtual_size_bytes = 10 * 1073741824;
|
|
|
|
|
|
$sourceBackup->sha256 = str_repeat('d', 64);
|
|
|
|
|
|
$sourceBackup->spec_json = json_encode(
|
|
|
|
|
|
$sourceBackupSpec,
|
|
|
|
|
|
JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES
|
|
|
|
|
|
);
|
|
|
|
|
|
$sourceBackup->sealed_storage_runtime = '';
|
|
|
|
|
|
$sourceBackup->sealed_credential = $instance->sealed_credential;
|
|
|
|
|
|
$sourceBackup->storage_revision = 0;
|
|
|
|
|
|
$sourceBackup->source_portable_restore_contract_json =
|
|
|
|
|
|
$portableRestoreCompatibility->json();
|
|
|
|
|
|
$sourceBackup->source_portable_restore_contract_sha256 =
|
|
|
|
|
|
$portableRestoreCompatibility->sha256();
|
|
|
|
|
|
$sourceBackup->ready_at = time();
|
|
|
|
|
|
$sourceBackup->created_at = $sourceBackup->updated_at = time();
|
|
|
|
|
|
$sourceBackup->save();
|
|
|
|
|
|
$storageCredential = \App\Services\Cloud\CloudSecrets::encrypt([
|
|
|
|
|
|
'access_key_id' => 'test-access-key',
|
|
|
|
|
|
'secret_access_key' => 'test-secret-key',
|
|
|
|
|
|
], 'object-storage-backend-credentials');
|
|
|
|
|
|
$capsule->table('cloud_storage_backends')->insert([
|
|
|
|
|
|
'id' => 401,
|
|
|
|
|
|
'backend_key' => str_repeat('4', 32),
|
|
|
|
|
|
'identity_sha256' => str_repeat('5', 64),
|
|
|
|
|
|
'name' => 'backup-primary',
|
|
|
|
|
|
'driver' => 's3',
|
|
|
|
|
|
'endpoint' => 'https://objects.example.test',
|
|
|
|
|
|
'region' => 'us-east-1',
|
|
|
|
|
|
'bucket' => 'virt-test',
|
|
|
|
|
|
'prefix' => 'virt-cloud',
|
|
|
|
|
|
'path_style' => 1,
|
|
|
|
|
|
'failure_domain' => 'test-primary',
|
|
|
|
|
|
'sealed_credentials' => $storageCredential,
|
|
|
|
|
|
'config_revision' => 1,
|
|
|
|
|
|
'tested_revision' => 1,
|
|
|
|
|
|
'admin_state' => 'active',
|
|
|
|
|
|
'health_state' => 'healthy',
|
|
|
|
|
|
'write_priority' => 10,
|
|
|
|
|
|
'consecutive_failures' => 0,
|
|
|
|
|
|
'consecutive_successes' => 1,
|
|
|
|
|
|
'last_error_code' => '',
|
|
|
|
|
|
'last_tested_at' => time(),
|
|
|
|
|
|
'last_healthy_at' => time(),
|
|
|
|
|
|
'next_probe_at' => time() + 300,
|
|
|
|
|
|
'created_at' => time(),
|
|
|
|
|
|
'updated_at' => time(),
|
|
|
|
|
|
]);
|
|
|
|
|
|
$capsule->table('cloud_storage_artifacts')->insert([
|
|
|
|
|
|
'id' => 501,
|
|
|
|
|
|
'artifact_key' => str_repeat('6', 32),
|
|
|
|
|
|
'kind' => 'backup',
|
|
|
|
|
|
'maximum_size_bytes' => 10 * 1073741824,
|
|
|
|
|
|
'size_bytes' => (int) $sourceBackup->object_size_bytes,
|
|
|
|
|
|
'sha256' => (string) $sourceBackup->sha256,
|
|
|
|
|
|
'durability_policy' => 'mirror_async_2',
|
|
|
|
|
|
'desired_copies' => 2,
|
|
|
|
|
|
'required_copies' => 1,
|
|
|
|
|
|
'state' => 'available',
|
|
|
|
|
|
'last_operation_id' => str_repeat('7', 32),
|
|
|
|
|
|
'verified_at' => time(),
|
|
|
|
|
|
'created_at' => time(),
|
|
|
|
|
|
'updated_at' => time(),
|
|
|
|
|
|
]);
|
|
|
|
|
|
$capsule->table('cloud_storage_publications')->insert([
|
|
|
|
|
|
'id' => 502,
|
|
|
|
|
|
'artifact_id' => 501,
|
|
|
|
|
|
'backup_id' => (int) $sourceBackup->id,
|
|
|
|
|
|
'state' => 'ready_to_finalize',
|
|
|
|
|
|
'created_at' => time(),
|
|
|
|
|
|
'updated_at' => time(),
|
|
|
|
|
|
]);
|
|
|
|
|
|
$capsule->table('cloud_storage_replicas')->insert([
|
|
|
|
|
|
'id' => 601,
|
|
|
|
|
|
'replica_key' => str_repeat('8', 32),
|
|
|
|
|
|
'artifact_id' => 501,
|
|
|
|
|
|
'backend_id' => 401,
|
|
|
|
|
|
'role' => 'primary',
|
|
|
|
|
|
'object_key' => (string) $sourceBackup->object_key,
|
|
|
|
|
|
'object_version_id' => '',
|
|
|
|
|
|
'object_etag' => '',
|
|
|
|
|
|
'size_bytes' => (int) $sourceBackup->object_size_bytes,
|
|
|
|
|
|
'sha256' => (string) $sourceBackup->sha256,
|
|
|
|
|
|
'state' => 'ready',
|
|
|
|
|
|
'publish_operation_id' => str_repeat('9', 32),
|
|
|
|
|
|
'delete_operation_id' => null,
|
|
|
|
|
|
'delete_generation' => 0,
|
|
|
|
|
|
'delete_cycle_attempts' => 0,
|
|
|
|
|
|
'delete_presence_confirmations' => 0,
|
|
|
|
|
|
'delete_last_checked_at' => 0,
|
|
|
|
|
|
'delete_next_check_at' => 0,
|
|
|
|
|
|
'delete_attention_at' => null,
|
|
|
|
|
|
'lease_token' => null,
|
|
|
|
|
|
'lease_until' => 0,
|
|
|
|
|
|
'error_code' => '',
|
|
|
|
|
|
'verified_at' => time(),
|
|
|
|
|
|
'created_at' => time(),
|
|
|
|
|
|
'updated_at' => time(),
|
|
|
|
|
|
]);
|
|
|
|
|
|
$orderCountBeforeDirectBackupRequest = CloudOrder::count();
|
|
|
|
|
|
try {
|
|
|
|
|
|
CloudOrderService::purchase([
|
|
|
|
|
|
'offer_id' => (int) $offer->id,
|
|
|
|
|
|
'source' => 'backup',
|
|
|
|
|
|
'backup_id' => (int) $sourceBackup->id,
|
|
|
|
|
|
'name' => 'forbidden-backup-source',
|
|
|
|
|
|
'automatic_backup' => false,
|
|
|
|
|
|
'idempotency_key' => 'test-direct-backup-source-0001',
|
|
|
|
|
|
], $actor);
|
|
|
|
|
|
throw new RuntimeException(
|
|
|
|
|
|
'A cloud backup was accepted as a public deployment source.'
|
|
|
|
|
|
);
|
|
|
|
|
|
} catch (CloudError $error) {
|
|
|
|
|
|
$assert(str_contains($error->getMessage(), '云快照'),
|
|
|
|
|
|
'Direct backup deployment did not direct the user to cloud snapshots.');
|
|
|
|
|
|
}
|
|
|
|
|
|
$assert(CloudOrder::count() === $orderCountBeforeDirectBackupRequest,
|
|
|
|
|
|
'Rejected direct backup deployment created finance state.');
|
|
|
|
|
|
// The guest system disk is authoritative. A restored source may no longer
|
|
|
|
|
|
// have a trustworthy panel credential envelope and must still be usable
|
|
|
|
|
|
// as a cloud snapshot deployment source.
|
|
|
|
|
|
$sourceBackup->sealed_credential = null;
|
|
|
|
|
|
$sourceBackup->updated_at = time();
|
|
|
|
|
|
$sourceBackup->save();
|
|
|
|
|
|
$deploymentSnapshot = new CloudDeploymentSnapshot();
|
|
|
|
|
|
$deploymentSnapshot->snapshot_key = str_repeat('e', 32);
|
|
|
|
|
|
$deploymentSnapshot->user_id = 1;
|
|
|
|
|
|
$deploymentSnapshot->backup_id = (int) $sourceBackup->id;
|
|
|
|
|
|
$deploymentSnapshot->source_instance_id = (int) $instance->id;
|
|
|
|
|
|
$deploymentSnapshot->name = 'deployment-snapshot-fixture';
|
|
|
|
|
|
$deploymentSnapshot->state = 'ready';
|
|
|
|
|
|
$deploymentSnapshot->idempotency_key =
|
|
|
|
|
|
'snapshot-promotion-test-0001';
|
|
|
|
|
|
$deploymentSnapshot->request_hash = str_repeat('f', 64);
|
|
|
|
|
|
$deploymentSnapshot->error_code = '';
|
|
|
|
|
|
$deploymentSnapshot->ready_at = time();
|
|
|
|
|
|
$deploymentSnapshot->created_at =
|
|
|
|
|
|
$deploymentSnapshot->updated_at = time();
|
|
|
|
|
|
$deploymentSnapshot->save();
|
|
|
|
|
|
$contractGateOrderCount = CloudOrder::count();
|
|
|
|
|
|
$contractGateInvoiceCount = CloudInvoice::count();
|
|
|
|
|
|
$sourceBackup->source_portable_restore_contract_json = '{}';
|
|
|
|
|
|
$sourceBackup->source_portable_restore_contract_sha256 = '';
|
|
|
|
|
|
$sourceBackup->save();
|
|
|
|
|
|
try {
|
|
|
|
|
|
CloudOrderService::purchase([
|
|
|
|
|
|
'offer_id' => (int) $offer->id,
|
|
|
|
|
|
'source' => 'deployment_snapshot',
|
|
|
|
|
|
'cloud_snapshot_key' => $deploymentSnapshot->snapshot_key,
|
|
|
|
|
|
'name' => 'vm-incompatible-backup',
|
|
|
|
|
|
'automatic_backup' => false,
|
|
|
|
|
|
'idempotency_key' => 'snapshot-source-contract-missing-0001',
|
|
|
|
|
|
], $actor);
|
|
|
|
|
|
throw new RuntimeException(
|
|
|
|
|
|
'Cloud snapshot without source compatibility evidence was sold.'
|
|
|
|
|
|
);
|
|
|
|
|
|
} catch (CloudError $error) {
|
|
|
|
|
|
$assert($error->httpStatus === 409,
|
|
|
|
|
|
'Missing source compatibility evidence returned the wrong status.');
|
|
|
|
|
|
}
|
|
|
|
|
|
$sourceBackup->source_portable_restore_contract_json =
|
|
|
|
|
|
$portableRestoreCompatibility->json();
|
|
|
|
|
|
$sourceBackup->source_portable_restore_contract_sha256 =
|
|
|
|
|
|
$portableRestoreCompatibility->sha256();
|
|
|
|
|
|
$sourceBackup->save();
|
|
|
|
|
|
$assert(CloudOrder::count() === $contractGateOrderCount
|
|
|
|
|
|
&& CloudInvoice::count() === $contractGateInvoiceCount,
|
|
|
|
|
|
'Rejected incompatible cloud snapshot created finance state.');
|
|
|
|
|
|
$snapshotOrderCount = CloudOrder::count();
|
|
|
|
|
|
$snapshotInvoiceCount = CloudInvoice::count();
|
|
|
|
|
|
$snapshotJobCount = CloudJob::count();
|
|
|
|
|
|
$snapshotReservationCount = CloudResourceReservation::count();
|
|
|
|
|
|
$snapshotBalance = (string) User::findOrFail(1)->money;
|
|
|
|
|
|
$snapshotStock = (int) CloudOffer::findOrFail($offer->id)->stock;
|
|
|
|
|
|
$connection = Capsule::connection('default');
|
|
|
|
|
|
$connection->beginTransaction();
|
|
|
|
|
|
try {
|
|
|
|
|
|
$snapshotOrder = CloudOrderService::purchase([
|
|
|
|
|
|
'offer_id' => (int) $offer->id,
|
|
|
|
|
|
'source' => 'deployment_snapshot',
|
|
|
|
|
|
'cloud_snapshot_key' => $deploymentSnapshot->snapshot_key,
|
|
|
|
|
|
'name' => 'vm-two',
|
|
|
|
|
|
'automatic_backup' => false,
|
|
|
|
|
|
'idempotency_key' => 'test-request-key-0002',
|
|
|
|
|
|
], $actor);
|
|
|
|
|
|
$snapshotReplay = CloudOrderService::purchase([
|
|
|
|
|
|
'offer_id' => (int) $offer->id,
|
|
|
|
|
|
'source' => 'deployment_snapshot',
|
|
|
|
|
|
'cloud_snapshot_key' => $deploymentSnapshot->snapshot_key,
|
|
|
|
|
|
'name' => 'vm-two',
|
|
|
|
|
|
'automatic_backup' => false,
|
|
|
|
|
|
'idempotency_key' => 'test-request-key-0002',
|
|
|
|
|
|
], $actor);
|
|
|
|
|
|
$snapshotInvoice = CloudInvoice::where(
|
|
|
|
|
|
'order_id', (int) $snapshotOrder->id
|
|
|
|
|
|
)->firstOrFail();
|
|
|
|
|
|
$snapshotOrderSpec = json_decode(
|
|
|
|
|
|
(string) $snapshotOrder->spec_json,
|
|
|
|
|
|
true,
|
|
|
|
|
|
64,
|
|
|
|
|
|
JSON_THROW_ON_ERROR
|
|
|
|
|
|
);
|
|
|
|
|
|
$assert((int) $snapshotReplay->id === (int) $snapshotOrder->id
|
|
|
|
|
|
&& (string) $snapshotOrder->status === 'awaiting_payment'
|
|
|
|
|
|
&& (int) $snapshotOrder->deployment_snapshot_id
|
|
|
|
|
|
=== (int) $deploymentSnapshot->id
|
|
|
|
|
|
&& (int) ($snapshotOrderSpec['backing_backup_id'] ?? 0)
|
|
|
|
|
|
=== (int) $sourceBackup->id
|
|
|
|
|
|
&& ($snapshotOrderSpec['cloud_snapshot_key'] ?? null)
|
|
|
|
|
|
=== (string) $deploymentSnapshot->snapshot_key
|
|
|
|
|
|
&& CloudOrder::count() === $snapshotOrderCount + 1
|
|
|
|
|
|
&& CloudInvoice::count() === $snapshotInvoiceCount + 1
|
|
|
|
|
|
&& CloudJob::count() === $snapshotJobCount
|
|
|
|
|
|
&& CloudResourceReservation::count()
|
|
|
|
|
|
=== $snapshotReservationCount
|
|
|
|
|
|
&& (string) User::findOrFail(1)->money === $snapshotBalance,
|
|
|
|
|
|
'Unpaid cloud-snapshot purchase was not idempotent or reserved runtime resources.');
|
|
|
|
|
|
|
|
|
|
|
|
Capsule::table('cloud_host_agents')->where(
|
|
|
|
|
|
'host_id', (int) $host->id
|
|
|
|
|
|
)->update([
|
|
|
|
|
|
'portable_restore_contract_json' => '{}',
|
|
|
|
|
|
'portable_restore_contract_sha256' => '',
|
|
|
|
|
|
'updated_at' => time(),
|
|
|
|
|
|
]);
|
|
|
|
|
|
try {
|
|
|
|
|
|
CloudInvoiceService::payBalance(
|
|
|
|
|
|
$snapshotInvoice,
|
|
|
|
|
|
$actor,
|
|
|
|
|
|
'snapshot-incompatible-target-0002'
|
|
|
|
|
|
);
|
|
|
|
|
|
throw new RuntimeException(
|
|
|
|
|
|
'Cloud snapshot payment selected an incompatible target Agent.'
|
|
|
|
|
|
);
|
|
|
|
|
|
} catch (CloudError $error) {
|
|
|
|
|
|
$assert($error->httpStatus === 409,
|
|
|
|
|
|
'Incompatible target Agent returned the wrong status.');
|
|
|
|
|
|
}
|
|
|
|
|
|
$snapshotOrder->refresh();
|
|
|
|
|
|
$snapshotInvoice->refresh();
|
|
|
|
|
|
$assert((string) $snapshotOrder->status === 'awaiting_payment'
|
|
|
|
|
|
&& (string) $snapshotInvoice->status === 'open'
|
|
|
|
|
|
&& CloudJob::count() === $snapshotJobCount
|
|
|
|
|
|
&& CloudResourceReservation::count()
|
|
|
|
|
|
=== $snapshotReservationCount
|
|
|
|
|
|
&& (string) User::findOrFail(1)->money === $snapshotBalance,
|
|
|
|
|
|
'Rejected target compatibility changed money, queue, or placement state.');
|
|
|
|
|
|
Capsule::table('cloud_host_agents')->where(
|
|
|
|
|
|
'host_id', (int) $host->id
|
|
|
|
|
|
)->update([
|
|
|
|
|
|
'portable_restore_contract_json' =>
|
|
|
|
|
|
$portableRestoreCompatibility->json(),
|
|
|
|
|
|
'portable_restore_contract_sha256' =>
|
|
|
|
|
|
$portableRestoreCompatibility->sha256(),
|
|
|
|
|
|
'updated_at' => time(),
|
|
|
|
|
|
]);
|
|
|
|
|
|
|
|
|
|
|
|
// Payment must retain the prepaid order's 20 GiB target even if an
|
|
|
|
|
|
// administrator edits the mutable plan before the invoice is paid.
|
|
|
|
|
|
$snapshotChangedPlan = Shop::findOrFail((int) $plan->id);
|
|
|
|
|
|
$snapshotChangedPlan->disk_gib = 5;
|
|
|
|
|
|
$snapshotChangedPlan->save();
|
|
|
|
|
|
CloudInvoiceService::payBalance(
|
|
|
|
|
|
$snapshotInvoice,
|
|
|
|
|
|
$actor,
|
|
|
|
|
|
'snapshot-provision-balance-0002'
|
|
|
|
|
|
);
|
|
|
|
|
|
$snapshotOrder->refresh();
|
|
|
|
|
|
$snapshotInvoice->refresh();
|
|
|
|
|
|
$snapshotJob = CloudJob::where(
|
|
|
|
|
|
'order_id', (int) $snapshotOrder->id
|
|
|
|
|
|
)->firstOrFail();
|
|
|
|
|
|
$snapshotReservation = CloudResourceReservation::where(
|
|
|
|
|
|
'order_id', (int) $snapshotOrder->id
|
|
|
|
|
|
)->firstOrFail();
|
|
|
|
|
|
$snapshotClaims = Capsule::table(
|
|
|
|
|
|
'cloud_resource_reservation_ips'
|
|
|
|
|
|
)->where('reservation_id', (int) $snapshotReservation->id)
|
|
|
|
|
|
->orderBy('address_family')->get();
|
|
|
|
|
|
$snapshotPayload = \App\Services\Cloud\CloudSecrets::decrypt(
|
|
|
|
|
|
(string) $snapshotJob->sealed_payload,
|
|
|
|
|
|
'job-payload'
|
|
|
|
|
|
);
|
|
|
|
|
|
$snapshotQueuedSpec = json_decode(
|
|
|
|
|
|
(string) $snapshotOrder->spec_json,
|
|
|
|
|
|
true,
|
|
|
|
|
|
64,
|
|
|
|
|
|
JSON_THROW_ON_ERROR
|
|
|
|
|
|
);
|
|
|
|
|
|
$snapshotNetwork = $snapshotQueuedSpec['public_network'] ?? null;
|
|
|
|
|
|
$snapshotRequestTarget = $snapshotPayload['request_target'] ?? null;
|
|
|
|
|
|
$assert((string) $snapshotOrder->status === 'queued'
|
|
|
|
|
|
&& (string) $snapshotInvoice->status === 'paid'
|
|
|
|
|
|
&& $snapshotOrder->sealed_payload === null
|
|
|
|
|
|
&& (string) $snapshotJob->kind === 'create'
|
|
|
|
|
|
&& (int) $snapshotJob->backup_id === (int) $sourceBackup->id
|
|
|
|
|
|
&& is_string($snapshotRequestTarget)
|
|
|
|
|
|
&& preg_match(
|
|
|
|
|
|
'#^/v1/vms/[a-f0-9-]{36}/restore-backup$#D',
|
|
|
|
|
|
$snapshotRequestTarget
|
|
|
|
|
|
) === 1
|
|
|
|
|
|
&& ($snapshotPayload['body']['backup_key'] ?? null)
|
|
|
|
|
|
=== (string) $sourceBackup->backup_key
|
|
|
|
|
|
&& ($snapshotPayload['body']['source_sha256'] ?? null)
|
|
|
|
|
|
=== (string) $sourceBackup->sha256
|
|
|
|
|
|
&& ($snapshotPayload['body']['disk_gib'] ?? null) === 20
|
|
|
|
|
|
&& ($snapshotQueuedSpec['target_agent_revision'] ?? null) === 1
|
|
|
|
|
|
&& ($snapshotQueuedSpec['target_agent_contract_sha256'] ?? null)
|
|
|
|
|
|
=== $portableRestoreCompatibility->sha256()
|
|
|
|
|
|
&& ($snapshotPayload['backup_metadata'][
|
|
|
|
|
|
'target_agent_revision'
|
|
|
|
|
|
] ?? null) === 1
|
|
|
|
|
|
&& ($snapshotPayload['backup_metadata'][
|
|
|
|
|
|
'target_agent_contract_sha256'
|
|
|
|
|
|
] ?? null) === $portableRestoreCompatibility->sha256()
|
|
|
|
|
|
&& (int) $snapshotReservation->disk_gib === 20
|
|
|
|
|
|
&& is_string($snapshotPayload['body']['source_url'] ?? null)
|
|
|
|
|
|
&& str_starts_with(
|
|
|
|
|
|
$snapshotPayload['body']['source_url'],
|
|
|
|
|
|
'https://objects.example.test/'
|
|
|
|
|
|
)
|
|
|
|
|
|
&& !array_key_exists('credential', $snapshotPayload)
|
|
|
|
|
|
&& is_array($snapshotNetwork)
|
|
|
|
|
|
&& ($snapshotPayload['body']['public_network'] ?? null)
|
|
|
|
|
|
=== $snapshotNetwork
|
|
|
|
|
|
&& ($snapshotNetwork['ipv4']['address_cidr'] ?? null)
|
|
|
|
|
|
=== '8.8.8.11/24'
|
|
|
|
|
|
&& ($snapshotNetwork['ipv6']['address_cidr'] ?? null)
|
|
|
|
|
|
=== '2404:c140:156:3a::11/64'
|
|
|
|
|
|
&& $snapshotClaims->count() === 2
|
|
|
|
|
|
&& $snapshotClaims->pluck('address_family')->all() === [4, 6]
|
|
|
|
|
|
&& $snapshotClaims->every(
|
|
|
|
|
|
static fn ($claim): bool =>
|
|
|
|
|
|
(string) $claim->state === 'reserved'
|
|
|
|
|
|
),
|
|
|
|
|
|
'Paid cloud-snapshot provisioning did not freeze one exact dual-stack restore task.');
|
|
|
|
|
|
|
|
|
|
|
|
$snapshotIdentity = CloudOperationRecoveryIdentity::expected(
|
|
|
|
|
|
$snapshotJob,
|
|
|
|
|
|
true
|
|
|
|
|
|
);
|
|
|
|
|
|
// Finalization must use the immutable target proof selected while the
|
|
|
|
|
|
// payment transaction held the Agent row lock. A later administrator
|
|
|
|
|
|
// edit may make the live Agent temporarily unready, but it cannot
|
|
|
|
|
|
// rewrite the identity of an already dispatched restore.
|
|
|
|
|
|
Capsule::table('cloud_host_agents')->where(
|
|
|
|
|
|
'host_id', (int) $host->id
|
|
|
|
|
|
)->update([
|
|
|
|
|
|
'revision' => 2,
|
|
|
|
|
|
'tested_revision' => 0,
|
|
|
|
|
|
'last_test_status' => 'never',
|
|
|
|
|
|
'portable_restore_contract_json' => '{}',
|
|
|
|
|
|
'portable_restore_contract_sha256' => '',
|
|
|
|
|
|
'updated_at' => time(),
|
|
|
|
|
|
]);
|
|
|
|
|
|
$snapshotCreatedVm = array_replace($createdVm, [
|
|
|
|
|
|
'uuid' => $snapshotIdentity['resource_id'],
|
|
|
|
|
|
'name' => '1:vm-snapshot-restore',
|
|
|
|
|
|
'public_network' => $snapshotNetwork,
|
|
|
|
|
|
'backup_key' => (string) $sourceBackup->backup_key,
|
|
|
|
|
|
'sha256' => (string) $sourceBackup->sha256,
|
|
|
|
|
|
]);
|
|
|
|
|
|
unset($snapshotCreatedVm['generated_password']);
|
|
|
|
|
|
$historyBeforeSnapshot = count($history);
|
|
|
|
|
|
$mock->append(new Response(200, [], json_encode([
|
|
|
|
|
|
'operation_id' => (string) $snapshotJob->operation_id,
|
|
|
|
|
|
'status' => 'succeeded',
|
|
|
|
|
|
'result' => $snapshotCreatedVm,
|
|
|
|
|
|
...$snapshotIdentity,
|
|
|
|
|
|
], JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES)));
|
|
|
|
|
|
$assert($worker->tick(),
|
|
|
|
|
|
'Paid cloud-snapshot provisioning task was not processed.');
|
|
|
|
|
|
$snapshotJob->refresh();
|
|
|
|
|
|
$snapshotOrder->refresh();
|
|
|
|
|
|
$snapshotReservation->refresh();
|
|
|
|
|
|
$snapshotInstance = CloudInstance::where(
|
|
|
|
|
|
'order_id', (int) $snapshotOrder->id
|
|
|
|
|
|
)->first();
|
|
|
|
|
|
$snapshotInstanceSpec = $snapshotInstance === null ? null
|
|
|
|
|
|
: json_decode(
|
|
|
|
|
|
(string) $snapshotInstance->spec_json,
|
|
|
|
|
|
true,
|
|
|
|
|
|
64,
|
|
|
|
|
|
JSON_THROW_ON_ERROR
|
|
|
|
|
|
);
|
|
|
|
|
|
$snapshotAgentRequest = $agentRequest(
|
|
|
|
|
|
$history[$historyBeforeSnapshot]
|
|
|
|
|
|
);
|
|
|
|
|
|
$snapshotPosted = json_decode(
|
|
|
|
|
|
$snapshotAgentRequest['body'],
|
|
|
|
|
|
true,
|
|
|
|
|
|
64,
|
|
|
|
|
|
JSON_THROW_ON_ERROR
|
|
|
|
|
|
);
|
|
|
|
|
|
$assert((string) $snapshotJob->phase === 'terminal'
|
|
|
|
|
|
&& (string) $snapshotJob->outcome === 'succeeded'
|
|
|
|
|
|
&& (int) $snapshotJob->dispatch_attempts === 1
|
|
|
|
|
|
&& (int) $snapshotJob->receipt_deadline_at
|
|
|
|
|
|
- (int) $snapshotJob->dispatched_at
|
|
|
|
|
|
=== CloudJobService::receiptDeadlineSeconds('backup_restore')
|
|
|
|
|
|
&& (string) $snapshotOrder->status === 'succeeded'
|
|
|
|
|
|
&& $snapshotInstance !== null
|
|
|
|
|
|
&& $snapshotInstance->sealed_credential === null
|
|
|
|
|
|
&& ($snapshotInstanceSpec['public_network'] ?? null)
|
|
|
|
|
|
=== $snapshotNetwork
|
|
|
|
|
|
&& ($snapshotInstanceSpec['restored_from_backup'] ?? null)
|
|
|
|
|
|
=== (string) $sourceBackup->backup_key
|
|
|
|
|
|
&& !array_key_exists('guest_hostname',
|
|
|
|
|
|
$snapshotInstanceSpec ?? [])
|
|
|
|
|
|
&& (string) $snapshotReservation->state === 'committed'
|
|
|
|
|
|
&& $bundleState(
|
|
|
|
|
|
$snapshotReservation,
|
|
|
|
|
|
'committed',
|
|
|
|
|
|
'assigned'
|
|
|
|
|
|
)
|
|
|
|
|
|
&& ($snapshotAgentRequest['request_target'] ?? null)
|
|
|
|
|
|
=== $snapshotRequestTarget
|
|
|
|
|
|
&& ($snapshotPosted['hostname'] ?? null) === ''
|
|
|
|
|
|
&& ($snapshotPosted['root_password'] ?? null) === ''
|
|
|
|
|
|
&& ($snapshotPosted['guest_ssh_public_key'] ?? null) === ''
|
|
|
|
|
|
&& ($snapshotPosted['public_network'] ?? null)
|
|
|
|
|
|
=== $snapshotNetwork,
|
|
|
|
|
|
'Cloud-snapshot Worker delivery replaced guest identity or failed to commit the exact dual-stack ledger.');
|
|
|
|
|
|
} finally {
|
|
|
|
|
|
$connection->rollBack();
|
|
|
|
|
|
}
|
|
|
|
|
|
$assert(CloudOrder::count() === $snapshotOrderCount
|
|
|
|
|
|
&& CloudInvoice::count() === $snapshotInvoiceCount
|
|
|
|
|
|
&& CloudJob::count() === $snapshotJobCount
|
|
|
|
|
|
&& CloudResourceReservation::count() === $snapshotReservationCount
|
|
|
|
|
|
&& (string) User::findOrFail(1)->money === $snapshotBalance
|
|
|
|
|
|
&& (int) CloudOffer::findOrFail($offer->id)->stock === $snapshotStock,
|
|
|
|
|
|
'Rolled-back cloud-snapshot fixture leaked finance or placement state.');
|
|
|
|
|
|
|
|
|
|
|
|
$queueSnapshotProvision = static function (
|
|
|
|
|
|
string $name,
|
|
|
|
|
|
string $requestKey,
|
|
|
|
|
|
string $paymentKey
|
|
|
|
|
|
) use ($actor, $offer, $deploymentSnapshot): array {
|
|
|
|
|
|
$order = CloudOrderService::purchase([
|
|
|
|
|
|
'offer_id' => (int) $offer->id,
|
|
|
|
|
|
'source' => 'deployment_snapshot',
|
|
|
|
|
|
'cloud_snapshot_key' => $deploymentSnapshot->snapshot_key,
|
|
|
|
|
|
'name' => $name,
|
|
|
|
|
|
'automatic_backup' => false,
|
|
|
|
|
|
'idempotency_key' => $requestKey,
|
|
|
|
|
|
], $actor);
|
|
|
|
|
|
CloudInvoiceService::payBalance(
|
|
|
|
|
|
CloudInvoice::where('order_id', (int) $order->id)->firstOrFail(),
|
|
|
|
|
|
$actor,
|
|
|
|
|
|
$paymentKey
|
|
|
|
|
|
);
|
|
|
|
|
|
return [
|
|
|
|
|
|
$order,
|
|
|
|
|
|
CloudJob::where('order_id', (int) $order->id)->firstOrFail(),
|
|
|
|
|
|
CloudResourceReservation::where(
|
|
|
|
|
|
'order_id', (int) $order->id
|
|
|
|
|
|
)->firstOrFail(),
|
|
|
|
|
|
];
|
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
|
|
$connection->beginTransaction();
|
|
|
|
|
|
try {
|
|
|
|
|
|
[$snapshotPreSendOrder, $snapshotPreSendJob,
|
|
|
|
|
|
$snapshotPreSendReservation] = $queueSnapshotProvision(
|
|
|
|
|
|
'vm-snapshot-presend',
|
|
|
|
|
|
'snapshot-presend-request-0001',
|
|
|
|
|
|
'snapshot-presend-payment-0001'
|
|
|
|
|
|
);
|
|
|
|
|
|
$mock->append(new ConnectException(
|
|
|
|
|
|
'simulated snapshot local connection failure',
|
|
|
|
|
|
new Request(
|
|
|
|
|
|
'POST',
|
|
|
|
|
|
'http://127.0.0.1:8001/internal/agent/request'
|
|
|
|
|
|
),
|
|
|
|
|
|
null,
|
|
|
|
|
|
[
|
|
|
|
|
|
'errno' => 7,
|
|
|
|
|
|
'url' =>
|
|
|
|
|
|
'http://127.0.0.1:8001/internal/agent/request',
|
|
|
|
|
|
'request_size' => 0,
|
|
|
|
|
|
'size_upload' => 0.0,
|
|
|
|
|
|
'size_download' => 0.0,
|
|
|
|
|
|
'header_size' => 0,
|
|
|
|
|
|
'http_code' => 0,
|
|
|
|
|
|
'redirect_count' => 0,
|
|
|
|
|
|
'connect_time' => 0.0,
|
|
|
|
|
|
'pretransfer_time' => 0.0,
|
|
|
|
|
|
'starttransfer_time' => 0.0,
|
|
|
|
|
|
]
|
|
|
|
|
|
));
|
|
|
|
|
|
$snapshotPreSendHistory = count($history);
|
|
|
|
|
|
$assert($worker->tick(),
|
|
|
|
|
|
'Cloud-snapshot pre-send failure was not processed.');
|
|
|
|
|
|
$snapshotPreSendJob->refresh();
|
|
|
|
|
|
$snapshotPreSendOrder->refresh();
|
|
|
|
|
|
$snapshotPreSendReservation->refresh();
|
|
|
|
|
|
$assert((string) $snapshotPreSendJob->phase === 'terminal'
|
|
|
|
|
|
&& (string) $snapshotPreSendJob->outcome === 'failed'
|
|
|
|
|
|
&& (string) $snapshotPreSendJob->effect_resolution
|
|
|
|
|
|
=== 'not_applied'
|
|
|
|
|
|
&& !(bool) $snapshotPreSendJob->effect_may_have_applied
|
|
|
|
|
|
&& $snapshotPreSendOrder->refunded_at !== null
|
|
|
|
|
|
&& (string) $snapshotPreSendReservation->state === 'released'
|
|
|
|
|
|
&& $bundleState(
|
|
|
|
|
|
$snapshotPreSendReservation,
|
|
|
|
|
|
'released',
|
|
|
|
|
|
'available'
|
|
|
|
|
|
)
|
|
|
|
|
|
&& (string) User::findOrFail(1)->money === $snapshotBalance
|
|
|
|
|
|
&& (int) CloudOffer::findOrFail($offer->id)->stock
|
|
|
|
|
|
=== $snapshotStock,
|
|
|
|
|
|
'Cloud-snapshot pre-send failure retained money or either address family.');
|
|
|
|
|
|
$worker->tick();
|
|
|
|
|
|
$assert(count($history) === $snapshotPreSendHistory + 1,
|
|
|
|
|
|
'Cloud-snapshot pre-send failure replayed its mutation.');
|
|
|
|
|
|
} finally {
|
|
|
|
|
|
$connection->rollBack();
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
$connection->beginTransaction();
|
|
|
|
|
|
try {
|
|
|
|
|
|
[$snapshotUnknownOrder, $snapshotUnknownJob,
|
|
|
|
|
|
$snapshotUnknownReservation] = $queueSnapshotProvision(
|
|
|
|
|
|
'vm-snapshot-unknown',
|
|
|
|
|
|
'snapshot-unknown-request-0001',
|
|
|
|
|
|
'snapshot-unknown-payment-0001'
|
|
|
|
|
|
);
|
|
|
|
|
|
$mock->append(new ConnectException(
|
|
|
|
|
|
'simulated snapshot dispatched timeout',
|
|
|
|
|
|
new Request('POST', 'http://127.0.0.1:8000')
|
|
|
|
|
|
));
|
|
|
|
|
|
$snapshotUnknownHistory = count($history);
|
|
|
|
|
|
$assert($worker->tick(),
|
|
|
|
|
|
'Cloud-snapshot unknown-effect failure was not processed.');
|
|
|
|
|
|
$snapshotUnknownJob->refresh();
|
|
|
|
|
|
$snapshotUnknownOrder->refresh();
|
|
|
|
|
|
$snapshotUnknownReservation->refresh();
|
|
|
|
|
|
$assert((string) $snapshotUnknownJob->phase === 'terminal'
|
|
|
|
|
|
&& (string) $snapshotUnknownJob->outcome === 'failed'
|
|
|
|
|
|
&& (string) $snapshotUnknownJob->effect_resolution === 'unknown'
|
|
|
|
|
|
&& (bool) $snapshotUnknownJob->effect_may_have_applied
|
|
|
|
|
|
&& !(bool) $snapshotUnknownJob->safe_to_resubmit
|
|
|
|
|
|
&& $snapshotUnknownOrder->refunded_at === null
|
|
|
|
|
|
&& (string) $snapshotUnknownReservation->state === 'uncertain'
|
|
|
|
|
|
&& $bundleState(
|
|
|
|
|
|
$snapshotUnknownReservation,
|
|
|
|
|
|
'uncertain',
|
|
|
|
|
|
'uncertain'
|
|
|
|
|
|
)
|
|
|
|
|
|
&& (string) User::findOrFail(1)->money === '80.02'
|
|
|
|
|
|
&& (int) CloudOffer::findOrFail($offer->id)->stock === 8,
|
|
|
|
|
|
'Cloud-snapshot dispatched timeout refunded or released an unknown dual-stack effect.');
|
|
|
|
|
|
$worker->tick();
|
|
|
|
|
|
$assert(count($history) === $snapshotUnknownHistory + 1,
|
|
|
|
|
|
'Cloud-snapshot unknown effect replayed its mutation.');
|
|
|
|
|
|
} finally {
|
|
|
|
|
|
$connection->rollBack();
|
|
|
|
|
|
}
|
|
|
|
|
|
$assert(CloudOrder::count() === $snapshotOrderCount
|
|
|
|
|
|
&& CloudInvoice::count() === $snapshotInvoiceCount
|
|
|
|
|
|
&& CloudJob::count() === $snapshotJobCount
|
|
|
|
|
|
&& CloudResourceReservation::count() === $snapshotReservationCount
|
|
|
|
|
|
&& (string) User::findOrFail(1)->money === $snapshotBalance
|
|
|
|
|
|
&& (int) CloudOffer::findOrFail($offer->id)->stock === $snapshotStock,
|
|
|
|
|
|
'Cloud-snapshot failure fixtures leaked finance or placement state.');
|
|
|
|
|
|
|
|
|
|
|
|
// A failed local TCP connection is not an ambiguous Agent dispatch.
|
|
|
|
|
|
// Exercise the full worker/financial path, not only client classification.
|
|
|
|
|
|
foreach ([7 => 'refused', 28 => 'connect-timeout'] as $curlError => $failure) {
|
|
|
|
|
|
$gatewayFailure = CloudOrderService::purchase(array_replace($input, [
|
|
|
|
|
|
'name' => 'vm-gateway-' . $failure,
|
|
|
|
|
|
'idempotency_key' => 'gateway-before-send-' . $failure,
|
|
|
|
|
|
]), $actor);
|
|
|
|
|
|
CloudInvoiceService::payBalance(
|
|
|
|
|
|
CloudInvoice::where('order_id', $gatewayFailure->id)->first(),
|
|
|
|
|
|
$actor, 'gateway-balance-' . $failure
|
|
|
|
|
|
);
|
|
|
|
|
|
$mock->append(new ConnectException('simulated local connection failure',
|
|
|
|
|
|
new Request('POST', 'http://127.0.0.1:8001/internal/agent/request'),
|
|
|
|
|
|
null, [
|
|
|
|
|
|
'errno' => $curlError,
|
|
|
|
|
|
'url' => 'http://127.0.0.1:8001/internal/agent/request',
|
|
|
|
|
|
'request_size' => 0, 'size_upload' => 0.0,
|
|
|
|
|
|
'size_download' => 0.0, 'header_size' => 0,
|
|
|
|
|
|
'http_code' => 0, 'redirect_count' => 0,
|
|
|
|
|
|
'connect_time' => 0.0, 'pretransfer_time' => 0.0,
|
|
|
|
|
|
'starttransfer_time' => 0.0,
|
|
|
|
|
|
]));
|
|
|
|
|
|
$beforeGatewayFailure = count($history);
|
|
|
|
|
|
$assert($worker->tick(), 'Gateway failure task was not processed.');
|
|
|
|
|
|
$gatewayFailureJob = CloudJob::where('order_id', $gatewayFailure->id)->first();
|
|
|
|
|
|
$gatewayFailureReservation = CloudResourceReservation::where(
|
|
|
|
|
|
'order_id', $gatewayFailure->id
|
|
|
|
|
|
)->first();
|
|
|
|
|
|
$assert($gatewayFailureJob->phase === 'terminal'
|
|
|
|
|
|
&& $gatewayFailureJob->outcome === 'failed'
|
|
|
|
|
|
&& $gatewayFailureJob->effect_resolution === 'not_applied'
|
|
|
|
|
|
&& !(bool) $gatewayFailureJob->effect_may_have_applied
|
|
|
|
|
|
&& !(bool) $gatewayFailureJob->finalization_incident
|
|
|
|
|
|
&& CloudOrder::find($gatewayFailure->id)->refunded_at !== null
|
|
|
|
|
|
&& $gatewayFailureReservation->state === 'released'
|
|
|
|
|
|
&& $bundleState(
|
|
|
|
|
|
$gatewayFailureReservation,
|
|
|
|
|
|
'released',
|
|
|
|
|
|
'available'
|
|
|
|
|
|
)
|
|
|
|
|
|
&& User::find(1)->money === '90.01'
|
|
|
|
|
|
&& CloudOffer::find(1)->stock === 9,
|
|
|
|
|
|
'Proven pre-send Gateway failure retained unknown effects, funds or placement.');
|
|
|
|
|
|
$worker->tick();
|
|
|
|
|
|
$assert(count($history) === $beforeGatewayFailure + 1
|
|
|
|
|
|
&& User::find(1)->money === '90.01'
|
|
|
|
|
|
&& CloudOffer::find(1)->stock === 9,
|
|
|
|
|
|
'Pre-send Gateway failure replayed its mutation or refunded twice.');
|
|
|
|
|
|
}
|
|
|
|
|
|
$uncertain = CloudOrderService::purchase(array_replace($input, ['name' => 'vm-three', 'idempotency_key' => 'test-request-key-0003']), $actor);
|
|
|
|
|
|
$assert(User::find(1)->money === '90.01'
|
|
|
|
|
|
&& CloudJob::where('order_id', $uncertain->id)->count() === 0,
|
|
|
|
|
|
'An unpaid ambiguous-case deployment charged or entered the worker queue.');
|
|
|
|
|
|
CloudInvoiceService::payBalance(CloudInvoice::where('order_id', $uncertain->id)->first(),
|
|
|
|
|
|
$actor, 'provision-balance-test-0003');
|
|
|
|
|
|
$mock->append(new ConnectException('simulated timeout', new Request('POST', 'http://127.0.0.1:8000')));
|
|
|
|
|
|
$worker->tick();
|
|
|
|
|
|
$uncertainJob = CloudJob::where('order_id', $uncertain->id)->first();
|
|
|
|
|
|
$uncertainInvoiceDisplay = CloudInvoiceService::display(
|
|
|
|
|
|
CloudInvoice::where('order_id', $uncertain->id)->first());
|
|
|
|
|
|
$uncertainReservation = CloudResourceReservation::where(
|
|
|
|
|
|
'order_id', $uncertain->id
|
|
|
|
|
|
)->first();
|
|
|
|
|
|
$assert($uncertainJob->phase === 'terminal'
|
|
|
|
|
|
&& $uncertainJob->outcome === 'failed'
|
|
|
|
|
|
&& $uncertainJob->status === 'failed'
|
|
|
|
|
|
&& (int) $uncertainJob->dispatch_attempts === 1
|
|
|
|
|
|
&& (bool) $uncertainJob->effect_may_have_applied === true
|
|
|
|
|
|
&& (bool) $uncertainJob->safe_to_resubmit === false
|
|
|
|
|
|
&& $uncertainJob->effect_resolution === 'unknown'
|
|
|
|
|
|
&& CloudOrder::find($uncertain->id)->status === 'failed'
|
|
|
|
|
|
&& CloudOrder::find($uncertain->id)->refunded_at === null
|
|
|
|
|
|
&& $uncertainReservation->state === 'uncertain'
|
|
|
|
|
|
&& $bundleState($uncertainReservation, 'uncertain', 'uncertain')
|
|
|
|
|
|
&& User::find(1)->money === '80.02'
|
|
|
|
|
|
&& ($uncertainInvoiceDisplay['order_status'] ?? null) === 'failed'
|
|
|
|
|
|
&& CloudMailOutbox::where('kind', 'provision_succeeded')->count() === 1,
|
|
|
|
|
|
'A dispatched create timeout did not become a terminal unknown-effect failure.');
|
|
|
|
|
|
$beforeFailedCreateRetry = count($history);
|
|
|
|
|
|
$worker->tick();
|
|
|
|
|
|
$uncertainJob->refresh();
|
|
|
|
|
|
$assert(count($history) === $beforeFailedCreateRetry
|
|
|
|
|
|
&& $uncertainJob->phase === 'terminal'
|
|
|
|
|
|
&& $uncertainJob->outcome === 'failed'
|
|
|
|
|
|
&& (int) $uncertainJob->dispatch_attempts === 1
|
|
|
|
|
|
&& CloudInstance::count() === 1
|
|
|
|
|
|
&& CloudResourceReservation::find($uncertainReservation->id)->state === 'uncertain'
|
|
|
|
|
|
&& User::find(1)->money === '80.02'
|
|
|
|
|
|
&& CloudMailOutbox::where('kind', 'provision_succeeded')->count() === 1,
|
|
|
|
|
|
'A terminal create failure was automatically polled, reconciled, or mutation-retried.');
|
|
|
|
|
|
|
|
|
|
|
|
// Simulate the crash window where the Agent already committed its
|
|
|
|
|
|
// not-applied fence receipt but PHP died before settling local ledgers. A
|
|
|
|
|
|
// later fresh inspection may consume the older matching receipt, while
|
|
|
|
|
|
// the immutable public task remains terminal failed.
|
|
|
|
|
|
$resolutionReason = '已通过新的只读检查确认虚拟机及面板管理资源均不存在,执行原子退款。';
|
|
|
|
|
|
$inspectionNow = time();
|
|
|
|
|
|
$freshInspection = [
|
|
|
|
|
|
'operation_id' => (string) $uncertainJob->operation_id,
|
|
|
|
|
|
'checked_at' => $inspectionNow,
|
|
|
|
|
|
'outcome' => 'failed',
|
|
|
|
|
|
'summary_code' => 'operation_failed',
|
|
|
|
|
|
'error_code' => 'creation_absent',
|
|
|
|
|
|
];
|
|
|
|
|
|
$persistedFenceReceipt = [
|
|
|
|
|
|
'operation_id' => (string) $uncertainJob->operation_id,
|
|
|
|
|
|
'resolution_id' => str_repeat('a', 32),
|
|
|
|
|
|
'resolution' => 'not_applied',
|
|
|
|
|
|
'inspection_checked_at' => $inspectionNow - 60,
|
|
|
|
|
|
'reason_sha256' => hash('sha256', $resolutionReason),
|
|
|
|
|
|
'resolved_at' => $inspectionNow - 50,
|
|
|
|
|
|
'resolved_by' => 7,
|
|
|
|
|
|
'replayed' => false,
|
|
|
|
|
|
];
|
|
|
|
|
|
$resolvedCreate = CloudAdminOperationsService::resolveEffect([
|
|
|
|
|
|
'user_id' => 99, 'is_admin' => true,
|
|
|
|
|
|
'is_super_admin' => true, 'can_manage' => true,
|
|
|
|
|
|
], (string) $uncertainJob->id, [
|
|
|
|
|
|
'resolution' => 'not_applied', 'reason' => $resolutionReason,
|
|
|
|
|
|
], $freshInspection, $persistedFenceReceipt);
|
|
|
|
|
|
$uncertainJob->refresh();
|
|
|
|
|
|
$assert($uncertainJob->phase === 'terminal'
|
|
|
|
|
|
&& $uncertainJob->outcome === 'failed'
|
|
|
|
|
|
&& $uncertainJob->status === 'failed'
|
|
|
|
|
|
&& $uncertainJob->effect_resolution === 'not_applied'
|
|
|
|
|
|
&& (int) $uncertainJob->effect_resolved_by === 99
|
|
|
|
|
|
&& ($resolvedCreate['job']['completed'] ?? false) === true
|
|
|
|
|
|
&& ($resolvedCreate['job']['status'] ?? null) === 'failed'
|
|
|
|
|
|
&& ($resolvedCreate['job']['can_resolve_effect'] ?? true) === false
|
|
|
|
|
|
&& CloudOrder::find($uncertain->id)->refunded_at !== null
|
|
|
|
|
|
&& CloudInvoice::where('order_id', $uncertain->id)
|
|
|
|
|
|
->value('status') === 'refunded'
|
|
|
|
|
|
&& CloudResourceReservation::find($uncertainReservation->id)
|
|
|
|
|
|
->state === 'released'
|
|
|
|
|
|
&& $bundleState(
|
|
|
|
|
|
CloudResourceReservation::find($uncertainReservation->id),
|
|
|
|
|
|
'released',
|
|
|
|
|
|
'available'
|
|
|
|
|
|
)
|
|
|
|
|
|
&& User::find(1)->money === '90.01'
|
|
|
|
|
|
&& CloudOffer::find(1)->stock === 9,
|
|
|
|
|
|
'A prior Agent fence commit was not recovered into one atomic local create settlement.');
|
|
|
|
|
|
|
|
|
|
|
|
// A completion may be durably accepted before PHP finishes the create
|
|
|
|
|
|
// ledgers. Keep the encrypted request escrow while the effect is unknown,
|
|
|
|
|
|
// then let the exact same authenticated receipt atomically materialize the
|
|
|
|
|
|
// instance and ACK its event after the local ledger fault is repaired.
|
|
|
|
|
|
$finiteStockBeforeUnlimitedCreate = (int) CloudOffer::findOrFail(1)->stock;
|
|
|
|
|
|
$unlimitedOffer = CloudOffer::findOrFail(1);
|
|
|
|
|
|
$offerCreatedAtBeforeUnlimitedCreate = (int) $unlimitedOffer->created_at;
|
|
|
|
|
|
$unlimitedOffer->stock = -1;
|
|
|
|
|
|
$unlimitedOffer->created_at = time() - 20;
|
|
|
|
|
|
$unlimitedOffer->updated_at = time() - 10;
|
|
|
|
|
|
$unlimitedOffer->save();
|
|
|
|
|
|
$lateCreate = CloudOrderService::purchase(array_replace($input, [
|
|
|
|
|
|
'name' => 'vm-late-create',
|
|
|
|
|
|
'idempotency_key' => 'test-request-key-late-create',
|
|
|
|
|
|
]), $actor);
|
|
|
|
|
|
$lateCreateSpec = json_decode(
|
|
|
|
|
|
(string) $lateCreate->spec_json,
|
|
|
|
|
|
true,
|
|
|
|
|
|
64,
|
|
|
|
|
|
JSON_THROW_ON_ERROR
|
|
|
|
|
|
);
|
|
|
|
|
|
$assert(($lateCreateSpec['stock_mode'] ?? null) === 'unlimited'
|
|
|
|
|
|
&& $lateCreate->stock_reserved_at === null
|
|
|
|
|
|
&& $lateCreate->stock_released_at === null
|
|
|
|
|
|
&& (int) CloudOffer::findOrFail(1)->stock === -1,
|
|
|
|
|
|
'An unlimited-stock purchase consumed or fabricated finite stock.');
|
|
|
|
|
|
$lateInvoice = CloudInvoice::where('order_id', $lateCreate->id)->first();
|
|
|
|
|
|
CloudInvoiceService::payBalance(
|
|
|
|
|
|
$lateInvoice,
|
|
|
|
|
|
$actor,
|
|
|
|
|
|
'provision-balance-late-create'
|
|
|
|
|
|
);
|
|
|
|
|
|
$lateCreateJob = CloudJob::where('order_id', $lateCreate->id)->first();
|
|
|
|
|
|
$lateCreateIdentity = CloudOperationRecoveryIdentity::expected(
|
|
|
|
|
|
$lateCreateJob,
|
|
|
|
|
|
true
|
|
|
|
|
|
);
|
|
|
|
|
|
$lateCreatePayload = \App\Services\Cloud\CloudSecrets::decrypt(
|
|
|
|
|
|
(string) $lateCreateJob->sealed_payload,
|
|
|
|
|
|
'job-payload'
|
|
|
|
|
|
);
|
|
|
|
|
|
$mock->append(new Response(202, [], json_encode([
|
|
|
|
|
|
'operation_id' => $lateCreateJob->operation_id,
|
|
|
|
|
|
'status' => 'reserved',
|
|
|
|
|
|
...$lateCreateIdentity,
|
|
|
|
|
|
'replayed' => false,
|
|
|
|
|
|
'retryable' => false,
|
|
|
|
|
|
])));
|
|
|
|
|
|
$worker->tick();
|
|
|
|
|
|
$lateCreateJob->refresh();
|
|
|
|
|
|
$assert((string) $lateCreateJob->phase === 'awaiting_receipt',
|
|
|
|
|
|
'Late-create fixture did not cross the Agent dispatch boundary.');
|
|
|
|
|
|
|
|
|
|
|
|
$lateOrder = CloudOrder::find($lateCreate->id);
|
|
|
|
|
|
$lateOrderSpec = json_decode(
|
|
|
|
|
|
(string) $lateOrder->spec_json,
|
|
|
|
|
|
true,
|
|
|
|
|
|
64,
|
|
|
|
|
|
JSON_THROW_ON_ERROR
|
|
|
|
|
|
);
|
|
|
|
|
|
$damagedLateOrderSpec = $lateOrderSpec;
|
|
|
|
|
|
unset($damagedLateOrderSpec['validity_days']);
|
|
|
|
|
|
$lateOrder->spec_json = json_encode(
|
|
|
|
|
|
$damagedLateOrderSpec,
|
|
|
|
|
|
JSON_THROW_ON_ERROR
|
|
|
|
|
|
);
|
|
|
|
|
|
$lateOrder->save();
|
|
|
|
|
|
$lateReceiptId = 'create-late-success-event-0001';
|
|
|
|
|
|
$lateCreatedVm = array_replace($createdVm, [
|
|
|
|
|
|
'uuid' => $lateCreateIdentity['resource_id'],
|
|
|
|
|
|
'name' => '1:vm-late-create',
|
|
|
|
|
|
'public_network' => $lateCreatePayload['body']['public_network'],
|
|
|
|
|
|
]);
|
|
|
|
|
|
unset($lateCreatedVm['generated_password']);
|
|
|
|
|
|
$lateReceipt = [
|
|
|
|
|
|
'type' => 'completion',
|
|
|
|
|
|
'id' => $lateReceiptId,
|
|
|
|
|
|
'agent_id' => (string) $lateCreateJob->host_id,
|
|
|
|
|
|
'operation_id' => (string) $lateCreateJob->operation_id,
|
|
|
|
|
|
...$lateCreateIdentity,
|
|
|
|
|
|
'status' => 'succeeded',
|
|
|
|
|
|
'effect_may_have_applied' => false,
|
|
|
|
|
|
'safe_to_resubmit' => false,
|
|
|
|
|
|
'error_code' => '',
|
|
|
|
|
|
'result' => $lateCreatedVm,
|
|
|
|
|
|
];
|
|
|
|
|
|
$firstLateReceipt = CloudOperationCompletionService::accept($lateReceipt);
|
|
|
|
|
|
$lateCreateJob->refresh();
|
|
|
|
|
|
$lateEscrow = \App\Services\Cloud\CloudSecrets::decrypt(
|
|
|
|
|
|
(string) $lateCreateJob->sealed_payload,
|
|
|
|
|
|
'job-payload'
|
|
|
|
|
|
);
|
|
|
|
|
|
$lateEvent = \App\Models\CloudAgentCompletionEvent::where(
|
|
|
|
|
|
'event_id',
|
|
|
|
|
|
$lateReceiptId
|
|
|
|
|
|
)->first();
|
|
|
|
|
|
$assert($firstLateReceipt === [
|
|
|
|
|
|
'accepted' => true,
|
|
|
|
|
|
'disposition' => 'received',
|
|
|
|
|
|
]
|
|
|
|
|
|
&& (string) $lateCreateJob->phase === 'terminal'
|
|
|
|
|
|
&& (string) $lateCreateJob->outcome === 'failed'
|
|
|
|
|
|
&& (string) $lateCreateJob->effect_resolution === 'unknown'
|
|
|
|
|
|
&& ($lateEscrow['credential']['password'] ?? null)
|
|
|
|
|
|
=== ($lateEscrow['body']['root_password'] ?? null)
|
|
|
|
|
|
&& (string) $lateEvent->disposition === 'received'
|
|
|
|
|
|
&& CloudInstance::where('order_id', $lateCreate->id)->count() === 0,
|
|
|
|
|
|
'A failed local create finalizer lost its exact receipt or encrypted recovery escrow.');
|
|
|
|
|
|
|
|
|
|
|
|
$lateWalletBeforeManualResolution = (string) User::find(1)->money;
|
|
|
|
|
|
$lateReservationBeforeManualResolution = CloudResourceReservation::where(
|
|
|
|
|
|
'order_id',
|
|
|
|
|
|
$lateCreate->id
|
|
|
|
|
|
)->first();
|
|
|
|
|
|
try {
|
|
|
|
|
|
\App\Services\Cloud\CloudCreateFailureFinalizer::resolveNotApplied(
|
|
|
|
|
|
(int) $lateCreateJob->id,
|
|
|
|
|
|
(string) $lateCreateJob->operation_id,
|
|
|
|
|
|
99,
|
|
|
|
|
|
'contradictory-manual-absence',
|
|
|
|
|
|
[
|
|
|
|
|
|
'operation_id' => (string) $lateCreateJob->operation_id,
|
|
|
|
|
|
'outcome' => 'failed',
|
|
|
|
|
|
'summary_code' => 'operation_failed',
|
|
|
|
|
|
'error_code' => 'creation_absent',
|
|
|
|
|
|
],
|
|
|
|
|
|
time()
|
|
|
|
|
|
);
|
|
|
|
|
|
throw new RuntimeException(
|
|
|
|
|
|
'Manual no-effect resolution ignored a durable create success.'
|
|
|
|
|
|
);
|
|
|
|
|
|
} catch (CloudError $error) {
|
|
|
|
|
|
$assert($error->httpStatus === 409,
|
|
|
|
|
|
'Create success/manual absence conflict returned wrong status.');
|
|
|
|
|
|
}
|
|
|
|
|
|
$assert((string) User::find(1)->money
|
|
|
|
|
|
=== $lateWalletBeforeManualResolution
|
|
|
|
|
|
&& (string) CloudOrder::find($lateCreate->id)->status === 'failed'
|
|
|
|
|
|
&& (string) CloudInvoice::where('order_id', $lateCreate->id)
|
|
|
|
|
|
->value('status') === 'paid'
|
|
|
|
|
|
&& (string) CloudResourceReservation::find(
|
|
|
|
|
|
$lateReservationBeforeManualResolution->id
|
|
|
|
|
|
)->state === (string) $lateReservationBeforeManualResolution->state,
|
|
|
|
|
|
'Conflicting manual absence resolution refunded or released a create.');
|
|
|
|
|
|
|
|
|
|
|
|
$legacyLateOrderSpec = $lateOrderSpec;
|
|
|
|
|
|
unset($legacyLateOrderSpec['stock_mode']);
|
|
|
|
|
|
$lateOrder->spec_json = json_encode(
|
|
|
|
|
|
$legacyLateOrderSpec,
|
|
|
|
|
|
JSON_THROW_ON_ERROR
|
|
|
|
|
|
);
|
|
|
|
|
|
$lateOrder->save();
|
|
|
|
|
|
$makeLateCreateRetryable = static function () use (
|
|
|
|
|
|
$lateCreateJob
|
|
|
|
|
|
): void {
|
|
|
|
|
|
Capsule::table('cloud_jobs')->where(
|
|
|
|
|
|
'id',
|
|
|
|
|
|
(int) $lateCreateJob->id
|
|
|
|
|
|
)->update([
|
|
|
|
|
|
'available_at' => 0,
|
|
|
|
|
|
'lease_token' => null,
|
|
|
|
|
|
'lease_until' => 0,
|
|
|
|
|
|
]);
|
|
|
|
|
|
};
|
|
|
|
|
|
$assertLateCreateStillPending = static function (
|
|
|
|
|
|
string $message
|
|
|
|
|
|
) use ($assert, $lateCreate, $lateReceiptId): void {
|
|
|
|
|
|
$assert((string) \App\Models\CloudAgentCompletionEvent::where(
|
|
|
|
|
|
'event_id',
|
|
|
|
|
|
$lateReceiptId
|
|
|
|
|
|
)->value('disposition') === 'received'
|
|
|
|
|
|
&& CloudInstance::where('order_id', $lateCreate->id)->count() === 0,
|
|
|
|
|
|
$message);
|
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
|
|
$finiteMarkerlessSpec = $legacyLateOrderSpec;
|
|
|
|
|
|
$finiteMarkerlessSpec['stock_mode'] = 'finite';
|
|
|
|
|
|
$lateOrder->spec_json = json_encode(
|
|
|
|
|
|
$finiteMarkerlessSpec,
|
|
|
|
|
|
JSON_THROW_ON_ERROR
|
|
|
|
|
|
);
|
|
|
|
|
|
$lateOrder->save();
|
|
|
|
|
|
$finiteGuardWallet = (string) User::findOrFail(1)->money;
|
|
|
|
|
|
$finiteGuardReservation = CloudResourceReservation::where(
|
|
|
|
|
|
'order_id',
|
|
|
|
|
|
$lateCreate->id
|
|
|
|
|
|
)->firstOrFail();
|
|
|
|
|
|
$makeLateCreateRetryable();
|
|
|
|
|
|
$assert((new CloudAutomaticOperationRecoveryService())->tick(),
|
|
|
|
|
|
'Markerless finite-stock create was not retried safely.');
|
|
|
|
|
|
$assertLateCreateStillPending(
|
|
|
|
|
|
'A markerless finite-stock order allowed late create materialisation or ACK.'
|
|
|
|
|
|
);
|
|
|
|
|
|
$assert(CloudOrder::findOrFail($lateCreate->id)->refunded_at === null
|
|
|
|
|
|
&& (string) CloudInvoice::findOrFail($lateInvoice->id)->status === 'paid'
|
|
|
|
|
|
&& (string) CloudResourceReservation::findOrFail(
|
|
|
|
|
|
$finiteGuardReservation->id
|
|
|
|
|
|
)->state === (string) $finiteGuardReservation->state
|
|
|
|
|
|
&& $bundleState(
|
|
|
|
|
|
CloudResourceReservation::findOrFail(
|
|
|
|
|
|
$finiteGuardReservation->id
|
|
|
|
|
|
),
|
|
|
|
|
|
'uncertain',
|
|
|
|
|
|
'uncertain'
|
|
|
|
|
|
)
|
|
|
|
|
|
&& (string) User::findOrFail(1)->money === $finiteGuardWallet
|
|
|
|
|
|
&& (int) CloudOffer::findOrFail(1)->stock === -1,
|
|
|
|
|
|
'A markerless finite-stock order released or refunded fenced ledgers.');
|
|
|
|
|
|
|
|
|
|
|
|
$unknownStockModeSpec = $legacyLateOrderSpec;
|
|
|
|
|
|
$unknownStockModeSpec['stock_mode'] = 'unknown';
|
|
|
|
|
|
$lateOrder->spec_json = json_encode(
|
|
|
|
|
|
$unknownStockModeSpec,
|
|
|
|
|
|
JSON_THROW_ON_ERROR
|
|
|
|
|
|
);
|
|
|
|
|
|
$lateOrder->save();
|
|
|
|
|
|
$makeLateCreateRetryable();
|
|
|
|
|
|
$assert((new CloudAutomaticOperationRecoveryService())->tick(),
|
|
|
|
|
|
'Unknown stock-mode create was not retried safely.');
|
|
|
|
|
|
$assertLateCreateStillPending(
|
|
|
|
|
|
'An unknown stock mode allowed late create materialisation or ACK.'
|
|
|
|
|
|
);
|
|
|
|
|
|
|
|
|
|
|
|
$nullStockModeSpec = $legacyLateOrderSpec;
|
|
|
|
|
|
$nullStockModeSpec['stock_mode'] = null;
|
|
|
|
|
|
$lateOrder->spec_json = json_encode(
|
|
|
|
|
|
$nullStockModeSpec,
|
|
|
|
|
|
JSON_THROW_ON_ERROR
|
|
|
|
|
|
);
|
|
|
|
|
|
$lateOrder->save();
|
|
|
|
|
|
$makeLateCreateRetryable();
|
|
|
|
|
|
$assert((new CloudAutomaticOperationRecoveryService())->tick(),
|
|
|
|
|
|
'Null stock-mode create was not retried safely.');
|
|
|
|
|
|
$assertLateCreateStillPending(
|
|
|
|
|
|
'A null stock mode allowed late create materialisation or ACK.'
|
|
|
|
|
|
);
|
|
|
|
|
|
|
|
|
|
|
|
$lateOrder->spec_json = json_encode(
|
|
|
|
|
|
$legacyLateOrderSpec,
|
|
|
|
|
|
JSON_THROW_ON_ERROR
|
|
|
|
|
|
);
|
|
|
|
|
|
$lateOrder->save();
|
|
|
|
|
|
$unlimitedOffer->updated_at = (int) $lateOrder->created_at;
|
|
|
|
|
|
$unlimitedOffer->save();
|
|
|
|
|
|
$makeLateCreateRetryable();
|
|
|
|
|
|
$assert((new CloudAutomaticOperationRecoveryService())->tick(),
|
|
|
|
|
|
'Ambiguous legacy unlimited-stock create was not retried safely.');
|
|
|
|
|
|
$assertLateCreateStillPending(
|
|
|
|
|
|
'A changed legacy offer allowed late create materialisation or ACK.'
|
|
|
|
|
|
);
|
|
|
|
|
|
$unlimitedOffer->updated_at = (int) $lateOrder->created_at - 1;
|
|
|
|
|
|
$unlimitedOffer->save();
|
|
|
|
|
|
|
|
|
|
|
|
$lateInvoice->purpose = 'renewal';
|
|
|
|
|
|
$lateInvoice->save();
|
|
|
|
|
|
$makeLateCreateRetryable();
|
|
|
|
|
|
$assert((new CloudAutomaticOperationRecoveryService())->tick(),
|
|
|
|
|
|
'Wrong-purpose create invoice was not retried safely.');
|
|
|
|
|
|
$assertLateCreateStillPending(
|
|
|
|
|
|
'A wrong-purpose invoice allowed late create materialisation or ACK.'
|
|
|
|
|
|
);
|
|
|
|
|
|
|
|
|
|
|
|
$lateInvoice->purpose = 'provision';
|
|
|
|
|
|
$lateInvoice->status = 'refunded';
|
|
|
|
|
|
$lateInvoice->save();
|
|
|
|
|
|
$makeLateCreateRetryable();
|
|
|
|
|
|
$assert((new CloudAutomaticOperationRecoveryService())->tick(),
|
|
|
|
|
|
'Refunded create invoice was not retried safely.');
|
|
|
|
|
|
$assertLateCreateStillPending(
|
|
|
|
|
|
'A refunded invoice allowed late create materialisation or ACK.'
|
|
|
|
|
|
);
|
|
|
|
|
|
|
|
|
|
|
|
$lateInvoice->status = 'paid';
|
|
|
|
|
|
$lateInvoice->save();
|
|
|
|
|
|
$lateInvoiceRow = (array) Capsule::table('cloud_invoices')
|
|
|
|
|
|
->where('id', (int) $lateInvoice->id)->first();
|
|
|
|
|
|
Capsule::table('cloud_invoices')->where(
|
|
|
|
|
|
'id',
|
|
|
|
|
|
(int) $lateInvoice->id
|
|
|
|
|
|
)->delete();
|
|
|
|
|
|
$makeLateCreateRetryable();
|
|
|
|
|
|
$assert((new CloudAutomaticOperationRecoveryService())->tick(),
|
|
|
|
|
|
'Missing create invoice was not retried safely.');
|
|
|
|
|
|
$assertLateCreateStillPending(
|
|
|
|
|
|
'A missing invoice allowed late create materialisation or ACK.'
|
|
|
|
|
|
);
|
|
|
|
|
|
Capsule::table('cloud_invoices')->insert($lateInvoiceRow);
|
|
|
|
|
|
$lateInvoice = CloudInvoice::findOrFail((int) $lateInvoice->id);
|
|
|
|
|
|
$makeLateCreateRetryable();
|
|
|
|
|
|
$lateRecoveryTick = (new CloudAutomaticOperationRecoveryService())->tick();
|
|
|
|
|
|
// The Agent may already have stopped retransmitting after the panel
|
|
|
|
|
|
// durably accepted the event. The recovery worker must consume that same
|
|
|
|
|
|
// stored receipt; a later duplicate is only an idempotent ACK check.
|
|
|
|
|
|
$secondLateReceipt = CloudOperationCompletionService::accept($lateReceipt);
|
|
|
|
|
|
$lateCreateJob->refresh();
|
|
|
|
|
|
$lateOrder->refresh();
|
|
|
|
|
|
$lateInvoice->refresh();
|
|
|
|
|
|
$lateInstance = CloudInstance::where('order_id', $lateCreate->id)->first();
|
|
|
|
|
|
$lateReservation = CloudResourceReservation::where(
|
|
|
|
|
|
'order_id',
|
|
|
|
|
|
$lateCreate->id
|
|
|
|
|
|
)->first();
|
|
|
|
|
|
$latePeriod = \App\Models\CloudServicePeriod::where(
|
|
|
|
|
|
'order_id',
|
|
|
|
|
|
$lateCreate->id
|
|
|
|
|
|
)->first();
|
|
|
|
|
|
$lateCredential = $lateInstance === null ? null
|
|
|
|
|
|
: \App\Services\Cloud\CloudSecrets::decrypt(
|
|
|
|
|
|
(string) $lateInstance->sealed_credential,
|
|
|
|
|
|
'instance-credential'
|
|
|
|
|
|
);
|
|
|
|
|
|
$lateEvent->refresh();
|
|
|
|
|
|
$lateFinalPayload = \App\Services\Cloud\CloudSecrets::decrypt(
|
|
|
|
|
|
(string) $lateCreateJob->sealed_payload,
|
|
|
|
|
|
'job-payload'
|
|
|
|
|
|
);
|
|
|
|
|
|
$lateInvoiceDisplay = CloudInvoiceService::display($lateInvoice);
|
|
|
|
|
|
$lateTaskDisplay = CloudJobService::display($lateCreateJob);
|
|
|
|
|
|
$assert($lateRecoveryTick
|
|
|
|
|
|
&& $secondLateReceipt === [
|
|
|
|
|
|
'accepted' => true,
|
|
|
|
|
|
'disposition' => 'applied',
|
|
|
|
|
|
]
|
|
|
|
|
|
&& $lateInstance !== null
|
|
|
|
|
|
&& (string) $lateCreateJob->phase === 'terminal'
|
|
|
|
|
|
&& (string) $lateCreateJob->outcome === 'failed'
|
|
|
|
|
|
&& (string) $lateCreateJob->status === 'failed'
|
|
|
|
|
|
&& (string) $lateCreateJob->effect_resolution === 'applied'
|
|
|
|
|
|
&& (string) $lateCreateJob->effect_resolution_reason
|
|
|
|
|
|
=== 'late_completion_receipt:' . $lateReceiptId
|
|
|
|
|
|
&& (string) $lateEvent->disposition === 'applied'
|
|
|
|
|
|
&& $lateEvent->applied_at !== null
|
|
|
|
|
|
&& (string) $lateOrder->status === 'succeeded'
|
|
|
|
|
|
&& (int) $lateOrder->instance_id === (int) $lateInstance->id
|
|
|
|
|
|
&& (int) $lateInvoice->instance_id === (int) $lateInstance->id
|
|
|
|
|
|
&& (string) $lateReservation->state === 'committed'
|
|
|
|
|
|
&& $latePeriod !== null
|
|
|
|
|
|
&& (string) $latePeriod->status === 'active'
|
|
|
|
|
|
&& (int) $latePeriod->instance_id === (int) $lateInstance->id
|
|
|
|
|
|
&& (int) $lateInstance->current_period_id === (int) $latePeriod->id
|
|
|
|
|
|
&& $lateCredential === $lateCreatePayload['credential']
|
|
|
|
|
|
&& !array_key_exists('credential', $lateFinalPayload)
|
|
|
|
|
|
&& !array_key_exists('root_password', $lateFinalPayload['body'] ?? [])
|
|
|
|
|
|
&& ($lateInvoiceDisplay['job_status'] ?? null) === 'succeeded'
|
|
|
|
|
|
&& ($lateInvoiceDisplay['fulfillment_status'] ?? null) === 'succeeded'
|
|
|
|
|
|
&& ($lateTaskDisplay['status'] ?? null) === 'failed'
|
|
|
|
|
|
&& ($lateTaskDisplay['effect_resolution'] ?? null) === 'applied',
|
|
|
|
|
|
'The exact late create receipt did not atomically converge PHP ledgers and public fulfillment state.');
|
|
|
|
|
|
$assert(CloudInvoiceService::deliverReminders(20) >= 1
|
|
|
|
|
|
&& (string) CloudMailOutbox::where(
|
|
|
|
|
|
'invoice_id',
|
|
|
|
|
|
$lateInvoice->id
|
|
|
|
|
|
)->where('kind', 'provision_succeeded')->value('status') === 'sent',
|
|
|
|
|
|
'Recovered create success was cancelled instead of delivering its provision notice.');
|
|
|
|
|
|
$assert($lateOrder->stock_reserved_at === null
|
|
|
|
|
|
&& $lateOrder->stock_released_at === null
|
|
|
|
|
|
&& (int) CloudOffer::findOrFail(1)->stock === -1,
|
|
|
|
|
|
'Unlimited-stock create recovery mutated finite inventory.');
|
|
|
|
|
|
$unlimitedOffer->stock = $finiteStockBeforeUnlimitedCreate;
|
|
|
|
|
|
$unlimitedOffer->created_at = $offerCreatedAtBeforeUnlimitedCreate;
|
|
|
|
|
|
$unlimitedOffer->updated_at = time();
|
|
|
|
|
|
$unlimitedOffer->save();
|
|
|
|
|
|
|
|
|
|
|
|
$actionJob = CloudJobService::enqueue($instance, $actor, 'action', 'POST', '/actions', ['action' => 'reboot']);
|
|
|
|
|
|
$mock->append(new Response(200, [], json_encode([
|
|
|
|
|
|
'operation_id' => $actionJob->operation_id,
|
|
|
|
|
|
'status' => 'succeeded',
|
|
|
|
|
|
...CloudOperationRecoveryIdentity::expected($actionJob, true),
|
|
|
|
|
|
'result' => [
|
|
|
|
|
|
'uuid' => (string) $instance->uuid,
|
|
|
|
|
|
'state' => 'running',
|
|
|
|
|
|
],
|
|
|
|
|
|
]))); $worker->tick();
|
|
|
|
|
|
$last = $agentRequest(end($history));
|
|
|
|
|
|
$signedActor = json_decode(base64_decode(strtr(
|
|
|
|
|
|
$last['headers']['X-Virt-Actor'], '-_', '+/'
|
|
|
|
|
|
)), true);
|
|
|
|
|
|
$assert(CloudJob::find($actionJob->id)->status === 'succeeded'
|
|
|
|
|
|
&& $signedActor['can_manage'] === true
|
|
|
|
|
|
&& ($signedActor['operation_id'] ?? null) === $actionJob->operation_id,
|
|
|
|
|
|
'Power task lost its typed Agent authority or did not complete.');
|
|
|
|
|
|
|
|
|
|
|
|
// Keep the panel/audit vocabulary stable while translating the force-off
|
|
|
|
|
|
// command to the Go Agent's strict PowerAction wire value.
|
|
|
|
|
|
$poweroffJob = CloudJobService::enqueue(
|
|
|
|
|
|
$instance, $actor, 'action', 'POST', '/actions', ['action' => 'poweroff']
|
|
|
|
|
|
);
|
|
|
|
|
|
$storedPoweroff = \App\Services\Cloud\CloudSecrets::decrypt(
|
|
|
|
|
|
(string) $poweroffJob->sealed_payload, 'job-payload'
|
|
|
|
|
|
);
|
|
|
|
|
|
$mock->append(new Response(200, [], json_encode([
|
|
|
|
|
|
'operation_id' => $poweroffJob->operation_id,
|
|
|
|
|
|
'status' => 'succeeded',
|
|
|
|
|
|
...CloudOperationRecoveryIdentity::expected($poweroffJob, true),
|
|
|
|
|
|
'result' => [
|
|
|
|
|
|
'uuid' => (string) $instance->uuid,
|
|
|
|
|
|
'state' => 'stopped',
|
|
|
|
|
|
],
|
|
|
|
|
|
])));
|
|
|
|
|
|
$worker->tick();
|
|
|
|
|
|
$poweroffRequest = $agentRequest(end($history));
|
|
|
|
|
|
$poweroffBody = json_decode(
|
|
|
|
|
|
(string) ($poweroffRequest['body'] ?? ''), true, 8,
|
|
|
|
|
|
JSON_THROW_ON_ERROR
|
|
|
|
|
|
);
|
|
|
|
|
|
$assert(CloudJob::find($poweroffJob->id)->status === 'succeeded'
|
|
|
|
|
|
&& ($storedPoweroff['body']['action'] ?? null) === 'poweroff'
|
|
|
|
|
|
&& $poweroffBody === ['action' => 'destroy'],
|
|
|
|
|
|
'Panel poweroff was not translated exactly once at the Agent wire boundary.');
|
|
|
|
|
|
|
|
|
|
|
|
// Keep one bounded Agent cause for administrator diagnosis while the
|
|
|
|
|
|
// public job contract continues to expose only the panel-owned error.
|
|
|
|
|
|
$diagnosticJob = CloudJobService::enqueue(
|
|
|
|
|
|
$instance, $actor, 'action', 'POST', '/actions', ['action' => 'reboot']
|
|
|
|
|
|
);
|
|
|
|
|
|
$mock->append(new Response(200, [], json_encode([
|
|
|
|
|
|
'operation_id' => $diagnosticJob->operation_id,
|
|
|
|
|
|
'status' => 'failed',
|
|
|
|
|
|
...CloudOperationRecoveryIdentity::expected($diagnosticJob),
|
|
|
|
|
|
'cause_code' => 'domain_start_failed',
|
|
|
|
|
|
'error' => [
|
|
|
|
|
|
'code' => 'libvirt_error',
|
|
|
|
|
|
'message' => 'private path /srv/libvirt/domain.xml',
|
|
|
|
|
|
],
|
|
|
|
|
|
'effect_may_have_applied' => false,
|
|
|
|
|
|
'safe_to_resubmit' => true,
|
|
|
|
|
|
'result' => ['state' => 'stopped'],
|
|
|
|
|
|
])));
|
|
|
|
|
|
$worker->tick();
|
|
|
|
|
|
$diagnosticRow = CloudJob::find($diagnosticJob->id);
|
|
|
|
|
|
$diagnosticResult = json_decode((string) $diagnosticRow->result_json,
|
|
|
|
|
|
true, flags: JSON_THROW_ON_ERROR);
|
|
|
|
|
|
$publicDiagnostic = CloudJobService::display($diagnosticRow);
|
|
|
|
|
|
$assert($diagnosticRow->status === 'failed'
|
|
|
|
|
|
&& $diagnosticRow->error_code === 'operation_failed'
|
|
|
|
|
|
&& ($diagnosticResult['cause_code'] ?? null) === 'domain_start_failed'
|
|
|
|
|
|
&& !isset($publicDiagnostic['result']['cause_code'])
|
|
|
|
|
|
&& !str_contains(json_encode($publicDiagnostic,
|
|
|
|
|
|
JSON_UNESCAPED_UNICODE), '/srv/libvirt'),
|
|
|
|
|
|
'Stable Agent cause was not persisted privately or leaked into the public job DTO.');
|
|
|
|
|
|
|
|
|
|
|
|
// A strict two-hop admission rejection proves that the mutation never
|
|
|
|
|
|
// reached the Agent journal. It is terminal and may permit an explicit
|
|
|
|
|
|
// future request, but the worker never re-dispatches this task.
|
|
|
|
|
|
$requeueJob = CloudJobService::enqueue(
|
|
|
|
|
|
$instance, $actor, 'action', 'POST', '/actions', ['action' => 'reboot']
|
|
|
|
|
|
);
|
|
|
|
|
|
$beforeRequeue = count($history);
|
|
|
|
|
|
$mock->append(new Response(503, [], json_encode(['error' => [
|
|
|
|
|
|
'code' => 'agent_busy',
|
|
|
|
|
|
'message' => 'Agent control request was not dispatched',
|
|
|
|
|
|
'retryable' => false,
|
|
|
|
|
|
'dispatch_state' => 'not_dispatched',
|
|
|
|
|
|
]])));
|
|
|
|
|
|
$worker->tick();
|
|
|
|
|
|
$requeueJob->refresh();
|
|
|
|
|
|
$assert($requeueJob->phase === 'terminal'
|
|
|
|
|
|
&& $requeueJob->outcome === 'failed'
|
|
|
|
|
|
&& $requeueJob->status === 'failed'
|
|
|
|
|
|
&& (int) $requeueJob->dispatch_attempts === 1
|
|
|
|
|
|
&& $requeueJob->dispatched_at !== null
|
|
|
|
|
|
&& $requeueJob->lease_token === null
|
|
|
|
|
|
&& (bool) $requeueJob->effect_may_have_applied === false
|
|
|
|
|
|
&& (bool) $requeueJob->safe_to_resubmit === true
|
|
|
|
|
|
&& $requeueJob->effect_resolution === 'not_applied',
|
|
|
|
|
|
'Definitive pre-dispatch backpressure did not become a safe terminal failure.');
|
|
|
|
|
|
$worker->tick();
|
|
|
|
|
|
$requeueRequests = array_slice($history, $beforeRequeue);
|
|
|
|
|
|
$assert(CloudJob::find($requeueJob->id)->status === 'failed'
|
|
|
|
|
|
&& count($requeueRequests) === 1
|
|
|
|
|
|
&& $agentRequest($requeueRequests[0])['method'] === 'POST',
|
|
|
|
|
|
'Safe pre-dispatch failure was automatically retried.');
|
|
|
|
|
|
|
|
|
|
|
|
// A strict DTO rejection happens before the Agent operation journal for
|
|
|
|
|
|
// every mutation kind, not only provisioning. It is terminal validation
|
|
|
|
|
|
// failure and must not strand a non-create task in uncertain/404 polling.
|
|
|
|
|
|
$invalidActionJob = CloudJobService::enqueue(
|
|
|
|
|
|
$instance, $actor, 'action', 'POST', '/actions', ['action' => 'reboot']
|
|
|
|
|
|
);
|
|
|
|
|
|
$beforeInvalidAction = count($history);
|
|
|
|
|
|
$mock->append(new Response(422, [], json_encode(['error' => [
|
|
|
|
|
|
'code' => 'invalid_json',
|
|
|
|
|
|
'message' => 'request contains an unknown field',
|
|
|
|
|
|
'retryable' => false,
|
|
|
|
|
|
'dispatch_state' => 'not_dispatched',
|
|
|
|
|
|
]])));
|
|
|
|
|
|
$worker->tick();
|
|
|
|
|
|
$invalidActionJob->refresh();
|
|
|
|
|
|
$assert($invalidActionJob->status === 'failed'
|
|
|
|
|
|
&& $invalidActionJob->dispatched_at !== null
|
|
|
|
|
|
&& count($history) === $beforeInvalidAction + 1,
|
|
|
|
|
|
'Definitive non-create validation failure became permanently uncertain.');
|
|
|
|
|
|
|
|
|
|
|
|
// A gateway secret mismatch is a terminal configuration/authentication
|
|
|
|
|
|
// rejection before WSS dispatch. It must not strand the task in uncertain
|
|
|
|
|
|
// polling or loop forever in the queue.
|
|
|
|
|
|
$gatewayAuthJob = CloudJobService::enqueue(
|
|
|
|
|
|
$instance, $actor, 'action', 'POST', '/actions', ['action' => 'reboot']
|
|
|
|
|
|
);
|
|
|
|
|
|
$beforeGatewayAuth = count($history);
|
|
|
|
|
|
$mock->append(new Response(403, [], json_encode(['error' => [
|
|
|
|
|
|
'code' => 'gateway_forbidden',
|
|
|
|
|
|
'message' => 'Agent gateway request is forbidden',
|
|
|
|
|
|
'retryable' => false,
|
|
|
|
|
|
'dispatch_state' => 'not_dispatched',
|
|
|
|
|
|
]])));
|
|
|
|
|
|
$worker->tick();
|
|
|
|
|
|
$gatewayAuthJob->refresh();
|
|
|
|
|
|
$assert($gatewayAuthJob->status === 'failed'
|
|
|
|
|
|
&& $gatewayAuthJob->dispatched_at !== null
|
|
|
|
|
|
&& $gatewayAuthJob->lease_token === null,
|
|
|
|
|
|
'Gateway authentication failure did not terminate before journal dispatch.');
|
|
|
|
|
|
$gatewayAuthRequests = array_slice($history, $beforeGatewayAuth);
|
|
|
|
|
|
$assert(count($gatewayAuthRequests) === 1
|
|
|
|
|
|
&& $agentRequest($gatewayAuthRequests[0])['method'] === 'POST',
|
|
|
|
|
|
'Gateway authentication failure was replayed after a terminal response.');
|
|
|
|
|
|
|
|
|
|
|
|
// A journal-backed error envelope for another operation must never finish
|
|
|
|
|
|
// this job, even when the foreign record itself is terminal.
|
|
|
|
|
|
$crossBoundJob = CloudJobService::enqueue(
|
|
|
|
|
|
$instance, $actor, 'action', 'POST', '/actions', ['action' => 'reset']
|
|
|
|
|
|
);
|
|
|
|
|
|
$mock->append(new Response(409, [], json_encode([
|
|
|
|
|
|
'error' => [
|
|
|
|
|
|
'code' => 'operation_payload_mismatch',
|
|
|
|
|
|
'retryable' => false,
|
|
|
|
|
|
'operation' => [
|
|
|
|
|
|
'operation_id' => $actionJob->operation_id,
|
|
|
|
|
|
'status' => 'failed',
|
|
|
|
|
|
'error' => ['code' => 'operation_payload_mismatch'],
|
|
|
|
|
|
],
|
|
|
|
|
|
],
|
|
|
|
|
|
])));
|
|
|
|
|
|
$worker->tick();
|
|
|
|
|
|
$crossBoundJob->refresh();
|
|
|
|
|
|
$assert($crossBoundJob->phase === 'terminal'
|
|
|
|
|
|
&& $crossBoundJob->outcome === 'failed'
|
|
|
|
|
|
&& $crossBoundJob->status === 'failed'
|
|
|
|
|
|
&& (bool) $crossBoundJob->effect_may_have_applied === true
|
|
|
|
|
|
&& (bool) $crossBoundJob->safe_to_resubmit === false
|
|
|
|
|
|
&& $crossBoundJob->effect_resolution === 'unknown'
|
|
|
|
|
|
&& $instance->fresh()->status === 'active',
|
|
|
|
|
|
'A foreign durable error envelope advanced the current job.');
|
|
|
|
|
|
$crossBoundJob->effect_resolution = 'not_applied';
|
|
|
|
|
|
$crossBoundJob->effect_resolved_at = time();
|
|
|
|
|
|
$crossBoundJob->effect_resolved_by = 99;
|
|
|
|
|
|
$crossBoundJob->effect_resolution_reason = 'test fixture fence resolution';
|
|
|
|
|
|
$crossBoundJob->save();
|
|
|
|
|
|
|
|
|
|
|
|
// Clean-break protocol: a legacy success shortcut without operation_id is
|
|
|
|
|
|
// ambiguous and cannot be used to advance any mutation.
|
|
|
|
|
|
$legacyShortcutJob = CloudJobService::enqueue(
|
|
|
|
|
|
$instance, $actor, 'action', 'POST', '/actions', ['action' => 'reboot']
|
|
|
|
|
|
);
|
|
|
|
|
|
$mock->append(new Response(200, [], json_encode([
|
|
|
|
|
|
'status' => 'succeeded',
|
|
|
|
|
|
'uuid' => (string) $instance->uuid,
|
|
|
|
|
|
'state' => 'running',
|
|
|
|
|
|
])));
|
|
|
|
|
|
$worker->tick();
|
|
|
|
|
|
$legacyShortcutJob->refresh();
|
|
|
|
|
|
$assert($legacyShortcutJob->phase === 'terminal'
|
|
|
|
|
|
&& $legacyShortcutJob->outcome === 'failed'
|
|
|
|
|
|
&& $legacyShortcutJob->status === 'failed'
|
|
|
|
|
|
&& $legacyShortcutJob->effect_resolution === 'unknown',
|
|
|
|
|
|
'A mutation response without operation_id was accepted as success.');
|
|
|
|
|
|
$legacyShortcutJob->effect_resolution = 'not_applied';
|
|
|
|
|
|
$legacyShortcutJob->effect_resolved_at = time();
|
|
|
|
|
|
$legacyShortcutJob->effect_resolved_by = 99;
|
|
|
|
|
|
$legacyShortcutJob->effect_resolution_reason = 'test fixture fence resolution';
|
|
|
|
|
|
$legacyShortcutJob->save();
|
|
|
|
|
|
|
|
|
|
|
|
// Terminal tasks are never polled through the retired Agent operation
|
|
|
|
|
|
// lookup/reconcile loop. Every Agent request issued above was the single
|
|
|
|
|
|
// mutation dispatch for its own durable task.
|
|
|
|
|
|
$lookupRequests = array_filter($history, static function (array $entry)
|
|
|
|
|
|
use ($agentRequest): bool {
|
|
|
|
|
|
return $agentRequest($entry)['method'] === 'GET';
|
|
|
|
|
|
});
|
|
|
|
|
|
$assert($lookupRequests === [],
|
|
|
|
|
|
'A terminal task triggered retired Agent lookup/reconcile traffic.');
|
|
|
|
|
|
$instance->network_status = 'active';
|
|
|
|
|
|
$instance->save();
|
|
|
|
|
|
|
|
|
|
|
|
// Expiry networking, graceful purge shutdown, and irreversible deletion
|
|
|
|
|
|
// deliberately use separate states. Only the four target-state policy
|
|
|
|
|
|
// jobs may converge; the final disk deletion is a one-shot mutation.
|
|
|
|
|
|
$makeLifecycleInstance = function (string $uuid, string $name,
|
|
|
|
|
|
string $serviceStatus, string $networkStatus) use ($instance) {
|
|
|
|
|
|
$copy = $instance->replicate();
|
|
|
|
|
|
$copy->order_id = null;
|
|
|
|
|
|
$copy->reservation_id = null;
|
|
|
|
|
|
$copy->current_period_id = null;
|
|
|
|
|
|
$copy->uuid = $uuid;
|
|
|
|
|
|
$copy->name = $name;
|
|
|
|
|
|
$copy->status = 'active';
|
|
|
|
|
|
$copy->service_status = $serviceStatus;
|
|
|
|
|
|
$copy->network_status = $networkStatus;
|
|
|
|
|
|
$copy->suspended_at = null;
|
|
|
|
|
|
$copy->network_suspended_at = null;
|
|
|
|
|
|
$copy->sealed_credential = \App\Services\Cloud\CloudSecrets::encrypt([
|
|
|
|
|
|
'password' => 'LifecycleStrongPassword!123', 'login_name' => 'root',
|
|
|
|
|
|
], 'instance-credential');
|
|
|
|
|
|
$copy->created_at = $copy->updated_at = time();
|
|
|
|
|
|
$copy->save();
|
|
|
|
|
|
return $copy;
|
|
|
|
|
|
};
|
|
|
|
|
|
$queuePolicy = function (CloudInstance $target, string $kind,
|
|
|
|
|
|
string $suffix, ?string $reason = null) {
|
|
|
|
|
|
$job = new CloudJob();
|
|
|
|
|
|
$job->operation_id = bin2hex(random_bytes(16));
|
|
|
|
|
|
$job->instance_id = $target->id;
|
|
|
|
|
|
$job->user_id = $target->user_id;
|
|
|
|
|
|
$job->host_id = $target->host_id;
|
|
|
|
|
|
$job->kind = $kind;
|
|
|
|
|
|
$job->execution_mode = 'agent';
|
|
|
|
|
|
$job->status = 'queued';
|
|
|
|
|
|
$job->sealed_payload = \App\Services\Cloud\CloudSecrets::encrypt([
|
|
|
|
|
|
'method' => 'POST',
|
|
|
|
|
|
'request_target' => '/v1/vms/' . $target->uuid . '/' . $suffix,
|
|
|
|
|
|
'body' => $reason === null ? [] : ['reason' => $reason],
|
|
|
|
|
|
'owner_id' => (int) $target->user_id,
|
|
|
|
|
|
'system_policy' => true, 'suspend_reason' => $reason,
|
|
|
|
|
|
], 'job-payload');
|
|
|
|
|
|
$job->available_at = $job->created_at = $job->updated_at = time();
|
|
|
|
|
|
$job->save();
|
|
|
|
|
|
return $job;
|
|
|
|
|
|
};
|
|
|
|
|
|
$adminUser = new User();
|
|
|
|
|
|
$adminUser->id = 99; $adminUser->money = '0.00'; $adminUser->enable = 1;
|
|
|
|
|
|
$adminUser->is_admin = true; $adminUser->save();
|
|
|
|
|
|
$adminActor = ['user_id' => 99, 'is_admin' => true,
|
|
|
|
|
|
'is_super_admin' => true, 'can_manage' => true, 'session_id' => ''];
|
|
|
|
|
|
|
|
|
|
|
|
$expiredNetwork = $makeLifecycleInstance(
|
|
|
|
|
|
'55555555-5555-4555-8555-555555555555', 'expired-network-vm',
|
|
|
|
|
|
'expired_grace', 'blocking_expired');
|
|
|
|
|
|
$expiredNetworkJob = $queuePolicy(
|
|
|
|
|
|
$expiredNetwork, 'network_suspend', 'network/suspend', 'expired');
|
|
|
|
|
|
$mock->append(new Response(200, [], json_encode([
|
|
|
|
|
|
'operation_id' => $expiredNetworkJob->operation_id,
|
|
|
|
|
|
'status' => 'succeeded',
|
|
|
|
|
|
...CloudOperationRecoveryIdentity::expected(
|
|
|
|
|
|
$expiredNetworkJob,
|
|
|
|
|
|
true
|
|
|
|
|
|
),
|
|
|
|
|
|
'result' => ['network_suspended' => true],
|
|
|
|
|
|
])));
|
|
|
|
|
|
$worker->tick();
|
|
|
|
|
|
$assert(CloudJob::find($expiredNetworkJob->id)->status === 'succeeded'
|
|
|
|
|
|
&& $expiredNetwork->fresh()->network_status === 'blocked_expired',
|
|
|
|
|
|
'Expiry network suspension was recorded as a traffic quota block.');
|
|
|
|
|
|
|
|
|
|
|
|
$purgeStopped = $makeLifecycleInstance(
|
|
|
|
|
|
'66666666-6666-4666-8666-666666666666', 'purge-stop-success-vm',
|
|
|
|
|
|
'purge_stopping', 'blocked_expired');
|
|
|
|
|
|
$purgeStopped->status = 'deleting';
|
|
|
|
|
|
$purgeStopped->save();
|
|
|
|
|
|
$purgeStopJob = $queuePolicy(
|
|
|
|
|
|
$purgeStopped, 'service_suspend', 'service/suspend', 'purge');
|
|
|
|
|
|
$mock->append(new Response(200, [], json_encode([
|
|
|
|
|
|
'operation_id' => $purgeStopJob->operation_id,
|
|
|
|
|
|
'status' => 'succeeded',
|
|
|
|
|
|
...CloudOperationRecoveryIdentity::expected($purgeStopJob, true),
|
|
|
|
|
|
'result' => ['service_was_running' => true],
|
|
|
|
|
|
])));
|
|
|
|
|
|
$worker->tick();
|
|
|
|
|
|
$purgeStopped->refresh();
|
|
|
|
|
|
$assert(CloudJob::find($purgeStopJob->id)->status === 'succeeded'
|
|
|
|
|
|
&& $purgeStopped->status === 'deleting'
|
|
|
|
|
|
&& $purgeStopped->service_status === 'purge_deleting'
|
|
|
|
|
|
&& $purgeStopped->suspended_at === null,
|
|
|
|
|
|
'Successful expiry shutdown did not advance to disk deletion safely.');
|
|
|
|
|
|
|
|
|
|
|
|
$purgeStopFailed = $makeLifecycleInstance(
|
|
|
|
|
|
'77777777-7777-4777-8777-777777777777', 'purge-stop-failed-vm',
|
|
|
|
|
|
'purge_stopping', 'blocked_expired');
|
|
|
|
|
|
$purgeStopFailed->status = 'deleting';
|
|
|
|
|
|
$purgeStopFailed->save();
|
|
|
|
|
|
$purgeStopFailedJob = $queuePolicy(
|
|
|
|
|
|
$purgeStopFailed, 'service_suspend', 'service/suspend', 'purge');
|
|
|
|
|
|
$mock->append(new Response(409, [], json_encode([
|
|
|
|
|
|
'operation_id' => $purgeStopFailedJob->operation_id,
|
|
|
|
|
|
'status' => 'failed',
|
|
|
|
|
|
...CloudOperationRecoveryIdentity::expected($purgeStopFailedJob),
|
|
|
|
|
|
'error' => ['code' => 'conflict', 'message' => 'shutdown rejected'],
|
|
|
|
|
|
'effect_may_have_applied' => false,
|
|
|
|
|
|
'safe_to_resubmit' => true,
|
|
|
|
|
|
'replayed' => false,
|
|
|
|
|
|
'retryable' => false,
|
|
|
|
|
|
])));
|
|
|
|
|
|
$worker->tick();
|
|
|
|
|
|
$purgeStopFailed->refresh();
|
|
|
|
|
|
$assert(CloudJob::find($purgeStopFailedJob->id)->status === 'failed'
|
|
|
|
|
|
&& $purgeStopFailed->status === 'deleting'
|
|
|
|
|
|
&& $purgeStopFailed->service_status === 'purge_stop_failed',
|
|
|
|
|
|
'Failed expiry shutdown incorrectly advanced or failed the disk purge.');
|
|
|
|
|
|
|
|
|
|
|
|
$purgeForceRequired = $makeLifecycleInstance(
|
|
|
|
|
|
'76767676-7676-4676-8676-767676767676',
|
|
|
|
|
|
'purge-force-required-vm', 'purge_stopping', 'blocked_expired');
|
|
|
|
|
|
$purgeForceRequired->status = 'deleting';
|
|
|
|
|
|
$purgeForceRequired->save();
|
|
|
|
|
|
$purgeForcePrior = $queuePolicy(
|
|
|
|
|
|
$purgeForceRequired, 'service_suspend', 'service/suspend', 'purge');
|
|
|
|
|
|
$mock->append(new Response(409, [], json_encode([
|
|
|
|
|
|
'operation_id' => $purgeForcePrior->operation_id,
|
|
|
|
|
|
'status' => 'failed',
|
|
|
|
|
|
...CloudOperationRecoveryIdentity::expected($purgeForcePrior),
|
|
|
|
|
|
'error' => ['code' => 'force_required'],
|
|
|
|
|
|
'replayed' => false,
|
|
|
|
|
|
'retryable' => false,
|
|
|
|
|
|
])));
|
|
|
|
|
|
$worker->tick();
|
|
|
|
|
|
$purgeForceRequired->refresh();
|
|
|
|
|
|
$purgeForcePrior->refresh();
|
|
|
|
|
|
$assert($purgeForcePrior->phase === 'terminal'
|
|
|
|
|
|
&& $purgeForcePrior->outcome === 'failed'
|
|
|
|
|
|
&& $purgeForcePrior->status === 'failed'
|
|
|
|
|
|
&& $purgeForcePrior->error_code === 'force_required'
|
|
|
|
|
|
&& $purgeForceRequired->status === 'deleting'
|
|
|
|
|
|
&& $purgeForceRequired->service_status === 'purge_force_required',
|
|
|
|
|
|
'Expiry graceful shutdown crossed the explicit force boundary.');
|
|
|
|
|
|
|
|
|
|
|
|
// A force-required graceful shutdown is already a terminal failed task.
|
|
|
|
|
|
// Any later destructive force action must be a separately audited task;
|
|
|
|
|
|
// inspection never rewrites the prior terminal receipt.
|
|
|
|
|
|
|
|
|
|
|
|
Capsule::table('cloud_backup_subscriptions')->insert([
|
|
|
|
|
|
'instance_id' => (int) $purgeForceRequired->id,
|
|
|
|
|
|
'user_id' => (int) $purgeForceRequired->user_id,
|
|
|
|
|
|
'offer_id' => (int) $purgeForceRequired->offer_id,
|
|
|
|
|
|
'enabled' => 1,
|
|
|
|
|
|
'renew_next_period' => 1,
|
|
|
|
|
|
'frequency' => 'daily',
|
|
|
|
|
|
'weekday' => 0,
|
|
|
|
|
|
'retention_count' => 2,
|
|
|
|
|
|
'timezone' => 'UTC',
|
|
|
|
|
|
'price' => '2.50',
|
|
|
|
|
|
'currency' => 'USD',
|
|
|
|
|
|
'price_version' => 1,
|
|
|
|
|
|
'next_run_at' => time() + 3600,
|
|
|
|
|
|
'created_at' => time(),
|
|
|
|
|
|
'updated_at' => time(),
|
|
|
|
|
|
]);
|
|
|
|
|
|
$purgeForceJob = CloudJobService::enqueueForceDelete(
|
|
|
|
|
|
$purgeForcePrior, $adminActor
|
|
|
|
|
|
);
|
|
|
|
|
|
$mock->append(new Response(200, [], json_encode([
|
|
|
|
|
|
'operation_id' => $purgeForceJob->operation_id,
|
|
|
|
|
|
'status' => 'succeeded',
|
|
|
|
|
|
...CloudOperationRecoveryIdentity::expected($purgeForceJob, true),
|
|
|
|
|
|
'result' => [
|
|
|
|
|
|
'deleted' => true, 'managed_artifacts_absent' => true,
|
|
|
|
|
|
'forced' => true,
|
|
|
|
|
|
],
|
|
|
|
|
|
])));
|
|
|
|
|
|
$worker->tick();
|
|
|
|
|
|
$purgeForcedRequest = $agentRequest(end($history));
|
|
|
|
|
|
$purgeForcedBody = json_decode($purgeForcedRequest['body'], true);
|
|
|
|
|
|
$assert($purgeForceJob->fresh()->status === 'succeeded'
|
|
|
|
|
|
&& $purgeForcePrior->fresh()->status === 'failed'
|
|
|
|
|
|
&& $purgeForcePrior->fresh()->error_code === 'force_required'
|
|
|
|
|
|
&& $purgeForceRequired->fresh()->status === 'deleted'
|
|
|
|
|
|
&& (int) Capsule::table('cloud_backup_subscriptions')
|
|
|
|
|
|
->where('instance_id', (int) $purgeForceRequired->id)
|
|
|
|
|
|
->value('enabled') === 0
|
|
|
|
|
|
&& (int) Capsule::table('cloud_backup_subscriptions')
|
|
|
|
|
|
->where('instance_id', (int) $purgeForceRequired->id)
|
|
|
|
|
|
->value('renew_next_period') === 0
|
|
|
|
|
|
&& $purgeForcedRequest['method'] === 'POST'
|
|
|
|
|
|
&& str_ends_with($purgeForcedRequest['request_target'], '/force-delete')
|
|
|
|
|
|
&& ($purgeForcedBody['prior_operation_id'] ?? null)
|
|
|
|
|
|
=== $purgeForcePrior->operation_id
|
|
|
|
|
|
&& ($purgeForcedBody['ack_data_loss'] ?? null) === true,
|
|
|
|
|
|
'Expiry force-required state did not reach the separately audited force route.');
|
|
|
|
|
|
|
|
|
|
|
|
$expiryDeleted = $makeLifecycleInstance(
|
|
|
|
|
|
'88888888-8888-4888-8888-888888888888', 'expiry-delete-success-vm',
|
|
|
|
|
|
'purge_deleting', 'blocked_expired');
|
|
|
|
|
|
$expiryDeleted->status = 'deleting';
|
|
|
|
|
|
$expiryDeleted->save();
|
|
|
|
|
|
$olderBacklog = CloudJobService::enqueue($instance, $actor, 'action', 'POST',
|
|
|
|
|
|
'/actions', ['action' => 'reboot']);
|
|
|
|
|
|
$expiryDeleteJob = CloudJobService::enqueueExpiryDelete($expiryDeleted);
|
|
|
|
|
|
$beforeExpiryDeleteRequests = count($history);
|
|
|
|
|
|
$user->enable = 0;
|
|
|
|
|
|
$user->save();
|
|
|
|
|
|
$mock->append(new Response(200, [], json_encode([
|
|
|
|
|
|
'operation_id' => $expiryDeleteJob->operation_id,
|
|
|
|
|
|
'status' => 'succeeded',
|
|
|
|
|
|
...CloudOperationRecoveryIdentity::expected($expiryDeleteJob, true),
|
|
|
|
|
|
'result' => [
|
|
|
|
|
|
'deleted' => true,
|
|
|
|
|
|
'managed_artifacts_absent' => true,
|
|
|
|
|
|
'forced' => true,
|
|
|
|
|
|
],
|
|
|
|
|
|
])));
|
|
|
|
|
|
$worker->tick();
|
|
|
|
|
|
$user->enable = 1;
|
|
|
|
|
|
$user->save();
|
|
|
|
|
|
$expiryDeleteRequests = array_slice($history, $beforeExpiryDeleteRequests);
|
|
|
|
|
|
$expiryDeleteRequest = count($expiryDeleteRequests) === 1
|
|
|
|
|
|
? $agentRequest($expiryDeleteRequests[0]) : [];
|
|
|
|
|
|
$expiryDeleteActor = count($expiryDeleteRequests) === 1
|
|
|
|
|
|
? json_decode(base64_decode(strtr(
|
|
|
|
|
|
$expiryDeleteRequest['headers']['X-Virt-Actor'],
|
|
|
|
|
|
'-_', '+/'
|
|
|
|
|
|
)), true) : [];
|
|
|
|
|
|
$expiryDeleted->refresh();
|
|
|
|
|
|
$assert(count($expiryDeleteRequests) === 1
|
|
|
|
|
|
&& $expiryDeleteRequest['method'] === 'DELETE'
|
|
|
|
|
|
&& !str_contains($expiryDeleteRequest['request_target'], '?')
|
|
|
|
|
|
&& json_decode($expiryDeleteRequest['body'], true)
|
|
|
|
|
|
=== ['remove_managed_disks' => true]
|
|
|
|
|
|
&& ($expiryDeleteActor['can_manage'] ?? false) === true
|
|
|
|
|
|
&& ($expiryDeleteActor['operation_id'] ?? null)
|
|
|
|
|
|
=== $expiryDeleteJob->operation_id
|
|
|
|
|
|
&& !array_key_exists('system_cleanup', $expiryDeleteActor)
|
|
|
|
|
|
&& $expiryDeleteActor['is_admin'] === false
|
|
|
|
|
|
&& CloudJob::find($expiryDeleteJob->id)->phase === 'terminal'
|
|
|
|
|
|
&& CloudJob::find($expiryDeleteJob->id)->outcome === 'failed'
|
|
|
|
|
|
&& CloudJob::find($expiryDeleteJob->id)->status === 'failed'
|
|
|
|
|
|
&& (bool) CloudJob::find($expiryDeleteJob->id)
|
|
|
|
|
|
->effect_may_have_applied === true
|
|
|
|
|
|
&& (bool) CloudJob::find($expiryDeleteJob->id)
|
|
|
|
|
|
->safe_to_resubmit === false
|
|
|
|
|
|
&& $expiryDeleted->status === 'delete_failed'
|
|
|
|
|
|
&& $expiryDeleted->service_status === 'purge_failed'
|
|
|
|
|
|
&& $expiryDeleted->sealed_credential !== null,
|
|
|
|
|
|
'Expiry cleanup accepted force evidence on the graceful route or released panel state.');
|
|
|
|
|
|
$olderBacklog->phase = 'terminal';
|
|
|
|
|
|
$olderBacklog->outcome = 'failed';
|
|
|
|
|
|
$olderBacklog->status = 'failed';
|
|
|
|
|
|
$olderBacklog->terminal_at = time();
|
|
|
|
|
|
$olderBacklog->effect_may_have_applied = 0;
|
|
|
|
|
|
$olderBacklog->safe_to_resubmit = 1;
|
|
|
|
|
|
$olderBacklog->effect_resolution = 'not_applied';
|
|
|
|
|
|
$olderBacklog->available_at = 0;
|
|
|
|
|
|
$olderBacklog->save();
|
|
|
|
|
|
$beforeExpiryDeleteRetry = count($history);
|
|
|
|
|
|
$worker->tick();
|
|
|
|
|
|
$assert(count($history) === $beforeExpiryDeleteRetry
|
|
|
|
|
|
&& $expiryDeleteJob->fresh()->status === 'failed'
|
|
|
|
|
|
&& (int) $expiryDeleteJob->fresh()->dispatch_attempts === 1,
|
|
|
|
|
|
'Terminal expiry deletion was automatically looked up or mutation-retried.');
|
|
|
|
|
|
|
|
|
|
|
|
$forgedCleanup = $makeLifecycleInstance(
|
|
|
|
|
|
'89898989-8989-4898-8898-898989898989', 'forged-expiry-delete-vm',
|
|
|
|
|
|
'purge_deleting', 'blocked_expired');
|
|
|
|
|
|
$forgedCleanup->status = 'deleting';
|
|
|
|
|
|
$forgedCleanup->save();
|
|
|
|
|
|
$forgedCleanupJob = CloudJobService::enqueueExpiryDelete($forgedCleanup);
|
|
|
|
|
|
$forgedPayload = \App\Services\Cloud\CloudSecrets::decrypt(
|
|
|
|
|
|
$forgedCleanupJob->sealed_payload,
|
|
|
|
|
|
'job-payload'
|
|
|
|
|
|
);
|
|
|
|
|
|
$forgedPayload['request_target'] = '/v1/vms/' . $instance->uuid;
|
|
|
|
|
|
$forgedCleanupJob->sealed_payload = \App\Services\Cloud\CloudSecrets::encrypt(
|
|
|
|
|
|
$forgedPayload,
|
|
|
|
|
|
'job-payload'
|
|
|
|
|
|
);
|
|
|
|
|
|
$forgedCleanupJob->save();
|
|
|
|
|
|
$beforeForgedCleanupRequests = count($history);
|
|
|
|
|
|
$worker->tick();
|
|
|
|
|
|
$forgedCleanup->refresh();
|
|
|
|
|
|
$assert(count($history) === $beforeForgedCleanupRequests
|
|
|
|
|
|
&& CloudJob::find($forgedCleanupJob->id)->status === 'failed'
|
|
|
|
|
|
&& $forgedCleanup->status === 'delete_failed'
|
|
|
|
|
|
&& $forgedCleanup->service_status === 'purge_failed',
|
|
|
|
|
|
'A forged cleanup payload crossed its bound instance or reached the Agent.');
|
|
|
|
|
|
|
|
|
|
|
|
$expiryDeleteFailed = $makeLifecycleInstance(
|
|
|
|
|
|
'99999999-9999-4999-8999-999999999999', 'expiry-delete-failed-vm',
|
|
|
|
|
|
'purge_deleting', 'blocked_expired');
|
|
|
|
|
|
$expiryDeleteFailed->status = 'deleting';
|
|
|
|
|
|
$expiryDeleteFailed->save();
|
|
|
|
|
|
$expiryDeleteFailedJob = CloudJobService::enqueueExpiryDelete($expiryDeleteFailed);
|
|
|
|
|
|
$mock->append(new Response(409, [], json_encode([
|
|
|
|
|
|
'operation_id' => $expiryDeleteFailedJob->operation_id,
|
|
|
|
|
|
'status' => 'failed',
|
|
|
|
|
|
...CloudOperationRecoveryIdentity::expected($expiryDeleteFailedJob),
|
|
|
|
|
|
'error' => [
|
|
|
|
|
|
'code' => 'conflict',
|
|
|
|
|
|
'message' => 'volume delete rejected before undefine',
|
|
|
|
|
|
],
|
|
|
|
|
|
'effect_may_have_applied' => false,
|
|
|
|
|
|
'safe_to_resubmit' => true,
|
|
|
|
|
|
'replayed' => false,
|
|
|
|
|
|
'retryable' => false,
|
|
|
|
|
|
])));
|
|
|
|
|
|
$worker->tick();
|
|
|
|
|
|
$expiryDeleteFailed->refresh();
|
|
|
|
|
|
$assert(CloudJob::find($expiryDeleteFailedJob->id)->status === 'failed'
|
|
|
|
|
|
&& $expiryDeleteFailed->status === 'delete_failed'
|
|
|
|
|
|
&& $expiryDeleteFailed->service_status === 'purge_failed'
|
|
|
|
|
|
&& $expiryDeleteFailed->sealed_credential !== null,
|
|
|
|
|
|
'Explicit expiry deletion failure did not enter the manual-review state.');
|
|
|
|
|
|
|
|
|
|
|
|
$beforeFailedExpiryRetry = count($history);
|
|
|
|
|
|
$worker->tick();
|
|
|
|
|
|
$expiryDeleteFailedJob->refresh();
|
|
|
|
|
|
$assert(count($history) === $beforeFailedExpiryRetry
|
|
|
|
|
|
&& $expiryDeleteFailedJob->phase === 'terminal'
|
|
|
|
|
|
&& $expiryDeleteFailedJob->outcome === 'failed'
|
|
|
|
|
|
&& (int) $expiryDeleteFailedJob->dispatch_attempts === 1,
|
|
|
|
|
|
'Terminal expiry failure was automatically looked up, reconciled, or mutation-retried.');
|
|
|
|
|
|
|
|
|
|
|
|
$reinstall = CloudJobService::enqueue($instance, $actor, 'reinstall', 'POST', '/reinstall', ['template_id' => $reinstallTemplate->id, 'confirmation_name' => $instance->name, 'root_password' => 'NewStrongPassword!123'], ['password' => 'NewStrongPassword!123', 'login_name' => 'root', 'auth_mode' => 'password']);
|
|
|
|
|
|
$oldCredential = $instance->sealed_credential;
|
|
|
|
|
|
$queuedReinstall = \App\Services\Cloud\CloudSecrets::decrypt(
|
|
|
|
|
|
(string) $reinstall->sealed_payload, 'job-payload'
|
|
|
|
|
|
);
|
|
|
|
|
|
$assert(($queuedReinstall['template_metadata']['template_id'] ?? null)
|
|
|
|
|
|
=== $reinstallTemplate->id
|
|
|
|
|
|
&& ($queuedReinstall['template_metadata']['os_source'] ?? null)
|
|
|
|
|
|
=== 'cloud_template'
|
|
|
|
|
|
&& ($queuedReinstall['template_metadata']['os_name'] ?? null)
|
|
|
|
|
|
=== 'Debian 13'
|
|
|
|
|
|
&& ($queuedReinstall['template_metadata']['login_name'] ?? null)
|
|
|
|
|
|
=== 'root'
|
|
|
|
|
|
&& ($queuedReinstall['template_metadata']['debian_12_network_fix']
|
|
|
|
|
|
?? null) === false
|
|
|
|
|
|
&& !array_key_exists('os_source', $queuedReinstall['body'] ?? [])
|
|
|
|
|
|
&& !array_key_exists('os_name', $queuedReinstall['body'] ?? [])
|
|
|
|
|
|
&& !array_key_exists('login_name', $queuedReinstall['body'] ?? []),
|
|
|
|
|
|
'Reinstall did not keep PHP template metadata outside the strict Agent DTO.');
|
|
|
|
|
|
$mock->append(new Response(202, [], json_encode([
|
|
|
|
|
|
'operation_id' => $reinstall->operation_id,
|
|
|
|
|
|
'status' => 'reserved',
|
|
|
|
|
|
...CloudOperationRecoveryIdentity::expected($reinstall),
|
|
|
|
|
|
'replayed' => false,
|
|
|
|
|
|
'retryable' => false,
|
|
|
|
|
|
]))); $worker->tick();
|
|
|
|
|
|
$reinstallRequest = $agentRequest(end($history));
|
|
|
|
|
|
$reinstallBody = json_decode(
|
|
|
|
|
|
(string) ($reinstallRequest['body'] ?? ''), true, 16,
|
|
|
|
|
|
JSON_THROW_ON_ERROR
|
|
|
|
|
|
);
|
|
|
|
|
|
$reinstallKeys = array_keys($reinstallBody);
|
|
|
|
|
|
sort($reinstallKeys);
|
|
|
|
|
|
$expectedReinstallKeys = [
|
|
|
|
|
|
'root_password', 'template_generation_token', 'template_id',
|
|
|
|
|
|
'template_revision',
|
|
|
|
|
|
];
|
|
|
|
|
|
sort($expectedReinstallKeys);
|
|
|
|
|
|
$assert(CloudJob::find($reinstall->id)->status === 'running' && $instance->fresh()->sealed_credential === $oldCredential, 'Pending reinstall replaced password prematurely.');
|
|
|
|
|
|
$assert($reinstallKeys === $expectedReinstallKeys
|
|
|
|
|
|
&& !array_key_exists('confirmation_name', $reinstallBody),
|
|
|
|
|
|
'Browser reinstall confirmation leaked into the strict Agent DTO.');
|
|
|
|
|
|
$beforeReinstallReceipt = count($history);
|
|
|
|
|
|
$reinstallReceipt = CloudOperationCompletionService::accept([
|
|
|
|
|
|
'type' => 'completion',
|
|
|
|
|
|
'id' => bin2hex(random_bytes(16)),
|
|
|
|
|
|
'agent_id' => (string) $instance->host_id,
|
|
|
|
|
|
'operation_id' => $reinstall->operation_id,
|
|
|
|
|
|
...CloudOperationRecoveryIdentity::expected($reinstall, true),
|
|
|
|
|
|
'status' => 'succeeded',
|
|
|
|
|
|
'effect_may_have_applied' => false,
|
|
|
|
|
|
'safe_to_resubmit' => false,
|
|
|
|
|
|
'error_code' => '',
|
|
|
|
|
|
'result' => [
|
|
|
|
|
|
'state' => 'running',
|
|
|
|
|
|
],
|
|
|
|
|
|
]);
|
|
|
|
|
|
$reinstalled = $instance->fresh();
|
|
|
|
|
|
$reinstalledSpec = json_decode((string) $reinstalled->spec_json, true,
|
|
|
|
|
|
flags: JSON_THROW_ON_ERROR);
|
|
|
|
|
|
$assert($reinstallReceipt === ['accepted' => true,
|
|
|
|
|
|
'disposition' => 'applied']
|
|
|
|
|
|
&& count($history) === $beforeReinstallReceipt
|
|
|
|
|
|
&& CloudJob::find($reinstall->id)->status === 'succeeded'
|
|
|
|
|
|
&& $reinstalled->sealed_credential !== $oldCredential
|
|
|
|
|
|
&& ($reinstalledSpec['template_id'] ?? null) === $reinstallTemplate->id
|
|
|
|
|
|
&& ($reinstalledSpec['public_network'] ?? null)
|
|
|
|
|
|
=== $openingSpec['public_network']
|
|
|
|
|
|
&& ($reinstalledSpec['os_source'] ?? null) === 'cloud_template'
|
|
|
|
|
|
&& ($reinstalledSpec['os_name'] ?? null) === 'Debian 13'
|
|
|
|
|
|
&& ($reinstalledSpec['login_name'] ?? null) === 'root'
|
|
|
|
|
|
&& ($reinstalledSpec['managed'] ?? null) === true
|
|
|
|
|
|
&& !isset($reinstalledSpec['install_iso'],
|
|
|
|
|
|
$reinstalledSpec['agent_metadata_version']),
|
|
|
|
|
|
'Completed reinstall lost its trusted template metadata.');
|
|
|
|
|
|
|
|
|
|
|
|
// A replacement guest may be running while old-volume cleanup remains
|
|
|
|
|
|
// unresolved. The worker must keep the job uncertain but still persist
|
|
|
|
|
|
// the new credential and template metadata; raw host warnings stay out of
|
|
|
|
|
|
// result_json.
|
|
|
|
|
|
$partialReinstall = CloudJobService::enqueue($instance->fresh(), $actor,
|
|
|
|
|
|
'reinstall', 'POST', '/reinstall', [
|
|
|
|
|
|
'template_id' => $input['template_id'],
|
|
|
|
|
|
'confirmation_name' => $instance->name,
|
|
|
|
|
|
'root_password' => 'PartialStrongPassword!123',
|
|
|
|
|
|
], ['password' => 'PartialStrongPassword!123', 'login_name' => 'root',
|
|
|
|
|
|
'auth_mode' => 'password']);
|
|
|
|
|
|
$mock->append(new Response(202, [], json_encode([
|
|
|
|
|
|
'operation_id' => $partialReinstall->operation_id,
|
|
|
|
|
|
'status' => 'reserved',
|
|
|
|
|
|
...CloudOperationRecoveryIdentity::expected($partialReinstall),
|
|
|
|
|
|
'replayed' => false,
|
|
|
|
|
|
'retryable' => false,
|
|
|
|
|
|
])));
|
|
|
|
|
|
$worker->tick();
|
|
|
|
|
|
$beforePartialReceipt = count($history);
|
|
|
|
|
|
$partialReceipt = CloudOperationCompletionService::accept([
|
|
|
|
|
|
'type' => 'completion',
|
|
|
|
|
|
'id' => bin2hex(random_bytes(16)),
|
|
|
|
|
|
'agent_id' => (string) $instance->host_id,
|
|
|
|
|
|
'operation_id' => $partialReinstall->operation_id,
|
|
|
|
|
|
...CloudOperationRecoveryIdentity::expected(
|
|
|
|
|
|
$partialReinstall,
|
|
|
|
|
|
true
|
|
|
|
|
|
),
|
|
|
|
|
|
'status' => 'succeeded',
|
|
|
|
|
|
'effect_may_have_applied' => false,
|
|
|
|
|
|
'safe_to_resubmit' => false,
|
|
|
|
|
|
'error_code' => '',
|
|
|
|
|
|
'result' => [
|
|
|
|
|
|
'reinstall_committed' => true,
|
|
|
|
|
|
'cleanup_pending' => true,
|
|
|
|
|
|
'vm' => ['uuid' => (string) $instance->uuid,
|
|
|
|
|
|
'name' => $instance->name, 'state' => 'running'],
|
|
|
|
|
|
],
|
|
|
|
|
|
]);
|
|
|
|
|
|
$partialJob = CloudJob::find($partialReinstall->id);
|
|
|
|
|
|
$assert($partialReceipt === ['accepted' => true,
|
|
|
|
|
|
'disposition' => 'received']
|
|
|
|
|
|
&& count($history) === $beforePartialReceipt
|
|
|
|
|
|
&& $partialJob->phase === 'terminal'
|
|
|
|
|
|
&& $partialJob->outcome === 'failed'
|
|
|
|
|
|
&& $partialJob->status === 'failed'
|
|
|
|
|
|
&& (int) $partialJob->dispatch_attempts === 1
|
|
|
|
|
|
&& (bool) $partialJob->effect_may_have_applied === true
|
|
|
|
|
|
&& (bool) $partialJob->safe_to_resubmit === false
|
|
|
|
|
|
&& $instance->fresh()->sealed_credential
|
|
|
|
|
|
=== $reinstalled->sealed_credential,
|
|
|
|
|
|
'Cleanup-pending reinstall was acknowledged or did not become a terminal fenced failure.');
|
|
|
|
|
|
$beforePartialRetry = count($history);
|
|
|
|
|
|
$worker->tick();
|
|
|
|
|
|
$assert(count($history) === $beforePartialRetry
|
|
|
|
|
|
&& CloudJob::find($partialReinstall->id)->status === 'failed',
|
|
|
|
|
|
'Cleanup-pending reinstall was automatically looked up or retried.');
|
|
|
|
|
|
$partialJob->effect_resolution = 'applied';
|
|
|
|
|
|
$partialJob->effect_resolved_at = time();
|
|
|
|
|
|
$partialJob->effect_resolved_by = 99;
|
|
|
|
|
|
$partialJob->effect_resolution_reason =
|
|
|
|
|
|
'test fixture confirms replacement guest committed';
|
|
|
|
|
|
$partialJob->save();
|
|
|
|
|
|
|
|
|
|
|
|
// A transition-owned network change crosses the complete Worker path
|
|
|
|
|
|
// exactly once and publishes only the exact typed public_network result.
|
|
|
|
|
|
$networkTargetIp = new CloudPublicIp();
|
|
|
|
|
|
$networkTargetIp->host_id = (int) $host->id;
|
|
|
|
|
|
$networkTargetIp->public_ip_pool_id = (int) $sourcePublicPool->id;
|
|
|
|
|
|
$networkTargetIp->address_family = 4;
|
|
|
|
|
|
$networkTargetIp->address = '8.8.8.200';
|
|
|
|
|
|
$networkTargetIp->address_binary = inet_pton('8.8.8.200');
|
|
|
|
|
|
$networkTargetIp->prefix_length = 24;
|
|
|
|
|
|
$networkTargetIp->state = 'available';
|
|
|
|
|
|
$networkTargetIp->created_at = $networkTargetIp->updated_at = time();
|
|
|
|
|
|
$networkTargetIp->save();
|
|
|
|
|
|
$networkDispatch = CloudPublicNetworkTransitionWorkflow::begin(
|
|
|
|
|
|
CloudPublicNetworkTransitionCommand::adminExact(
|
|
|
|
|
|
(int) $instance->id,
|
|
|
|
|
|
99,
|
|
|
|
|
|
'financial-network-transition-0001',
|
|
|
|
|
|
[4 => [
|
|
|
|
|
|
'pool_key' => $sourcePoolKey,
|
|
|
|
|
|
'address_cidr' => '8.8.8.200/24',
|
|
|
|
|
|
]]
|
|
|
|
|
|
)
|
|
|
|
|
|
);
|
|
|
|
|
|
$networkJob = CloudJob::find($networkDispatch->jobId());
|
|
|
|
|
|
$networkPayload = \App\Services\Cloud\CloudSecrets::decrypt(
|
|
|
|
|
|
(string) $networkJob->sealed_payload,
|
|
|
|
|
|
'job-payload'
|
|
|
|
|
|
);
|
|
|
|
|
|
$networkIdentity = CloudOperationRecoveryIdentity::expectedFromPayload(
|
|
|
|
|
|
$networkJob,
|
|
|
|
|
|
$networkPayload,
|
|
|
|
|
|
true
|
|
|
|
|
|
);
|
|
|
|
|
|
$networkTarget = $networkDispatch->target()->toArray();
|
|
|
|
|
|
$mock->append(new Response(200, [], json_encode([
|
|
|
|
|
|
'operation_id' => (string) $networkJob->operation_id,
|
|
|
|
|
|
'status' => 'succeeded',
|
|
|
|
|
|
...$networkIdentity,
|
|
|
|
|
|
'result' => [
|
|
|
|
|
|
'uuid' => (string) $instance->uuid,
|
|
|
|
|
|
'network_changed' => true,
|
|
|
|
|
|
'public_network' => $networkTarget,
|
|
|
|
|
|
],
|
|
|
|
|
|
], JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES)));
|
|
|
|
|
|
$beforeNetworkDispatch = count($history);
|
|
|
|
|
|
$assert($worker->tick(),
|
|
|
|
|
|
'Transition-owned network task was not consumed by the Worker.');
|
|
|
|
|
|
$networkRequest = $agentRequest($history[$beforeNetworkDispatch]);
|
|
|
|
|
|
$networkBody = json_decode(
|
|
|
|
|
|
(string) $networkRequest['body'],
|
|
|
|
|
|
true,
|
|
|
|
|
|
32,
|
|
|
|
|
|
JSON_THROW_ON_ERROR
|
|
|
|
|
|
);
|
|
|
|
|
|
$networkSpec = json_decode(
|
|
|
|
|
|
(string) $instance->fresh()->spec_json,
|
|
|
|
|
|
true,
|
|
|
|
|
|
64,
|
|
|
|
|
|
JSON_THROW_ON_ERROR
|
|
|
|
|
|
);
|
|
|
|
|
|
$assert(count($history) === $beforeNetworkDispatch + 1
|
|
|
|
|
|
&& ($networkBody['public_network'] ?? null) === $networkTarget
|
|
|
|
|
|
&& CloudJob::find($networkJob->id)->phase === 'terminal'
|
|
|
|
|
|
&& CloudJob::find($networkJob->id)->outcome === 'succeeded'
|
|
|
|
|
|
&& (int) CloudJob::find($networkJob->id)->dispatch_attempts === 1
|
|
|
|
|
|
&& Capsule::table('cloud_public_network_transitions')
|
|
|
|
|
|
->where('id', $networkDispatch->transitionId())
|
|
|
|
|
|
->value('state') === 'succeeded'
|
|
|
|
|
|
&& ($networkSpec['public_network'] ?? null) === $networkTarget,
|
|
|
|
|
|
'Worker did not dispatch/finalize the exact transition projection.');
|
|
|
|
|
|
$afterNetworkDispatch = count($history);
|
|
|
|
|
|
$worker->tick();
|
|
|
|
|
|
$assert(count($history) === $afterNetworkDispatch
|
|
|
|
|
|
&& (int) CloudJob::find($networkJob->id)->dispatch_attempts === 1,
|
|
|
|
|
|
'A terminal transition-owned network task was replayed.');
|
|
|
|
|
|
|
|
|
|
|
|
// A legacy scalar task that lacks transition ownership fails before the
|
|
|
|
|
|
// Agent. Use an isolated instance so its durable incident cannot affect
|
|
|
|
|
|
// the canonical instance's later lifecycle tests.
|
|
|
|
|
|
$legacyNetworkInstance = $makeLifecycleInstance(
|
|
|
|
|
|
'45454545-4545-4545-8545-454545454545',
|
|
|
|
|
|
'legacy-network-gate-vm',
|
|
|
|
|
|
'active',
|
|
|
|
|
|
'active'
|
|
|
|
|
|
);
|
|
|
|
|
|
$legacyNetworkSpec = (string) $legacyNetworkInstance->spec_json;
|
|
|
|
|
|
$legacyNetworkIpLedger = Capsule::table('cloud_public_ips')
|
|
|
|
|
|
->orderBy('id')->get([
|
|
|
|
|
|
'id', 'state', 'reservation_id', 'instance_id',
|
|
|
|
|
|
])->map(static fn ($row): array => (array) $row)->all();
|
|
|
|
|
|
$legacyNetworkClaimLedger = Capsule::table(
|
|
|
|
|
|
'cloud_resource_reservation_ips'
|
|
|
|
|
|
)->orderBy('id')->get([
|
|
|
|
|
|
'id', 'state', 'reservation_id', 'public_ip_id', 'instance_id',
|
|
|
|
|
|
])->map(static fn ($row): array => (array) $row)->all();
|
|
|
|
|
|
$legacyWorkerJob = new CloudJob();
|
|
|
|
|
|
$legacyWorkerJob->operation_id = bin2hex(random_bytes(16));
|
|
|
|
|
|
$legacyWorkerJob->request_idempotency_key =
|
|
|
|
|
|
'legacy-network-worker-gate-0001';
|
|
|
|
|
|
$legacyWorkerJob->request_fingerprint = hash(
|
|
|
|
|
|
'sha256',
|
|
|
|
|
|
'legacy-network-worker-gate-0001'
|
|
|
|
|
|
);
|
|
|
|
|
|
$legacyWorkerJob->instance_id = (int) $legacyNetworkInstance->id;
|
|
|
|
|
|
$legacyWorkerJob->user_id = (int) $legacyNetworkInstance->user_id;
|
|
|
|
|
|
$legacyWorkerJob->host_id = (int) $legacyNetworkInstance->host_id;
|
|
|
|
|
|
$legacyWorkerJob->kind = 'network_change';
|
|
|
|
|
|
$legacyWorkerJob->execution_mode = 'agent';
|
|
|
|
|
|
$legacyWorkerJob->status = 'queued';
|
|
|
|
|
|
$legacyWorkerJob->public_network_transition_id = null;
|
|
|
|
|
|
$legacyWorkerJob->sealed_payload =
|
|
|
|
|
|
\App\Services\Cloud\CloudSecrets::encrypt([
|
|
|
|
|
|
'method' => 'POST',
|
|
|
|
|
|
'request_target' => '/v1/vms/'
|
|
|
|
|
|
. rawurlencode((string) $legacyNetworkInstance->uuid)
|
|
|
|
|
|
. '/network',
|
|
|
|
|
|
'body' => [
|
|
|
|
|
|
'public_ip_cidr' => '9.9.9.10/24',
|
|
|
|
|
|
'confirm_restart' => true,
|
|
|
|
|
|
],
|
|
|
|
|
|
'credential' => null,
|
|
|
|
|
|
'owner_id' => (int) $legacyNetworkInstance->user_id,
|
|
|
|
|
|
'can_manage' => false,
|
|
|
|
|
|
], 'job-payload');
|
|
|
|
|
|
$legacyWorkerJob->available_at = $legacyWorkerJob->created_at =
|
|
|
|
|
|
$legacyWorkerJob->updated_at = time();
|
|
|
|
|
|
$legacyWorkerJob->save();
|
|
|
|
|
|
$beforeLegacyWorker = count($history);
|
|
|
|
|
|
$assert($worker->tick(),
|
|
|
|
|
|
'Legacy network task was not consumed by the local fail-closed path.');
|
|
|
|
|
|
$legacyWorkerJob->refresh();
|
|
|
|
|
|
$assert(count($history) === $beforeLegacyWorker
|
|
|
|
|
|
&& (int) $legacyWorkerJob->dispatch_attempts === 0
|
|
|
|
|
|
&& $legacyWorkerJob->phase === 'terminal'
|
|
|
|
|
|
&& $legacyWorkerJob->outcome === 'failed'
|
|
|
|
|
|
&& (string) $legacyNetworkInstance->fresh()->spec_json
|
|
|
|
|
|
=== $legacyNetworkSpec
|
|
|
|
|
|
&& Capsule::table('cloud_public_ips')->orderBy('id')->get([
|
|
|
|
|
|
'id', 'state', 'reservation_id', 'instance_id',
|
|
|
|
|
|
])->map(static fn ($row): array => (array) $row)->all()
|
|
|
|
|
|
=== $legacyNetworkIpLedger
|
|
|
|
|
|
&& Capsule::table('cloud_resource_reservation_ips')
|
|
|
|
|
|
->orderBy('id')->get([
|
|
|
|
|
|
'id', 'state', 'reservation_id', 'public_ip_id', 'instance_id',
|
|
|
|
|
|
])->map(static fn ($row): array => (array) $row)->all()
|
|
|
|
|
|
=== $legacyNetworkClaimLedger,
|
|
|
|
|
|
'Legacy network task reached the Agent, remained claimable, or changed '
|
|
|
|
|
|
. 'the canonical instance/address ledgers.');
|
|
|
|
|
|
$worker->tick();
|
|
|
|
|
|
$assert(count($history) === $beforeLegacyWorker
|
|
|
|
|
|
&& (int) $legacyWorkerJob->fresh()->dispatch_attempts === 0,
|
|
|
|
|
|
'Legacy network task was replayed after local rejection.');
|
|
|
|
|
|
|
|
|
|
|
|
// A stale migration row that still carries the retired scalar public-IP
|
|
|
|
|
|
// request shape must fail before the accepted-to-dispatching CAS. The
|
|
|
|
|
|
// immutable migration/instance/reservation ledgers are the only authority
|
|
|
|
|
|
// for a PeerMigration receive request.
|
|
|
|
|
|
$legacyMigrationJob = new CloudJob();
|
|
|
|
|
|
$legacyMigrationJob->operation_id = bin2hex(random_bytes(16));
|
|
|
|
|
|
$legacyMigrationJob->request_idempotency_key =
|
|
|
|
|
|
'legacy-migration-worker-gate-0001';
|
|
|
|
|
|
$legacyMigrationJob->request_fingerprint = hash(
|
|
|
|
|
|
'sha256',
|
|
|
|
|
|
'legacy-migration-worker-gate-0001'
|
|
|
|
|
|
);
|
|
|
|
|
|
$legacyMigrationJob->instance_id = (int) $legacyNetworkInstance->id;
|
|
|
|
|
|
$legacyMigrationJob->user_id = (int) $legacyNetworkInstance->user_id;
|
|
|
|
|
|
$legacyMigrationJob->host_id = (int) $legacyNetworkInstance->host_id;
|
|
|
|
|
|
$legacyMigrationJob->kind = 'instance_migration_transfer';
|
|
|
|
|
|
$legacyMigrationJob->execution_mode = 'agent';
|
|
|
|
|
|
$legacyMigrationJob->status = 'queued';
|
|
|
|
|
|
$legacyMigrationStoredIdentity = [
|
|
|
|
|
|
'version' => 1,
|
|
|
|
|
|
'action' => 'peer-migration-receive',
|
|
|
|
|
|
'resource_id' => 'migration:987654',
|
|
|
|
|
|
'request_fingerprint' => str_repeat('a', 64),
|
|
|
|
|
|
];
|
|
|
|
|
|
$legacyMigrationJob->sealed_payload =
|
|
|
|
|
|
\App\Services\Cloud\CloudSecrets::encrypt([
|
|
|
|
|
|
\App\Services\Cloud\CloudOperationRecoveryIdentity::payloadKey()
|
|
|
|
|
|
=> $legacyMigrationStoredIdentity,
|
|
|
|
|
|
'_operation_recovery' => [
|
|
|
|
|
|
'version' => 1,
|
|
|
|
|
|
'kind' => 'instance_migration_transfer',
|
|
|
|
|
|
'prepared_marker' => 'prepared:test:legacy-migration',
|
|
|
|
|
|
'operation' => [
|
|
|
|
|
|
'operation_id' =>
|
|
|
|
|
|
(string) $legacyMigrationJob->operation_id,
|
|
|
|
|
|
'action' => $legacyMigrationStoredIdentity['action'],
|
|
|
|
|
|
'resource_id' =>
|
|
|
|
|
|
$legacyMigrationStoredIdentity['resource_id'],
|
|
|
|
|
|
'request_fingerprint' =>
|
|
|
|
|
|
$legacyMigrationStoredIdentity[
|
|
|
|
|
|
'request_fingerprint'
|
|
|
|
|
|
],
|
|
|
|
|
|
],
|
|
|
|
|
|
],
|
|
|
|
|
|
'method' => 'POST',
|
|
|
|
|
|
'request_target' => '/v1/vms/'
|
|
|
|
|
|
. rawurlencode((string) $legacyNetworkInstance->uuid)
|
|
|
|
|
|
. '/peer-migration/receive',
|
|
|
|
|
|
'body' => [
|
|
|
|
|
|
'operation_id' => (string) $legacyMigrationJob->operation_id,
|
|
|
|
|
|
'public_ip_cidr' => '9.9.9.10/24',
|
|
|
|
|
|
'gateway' => '9.9.9.1',
|
|
|
|
|
|
],
|
|
|
|
|
|
'maintenance_migration_id' => 987654,
|
|
|
|
|
|
'owner_id' => (int) $legacyNetworkInstance->user_id,
|
|
|
|
|
|
'can_manage' => false,
|
|
|
|
|
|
], 'job-payload');
|
|
|
|
|
|
$legacyMigrationJob->available_at = $legacyMigrationJob->created_at =
|
|
|
|
|
|
$legacyMigrationJob->updated_at = time();
|
|
|
|
|
|
$legacyMigrationJob->save();
|
|
|
|
|
|
$beforeLegacyMigration = count($history);
|
|
|
|
|
|
$assert($worker->tick(),
|
|
|
|
|
|
'Legacy migration task was not consumed by local admission.');
|
|
|
|
|
|
$legacyMigrationJob->refresh();
|
|
|
|
|
|
$assert(count($history) === $beforeLegacyMigration
|
|
|
|
|
|
&& (int) $legacyMigrationJob->dispatch_attempts === 0
|
|
|
|
|
|
&& $legacyMigrationJob->phase === 'terminal'
|
|
|
|
|
|
&& $legacyMigrationJob->outcome === 'failed',
|
|
|
|
|
|
'Legacy migration task reached the Agent or remained claimable.');
|
|
|
|
|
|
$worker->tick();
|
|
|
|
|
|
$assert(count($history) === $beforeLegacyMigration
|
|
|
|
|
|
&& (int) $legacyMigrationJob->fresh()->dispatch_attempts === 0,
|
|
|
|
|
|
'Legacy migration task was replayed after local rejection.');
|
|
|
|
|
|
|
|
|
|
|
|
// The ordinary job enqueue path is permanently closed to legacy scalar
|
|
|
|
|
|
// network changes. Canonical transitions have their own durable workflow
|
|
|
|
|
|
// and are tested in the dedicated begin/finalization behavior suites.
|
|
|
|
|
|
$networkInput = ['public_ip_cidr' => '9.9.9.10/24',
|
|
|
|
|
|
'public_pool_key' => $targetPoolKey, 'confirm_restart' => true];
|
|
|
|
|
|
$beforeLegacyNetworkJobs = CloudJob::count();
|
|
|
|
|
|
try {
|
|
|
|
|
|
CloudJobService::enqueue(
|
|
|
|
|
|
$instance->fresh(),
|
|
|
|
|
|
$actor,
|
|
|
|
|
|
'network_change',
|
|
|
|
|
|
'POST',
|
|
|
|
|
|
'/network',
|
|
|
|
|
|
$networkInput
|
|
|
|
|
|
);
|
|
|
|
|
|
throw new RuntimeException(
|
|
|
|
|
|
'Legacy scalar network writer bypassed the transition gate.'
|
|
|
|
|
|
);
|
|
|
|
|
|
} catch (CloudError $error) {
|
|
|
|
|
|
$assert($error->httpStatus === 409
|
|
|
|
|
|
&& CloudJob::count() === $beforeLegacyNetworkJobs,
|
|
|
|
|
|
'Transition gate created a network job before rejecting it.');
|
|
|
|
|
|
}
|
|
|
|
|
|
// Stage 5 keeps one executable legacy-gate assertion above; paid charge,
|
|
|
|
|
|
// idempotency, not-applied refund and unknown-effect recovery are covered
|
|
|
|
|
|
// by the canonical transition Workflow and Finalization behavior suites.
|
|
|
|
|
|
// Deletion consumes the unchanged exact dual-stack authority and never
|
|
|
|
|
|
// reconstructs it from retained singular reservation columns.
|
|
|
|
|
|
$canonicalClaimSet =
|
|
|
|
|
|
\App\Services\Cloud\CloudPublicNetworkAllocationService::currentClaimSet(
|
|
|
|
|
|
(int) $openingReservation->id,
|
|
|
|
|
|
'committed',
|
|
|
|
|
|
(int) $instance->id
|
|
|
|
|
|
);
|
|
|
|
|
|
$canonicalClaimIds = [
|
|
|
|
|
|
$canonicalClaimSet->claimId(4),
|
|
|
|
|
|
$canonicalClaimSet->claimId(6),
|
|
|
|
|
|
];
|
|
|
|
|
|
$canonicalPublicIpIds = [
|
|
|
|
|
|
$canonicalClaimSet->publicIpId(4),
|
|
|
|
|
|
$canonicalClaimSet->publicIpId(6),
|
|
|
|
|
|
];
|
|
|
|
|
|
|
|
|
|
|
|
$interruptedDelete = CloudJobService::enqueue($instance, $actor,
|
|
|
|
|
|
'delete', 'DELETE', '',
|
|
|
|
|
|
['remove_managed_disks' => true]);
|
|
|
|
|
|
$mock->append(new Response(422, [], json_encode([
|
|
|
|
|
|
'operation_id' => $interruptedDelete->operation_id,
|
|
|
|
|
|
'status' => 'failed',
|
|
|
|
|
|
...CloudOperationRecoveryIdentity::expected($interruptedDelete),
|
|
|
|
|
|
'error' => ['code' => 'graceful_shutdown_unconfirmed'],
|
|
|
|
|
|
'effect_may_have_applied' => false,
|
|
|
|
|
|
'safe_to_resubmit' => true,
|
|
|
|
|
|
'replayed' => false,
|
|
|
|
|
|
'retryable' => false,
|
|
|
|
|
|
])));
|
|
|
|
|
|
$worker->tick();
|
|
|
|
|
|
$interruptedDelete->refresh();
|
|
|
|
|
|
$assert($interruptedDelete->status === 'failed'
|
|
|
|
|
|
&& $interruptedDelete->error_code === 'graceful_shutdown_unconfirmed'
|
|
|
|
|
|
&& $instance->fresh()->status === 'active',
|
|
|
|
|
|
'An unconfirmed graceful delete did not close safely for a new normal retry.');
|
|
|
|
|
|
|
|
|
|
|
|
$delete = CloudJobService::enqueue($instance->fresh(), $actor,
|
|
|
|
|
|
'delete', 'DELETE', '',
|
|
|
|
|
|
['remove_managed_disks' => true]);
|
|
|
|
|
|
$assert($delete->operation_id !== $interruptedDelete->operation_id,
|
|
|
|
|
|
'A graceful retry reused the interrupted delete operation identifier.');
|
|
|
|
|
|
$mock->append(new Response(409, [], json_encode([
|
|
|
|
|
|
'operation_id' => $delete->operation_id,
|
|
|
|
|
|
'status' => 'failed',
|
|
|
|
|
|
...CloudOperationRecoveryIdentity::expected($delete),
|
|
|
|
|
|
'error' => ['code' => 'force_required'],
|
|
|
|
|
|
'replayed' => false,
|
|
|
|
|
|
'retryable' => false,
|
|
|
|
|
|
])));
|
|
|
|
|
|
$worker->tick();
|
|
|
|
|
|
$graceful = $agentRequest(end($history));
|
|
|
|
|
|
$delete->refresh();
|
|
|
|
|
|
$assert(json_decode($graceful['body'], true)
|
|
|
|
|
|
=== ['remove_managed_disks' => true]
|
|
|
|
|
|
&& !str_contains($graceful['request_target'], '?')
|
|
|
|
|
|
&& $delete->phase === 'terminal'
|
|
|
|
|
|
&& $delete->outcome === 'failed'
|
|
|
|
|
|
&& $delete->status === 'failed'
|
|
|
|
|
|
&& $delete->error_code === 'force_required'
|
|
|
|
|
|
&& $instance->fresh()->status === 'active',
|
|
|
|
|
|
'Graceful delete did not stop at the explicit force-required boundary.');
|
|
|
|
|
|
|
|
|
|
|
|
$failedForceDelete = CloudJobService::enqueueForceDelete(
|
|
|
|
|
|
$delete,
|
|
|
|
|
|
$adminActor
|
|
|
|
|
|
);
|
|
|
|
|
|
$mock->append(new Response(422, [], json_encode([
|
|
|
|
|
|
'operation_id' => $failedForceDelete->operation_id,
|
|
|
|
|
|
'status' => 'failed',
|
|
|
|
|
|
...CloudOperationRecoveryIdentity::expected($failedForceDelete),
|
|
|
|
|
|
'error' => ['code' => 'invalid_request'],
|
|
|
|
|
|
'effect_may_have_applied' => false,
|
|
|
|
|
|
'safe_to_resubmit' => true,
|
|
|
|
|
|
'replayed' => false,
|
|
|
|
|
|
'retryable' => false,
|
|
|
|
|
|
])));
|
|
|
|
|
|
$worker->tick();
|
|
|
|
|
|
$delete->refresh();
|
|
|
|
|
|
$failedForceDelete->refresh();
|
|
|
|
|
|
$assert($failedForceDelete->status === 'failed'
|
|
|
|
|
|
&& $delete->status === 'failed'
|
|
|
|
|
|
&& $delete->error_code === 'force_required'
|
|
|
|
|
|
&& $instance->fresh()->status === 'active',
|
|
|
|
|
|
'A definitely failed force attempt did not safely reopen the explicit force gate.');
|
|
|
|
|
|
|
|
|
|
|
|
$forceDelete = CloudJobService::enqueueForceDelete(
|
|
|
|
|
|
$delete,
|
|
|
|
|
|
$adminActor
|
|
|
|
|
|
);
|
|
|
|
|
|
$mock->append(new Response(200, [], json_encode([
|
|
|
|
|
|
'operation_id' => $forceDelete->operation_id,
|
|
|
|
|
|
'status' => 'succeeded',
|
|
|
|
|
|
...CloudOperationRecoveryIdentity::expected($forceDelete, true),
|
|
|
|
|
|
'result' => [
|
|
|
|
|
|
'uuid' => (string) $instance->uuid,
|
|
|
|
|
|
'deleted' => true,
|
|
|
|
|
|
'managed_artifacts_absent' => true,
|
|
|
|
|
|
'forced' => true,
|
|
|
|
|
|
],
|
|
|
|
|
|
])));
|
|
|
|
|
|
$worker->tick();
|
|
|
|
|
|
$forced = $agentRequest(end($history));
|
|
|
|
|
|
$forcedBody = json_decode($forced['body'], true);
|
|
|
|
|
|
$retainedDeleteReservation = CloudResourceReservation::where(
|
|
|
|
|
|
'order_id', $first->id
|
|
|
|
|
|
)->first();
|
|
|
|
|
|
$assert($forced['method'] === 'POST'
|
|
|
|
|
|
&& str_ends_with($forced['request_target'], '/force-delete')
|
|
|
|
|
|
&& $forceDelete->operation_id !== $failedForceDelete->operation_id
|
|
|
|
|
|
&& ($forcedBody['operation_id'] ?? null) === $forceDelete->operation_id
|
|
|
|
|
|
&& ($forcedBody['prior_operation_id'] ?? null) === $delete->operation_id
|
|
|
|
|
|
&& ($forcedBody['ack_data_loss'] ?? null) === true
|
|
|
|
|
|
&& ($forcedBody['remove_managed_disks'] ?? null) === true
|
|
|
|
|
|
&& $forceDelete->fresh()->status === 'succeeded'
|
|
|
|
|
|
&& $delete->fresh()->status === 'failed'
|
|
|
|
|
|
&& $delete->fresh()->error_code === 'force_required'
|
|
|
|
|
|
&& $instance->fresh()->status === 'deleted'
|
|
|
|
|
|
&& $instance->fresh()->sealed_credential === null
|
|
|
|
|
|
&& $retainedDeleteReservation->state === 'released'
|
|
|
|
|
|
&& CloudPublicIp::whereIn('id', $canonicalPublicIpIds)
|
|
|
|
|
|
->where('state', 'available')
|
|
|
|
|
|
->whereNull('reservation_id')
|
|
|
|
|
|
->whereNull('reservation_ip_claim_id')
|
|
|
|
|
|
->whereNull('instance_id')
|
|
|
|
|
|
->whereNull('operation_id')->count() === 2
|
|
|
|
|
|
&& Capsule::table('cloud_resource_reservation_ips')
|
|
|
|
|
|
->whereIn('id', $canonicalClaimIds)
|
|
|
|
|
|
->where('state', 'released')
|
|
|
|
|
|
->whereNull('active_claim')->count() === 2,
|
|
|
|
|
|
'Force-delete identity, acknowledgement, cleanup, or immutable prior receipt failed.');
|
|
|
|
|
|
$beforeForceReplay = count($history);
|
|
|
|
|
|
$worker->tick();
|
|
|
|
|
|
$assert(count($history) === $beforeForceReplay
|
|
|
|
|
|
&& $forceDelete->fresh()->phase === 'terminal'
|
|
|
|
|
|
&& (int) $forceDelete->fresh()->dispatch_attempts === 1,
|
|
|
|
|
|
'Successful force-delete task was automatically re-dispatched.');
|
|
|
|
|
|
|
|
|
|
|
|
// Independent backup restores have no purchase order. Their committed
|
|
|
|
|
|
// capacity and fresh IP must still be released after a proven deletion.
|
|
|
|
|
|
$restoreOperationId = str_repeat('e', 32);
|
|
|
|
|
|
$restoredUuid = 'eeeeeeee-eeee-4eee-8eee-eeeeeeeeeeee';
|
|
|
|
|
|
$backupRestore = new CloudBackupRestore();
|
|
|
|
|
|
$backupRestore->restore_key = str_repeat('f', 32);
|
|
|
|
|
|
$backupRestore->backup_id = 9001;
|
|
|
|
|
|
$backupRestore->target_host_id = (int) $host->id;
|
|
|
|
|
|
$backupRestore->target_instance_id = null;
|
|
|
|
|
|
$backupRestore->reservation_id = null;
|
|
|
|
|
|
$backupRestore->job_id = null;
|
|
|
|
|
|
$backupRestore->operation_id = $restoreOperationId;
|
|
|
|
|
|
$backupRestore->new_uuid = $restoredUuid;
|
|
|
|
|
|
$backupRestore->new_name = 'restored-delete-vm';
|
|
|
|
|
|
$backupRestore->state = 'ready';
|
|
|
|
|
|
$backupRestore->error_code = '';
|
|
|
|
|
|
$backupRestore->ready_at = time();
|
|
|
|
|
|
$backupRestore->created_at = $backupRestore->updated_at = time();
|
|
|
|
|
|
$backupRestore->save();
|
|
|
|
|
|
$restoredIp = new CloudPublicIp();
|
|
|
|
|
|
$restoredIp->host_id = (int) $host->id;
|
|
|
|
|
|
$restoredIp->public_ip_pool_id = (int) $targetPublicPool->id;
|
|
|
|
|
|
$restoredIp->address_family = 4;
|
|
|
|
|
|
$restoredIp->address = '9.9.9.12';
|
|
|
|
|
|
$restoredIp->address_binary = inet_pton('9.9.9.12');
|
|
|
|
|
|
$restoredIp->prefix_length = 24;
|
|
|
|
|
|
$restoredIp->state = 'assigned';
|
|
|
|
|
|
$restoredIp->reservation_id = null;
|
|
|
|
|
|
$restoredIp->instance_id = null;
|
|
|
|
|
|
$restoredIp->operation_id = null;
|
|
|
|
|
|
$restoredIp->created_at = $restoredIp->updated_at = time();
|
|
|
|
|
|
$restoredIp->save();
|
|
|
|
|
|
$restoredReservation = new CloudResourceReservation();
|
|
|
|
|
|
$restoredReservation->reservation_key = str_repeat('d', 32);
|
|
|
|
|
|
$restoredReservation->operation_id = $restoreOperationId;
|
|
|
|
|
|
$restoredReservation->order_id = null;
|
|
|
|
|
|
$restoredReservation->backup_restore_id = (int) $backupRestore->id;
|
|
|
|
|
|
$restoredReservation->offer_id = null;
|
|
|
|
|
|
$restoredReservation->resource_pool_id = (int) $pool->id;
|
|
|
|
|
|
$restoredReservation->host_id = (int) $host->id;
|
|
|
|
|
|
$restoredReservation->user_id = 1;
|
|
|
|
|
|
$restoredReservation->instance_id = null;
|
|
|
|
|
|
$restoredReservation->network_attachment_id =
|
|
|
|
|
|
(int) $targetAttachment->id;
|
|
|
|
|
|
$restoredReservation->public_ip_pool_id = (int) $targetPublicPool->id;
|
|
|
|
|
|
$restoredReservation->public_ip_id = (int) $restoredIp->id;
|
|
|
|
|
|
$restoredReservation->cpu_cores = 2;
|
|
|
|
|
|
$restoredReservation->memory_mib = 2048;
|
|
|
|
|
|
$restoredReservation->disk_gib = 20;
|
|
|
|
|
|
$restoredReservation->state = 'committed';
|
|
|
|
|
|
$restoredReservation->expires_at = null;
|
|
|
|
|
|
$restoredReservation->released_at = null;
|
|
|
|
|
|
$restoredReservation->release_reason = '';
|
|
|
|
|
|
$restoredReservation->created_at = $restoredReservation->updated_at = time();
|
|
|
|
|
|
$restoredReservation->save();
|
|
|
|
|
|
$restoredInstance = $instance->replicate();
|
|
|
|
|
|
$restoredInstance->uuid = $restoredUuid;
|
|
|
|
|
|
$restoredInstance->host_id = (int) $host->id;
|
|
|
|
|
|
$restoredInstance->user_id = 1;
|
|
|
|
|
|
$restoredInstance->order_id = null;
|
|
|
|
|
|
$restoredInstance->plan_id = null;
|
|
|
|
|
|
$restoredInstance->offer_id = null;
|
|
|
|
|
|
$restoredInstance->resource_pool_id = (int) $pool->id;
|
|
|
|
|
|
$restoredInstance->reservation_id = (int) $restoredReservation->id;
|
|
|
|
|
|
$restoredInstance->current_period_id = null;
|
|
|
|
|
|
$restoredInstance->name = 'restored-delete-vm';
|
|
|
|
|
|
$restoredInstance->status = 'active';
|
|
|
|
|
|
$restoredInstance->service_status = 'exempt';
|
|
|
|
|
|
$restoredInstance->network_status = 'active';
|
|
|
|
|
|
$restoredInstance->expires_at = null;
|
|
|
|
|
|
$restoredInstance->spec_json = json_encode([
|
|
|
|
|
|
'managed' => true,
|
|
|
|
|
|
'vcpu' => 2,
|
|
|
|
|
|
'memory_mib' => 2048,
|
|
|
|
|
|
'disk_gib' => 20,
|
|
|
|
|
|
'public_ip_cidr' => '9.9.9.12/24',
|
|
|
|
|
|
], JSON_THROW_ON_ERROR);
|
|
|
|
|
|
$restoredInstance->sealed_credential = 'sealed-restored-credential';
|
|
|
|
|
|
$restoredInstance->created_at = $restoredInstance->updated_at = time();
|
|
|
|
|
|
$restoredInstance->save();
|
|
|
|
|
|
$restoredReservation->instance_id = (int) $restoredInstance->id;
|
|
|
|
|
|
$restoredReservation->save();
|
|
|
|
|
|
$restoredIp->reservation_id = (int) $restoredReservation->id;
|
|
|
|
|
|
$restoredIp->instance_id = (int) $restoredInstance->id;
|
|
|
|
|
|
$restoredIp->operation_id = $restoreOperationId;
|
|
|
|
|
|
$restoredIp->save();
|
|
|
|
|
|
$backupRestore->target_instance_id = (int) $restoredInstance->id;
|
|
|
|
|
|
$backupRestore->reservation_id = (int) $restoredReservation->id;
|
|
|
|
|
|
$backupRestore->save();
|
|
|
|
|
|
$beforeRestoredDeleteJobs = CloudJob::count();
|
|
|
|
|
|
$beforeRestoredDeleteRequests = count($history);
|
|
|
|
|
|
try {
|
|
|
|
|
|
CloudJobService::enqueue(
|
|
|
|
|
|
$restoredInstance,
|
|
|
|
|
|
$adminActor,
|
|
|
|
|
|
'delete',
|
|
|
|
|
|
'DELETE',
|
|
|
|
|
|
'',
|
|
|
|
|
|
['remove_managed_disks' => true]
|
|
|
|
|
|
);
|
|
|
|
|
|
throw new RuntimeException(
|
|
|
|
|
|
'A restored instance bypassed the dual-stack transition gate.'
|
|
|
|
|
|
);
|
|
|
|
|
|
} catch (CloudError $error) {
|
|
|
|
|
|
$assert($error->httpStatus === 409,
|
|
|
|
|
|
'Restored-instance delete gate returned the wrong status.');
|
|
|
|
|
|
}
|
|
|
|
|
|
$restoredInstance->refresh();
|
|
|
|
|
|
$restoredReservation->refresh();
|
|
|
|
|
|
$restoredIp->refresh();
|
|
|
|
|
|
$assert(CloudJob::count() === $beforeRestoredDeleteJobs
|
|
|
|
|
|
&& count($history) === $beforeRestoredDeleteRequests
|
|
|
|
|
|
&& $restoredInstance->status === 'active'
|
|
|
|
|
|
&& $restoredReservation->state === 'committed'
|
|
|
|
|
|
&& $restoredIp->state === 'assigned'
|
|
|
|
|
|
&& (int) $restoredIp->reservation_id
|
|
|
|
|
|
=== (int) $restoredReservation->id
|
|
|
|
|
|
&& (int) $restoredIp->instance_id === (int) $restoredInstance->id,
|
|
|
|
|
|
'Restored-instance delete gate created work or changed its old ledger.');
|
|
|
|
|
|
$restoredInstance->status = 'deleting';
|
|
|
|
|
|
$restoredInstance->service_status = 'purge_deleting';
|
|
|
|
|
|
$restoredInstance->save();
|
|
|
|
|
|
try {
|
|
|
|
|
|
CloudJobService::enqueueExpiryDelete($restoredInstance);
|
|
|
|
|
|
throw new RuntimeException(
|
|
|
|
|
|
'A restored instance bypassed the expiry-delete transition gate.'
|
|
|
|
|
|
);
|
|
|
|
|
|
} catch (CloudError $error) {
|
|
|
|
|
|
$assert($error->httpStatus === 409,
|
|
|
|
|
|
'Restored-instance expiry-delete gate returned the wrong status.');
|
|
|
|
|
|
}
|
|
|
|
|
|
$assert(CloudJob::count() === $beforeRestoredDeleteJobs
|
|
|
|
|
|
&& count($history) === $beforeRestoredDeleteRequests,
|
|
|
|
|
|
'Restored-instance expiry-delete gate created durable work.');
|
|
|
|
|
|
// This clean-break fixture intentionally models the retired singular
|
|
|
|
|
|
// restore shape. Remove it explicitly after proving the gate so it cannot
|
|
|
|
|
|
// poison later active-pair placement scans in this long scenario.
|
|
|
|
|
|
$restoredInstance->delete();
|
|
|
|
|
|
$restoredIp->delete();
|
|
|
|
|
|
$restoredReservation->delete();
|
|
|
|
|
|
$backupRestore->delete();
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
// Provider callbacks credit the wallet before the invoice bridge runs.
|
|
|
|
|
|
// That credit must be consumed once, queue one create job, and remain
|
|
|
|
|
|
// idempotent when the payment provider retries its callback.
|
|
|
|
|
|
$providerOrder = CloudOrderService::purchase(array_replace($input, [
|
|
|
|
|
|
'name' => 'vm-provider-payment',
|
|
|
|
|
|
'idempotency_key' => 'test-request-key-provider-payment',
|
|
|
|
|
|
]), $actor);
|
|
|
|
|
|
$providerInvoice = CloudInvoice::where('order_id', $providerOrder->id)->first();
|
|
|
|
|
|
$providerPayment = new CloudInvoicePayment();
|
|
|
|
|
|
$providerPayment->invoice_id = $providerInvoice->id;
|
|
|
|
|
|
$providerPayment->user_id = 1;
|
|
|
|
|
|
$providerPayment->provider = 'paytaro';
|
|
|
|
|
|
$providerPayment->provider_order_no = 'PT-PROVISION-0001';
|
|
|
|
|
|
$providerPayment->idempotency_key = 'provision-provider-payment-0001';
|
|
|
|
|
|
$providerPayment->amount = '9.99';
|
|
|
|
|
|
$providerPayment->status = 'pending';
|
|
|
|
|
|
$providerPayment->created_at = $providerPayment->updated_at = time();
|
|
|
|
|
|
$providerPayment->save();
|
|
|
|
|
|
$balanceBeforeProviderCredit = CloudMoney::cents((string) User::find(1)->money);
|
|
|
|
|
|
$user = User::find(1);
|
|
|
|
|
|
$user->money = CloudMoney::decimal($balanceBeforeProviderCredit + 999);
|
|
|
|
|
|
$user->save();
|
|
|
|
|
|
$providerOutcome = CloudInvoicePaymentService::settleForProvider(
|
|
|
|
|
|
'paytaro', 'PT-PROVISION-0001');
|
|
|
|
|
|
$providerOutcomeAgain = CloudInvoicePaymentService::settleForProvider(
|
|
|
|
|
|
'paytaro', 'PT-PROVISION-0001');
|
|
|
|
|
|
$providerQueuedCreate = \App\Services\Cloud\CloudSecrets::decrypt(
|
|
|
|
|
|
(string) CloudJob::where('order_id', $providerOrder->id)
|
|
|
|
|
|
->first()->sealed_payload,
|
|
|
|
|
|
'job-payload'
|
|
|
|
|
|
);
|
|
|
|
|
|
$providerPolicy = $providerQueuedCreate['body']
|
|
|
|
|
|
['platform_network_policy_snapshot'] ?? null;
|
|
|
|
|
|
$assert($providerOutcome === 'paid_invoice'
|
|
|
|
|
|
&& $providerOutcomeAgain === 'paid_invoice'
|
|
|
|
|
|
&& CloudMoney::cents((string) User::find(1)->money) === $balanceBeforeProviderCredit
|
|
|
|
|
|
&& CloudInvoice::find($providerInvoice->id)->status === 'paid'
|
|
|
|
|
|
&& CloudOrder::find($providerOrder->id)->status === 'queued'
|
|
|
|
|
|
&& CloudJob::where('order_id', $providerOrder->id)->count() === 1
|
|
|
|
|
|
&& ($providerPolicy['catalog_revision'] ?? null) === 2
|
|
|
|
|
|
&& count($providerPolicy['rules'] ?? []) === 9,
|
|
|
|
|
|
'Provider callback did not settle an opening invoice exactly once.');
|
|
|
|
|
|
|
|
|
|
|
|
// A blank optional label is normalized before hashing, but generated only
|
|
|
|
|
|
// for a genuinely new order. Replaying the same request with an explicit
|
|
|
|
|
|
// empty field must therefore return the original generated display name.
|
|
|
|
|
|
$automaticInput = array_replace($input, [
|
|
|
|
|
|
'idempotency_key' => 'test-request-key-automatic-name',
|
|
|
|
|
|
]);
|
|
|
|
|
|
unset($automaticInput['name']);
|
|
|
|
|
|
$automatic = CloudOrderService::purchase($automaticInput, $actor);
|
|
|
|
|
|
$automaticReplay = CloudOrderService::purchase(
|
|
|
|
|
|
$automaticInput + ['name' => ''], $actor
|
|
|
|
|
|
);
|
|
|
|
|
|
$automaticSpec = json_decode((string) $automatic->spec_json, true,
|
|
|
|
|
|
flags: JSON_THROW_ON_ERROR);
|
|
|
|
|
|
$assert($automatic->id === $automaticReplay->id
|
|
|
|
|
|
&& preg_match('/^vm-[abcdefghjkmnpqrstuvwxyz23456789]{6}$/D',
|
|
|
|
|
|
(string) ($automaticSpec['name'] ?? '')) === 1,
|
|
|
|
|
|
'Blank instance names were not generated securely or idempotently.');
|
|
|
|
|
|
|
|
|
|
|
|
// Repeated provider settlement reuses the one accepted provisioning task.
|
|
|
|
|
|
// It never creates a recovery generation or an automatic retry.
|
|
|
|
|
|
$providerJob = CloudJob::where('order_id', $providerOrder->id)->first();
|
|
|
|
|
|
$providerOrderAgain = CloudOrderService::queuePaidProvision(
|
|
|
|
|
|
CloudInvoice::find($providerInvoice->id),
|
|
|
|
|
|
CloudOrder::find($providerOrder->id)
|
|
|
|
|
|
);
|
|
|
|
|
|
$assert($providerJob !== null
|
|
|
|
|
|
&& $providerJob->phase === 'accepted'
|
|
|
|
|
|
&& $providerJob->outcome === null
|
|
|
|
|
|
&& $providerJob->status === 'queued'
|
|
|
|
|
|
&& (int) $providerJob->dispatch_attempts === 0
|
|
|
|
|
|
&& (string) $providerOrderAgain->status === 'queued'
|
|
|
|
|
|
&& CloudJob::where('order_id', $providerOrder->id)->count() === 1,
|
|
|
|
|
|
'A repeated paid callback duplicated or retried its durable create task.');
|
|
|
|
|
|
|
|
|
|
|
|
// A definitively not-applied lifecycle failure starts a new audited
|
|
|
|
|
|
// generation after backoff. The failed job and its operation ID remain
|
|
|
|
|
|
// immutable; unknown, applied, or sticky failures remain fail-closed.
|
|
|
|
|
|
$lifecycle = new CloudLifecycleService();
|
|
|
|
|
|
$reconcile = (new ReflectionClass(CloudLifecycleService::class))
|
|
|
|
|
|
->getMethod('reconcile');
|
|
|
|
|
|
$oldPurgeOperation = (string) $purgeStopFailedJob->operation_id;
|
|
|
|
|
|
$purgeCount = CloudJob::where('instance_id', $purgeStopFailed->id)
|
|
|
|
|
|
->where('kind', 'service_suspend')->count();
|
|
|
|
|
|
$assert($reconcile->invoke($lifecycle, (int) $purgeStopFailed->id) === false
|
|
|
|
|
|
&& CloudJob::where('instance_id', $purgeStopFailed->id)
|
|
|
|
|
|
->where('kind', 'service_suspend')->count() === $purgeCount,
|
|
|
|
|
|
'Lifecycle policy failure ignored the mandatory retry backoff.');
|
|
|
|
|
|
Capsule::table('cloud_jobs')->where('id', $purgeStopFailedJob->id)
|
|
|
|
|
|
->update(['terminal_at' => time() - 310, 'updated_at' => time() - 310]);
|
|
|
|
|
|
$assert($reconcile->invoke($lifecycle, (int) $purgeStopFailed->id) === true,
|
|
|
|
|
|
'A definitively not-applied purge stop did not create a new generation: '
|
|
|
|
|
|
. json_encode($purgeStopFailedJob->fresh()->toArray(),
|
|
|
|
|
|
JSON_THROW_ON_ERROR));
|
|
|
|
|
|
$purgeRetry = CloudJob::where('instance_id', $purgeStopFailed->id)
|
|
|
|
|
|
->where('kind', 'service_suspend')->orderByDesc('id')->first();
|
|
|
|
|
|
$assert($purgeRetry !== null
|
|
|
|
|
|
&& (int) $purgeRetry->id !== (int) $purgeStopFailedJob->id
|
|
|
|
|
|
&& (string) $purgeRetry->operation_id !== $oldPurgeOperation
|
|
|
|
|
|
&& (string) $purgeStopFailedJob->fresh()->operation_id
|
|
|
|
|
|
=== $oldPurgeOperation
|
|
|
|
|
|
&& (string) $purgeStopFailedJob->fresh()->phase === 'terminal'
|
|
|
|
|
|
&& (string) $purgeStopFailed->fresh()->service_status
|
|
|
|
|
|
=== 'purge_stopping',
|
|
|
|
|
|
'Purge retry replayed or rewrote the failed generation.');
|
|
|
|
|
|
|
|
|
|
|
|
$oldDeleteOperation = (string) $expiryDeleteFailedJob->operation_id;
|
|
|
|
|
|
Capsule::table('cloud_jobs')->where('id', $expiryDeleteFailedJob->id)
|
|
|
|
|
|
->update(['terminal_at' => time() - 310, 'updated_at' => time() - 310]);
|
|
|
|
|
|
$assert($reconcile->invoke($lifecycle, (int) $expiryDeleteFailed->id) === true,
|
|
|
|
|
|
'A definitively not-applied expiry delete did not create a new generation.');
|
|
|
|
|
|
$deleteRetry = CloudJob::where('instance_id', $expiryDeleteFailed->id)
|
|
|
|
|
|
->where('kind', 'expiry_delete')->orderByDesc('id')->first();
|
|
|
|
|
|
$assert($deleteRetry !== null
|
|
|
|
|
|
&& (int) $deleteRetry->id !== (int) $expiryDeleteFailedJob->id
|
|
|
|
|
|
&& (string) $deleteRetry->operation_id !== $oldDeleteOperation
|
|
|
|
|
|
&& (string) $expiryDeleteFailedJob->fresh()->operation_id
|
|
|
|
|
|
=== $oldDeleteOperation
|
|
|
|
|
|
&& (string) $expiryDeleteFailedJob->fresh()->phase === 'terminal'
|
|
|
|
|
|
&& (string) $expiryDeleteFailed->fresh()->status === 'deleting'
|
|
|
|
|
|
&& (string) $expiryDeleteFailed->fresh()->service_status
|
|
|
|
|
|
=== 'purge_deleting',
|
|
|
|
|
|
'Expiry-delete retry replayed or rewrote the failed generation.');
|
|
|
|
|
|
|
|
|
|
|
|
$finalizeUndispatchedFailure = static function (
|
|
|
|
|
|
CloudJob $job
|
|
|
|
|
|
): CloudJob {
|
|
|
|
|
|
$past = time() - 310;
|
|
|
|
|
|
$job->attempts = 0;
|
|
|
|
|
|
$job->dispatch_attempts = 0;
|
|
|
|
|
|
$job->dispatch_started_at = null;
|
|
|
|
|
|
$job->agent_accepted_at = null;
|
|
|
|
|
|
$job->dispatched_at = null;
|
|
|
|
|
|
$job->receipt_deadline_at = null;
|
|
|
|
|
|
$job->completion_event_id = null;
|
|
|
|
|
|
$job->phase = 'terminal';
|
|
|
|
|
|
$job->outcome = 'failed';
|
|
|
|
|
|
$job->status = 'failed';
|
|
|
|
|
|
$job->terminal_at = $past;
|
|
|
|
|
|
$job->effect_may_have_applied = 0;
|
|
|
|
|
|
$job->safe_to_resubmit = 1;
|
|
|
|
|
|
$job->effect_resolution = 'not_applied';
|
|
|
|
|
|
$job->error_code = 'dispatch_not_sent';
|
|
|
|
|
|
$job->finalization_incident = 0;
|
|
|
|
|
|
$job->finalization_error_code = null;
|
|
|
|
|
|
$job->updated_at = $past;
|
|
|
|
|
|
$job->save();
|
|
|
|
|
|
return $job;
|
|
|
|
|
|
};
|
|
|
|
|
|
$undispatchedDelete = $makeLifecycleInstance(
|
|
|
|
|
|
'90909090-9090-4090-8090-909090909090',
|
|
|
|
|
|
'undispatched-delete-vm',
|
|
|
|
|
|
'purge_deleting',
|
|
|
|
|
|
'blocked_expired'
|
|
|
|
|
|
);
|
|
|
|
|
|
$undispatchedDelete->status = 'deleting';
|
|
|
|
|
|
$undispatchedDelete->save();
|
|
|
|
|
|
$oldUndispatchedDelete = $finalizeUndispatchedFailure(
|
|
|
|
|
|
CloudJobService::enqueueExpiryDelete($undispatchedDelete)
|
|
|
|
|
|
);
|
|
|
|
|
|
$undispatchedDelete->status = 'delete_failed';
|
|
|
|
|
|
$undispatchedDelete->service_status = 'purge_failed';
|
|
|
|
|
|
$undispatchedDelete->save();
|
|
|
|
|
|
$assert($reconcile->invoke(
|
|
|
|
|
|
$lifecycle,
|
|
|
|
|
|
(int) $undispatchedDelete->id
|
|
|
|
|
|
) === true, 'A generation-zero expiry failure did not create a safe new generation.');
|
|
|
|
|
|
$undispatchedDeleteRetry = CloudJob::where(
|
|
|
|
|
|
'instance_id',
|
|
|
|
|
|
$undispatchedDelete->id
|
|
|
|
|
|
)->where('kind', 'expiry_delete')->orderByDesc('id')->first();
|
|
|
|
|
|
$assert($undispatchedDeleteRetry !== null
|
|
|
|
|
|
&& (int) $undispatchedDeleteRetry->id
|
|
|
|
|
|
!== (int) $oldUndispatchedDelete->id
|
|
|
|
|
|
&& (string) $undispatchedDeleteRetry->operation_id
|
|
|
|
|
|
!== (string) $oldUndispatchedDelete->operation_id
|
|
|
|
|
|
&& (int) $oldUndispatchedDelete->fresh()->dispatch_attempts === 0,
|
|
|
|
|
|
'Generation-zero expiry retry replayed or rewrote the prior operation.');
|
|
|
|
|
|
|
|
|
|
|
|
$undispatchedPurgeStop = $makeLifecycleInstance(
|
|
|
|
|
|
'97979797-9797-4797-8797-979797979797',
|
|
|
|
|
|
'undispatched-purge-stop-vm',
|
|
|
|
|
|
'purge_stopping',
|
|
|
|
|
|
'blocked_expired'
|
|
|
|
|
|
);
|
|
|
|
|
|
$undispatchedPurgeStop->status = 'deleting';
|
|
|
|
|
|
$undispatchedPurgeStop->save();
|
|
|
|
|
|
$oldUndispatchedPurgeStop = $finalizeUndispatchedFailure($queuePolicy(
|
|
|
|
|
|
$undispatchedPurgeStop,
|
|
|
|
|
|
'service_suspend',
|
|
|
|
|
|
'service/suspend',
|
|
|
|
|
|
'purge'
|
|
|
|
|
|
));
|
|
|
|
|
|
$undispatchedPurgeStop->service_status = 'purge_stop_failed';
|
|
|
|
|
|
$undispatchedPurgeStop->save();
|
|
|
|
|
|
$assert($reconcile->invoke(
|
|
|
|
|
|
$lifecycle,
|
|
|
|
|
|
(int) $undispatchedPurgeStop->id
|
|
|
|
|
|
) === true, 'A generation-zero purge stop did not create a safe new generation.');
|
|
|
|
|
|
$undispatchedPurgeStopRetry = CloudJob::where(
|
|
|
|
|
|
'instance_id',
|
|
|
|
|
|
$undispatchedPurgeStop->id
|
|
|
|
|
|
)->where('kind', 'service_suspend')->orderByDesc('id')->first();
|
|
|
|
|
|
$assert($undispatchedPurgeStopRetry !== null
|
|
|
|
|
|
&& (int) $undispatchedPurgeStopRetry->id
|
|
|
|
|
|
!== (int) $oldUndispatchedPurgeStop->id
|
|
|
|
|
|
&& (string) $undispatchedPurgeStopRetry->operation_id
|
|
|
|
|
|
!== (string) $oldUndispatchedPurgeStop->operation_id
|
|
|
|
|
|
&& (int) $oldUndispatchedPurgeStop->fresh()->dispatch_attempts === 0
|
|
|
|
|
|
&& (string) $undispatchedPurgeStop->fresh()->service_status
|
|
|
|
|
|
=== 'purge_stopping',
|
|
|
|
|
|
'Generation-zero purge-stop retry replayed or rewrote the prior operation.');
|
|
|
|
|
|
|
|
|
|
|
|
$preparePolicyFailure = static function (
|
|
|
|
|
|
CloudInstance $target,
|
|
|
|
|
|
string $kind,
|
|
|
|
|
|
string $suffix,
|
|
|
|
|
|
?string $reason
|
|
|
|
|
|
) use ($queuePolicy): CloudJob {
|
|
|
|
|
|
$job = $queuePolicy($target, $kind, $suffix, $reason);
|
|
|
|
|
|
$identity = CloudOperationRecoveryIdentity::expected($job);
|
|
|
|
|
|
$past = time() - 310;
|
|
|
|
|
|
$job->request_fingerprint = $identity['request_fingerprint'];
|
|
|
|
|
|
$job->attempts = 1;
|
|
|
|
|
|
$job->dispatch_attempts = 1;
|
|
|
|
|
|
$job->dispatch_started_at = $past - 30;
|
|
|
|
|
|
$job->agent_accepted_at = $past - 20;
|
|
|
|
|
|
$job->dispatched_at = $past - 20;
|
|
|
|
|
|
$job->receipt_deadline_at = $past - 10;
|
|
|
|
|
|
$job->phase = 'terminal';
|
|
|
|
|
|
$job->outcome = 'failed';
|
|
|
|
|
|
$job->status = 'failed';
|
|
|
|
|
|
$job->terminal_at = $past;
|
|
|
|
|
|
$job->effect_may_have_applied = 0;
|
|
|
|
|
|
$job->safe_to_resubmit = 1;
|
|
|
|
|
|
$job->effect_resolution = 'not_applied';
|
|
|
|
|
|
$job->error_code = 'operation_failed';
|
|
|
|
|
|
$job->finalization_incident = 0;
|
|
|
|
|
|
$job->finalization_error_code = null;
|
|
|
|
|
|
$job->updated_at = $past;
|
|
|
|
|
|
$job->save();
|
|
|
|
|
|
return $job;
|
|
|
|
|
|
};
|
|
|
|
|
|
$setPolicyResolution = static function (
|
|
|
|
|
|
CloudJob $job,
|
|
|
|
|
|
string $resolution,
|
|
|
|
|
|
bool $incident = false
|
|
|
|
|
|
): CloudJob {
|
|
|
|
|
|
if ($resolution === 'applied') {
|
|
|
|
|
|
$job->outcome = 'succeeded';
|
|
|
|
|
|
$job->status = 'succeeded';
|
|
|
|
|
|
$job->effect_may_have_applied = 0;
|
|
|
|
|
|
$job->safe_to_resubmit = 0;
|
|
|
|
|
|
$job->error_code = null;
|
|
|
|
|
|
} elseif ($resolution === 'unknown') {
|
|
|
|
|
|
$job->outcome = 'failed';
|
|
|
|
|
|
$job->status = 'failed';
|
|
|
|
|
|
$job->effect_may_have_applied = 1;
|
|
|
|
|
|
$job->safe_to_resubmit = 0;
|
|
|
|
|
|
$job->error_code = 'operation_failed_effect_unknown';
|
|
|
|
|
|
}
|
|
|
|
|
|
$job->effect_resolution = $resolution;
|
|
|
|
|
|
$job->finalization_incident = $incident ? 1 : 0;
|
|
|
|
|
|
$job->finalization_error_code = $incident
|
|
|
|
|
|
? 'recovery_ledger_conflict' : null;
|
|
|
|
|
|
$job->save();
|
|
|
|
|
|
return $job;
|
|
|
|
|
|
};
|
|
|
|
|
|
$insertScheduledPeriod = static function (
|
|
|
|
|
|
CloudInstance $target
|
|
|
|
|
|
): int {
|
|
|
|
|
|
$now = time();
|
|
|
|
|
|
return (int) Capsule::table('cloud_service_periods')->insertGetId([
|
|
|
|
|
|
'instance_id' => $target->id,
|
|
|
|
|
|
'user_id' => $target->user_id,
|
|
|
|
|
|
'plan_id' => $target->plan_id,
|
|
|
|
|
|
'starts_at' => $now - 10,
|
|
|
|
|
|
'ends_at' => $now + 3600,
|
|
|
|
|
|
'traffic_quota_bytes' => 1000,
|
|
|
|
|
|
'download_base_bytes' => null,
|
|
|
|
|
|
'upload_base_bytes' => null,
|
|
|
|
|
|
'download_used_bytes' => 0,
|
|
|
|
|
|
'upload_used_bytes' => 0,
|
|
|
|
|
|
'status' => 'scheduled',
|
|
|
|
|
|
'created_at' => $now - 100,
|
|
|
|
|
|
'updated_at' => $now - 100,
|
|
|
|
|
|
]);
|
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
|
|
// Activating a scheduled paid period may update the billing ledger, but
|
|
|
|
|
|
// it must not overwrite a policy projection whose Agent conclusion is
|
|
|
|
|
|
// still unknown, contradictory, or sticky.
|
|
|
|
|
|
$scheduledSuspend = $makeLifecycleInstance(
|
|
|
|
|
|
'80808080-8080-4080-8080-808080808080',
|
|
|
|
|
|
'scheduled-suspend-conflict-vm',
|
|
|
|
|
|
'suspend_failed',
|
|
|
|
|
|
'active'
|
|
|
|
|
|
);
|
|
|
|
|
|
$scheduledSuspend->expires_at = time() + 3600;
|
|
|
|
|
|
$scheduledSuspend->current_period_id = null;
|
|
|
|
|
|
$scheduledSuspend->save();
|
|
|
|
|
|
$scheduledSuspendJob = $setPolicyResolution($preparePolicyFailure(
|
|
|
|
|
|
$scheduledSuspend,
|
|
|
|
|
|
'service_suspend',
|
|
|
|
|
|
'service/suspend',
|
|
|
|
|
|
'purge'
|
|
|
|
|
|
), 'unknown');
|
|
|
|
|
|
$scheduledSuspendPeriod = $insertScheduledPeriod($scheduledSuspend);
|
|
|
|
|
|
$reconcile->invoke($lifecycle, (int) $scheduledSuspend->id);
|
|
|
|
|
|
$assert((int) $scheduledSuspend->fresh()->current_period_id
|
|
|
|
|
|
=== $scheduledSuspendPeriod
|
|
|
|
|
|
&& (string) CloudServicePeriod::find($scheduledSuspendPeriod)->status
|
|
|
|
|
|
=== 'active'
|
|
|
|
|
|
&& (string) $scheduledSuspend->fresh()->service_status
|
|
|
|
|
|
=== 'suspend_failed'
|
|
|
|
|
|
&& (string) $scheduledSuspendJob->fresh()->effect_resolution
|
|
|
|
|
|
=== 'unknown',
|
|
|
|
|
|
'Scheduled activation falsely cleared an unknown service suspension.');
|
|
|
|
|
|
|
|
|
|
|
|
$scheduledBlock = $makeLifecycleInstance(
|
|
|
|
|
|
'81818181-8181-4181-8181-818181818181',
|
|
|
|
|
|
'scheduled-block-conflict-vm',
|
|
|
|
|
|
'expired_grace',
|
|
|
|
|
|
'block_failed'
|
|
|
|
|
|
);
|
|
|
|
|
|
$scheduledBlock->expires_at = time() + 3600;
|
|
|
|
|
|
$scheduledBlock->current_period_id = null;
|
|
|
|
|
|
$scheduledBlock->save();
|
|
|
|
|
|
$scheduledBlockJob = $setPolicyResolution($preparePolicyFailure(
|
|
|
|
|
|
$scheduledBlock,
|
|
|
|
|
|
'network_suspend',
|
|
|
|
|
|
'network/suspend',
|
|
|
|
|
|
'expired'
|
|
|
|
|
|
), 'applied');
|
|
|
|
|
|
$scheduledBlockPeriod = $insertScheduledPeriod($scheduledBlock);
|
|
|
|
|
|
$reconcile->invoke($lifecycle, (int) $scheduledBlock->id);
|
|
|
|
|
|
$assert((int) $scheduledBlock->fresh()->current_period_id
|
|
|
|
|
|
=== $scheduledBlockPeriod
|
|
|
|
|
|
&& (string) $scheduledBlock->fresh()->network_status
|
|
|
|
|
|
=== 'block_failed'
|
|
|
|
|
|
&& (bool) $scheduledBlockJob->fresh()->finalization_incident
|
|
|
|
|
|
&& (string) $scheduledBlockJob->fresh()->finalization_error_code
|
|
|
|
|
|
=== 'recovery_instance_state_conflict',
|
|
|
|
|
|
'Scheduled activation hid an applied suspension/local-state conflict.');
|
|
|
|
|
|
|
|
|
|
|
|
$scheduledRestore = $makeLifecycleInstance(
|
|
|
|
|
|
'82828282-8282-4282-8282-828282828282',
|
|
|
|
|
|
'scheduled-restore-conflict-vm',
|
|
|
|
|
|
'expired_grace',
|
|
|
|
|
|
'restore_failed'
|
|
|
|
|
|
);
|
|
|
|
|
|
$scheduledRestore->expires_at = time() + 3600;
|
|
|
|
|
|
$scheduledRestore->current_period_id = null;
|
|
|
|
|
|
$scheduledRestore->save();
|
|
|
|
|
|
$scheduledRestoreJob = $setPolicyResolution($preparePolicyFailure(
|
|
|
|
|
|
$scheduledRestore,
|
|
|
|
|
|
'network_resume',
|
|
|
|
|
|
'network/resume',
|
|
|
|
|
|
null
|
|
|
|
|
|
), 'not_applied', true);
|
|
|
|
|
|
$scheduledRestorePeriod = $insertScheduledPeriod($scheduledRestore);
|
|
|
|
|
|
$reconcile->invoke($lifecycle, (int) $scheduledRestore->id);
|
|
|
|
|
|
$assert((int) $scheduledRestore->fresh()->current_period_id
|
|
|
|
|
|
=== $scheduledRestorePeriod
|
|
|
|
|
|
&& (string) $scheduledRestore->fresh()->network_status
|
|
|
|
|
|
=== 'restore_failed'
|
|
|
|
|
|
&& (bool) $scheduledRestoreJob->fresh()->finalization_incident,
|
|
|
|
|
|
'Scheduled activation falsely cleared a sticky network restore failure.');
|
|
|
|
|
|
|
|
|
|
|
|
$makeImmediatelyRenewable = static function (
|
|
|
|
|
|
CloudInstance $target
|
|
|
|
|
|
): void {
|
|
|
|
|
|
$past = time() - 60;
|
|
|
|
|
|
$target->current_period_id = null;
|
|
|
|
|
|
$target->expires_at = $past;
|
|
|
|
|
|
$target->renewal_days = 30;
|
|
|
|
|
|
$target->renewal_price = '1.00';
|
|
|
|
|
|
$target->renewal_traffic_bytes = 1000;
|
|
|
|
|
|
$target->traffic_synced_at = time() + 10;
|
|
|
|
|
|
$target->save();
|
|
|
|
|
|
Capsule::table('cloud_service_periods')->insert([
|
|
|
|
|
|
'instance_id' => $target->id,
|
|
|
|
|
|
'user_id' => $target->user_id,
|
|
|
|
|
|
'plan_id' => $target->plan_id,
|
|
|
|
|
|
'starts_at' => $past - 86400,
|
|
|
|
|
|
'ends_at' => $past,
|
|
|
|
|
|
'traffic_quota_bytes' => 1000,
|
|
|
|
|
|
'download_base_bytes' => 0,
|
|
|
|
|
|
'upload_base_bytes' => 0,
|
|
|
|
|
|
'download_used_bytes' => 0,
|
|
|
|
|
|
'upload_used_bytes' => 0,
|
|
|
|
|
|
'status' => 'finished',
|
|
|
|
|
|
'created_at' => $past - 86400,
|
|
|
|
|
|
'updated_at' => $past,
|
|
|
|
|
|
]);
|
|
|
|
|
|
$issueOne = new ReflectionMethod(
|
|
|
|
|
|
CloudInvoiceService::class,
|
|
|
|
|
|
'issueOne'
|
|
|
|
|
|
);
|
|
|
|
|
|
if ($issueOne->invoke(null, (int) $target->id, time()) !== true) {
|
|
|
|
|
|
throw new RuntimeException(
|
|
|
|
|
|
'Failed to create the renewal invoice fixture.'
|
|
|
|
|
|
);
|
|
|
|
|
|
}
|
|
|
|
|
|
};
|
|
|
|
|
|
$user->money = '1000.00';
|
|
|
|
|
|
$user->save();
|
|
|
|
|
|
|
|
|
|
|
|
$renewUnknownSuspend = $makeLifecycleInstance(
|
|
|
|
|
|
'83838383-8383-4383-8383-838383838383',
|
|
|
|
|
|
'renew-unknown-suspend-vm',
|
|
|
|
|
|
'suspend_failed',
|
|
|
|
|
|
'active'
|
|
|
|
|
|
);
|
|
|
|
|
|
$makeImmediatelyRenewable($renewUnknownSuspend);
|
|
|
|
|
|
$setPolicyResolution($preparePolicyFailure(
|
|
|
|
|
|
$renewUnknownSuspend,
|
|
|
|
|
|
'service_suspend',
|
|
|
|
|
|
'service/suspend',
|
|
|
|
|
|
'purge'
|
|
|
|
|
|
), 'unknown');
|
|
|
|
|
|
$unknownRenewal = CloudRenewalService::renew(
|
|
|
|
|
|
$renewUnknownSuspend,
|
|
|
|
|
|
['idempotency_key' => 'renew-unknown-suspend-0001'],
|
|
|
|
|
|
$actor
|
|
|
|
|
|
);
|
|
|
|
|
|
$assert((string) CloudServicePeriod::find(
|
|
|
|
|
|
$unknownRenewal->period_id
|
|
|
|
|
|
)->status === 'active'
|
|
|
|
|
|
&& (string) $renewUnknownSuspend->fresh()->service_status
|
|
|
|
|
|
=== 'suspend_failed',
|
|
|
|
|
|
'Immediate renewal falsely cleared an unknown service suspension.');
|
|
|
|
|
|
|
|
|
|
|
|
$renewAppliedBlock = $makeLifecycleInstance(
|
|
|
|
|
|
'84848484-8484-4484-8484-848484848484',
|
|
|
|
|
|
'renew-applied-block-vm',
|
|
|
|
|
|
'expired_grace',
|
|
|
|
|
|
'block_failed'
|
|
|
|
|
|
);
|
|
|
|
|
|
$makeImmediatelyRenewable($renewAppliedBlock);
|
|
|
|
|
|
$setPolicyResolution($preparePolicyFailure(
|
|
|
|
|
|
$renewAppliedBlock,
|
|
|
|
|
|
'network_suspend',
|
|
|
|
|
|
'network/suspend',
|
|
|
|
|
|
'expired'
|
|
|
|
|
|
), 'applied');
|
|
|
|
|
|
CloudRenewalService::renew(
|
|
|
|
|
|
$renewAppliedBlock,
|
|
|
|
|
|
['idempotency_key' => 'renew-applied-block-0001'],
|
|
|
|
|
|
$actor
|
|
|
|
|
|
);
|
|
|
|
|
|
$assert((string) $renewAppliedBlock->fresh()->network_status
|
|
|
|
|
|
=== 'block_failed',
|
|
|
|
|
|
'Immediate renewal hid an applied network suspension conflict.');
|
|
|
|
|
|
|
|
|
|
|
|
$renewStickyRestore = $makeLifecycleInstance(
|
|
|
|
|
|
'85858585-8585-4585-8585-858585858585',
|
|
|
|
|
|
'renew-sticky-restore-vm',
|
|
|
|
|
|
'expired_grace',
|
|
|
|
|
|
'restore_failed'
|
|
|
|
|
|
);
|
|
|
|
|
|
$makeImmediatelyRenewable($renewStickyRestore);
|
|
|
|
|
|
$setPolicyResolution($preparePolicyFailure(
|
|
|
|
|
|
$renewStickyRestore,
|
|
|
|
|
|
'network_resume',
|
|
|
|
|
|
'network/resume',
|
|
|
|
|
|
null
|
|
|
|
|
|
), 'not_applied', true);
|
|
|
|
|
|
CloudRenewalService::renew(
|
|
|
|
|
|
$renewStickyRestore,
|
|
|
|
|
|
['idempotency_key' => 'renew-sticky-restore-0001'],
|
|
|
|
|
|
$actor
|
|
|
|
|
|
);
|
|
|
|
|
|
$assert((string) $renewStickyRestore->fresh()->network_status
|
|
|
|
|
|
=== 'restore_failed',
|
|
|
|
|
|
'Immediate renewal cleared a sticky network restore failure.');
|
|
|
|
|
|
|
|
|
|
|
|
$renewSafeSuspend = $makeLifecycleInstance(
|
|
|
|
|
|
'86868686-8686-4686-8686-868686868686',
|
|
|
|
|
|
'renew-safe-suspend-vm',
|
|
|
|
|
|
'suspend_failed',
|
|
|
|
|
|
'active'
|
|
|
|
|
|
);
|
|
|
|
|
|
$makeImmediatelyRenewable($renewSafeSuspend);
|
|
|
|
|
|
$safeSuspendJob = $preparePolicyFailure(
|
|
|
|
|
|
$renewSafeSuspend,
|
|
|
|
|
|
'service_suspend',
|
|
|
|
|
|
'service/suspend',
|
|
|
|
|
|
'purge'
|
|
|
|
|
|
);
|
|
|
|
|
|
CloudRenewalService::renew(
|
|
|
|
|
|
$renewSafeSuspend,
|
|
|
|
|
|
['idempotency_key' => 'renew-safe-suspend-0001'],
|
|
|
|
|
|
$actor
|
|
|
|
|
|
);
|
|
|
|
|
|
$assert((string) $renewSafeSuspend->fresh()->service_status
|
|
|
|
|
|
=== 'suspend_failed',
|
|
|
|
|
|
'Immediate renewal bypassed lifecycle verification of a safe failure.');
|
|
|
|
|
|
$reconcile->invoke($lifecycle, (int) $renewSafeSuspend->id);
|
|
|
|
|
|
$assert((string) $renewSafeSuspend->fresh()->service_status === 'active'
|
|
|
|
|
|
&& (string) $safeSuspendJob->fresh()->phase === 'terminal',
|
|
|
|
|
|
'Lifecycle did not clear a canonical not-applied failure after renewal.');
|
|
|
|
|
|
|
|
|
|
|
|
$trafficRetryInstance = $makeLifecycleInstance(
|
|
|
|
|
|
'91919191-9191-4191-8191-919191919191',
|
|
|
|
|
|
'traffic-retry-vm',
|
|
|
|
|
|
'active',
|
|
|
|
|
|
'block_failed'
|
|
|
|
|
|
);
|
|
|
|
|
|
$trafficRetryInstance->expires_at = time() + 86400;
|
|
|
|
|
|
$trafficRetryInstance->lifetime_download_bytes = 200;
|
|
|
|
|
|
$trafficRetryInstance->lifetime_upload_bytes = 0;
|
|
|
|
|
|
$trafficRetryInstance->traffic_synced_at = time();
|
|
|
|
|
|
$trafficRetryInstance->save();
|
|
|
|
|
|
$trafficPeriodId = Capsule::table('cloud_service_periods')->insertGetId([
|
|
|
|
|
|
'instance_id' => $trafficRetryInstance->id,
|
|
|
|
|
|
'user_id' => $trafficRetryInstance->user_id,
|
|
|
|
|
|
'starts_at' => time() - 3600,
|
|
|
|
|
|
'ends_at' => time() + 3600,
|
|
|
|
|
|
'traffic_quota_bytes' => 100,
|
|
|
|
|
|
'download_base_bytes' => 0,
|
|
|
|
|
|
'upload_base_bytes' => 0,
|
|
|
|
|
|
'download_used_bytes' => 200,
|
|
|
|
|
|
'upload_used_bytes' => 0,
|
|
|
|
|
|
'status' => 'exhausted',
|
|
|
|
|
|
'created_at' => time() - 3600,
|
|
|
|
|
|
'updated_at' => time() - 310,
|
|
|
|
|
|
]);
|
|
|
|
|
|
$trafficRetryInstance->current_period_id = $trafficPeriodId;
|
|
|
|
|
|
$trafficRetryInstance->save();
|
|
|
|
|
|
$oldTrafficJob = $preparePolicyFailure(
|
|
|
|
|
|
$trafficRetryInstance,
|
|
|
|
|
|
'network_suspend',
|
|
|
|
|
|
'network/suspend',
|
|
|
|
|
|
'traffic'
|
|
|
|
|
|
);
|
|
|
|
|
|
$assert($reconcile->invoke(
|
|
|
|
|
|
$lifecycle,
|
|
|
|
|
|
(int) $trafficRetryInstance->id
|
|
|
|
|
|
) === true, 'A still-required traffic block did not create a new generation.');
|
|
|
|
|
|
$trafficRetry = CloudJob::where('instance_id', $trafficRetryInstance->id)
|
|
|
|
|
|
->where('kind', 'network_suspend')->orderByDesc('id')->first();
|
|
|
|
|
|
$assert($trafficRetry !== null
|
|
|
|
|
|
&& (int) $trafficRetry->id !== (int) $oldTrafficJob->id
|
|
|
|
|
|
&& (string) $trafficRetry->operation_id
|
|
|
|
|
|
!== (string) $oldTrafficJob->operation_id
|
|
|
|
|
|
&& (string) $oldTrafficJob->fresh()->phase === 'terminal'
|
|
|
|
|
|
&& (string) $trafficRetryInstance->fresh()->network_status
|
|
|
|
|
|
=== 'blocking',
|
|
|
|
|
|
'Traffic-block retry mutated or replayed its prior generation.');
|
|
|
|
|
|
|
|
|
|
|
|
$undispatchedTraffic = $makeLifecycleInstance(
|
|
|
|
|
|
'96969696-9696-4696-8696-969696969696',
|
|
|
|
|
|
'undispatched-traffic-vm',
|
|
|
|
|
|
'active',
|
|
|
|
|
|
'block_failed'
|
|
|
|
|
|
);
|
|
|
|
|
|
$undispatchedTraffic->expires_at = time() + 86400;
|
|
|
|
|
|
$undispatchedTraffic->lifetime_download_bytes = 200;
|
|
|
|
|
|
$undispatchedTraffic->lifetime_upload_bytes = 0;
|
|
|
|
|
|
$undispatchedTraffic->traffic_synced_at = time();
|
|
|
|
|
|
$undispatchedTraffic->save();
|
|
|
|
|
|
$undispatchedTrafficPeriod = Capsule::table(
|
|
|
|
|
|
'cloud_service_periods'
|
|
|
|
|
|
)->insertGetId([
|
|
|
|
|
|
'instance_id' => $undispatchedTraffic->id,
|
|
|
|
|
|
'user_id' => $undispatchedTraffic->user_id,
|
|
|
|
|
|
'starts_at' => time() - 3600,
|
|
|
|
|
|
'ends_at' => time() + 3600,
|
|
|
|
|
|
'traffic_quota_bytes' => 100,
|
|
|
|
|
|
'download_base_bytes' => 0,
|
|
|
|
|
|
'upload_base_bytes' => 0,
|
|
|
|
|
|
'download_used_bytes' => 200,
|
|
|
|
|
|
'upload_used_bytes' => 0,
|
|
|
|
|
|
'status' => 'exhausted',
|
|
|
|
|
|
'created_at' => time() - 3600,
|
|
|
|
|
|
'updated_at' => time() - 310,
|
|
|
|
|
|
]);
|
|
|
|
|
|
$undispatchedTraffic->current_period_id = $undispatchedTrafficPeriod;
|
|
|
|
|
|
$undispatchedTraffic->save();
|
|
|
|
|
|
$oldUndispatchedTraffic = $finalizeUndispatchedFailure($queuePolicy(
|
|
|
|
|
|
$undispatchedTraffic,
|
|
|
|
|
|
'network_suspend',
|
|
|
|
|
|
'network/suspend',
|
|
|
|
|
|
'traffic'
|
|
|
|
|
|
));
|
|
|
|
|
|
$assert($reconcile->invoke(
|
|
|
|
|
|
$lifecycle,
|
|
|
|
|
|
(int) $undispatchedTraffic->id
|
|
|
|
|
|
) === true, 'A generation-zero traffic failure did not create a safe new generation.');
|
|
|
|
|
|
$undispatchedTrafficRetry = CloudJob::where(
|
|
|
|
|
|
'instance_id',
|
|
|
|
|
|
$undispatchedTraffic->id
|
|
|
|
|
|
)->where('kind', 'network_suspend')->orderByDesc('id')->first();
|
|
|
|
|
|
$assert($undispatchedTrafficRetry !== null
|
|
|
|
|
|
&& (int) $undispatchedTrafficRetry->id
|
|
|
|
|
|
!== (int) $oldUndispatchedTraffic->id
|
|
|
|
|
|
&& (string) $undispatchedTrafficRetry->operation_id
|
|
|
|
|
|
!== (string) $oldUndispatchedTraffic->operation_id
|
|
|
|
|
|
&& (int) $oldUndispatchedTraffic->fresh()->dispatch_attempts === 0,
|
|
|
|
|
|
'Generation-zero traffic retry replayed or rewrote the prior operation.');
|
|
|
|
|
|
|
|
|
|
|
|
$resumeRetryInstance = $makeLifecycleInstance(
|
|
|
|
|
|
'92929292-9292-4292-8292-929292929292',
|
|
|
|
|
|
'resume-retry-vm',
|
|
|
|
|
|
'active',
|
|
|
|
|
|
'restore_failed'
|
|
|
|
|
|
);
|
|
|
|
|
|
$resumeRetryInstance->expires_at = time() + 86400;
|
|
|
|
|
|
$resumeRetryInstance->lifetime_download_bytes = 50;
|
|
|
|
|
|
$resumeRetryInstance->lifetime_upload_bytes = 0;
|
|
|
|
|
|
$resumeRetryInstance->traffic_synced_at = time();
|
|
|
|
|
|
$resumeRetryInstance->save();
|
|
|
|
|
|
$resumePeriodId = Capsule::table('cloud_service_periods')->insertGetId([
|
|
|
|
|
|
'instance_id' => $resumeRetryInstance->id,
|
|
|
|
|
|
'user_id' => $resumeRetryInstance->user_id,
|
|
|
|
|
|
'starts_at' => time() - 3600,
|
|
|
|
|
|
'ends_at' => time() + 3600,
|
|
|
|
|
|
'traffic_quota_bytes' => 1000,
|
|
|
|
|
|
'download_base_bytes' => 0,
|
|
|
|
|
|
'upload_base_bytes' => 0,
|
|
|
|
|
|
'download_used_bytes' => 50,
|
|
|
|
|
|
'upload_used_bytes' => 0,
|
|
|
|
|
|
'status' => 'active',
|
|
|
|
|
|
'created_at' => time() - 3600,
|
|
|
|
|
|
'updated_at' => time() - 310,
|
|
|
|
|
|
]);
|
|
|
|
|
|
$resumeRetryInstance->current_period_id = $resumePeriodId;
|
|
|
|
|
|
$resumeRetryInstance->save();
|
|
|
|
|
|
$oldResumeJob = $preparePolicyFailure(
|
|
|
|
|
|
$resumeRetryInstance,
|
|
|
|
|
|
'network_resume',
|
|
|
|
|
|
'network/resume',
|
|
|
|
|
|
null
|
|
|
|
|
|
);
|
|
|
|
|
|
$assert($reconcile->invoke(
|
|
|
|
|
|
$lifecycle,
|
|
|
|
|
|
(int) $resumeRetryInstance->id
|
|
|
|
|
|
) === true, 'A still-required network restore did not create a new generation.');
|
|
|
|
|
|
$resumeRetry = CloudJob::where('instance_id', $resumeRetryInstance->id)
|
|
|
|
|
|
->where('kind', 'network_resume')->orderByDesc('id')->first();
|
|
|
|
|
|
$assert($resumeRetry !== null
|
|
|
|
|
|
&& (int) $resumeRetry->id !== (int) $oldResumeJob->id
|
|
|
|
|
|
&& (string) $resumeRetry->operation_id
|
|
|
|
|
|
!== (string) $oldResumeJob->operation_id
|
|
|
|
|
|
&& (string) $oldResumeJob->fresh()->phase === 'terminal'
|
|
|
|
|
|
&& (string) $resumeRetryInstance->fresh()->network_status
|
|
|
|
|
|
=== 'restoring',
|
|
|
|
|
|
'Network-restore retry mutated or replayed its prior generation.');
|
|
|
|
|
|
|
|
|
|
|
|
$attachActivePeriod = static function (CloudInstance $target): void {
|
|
|
|
|
|
$target->expires_at = time() + 86400;
|
|
|
|
|
|
$target->lifetime_download_bytes = 50;
|
|
|
|
|
|
$target->lifetime_upload_bytes = 0;
|
|
|
|
|
|
$target->traffic_synced_at = time();
|
|
|
|
|
|
$target->save();
|
|
|
|
|
|
$periodId = Capsule::table('cloud_service_periods')->insertGetId([
|
|
|
|
|
|
'instance_id' => $target->id,
|
|
|
|
|
|
'user_id' => $target->user_id,
|
|
|
|
|
|
'starts_at' => time() - 3600,
|
|
|
|
|
|
'ends_at' => time() + 3600,
|
|
|
|
|
|
'traffic_quota_bytes' => 1000,
|
|
|
|
|
|
'download_base_bytes' => 0,
|
|
|
|
|
|
'upload_base_bytes' => 0,
|
|
|
|
|
|
'download_used_bytes' => 50,
|
|
|
|
|
|
'upload_used_bytes' => 0,
|
|
|
|
|
|
'status' => 'active',
|
|
|
|
|
|
'created_at' => time() - 3600,
|
|
|
|
|
|
'updated_at' => time() - 310,
|
|
|
|
|
|
]);
|
|
|
|
|
|
$target->current_period_id = $periodId;
|
|
|
|
|
|
$target->save();
|
|
|
|
|
|
};
|
|
|
|
|
|
$undispatchedResume = $makeLifecycleInstance(
|
|
|
|
|
|
'98989898-9898-4898-8898-989898989898',
|
|
|
|
|
|
'undispatched-resume-vm',
|
|
|
|
|
|
'active',
|
|
|
|
|
|
'restore_failed'
|
|
|
|
|
|
);
|
|
|
|
|
|
$attachActivePeriod($undispatchedResume);
|
|
|
|
|
|
$oldUndispatchedResume = $finalizeUndispatchedFailure($queuePolicy(
|
|
|
|
|
|
$undispatchedResume,
|
|
|
|
|
|
'network_resume',
|
|
|
|
|
|
'network/resume'
|
|
|
|
|
|
));
|
|
|
|
|
|
$assert($reconcile->invoke(
|
|
|
|
|
|
$lifecycle,
|
|
|
|
|
|
(int) $undispatchedResume->id
|
|
|
|
|
|
) === true, 'A generation-zero network restore did not create a safe new generation.');
|
|
|
|
|
|
$undispatchedResumeRetry = CloudJob::where(
|
|
|
|
|
|
'instance_id',
|
|
|
|
|
|
$undispatchedResume->id
|
|
|
|
|
|
)->where('kind', 'network_resume')->orderByDesc('id')->first();
|
|
|
|
|
|
$assert($undispatchedResumeRetry !== null
|
|
|
|
|
|
&& (int) $undispatchedResumeRetry->id
|
|
|
|
|
|
!== (int) $oldUndispatchedResume->id
|
|
|
|
|
|
&& (string) $undispatchedResumeRetry->operation_id
|
|
|
|
|
|
!== (string) $oldUndispatchedResume->operation_id
|
|
|
|
|
|
&& (int) $oldUndispatchedResume->fresh()->dispatch_attempts === 0
|
|
|
|
|
|
&& (string) $undispatchedResume->fresh()->network_status
|
|
|
|
|
|
=== 'restoring',
|
|
|
|
|
|
'Generation-zero network-restore retry replayed or rewrote the prior operation.');
|
|
|
|
|
|
|
|
|
|
|
|
$obsoleteSuspend = $makeLifecycleInstance(
|
|
|
|
|
|
'94949494-9494-4494-8494-949494949494',
|
|
|
|
|
|
'obsolete-suspend-vm',
|
|
|
|
|
|
'suspend_failed',
|
|
|
|
|
|
'active'
|
|
|
|
|
|
);
|
|
|
|
|
|
$attachActivePeriod($obsoleteSuspend);
|
|
|
|
|
|
$oldObsoleteSuspend = $preparePolicyFailure(
|
|
|
|
|
|
$obsoleteSuspend,
|
|
|
|
|
|
'service_suspend',
|
|
|
|
|
|
'service/suspend',
|
|
|
|
|
|
'purge'
|
|
|
|
|
|
);
|
|
|
|
|
|
$obsoleteCount = CloudJob::where(
|
|
|
|
|
|
'instance_id',
|
|
|
|
|
|
$obsoleteSuspend->id
|
|
|
|
|
|
)->count();
|
|
|
|
|
|
$assert($reconcile->invoke(
|
|
|
|
|
|
$lifecycle,
|
|
|
|
|
|
(int) $obsoleteSuspend->id
|
|
|
|
|
|
) === false
|
|
|
|
|
|
&& (string) $obsoleteSuspend->fresh()->service_status === 'active'
|
|
|
|
|
|
&& CloudJob::where('instance_id', $obsoleteSuspend->id)->count()
|
|
|
|
|
|
=== $obsoleteCount
|
|
|
|
|
|
&& (string) $oldObsoleteSuspend->fresh()->phase === 'terminal',
|
|
|
|
|
|
'A no-longer-required, definitively not-applied suspend was not safely cleared.');
|
|
|
|
|
|
|
|
|
|
|
|
$uncertainSuspend = $makeLifecycleInstance(
|
|
|
|
|
|
'95959595-9595-4595-8595-959595959595',
|
|
|
|
|
|
'uncertain-suspend-vm',
|
|
|
|
|
|
'suspend_failed',
|
|
|
|
|
|
'active'
|
|
|
|
|
|
);
|
|
|
|
|
|
$attachActivePeriod($uncertainSuspend);
|
|
|
|
|
|
$oldUncertainSuspend = $preparePolicyFailure(
|
|
|
|
|
|
$uncertainSuspend,
|
|
|
|
|
|
'service_suspend',
|
|
|
|
|
|
'service/suspend',
|
|
|
|
|
|
'purge'
|
|
|
|
|
|
);
|
|
|
|
|
|
$oldUncertainSuspend->effect_resolution = 'unknown';
|
|
|
|
|
|
$oldUncertainSuspend->effect_may_have_applied = 1;
|
|
|
|
|
|
$oldUncertainSuspend->safe_to_resubmit = 0;
|
|
|
|
|
|
$oldUncertainSuspend->save();
|
|
|
|
|
|
$assert($reconcile->invoke(
|
|
|
|
|
|
$lifecycle,
|
|
|
|
|
|
(int) $uncertainSuspend->id
|
|
|
|
|
|
) === false
|
|
|
|
|
|
&& (string) $uncertainSuspend->fresh()->service_status
|
|
|
|
|
|
=== 'suspend_failed'
|
|
|
|
|
|
&& (string) $oldUncertainSuspend->fresh()->effect_resolution
|
|
|
|
|
|
=== 'unknown',
|
|
|
|
|
|
'An unknown-effect suspend failure was falsely unlocked by a paid period.');
|
|
|
|
|
|
|
|
|
|
|
|
foreach ([
|
|
|
|
|
|
['unknown', 1, 0, false],
|
|
|
|
|
|
['applied', 0, 0, false],
|
|
|
|
|
|
['not_applied', 0, 1, true],
|
|
|
|
|
|
] as $index => [$resolution, $effect, $safe, $incident]) {
|
|
|
|
|
|
$guarded = $makeLifecycleInstance(
|
|
|
|
|
|
sprintf('93939393-9393-4393-8393-%012d', $index + 1),
|
|
|
|
|
|
'guarded-lifecycle-' . $index,
|
|
|
|
|
|
'purge_deleting',
|
|
|
|
|
|
'blocked_expired'
|
|
|
|
|
|
);
|
|
|
|
|
|
$guarded->status = 'deleting';
|
|
|
|
|
|
$guarded->save();
|
|
|
|
|
|
$guardedJob = CloudJobService::enqueueExpiryDelete($guarded);
|
|
|
|
|
|
$identity = CloudOperationRecoveryIdentity::expected($guardedJob);
|
|
|
|
|
|
$past = time() - 310;
|
|
|
|
|
|
$guardedJob->request_fingerprint = $identity['request_fingerprint'];
|
|
|
|
|
|
$guardedJob->attempts = 1;
|
|
|
|
|
|
$guardedJob->dispatch_attempts = 1;
|
|
|
|
|
|
$guardedJob->dispatch_started_at = $past - 30;
|
|
|
|
|
|
$guardedJob->agent_accepted_at = $past - 20;
|
|
|
|
|
|
$guardedJob->dispatched_at = $past - 20;
|
|
|
|
|
|
$guardedJob->receipt_deadline_at = $past - 10;
|
|
|
|
|
|
$guardedJob->phase = 'terminal';
|
|
|
|
|
|
$guardedJob->terminal_at = $past;
|
|
|
|
|
|
$guardedJob->error_code = 'operation_failed';
|
|
|
|
|
|
$guardedJob->updated_at = $past;
|
|
|
|
|
|
$guardedJob->effect_resolution = $resolution;
|
|
|
|
|
|
$guardedJob->effect_may_have_applied = $effect;
|
|
|
|
|
|
$guardedJob->safe_to_resubmit = $safe;
|
|
|
|
|
|
$guardedJob->outcome = $resolution === 'applied'
|
|
|
|
|
|
? 'succeeded' : 'failed';
|
|
|
|
|
|
$guardedJob->status = $guardedJob->outcome;
|
|
|
|
|
|
$guardedJob->finalization_incident = $incident ? 1 : 0;
|
|
|
|
|
|
$guardedJob->finalization_error_code = $incident
|
|
|
|
|
|
? 'recovery_ledger_conflict' : null;
|
|
|
|
|
|
$guardedJob->save();
|
|
|
|
|
|
$guarded->status = 'delete_failed';
|
|
|
|
|
|
$guarded->service_status = 'purge_failed';
|
|
|
|
|
|
$guarded->save();
|
|
|
|
|
|
$beforeGuarded = CloudJob::where('instance_id', $guarded->id)->count();
|
|
|
|
|
|
$assert($reconcile->invoke($lifecycle, (int) $guarded->id) === false
|
|
|
|
|
|
&& CloudJob::where('instance_id', $guarded->id)->count()
|
|
|
|
|
|
=== $beforeGuarded
|
|
|
|
|
|
&& (string) $guardedJob->fresh()->operation_id
|
|
|
|
|
|
=== (string) $guardedJob->operation_id,
|
|
|
|
|
|
'Unknown, applied, or sticky lifecycle evidence created a new mutation generation.');
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
// A create which crossed the Agent boundary may time out locally before
|
|
|
|
|
|
// the authenticated terminal failure reaches PHP. Exact Agent proof that
|
|
|
|
|
|
// both the guest and every managed artifact are absent must finish the
|
|
|
|
|
|
// original paid order locally; it must never dispatch a second create.
|
|
|
|
|
|
foreach (['8.8.8.20', '8.8.8.21', '8.8.8.22'] as $address) {
|
|
|
|
|
|
if (!CloudPublicIp::where('address', $address)->exists()) {
|
|
|
|
|
|
$ip = new CloudPublicIp();
|
|
|
|
|
|
$ip->host_id = $host->id;
|
|
|
|
|
|
$ip->public_ip_pool_id = $sourcePublicPool->id;
|
|
|
|
|
|
$ip->address_family = 4;
|
|
|
|
|
|
$ip->address = $address;
|
|
|
|
|
|
$ip->address_binary = inet_pton($address);
|
|
|
|
|
|
$ip->prefix_length = 24;
|
|
|
|
|
|
$ip->state = 'available';
|
|
|
|
|
|
$ip->created_at = $ip->updated_at = time();
|
|
|
|
|
|
$ip->save();
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
$lateFailureOrder = CloudOrderService::purchase(array_replace($input, [
|
|
|
|
|
|
'name' => 'vm-late-failure',
|
|
|
|
|
|
'idempotency_key' => 'test-request-key-late-failure',
|
|
|
|
|
|
]), $actor);
|
|
|
|
|
|
$lateFailureInvoice = CloudInvoice::where(
|
|
|
|
|
|
'order_id',
|
|
|
|
|
|
$lateFailureOrder->id
|
|
|
|
|
|
)->firstOrFail();
|
|
|
|
|
|
CloudInvoiceService::payBalance(
|
|
|
|
|
|
$lateFailureInvoice,
|
|
|
|
|
|
$actor,
|
|
|
|
|
|
'provision-balance-late-failure'
|
|
|
|
|
|
);
|
|
|
|
|
|
$lateFailureJob = CloudJob::where(
|
|
|
|
|
|
'order_id',
|
|
|
|
|
|
$lateFailureOrder->id
|
|
|
|
|
|
)->firstOrFail();
|
|
|
|
|
|
// Earlier lifecycle tests intentionally leave separately audited retries
|
|
|
|
|
|
// queued. Park those unrelated rows so this tracer observes exactly one
|
|
|
|
|
|
// create dispatch rather than depending on suite-wide queue order.
|
|
|
|
|
|
Capsule::table('cloud_jobs')
|
|
|
|
|
|
->where('id', '<>', (int) $lateFailureJob->id)
|
|
|
|
|
|
->where('phase', '<>', 'terminal')
|
|
|
|
|
|
->update([
|
|
|
|
|
|
'available_at' => time() + 3600,
|
|
|
|
|
|
'dispatch_deadline_at' => time() + 3600,
|
|
|
|
|
|
'receipt_deadline_at' => time() + 3600,
|
|
|
|
|
|
]);
|
|
|
|
|
|
Capsule::table('cloud_jobs')->where(
|
|
|
|
|
|
'id',
|
|
|
|
|
|
(int) $lateFailureJob->id
|
|
|
|
|
|
)->update(['available_at' => 0]);
|
|
|
|
|
|
$lateFailureJob->refresh();
|
|
|
|
|
|
$lateFailureIdentity = CloudOperationRecoveryIdentity::expected(
|
|
|
|
|
|
$lateFailureJob,
|
|
|
|
|
|
false
|
|
|
|
|
|
);
|
|
|
|
|
|
$mock->append(new Response(202, [], json_encode([
|
|
|
|
|
|
'operation_id' => $lateFailureJob->operation_id,
|
|
|
|
|
|
'status' => 'reserved',
|
|
|
|
|
|
...$lateFailureIdentity,
|
|
|
|
|
|
'replayed' => false,
|
|
|
|
|
|
'retryable' => false,
|
|
|
|
|
|
], JSON_THROW_ON_ERROR)));
|
|
|
|
|
|
$beforeLateFailureDispatch = count($history);
|
|
|
|
|
|
$assert($worker->tick(),
|
|
|
|
|
|
'Late-failure create did not cross the Agent dispatch boundary.');
|
|
|
|
|
|
$lateFailureJob->refresh();
|
|
|
|
|
|
$assert((string) $lateFailureJob->phase === 'awaiting_receipt'
|
|
|
|
|
|
&& (int) $lateFailureJob->dispatch_attempts === 1
|
|
|
|
|
|
&& count($history) === $beforeLateFailureDispatch + 1,
|
|
|
|
|
|
'Late-failure create did not persist its single dispatch generation.');
|
|
|
|
|
|
Capsule::table('cloud_jobs')->where(
|
|
|
|
|
|
'id',
|
|
|
|
|
|
(int) $lateFailureJob->id
|
|
|
|
|
|
)->update([
|
|
|
|
|
|
'receipt_deadline_at' => time() - 1,
|
|
|
|
|
|
'lease_token' => null,
|
|
|
|
|
|
'lease_until' => 0,
|
|
|
|
|
|
]);
|
|
|
|
|
|
$assert(CloudWorkerService::expireDueTaskForAdmin(
|
|
|
|
|
|
(int) $lateFailureJob->id
|
|
|
|
|
|
) === 1, 'Late-failure create did not reach its local deadline.');
|
|
|
|
|
|
$lateFailureJob->refresh();
|
|
|
|
|
|
$lateFailureOrder->refresh();
|
|
|
|
|
|
$lateFailureInvoice->refresh();
|
|
|
|
|
|
$lateFailureReservation = CloudResourceReservation::where(
|
|
|
|
|
|
'order_id',
|
|
|
|
|
|
$lateFailureOrder->id
|
|
|
|
|
|
)->firstOrFail();
|
|
|
|
|
|
$walletWhileFenced = (string) User::findOrFail(1)->money;
|
|
|
|
|
|
$stockWhileFenced = (int) CloudOffer::findOrFail(1)->stock;
|
|
|
|
|
|
$assert((string) $lateFailureJob->phase === 'terminal'
|
|
|
|
|
|
&& (string) $lateFailureJob->outcome === 'failed'
|
|
|
|
|
|
&& (string) $lateFailureJob->effect_resolution === 'unknown'
|
|
|
|
|
|
&& (string) $lateFailureOrder->status === 'failed'
|
|
|
|
|
|
&& $lateFailureOrder->refunded_at === null
|
|
|
|
|
|
&& (string) $lateFailureInvoice->status === 'paid'
|
|
|
|
|
|
&& (string) $lateFailureReservation->state === 'uncertain'
|
|
|
|
|
|
&& $bundleState(
|
|
|
|
|
|
$lateFailureReservation,
|
|
|
|
|
|
'uncertain',
|
|
|
|
|
|
'uncertain'
|
|
|
|
|
|
),
|
|
|
|
|
|
'Timed-out dispatched create did not retain its financial fence.');
|
|
|
|
|
|
|
|
|
|
|
|
$lateFailureEventId = 'create-late-failure-event-0001';
|
|
|
|
|
|
$lateFailureReceipt = [
|
|
|
|
|
|
'type' => 'completion',
|
|
|
|
|
|
'id' => $lateFailureEventId,
|
|
|
|
|
|
'agent_id' => (string) $lateFailureJob->host_id,
|
|
|
|
|
|
'operation_id' => (string) $lateFailureJob->operation_id,
|
|
|
|
|
|
...$lateFailureIdentity,
|
|
|
|
|
|
'status' => 'failed',
|
|
|
|
|
|
'effect_may_have_applied' => false,
|
|
|
|
|
|
'safe_to_resubmit' => false,
|
|
|
|
|
|
'error_code' => 'creation_absent',
|
|
|
|
|
|
'result' => [
|
|
|
|
|
|
'uuid' => $lateFailureIdentity['resource_id'],
|
|
|
|
|
|
'creation_absent' => true,
|
|
|
|
|
|
'managed_artifacts_absent' => true,
|
|
|
|
|
|
],
|
|
|
|
|
|
];
|
|
|
|
|
|
$lateFailureAccepted = CloudOperationCompletionService::accept(
|
|
|
|
|
|
$lateFailureReceipt
|
|
|
|
|
|
);
|
|
|
|
|
|
$lateFailureJob->refresh();
|
|
|
|
|
|
$lateFailureOrder->refresh();
|
|
|
|
|
|
$lateFailureInvoice->refresh();
|
|
|
|
|
|
$lateFailureReservation->refresh();
|
|
|
|
|
|
$lateFailureEvent = \App\Models\CloudAgentCompletionEvent::where(
|
|
|
|
|
|
'event_id',
|
|
|
|
|
|
$lateFailureEventId
|
|
|
|
|
|
)->firstOrFail();
|
|
|
|
|
|
$assert($lateFailureAccepted === [
|
|
|
|
|
|
'accepted' => true,
|
|
|
|
|
|
'disposition' => 'applied',
|
|
|
|
|
|
]
|
|
|
|
|
|
&& (string) $lateFailureEvent->disposition === 'applied'
|
|
|
|
|
|
&& $lateFailureEvent->applied_at !== null
|
|
|
|
|
|
&& (string) $lateFailureJob->effect_resolution === 'not_applied'
|
|
|
|
|
|
&& !(bool) $lateFailureJob->finalization_incident
|
|
|
|
|
|
&& (string) $lateFailureOrder->status === 'failed'
|
|
|
|
|
|
&& $lateFailureOrder->refunded_at !== null
|
|
|
|
|
|
&& (string) $lateFailureInvoice->status === 'refunded'
|
|
|
|
|
|
&& (string) $lateFailureReservation->state === 'released'
|
|
|
|
|
|
&& $bundleState(
|
|
|
|
|
|
$lateFailureReservation,
|
|
|
|
|
|
'released',
|
|
|
|
|
|
'available'
|
|
|
|
|
|
)
|
|
|
|
|
|
&& (string) User::findOrFail(1)->money
|
|
|
|
|
|
=== CloudMoney::decimal(
|
|
|
|
|
|
CloudMoney::cents($walletWhileFenced) + 999
|
|
|
|
|
|
)
|
|
|
|
|
|
&& (int) CloudOffer::findOrFail(1)->stock
|
|
|
|
|
|
=== $stockWhileFenced + 1
|
|
|
|
|
|
&& count($history) === $beforeLateFailureDispatch + 1,
|
|
|
|
|
|
'Exact late create failure did not atomically refund, release, ACK, '
|
|
|
|
|
|
. 'or replayed the original mutation.');
|
|
|
|
|
|
$lateFailureWalletAfter = (string) User::findOrFail(1)->money;
|
|
|
|
|
|
$lateFailureStockAfter = (int) CloudOffer::findOrFail(1)->stock;
|
|
|
|
|
|
$lateFailureRefundedAt = (int) $lateFailureOrder->refunded_at;
|
|
|
|
|
|
$lateFailureJobUpdatedAt = (int) $lateFailureJob->updated_at;
|
|
|
|
|
|
$lateFailureAppliedAt = (int) $lateFailureEvent->applied_at;
|
|
|
|
|
|
$lateFailureDuplicate = CloudOperationCompletionService::accept(
|
|
|
|
|
|
$lateFailureReceipt
|
|
|
|
|
|
);
|
|
|
|
|
|
$assert($lateFailureDuplicate === [
|
|
|
|
|
|
'accepted' => true,
|
|
|
|
|
|
'disposition' => 'applied',
|
|
|
|
|
|
]
|
|
|
|
|
|
&& (string) User::findOrFail(1)->money
|
|
|
|
|
|
=== $lateFailureWalletAfter
|
|
|
|
|
|
&& (int) CloudOffer::findOrFail(1)->stock
|
|
|
|
|
|
=== $lateFailureStockAfter
|
|
|
|
|
|
&& (int) CloudOrder::findOrFail(
|
|
|
|
|
|
(int) $lateFailureOrder->id
|
|
|
|
|
|
)->refunded_at === $lateFailureRefundedAt
|
|
|
|
|
|
&& (int) CloudJob::findOrFail(
|
|
|
|
|
|
(int) $lateFailureJob->id
|
|
|
|
|
|
)->updated_at === $lateFailureJobUpdatedAt
|
|
|
|
|
|
&& (int) \App\Models\CloudAgentCompletionEvent::findOrFail(
|
|
|
|
|
|
$lateFailureEventId
|
|
|
|
|
|
)->applied_at === $lateFailureAppliedAt
|
|
|
|
|
|
&& count($history) === $beforeLateFailureDispatch + 1,
|
|
|
|
|
|
'Duplicate late create failure refunded, released, ACKed, or dispatched '
|
|
|
|
|
|
. 'more than once.');
|
|
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
|
* Create one paid order, cross the Agent boundary exactly once, and force
|
|
|
|
|
|
* the panel-side receipt deadline to the terminal unknown-effect state.
|
|
|
|
|
|
* Later recovery assertions can then focus on finance and evidence rather
|
|
|
|
|
|
* than reproducing dispatch bookkeeping by hand.
|
|
|
|
|
|
*
|
|
|
|
|
|
* @return array<string,mixed>
|
|
|
|
|
|
*/
|
|
|
|
|
|
$makeTimedOutCreate = static function (
|
|
|
|
|
|
string $suffix
|
|
|
|
|
|
) use (
|
|
|
|
|
|
$input,
|
|
|
|
|
|
$actor,
|
|
|
|
|
|
$mock,
|
|
|
|
|
|
$worker,
|
|
|
|
|
|
&$history,
|
|
|
|
|
|
$assert,
|
|
|
|
|
|
$bundleState
|
|
|
|
|
|
): array {
|
|
|
|
|
|
$order = CloudOrderService::purchase(array_replace($input, [
|
|
|
|
|
|
'name' => 'vm-create-recovery-' . $suffix,
|
|
|
|
|
|
'idempotency_key' => 'test-create-recovery-' . $suffix,
|
|
|
|
|
|
]), $actor);
|
|
|
|
|
|
$invoice = CloudInvoice::where('order_id', $order->id)
|
|
|
|
|
|
->firstOrFail();
|
|
|
|
|
|
CloudInvoiceService::payBalance(
|
|
|
|
|
|
$invoice,
|
|
|
|
|
|
$actor,
|
|
|
|
|
|
'provision-create-recovery-' . $suffix
|
|
|
|
|
|
);
|
|
|
|
|
|
$job = CloudJob::where('order_id', $order->id)->firstOrFail();
|
|
|
|
|
|
$future = time() + 3600;
|
|
|
|
|
|
Capsule::table('cloud_jobs')
|
|
|
|
|
|
->where('id', '<>', (int) $job->id)
|
|
|
|
|
|
->update([
|
|
|
|
|
|
'available_at' => $future,
|
|
|
|
|
|
'lease_until' => $future,
|
|
|
|
|
|
]);
|
|
|
|
|
|
Capsule::table('cloud_jobs')->where('id', (int) $job->id)
|
|
|
|
|
|
->update(['available_at' => 0, 'lease_until' => 0]);
|
|
|
|
|
|
$job->refresh();
|
|
|
|
|
|
$identity = CloudOperationRecoveryIdentity::expected($job, false);
|
|
|
|
|
|
$mock->append(new Response(202, [], json_encode([
|
|
|
|
|
|
'operation_id' => $job->operation_id,
|
|
|
|
|
|
'status' => 'reserved',
|
|
|
|
|
|
...$identity,
|
|
|
|
|
|
'replayed' => false,
|
|
|
|
|
|
'retryable' => false,
|
|
|
|
|
|
], JSON_THROW_ON_ERROR)));
|
|
|
|
|
|
$historyBefore = count($history);
|
|
|
|
|
|
$assert($worker->tick(),
|
|
|
|
|
|
'Create recovery fixture did not dispatch: ' . $suffix);
|
|
|
|
|
|
$job->refresh();
|
|
|
|
|
|
$assert((string) $job->phase === 'awaiting_receipt'
|
|
|
|
|
|
&& (int) $job->dispatch_attempts === 1
|
|
|
|
|
|
&& count($history) === $historyBefore + 1,
|
|
|
|
|
|
'Create recovery fixture did not persist one dispatch: '
|
|
|
|
|
|
. $suffix);
|
|
|
|
|
|
Capsule::table('cloud_jobs')->where('id', (int) $job->id)
|
|
|
|
|
|
->update([
|
|
|
|
|
|
'receipt_deadline_at' => time() - 1,
|
|
|
|
|
|
'lease_token' => null,
|
|
|
|
|
|
'lease_until' => 0,
|
|
|
|
|
|
]);
|
|
|
|
|
|
$assert(CloudWorkerService::expireDueTaskForAdmin((int) $job->id)
|
|
|
|
|
|
=== 1, 'Create recovery fixture did not expire: ' . $suffix);
|
|
|
|
|
|
$job->refresh();
|
|
|
|
|
|
$order->refresh();
|
|
|
|
|
|
$invoice->refresh();
|
|
|
|
|
|
$reservation = CloudResourceReservation::where(
|
|
|
|
|
|
'order_id',
|
|
|
|
|
|
$order->id
|
|
|
|
|
|
)->firstOrFail();
|
|
|
|
|
|
$assert((string) $job->phase === 'terminal'
|
|
|
|
|
|
&& (string) $job->outcome === 'failed'
|
|
|
|
|
|
&& (string) $job->effect_resolution === 'unknown'
|
|
|
|
|
|
&& (string) $order->status === 'failed'
|
|
|
|
|
|
&& $order->refunded_at === null
|
|
|
|
|
|
&& (string) $invoice->status === 'paid'
|
|
|
|
|
|
&& (string) $reservation->state === 'uncertain'
|
|
|
|
|
|
&& $bundleState($reservation, 'uncertain', 'uncertain'),
|
|
|
|
|
|
'Create recovery fixture did not retain its safety fence: '
|
|
|
|
|
|
. $suffix);
|
|
|
|
|
|
return compact(
|
|
|
|
|
|
'order', 'invoice', 'job', 'identity', 'reservation',
|
|
|
|
|
|
'historyBefore'
|
|
|
|
|
|
) + [
|
|
|
|
|
|
'walletBefore' => (string) User::findOrFail(1)->money,
|
|
|
|
|
|
'stockBefore' => (int) CloudOffer::findOrFail(1)->stock,
|
|
|
|
|
|
];
|
|
|
|
|
|
};
|
|
|
|
|
|
$makeRecoveryReady = static function (CloudJob $job): void {
|
|
|
|
|
|
$future = time() + 3600;
|
|
|
|
|
|
Capsule::table('cloud_jobs')
|
|
|
|
|
|
->where('id', '<>', (int) $job->id)
|
|
|
|
|
|
->update([
|
|
|
|
|
|
'available_at' => $future,
|
|
|
|
|
|
'lease_until' => $future,
|
|
|
|
|
|
]);
|
|
|
|
|
|
Capsule::table('cloud_jobs')->where('id', (int) $job->id)
|
|
|
|
|
|
->update([
|
|
|
|
|
|
'available_at' => 0,
|
|
|
|
|
|
'lease_token' => null,
|
|
|
|
|
|
'lease_until' => 0,
|
|
|
|
|
|
]);
|
|
|
|
|
|
$job->refresh();
|
|
|
|
|
|
};
|
|
|
|
|
|
$inspectionFor = static function (
|
|
|
|
|
|
CloudJob $job,
|
|
|
|
|
|
array $identity,
|
|
|
|
|
|
bool $fenced = false
|
|
|
|
|
|
): array {
|
|
|
|
|
|
return [
|
|
|
|
|
|
'operation_id' => (string) $job->operation_id,
|
|
|
|
|
|
...$identity,
|
|
|
|
|
|
'operation_status' => 'failed',
|
|
|
|
|
|
'checked_at' => time(),
|
|
|
|
|
|
'outcome' => 'failed',
|
|
|
|
|
|
'summary_code' => 'operation_failed',
|
|
|
|
|
|
'error_code' => 'creation_absent',
|
|
|
|
|
|
'effect_resolution' => 'not_applied',
|
|
|
|
|
|
'fence_state' => $fenced ? 'active' : 'not_required',
|
|
|
|
|
|
'fence_resolution_required' => $fenced,
|
|
|
|
|
|
'effect_projection' => [
|
|
|
|
|
|
'creation_absent' => true,
|
|
|
|
|
|
'managed_artifacts_absent' => true,
|
|
|
|
|
|
],
|
|
|
|
|
|
'abort_job' => ['available' => false],
|
|
|
|
|
|
'cleanup' => ['available' => false],
|
|
|
|
|
|
];
|
|
|
|
|
|
};
|
|
|
|
|
|
$assertCreateSettled = static function (
|
|
|
|
|
|
array $case,
|
|
|
|
|
|
string $message
|
|
|
|
|
|
) use ($assert, $bundleState, &$history): void {
|
|
|
|
|
|
$case['job']->refresh();
|
|
|
|
|
|
$case['order']->refresh();
|
|
|
|
|
|
$case['invoice']->refresh();
|
|
|
|
|
|
$case['reservation']->refresh();
|
|
|
|
|
|
$assert((string) $case['job']->effect_resolution === 'not_applied'
|
|
|
|
|
|
&& !(bool) $case['job']->finalization_incident
|
|
|
|
|
|
&& $case['job']->lease_token === null
|
|
|
|
|
|
&& (string) $case['order']->status === 'failed'
|
|
|
|
|
|
&& $case['order']->refunded_at !== null
|
|
|
|
|
|
&& (string) $case['invoice']->status === 'refunded'
|
|
|
|
|
|
&& (string) $case['reservation']->state === 'released'
|
|
|
|
|
|
&& $bundleState(
|
|
|
|
|
|
$case['reservation'],
|
|
|
|
|
|
'released',
|
|
|
|
|
|
'available'
|
|
|
|
|
|
)
|
|
|
|
|
|
&& (string) User::findOrFail(1)->money
|
|
|
|
|
|
=== CloudMoney::decimal(
|
|
|
|
|
|
CloudMoney::cents($case['walletBefore']) + 999
|
|
|
|
|
|
)
|
|
|
|
|
|
&& (int) CloudOffer::findOrFail(1)->stock
|
|
|
|
|
|
=== $case['stockBefore'] + 1
|
|
|
|
|
|
&& count($history) === $case['historyBefore'] + 1,
|
|
|
|
|
|
$message);
|
|
|
|
|
|
};
|
|
|
|
|
|
$failedCreateReceipt = static function (
|
|
|
|
|
|
array $case,
|
|
|
|
|
|
string $eventId
|
|
|
|
|
|
): array {
|
|
|
|
|
|
return [
|
|
|
|
|
|
'type' => 'completion',
|
|
|
|
|
|
'id' => $eventId,
|
|
|
|
|
|
'agent_id' => (string) $case['job']->host_id,
|
|
|
|
|
|
'operation_id' => (string) $case['job']->operation_id,
|
|
|
|
|
|
...$case['identity'],
|
|
|
|
|
|
'status' => 'failed',
|
|
|
|
|
|
'effect_may_have_applied' => false,
|
|
|
|
|
|
'safe_to_resubmit' => false,
|
|
|
|
|
|
'error_code' => 'creation_absent',
|
|
|
|
|
|
'result' => [
|
|
|
|
|
|
'uuid' => $case['identity']['resource_id'],
|
|
|
|
|
|
'creation_absent' => true,
|
|
|
|
|
|
'managed_artifacts_absent' => true,
|
|
|
|
|
|
],
|
|
|
|
|
|
];
|
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
|
|
// Immutable host absence evidence must wait for a temporarily inconsistent
|
|
|
|
|
|
// local invoice. Keeping the event received allows the exact same receipt
|
|
|
|
|
|
// to finish atomically after the PHP ledger is repaired.
|
|
|
|
|
|
$invoiceGuardCase = $makeTimedOutCreate('receipt-invoice-guard');
|
|
|
|
|
|
$invoiceGuardCase['invoice']->purpose = 'renewal';
|
|
|
|
|
|
$invoiceGuardCase['invoice']->save();
|
|
|
|
|
|
$invoiceGuardReceipt = $failedCreateReceipt(
|
|
|
|
|
|
$invoiceGuardCase,
|
|
|
|
|
|
'create-invoice-guard-failure-event-0001'
|
|
|
|
|
|
);
|
|
|
|
|
|
$invoiceGuardPending = CloudOperationCompletionService::accept(
|
|
|
|
|
|
$invoiceGuardReceipt
|
|
|
|
|
|
);
|
|
|
|
|
|
$invoiceGuardCase['job']->refresh();
|
|
|
|
|
|
$invoiceGuardCase['order']->refresh();
|
|
|
|
|
|
$invoiceGuardCase['reservation']->refresh();
|
|
|
|
|
|
$invoiceGuardEvent = \App\Models\CloudAgentCompletionEvent::findOrFail(
|
|
|
|
|
|
$invoiceGuardReceipt['id']
|
|
|
|
|
|
);
|
|
|
|
|
|
$assert($invoiceGuardPending === [
|
|
|
|
|
|
'accepted' => true,
|
|
|
|
|
|
'disposition' => 'received',
|
|
|
|
|
|
]
|
|
|
|
|
|
&& (string) $invoiceGuardEvent->disposition === 'received'
|
|
|
|
|
|
&& $invoiceGuardEvent->applied_at === null
|
|
|
|
|
|
&& (string) $invoiceGuardCase['job']->effect_resolution === 'unknown'
|
|
|
|
|
|
&& $invoiceGuardCase['order']->refunded_at === null
|
|
|
|
|
|
&& (string) $invoiceGuardCase['reservation']->state === 'uncertain'
|
|
|
|
|
|
&& $bundleState(
|
|
|
|
|
|
$invoiceGuardCase['reservation'],
|
|
|
|
|
|
'uncertain',
|
|
|
|
|
|
'uncertain'
|
|
|
|
|
|
)
|
|
|
|
|
|
&& (string) User::findOrFail(1)->money
|
|
|
|
|
|
=== $invoiceGuardCase['walletBefore']
|
|
|
|
|
|
&& (int) CloudOffer::findOrFail(1)->stock
|
|
|
|
|
|
=== $invoiceGuardCase['stockBefore'],
|
|
|
|
|
|
'Exact create failure bypassed an inconsistent local invoice: '
|
|
|
|
|
|
. json_encode([
|
|
|
|
|
|
'accepted' => $invoiceGuardPending,
|
|
|
|
|
|
'event_disposition' => $invoiceGuardEvent->disposition,
|
|
|
|
|
|
'event_applied_at' => $invoiceGuardEvent->applied_at,
|
|
|
|
|
|
'job_resolution' =>
|
|
|
|
|
|
$invoiceGuardCase['job']->effect_resolution,
|
|
|
|
|
|
'refunded_at' => $invoiceGuardCase['order']->refunded_at,
|
|
|
|
|
|
'reservation' => $invoiceGuardCase['reservation']->state,
|
|
|
|
|
|
'bundle_uncertain' => $bundleState(
|
|
|
|
|
|
$invoiceGuardCase['reservation'],
|
|
|
|
|
|
'uncertain',
|
|
|
|
|
|
'uncertain'
|
|
|
|
|
|
),
|
|
|
|
|
|
'wallet_before' => $invoiceGuardCase['walletBefore'],
|
|
|
|
|
|
'wallet_after' => User::findOrFail(1)->money,
|
|
|
|
|
|
'stock_before' => $invoiceGuardCase['stockBefore'],
|
|
|
|
|
|
'stock_after' => CloudOffer::findOrFail(1)->stock,
|
|
|
|
|
|
], JSON_THROW_ON_ERROR));
|
|
|
|
|
|
$invoiceGuardCase['invoice']->purpose = 'provision';
|
|
|
|
|
|
$invoiceGuardCase['invoice']->save();
|
|
|
|
|
|
$assert(CloudOperationCompletionService::accept($invoiceGuardReceipt)
|
|
|
|
|
|
=== ['accepted' => true, 'disposition' => 'applied'],
|
|
|
|
|
|
'Repaired local invoice did not resume the same create receipt.');
|
|
|
|
|
|
$assertCreateSettled(
|
|
|
|
|
|
$invoiceGuardCase,
|
|
|
|
|
|
'Repaired local invoice did not atomically settle create finance.'
|
|
|
|
|
|
);
|
|
|
|
|
|
|
|
|
|
|
|
// The specialised create finalizer must validate the complete dispatched
|
|
|
|
|
|
// envelope itself. Generic recovery policy deliberately does not own
|
|
|
|
|
|
// create, so neither a damaged attempt count nor a missing receipt
|
|
|
|
|
|
// deadline may reach the refund path.
|
|
|
|
|
|
foreach ([
|
|
|
|
|
|
'attempts' => 2,
|
|
|
|
|
|
'receipt_deadline_at' => null,
|
|
|
|
|
|
] as $damagedField => $damagedValue) {
|
|
|
|
|
|
$suffix = 'receipt-envelope-' . str_replace('_', '-', $damagedField);
|
|
|
|
|
|
$envelopeCase = $makeTimedOutCreate($suffix);
|
|
|
|
|
|
$eventId = 'create-envelope-' . str_replace('_', '-', $damagedField)
|
|
|
|
|
|
. '-event-0001';
|
|
|
|
|
|
$envelopeReceipt = $failedCreateReceipt($envelopeCase, $eventId);
|
|
|
|
|
|
$originalValue = $envelopeCase['job']->{$damagedField};
|
|
|
|
|
|
Capsule::table('cloud_jobs')->where(
|
|
|
|
|
|
'id',
|
|
|
|
|
|
(int) $envelopeCase['job']->id
|
|
|
|
|
|
)->update([$damagedField => $damagedValue]);
|
|
|
|
|
|
$envelopePending = CloudOperationCompletionService::accept(
|
|
|
|
|
|
$envelopeReceipt
|
|
|
|
|
|
);
|
|
|
|
|
|
$envelopeCase['job']->refresh();
|
|
|
|
|
|
$envelopeCase['order']->refresh();
|
|
|
|
|
|
$envelopeCase['invoice']->refresh();
|
|
|
|
|
|
$envelopeCase['reservation']->refresh();
|
|
|
|
|
|
$envelopeEvent = \App\Models\CloudAgentCompletionEvent::findOrFail(
|
|
|
|
|
|
$eventId
|
|
|
|
|
|
);
|
|
|
|
|
|
$assert($envelopePending === [
|
|
|
|
|
|
'accepted' => true,
|
|
|
|
|
|
'disposition' => 'received',
|
|
|
|
|
|
]
|
|
|
|
|
|
&& (string) $envelopeEvent->disposition === 'received'
|
|
|
|
|
|
&& $envelopeEvent->applied_at === null
|
|
|
|
|
|
&& (string) $envelopeCase['job']->effect_resolution === 'unknown'
|
|
|
|
|
|
&& $envelopeCase['order']->refunded_at === null
|
|
|
|
|
|
&& (string) $envelopeCase['invoice']->status === 'paid'
|
|
|
|
|
|
&& (string) $envelopeCase['reservation']->state === 'uncertain'
|
|
|
|
|
|
&& $bundleState(
|
|
|
|
|
|
$envelopeCase['reservation'],
|
|
|
|
|
|
'uncertain',
|
|
|
|
|
|
'uncertain'
|
|
|
|
|
|
)
|
|
|
|
|
|
&& (string) User::findOrFail(1)->money
|
|
|
|
|
|
=== $envelopeCase['walletBefore']
|
|
|
|
|
|
&& (int) CloudOffer::findOrFail(1)->stock
|
|
|
|
|
|
=== $envelopeCase['stockBefore'],
|
|
|
|
|
|
'Damaged create dispatch envelope released finance: '
|
|
|
|
|
|
. $damagedField);
|
|
|
|
|
|
Capsule::table('cloud_jobs')->where(
|
|
|
|
|
|
'id',
|
|
|
|
|
|
(int) $envelopeCase['job']->id
|
|
|
|
|
|
)->update([$damagedField => $originalValue]);
|
|
|
|
|
|
$assert(CloudOperationCompletionService::accept($envelopeReceipt)
|
|
|
|
|
|
=== ['accepted' => true, 'disposition' => 'applied'],
|
|
|
|
|
|
'Repaired create dispatch envelope did not resume receipt: '
|
|
|
|
|
|
. $damagedField);
|
|
|
|
|
|
$assertCreateSettled(
|
|
|
|
|
|
$envelopeCase,
|
|
|
|
|
|
'Repaired dispatch envelope did not settle create finance: '
|
|
|
|
|
|
. $damagedField
|
|
|
|
|
|
);
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
// A fresh read-only Journal failure with complete absence projection can
|
|
|
|
|
|
// finish a timed-out create even when no completion event ever arrived.
|
|
|
|
|
|
$inspectionCase = $makeTimedOutCreate('inspection-no-receipt');
|
|
|
|
|
|
$makeRecoveryReady($inspectionCase['job']);
|
|
|
|
|
|
$inspectionClient = new FinancialCreateRecoveryAgentClient();
|
|
|
|
|
|
$inspectionClient->responses[] = $inspectionFor(
|
|
|
|
|
|
$inspectionCase['job'],
|
|
|
|
|
|
$inspectionCase['identity']
|
|
|
|
|
|
);
|
|
|
|
|
|
CloudAgentSettingsService::$hostId =
|
|
|
|
|
|
(int) $inspectionCase['job']->host_id;
|
|
|
|
|
|
CloudAgentSettingsService::$client = $inspectionClient;
|
|
|
|
|
|
$inspectionRecovery = new CloudAutomaticOperationRecoveryService();
|
|
|
|
|
|
$assert($inspectionRecovery->tick(),
|
|
|
|
|
|
'No-receipt create was not claimed for read-only recovery.');
|
|
|
|
|
|
$assert(count($inspectionClient->requests) === 1
|
|
|
|
|
|
&& $inspectionClient->requests[0]['method'] === 'GET'
|
|
|
|
|
|
&& str_ends_with(
|
|
|
|
|
|
$inspectionClient->requests[0]['target'],
|
|
|
|
|
|
'/inspect'
|
|
|
|
|
|
)
|
|
|
|
|
|
&& $inspectionClient->requests[0]['payload'] === null,
|
|
|
|
|
|
'No-receipt create recovery used something other than one read-only '
|
|
|
|
|
|
. 'inspection.');
|
|
|
|
|
|
$assertCreateSettled(
|
|
|
|
|
|
$inspectionCase,
|
|
|
|
|
|
'Read-only create absence evidence did not atomically settle finance.'
|
|
|
|
|
|
);
|
|
|
|
|
|
|
|
|
|
|
|
// Once inspection has settled the ledger, the first later failure receipt
|
|
|
|
|
|
// is an idempotent ACK of the late delivery. It must not re-enter the received queue,
|
|
|
|
|
|
// rewrite the job incident, or credit/release anything twice.
|
|
|
|
|
|
$inspectionCase['job']->refresh();
|
|
|
|
|
|
$inspectionJobUpdatedAt = (int) $inspectionCase['job']->updated_at;
|
|
|
|
|
|
$inspectionWalletAfter = (string) User::findOrFail(1)->money;
|
|
|
|
|
|
$inspectionStockAfter = (int) CloudOffer::findOrFail(1)->stock;
|
|
|
|
|
|
$inspectionLateReceipt = [
|
|
|
|
|
|
'type' => 'completion',
|
|
|
|
|
|
'id' => 'create-inspection-late-failure-event-0001',
|
|
|
|
|
|
'agent_id' => (string) $inspectionCase['job']->host_id,
|
|
|
|
|
|
'operation_id' => (string) $inspectionCase['job']->operation_id,
|
|
|
|
|
|
...$inspectionCase['identity'],
|
|
|
|
|
|
'status' => 'failed',
|
|
|
|
|
|
'effect_may_have_applied' => false,
|
|
|
|
|
|
'safe_to_resubmit' => false,
|
|
|
|
|
|
'error_code' => 'creation_absent',
|
|
|
|
|
|
'result' => [
|
|
|
|
|
|
'uuid' => $inspectionCase['identity']['resource_id'],
|
|
|
|
|
|
'creation_absent' => true,
|
|
|
|
|
|
'managed_artifacts_absent' => true,
|
|
|
|
|
|
],
|
|
|
|
|
|
];
|
|
|
|
|
|
$inspectionLateAccepted = CloudOperationCompletionService::accept(
|
|
|
|
|
|
$inspectionLateReceipt
|
|
|
|
|
|
);
|
|
|
|
|
|
$inspectionLateDuplicate = CloudOperationCompletionService::accept(
|
|
|
|
|
|
$inspectionLateReceipt
|
|
|
|
|
|
);
|
|
|
|
|
|
$inspectionCase['job']->refresh();
|
|
|
|
|
|
$inspectionLateEvent = \App\Models\CloudAgentCompletionEvent::findOrFail(
|
|
|
|
|
|
$inspectionLateReceipt['id']
|
|
|
|
|
|
);
|
|
|
|
|
|
$assert($inspectionLateAccepted === [
|
|
|
|
|
|
'accepted' => true,
|
|
|
|
|
|
'disposition' => 'applied',
|
|
|
|
|
|
]
|
|
|
|
|
|
&& $inspectionLateDuplicate === $inspectionLateAccepted
|
|
|
|
|
|
&& (string) $inspectionLateEvent->disposition === 'applied'
|
|
|
|
|
|
&& $inspectionLateEvent->applied_at !== null
|
|
|
|
|
|
&& (int) $inspectionCase['job']->updated_at
|
|
|
|
|
|
=== $inspectionJobUpdatedAt
|
|
|
|
|
|
&& !(bool) $inspectionCase['job']->finalization_incident
|
|
|
|
|
|
&& (string) User::findOrFail(1)->money === $inspectionWalletAfter
|
|
|
|
|
|
&& (int) CloudOffer::findOrFail(1)->stock === $inspectionStockAfter,
|
|
|
|
|
|
'A failure receipt arriving after inspection settlement was requeued '
|
|
|
|
|
|
. 'or changed the financial ledger: ' . json_encode([
|
|
|
|
|
|
'first' => $inspectionLateAccepted,
|
|
|
|
|
|
'duplicate' => $inspectionLateDuplicate,
|
|
|
|
|
|
'event_disposition' => $inspectionLateEvent->disposition,
|
|
|
|
|
|
'event_applied_at' => $inspectionLateEvent->applied_at,
|
|
|
|
|
|
'job_updated_at_before' => $inspectionJobUpdatedAt,
|
|
|
|
|
|
'job_updated_at_after' => $inspectionCase['job']->updated_at,
|
|
|
|
|
|
'job_incident' => $inspectionCase['job']->finalization_incident,
|
|
|
|
|
|
'wallet_before' => $inspectionWalletAfter,
|
|
|
|
|
|
'wallet_after' => User::findOrFail(1)->money,
|
|
|
|
|
|
'stock_before' => $inspectionStockAfter,
|
|
|
|
|
|
'stock_after' => CloudOffer::findOrFail(1)->stock,
|
|
|
|
|
|
], JSON_THROW_ON_ERROR));
|
|
|
|
|
|
|
|
|
|
|
|
// An incomplete failed receipt is retained as a late audit event, then a
|
|
|
|
|
|
// fresh read-only inspection supplies the missing managed-artifact proof.
|
|
|
|
|
|
$incompleteCase = $makeTimedOutCreate('incomplete-receipt');
|
|
|
|
|
|
$incompleteReceipt = [
|
|
|
|
|
|
'type' => 'completion',
|
|
|
|
|
|
'id' => 'create-incomplete-failure-event-0001',
|
|
|
|
|
|
'agent_id' => (string) $incompleteCase['job']->host_id,
|
|
|
|
|
|
'operation_id' => (string) $incompleteCase['job']->operation_id,
|
|
|
|
|
|
...$incompleteCase['identity'],
|
|
|
|
|
|
'status' => 'failed',
|
|
|
|
|
|
'effect_may_have_applied' => false,
|
|
|
|
|
|
'safe_to_resubmit' => false,
|
|
|
|
|
|
'error_code' => 'creation_absent',
|
|
|
|
|
|
'result' => [
|
|
|
|
|
|
'uuid' => $incompleteCase['identity']['resource_id'],
|
|
|
|
|
|
'creation_absent' => true,
|
|
|
|
|
|
],
|
|
|
|
|
|
];
|
|
|
|
|
|
$assert(CloudOperationCompletionService::accept($incompleteReceipt)
|
|
|
|
|
|
=== ['accepted' => true, 'disposition' => 'late'],
|
|
|
|
|
|
'Incomplete create absence receipt was not ACKed as late.');
|
|
|
|
|
|
$makeRecoveryReady($incompleteCase['job']);
|
|
|
|
|
|
$incompleteClient = new FinancialCreateRecoveryAgentClient();
|
|
|
|
|
|
$incompleteClient->responses[] = $inspectionFor(
|
|
|
|
|
|
$incompleteCase['job'],
|
|
|
|
|
|
$incompleteCase['identity']
|
|
|
|
|
|
);
|
|
|
|
|
|
CloudAgentSettingsService::$client = $incompleteClient;
|
|
|
|
|
|
$assert((new CloudAutomaticOperationRecoveryService())->tick(),
|
|
|
|
|
|
'Incomplete receipt create did not enter fresh inspection recovery.');
|
|
|
|
|
|
$assertCreateSettled(
|
|
|
|
|
|
$incompleteCase,
|
|
|
|
|
|
'Fresh inspection did not finish an incomplete failed receipt create.'
|
|
|
|
|
|
);
|
|
|
|
|
|
|
|
|
|
|
|
// If PHP fails after durably preparing a no-fence conclusion, a later
|
|
|
|
|
|
// tick must finish locally even while the Agent is offline.
|
|
|
|
|
|
$noFenceRetry = $makeTimedOutCreate('nofence-db-retry');
|
|
|
|
|
|
$noFenceRetry['invoice']->purpose = 'renewal';
|
|
|
|
|
|
$noFenceRetry['invoice']->save();
|
|
|
|
|
|
$makeRecoveryReady($noFenceRetry['job']);
|
|
|
|
|
|
$noFenceClient = new FinancialCreateRecoveryAgentClient();
|
|
|
|
|
|
$noFenceClient->responses[] = $inspectionFor(
|
|
|
|
|
|
$noFenceRetry['job'],
|
|
|
|
|
|
$noFenceRetry['identity']
|
|
|
|
|
|
);
|
|
|
|
|
|
CloudAgentSettingsService::$client = $noFenceClient;
|
|
|
|
|
|
$assert((new CloudAutomaticOperationRecoveryService())->tick(),
|
|
|
|
|
|
'No-fence DB-failure create was not claimed.');
|
|
|
|
|
|
$noFenceRetry['job']->refresh();
|
|
|
|
|
|
$noFenceRetry['order']->refresh();
|
|
|
|
|
|
$noFenceRetry['invoice']->refresh();
|
|
|
|
|
|
$assert(preg_match(
|
|
|
|
|
|
'/^prepared:create_not_applied:v1:[a-f0-9]{64}$/D',
|
|
|
|
|
|
(string) $noFenceRetry['job']->effect_resolution_reason
|
|
|
|
|
|
) === 1
|
|
|
|
|
|
&& (string) $noFenceRetry['job']->effect_resolution === 'unknown'
|
|
|
|
|
|
&& $noFenceRetry['order']->refunded_at === null
|
|
|
|
|
|
&& (string) $noFenceRetry['invoice']->status === 'paid'
|
|
|
|
|
|
&& count($noFenceClient->requests) === 1,
|
|
|
|
|
|
'No-fence financial failure did not retain its durable prepare.');
|
|
|
|
|
|
$noFenceRetry['invoice']->purpose = 'provision';
|
|
|
|
|
|
$noFenceRetry['invoice']->save();
|
|
|
|
|
|
$makeRecoveryReady($noFenceRetry['job']);
|
|
|
|
|
|
// Deliberately leave the response queue empty: any Agent call now fails.
|
|
|
|
|
|
$assert((new CloudAutomaticOperationRecoveryService())->tick(),
|
|
|
|
|
|
'Prepared no-fence create was not retried locally.');
|
|
|
|
|
|
$assert(count($noFenceClient->requests) === 1,
|
|
|
|
|
|
'Prepared no-fence retry contacted an offline Agent.');
|
|
|
|
|
|
$assertCreateSettled(
|
|
|
|
|
|
$noFenceRetry,
|
|
|
|
|
|
'Prepared no-fence create did not settle after the DB fault cleared.'
|
|
|
|
|
|
);
|
|
|
|
|
|
|
|
|
|
|
|
// If the Agent fence write committed but the same PHP transaction later
|
|
|
|
|
|
// failed, recovery reads that immutable verdict once and never POSTs it a
|
|
|
|
|
|
// second time.
|
|
|
|
|
|
$fenceRetry = $makeTimedOutCreate('fence-db-retry');
|
|
|
|
|
|
$fenceRetry['invoice']->purpose = 'renewal';
|
|
|
|
|
|
$fenceRetry['invoice']->save();
|
|
|
|
|
|
$makeRecoveryReady($fenceRetry['job']);
|
|
|
|
|
|
$fenceClient = new FinancialCreateRecoveryAgentClient();
|
|
|
|
|
|
$fenceClient->responses[] = $inspectionFor(
|
|
|
|
|
|
$fenceRetry['job'],
|
|
|
|
|
|
$fenceRetry['identity'],
|
|
|
|
|
|
true
|
|
|
|
|
|
);
|
|
|
|
|
|
$fenceClient->responses[] = new CloudAgentError(
|
|
|
|
|
|
'no committed fence',
|
|
|
|
|
|
404,
|
|
|
|
|
|
'not_found',
|
|
|
|
|
|
false,
|
|
|
|
|
|
404
|
|
|
|
|
|
);
|
|
|
|
|
|
$committedFence = null;
|
|
|
|
|
|
$fenceClient->responses[] = static function (
|
|
|
|
|
|
array $request
|
|
|
|
|
|
) use (&$committedFence, $fenceRetry): array {
|
|
|
|
|
|
$payload = $request['payload'];
|
|
|
|
|
|
$committedFence = [
|
|
|
|
|
|
'operation_id' => (string) $fenceRetry['job']->operation_id,
|
|
|
|
|
|
'resolution_id' => (string) $payload['resolution_id'],
|
|
|
|
|
|
'resolution' => (string) $payload['resolution'],
|
|
|
|
|
|
'inspection_checked_at' =>
|
|
|
|
|
|
(int) $payload['inspection_checked_at'],
|
|
|
|
|
|
'reason_sha256' => (string) $payload['reason_sha256'],
|
|
|
|
|
|
'resolved_at' => time(),
|
|
|
|
|
|
'resolved_by' => 0,
|
|
|
|
|
|
'replayed' => false,
|
|
|
|
|
|
];
|
|
|
|
|
|
return $committedFence;
|
|
|
|
|
|
};
|
|
|
|
|
|
CloudAgentSettingsService::$client = $fenceClient;
|
|
|
|
|
|
$assert((new CloudAutomaticOperationRecoveryService())->tick(),
|
|
|
|
|
|
'Active-fence DB-failure create was not claimed.');
|
|
|
|
|
|
$fenceRetry['job']->refresh();
|
|
|
|
|
|
$assert(is_array($committedFence)
|
|
|
|
|
|
&& count($fenceClient->requests) === 3
|
|
|
|
|
|
&& $fenceClient->requests[0]['method'] === 'GET'
|
|
|
|
|
|
&& str_ends_with($fenceClient->requests[0]['target'], '/inspect')
|
|
|
|
|
|
&& $fenceClient->requests[1]['method'] === 'GET'
|
|
|
|
|
|
&& str_ends_with(
|
|
|
|
|
|
$fenceClient->requests[1]['target'],
|
|
|
|
|
|
'/fence-resolution'
|
|
|
|
|
|
)
|
|
|
|
|
|
&& $fenceClient->requests[2]['method'] === 'POST'
|
|
|
|
|
|
&& str_ends_with(
|
|
|
|
|
|
$fenceClient->requests[2]['target'],
|
|
|
|
|
|
'/resolve-fence'
|
|
|
|
|
|
)
|
|
|
|
|
|
&& preg_match(
|
|
|
|
|
|
'/^prepared:create_not_applied:v1:[a-f0-9]{64}$/D',
|
|
|
|
|
|
(string) $fenceRetry['job']->effect_resolution_reason
|
|
|
|
|
|
) === 1
|
|
|
|
|
|
&& (string) $fenceRetry['job']->effect_resolution === 'unknown',
|
|
|
|
|
|
'Fence commit plus PHP failure did not preserve resumable evidence.');
|
|
|
|
|
|
$fenceRetry['invoice']->purpose = 'provision';
|
|
|
|
|
|
$fenceRetry['invoice']->save();
|
|
|
|
|
|
$makeRecoveryReady($fenceRetry['job']);
|
|
|
|
|
|
$fenceClient->responses[] = $committedFence;
|
|
|
|
|
|
$assert((new CloudAutomaticOperationRecoveryService())->tick(),
|
|
|
|
|
|
'Prepared active-fence create was not retried.');
|
|
|
|
|
|
$assert(count($fenceClient->requests) === 4
|
|
|
|
|
|
&& $fenceClient->requests[3]['method'] === 'GET'
|
|
|
|
|
|
&& str_ends_with(
|
|
|
|
|
|
$fenceClient->requests[3]['target'],
|
|
|
|
|
|
'/fence-resolution'
|
|
|
|
|
|
),
|
|
|
|
|
|
'Prepared active-fence retry wrote or re-inspected Agent state.');
|
|
|
|
|
|
$assertCreateSettled(
|
|
|
|
|
|
$fenceRetry,
|
|
|
|
|
|
'Prepared active-fence create did not settle after DB recovery.'
|
|
|
|
|
|
);
|
|
|
|
|
|
|
|
|
|
|
|
// A fence write can fail before the Agent commits it. The next tick first
|
|
|
|
|
|
// proves that no prior verdict exists, then obtains a fresh inspection.
|
|
|
|
|
|
// checked_at is freshness metadata, not part of the durable conclusion:
|
|
|
|
|
|
// the same facts must produce the same marker and fence reason.
|
|
|
|
|
|
$stableCase = $makeTimedOutCreate('stable-inspection-marker');
|
|
|
|
|
|
$makeRecoveryReady($stableCase['job']);
|
|
|
|
|
|
$stableClient = new FinancialCreateRecoveryAgentClient();
|
|
|
|
|
|
$stableFirstInspection = $inspectionFor(
|
|
|
|
|
|
$stableCase['job'],
|
|
|
|
|
|
$stableCase['identity'],
|
|
|
|
|
|
true
|
|
|
|
|
|
);
|
|
|
|
|
|
$stableClient->responses = [
|
|
|
|
|
|
$stableFirstInspection,
|
|
|
|
|
|
new CloudAgentError('no committed fence', 404, 'not_found', false, 404),
|
|
|
|
|
|
new CloudAgentError('fence write unavailable', 503, 'unavailable', true, 503),
|
|
|
|
|
|
new CloudAgentError('no committed fence', 404, 'not_found', false, 404),
|
|
|
|
|
|
];
|
|
|
|
|
|
CloudAgentSettingsService::$client = $stableClient;
|
|
|
|
|
|
$assert((new CloudAutomaticOperationRecoveryService())->tick(),
|
|
|
|
|
|
'Stable-marker first recovery attempt was not claimed.');
|
|
|
|
|
|
$stableCase['job']->refresh();
|
|
|
|
|
|
$stableMarker = (string) $stableCase['job']->effect_resolution_reason;
|
|
|
|
|
|
$assert(preg_match(
|
|
|
|
|
|
'/^prepared:create_not_applied:v1:[a-f0-9]{64}$/D',
|
|
|
|
|
|
$stableMarker
|
|
|
|
|
|
) === 1
|
|
|
|
|
|
&& (string) $stableCase['job']->effect_resolution === 'unknown'
|
|
|
|
|
|
&& count($stableClient->requests) === 4,
|
|
|
|
|
|
'Failed fence write did not retain a stable prepared create marker.');
|
|
|
|
|
|
$makeRecoveryReady($stableCase['job']);
|
|
|
|
|
|
$stableSecondInspection = $stableFirstInspection;
|
|
|
|
|
|
$stableSecondInspection['checked_at'] =
|
|
|
|
|
|
(int) $stableFirstInspection['checked_at'] + 1;
|
|
|
|
|
|
$stableCommittedFence = null;
|
|
|
|
|
|
$stableClient->responses = [
|
|
|
|
|
|
new CloudAgentError('no committed fence', 404, 'not_found', false, 404),
|
|
|
|
|
|
$stableSecondInspection,
|
|
|
|
|
|
new CloudAgentError('no committed fence', 404, 'not_found', false, 404),
|
|
|
|
|
|
static function (array $request) use (
|
|
|
|
|
|
&$stableCommittedFence,
|
|
|
|
|
|
$stableCase
|
|
|
|
|
|
): array {
|
|
|
|
|
|
$payload = $request['payload'];
|
|
|
|
|
|
$stableCommittedFence = [
|
|
|
|
|
|
'operation_id' => (string) $stableCase['job']->operation_id,
|
|
|
|
|
|
'resolution_id' => (string) $payload['resolution_id'],
|
|
|
|
|
|
'resolution' => (string) $payload['resolution'],
|
|
|
|
|
|
'inspection_checked_at' =>
|
|
|
|
|
|
(int) $payload['inspection_checked_at'],
|
|
|
|
|
|
'reason_sha256' => (string) $payload['reason_sha256'],
|
|
|
|
|
|
'resolved_at' => time(),
|
|
|
|
|
|
'resolved_by' => 0,
|
|
|
|
|
|
'replayed' => false,
|
|
|
|
|
|
];
|
|
|
|
|
|
return $stableCommittedFence;
|
|
|
|
|
|
},
|
|
|
|
|
|
];
|
|
|
|
|
|
$assert((new CloudAutomaticOperationRecoveryService())->tick(),
|
|
|
|
|
|
'Stable-marker create did not resume from fresh inspection.');
|
|
|
|
|
|
$stableExpectedReason =
|
|
|
|
|
|
'自动恢复已确认原开通操作未创建虚拟机或面板管理资源;operation_id='
|
|
|
|
|
|
. (string) $stableCase['job']->operation_id . ';evidence='
|
|
|
|
|
|
. substr($stableMarker, -64);
|
|
|
|
|
|
$assert(is_array($stableCommittedFence)
|
|
|
|
|
|
&& count($stableClient->requests) === 8
|
|
|
|
|
|
&& $stableClient->requests[4]['method'] === 'GET'
|
|
|
|
|
|
&& str_ends_with(
|
|
|
|
|
|
$stableClient->requests[4]['target'],
|
|
|
|
|
|
'/fence-resolution'
|
|
|
|
|
|
)
|
|
|
|
|
|
&& $stableClient->requests[5]['method'] === 'GET'
|
|
|
|
|
|
&& str_ends_with($stableClient->requests[5]['target'], '/inspect')
|
|
|
|
|
|
&& $stableClient->requests[7]['method'] === 'POST'
|
|
|
|
|
|
&& (string) $stableClient->requests[7]['payload']['reason_sha256']
|
|
|
|
|
|
=== hash('sha256', $stableExpectedReason),
|
|
|
|
|
|
'Fresh equivalent inspection changed the prepared fence reason or '
|
|
|
|
|
|
. 'replayed the original create.');
|
|
|
|
|
|
$assertCreateSettled(
|
|
|
|
|
|
$stableCase,
|
|
|
|
|
|
'Stable-marker retry did not finish the create financial ledger.'
|
|
|
|
|
|
);
|
|
|
|
|
|
|
|
|
|
|
|
// Partial/residual host state and an identity mismatch are both
|
|
|
|
|
|
// contradictions, not refund evidence. They remain fenced and retryable.
|
|
|
|
|
|
$conflictCase = $makeTimedOutCreate('inspection-conflicts');
|
|
|
|
|
|
$makeRecoveryReady($conflictCase['job']);
|
|
|
|
|
|
$conflictClient = new FinancialCreateRecoveryAgentClient();
|
|
|
|
|
|
$partialInspection = $inspectionFor(
|
|
|
|
|
|
$conflictCase['job'],
|
|
|
|
|
|
$conflictCase['identity'],
|
|
|
|
|
|
true
|
|
|
|
|
|
);
|
|
|
|
|
|
$partialInspection['outcome'] = 'uncertain';
|
|
|
|
|
|
$partialInspection['summary_code'] = 'evidence_incomplete';
|
|
|
|
|
|
$partialInspection['effect_resolution'] = 'partial';
|
|
|
|
|
|
$partialInspection['error_code'] = 'cleanup_required';
|
|
|
|
|
|
$partialInspection['cleanup'] = [
|
|
|
|
|
|
'available' => true,
|
|
|
|
|
|
'plan_id' => str_repeat('a', 64),
|
|
|
|
|
|
'steps' => ['delete_managed_volumes'],
|
|
|
|
|
|
];
|
|
|
|
|
|
unset($partialInspection['effect_projection']);
|
|
|
|
|
|
$conflictClient->responses[] = $partialInspection;
|
|
|
|
|
|
CloudAgentSettingsService::$client = $conflictClient;
|
|
|
|
|
|
$assert((new CloudAutomaticOperationRecoveryService())->tick(),
|
|
|
|
|
|
'Partial create evidence was not safely deferred.');
|
|
|
|
|
|
$conflictCase['job']->refresh();
|
|
|
|
|
|
$conflictCase['order']->refresh();
|
|
|
|
|
|
$conflictCase['invoice']->refresh();
|
|
|
|
|
|
$assert((string) $conflictCase['job']->effect_resolution === 'unknown'
|
|
|
|
|
|
&& (bool) $conflictCase['job']->finalization_incident
|
|
|
|
|
|
&& $conflictCase['order']->refunded_at === null
|
|
|
|
|
|
&& (string) $conflictCase['invoice']->status === 'paid'
|
|
|
|
|
|
&& (string) User::findOrFail(1)->money
|
|
|
|
|
|
=== $conflictCase['walletBefore']
|
|
|
|
|
|
&& (int) CloudOffer::findOrFail(1)->stock
|
|
|
|
|
|
=== $conflictCase['stockBefore'],
|
|
|
|
|
|
'Partial create evidence released a financial/resource fence.');
|
|
|
|
|
|
$makeRecoveryReady($conflictCase['job']);
|
|
|
|
|
|
$wrongIdentity = $inspectionFor(
|
|
|
|
|
|
$conflictCase['job'],
|
|
|
|
|
|
$conflictCase['identity']
|
|
|
|
|
|
);
|
|
|
|
|
|
$wrongIdentity['resource_id'] = 'wrong-create-resource';
|
|
|
|
|
|
$conflictClient->responses[] = $wrongIdentity;
|
|
|
|
|
|
$assert((new CloudAutomaticOperationRecoveryService())->tick(),
|
|
|
|
|
|
'Create identity contradiction was not safely deferred.');
|
|
|
|
|
|
$conflictCase['job']->refresh();
|
|
|
|
|
|
$assert((string) $conflictCase['job']->effect_resolution === 'unknown'
|
|
|
|
|
|
&& (bool) $conflictCase['job']->finalization_incident
|
|
|
|
|
|
&& $conflictCase['order']->fresh()->refunded_at === null
|
|
|
|
|
|
&& (string) User::findOrFail(1)->money
|
|
|
|
|
|
=== $conflictCase['walletBefore']
|
|
|
|
|
|
&& (int) CloudOffer::findOrFail(1)->stock
|
|
|
|
|
|
=== $conflictCase['stockBefore'],
|
|
|
|
|
|
'Create identity contradiction released a financial/resource fence.');
|
|
|
|
|
|
|
|
|
|
|
|
// A caller cannot reuse an automatic claim after its lease has expired.
|
|
|
|
|
|
$leaseToken = bin2hex(random_bytes(16));
|
|
|
|
|
|
Capsule::table('cloud_jobs')->where('id', (int) $conflictCase['job']->id)
|
|
|
|
|
|
->update([
|
|
|
|
|
|
'available_at' => 0,
|
|
|
|
|
|
'lease_token' => $leaseToken,
|
|
|
|
|
|
'lease_until' => time() + 180,
|
|
|
|
|
|
]);
|
|
|
|
|
|
$staleClaim = CloudJob::findOrFail((int) $conflictCase['job']->id);
|
|
|
|
|
|
Capsule::table('cloud_jobs')->where('id', (int) $conflictCase['job']->id)
|
|
|
|
|
|
->update(['lease_until' => time() - 1]);
|
|
|
|
|
|
$staleRequestCount = count($conflictClient->requests);
|
|
|
|
|
|
$staleRejected = false;
|
|
|
|
|
|
try {
|
|
|
|
|
|
CloudCreateFailureRecoveryService::recover($staleClaim);
|
|
|
|
|
|
} catch (CloudError $error) {
|
|
|
|
|
|
$staleRejected = true;
|
|
|
|
|
|
}
|
|
|
|
|
|
$assert($staleRejected
|
|
|
|
|
|
&& count($conflictClient->requests) === $staleRequestCount
|
|
|
|
|
|
&& (string) CloudJob::findOrFail(
|
|
|
|
|
|
(int) $conflictCase['job']->id
|
|
|
|
|
|
)->effect_resolution === 'unknown'
|
|
|
|
|
|
&& $conflictCase['order']->fresh()->refunded_at === null,
|
|
|
|
|
|
'Expired create recovery lease contacted Agent or released finance.');
|
|
|
|
|
|
|
|
|
|
|
|
// Real address exhaustion on the first scored host is a scheduler miss,
|
|
|
|
|
|
// not a corrupt-order failure. The failed host must leave neither its
|
|
|
|
|
|
// temporary reservation nor a partial address claim before placement
|
|
|
|
|
|
// continues with the next host.
|
|
|
|
|
|
$attachment->state = 'active';
|
|
|
|
|
|
$attachment->verification_state = 'verified';
|
|
|
|
|
|
$attachment->verified_agent_revision = 1;
|
|
|
|
|
|
$attachment->last_verified_at = time();
|
|
|
|
|
|
$attachment->save();
|
|
|
|
|
|
$sourcePublicPool->state = 'active';
|
|
|
|
|
|
$sourcePublicPool->sync_state = 'applied';
|
|
|
|
|
|
$sourcePublicPool->desired_revision = 1;
|
|
|
|
|
|
$sourcePublicPool->observed_revision = 1;
|
|
|
|
|
|
$sourcePublicPool->save();
|
|
|
|
|
|
$sourceIpv6Pool->state = 'active';
|
|
|
|
|
|
$sourceIpv6Pool->sync_state = 'applied';
|
|
|
|
|
|
$sourceIpv6Pool->desired_revision = 1;
|
|
|
|
|
|
$sourceIpv6Pool->observed_revision = 1;
|
|
|
|
|
|
$sourceIpv6Pool->allocation_start = '2404:c140:156:3a::10';
|
|
|
|
|
|
$sourceIpv6Pool->allocation_end = '2404:c140:156:3a::10';
|
|
|
|
|
|
$sourceIpv6Pool->allocation_cursor = null;
|
|
|
|
|
|
$sourceIpv6Pool->max_allocations = 1;
|
|
|
|
|
|
$sourceIpv6Pool->save();
|
|
|
|
|
|
Capsule::table('cloud_public_ip_pools')
|
|
|
|
|
|
->where('id', (int) $sourceIpv6Pool->id)
|
|
|
|
|
|
->update(['allocation_cursor' => null]);
|
|
|
|
|
|
CloudPublicIp::where('public_ip_pool_id', $sourceIpv6Pool->id)->update([
|
|
|
|
|
|
'state' => 'disabled',
|
|
|
|
|
|
'reservation_id' => null,
|
|
|
|
|
|
'reservation_ip_claim_id' => null,
|
|
|
|
|
|
'instance_id' => null,
|
|
|
|
|
|
'operation_id' => null,
|
|
|
|
|
|
'updated_at' => time(),
|
|
|
|
|
|
]);
|
|
|
|
|
|
// Earlier scalar network-change scenarios deliberately reuse and then
|
|
|
|
|
|
// disable concrete IPv4 rows. New provisioning now validates the whole
|
|
|
|
|
|
// pool ledger, so return those test-only rows to the real disabled shape
|
|
|
|
|
|
// before exercising host-capacity fallback.
|
|
|
|
|
|
CloudPublicIp::where('state', 'disabled')->update([
|
|
|
|
|
|
'reservation_id' => null,
|
|
|
|
|
|
'reservation_ip_claim_id' => null,
|
|
|
|
|
|
'instance_id' => null,
|
|
|
|
|
|
'operation_id' => null,
|
|
|
|
|
|
'updated_at' => time(),
|
|
|
|
|
|
]);
|
|
|
|
|
|
if (!CloudPublicIp::where('address_family', 6)
|
|
|
|
|
|
->where('address_binary', inet_pton('2404:c140:156:3a::10'))
|
|
|
|
|
|
->exists()) {
|
|
|
|
|
|
$blockedIpv6 = new CloudPublicIp();
|
|
|
|
|
|
$blockedIpv6->host_id = (int) $host->id;
|
|
|
|
|
|
$blockedIpv6->public_ip_pool_id = (int) $sourceIpv6Pool->id;
|
|
|
|
|
|
$blockedIpv6->address_family = 6;
|
|
|
|
|
|
$blockedIpv6->address = '2404:c140:156:3a::10';
|
|
|
|
|
|
$blockedIpv6->address_binary = inet_pton(
|
|
|
|
|
|
'2404:c140:156:3a::10'
|
|
|
|
|
|
);
|
|
|
|
|
|
$blockedIpv6->prefix_length = 64;
|
|
|
|
|
|
$blockedIpv6->state = 'disabled';
|
|
|
|
|
|
$blockedIpv6->created_at = $blockedIpv6->updated_at = time();
|
|
|
|
|
|
$blockedIpv6->save();
|
|
|
|
|
|
}
|
|
|
|
|
|
if (!CloudPublicIp::where('address_family', 4)
|
|
|
|
|
|
->where('address_binary', inet_pton('8.8.8.23'))->exists()) {
|
|
|
|
|
|
$fallbackProbeIpv4 = new CloudPublicIp();
|
|
|
|
|
|
$fallbackProbeIpv4->host_id = (int) $host->id;
|
|
|
|
|
|
$fallbackProbeIpv4->public_ip_pool_id =
|
|
|
|
|
|
(int) $sourcePublicPool->id;
|
|
|
|
|
|
$fallbackProbeIpv4->address_family = 4;
|
|
|
|
|
|
$fallbackProbeIpv4->address = '8.8.8.23';
|
|
|
|
|
|
$fallbackProbeIpv4->address_binary = inet_pton('8.8.8.23');
|
|
|
|
|
|
$fallbackProbeIpv4->prefix_length = 24;
|
|
|
|
|
|
$fallbackProbeIpv4->state = 'available';
|
|
|
|
|
|
$fallbackProbeIpv4->created_at =
|
|
|
|
|
|
$fallbackProbeIpv4->updated_at = time();
|
|
|
|
|
|
$fallbackProbeIpv4->save();
|
|
|
|
|
|
}
|
|
|
|
|
|
$targetAttachment->state = 'draining';
|
|
|
|
|
|
$targetAttachment->save();
|
|
|
|
|
|
$firstHostPolicy = CloudHostCapacityPolicy::findOrFail((int) $host->id);
|
|
|
|
|
|
$firstHostPolicy->configured_cpu_cores = 1000000;
|
|
|
|
|
|
$firstHostPolicy->configured_memory_mib = 1000000000;
|
|
|
|
|
|
$firstHostPolicy->configured_disk_gib = 1000000000;
|
|
|
|
|
|
$firstHostPolicy->safety_cpu_cores = 0;
|
|
|
|
|
|
$firstHostPolicy->safety_memory_mib = 0;
|
|
|
|
|
|
$firstHostPolicy->safety_disk_gib = 0;
|
|
|
|
|
|
$firstHostPolicy->max_instances = 1000000;
|
|
|
|
|
|
$firstHostPolicy->save();
|
|
|
|
|
|
|
|
|
|
|
|
$fallbackHost = new CloudHost();
|
|
|
|
|
|
$fallbackHost->name = 'fallback-host';
|
|
|
|
|
|
$fallbackHost->enabled = true;
|
|
|
|
|
|
$fallbackHost->resource_pool_id = $pool->id;
|
|
|
|
|
|
$fallbackHost->scheduler_state = 'active';
|
|
|
|
|
|
$fallbackHost->default_network = 'bridge:br0';
|
|
|
|
|
|
$fallbackHost->created_at = $fallbackHost->updated_at = time();
|
|
|
|
|
|
$fallbackHost->save();
|
|
|
|
|
|
Capsule::table('cloud_host_agents')->insert([
|
|
|
|
|
|
'host_id' => $fallbackHost->id,
|
|
|
|
|
|
'sealed_api_secret' => 'fallback-test-only-agent-secret',
|
|
|
|
|
|
'enabled' => 1,
|
|
|
|
|
|
'revision' => 1,
|
|
|
|
|
|
'tested_revision' => 1,
|
|
|
|
|
|
'last_test_status' => 'healthy',
|
|
|
|
|
|
'last_error_code' => '',
|
|
|
|
|
|
'last_agent_version' => 'test',
|
|
|
|
|
|
'network_selectors_json' => '["bridge:br0"]',
|
|
|
|
|
|
'public_network_capabilities_json' => json_encode([
|
|
|
|
|
|
'contract' => 1,
|
|
|
|
|
|
'dual_stack_required' => true,
|
|
|
|
|
|
'project_owned_filter' => true,
|
|
|
|
|
|
'ndp_guard' => true,
|
|
|
|
|
|
], JSON_THROW_ON_ERROR),
|
|
|
|
|
|
'last_tested_at' => time(),
|
|
|
|
|
|
'created_at' => time(),
|
|
|
|
|
|
'updated_at' => time(),
|
|
|
|
|
|
]);
|
|
|
|
|
|
Capsule::table('cloud_host_storage_configurations')->insert([
|
|
|
|
|
|
'host_id' => $fallbackHost->id,
|
|
|
|
|
|
'revision' => 1,
|
|
|
|
|
|
'state' => 'applied',
|
|
|
|
|
|
]);
|
|
|
|
|
|
$fallbackMembership = new CloudHostResourcePoolMembership();
|
|
|
|
|
|
$fallbackMembership->host_id = $fallbackHost->id;
|
|
|
|
|
|
$fallbackMembership->resource_pool_id = $pool->id;
|
|
|
|
|
|
$fallbackMembership->scheduler_state = 'active';
|
|
|
|
|
|
$fallbackMembership->priority = 0;
|
|
|
|
|
|
$fallbackMembership->weight = 100;
|
|
|
|
|
|
$fallbackMembership->created_at =
|
|
|
|
|
|
$fallbackMembership->updated_at = time();
|
|
|
|
|
|
$fallbackMembership->save();
|
|
|
|
|
|
$fallbackAttachment = new CloudHostNetworkAttachment();
|
|
|
|
|
|
$fallbackAttachment->attachment_key = str_repeat('9', 32);
|
|
|
|
|
|
$fallbackAttachment->host_id = $fallbackHost->id;
|
|
|
|
|
|
$fallbackAttachment->line_id = $line->id;
|
|
|
|
|
|
$fallbackAttachment->name = 'Fallback dual stack';
|
|
|
|
|
|
$fallbackAttachment->network_mode = 'selector';
|
|
|
|
|
|
$fallbackAttachment->network_selector = 'bridge:br0';
|
|
|
|
|
|
$fallbackAttachment->state = 'active';
|
|
|
|
|
|
$fallbackAttachment->priority = 0;
|
|
|
|
|
|
$fallbackAttachment->config_revision = 1;
|
|
|
|
|
|
$fallbackAttachment->verified_agent_revision = 1;
|
|
|
|
|
|
$fallbackAttachment->verification_state = 'verified';
|
|
|
|
|
|
$fallbackAttachment->last_verified_at = time();
|
|
|
|
|
|
$fallbackAttachment->verification_error_code = '';
|
|
|
|
|
|
$fallbackAttachment->desired_revision = 1;
|
|
|
|
|
|
$fallbackAttachment->observed_revision = 1;
|
|
|
|
|
|
$fallbackAttachment->sync_state = 'applied';
|
|
|
|
|
|
$fallbackAttachment->last_error_code = '';
|
|
|
|
|
|
$fallbackAttachment->created_at =
|
|
|
|
|
|
$fallbackAttachment->updated_at = time();
|
|
|
|
|
|
$fallbackAttachment->save();
|
|
|
|
|
|
$fallbackIpv4Pool = new CloudPublicIpPool();
|
|
|
|
|
|
$fallbackIpv4Pool->pool_key = str_repeat('8', 32);
|
|
|
|
|
|
$fallbackIpv4Pool->host_id = $fallbackHost->id;
|
|
|
|
|
|
$fallbackIpv4Pool->network_attachment_id = $fallbackAttachment->id;
|
|
|
|
|
|
$fallbackIpv4Pool->name = 'Fallback IPv4';
|
|
|
|
|
|
$fallbackIpv4Pool->address_family = 4;
|
|
|
|
|
|
$fallbackIpv4Pool->cidr = '11.0.0.0/24';
|
|
|
|
|
|
$fallbackIpv4Pool->gateway = '11.0.0.1';
|
|
|
|
|
|
$fallbackIpv4Pool->gateway_on_link = false;
|
|
|
|
|
|
$fallbackIpv4Pool->guest_prefix_length = 24;
|
|
|
|
|
|
$fallbackIpv4Pool->dns_servers_json = '["1.1.1.1"]';
|
|
|
|
|
|
$fallbackIpv4Pool->reserved_ips_json = '[]';
|
|
|
|
|
|
$fallbackIpv4Pool->allocation_start = null;
|
|
|
|
|
|
$fallbackIpv4Pool->allocation_end = null;
|
|
|
|
|
|
$fallbackIpv4Pool->allocation_cursor = null;
|
|
|
|
|
|
$fallbackIpv4Pool->max_allocations = 0;
|
|
|
|
|
|
$fallbackIpv4Pool->state = 'active';
|
|
|
|
|
|
$fallbackIpv4Pool->priority = 0;
|
|
|
|
|
|
$fallbackIpv4Pool->desired_revision = 1;
|
|
|
|
|
|
$fallbackIpv4Pool->observed_revision = 1;
|
|
|
|
|
|
$fallbackIpv4Pool->sync_state = 'applied';
|
|
|
|
|
|
$fallbackIpv4Pool->last_error_code = '';
|
|
|
|
|
|
$fallbackIpv4Pool->created_at =
|
|
|
|
|
|
$fallbackIpv4Pool->updated_at = time();
|
|
|
|
|
|
$fallbackIpv4Pool->save();
|
|
|
|
|
|
$fallbackIpv6Pool = new CloudPublicIpPool();
|
|
|
|
|
|
$fallbackIpv6Pool->pool_key = str_repeat('7', 32);
|
|
|
|
|
|
$fallbackIpv6Pool->host_id = $fallbackHost->id;
|
|
|
|
|
|
$fallbackIpv6Pool->network_attachment_id = $fallbackAttachment->id;
|
|
|
|
|
|
$fallbackIpv6Pool->name = 'Fallback IPv6';
|
|
|
|
|
|
$fallbackIpv6Pool->address_family = 6;
|
|
|
|
|
|
$fallbackIpv6Pool->cidr = '2606:4700:4700:200::/64';
|
|
|
|
|
|
$fallbackIpv6Pool->gateway = '2606:4700:4700:200::1';
|
|
|
|
|
|
$fallbackIpv6Pool->gateway_on_link = false;
|
|
|
|
|
|
$fallbackIpv6Pool->guest_prefix_length = 64;
|
|
|
|
|
|
$fallbackIpv6Pool->dns_servers_json = '["2606:4700:4700::1111"]';
|
|
|
|
|
|
$fallbackIpv6Pool->reserved_ips_json = '[]';
|
|
|
|
|
|
$fallbackIpv6Pool->allocation_start = '2606:4700:4700:200::10';
|
|
|
|
|
|
$fallbackIpv6Pool->allocation_end = '2606:4700:4700:200::20';
|
|
|
|
|
|
$fallbackIpv6Pool->allocation_cursor = null;
|
|
|
|
|
|
$fallbackIpv6Pool->max_allocations = 4;
|
|
|
|
|
|
$fallbackIpv6Pool->state = 'active';
|
|
|
|
|
|
$fallbackIpv6Pool->priority = 0;
|
|
|
|
|
|
$fallbackIpv6Pool->desired_revision = 1;
|
|
|
|
|
|
$fallbackIpv6Pool->observed_revision = 1;
|
|
|
|
|
|
$fallbackIpv6Pool->sync_state = 'applied';
|
|
|
|
|
|
$fallbackIpv6Pool->last_error_code = '';
|
|
|
|
|
|
$fallbackIpv6Pool->created_at =
|
|
|
|
|
|
$fallbackIpv6Pool->updated_at = time();
|
|
|
|
|
|
$fallbackIpv6Pool->save();
|
|
|
|
|
|
$fallbackIpv4 = new CloudPublicIp();
|
|
|
|
|
|
$fallbackIpv4->host_id = $fallbackHost->id;
|
|
|
|
|
|
$fallbackIpv4->public_ip_pool_id = $fallbackIpv4Pool->id;
|
|
|
|
|
|
$fallbackIpv4->address_family = 4;
|
|
|
|
|
|
$fallbackIpv4->address = '11.0.0.10';
|
|
|
|
|
|
$fallbackIpv4->address_binary = inet_pton('11.0.0.10');
|
|
|
|
|
|
$fallbackIpv4->prefix_length = 24;
|
|
|
|
|
|
$fallbackIpv4->state = 'available';
|
|
|
|
|
|
$fallbackIpv4->created_at = $fallbackIpv4->updated_at = time();
|
|
|
|
|
|
$fallbackIpv4->save();
|
|
|
|
|
|
$fallbackPolicy = new CloudHostCapacityPolicy();
|
|
|
|
|
|
$fallbackPolicy->host_id = $fallbackHost->id;
|
|
|
|
|
|
$fallbackPolicy->configured_cpu_cores = 4;
|
|
|
|
|
|
$fallbackPolicy->configured_memory_mib = 4096;
|
|
|
|
|
|
$fallbackPolicy->configured_disk_gib = 40;
|
|
|
|
|
|
$fallbackPolicy->safety_cpu_cores = 0;
|
|
|
|
|
|
$fallbackPolicy->safety_memory_mib = 0;
|
|
|
|
|
|
$fallbackPolicy->safety_disk_gib = 0;
|
|
|
|
|
|
$fallbackPolicy->max_instances = 2;
|
|
|
|
|
|
$fallbackPolicy->created_at = $fallbackPolicy->updated_at = time();
|
|
|
|
|
|
$fallbackPolicy->save();
|
|
|
|
|
|
$fallbackScoreOrder = new CloudOrder();
|
|
|
|
|
|
$fallbackScoreOrder->user_id = 1;
|
|
|
|
|
|
$fallbackScoreOrder->plan_id = $plan->id;
|
|
|
|
|
|
$fallbackScoreOrder->offer_id = $offer->id;
|
|
|
|
|
|
$fallbackScoreOrder->resource_pool_id = $pool->id;
|
|
|
|
|
|
$fallbackScoreOrder->host_id = $fallbackHost->id;
|
|
|
|
|
|
$fallbackScoreOrder->reservation_id = null;
|
|
|
|
|
|
$fallbackScoreOrder->deployment_snapshot_id = null;
|
|
|
|
|
|
$fallbackScoreOrder->idempotency_key = 'fallback-score-fixture';
|
|
|
|
|
|
$fallbackScoreOrder->request_hash = str_repeat('4', 64);
|
|
|
|
|
|
$fallbackScoreOrder->amount = '0.00';
|
|
|
|
|
|
$fallbackScoreOrder->currency = 'USD';
|
|
|
|
|
|
$fallbackScoreOrder->status = 'succeeded';
|
|
|
|
|
|
$fallbackScoreOrder->spec_json = '{}';
|
|
|
|
|
|
$fallbackScoreOrder->sealed_payload = null;
|
|
|
|
|
|
$fallbackScoreOrder->instance_id = 999999;
|
|
|
|
|
|
$fallbackScoreOrder->refunded_at = null;
|
|
|
|
|
|
$fallbackScoreOrder->stock_reserved_at = null;
|
|
|
|
|
|
$fallbackScoreOrder->stock_released_at = null;
|
|
|
|
|
|
$fallbackScoreOrder->created_at =
|
|
|
|
|
|
$fallbackScoreOrder->updated_at = time();
|
|
|
|
|
|
$fallbackScoreOrder->save();
|
|
|
|
|
|
$fallbackScoreReservation = new CloudResourceReservation();
|
|
|
|
|
|
$fallbackScoreReservation->reservation_key = str_repeat('6', 32);
|
|
|
|
|
|
$fallbackScoreReservation->operation_id = str_repeat('5', 32);
|
|
|
|
|
|
$fallbackScoreReservation->order_id = $fallbackScoreOrder->id;
|
|
|
|
|
|
$fallbackScoreReservation->backup_restore_id = null;
|
|
|
|
|
|
$fallbackScoreReservation->offer_id = $offer->id;
|
|
|
|
|
|
$fallbackScoreReservation->resource_pool_id = $pool->id;
|
|
|
|
|
|
$fallbackScoreReservation->host_id = $fallbackHost->id;
|
|
|
|
|
|
$fallbackScoreReservation->user_id = 1;
|
|
|
|
|
|
$fallbackScoreReservation->instance_id = 999999;
|
|
|
|
|
|
$fallbackScoreReservation->network_attachment_id = null;
|
|
|
|
|
|
$fallbackScoreReservation->public_ip_pool_id = null;
|
|
|
|
|
|
$fallbackScoreReservation->public_ip_id = null;
|
|
|
|
|
|
$fallbackScoreReservation->cpu_cores = 1;
|
|
|
|
|
|
$fallbackScoreReservation->memory_mib = 512;
|
|
|
|
|
|
$fallbackScoreReservation->disk_gib = 1;
|
|
|
|
|
|
$fallbackScoreReservation->state = 'committed';
|
|
|
|
|
|
$fallbackScoreReservation->expires_at = null;
|
|
|
|
|
|
$fallbackScoreReservation->released_at = null;
|
|
|
|
|
|
$fallbackScoreReservation->release_reason = '';
|
|
|
|
|
|
$fallbackScoreReservation->created_at =
|
|
|
|
|
|
$fallbackScoreReservation->updated_at = time();
|
|
|
|
|
|
$fallbackScoreReservation->save();
|
|
|
|
|
|
$fallbackScoreOrder->reservation_id = $fallbackScoreReservation->id;
|
|
|
|
|
|
$fallbackScoreOrder->save();
|
|
|
|
|
|
Capsule::table('cloud_template_replicas')->insert([
|
|
|
|
|
|
'host_id' => $fallbackHost->id,
|
|
|
|
|
|
'template_id' => $template->id,
|
|
|
|
|
|
'revision' => $template->revision,
|
|
|
|
|
|
'generation_token' => $template->generation_token,
|
|
|
|
|
|
'state' => 'active',
|
|
|
|
|
|
'image_bytes' => $template->image_bytes,
|
|
|
|
|
|
'sha256' => $template->sha256,
|
|
|
|
|
|
'operation_id' => 'fixture-fallback-template-import',
|
|
|
|
|
|
'error_code' => '',
|
|
|
|
|
|
'created_at' => time(),
|
|
|
|
|
|
'updated_at' => time(),
|
|
|
|
|
|
]);
|
|
|
|
|
|
$offerForFallback = CloudOffer::findOrFail((int) $offer->id);
|
|
|
|
|
|
$offerForFallback->stock = -1;
|
|
|
|
|
|
$offerForFallback->save();
|
|
|
|
|
|
$fallbackUser = User::findOrFail(1);
|
|
|
|
|
|
$fallbackUser->money = '100.00';
|
|
|
|
|
|
$fallbackUser->save();
|
|
|
|
|
|
$beforeFallbackReservationId = (int) (
|
|
|
|
|
|
CloudResourceReservation::max('id') ?? 0
|
|
|
|
|
|
);
|
|
|
|
|
|
$fallbackOrder = CloudOrderService::purchase(array_replace($input, [
|
|
|
|
|
|
'name' => 'vm-host-capacity-fallback',
|
|
|
|
|
|
'idempotency_key' => 'test-host-capacity-fallback',
|
|
|
|
|
|
]), $actor);
|
|
|
|
|
|
CloudInvoiceService::payBalance(
|
|
|
|
|
|
CloudInvoice::where('order_id', $fallbackOrder->id)->firstOrFail(),
|
|
|
|
|
|
$actor,
|
|
|
|
|
|
'provision-host-capacity-fallback'
|
|
|
|
|
|
);
|
|
|
|
|
|
$fallbackReservation = CloudResourceReservation::where(
|
|
|
|
|
|
'order_id',
|
|
|
|
|
|
$fallbackOrder->id
|
|
|
|
|
|
)->firstOrFail();
|
|
|
|
|
|
$assert((int) $fallbackOrder->fresh()->host_id
|
|
|
|
|
|
=== (int) $fallbackHost->id
|
|
|
|
|
|
&& (int) $fallbackReservation->host_id === (int) $fallbackHost->id
|
|
|
|
|
|
&& (int) $fallbackReservation->id
|
|
|
|
|
|
>= $beforeFallbackReservationId + 2
|
|
|
|
|
|
&& Capsule::table('cloud_resource_reservation_ips')
|
|
|
|
|
|
->where('reservation_id', (int) $fallbackReservation->id)
|
|
|
|
|
|
->count() === 2
|
|
|
|
|
|
&& CloudResourceReservation::where('order_id', $fallbackOrder->id)
|
|
|
|
|
|
->where('host_id', $host->id)->count() === 0
|
|
|
|
|
|
&& Capsule::table('cloud_resource_reservation_ips as claim')
|
|
|
|
|
|
->leftJoin('cloud_resource_reservations as reservation',
|
|
|
|
|
|
'reservation.id', '=', 'claim.reservation_id')
|
|
|
|
|
|
->where('claim.reservation_id', '>', $beforeFallbackReservationId)
|
|
|
|
|
|
->whereNull('reservation.id')->count() === 0,
|
|
|
|
|
|
'Capacity fallback did not continue to host two or left host-one ledger residue.');
|
|
|
|
|
|
|
|
|
|
|
|
echo "Cloud SQLite purchase, provisioning, refund, ambiguity, quota and tenant checks passed (crypto double only).\n";
|
|
|
|
|
|
}
|