You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
1291 lines
67 KiB
PHP
1291 lines
67 KiB
PHP
<?php
|
|
|
|
namespace App\Services\Cloud {
|
|
final class CloudEngineClient
|
|
{
|
|
public static string $power = 'running';
|
|
public static array $commands = [];
|
|
public static array $operations = [];
|
|
public static array $operationOwners = [];
|
|
public static array $postAttempts = [];
|
|
public static array $lookups = [];
|
|
public static ?CloudError $postFailure = null;
|
|
public static ?CloudError $lookupFailure = null;
|
|
public static ?CloudError $adminLookupFailure = null;
|
|
public static bool $loseReceipt = false;
|
|
public static bool $failState = false;
|
|
public static int $stateReads = 0;
|
|
public static ?int $failStateAtRead = null;
|
|
public static bool $startChangesPower = true;
|
|
public static $vpcRespond;
|
|
public static int $applies = 0;
|
|
public static int $readbacks = 0;
|
|
public static $vpcRead;
|
|
|
|
public function __construct(...$args) {}
|
|
private static function assertPowerActor(array $actor, ?string $operationId = null,
|
|
bool $adminLookup = false): void
|
|
{
|
|
if ($adminLookup && ($actor['is_admin'] ?? null) === true
|
|
&& ($actor['is_super_admin'] ?? null) === true
|
|
&& ($actor['can_manage'] ?? null) === true
|
|
&& is_int($actor['user_id'] ?? null) && $actor['user_id'] > 0
|
|
&& ($actor['operation_id'] ?? null) === $operationId) {
|
|
return;
|
|
}
|
|
if (!is_int($actor['user_id'] ?? null) || $actor['user_id'] < 1
|
|
|| ($actor['is_admin'] ?? null) !== false
|
|
|| ($actor['is_super_admin'] ?? null) !== false
|
|
|| ($actor['can_manage'] ?? null) !== true
|
|
|| ($actor['session_id'] ?? null) !== ''
|
|
|| ($operationId !== null && ($actor['operation_id'] ?? null) !== $operationId)) {
|
|
throw new \RuntimeException('Power request did not use a bounded owner actor.');
|
|
}
|
|
}
|
|
public function host(int $host, string $method, string $path, array $actor,
|
|
?array $body = null, ?string $operationId = null): array
|
|
{
|
|
self::assertPowerActor($actor, $operationId);
|
|
$uuid = explode('/', $path)[1];
|
|
if ($method === 'GET') {
|
|
self::$stateReads++;
|
|
if (self::$failState
|
|
|| self::$failStateAtRead === self::$stateReads) {
|
|
self::$failStateAtRead = null;
|
|
throw new CloudError('offline', 502);
|
|
}
|
|
return ['uuid' => $uuid, 'state' => self::$power];
|
|
}
|
|
self::$postAttempts[] = [$operationId, $body['action'], $actor];
|
|
if (isset(self::$operationOwners[$operationId])
|
|
&& self::$operationOwners[$operationId] !== $actor['user_id']) {
|
|
throw new CloudError('missing', 404);
|
|
}
|
|
if (self::$postFailure !== null) {
|
|
throw self::$postFailure;
|
|
}
|
|
self::$commands[] = [$operationId, $body['action'], $uuid];
|
|
$result = ['operation_id' => $operationId, 'status' => 'succeeded',
|
|
'result' => ['uuid' => $uuid, 'action' => $body['action'], 'command_accepted' => true]];
|
|
self::$operations[$operationId] = $result;
|
|
self::$operationOwners[$operationId] = $actor['user_id'];
|
|
if ($body['action'] === 'start' && self::$startChangesPower) {
|
|
self::$power = 'running';
|
|
}
|
|
if (self::$loseReceipt) {
|
|
self::$loseReceipt = false;
|
|
throw new CloudError('lost response', 502, 'engine_transport_uncertain');
|
|
}
|
|
return $result;
|
|
}
|
|
public function request(string $method, string $path, array $actor,
|
|
?array $body = null, ?string $operationId = null): array
|
|
{
|
|
self::assertPowerActor($actor, $operationId, true);
|
|
self::$lookups[] = [$operationId, $actor];
|
|
if (($actor['is_admin'] ?? false) === true && self::$adminLookupFailure !== null) {
|
|
throw self::$adminLookupFailure;
|
|
}
|
|
if (self::$lookupFailure !== null) {
|
|
throw self::$lookupFailure;
|
|
}
|
|
if (isset(self::$operationOwners[basename($path)])
|
|
&& self::$operationOwners[basename($path)] !== $actor['user_id']
|
|
&& ($actor['is_admin'] ?? false) !== true) {
|
|
throw new CloudError('missing', 404);
|
|
}
|
|
return self::$operations[basename($path)]
|
|
?? throw new CloudError('missing', 404);
|
|
}
|
|
public function vpc(string $method, string $path, array $actor,
|
|
?array $body = null, ?string $operationId = null): array
|
|
{
|
|
if ($method === 'GET') {
|
|
self::$readbacks++;
|
|
return self::$vpcRead !== null ? (self::$vpcRead)($path)
|
|
: throw new CloudError('missing', 404);
|
|
}
|
|
self::$applies++;
|
|
return (self::$vpcRespond)($body);
|
|
}
|
|
}
|
|
}
|
|
|
|
namespace {
|
|
use App\Models\CloudInstance;
|
|
use App\Models\CloudVpc;
|
|
use App\Models\CloudVpcMember;
|
|
use App\Models\CloudVpcOperation;
|
|
use App\Services\Cloud\CloudEngineClient;
|
|
use App\Services\Cloud\CloudError;
|
|
use App\Services\Cloud\CloudVpcPowerWorkflow as Workflow;
|
|
use App\Services\Cloud\CloudVpcService;
|
|
use App\Services\Cloud\CloudVpcWorkerService;
|
|
use Illuminate\Database\Capsule\Manager as Capsule;
|
|
|
|
require __DIR__ . '/CloudVpcMemberReuseTest.php';
|
|
if (!isset($capsule)) { return; }
|
|
|
|
$make = static function (int $id, string $power = 'running') use ($seedVpc, $seedInstance, $actor): CloudVpcOperation {
|
|
$vpc = $seedVpc($id, '10.121.' . ($id % 250) . '.0/24',
|
|
'10.121.' . ($id % 250) . '.0/24', 10);
|
|
$seedInstance($id * 100, 10);
|
|
CloudVpcService::attachMember($vpc['vpc_key'], $actor, [
|
|
'instance_id' => $id * 100, 'request_key' => 'auto-member-' . $id,
|
|
]);
|
|
$operation = CloudVpcOperation::where('vpc_id', $id)
|
|
->orderByDesc('desired_revision')->first();
|
|
$instance = CloudInstance::find($id * 100);
|
|
$member = CloudVpcMember::where('instance_id', $id * 100)->first();
|
|
// Manufacture a pre-refactor row to verify compatibility recovery. New
|
|
// public attach/detach requests never initialize this workflow.
|
|
Workflow::initialize($operation, $instance, $member);
|
|
CloudEngineClient::$power = $power;
|
|
return $operation->fresh();
|
|
};
|
|
$legacyDetach = static function (CloudVpc $vpc, CloudVpcMember $member,
|
|
string $requestKey) use ($actor): CloudVpcOperation {
|
|
CloudVpcService::detachMember($vpc->vpc_key, $member->member_key, $actor, [
|
|
'request_key' => $requestKey,
|
|
]);
|
|
$operation = CloudVpcOperation::where('vpc_id', $vpc->id)
|
|
->orderByDesc('desired_revision')->first();
|
|
Workflow::initialize(
|
|
$operation,
|
|
CloudInstance::find((int) $member->instance_id),
|
|
$member->fresh()
|
|
);
|
|
return $operation->fresh();
|
|
};
|
|
$claim = static function (CloudVpcOperation $operation): array {
|
|
$operation = $operation->fresh();
|
|
$operation->lease_token = bin2hex(random_bytes(16));
|
|
$operation->lease_until = time() + 120;
|
|
$operation->status = 'running';
|
|
$operation->save();
|
|
return [$operation, $operation->lease_token];
|
|
};
|
|
$observed = static function (CloudVpcOperation $operation): void {
|
|
Workflow::configurationObserved($operation->fresh(), true);
|
|
};
|
|
$phase = static fn (CloudVpcOperation $operation): string => Workflow::state($operation->fresh())['phase'];
|
|
|
|
// Normal running guest: command acknowledgement, separate state observation,
|
|
// exact configuration gate, then one automatic restart.
|
|
$op = $make(1201);
|
|
[$op, $lease] = $claim($op);
|
|
$assert(Workflow::beforeApply($op, $lease, $actor, false) === false
|
|
&& $phase($op) === 'waiting_stopped', 'Shutdown must finish only the command step.');
|
|
$assert($op->fresh()->lease_token === null, 'Waiting for guest shutdown held its lease.');
|
|
$count = count(CloudEngineClient::$commands);
|
|
[$op, $lease] = $claim($op);
|
|
$assert(!Workflow::beforeApply($op, $lease, $actor, false)
|
|
&& count(CloudEngineClient::$commands) === $count, 'Polling resent shutdown.');
|
|
CloudEngineClient::$power = 'shutoff';
|
|
[$op, $lease] = $claim($op);
|
|
$reads = CloudEngineClient::$stateReads;
|
|
CloudEngineClient::$failStateAtRead = $reads + 2;
|
|
$assert(Workflow::beforeApply($op, $lease, $actor, false)
|
|
&& CloudEngineClient::$stateReads === $reads + 1,
|
|
'Stopped guest did not enter apply with one state read.');
|
|
CloudEngineClient::$failStateAtRead = null;
|
|
Workflow::afterApply($op, $lease, $actor);
|
|
$assert(count(CloudEngineClient::$commands) === $count, 'Guest started before exact configuration proof.');
|
|
[$op, $lease] = $claim($op);
|
|
$assert(Workflow::beforeApply($op, $lease, $actor, true), 'Explicit retry did not resume configuration.');
|
|
$observed($op);
|
|
Workflow::afterApply($op, $lease, $actor);
|
|
$assert($phase($op) === 'starting' && count(CloudEngineClient::$commands) === $count + 1
|
|
&& end(CloudEngineClient::$commands)[1] === 'start',
|
|
'Accepted start was not retained for power readback.');
|
|
[$op, $lease] = $claim($op);
|
|
$assert(!Workflow::beforeApply($op, $lease, $actor, false)
|
|
&& $phase($op) === 'completed', 'Running readback did not finish automatic restoration.');
|
|
$completedOp = $op;
|
|
|
|
// An accepted shutdown survives one transient state read and converges using
|
|
// the same child ID and fixed deadline. No manual retry or second POST occurs.
|
|
$op = $make(1275);
|
|
[$op, $lease] = $claim($op); Workflow::beforeApply($op, $lease, $actor, false);
|
|
$state = Workflow::state($op->fresh());
|
|
$deadline = $state['shutdown_deadline']; $childId = $state['shutdown_id'];
|
|
$commands = count(CloudEngineClient::$commands);
|
|
CloudEngineClient::$failStateAtRead = CloudEngineClient::$stateReads + 1;
|
|
[$op, $lease] = $claim($op);
|
|
$assert(!Workflow::beforeApply($op, $lease, $actor, false)
|
|
&& $phase($op) === 'waiting_stopped'
|
|
&& $op->fresh()->status === 'waiting_observed'
|
|
&& Workflow::state($op->fresh())['shutdown_deadline'] === $deadline
|
|
&& Workflow::state($op->fresh())['shutdown_id'] === $childId
|
|
&& count(CloudEngineClient::$commands) === $commands,
|
|
'Accepted shutdown state read did not enter bounded automatic recovery.');
|
|
CloudEngineClient::$power = 'shutoff';
|
|
[$op, $lease] = $claim($op);
|
|
$assert(Workflow::beforeApply($op, $lease, $actor, false)
|
|
&& $phase($op) === 'applying'
|
|
&& count(CloudEngineClient::$commands) === $commands,
|
|
'Accepted shutdown did not recover after a transient state failure.');
|
|
|
|
// A historical auto-power request remains exactly replayable during a rolling
|
|
// deployment, but a new request can no longer create that workflow.
|
|
$vpc = CloudVpc::find($completedOp->vpc_id);
|
|
$legacyPayload = json_decode((string) $completedOp->payload_json, true);
|
|
$legacySubnetKey = (string) ($legacyPayload['members'][0]['subnet_id'] ?? '');
|
|
$completedOp->request_hash = hash('sha256', $canonicalJson([
|
|
'kind' => 'member_attach',
|
|
'input' => [
|
|
'subnet_key' => $legacySubnetKey,
|
|
'instance_id' => 120100,
|
|
'private_ip' => null,
|
|
'auto_power' => true,
|
|
],
|
|
]));
|
|
$completedOp->save();
|
|
CloudVpcService::attachMember($vpc->vpc_key, $actor, [
|
|
'instance_id' => 120100, 'request_key' => 'auto-member-1201', 'auto_power' => true,
|
|
]);
|
|
$expectCloudError(static fn () => CloudVpcService::attachMember($vpc->vpc_key, $actor, [
|
|
'instance_id' => 120100, 'request_key' => 'auto-member-1201', 'auto_power' => false,
|
|
]), 409);
|
|
$retired = $expectCloudError(static fn () => CloudVpcService::attachMember(
|
|
$vpc->vpc_key, $actor, [
|
|
'instance_id' => 120100, 'request_key' => 'new-auto-member-1201',
|
|
'auto_power' => true,
|
|
]
|
|
), 422);
|
|
$assert($retired->errorCode === 'vpc_auto_power_retired',
|
|
'A new automatic-power workflow was still admitted.');
|
|
$expectCloudError(static fn () => CloudVpcService::attachMember($vpc->vpc_key, $actor, [
|
|
'instance_id' => 120100, 'request_key' => 'auto-member-1201', 'auto_power' => true,
|
|
'subnet_key' => str_repeat('a', 32),
|
|
]), 422);
|
|
foreach (['true', 1, null] as $badFlag) {
|
|
$expectCloudError(static fn () => Workflow::requested(['auto_power' => $badFlag]), 422);
|
|
}
|
|
|
|
// A previously stopped or already shutting-down guest must never be restarted.
|
|
foreach (['shutoff', 'shutdown'] as $offset => $initial) {
|
|
$op = $make(1202 + $offset, $initial);
|
|
$count = count(CloudEngineClient::$commands);
|
|
[$op, $lease] = $claim($op);
|
|
$ready = Workflow::beforeApply($op, $lease, $actor, false);
|
|
if (!$ready) {
|
|
CloudEngineClient::$power = 'shutoff';
|
|
[$op, $lease] = $claim($op);
|
|
$assert(Workflow::beforeApply($op, $lease, $actor, false), 'Existing shutdown did not converge.');
|
|
}
|
|
$observed($op);
|
|
Workflow::afterApply($op, $lease, $actor);
|
|
$assert($phase($op) === 'completed' && count(CloudEngineClient::$commands) === $count,
|
|
'A pre-existing stopped/shutdown state generated power commands.');
|
|
}
|
|
|
|
// Lost receipt: repeat uses the same durable engine operation, no second POST.
|
|
$op = $make(1204);
|
|
CloudEngineClient::$loseReceipt = true;
|
|
[$op, $lease] = $claim($op);
|
|
Workflow::beforeApply($op, $lease, $actor, false);
|
|
$assert($phase($op) === 'shutting_down'
|
|
&& $op->fresh()->status === 'waiting_observed',
|
|
'Lost receipt did not enter bounded fixed-ID recovery.');
|
|
$count = count(CloudEngineClient::$commands);
|
|
[$op, $lease] = $claim($op);
|
|
Workflow::beforeApply($op, $lease, $actor, false);
|
|
$assert($phase($op) === 'waiting_stopped' && count(CloudEngineClient::$commands) === $count,
|
|
'Lost accepted command was submitted again.');
|
|
|
|
// Graceful timeout pauses, permits force-off, and cancels automatic restoration.
|
|
$state = Workflow::state($op->fresh());
|
|
$state['shutdown_deadline'] = time() - 1;
|
|
$op->power_workflow_json = json_encode($state); $op->save();
|
|
[$op, $lease] = $claim($op);
|
|
Workflow::beforeApply($op, $lease, $actor, false);
|
|
$assert($phase($op) === 'paused' && $op->fresh()->status === 'paused', 'Timeout became an uncertain mutation.');
|
|
$instance = CloudInstance::find(120400);
|
|
$instance->getConnection()->transaction(static function () use ($instance): void {
|
|
Workflow::lockForInstance($instance);
|
|
$locked = CloudInstance::where('id', $instance->id)->lockForUpdate()->first();
|
|
Workflow::manualIntentLocked($locked, 'action', 'poweroff');
|
|
});
|
|
CloudEngineClient::$power = 'shutoff';
|
|
[$op, $lease] = $claim($op);
|
|
$assert(Workflow::beforeApply($op, $lease, $actor, true), 'Manual force-off cannot resume configuration.');
|
|
$observed($op); Workflow::afterApply($op, $lease, $actor);
|
|
$assert($phase($op) === 'completed' && count(CloudEngineClient::$commands) === $count,
|
|
'Automation undid newer manual power intent.');
|
|
|
|
// Only the active revision can continue; unfinished workflows forbid replacing
|
|
// their VPC plan, including paused steps and last-member removal.
|
|
$op = $make(1205, 'shutoff');
|
|
$vpc = CloudVpc::find($op->vpc_id);
|
|
$expectCloudError(static fn () => Workflow::assertMutationAllowed($vpc), 409);
|
|
[$op, $lease] = $claim($op);
|
|
Capsule::table('cloud_vpcs')->where('id', $vpc->id)->update(['desired_revision' => 999]);
|
|
$count = count(CloudEngineClient::$commands);
|
|
$expectCloudError(static fn () => Workflow::beforeApply($op, $lease, $actor, false), 409);
|
|
$assert(count(CloudEngineClient::$commands) === $count, 'Stale revision sent a power command.');
|
|
|
|
// Read-only failures pause without a child action or a lingering task lease.
|
|
$op = $make(1206);
|
|
CloudEngineClient::$failState = true;
|
|
[$op, $lease] = $claim($op);
|
|
try { Workflow::beforeApply($op, $lease, $actor, false); }
|
|
catch (CloudError $error) { Workflow::handleFailure($op, $lease, $error); }
|
|
CloudEngineClient::$failState = false;
|
|
$assert($op->fresh()->status === 'paused' && $op->fresh()->lease_token === null,
|
|
'Read-only preflight failure retained a lease.');
|
|
|
|
// Crash during apply has a bounded lease, so later manual power stays possible.
|
|
$op = $make(1207, 'shutoff');
|
|
[$op, $lease] = $claim($op); Workflow::beforeApply($op, $lease, $actor, false);
|
|
$op = $op->fresh(); $op->lease_until = time() - 1; $op->save();
|
|
$instance = CloudInstance::find(120700);
|
|
$instance->getConnection()->transaction(static function () use ($instance): void {
|
|
Workflow::lockForInstance($instance);
|
|
Workflow::manualIntentLocked($instance, 'action', 'poweroff');
|
|
});
|
|
$assert(Workflow::state($op->fresh())['resume_cancelled'] === true,
|
|
'Expired apply worker permanently blocked ordinary power.');
|
|
|
|
// Bare apply coordination failures are configuration incidents, not power
|
|
// outcomes. Pause with a dedicated code and never schedule a power replay.
|
|
foreach ([
|
|
[1276, 409, 'coordination_busy', 'vpc_apply_coordination_busy'],
|
|
[1277, 503, 'coordination_uncertain', 'vpc_apply_coordination_uncertain'],
|
|
[1278, 503, 'coordination_unavailable', 'vpc_apply_coordination_unavailable'],
|
|
] as [$id, $status, $remoteCode, $pauseCode]) {
|
|
$op = $make($id, 'shutoff');
|
|
[$op, $lease] = $claim($op);
|
|
$assert(Workflow::beforeApply($op, $lease, $actor, false)
|
|
&& $phase($op) === 'applying',
|
|
'Stopped guest did not reach the VPC apply stage.');
|
|
$commands = count(CloudEngineClient::$commands);
|
|
Workflow::handleFailure($op, $lease,
|
|
new CloudError('private coordination diagnostic', $status, $remoteCode));
|
|
$state = Workflow::state($op->fresh());
|
|
$root = CloudVpc::find($op->vpc_id);
|
|
$assert($state['phase'] === 'paused'
|
|
&& $state['paused_from'] === 'applying'
|
|
&& $state['error_code'] === $pauseCode
|
|
&& $op->fresh()->status === 'paused'
|
|
&& $root->last_error_code === $pauseCode
|
|
&& count(CloudEngineClient::$commands) === $commands
|
|
&& !str_contains($op->fresh()->power_workflow_json, 'private coordination diagnostic'),
|
|
'VPC apply coordination failure was mislabeled as a power result or replayed.');
|
|
}
|
|
|
|
// Corrupt child identity cannot be used to query or send an unrelated command.
|
|
$op = $make(1208);
|
|
$state = Workflow::state($op); $state['shutdown_id'] = str_repeat('f', 32);
|
|
$op->power_workflow_json = json_encode($state); $op->save();
|
|
$expectCloudError(static fn () => Workflow::state($op), 503);
|
|
|
|
// A crash before the first shutdown dispatch cannot overrule a newer start.
|
|
$op = $make(1209);
|
|
$state = Workflow::state($op);
|
|
$state['phase'] = 'shutting_down'; $state['original_running'] = true;
|
|
$state['shutdown_deadline'] = time() + 120;
|
|
$op->power_workflow_json = json_encode($state); $op->lease_until = 0; $op->save();
|
|
$instance = CloudInstance::find(120900);
|
|
$instance->getConnection()->transaction(static function () use ($instance): void {
|
|
Workflow::lockForInstance($instance);
|
|
Workflow::manualIntentLocked($instance, 'action', 'start');
|
|
});
|
|
$count = count(CloudEngineClient::$commands);
|
|
[$op, $lease] = $claim($op);
|
|
Workflow::beforeApply($op, $lease, $actor, false);
|
|
$assert($phase($op) === 'waiting_stopped' && count(CloudEngineClient::$commands) === $count,
|
|
'Recovered undispatched shutdown overruled a newer manual start.');
|
|
|
|
// A definitely undispatched rejection can proceed after manual stop; an
|
|
// uncertain/in-flight child is never replaced by a fresh command identifier.
|
|
$op = $make(1212);
|
|
$state = Workflow::state($op);
|
|
CloudEngineClient::$operations[$state['shutdown_id']] = [
|
|
'operation_id' => $state['shutdown_id'], 'status' => 'failed',
|
|
'execution_outcome' => 'not_dispatched', 'mutation_started' => false,
|
|
];
|
|
[$op, $lease] = $claim($op); Workflow::beforeApply($op, $lease, $actor, false);
|
|
$assert($phase($op) === 'paused' && Workflow::state($op->fresh())['error_code'] === 'vpc_power_command_rejected',
|
|
'Definitive rejection was mislabeled as unknown.');
|
|
CloudEngineClient::$power = 'shutoff';
|
|
[$op, $lease] = $claim($op); Workflow::beforeApply($op, $lease, $actor, true);
|
|
$assert($phase($op) === 'applying', 'Known undispatched shutdown could not proceed after manual stop.');
|
|
|
|
$op = $make(1213);
|
|
$state = Workflow::state($op);
|
|
CloudEngineClient::$operations[$state['shutdown_id']] = [
|
|
'operation_id' => $state['shutdown_id'], 'status' => 'running',
|
|
];
|
|
[$op, $lease] = $claim($op); Workflow::beforeApply($op, $lease, $actor, false);
|
|
$assert($op->fresh()->status === 'waiting_observed' && $phase($op) === 'shutting_down',
|
|
'In-flight power acknowledgement requires automatic polling, not a manual pause.');
|
|
CloudEngineClient::$operations[$state['shutdown_id']]['status'] = 'uncertain';
|
|
[$op, $lease] = $claim($op); Workflow::beforeApply($op, $lease, $actor, false);
|
|
$assert($phase($op) === 'paused' && count(CloudEngineClient::$commands) === $count,
|
|
'Unknown shutdown outcome was replayed.');
|
|
|
|
$op = $make(1214);
|
|
[$op, $lease] = $claim($op); Workflow::beforeApply($op, $lease, $actor, false);
|
|
CloudEngineClient::$power = 'shutoff';
|
|
[$op, $lease] = $claim($op); Workflow::beforeApply($op, $lease, $actor, false);
|
|
$observed($op); $state = Workflow::state($op->fresh());
|
|
CloudEngineClient::$operations[$state['start_id']] = [
|
|
'operation_id' => $state['start_id'], 'status' => 'failed',
|
|
'execution_outcome' => 'not_dispatched', 'mutation_started' => false,
|
|
];
|
|
Workflow::afterApply($op, $lease, $actor);
|
|
$instance = CloudInstance::find(121400);
|
|
$instance->getConnection()->transaction(static function () use ($instance): void {
|
|
Workflow::lockForInstance($instance);
|
|
Workflow::manualIntentLocked($instance, 'action', 'start');
|
|
});
|
|
[$op, $lease] = $claim($op); Workflow::beforeApply($op, $lease, $actor, true);
|
|
$assert($phase($op) === 'completed', 'Manual start could not finish rejected automatic restoration.');
|
|
|
|
// Browser retries carry no VM write capability. Authorization is checked in
|
|
// PHP, then every power read/child operation uses the same non-admin owner.
|
|
$tenantActor = $actor + ['is_admin' => false, 'is_super_admin' => false,
|
|
'can_manage' => false, 'session_id' => 'browser-session'];
|
|
$workerActor = array_replace($tenantActor, ['can_manage' => true, 'session_id' => '']);
|
|
$adminActor = ['user_id' => 1, 'is_admin' => true, 'is_super_admin' => true,
|
|
'can_manage' => true, 'session_id' => 'admin-session'];
|
|
$op = $make(1300);
|
|
[$op, $lease] = $claim($op); Workflow::beforeApply($op, $lease, $tenantActor, false);
|
|
$assert($phase($op) === 'waiting_stopped'
|
|
&& end(CloudEngineClient::$postAttempts)[2] === $workerActor
|
|
+ ['operation_id' => Workflow::state($op->fresh())['shutdown_id']],
|
|
'A tenant retry did not receive a bounded owner power actor.');
|
|
$op = $make(1301);
|
|
CloudEngineClient::$loseReceipt = true;
|
|
[$op, $lease] = $claim($op); Workflow::beforeApply($op, $lease, $adminActor, false);
|
|
$state = Workflow::state($op->fresh());
|
|
$count = count(CloudEngineClient::$commands);
|
|
$assert($phase($op) === 'shutting_down'
|
|
&& $op->fresh()->status === 'waiting_observed'
|
|
&& CloudEngineClient::$operationOwners[$state['shutdown_id']] === $actor['user_id'],
|
|
'Admin review created a power receipt under the wrong owner.');
|
|
[$op, $lease] = $claim($op); Workflow::beforeApply($op, $lease, $workerActor, false);
|
|
$assert($phase($op) === 'waiting_stopped' && count(CloudEngineClient::$commands) === $count,
|
|
'The owner worker could not recover an admin-reviewed child receipt.');
|
|
foreach (['caller', 'instance', 'root', 'operation'] as $offset => $fault) {
|
|
$op = $make(1302 + $offset);
|
|
$incomingActor = $tenantActor;
|
|
if ($fault === 'caller') {
|
|
$incomingActor['user_id'] = 999;
|
|
} elseif ($fault === 'instance') {
|
|
CloudInstance::where('id', $op->power_instance_id)->update(['user_id' => 999]);
|
|
} elseif ($fault === 'root') {
|
|
CloudVpc::where('id', $op->vpc_id)->update(['user_id' => 999]);
|
|
} else {
|
|
CloudVpcOperation::where('id', $op->id)->update(['user_id' => 999]);
|
|
}
|
|
$attempts = count(CloudEngineClient::$postAttempts);
|
|
$lookups = count(CloudEngineClient::$lookups);
|
|
[$op, $lease] = $claim($op);
|
|
$expectCloudError(static fn () => Workflow::beforeApply(
|
|
$op, $lease, $incomingActor, false
|
|
), $fault === 'caller' ? 403 : 409);
|
|
$assert(count(CloudEngineClient::$postAttempts) === $attempts
|
|
&& count(CloudEngineClient::$lookups) === $lookups,
|
|
'A changed ' . $fault . ' owner reached a power request.');
|
|
}
|
|
|
|
$prepareCommand = static function (int $id, string $action) use ($make): CloudVpcOperation {
|
|
$operation = $make($id);
|
|
if ($action === 'start') {
|
|
$state = Workflow::state($operation);
|
|
$state['original_running'] = true;
|
|
$state['config_confirmed'] = true;
|
|
$state['phase'] = 'configured';
|
|
$operation->power_workflow_json = json_encode($state);
|
|
$operation->save();
|
|
CloudEngineClient::$power = 'shutoff';
|
|
}
|
|
return $operation;
|
|
};
|
|
$advance = static function (CloudVpcOperation $operation) use ($claim, $workerActor): void {
|
|
[$operation, $lease] = $claim($operation);
|
|
Workflow::beforeApply($operation, $lease, $workerActor, false);
|
|
};
|
|
|
|
// A successful start receipt remains `starting` until the state endpoint
|
|
// independently reports running. The deadline remains fixed across polls.
|
|
CloudEngineClient::$startChangesPower = false;
|
|
$op = $prepareCommand(1328, 'start');
|
|
$count = count(CloudEngineClient::$commands); $lookups = count(CloudEngineClient::$lookups);
|
|
$advance($op);
|
|
$state = Workflow::state($op->fresh());
|
|
$deadline = $state['start_deadline']; $childId = $state['start_id'];
|
|
$assert($state['phase'] === 'starting' && $state['start_accepted'] === true
|
|
&& CloudEngineClient::$power === 'shutoff'
|
|
&& count(CloudEngineClient::$commands) === $count + 1,
|
|
'Accepted start was treated as confirmed running state.');
|
|
$advance($op);
|
|
$state = Workflow::state($op->fresh());
|
|
$assert($state['phase'] === 'starting' && $state['start_deadline'] === $deadline
|
|
&& $state['start_id'] === $childId
|
|
&& count(CloudEngineClient::$commands) === $count + 1
|
|
&& count(CloudEngineClient::$lookups) === $lookups + 1,
|
|
'Start state polling changed the child identity/deadline or repeated its POST.');
|
|
CloudEngineClient::$power = 'running';
|
|
$advance($op);
|
|
$assert($phase($op) === 'completed' && count(CloudEngineClient::$commands) === $count + 1,
|
|
'Running state readback did not complete an accepted start.');
|
|
|
|
// A manual poweroff can land after the start gate authorized its already
|
|
// accepted child but before the command fast path reacquires the instance.
|
|
// That newer intent completes the workflow without another read or POST.
|
|
CloudEngineClient::$startChangesPower = false;
|
|
$op = $prepareCommand(1349, 'start');
|
|
$advance($op);
|
|
$instance = CloudInstance::find(134900);
|
|
$instance->getConnection()->transaction(static function () use ($instance): void {
|
|
Workflow::lockForInstance($instance);
|
|
Workflow::manualIntentLocked($instance, 'action', 'poweroff');
|
|
});
|
|
[$op, $lease] = $claim($op);
|
|
$commands = count(CloudEngineClient::$commands);
|
|
$reads = CloudEngineClient::$stateReads;
|
|
$method = new ReflectionMethod(Workflow::class, 'command');
|
|
$method->invoke(null, $op, $lease, $workerActor, 'start', new CloudEngineClient());
|
|
$assert($phase($op) === 'completed'
|
|
&& Workflow::state($op->fresh())['resume_cancelled'] === true
|
|
&& count(CloudEngineClient::$commands) === $commands
|
|
&& CloudEngineClient::$stateReads === $reads,
|
|
'Accepted start ignored a newer manual poweroff intent.');
|
|
|
|
$op = $prepareCommand(1329, 'start');
|
|
$advance($op);
|
|
$state = Workflow::state($op->fresh());
|
|
$state['start_deadline'] = time() - 1;
|
|
$op->power_workflow_json = json_encode($state); $op->save();
|
|
$rootBeforePause = CloudVpc::find($op->vpc_id);
|
|
$rootBeforePause->state = 'degraded';
|
|
$rootBeforePause->sync_state = 'uncertain';
|
|
$rootBeforePause->last_error_code = 'vpc_host_configuration_mismatch';
|
|
$rootBeforePause->save();
|
|
$rootProjectionBeforePause = [
|
|
(string) $rootBeforePause->state,
|
|
(string) $rootBeforePause->sync_state,
|
|
(string) $rootBeforePause->last_error_code,
|
|
];
|
|
$advance($op);
|
|
$rootAfterPause = CloudVpc::find($op->vpc_id);
|
|
$assert($phase($op) === 'paused'
|
|
&& Workflow::state($op->fresh())['error_code'] === 'vpc_power_start_timeout'
|
|
&& [(string) $rootAfterPause->state, (string) $rootAfterPause->sync_state,
|
|
(string) $rootAfterPause->last_error_code] === $rootProjectionBeforePause,
|
|
'Accepted start without running readback did not stop at its fixed deadline.');
|
|
CloudEngineClient::$startChangesPower = true;
|
|
|
|
// Pre-upgrade admin-owned children are recoverable only with a currently
|
|
// authorized admin read of the original ID, never a privileged redispatch.
|
|
foreach (['shutdown', 'start'] as $actionOffset => $action) {
|
|
foreach (['succeeded', 'running', 'uncertain', 'wrong_uuid', 'wrong_action'] as $offset => $outcome) {
|
|
$op = $prepareCommand(1330 + $actionOffset * 5 + $offset, $action);
|
|
$state = Workflow::state($op->fresh());
|
|
$id = $state[$action . '_id'];
|
|
CloudEngineClient::$operationOwners[$id] = $adminActor['user_id'];
|
|
CloudEngineClient::$operations[$id] = [
|
|
'operation_id' => $id,
|
|
'status' => in_array($outcome, ['running', 'uncertain'], true) ? $outcome : 'succeeded',
|
|
'result' => ['uuid' => $outcome === 'wrong_uuid' ? str_repeat('a', 36) : $state['uuid'],
|
|
'action' => $outcome === 'wrong_action' ? 'reboot' : $action,
|
|
'command_accepted' => true],
|
|
];
|
|
$attempts = count(CloudEngineClient::$postAttempts);
|
|
$lookups = count(CloudEngineClient::$lookups);
|
|
[$op, $lease] = $claim($op); Workflow::beforeApply($op, $lease, $adminActor, false);
|
|
$expected = $outcome === 'succeeded'
|
|
? ($action === 'start' ? 'starting' : 'waiting_stopped')
|
|
: ($outcome === 'running' ? ($action === 'start' ? 'starting' : 'shutting_down') : 'paused');
|
|
$assert($phase($op) === $expected
|
|
&& count(CloudEngineClient::$postAttempts) === $attempts
|
|
&& count(CloudEngineClient::$lookups) === $lookups + 2
|
|
&& end(CloudEngineClient::$lookups)[1]['user_id'] === $adminActor['user_id'],
|
|
'Legacy admin-owned ' . $action . '/' . $outcome . ' was not safely recovered.');
|
|
}
|
|
}
|
|
foreach ([403, 502] as $offset => $status) {
|
|
$op = $prepareCommand(1340 + $offset, 'shutdown');
|
|
CloudEngineClient::$adminLookupFailure = new CloudError('private-admin-error', $status);
|
|
$attempts = count(CloudEngineClient::$postAttempts);
|
|
[$op, $lease] = $claim($op); Workflow::beforeApply($op, $lease, $adminActor, false);
|
|
CloudEngineClient::$adminLookupFailure = null;
|
|
$state = Workflow::state($op->fresh());
|
|
$assert($phase($op) === ($status === 502 ? 'shutting_down' : 'paused')
|
|
&& $op->fresh()->status === ($status === 502 ? 'waiting_observed' : 'paused')
|
|
&& count(CloudEngineClient::$postAttempts) === $attempts
|
|
&& $state['last_command_error']['stage'] === 'lookup'
|
|
&& $state['last_command_error']['http_status'] === $status,
|
|
'A failed legacy receipt read authorized a new dispatch.');
|
|
}
|
|
$op = $prepareCommand(1342, 'shutdown');
|
|
$state = Workflow::state($op->fresh());
|
|
$id = $state['shutdown_id'];
|
|
CloudEngineClient::$operationOwners[$id] = $adminActor['user_id'];
|
|
CloudEngineClient::$operations[$id] = ['operation_id' => $id, 'status' => 'uncertain'];
|
|
$lookups = count(CloudEngineClient::$lookups);
|
|
[$op, $lease] = $claim($op); Workflow::beforeApply($op, $lease, $tenantActor, false);
|
|
$assert($phase($op) === 'paused' && count(CloudEngineClient::$lookups) === $lookups + 1
|
|
&& end(CloudEngineClient::$lookups)[1]['is_admin'] === false
|
|
&& CloudEngineClient::$operations[$id]['status'] === 'uncertain',
|
|
'A tenant elevated its lookup or replaced an unreadable legacy child.');
|
|
|
|
// Both lookup and dispatch admission can be busy before a durable child row
|
|
// exists. Retry with the same identity, without extending the command bound.
|
|
foreach (['shutdown', 'start'] as $actionOffset => $action) {
|
|
foreach (['lookup', 'dispatch'] as $stageOffset => $stage) {
|
|
$op = $prepareCommand(1310 + $actionOffset * 2 + $stageOffset, $action);
|
|
$failureField = $stage === 'lookup' ? 'lookupFailure' : 'postFailure';
|
|
CloudEngineClient::${$failureField} = new CloudError('busy-private-detail', 409, 'coordination_busy');
|
|
$count = count(CloudEngineClient::$commands);
|
|
$advance($op);
|
|
$state = Workflow::state($op->fresh());
|
|
$expectedPhase = $action === 'start' ? 'starting' : 'shutting_down';
|
|
$childId = $state[$action . '_id'];
|
|
$deadline = $state[$action . '_deadline'];
|
|
$assert($state['phase'] === $expectedPhase
|
|
&& $op->fresh()->status === 'waiting_observed'
|
|
&& $op->fresh()->lease_token === null
|
|
&& !isset(CloudEngineClient::$operations[$childId])
|
|
&& $state['last_command_error'] === ['action' => $action, 'stage' => $stage,
|
|
'http_status' => 409, 'error_code' => 'coordination_busy'],
|
|
'Busy ' . $action . '/' . $stage . ' did not release for bounded polling.');
|
|
$assert(!str_contains($op->fresh()->power_workflow_json, 'busy-private-detail')
|
|
&& !array_key_exists('last_command_error', Workflow::publicState($op->fresh())),
|
|
'Private power diagnostics leaked message text or entered the public DTO.');
|
|
$advance($op);
|
|
$state = Workflow::state($op->fresh());
|
|
$assert($state['phase'] === $expectedPhase
|
|
&& $state[$action . '_id'] === $childId
|
|
&& $state[$action . '_deadline'] === $deadline
|
|
&& count(CloudEngineClient::$commands) === $count,
|
|
'Busy polling changed the original command identity or deadline.');
|
|
CloudEngineClient::${$failureField} = null;
|
|
$advance($op);
|
|
$assert($phase($op) === ($action === 'start' ? 'starting' : 'waiting_stopped')
|
|
&& count(CloudEngineClient::$commands) === $count + 1
|
|
&& end(CloudEngineClient::$commands)[0] === $childId,
|
|
'Cleared admission did not dispatch exactly the original command.');
|
|
if ($action === 'start') {
|
|
$advance($op);
|
|
$assert($phase($op) === 'completed'
|
|
&& count(CloudEngineClient::$commands) === $count + 1,
|
|
'Accepted start did not complete from running state readback.');
|
|
}
|
|
}
|
|
}
|
|
foreach (['shutdown', 'start'] as $offset => $action) {
|
|
$op = $prepareCommand(1314 + $offset, $action);
|
|
CloudEngineClient::$postFailure = new CloudError('busy', 409, 'coordination_busy');
|
|
$advance($op);
|
|
$state = Workflow::state($op->fresh());
|
|
$state[$action . '_deadline'] = time() - 1;
|
|
$op->power_workflow_json = json_encode($state); $op->save();
|
|
CloudEngineClient::$postFailure = null;
|
|
$attempts = count(CloudEngineClient::$postAttempts);
|
|
$advance($op);
|
|
$assert($phase($op) === 'paused' && $op->fresh()->lease_token === null
|
|
&& Workflow::state($op->fresh())['error_code'] === 'vpc_power_command_rejected'
|
|
&& count(CloudEngineClient::$postAttempts) === $attempts,
|
|
'Expired ' . $action . ' admission dispatched a late command.');
|
|
}
|
|
foreach (['shutdown', 'start'] as $offset => $action) {
|
|
$op = $prepareCommand(1343 + $offset, $action);
|
|
CloudEngineClient::$postFailure = new CloudError('busy', 409, 'coordination_busy');
|
|
$advance($op);
|
|
CloudEngineClient::$postFailure = null;
|
|
$state = Workflow::state($op->fresh());
|
|
$state[$action . '_deadline'] = time() - 1;
|
|
$op->power_workflow_json = json_encode($state); $op->save();
|
|
$id = $state[$action . '_id'];
|
|
CloudEngineClient::$operations[$id] = ['operation_id' => $id, 'status' => 'succeeded',
|
|
'result' => ['uuid' => $state['uuid'], 'action' => $action, 'command_accepted' => true]];
|
|
CloudEngineClient::$operationOwners[$id] = $actor['user_id'];
|
|
if ($action === 'start') { CloudEngineClient::$power = 'running'; }
|
|
$attempts = count(CloudEngineClient::$postAttempts);
|
|
$advance($op);
|
|
if ($action === 'start') { $advance($op); }
|
|
$assert($phase($op) === ($action === 'start' ? 'completed' : 'waiting_stopped')
|
|
&& count(CloudEngineClient::$postAttempts) === $attempts,
|
|
'The deadline discarded an exact accepted ' . $action . ' receipt.');
|
|
}
|
|
foreach ([[409, 'coordination_uncertain'], [503, 'coordination_busy'],
|
|
[503, 'coordination_unavailable'], [502, 'engine_transport_uncertain'],
|
|
[502, null], [403, null], [409, 'private-secret-code']] as $offset => [$status, $code]) {
|
|
$op = $prepareCommand(1316 + $offset, 'shutdown');
|
|
CloudEngineClient::$postFailure = new CloudError('private-secret-message', $status, $code);
|
|
$advance($op);
|
|
$state = Workflow::state($op->fresh());
|
|
$retryable = ($code === 'coordination_busy' && $status === 409)
|
|
|| ($code === 'engine_transport_uncertain' && $status === 502);
|
|
$assert($phase($op) === ($retryable ? 'shutting_down' : 'paused')
|
|
&& $op->fresh()->status === ($retryable ? 'waiting_observed' : 'paused')
|
|
&& $op->fresh()->lease_token === null
|
|
&& $state['error_code'] === ($retryable ? null : 'vpc_power_result_uncertain')
|
|
&& $state['last_command_error']['http_status'] === $status
|
|
&& !str_contains($op->fresh()->power_workflow_json, 'private-secret'),
|
|
'A command transport failure was misclassified or leaked upstream diagnostics.');
|
|
CloudEngineClient::$postFailure = null;
|
|
}
|
|
|
|
// End-to-end service/worker lane with the existing strict host/member proofs.
|
|
// Quiesce unrelated legacy fixture operations so the worker selects this intent.
|
|
Capsule::table('cloud_vpc_operations')->update(['status' => 'superseded', 'lease_until' => 0]);
|
|
CloudEngineClient::$vpcRespond = static function (array $payload) use ($activeHostEvidence): array {
|
|
$hosts = [];
|
|
foreach ($payload['hosts'] as $host) {
|
|
$id = (string) $host['host_id'];
|
|
if ($host['members'] === []) {
|
|
$hosts[$id] = [
|
|
'vpc_id' => $payload['vpc_id'], 'host_id' => $id,
|
|
'desired_revision' => $payload['desired_revision'], 'desired_digest' => $payload['desired_digest'],
|
|
'observed_revision' => $payload['desired_revision'], 'observed_digest' => $payload['desired_digest'],
|
|
'state' => 'active', 'transport_verified' => true,
|
|
'private_dataplane_verified' => true, 'removed' => true,
|
|
'bridge' => null, 'bridge_owner' => null, 'nft_table' => null,
|
|
'rollback' => 'none', 'readback' => 'confirmed',
|
|
'member_nics' => [], 'routes' => [], 'wg_allowed_ips' => [], 'proxy_ips' => [],
|
|
];
|
|
} else {
|
|
$item = $activeHostEvidence($payload['vpc_id'], (int) $id,
|
|
$payload['desired_revision'], $payload['desired_digest'], $payload['cidr'], $payload['members']);
|
|
if (CloudEngineClient::$power === 'shutoff') {
|
|
$item['state'] = 'degraded';
|
|
$item['private_dataplane_verified'] = false;
|
|
$item['last_error_code'] = 'vpc_private_guest_stopped';
|
|
foreach ($item['member_nics'] as &$nic) {
|
|
$nic['domain_running'] = false;
|
|
$nic['guest_network_verified'] = false;
|
|
$nic['guest_reason_code'] = 'vpc_private_guest_stopped';
|
|
}
|
|
unset($nic);
|
|
}
|
|
$hosts[$id] = $item;
|
|
}
|
|
}
|
|
$active = $hosts !== [];
|
|
foreach ($hosts as $host) {
|
|
$active = $active && $host['state'] === 'active'
|
|
&& $host['private_dataplane_verified'] === true;
|
|
}
|
|
return [
|
|
'vpc_id' => $payload['vpc_id'], 'operation_id' => $payload['operation_id'],
|
|
'desired_revision' => $payload['desired_revision'], 'desired_digest' => $payload['desired_digest'],
|
|
'observed_revision' => $payload['desired_revision'], 'observed_digest' => $payload['desired_digest'],
|
|
'state' => $active ? 'active' : 'degraded',
|
|
'next_stage_allowed' => $active, 'private_dataplane_verified' => $active,
|
|
'hosts' => array_values($hosts),
|
|
'observed' => ['evidence_version' => CloudVpcService::OBSERVATION_EVIDENCE_VERSION,
|
|
'revision' => $payload['desired_revision'], 'digest' => $payload['desired_digest'],
|
|
'state' => $active ? 'active' : 'degraded', 'next_stage_allowed' => $active,
|
|
'private_dataplane_verified' => $active, 'fabric_gate' => ['active' => true], 'hosts' => $hosts],
|
|
];
|
|
};
|
|
$tick = static function (CloudVpcOperation $operation) use ($assert): void {
|
|
Capsule::table('cloud_vpc_operations')->where('id', $operation->id)
|
|
->update(['updated_at' => time() - 31, 'lease_until' => 0]);
|
|
$assert((new CloudVpcWorkerService())->tick(), 'Independent VPC worker did not select current workflow.');
|
|
};
|
|
$op = $make(1210);
|
|
$tick($op);
|
|
$assert($phase($op) === 'waiting_stopped' && CloudEngineClient::$applies === 0,
|
|
'VPC applied configuration before stopping the guest.');
|
|
CloudEngineClient::$power = 'shutoff';
|
|
$tick($op);
|
|
$assert($phase($op) === 'starting' && CloudEngineClient::$power === 'running',
|
|
'Accepted automatic start was not retained for independent state readback.');
|
|
$tick($op);
|
|
$assert($phase($op) === 'completed' && $op->fresh()->status === 'succeeded'
|
|
&& CloudEngineClient::$power === 'running',
|
|
'Service/worker did not apply and restore a running member.');
|
|
$root = CloudVpc::find($op->vpc_id);
|
|
$member = CloudVpcMember::where('instance_id', 121000)->first();
|
|
$assert($root->state === 'active' && $root->sync_state === 'applied'
|
|
&& $root->last_error_code === null
|
|
&& $member->state === 'stopped'
|
|
&& $member->last_error_code === 'vpc_private_guest_stopped',
|
|
'Guest-only stopped evidence leaked into the applied root configuration.');
|
|
$commands = count(CloudEngineClient::$commands);
|
|
$applies = CloudEngineClient::$applies;
|
|
$attempts = (int) $op->fresh()->attempts;
|
|
// A completed, config-exact workflow is terminal for the apply worker. A
|
|
// later independent guest-health receipt may update only member/IP runtime.
|
|
Capsule::table('cloud_vpc_operations')->where('id', '<>', $op->id)
|
|
->update(['status' => 'superseded', 'lease_until' => 0]);
|
|
Capsule::table('cloud_vpc_operations')->where('id', $op->id)
|
|
->update(['updated_at' => time() - 31, 'lease_until' => 0]);
|
|
$assert((new CloudVpcWorkerService())->tick() === false
|
|
&& (int) $op->fresh()->attempts === $attempts
|
|
&& CloudEngineClient::$applies === $applies
|
|
&& count(CloudEngineClient::$commands) === $commands,
|
|
'The worker replayed a succeeded operation solely for guest health.');
|
|
$root->refresh(); $member->refresh();
|
|
$private = \App\Models\CloudVpcPrivateIp::find($member->private_ip_id);
|
|
$assert($phase($op) === 'completed' && $op->fresh()->status === 'succeeded'
|
|
&& (int) $op->fresh()->attempts === $attempts
|
|
&& $root->state === 'active' && $root->sync_state === 'applied'
|
|
&& $root->last_error_code === null
|
|
&& (int) $root->observed_revision === (int) $root->desired_revision
|
|
&& $member->state === 'stopped'
|
|
&& $member->last_error_code === 'vpc_private_guest_stopped'
|
|
&& $private->state === 'stopped'
|
|
&& $private->last_error_code === 'vpc_private_guest_stopped'
|
|
&& CloudEngineClient::$applies === $applies
|
|
&& count(CloudEngineClient::$commands) === $commands,
|
|
'A guest-only stopped state was not retained without replaying apply.');
|
|
$detach = $legacyDetach($root, $member, 'auto-detach-1210');
|
|
$tick($detach);
|
|
CloudEngineClient::$power = 'shutoff';
|
|
// Losing the start response keeps the fixed child in automatic receipt/state
|
|
// recovery after exact last-member teardown released membership/IP rows.
|
|
CloudEngineClient::$loseReceipt = true;
|
|
$tick($detach);
|
|
$assert($phase($detach) === 'starting' && $detach->fresh()->status === 'waiting_observed'
|
|
&& CloudVpcMember::where('instance_id', 121000)->count() === 0,
|
|
'Last-member teardown did not preserve restoration workflow after removal.');
|
|
$other = $seedVpc(1211, '10.121.211.0/24', '10.121.211.0/24', 10);
|
|
$expectCloudError(static fn () => CloudVpcService::attachMember($other['vpc_key'], $actor, [
|
|
'instance_id' => 121000, 'request_key' => 'cross-attach-121000',
|
|
]), 409);
|
|
$count = count(CloudEngineClient::$commands);
|
|
$tick($detach); $tick($detach);
|
|
$assert($phase($detach) === 'completed' && $detach->fresh()->status === 'succeeded'
|
|
&& count(CloudEngineClient::$commands) === $count,
|
|
'Last-member paused start could not recover its original receipt.');
|
|
|
|
// The departing host can finish while a surviving host is still converging.
|
|
// Keep that exact accepted target in the GET lane until the complete proof
|
|
// arrives; a VM must not remain off merely because the first receipt is partial.
|
|
$ordinaryVpcRespond = CloudEngineClient::$vpcRespond;
|
|
$runningVpcProof = static function (array $payload) use ($ordinaryVpcRespond, $activeHostEvidence): array {
|
|
$result = $ordinaryVpcRespond($payload);
|
|
$hosts = [];
|
|
foreach ($result['hosts'] as $host) {
|
|
$hostId = (string) $host['host_id'];
|
|
$hosts[$hostId] = ($host['removed'] ?? false) === true ? $host
|
|
: $activeHostEvidence($payload['vpc_id'], (int) $hostId,
|
|
$payload['desired_revision'], $payload['desired_digest'],
|
|
$payload['cidr'], $payload['members']);
|
|
}
|
|
$result['state'] = $result['observed']['state'] = 'active';
|
|
$result['next_stage_allowed'] = $result['observed']['next_stage_allowed'] = true;
|
|
$result['private_dataplane_verified'] = $result['observed']['private_dataplane_verified'] = true;
|
|
$result['hosts'] = array_values($hosts);
|
|
$result['observed']['hosts'] = $hosts;
|
|
return $result;
|
|
};
|
|
|
|
// A multi-host attach may return a bounded recovery snapshot while one host
|
|
// revalidates after an engine restart. Exact target/operation/revision/digest
|
|
// and the complete host closure keep it in the same read-only window.
|
|
Capsule::table('cloud_vpc_operations')->update(['status' => 'superseded', 'lease_until' => 0]);
|
|
$seedInstance(128000, 20);
|
|
$multiHostVpc = $seedVpc(1280, '10.128.0.0/24', '10.128.0.0/24', 20,
|
|
'active', 'active', [
|
|
'member_row_id' => 928000, 'instance_id' => 128000,
|
|
'private_ip' => '10.128.0.1',
|
|
]);
|
|
$seedPair(928000, 938000, $multiHostVpc, 128000, 20, '10.128.0.1', 'active');
|
|
$seedInstance(128001, 10);
|
|
CloudEngineClient::$power = 'running';
|
|
CloudVpcService::attachMember($multiHostVpc['vpc_key'], $actor, [
|
|
'instance_id' => 128001, 'request_key' => 'multi-host-auto-attach-128001',
|
|
]);
|
|
$multiAttach = CloudVpcOperation::where('vpc_id', 1280)
|
|
->orderByDesc('desired_revision')->first();
|
|
Workflow::initialize(
|
|
$multiAttach,
|
|
CloudInstance::find(128001),
|
|
CloudVpcMember::where('instance_id', 128001)->first()
|
|
);
|
|
$multiAttach->refresh();
|
|
$tick($multiAttach);
|
|
CloudEngineClient::$power = 'shutoff';
|
|
$multiPayload = json_decode($multiAttach->payload_json, true);
|
|
$multiUncertain = $ordinaryVpcRespond($multiPayload);
|
|
$multiUncertain['state'] = $multiUncertain['observed']['state'] = 'uncertain';
|
|
$multiUncertain['next_stage_allowed'] =
|
|
$multiUncertain['observed']['next_stage_allowed'] = false;
|
|
$multiUncertain['private_dataplane_verified'] =
|
|
$multiUncertain['observed']['private_dataplane_verified'] = false;
|
|
$multiUncertain['observed']['last_error_code'] = 'stale_pre_restart_failure';
|
|
$multiUncertain['observed']['fabric_gate'] = [
|
|
'active' => false, 'available' => true,
|
|
'error_code' => 'vpc_recovery_pending',
|
|
];
|
|
$recoveringHost = $multiUncertain['observed']['hosts']['20'];
|
|
$recoveringHost['state'] = 'uncertain';
|
|
$recoveringHost['transport_verified'] = false;
|
|
$recoveringHost['private_dataplane_verified'] = false;
|
|
$recoveringHost['config_verified'] = false;
|
|
$recoveringHost['observed_revision'] = 0;
|
|
$recoveringHost['observed_digest'] = null;
|
|
$recoveringHost['config_observed_revision'] = 0;
|
|
$recoveringHost['config_observed_digest'] = null;
|
|
// VpcCoordinator.recover() closes the root Fabric gate and marks each host
|
|
// uncertain, but deliberately does not manufacture a host-level error code.
|
|
$recoveringHost['last_error_code'] = null;
|
|
$multiUncertain['observed']['hosts']['20'] = $recoveringHost;
|
|
$multiUncertain['hosts'] = array_values($multiUncertain['observed']['hosts']);
|
|
CloudEngineClient::$vpcRespond = static fn () => $multiUncertain;
|
|
$applies = CloudEngineClient::$applies; $commands = count(CloudEngineClient::$commands);
|
|
$tick($multiAttach);
|
|
$state = Workflow::state($multiAttach->fresh());
|
|
$deadline = $state['configuration_deadline'] ?? 0;
|
|
$assert($state['phase'] === 'applying' && !$state['config_confirmed']
|
|
&& $deadline > time() && $multiAttach->fresh()->status === 'waiting_observed'
|
|
&& CloudEngineClient::$applies === $applies + 1
|
|
&& count(CloudEngineClient::$commands) === $commands,
|
|
'Reviewed multi-host attach recovery evidence paused the automatic workflow.');
|
|
CloudEngineClient::$vpcRead = static fn () => $multiUncertain;
|
|
$tick($multiAttach);
|
|
$state = Workflow::state($multiAttach->fresh());
|
|
$assert($state['phase'] === 'applying'
|
|
&& $state['configuration_deadline'] === $deadline
|
|
&& CloudEngineClient::$applies === $applies + 1
|
|
&& count(CloudEngineClient::$commands) === $commands,
|
|
'Multi-host attach recovery resent apply or extended its read-only deadline.');
|
|
$multiExact = $ordinaryVpcRespond($multiPayload);
|
|
CloudEngineClient::$vpcRead = static fn () => $multiExact;
|
|
$tick($multiAttach); $tick($multiAttach);
|
|
$assert($phase($multiAttach) === 'completed'
|
|
&& CloudEngineClient::$power === 'running'
|
|
&& CloudEngineClient::$applies === $applies + 1
|
|
&& count(CloudEngineClient::$commands) === $commands + 1,
|
|
'Multi-host attach did not converge from exact later readback and restore power once.');
|
|
CloudEngineClient::$vpcRespond = $ordinaryVpcRespond;
|
|
CloudEngineClient::$vpcRead = null;
|
|
|
|
$partialDetach = static function (int $id, bool $sameHost = false, ?string $fault = null) use (
|
|
$make, $tick, $actor, $seedInstance, $ordinaryVpcRespond, $runningVpcProof,
|
|
$legacyDetach, $assert
|
|
): array {
|
|
Capsule::table('cloud_vpc_operations')->update(['status' => 'superseded', 'lease_until' => 0]);
|
|
CloudEngineClient::$vpcRespond = $ordinaryVpcRespond;
|
|
CloudEngineClient::$vpcRead = null;
|
|
$attach = $make($id);
|
|
$tick($attach);
|
|
CloudEngineClient::$power = 'shutoff';
|
|
$tick($attach); $tick($attach);
|
|
$vpc = CloudVpc::find($attach->vpc_id);
|
|
foreach ([1, 2] as $offset) {
|
|
$seedInstance($id * 100 + $offset, $sameHost ? 10 : 20);
|
|
CloudVpcService::attachMember($vpc->vpc_key, $actor, [
|
|
'instance_id' => $id * 100 + $offset,
|
|
'request_key' => 'retained-sibling-' . $id . '-' . $offset,
|
|
]);
|
|
CloudVpcService::reconcile($vpc->vpc_key, $actor, true, true);
|
|
}
|
|
$target = CloudVpcMember::where('instance_id', $id * 100)->first();
|
|
$operation = $legacyDetach($vpc, $target, 'partial-detach-' . $id);
|
|
$tick($operation);
|
|
CloudEngineClient::$power = 'shutoff';
|
|
$full = $runningVpcProof(json_decode($operation->payload_json, true));
|
|
$partial = $full;
|
|
$partial['state'] = $partial['observed']['state'] = 'degraded';
|
|
$partial['next_stage_allowed'] = $partial['observed']['next_stage_allowed'] = false;
|
|
$partial['private_dataplane_verified'] = $partial['observed']['private_dataplane_verified'] = false;
|
|
$partial['observed']['fabric_gate'] = ['active' => false, 'available' => true];
|
|
foreach ($partial['hosts'] as &$host) {
|
|
if (($host['removed'] ?? false) === true) { continue; }
|
|
$host['transport_verified'] = $host['config_verified'] = $host['private_dataplane_verified'] = false;
|
|
$host['observed_revision'] = $host['config_observed_revision'] = 0;
|
|
$host['observed_digest'] = $host['config_observed_digest'] = null;
|
|
$host['last_error_code'] = 'vpc_private_dataplane_unverified';
|
|
$host['wg_allowed_ips'] = [];
|
|
}
|
|
unset($host);
|
|
$partial['observed']['hosts'] = [];
|
|
if ($fault === 'missing_host') {
|
|
array_pop($partial['hosts']);
|
|
} elseif ($fault === 'host_digest') {
|
|
$partial['hosts'][count($partial['hosts']) - 1]['desired_digest'] = str_repeat('f', 64);
|
|
} elseif ($fault === 'host_failed') {
|
|
$partial['hosts'][count($partial['hosts']) - 1]['last_error_code'] = 'vpc_private_dataplane_apply_failed';
|
|
} elseif ($fault === 'host_uncertain_unknown') {
|
|
$partial['hosts'][count($partial['hosts']) - 1]['state'] = 'uncertain';
|
|
$partial['hosts'][count($partial['hosts']) - 1]['last_error_code'] =
|
|
'unreviewed_host_failure';
|
|
} elseif ($fault === 'root_uncertain_unknown') {
|
|
$partial['state'] = $partial['observed']['state'] = 'uncertain';
|
|
$partial['observed']['fabric_gate']['error_code'] =
|
|
'unreviewed_root_failure';
|
|
}
|
|
CloudEngineClient::$vpcRespond = static fn () => $partial;
|
|
$tick($operation);
|
|
return [$operation, $partial, $full];
|
|
};
|
|
[$op, $partial, $full] = $partialDetach(1260);
|
|
$state = Workflow::state($op->fresh());
|
|
$deadline = $state['configuration_deadline'] ?? 0;
|
|
$applies = CloudEngineClient::$applies; $count = count(CloudEngineClient::$commands);
|
|
$assert($state['phase'] === 'applying' && !$state['config_confirmed'] && $deadline > time()
|
|
&& $op->fresh()->status === 'waiting_observed' && $op->fresh()->lease_token === null
|
|
&& CloudVpcMember::where('instance_id', 126000)->count() === 1,
|
|
'The partial two-host detach paused or released an unconfirmed member.');
|
|
foreach (['partial', 'missing', 'older', 'offline'] as $mode) {
|
|
CloudEngineClient::$vpcRead = static function () use ($partial, $mode): array {
|
|
if ($mode === 'missing') { throw new CloudError('missing', 404); }
|
|
if ($mode === 'offline') { throw new CloudError('offline', 502); }
|
|
if ($mode === 'older') { $partial['desired_revision']--; }
|
|
return $partial;
|
|
};
|
|
$tick($op);
|
|
$state = Workflow::state($op->fresh());
|
|
$assert($state['phase'] === 'applying' && !$state['config_confirmed']
|
|
&& $state['configuration_deadline'] === $deadline
|
|
&& $op->fresh()->status === 'waiting_observed'
|
|
&& CloudEngineClient::$applies === $applies && count(CloudEngineClient::$commands) === $count,
|
|
'Accepted configuration was resubmitted, restarted or extended on ' . $mode);
|
|
}
|
|
CloudEngineClient::$vpcRead = static fn () => $full;
|
|
$tick($op);
|
|
$assert($phase($op) === 'starting',
|
|
'Exact detach proof did not retain accepted start for state readback.');
|
|
$tick($op);
|
|
$assert($phase($op) === 'completed' && CloudEngineClient::$power === 'running'
|
|
&& CloudVpcMember::where('instance_id', 126000)->count() === 0
|
|
&& CloudVpcMember::whereIn('instance_id', [126001, 126002])->where('state', 'active')->count() === 2
|
|
&& CloudEngineClient::$applies === $applies && count(CloudEngineClient::$commands) === $count + 1,
|
|
'Later exact detach proof did not release only the target and restore it once.');
|
|
|
|
// The same mechanism also preserves siblings when the departing host stays live.
|
|
[$op, $partial, $full] = $partialDetach(1261, true);
|
|
$applies = CloudEngineClient::$applies;
|
|
CloudEngineClient::$vpcRead = static fn () => $full;
|
|
$tick($op);
|
|
$tick($op);
|
|
$assert($phase($op) === 'completed'
|
|
&& CloudVpcMember::where('instance_id', 126100)->count() === 0
|
|
&& CloudVpcMember::whereIn('instance_id', [126101, 126102])->where('state', 'active')->count() === 2
|
|
&& CloudEngineClient::$applies === $applies,
|
|
'Same-host detach required removing the host or lost its retained siblings.');
|
|
|
|
foreach (['partial', 'missing', 'exact'] as $offset => $mode) {
|
|
[$op, $partial, $full] = $partialDetach(1262 + $offset);
|
|
$state = Workflow::state($op->fresh());
|
|
$state['configuration_deadline'] = time() - 1;
|
|
$op->power_workflow_json = json_encode($state); $op->save();
|
|
$applies = CloudEngineClient::$applies; $count = count(CloudEngineClient::$commands);
|
|
CloudEngineClient::$vpcRead = static function () use ($mode, $partial, $full): array {
|
|
if ($mode === 'missing') { throw new CloudError('missing', 404); }
|
|
return $mode === 'exact' ? $full : $partial;
|
|
};
|
|
$tick($op);
|
|
if ($mode === 'exact') { $tick($op); }
|
|
$assert($phase($op) === ($mode === 'exact' ? 'completed' : 'paused')
|
|
&& CloudEngineClient::$applies === $applies
|
|
&& count(CloudEngineClient::$commands) === $count + ($mode === 'exact' ? 1 : 0),
|
|
'Configuration deadline extended, resent a POST or discarded exact proof: ' . $mode);
|
|
if ($mode !== 'exact') {
|
|
$assert(Workflow::state($op->fresh())['configuration_deadline'] === $state['configuration_deadline'],
|
|
'A deadline was refreshed by an ordinary worker poll.');
|
|
CloudEngineClient::$vpcRead = static fn () => $partial;
|
|
CloudVpcService::reconcile(CloudVpc::find($op->vpc_id)->vpc_key, $actor, true, true);
|
|
$assert($phase($op) === 'applying'
|
|
&& Workflow::state($op->fresh())['configuration_deadline'] > time()
|
|
&& CloudEngineClient::$applies === $applies,
|
|
'Explicit retry could not open a bounded read-only recovery window.');
|
|
}
|
|
}
|
|
[$op, $partial, $full] = $partialDetach(1265);
|
|
$instance = CloudInstance::find(126500);
|
|
$instance->getConnection()->transaction(static function () use ($instance): void {
|
|
Workflow::lockForInstance($instance);
|
|
Workflow::manualIntentLocked($instance, 'action', 'start');
|
|
});
|
|
$applies = CloudEngineClient::$applies; $count = count(CloudEngineClient::$commands);
|
|
CloudEngineClient::$vpcRead = static fn () => $full;
|
|
$tick($op);
|
|
$assert($phase($op) === 'completed' && Workflow::state($op->fresh())['resume_cancelled']
|
|
&& CloudEngineClient::$applies === $applies && count(CloudEngineClient::$commands) === $count,
|
|
'Configuration polling overrode a newer manual power intent.');
|
|
foreach ([[1266, 'missing_host'], [1267, 'host_digest'], [1268, 'host_failed'],
|
|
[1269, 'host_uncertain_unknown'], [1290, 'root_uncertain_unknown']]
|
|
as [$caseId, $fault]) {
|
|
[$op] = $partialDetach($caseId, false, $fault);
|
|
$state = Workflow::state($op->fresh());
|
|
$assert($state['phase'] === 'paused' && !$state['config_confirmed']
|
|
&& !isset($state['configuration_deadline']) && CloudEngineClient::$power === 'shutoff',
|
|
'Invalid partial proof opened an automatic waiting window: ' . $fault);
|
|
}
|
|
CloudEngineClient::$vpcRespond = $ordinaryVpcRespond;
|
|
CloudEngineClient::$vpcRead = null;
|
|
|
|
// Recover an interrupted apply from the exact durable engine receipt before
|
|
// the stopped-guest mutation precondition. A manual start needs no second stop.
|
|
$interrupted = static function (int $id, bool $paused = true) use ($make): CloudVpcOperation {
|
|
$operation = $make($id, 'running');
|
|
$state = Workflow::state($operation);
|
|
$state['phase'] = $paused ? 'paused' : 'applying';
|
|
$state['paused_from'] = 'applying';
|
|
$state['original_running'] = false;
|
|
$state['resume_cancelled'] = true;
|
|
$state['error_code'] = 'vpc_power_manual_override';
|
|
$operation->power_workflow_json = json_encode($state);
|
|
$operation->status = $paused ? 'paused' : 'waiting_observed';
|
|
$operation->lease_token = null; $operation->lease_until = 0;
|
|
$operation->save();
|
|
return $operation;
|
|
};
|
|
$receipt = static function (CloudVpcOperation $operation): array {
|
|
$result = (CloudEngineClient::$vpcRespond)(json_decode($operation->payload_json, true));
|
|
$result['observed']['state'] = 'active';
|
|
$result['observed']['next_stage_allowed'] = true;
|
|
$result['observed']['private_dataplane_verified'] = true;
|
|
// Exercise the engine's real compact observed + full top-level hosts DTO.
|
|
$result['hosts'] = $result['observed']['hosts'];
|
|
$result['observed']['hosts'] = [];
|
|
return $result;
|
|
};
|
|
foreach ([true, false] as $offset => $paused) {
|
|
$op = $interrupted(1220 + $offset, $paused);
|
|
$proof = $receipt($op);
|
|
CloudEngineClient::$vpcRead = static fn () => $proof;
|
|
$applies = CloudEngineClient::$applies; $reads = CloudEngineClient::$readbacks;
|
|
$count = count(CloudEngineClient::$commands);
|
|
CloudEngineClient::$failState = true;
|
|
CloudVpcService::reconcile(CloudVpc::find($op->vpc_id)->vpc_key, $actor, true, $paused);
|
|
CloudEngineClient::$failState = false;
|
|
$root = CloudVpc::find($op->vpc_id);
|
|
$assert($phase($op) === 'completed' && $op->fresh()->status === 'succeeded'
|
|
&& $root->state === 'active' && $root->observed_revision === $root->desired_revision
|
|
&& CloudEngineClient::$readbacks === $reads + 1
|
|
&& CloudEngineClient::$applies === $applies && count(CloudEngineClient::$commands) === $count
|
|
&& Workflow::state($op->fresh())['error_code'] === null,
|
|
'Exact interrupted apply did not recover without another mutation/power query.');
|
|
}
|
|
|
|
// A read failure/mismatch must not corrupt paused_from or trigger a new apply.
|
|
foreach (['missing', 'older', 'offline', 'vpc', 'operation', 'digest', 'newer', 'invalid_revision',
|
|
'host', 'nic', 'acl', 'fractional_revision'] as $offset => $fault) {
|
|
$op = $interrupted(1230 + $offset);
|
|
$proof = $receipt($op);
|
|
switch ($fault) {
|
|
case 'vpc': $proof['vpc_id'] = str_repeat('f', 32); break;
|
|
case 'operation': $proof['operation_id'] = str_repeat('f', 32); break;
|
|
case 'digest': $proof['desired_digest'] = str_repeat('f', 64); break;
|
|
case 'older': $proof['desired_revision']--; break;
|
|
case 'newer': $proof['desired_revision']++; break;
|
|
case 'invalid_revision': $proof['desired_revision'] = 'garbage'; break;
|
|
case 'fractional_revision': $proof['desired_revision'] += 0.9; break;
|
|
case 'host': $proof['hosts'] = []; break;
|
|
case 'nic': $proof['hosts']['10']['member_nics'] = []; break;
|
|
case 'acl': $proof['hosts']['10']['acl_revision'] = null; break;
|
|
}
|
|
CloudEngineClient::$vpcRead = static function () use ($proof, $fault): array {
|
|
if (in_array($fault, ['missing', 'offline'], true)) {
|
|
throw new CloudError('read failed', $fault === 'missing' ? 404 : 502);
|
|
}
|
|
return $proof;
|
|
};
|
|
$applies = CloudEngineClient::$applies; $count = count(CloudEngineClient::$commands);
|
|
CloudVpcService::reconcile(CloudVpc::find($op->vpc_id)->vpc_key, $actor, true, true);
|
|
$state = Workflow::state($op->fresh());
|
|
$assert($phase($op) === 'paused' && $state['paused_from'] === 'applying'
|
|
&& !$state['config_confirmed'] && CloudEngineClient::$applies === $applies
|
|
&& count(CloudEngineClient::$commands) === $count,
|
|
'Invalid/missing receipt bypassed stopped guest gate or corrupted phase: ' . $fault);
|
|
}
|
|
|
|
// The same absent receipt may proceed through the original mutation lane only
|
|
// after the guest is stopped. Unverified running proof never authorizes apply.
|
|
$op = $interrupted(1251);
|
|
CloudEngineClient::$power = 'shutoff';
|
|
CloudEngineClient::$vpcRead = null;
|
|
$applies = CloudEngineClient::$applies;
|
|
CloudVpcService::reconcile(CloudVpc::find($op->vpc_id)->vpc_key, $actor, true, true);
|
|
$assert($phase($op) === 'completed' && CloudEngineClient::$applies === $applies + 1,
|
|
'Missing receipt prevented legitimate stopped-guest configuration.');
|
|
|
|
// A rejected first submission has no accepted partial receipt. Preserve the
|
|
// original explicit-retry lane instead of fencing it behind a permanent GET.
|
|
$op = $interrupted(1270);
|
|
CloudEngineClient::$power = 'shutoff';
|
|
CloudEngineClient::$vpcRead = null;
|
|
CloudEngineClient::$vpcRespond = static function (): array {
|
|
throw new CloudError('rejected before apply', 422);
|
|
};
|
|
$applies = CloudEngineClient::$applies;
|
|
CloudVpcService::reconcile(CloudVpc::find($op->vpc_id)->vpc_key, $actor, true, true);
|
|
$assert($phase($op) === 'paused'
|
|
&& !isset(Workflow::state($op->fresh())['configuration_deadline'])
|
|
&& CloudEngineClient::$applies === $applies + 1,
|
|
'A rejected first POST acquired an accepted-configuration fence.');
|
|
CloudEngineClient::$vpcRespond = $ordinaryVpcRespond;
|
|
CloudVpcService::reconcile(CloudVpc::find($op->vpc_id)->vpc_key, $actor, true, true);
|
|
$assert($phase($op) === 'completed' && CloudEngineClient::$applies === $applies + 2,
|
|
'Explicit retry could not resubmit after a definite first-POST rejection.');
|
|
|
|
// New manual intent during the read wins even if the original guest ran.
|
|
$op = $interrupted(1252);
|
|
$state = Workflow::state($op); $state['original_running'] = true; $state['resume_cancelled'] = false;
|
|
$op->power_workflow_json = json_encode($state); $op->save();
|
|
$proof = $receipt($op);
|
|
CloudEngineClient::$vpcRead = static function () use ($op, $proof): array {
|
|
$instance = CloudInstance::find(125200);
|
|
$instance->getConnection()->transaction(static function () use ($instance): void {
|
|
Workflow::lockForInstance($instance);
|
|
Workflow::manualIntentLocked($instance, 'action', 'start');
|
|
});
|
|
return $proof;
|
|
};
|
|
$count = count(CloudEngineClient::$commands);
|
|
CloudVpcService::reconcile(CloudVpc::find($op->vpc_id)->vpc_key, $actor, true, true);
|
|
$assert($phase($op) === 'completed' && count(CloudEngineClient::$commands) === $count
|
|
&& Workflow::state($op->fresh())['resume_cancelled'], 'Receipt recovery overrode concurrent manual intent.');
|
|
|
|
// A late GET cannot publish evidence after ownership or instance identity moves.
|
|
foreach (['lease', 'revision', 'instance'] as $offset => $change) {
|
|
$op = $interrupted(1253 + $offset);
|
|
$proof = $receipt($op); $root = CloudVpc::find($op->vpc_id);
|
|
$previous = [$root->observed_revision, $root->observed_digest];
|
|
CloudEngineClient::$vpcRead = static function () use ($op, $proof, $change): array {
|
|
if ($change === 'lease') {
|
|
Capsule::table('cloud_vpc_operations')->where('id', $op->id)->update(['lease_token' => str_repeat('f', 32)]);
|
|
} elseif ($change === 'revision') {
|
|
Capsule::table('cloud_vpcs')->where('id', $op->vpc_id)->increment('desired_revision');
|
|
} else {
|
|
Capsule::table('cloud_instances')->where('id', $op->power_instance_id)->update(['host_id' => 11]);
|
|
}
|
|
return $proof;
|
|
};
|
|
$applies = CloudEngineClient::$applies; $count = count(CloudEngineClient::$commands);
|
|
try { CloudVpcService::reconcile($root->vpc_key, $actor, true, true); }
|
|
catch (CloudError $error) { /* Stale owner is deliberately unable to release its successor's lease. */ }
|
|
$root->refresh();
|
|
$assert([$root->observed_revision, $root->observed_digest] === $previous
|
|
&& !Workflow::state($op->fresh())['config_confirmed']
|
|
&& CloudEngineClient::$applies === $applies && count(CloudEngineClient::$commands) === $count,
|
|
'Late receipt updated evidence after a changed ' . $change);
|
|
}
|
|
CloudEngineClient::$vpcRead = null;
|
|
|
|
// An installed pre-workflow schema continues to admit ordinary power, but
|
|
// cannot accept a new automatic maintenance request.
|
|
$instance = CloudInstance::find(121000);
|
|
$index = 'cloud_vpc_operations_power_instance_id_index';
|
|
$assert($schema->hasIndex('cloud_vpc_operations', ['power_instance_id']),
|
|
'Workflow lookup has no index to bound its locking read.');
|
|
$schema->table('cloud_vpc_operations', static function ($table) use ($index): void { $table->dropIndex($index); });
|
|
$expectCloudError(static fn () => Workflow::lockForInstance($instance), 503);
|
|
$schema->table('cloud_vpc_operations', static function ($table): void { $table->index('power_instance_id'); });
|
|
$detach->refresh();
|
|
$detach->power_instance_id = 999;
|
|
$expectCloudError(static fn () => Workflow::state($detach), 503);
|
|
$detach->refresh();
|
|
$schema->table('cloud_vpc_operations', static function ($table) use ($index): void {
|
|
$table->dropIndex($index); $table->dropColumn('power_instance_id');
|
|
});
|
|
$expectCloudError(static fn () => Workflow::lockForInstance($instance), 503);
|
|
$schema->table('cloud_vpc_operations', static function ($table): void { $table->dropColumn('power_workflow_json'); });
|
|
Workflow::lockForInstance($instance);
|
|
Workflow::manualIntentLocked($instance, 'action', 'poweroff');
|
|
$expectCloudError(static fn () => Workflow::initialize($detach, $instance, $member), 503);
|
|
|
|
echo "VPC automatic power workflow contracts passed.\n";
|
|
}
|