0 && ($actor['operation_id'] ?? null) === $operationId) { return; } if (!is_int($actor['user_id'] ?? null) || $actor['user_id'] < 1 || ($actor['is_admin'] ?? null) !== false || ($actor['is_super_admin'] ?? null) !== false || ($actor['can_manage'] ?? null) !== true || ($actor['session_id'] ?? null) !== '' || ($operationId !== null && ($actor['operation_id'] ?? null) !== $operationId)) { throw new \RuntimeException('Power request did not use a bounded owner actor.'); } } public function host(int $host, string $method, string $path, array $actor, ?array $body = null, ?string $operationId = null): array { self::assertPowerActor($actor, $operationId); $uuid = explode('/', $path)[1]; if ($method === 'GET') { self::$stateReads++; if (self::$failState || self::$failStateAtRead === self::$stateReads) { self::$failStateAtRead = null; throw new CloudError('offline', 502); } return ['uuid' => $uuid, 'state' => self::$power]; } self::$postAttempts[] = [$operationId, $body['action'], $actor]; if (isset(self::$operationOwners[$operationId]) && self::$operationOwners[$operationId] !== $actor['user_id']) { throw new CloudError('missing', 404); } if (self::$postFailure !== null) { throw self::$postFailure; } self::$commands[] = [$operationId, $body['action'], $uuid]; $result = ['operation_id' => $operationId, 'status' => 'succeeded', 'result' => ['uuid' => $uuid, 'action' => $body['action'], 'command_accepted' => true]]; self::$operations[$operationId] = $result; self::$operationOwners[$operationId] = $actor['user_id']; if ($body['action'] === 'start' && self::$startChangesPower) { self::$power = 'running'; } if (self::$loseReceipt) { self::$loseReceipt = false; throw new CloudError('lost response', 502, 'engine_transport_uncertain'); } return $result; } public function request(string $method, string $path, array $actor, ?array $body = null, ?string $operationId = null): array { self::assertPowerActor($actor, $operationId, true); self::$lookups[] = [$operationId, $actor]; if (($actor['is_admin'] ?? false) === true && self::$adminLookupFailure !== null) { throw self::$adminLookupFailure; } if (self::$lookupFailure !== null) { throw self::$lookupFailure; } if (isset(self::$operationOwners[basename($path)]) && self::$operationOwners[basename($path)] !== $actor['user_id'] && ($actor['is_admin'] ?? false) !== true) { throw new CloudError('missing', 404); } return self::$operations[basename($path)] ?? throw new CloudError('missing', 404); } public function vpc(string $method, string $path, array $actor, ?array $body = null, ?string $operationId = null): array { if ($method === 'GET') { self::$readbacks++; return self::$vpcRead !== null ? (self::$vpcRead)($path) : throw new CloudError('missing', 404); } self::$applies++; return (self::$vpcRespond)($body); } } } namespace { use App\Models\CloudInstance; use App\Models\CloudVpc; use App\Models\CloudVpcMember; use App\Models\CloudVpcOperation; use App\Services\Cloud\CloudEngineClient; use App\Services\Cloud\CloudError; use App\Services\Cloud\CloudVpcPowerWorkflow as Workflow; use App\Services\Cloud\CloudVpcService; use App\Services\Cloud\CloudVpcWorkerService; use Illuminate\Database\Capsule\Manager as Capsule; require __DIR__ . '/CloudVpcMemberReuseTest.php'; if (!isset($capsule)) { return; } $make = static function (int $id, string $power = 'running') use ($seedVpc, $seedInstance, $actor): CloudVpcOperation { $vpc = $seedVpc($id, '10.121.' . ($id % 250) . '.0/24', '10.121.' . ($id % 250) . '.0/24', 10); $seedInstance($id * 100, 10); CloudVpcService::attachMember($vpc['vpc_key'], $actor, [ 'instance_id' => $id * 100, 'request_key' => 'auto-member-' . $id, ]); $operation = CloudVpcOperation::where('vpc_id', $id) ->orderByDesc('desired_revision')->first(); $instance = CloudInstance::find($id * 100); $member = CloudVpcMember::where('instance_id', $id * 100)->first(); // Manufacture a pre-refactor row to verify compatibility recovery. New // public attach/detach requests never initialize this workflow. Workflow::initialize($operation, $instance, $member); CloudEngineClient::$power = $power; return $operation->fresh(); }; $legacyDetach = static function (CloudVpc $vpc, CloudVpcMember $member, string $requestKey) use ($actor): CloudVpcOperation { CloudVpcService::detachMember($vpc->vpc_key, $member->member_key, $actor, [ 'request_key' => $requestKey, ]); $operation = CloudVpcOperation::where('vpc_id', $vpc->id) ->orderByDesc('desired_revision')->first(); Workflow::initialize( $operation, CloudInstance::find((int) $member->instance_id), $member->fresh() ); return $operation->fresh(); }; $claim = static function (CloudVpcOperation $operation): array { $operation = $operation->fresh(); $operation->lease_token = bin2hex(random_bytes(16)); $operation->lease_until = time() + 120; $operation->status = 'running'; $operation->save(); return [$operation, $operation->lease_token]; }; $observed = static function (CloudVpcOperation $operation): void { Workflow::configurationObserved($operation->fresh(), true); }; $phase = static fn (CloudVpcOperation $operation): string => Workflow::state($operation->fresh())['phase']; // Normal running guest: command acknowledgement, separate state observation, // exact configuration gate, then one automatic restart. $op = $make(1201); [$op, $lease] = $claim($op); $assert(Workflow::beforeApply($op, $lease, $actor, false) === false && $phase($op) === 'waiting_stopped', 'Shutdown must finish only the command step.'); $assert($op->fresh()->lease_token === null, 'Waiting for guest shutdown held its lease.'); $count = count(CloudEngineClient::$commands); [$op, $lease] = $claim($op); $assert(!Workflow::beforeApply($op, $lease, $actor, false) && count(CloudEngineClient::$commands) === $count, 'Polling resent shutdown.'); CloudEngineClient::$power = 'shutoff'; [$op, $lease] = $claim($op); $reads = CloudEngineClient::$stateReads; CloudEngineClient::$failStateAtRead = $reads + 2; $assert(Workflow::beforeApply($op, $lease, $actor, false) && CloudEngineClient::$stateReads === $reads + 1, 'Stopped guest did not enter apply with one state read.'); CloudEngineClient::$failStateAtRead = null; Workflow::afterApply($op, $lease, $actor); $assert(count(CloudEngineClient::$commands) === $count, 'Guest started before exact configuration proof.'); [$op, $lease] = $claim($op); $assert(Workflow::beforeApply($op, $lease, $actor, true), 'Explicit retry did not resume configuration.'); $observed($op); Workflow::afterApply($op, $lease, $actor); $assert($phase($op) === 'starting' && count(CloudEngineClient::$commands) === $count + 1 && end(CloudEngineClient::$commands)[1] === 'start', 'Accepted start was not retained for power readback.'); [$op, $lease] = $claim($op); $assert(!Workflow::beforeApply($op, $lease, $actor, false) && $phase($op) === 'completed', 'Running readback did not finish automatic restoration.'); $completedOp = $op; // An accepted shutdown survives one transient state read and converges using // the same child ID and fixed deadline. No manual retry or second POST occurs. $op = $make(1275); [$op, $lease] = $claim($op); Workflow::beforeApply($op, $lease, $actor, false); $state = Workflow::state($op->fresh()); $deadline = $state['shutdown_deadline']; $childId = $state['shutdown_id']; $commands = count(CloudEngineClient::$commands); CloudEngineClient::$failStateAtRead = CloudEngineClient::$stateReads + 1; [$op, $lease] = $claim($op); $assert(!Workflow::beforeApply($op, $lease, $actor, false) && $phase($op) === 'waiting_stopped' && $op->fresh()->status === 'waiting_observed' && Workflow::state($op->fresh())['shutdown_deadline'] === $deadline && Workflow::state($op->fresh())['shutdown_id'] === $childId && count(CloudEngineClient::$commands) === $commands, 'Accepted shutdown state read did not enter bounded automatic recovery.'); CloudEngineClient::$power = 'shutoff'; [$op, $lease] = $claim($op); $assert(Workflow::beforeApply($op, $lease, $actor, false) && $phase($op) === 'applying' && count(CloudEngineClient::$commands) === $commands, 'Accepted shutdown did not recover after a transient state failure.'); // A historical auto-power request remains exactly replayable during a rolling // deployment, but a new request can no longer create that workflow. $vpc = CloudVpc::find($completedOp->vpc_id); $legacyPayload = json_decode((string) $completedOp->payload_json, true); $legacySubnetKey = (string) ($legacyPayload['members'][0]['subnet_id'] ?? ''); $completedOp->request_hash = hash('sha256', $canonicalJson([ 'kind' => 'member_attach', 'input' => [ 'subnet_key' => $legacySubnetKey, 'instance_id' => 120100, 'private_ip' => null, 'auto_power' => true, ], ])); $completedOp->save(); CloudVpcService::attachMember($vpc->vpc_key, $actor, [ 'instance_id' => 120100, 'request_key' => 'auto-member-1201', 'auto_power' => true, ]); $expectCloudError(static fn () => CloudVpcService::attachMember($vpc->vpc_key, $actor, [ 'instance_id' => 120100, 'request_key' => 'auto-member-1201', 'auto_power' => false, ]), 409); $retired = $expectCloudError(static fn () => CloudVpcService::attachMember( $vpc->vpc_key, $actor, [ 'instance_id' => 120100, 'request_key' => 'new-auto-member-1201', 'auto_power' => true, ] ), 422); $assert($retired->errorCode === 'vpc_auto_power_retired', 'A new automatic-power workflow was still admitted.'); $expectCloudError(static fn () => CloudVpcService::attachMember($vpc->vpc_key, $actor, [ 'instance_id' => 120100, 'request_key' => 'auto-member-1201', 'auto_power' => true, 'subnet_key' => str_repeat('a', 32), ]), 422); foreach (['true', 1, null] as $badFlag) { $expectCloudError(static fn () => Workflow::requested(['auto_power' => $badFlag]), 422); } // A previously stopped or already shutting-down guest must never be restarted. foreach (['shutoff', 'shutdown'] as $offset => $initial) { $op = $make(1202 + $offset, $initial); $count = count(CloudEngineClient::$commands); [$op, $lease] = $claim($op); $ready = Workflow::beforeApply($op, $lease, $actor, false); if (!$ready) { CloudEngineClient::$power = 'shutoff'; [$op, $lease] = $claim($op); $assert(Workflow::beforeApply($op, $lease, $actor, false), 'Existing shutdown did not converge.'); } $observed($op); Workflow::afterApply($op, $lease, $actor); $assert($phase($op) === 'completed' && count(CloudEngineClient::$commands) === $count, 'A pre-existing stopped/shutdown state generated power commands.'); } // Lost receipt: repeat uses the same durable engine operation, no second POST. $op = $make(1204); CloudEngineClient::$loseReceipt = true; [$op, $lease] = $claim($op); Workflow::beforeApply($op, $lease, $actor, false); $assert($phase($op) === 'shutting_down' && $op->fresh()->status === 'waiting_observed', 'Lost receipt did not enter bounded fixed-ID recovery.'); $count = count(CloudEngineClient::$commands); [$op, $lease] = $claim($op); Workflow::beforeApply($op, $lease, $actor, false); $assert($phase($op) === 'waiting_stopped' && count(CloudEngineClient::$commands) === $count, 'Lost accepted command was submitted again.'); // Graceful timeout pauses, permits force-off, and cancels automatic restoration. $state = Workflow::state($op->fresh()); $state['shutdown_deadline'] = time() - 1; $op->power_workflow_json = json_encode($state); $op->save(); [$op, $lease] = $claim($op); Workflow::beforeApply($op, $lease, $actor, false); $assert($phase($op) === 'paused' && $op->fresh()->status === 'paused', 'Timeout became an uncertain mutation.'); $instance = CloudInstance::find(120400); $instance->getConnection()->transaction(static function () use ($instance): void { Workflow::lockForInstance($instance); $locked = CloudInstance::where('id', $instance->id)->lockForUpdate()->first(); Workflow::manualIntentLocked($locked, 'action', 'poweroff'); }); CloudEngineClient::$power = 'shutoff'; [$op, $lease] = $claim($op); $assert(Workflow::beforeApply($op, $lease, $actor, true), 'Manual force-off cannot resume configuration.'); $observed($op); Workflow::afterApply($op, $lease, $actor); $assert($phase($op) === 'completed' && count(CloudEngineClient::$commands) === $count, 'Automation undid newer manual power intent.'); // Only the active revision can continue; unfinished workflows forbid replacing // their VPC plan, including paused steps and last-member removal. $op = $make(1205, 'shutoff'); $vpc = CloudVpc::find($op->vpc_id); $expectCloudError(static fn () => Workflow::assertMutationAllowed($vpc), 409); [$op, $lease] = $claim($op); Capsule::table('cloud_vpcs')->where('id', $vpc->id)->update(['desired_revision' => 999]); $count = count(CloudEngineClient::$commands); $expectCloudError(static fn () => Workflow::beforeApply($op, $lease, $actor, false), 409); $assert(count(CloudEngineClient::$commands) === $count, 'Stale revision sent a power command.'); // Read-only failures pause without a child action or a lingering task lease. $op = $make(1206); CloudEngineClient::$failState = true; [$op, $lease] = $claim($op); try { Workflow::beforeApply($op, $lease, $actor, false); } catch (CloudError $error) { Workflow::handleFailure($op, $lease, $error); } CloudEngineClient::$failState = false; $assert($op->fresh()->status === 'paused' && $op->fresh()->lease_token === null, 'Read-only preflight failure retained a lease.'); // Crash during apply has a bounded lease, so later manual power stays possible. $op = $make(1207, 'shutoff'); [$op, $lease] = $claim($op); Workflow::beforeApply($op, $lease, $actor, false); $op = $op->fresh(); $op->lease_until = time() - 1; $op->save(); $instance = CloudInstance::find(120700); $instance->getConnection()->transaction(static function () use ($instance): void { Workflow::lockForInstance($instance); Workflow::manualIntentLocked($instance, 'action', 'poweroff'); }); $assert(Workflow::state($op->fresh())['resume_cancelled'] === true, 'Expired apply worker permanently blocked ordinary power.'); // Bare apply coordination failures are configuration incidents, not power // outcomes. Pause with a dedicated code and never schedule a power replay. foreach ([ [1276, 409, 'coordination_busy', 'vpc_apply_coordination_busy'], [1277, 503, 'coordination_uncertain', 'vpc_apply_coordination_uncertain'], [1278, 503, 'coordination_unavailable', 'vpc_apply_coordination_unavailable'], ] as [$id, $status, $remoteCode, $pauseCode]) { $op = $make($id, 'shutoff'); [$op, $lease] = $claim($op); $assert(Workflow::beforeApply($op, $lease, $actor, false) && $phase($op) === 'applying', 'Stopped guest did not reach the VPC apply stage.'); $commands = count(CloudEngineClient::$commands); Workflow::handleFailure($op, $lease, new CloudError('private coordination diagnostic', $status, $remoteCode)); $state = Workflow::state($op->fresh()); $root = CloudVpc::find($op->vpc_id); $assert($state['phase'] === 'paused' && $state['paused_from'] === 'applying' && $state['error_code'] === $pauseCode && $op->fresh()->status === 'paused' && $root->last_error_code === $pauseCode && count(CloudEngineClient::$commands) === $commands && !str_contains($op->fresh()->power_workflow_json, 'private coordination diagnostic'), 'VPC apply coordination failure was mislabeled as a power result or replayed.'); } // Corrupt child identity cannot be used to query or send an unrelated command. $op = $make(1208); $state = Workflow::state($op); $state['shutdown_id'] = str_repeat('f', 32); $op->power_workflow_json = json_encode($state); $op->save(); $expectCloudError(static fn () => Workflow::state($op), 503); // A crash before the first shutdown dispatch cannot overrule a newer start. $op = $make(1209); $state = Workflow::state($op); $state['phase'] = 'shutting_down'; $state['original_running'] = true; $state['shutdown_deadline'] = time() + 120; $op->power_workflow_json = json_encode($state); $op->lease_until = 0; $op->save(); $instance = CloudInstance::find(120900); $instance->getConnection()->transaction(static function () use ($instance): void { Workflow::lockForInstance($instance); Workflow::manualIntentLocked($instance, 'action', 'start'); }); $count = count(CloudEngineClient::$commands); [$op, $lease] = $claim($op); Workflow::beforeApply($op, $lease, $actor, false); $assert($phase($op) === 'waiting_stopped' && count(CloudEngineClient::$commands) === $count, 'Recovered undispatched shutdown overruled a newer manual start.'); // A definitely undispatched rejection can proceed after manual stop; an // uncertain/in-flight child is never replaced by a fresh command identifier. $op = $make(1212); $state = Workflow::state($op); CloudEngineClient::$operations[$state['shutdown_id']] = [ 'operation_id' => $state['shutdown_id'], 'status' => 'failed', 'execution_outcome' => 'not_dispatched', 'mutation_started' => false, ]; [$op, $lease] = $claim($op); Workflow::beforeApply($op, $lease, $actor, false); $assert($phase($op) === 'paused' && Workflow::state($op->fresh())['error_code'] === 'vpc_power_command_rejected', 'Definitive rejection was mislabeled as unknown.'); CloudEngineClient::$power = 'shutoff'; [$op, $lease] = $claim($op); Workflow::beforeApply($op, $lease, $actor, true); $assert($phase($op) === 'applying', 'Known undispatched shutdown could not proceed after manual stop.'); $op = $make(1213); $state = Workflow::state($op); CloudEngineClient::$operations[$state['shutdown_id']] = [ 'operation_id' => $state['shutdown_id'], 'status' => 'running', ]; [$op, $lease] = $claim($op); Workflow::beforeApply($op, $lease, $actor, false); $assert($op->fresh()->status === 'waiting_observed' && $phase($op) === 'shutting_down', 'In-flight power acknowledgement requires automatic polling, not a manual pause.'); CloudEngineClient::$operations[$state['shutdown_id']]['status'] = 'uncertain'; [$op, $lease] = $claim($op); Workflow::beforeApply($op, $lease, $actor, false); $assert($phase($op) === 'paused' && count(CloudEngineClient::$commands) === $count, 'Unknown shutdown outcome was replayed.'); $op = $make(1214); [$op, $lease] = $claim($op); Workflow::beforeApply($op, $lease, $actor, false); CloudEngineClient::$power = 'shutoff'; [$op, $lease] = $claim($op); Workflow::beforeApply($op, $lease, $actor, false); $observed($op); $state = Workflow::state($op->fresh()); CloudEngineClient::$operations[$state['start_id']] = [ 'operation_id' => $state['start_id'], 'status' => 'failed', 'execution_outcome' => 'not_dispatched', 'mutation_started' => false, ]; Workflow::afterApply($op, $lease, $actor); $instance = CloudInstance::find(121400); $instance->getConnection()->transaction(static function () use ($instance): void { Workflow::lockForInstance($instance); Workflow::manualIntentLocked($instance, 'action', 'start'); }); [$op, $lease] = $claim($op); Workflow::beforeApply($op, $lease, $actor, true); $assert($phase($op) === 'completed', 'Manual start could not finish rejected automatic restoration.'); // Browser retries carry no VM write capability. Authorization is checked in // PHP, then every power read/child operation uses the same non-admin owner. $tenantActor = $actor + ['is_admin' => false, 'is_super_admin' => false, 'can_manage' => false, 'session_id' => 'browser-session']; $workerActor = array_replace($tenantActor, ['can_manage' => true, 'session_id' => '']); $adminActor = ['user_id' => 1, 'is_admin' => true, 'is_super_admin' => true, 'can_manage' => true, 'session_id' => 'admin-session']; $op = $make(1300); [$op, $lease] = $claim($op); Workflow::beforeApply($op, $lease, $tenantActor, false); $assert($phase($op) === 'waiting_stopped' && end(CloudEngineClient::$postAttempts)[2] === $workerActor + ['operation_id' => Workflow::state($op->fresh())['shutdown_id']], 'A tenant retry did not receive a bounded owner power actor.'); $op = $make(1301); CloudEngineClient::$loseReceipt = true; [$op, $lease] = $claim($op); Workflow::beforeApply($op, $lease, $adminActor, false); $state = Workflow::state($op->fresh()); $count = count(CloudEngineClient::$commands); $assert($phase($op) === 'shutting_down' && $op->fresh()->status === 'waiting_observed' && CloudEngineClient::$operationOwners[$state['shutdown_id']] === $actor['user_id'], 'Admin review created a power receipt under the wrong owner.'); [$op, $lease] = $claim($op); Workflow::beforeApply($op, $lease, $workerActor, false); $assert($phase($op) === 'waiting_stopped' && count(CloudEngineClient::$commands) === $count, 'The owner worker could not recover an admin-reviewed child receipt.'); foreach (['caller', 'instance', 'root', 'operation'] as $offset => $fault) { $op = $make(1302 + $offset); $incomingActor = $tenantActor; if ($fault === 'caller') { $incomingActor['user_id'] = 999; } elseif ($fault === 'instance') { CloudInstance::where('id', $op->power_instance_id)->update(['user_id' => 999]); } elseif ($fault === 'root') { CloudVpc::where('id', $op->vpc_id)->update(['user_id' => 999]); } else { CloudVpcOperation::where('id', $op->id)->update(['user_id' => 999]); } $attempts = count(CloudEngineClient::$postAttempts); $lookups = count(CloudEngineClient::$lookups); [$op, $lease] = $claim($op); $expectCloudError(static fn () => Workflow::beforeApply( $op, $lease, $incomingActor, false ), $fault === 'caller' ? 403 : 409); $assert(count(CloudEngineClient::$postAttempts) === $attempts && count(CloudEngineClient::$lookups) === $lookups, 'A changed ' . $fault . ' owner reached a power request.'); } $prepareCommand = static function (int $id, string $action) use ($make): CloudVpcOperation { $operation = $make($id); if ($action === 'start') { $state = Workflow::state($operation); $state['original_running'] = true; $state['config_confirmed'] = true; $state['phase'] = 'configured'; $operation->power_workflow_json = json_encode($state); $operation->save(); CloudEngineClient::$power = 'shutoff'; } return $operation; }; $advance = static function (CloudVpcOperation $operation) use ($claim, $workerActor): void { [$operation, $lease] = $claim($operation); Workflow::beforeApply($operation, $lease, $workerActor, false); }; // A successful start receipt remains `starting` until the state endpoint // independently reports running. The deadline remains fixed across polls. CloudEngineClient::$startChangesPower = false; $op = $prepareCommand(1328, 'start'); $count = count(CloudEngineClient::$commands); $lookups = count(CloudEngineClient::$lookups); $advance($op); $state = Workflow::state($op->fresh()); $deadline = $state['start_deadline']; $childId = $state['start_id']; $assert($state['phase'] === 'starting' && $state['start_accepted'] === true && CloudEngineClient::$power === 'shutoff' && count(CloudEngineClient::$commands) === $count + 1, 'Accepted start was treated as confirmed running state.'); $advance($op); $state = Workflow::state($op->fresh()); $assert($state['phase'] === 'starting' && $state['start_deadline'] === $deadline && $state['start_id'] === $childId && count(CloudEngineClient::$commands) === $count + 1 && count(CloudEngineClient::$lookups) === $lookups + 1, 'Start state polling changed the child identity/deadline or repeated its POST.'); CloudEngineClient::$power = 'running'; $advance($op); $assert($phase($op) === 'completed' && count(CloudEngineClient::$commands) === $count + 1, 'Running state readback did not complete an accepted start.'); // A manual poweroff can land after the start gate authorized its already // accepted child but before the command fast path reacquires the instance. // That newer intent completes the workflow without another read or POST. CloudEngineClient::$startChangesPower = false; $op = $prepareCommand(1349, 'start'); $advance($op); $instance = CloudInstance::find(134900); $instance->getConnection()->transaction(static function () use ($instance): void { Workflow::lockForInstance($instance); Workflow::manualIntentLocked($instance, 'action', 'poweroff'); }); [$op, $lease] = $claim($op); $commands = count(CloudEngineClient::$commands); $reads = CloudEngineClient::$stateReads; $method = new ReflectionMethod(Workflow::class, 'command'); $method->invoke(null, $op, $lease, $workerActor, 'start', new CloudEngineClient()); $assert($phase($op) === 'completed' && Workflow::state($op->fresh())['resume_cancelled'] === true && count(CloudEngineClient::$commands) === $commands && CloudEngineClient::$stateReads === $reads, 'Accepted start ignored a newer manual poweroff intent.'); $op = $prepareCommand(1329, 'start'); $advance($op); $state = Workflow::state($op->fresh()); $state['start_deadline'] = time() - 1; $op->power_workflow_json = json_encode($state); $op->save(); $rootBeforePause = CloudVpc::find($op->vpc_id); $rootBeforePause->state = 'degraded'; $rootBeforePause->sync_state = 'uncertain'; $rootBeforePause->last_error_code = 'vpc_host_configuration_mismatch'; $rootBeforePause->save(); $rootProjectionBeforePause = [ (string) $rootBeforePause->state, (string) $rootBeforePause->sync_state, (string) $rootBeforePause->last_error_code, ]; $advance($op); $rootAfterPause = CloudVpc::find($op->vpc_id); $assert($phase($op) === 'paused' && Workflow::state($op->fresh())['error_code'] === 'vpc_power_start_timeout' && [(string) $rootAfterPause->state, (string) $rootAfterPause->sync_state, (string) $rootAfterPause->last_error_code] === $rootProjectionBeforePause, 'Accepted start without running readback did not stop at its fixed deadline.'); CloudEngineClient::$startChangesPower = true; // Pre-upgrade admin-owned children are recoverable only with a currently // authorized admin read of the original ID, never a privileged redispatch. foreach (['shutdown', 'start'] as $actionOffset => $action) { foreach (['succeeded', 'running', 'uncertain', 'wrong_uuid', 'wrong_action'] as $offset => $outcome) { $op = $prepareCommand(1330 + $actionOffset * 5 + $offset, $action); $state = Workflow::state($op->fresh()); $id = $state[$action . '_id']; CloudEngineClient::$operationOwners[$id] = $adminActor['user_id']; CloudEngineClient::$operations[$id] = [ 'operation_id' => $id, 'status' => in_array($outcome, ['running', 'uncertain'], true) ? $outcome : 'succeeded', 'result' => ['uuid' => $outcome === 'wrong_uuid' ? str_repeat('a', 36) : $state['uuid'], 'action' => $outcome === 'wrong_action' ? 'reboot' : $action, 'command_accepted' => true], ]; $attempts = count(CloudEngineClient::$postAttempts); $lookups = count(CloudEngineClient::$lookups); [$op, $lease] = $claim($op); Workflow::beforeApply($op, $lease, $adminActor, false); $expected = $outcome === 'succeeded' ? ($action === 'start' ? 'starting' : 'waiting_stopped') : ($outcome === 'running' ? ($action === 'start' ? 'starting' : 'shutting_down') : 'paused'); $assert($phase($op) === $expected && count(CloudEngineClient::$postAttempts) === $attempts && count(CloudEngineClient::$lookups) === $lookups + 2 && end(CloudEngineClient::$lookups)[1]['user_id'] === $adminActor['user_id'], 'Legacy admin-owned ' . $action . '/' . $outcome . ' was not safely recovered.'); } } foreach ([403, 502] as $offset => $status) { $op = $prepareCommand(1340 + $offset, 'shutdown'); CloudEngineClient::$adminLookupFailure = new CloudError('private-admin-error', $status); $attempts = count(CloudEngineClient::$postAttempts); [$op, $lease] = $claim($op); Workflow::beforeApply($op, $lease, $adminActor, false); CloudEngineClient::$adminLookupFailure = null; $state = Workflow::state($op->fresh()); $assert($phase($op) === ($status === 502 ? 'shutting_down' : 'paused') && $op->fresh()->status === ($status === 502 ? 'waiting_observed' : 'paused') && count(CloudEngineClient::$postAttempts) === $attempts && $state['last_command_error']['stage'] === 'lookup' && $state['last_command_error']['http_status'] === $status, 'A failed legacy receipt read authorized a new dispatch.'); } $op = $prepareCommand(1342, 'shutdown'); $state = Workflow::state($op->fresh()); $id = $state['shutdown_id']; CloudEngineClient::$operationOwners[$id] = $adminActor['user_id']; CloudEngineClient::$operations[$id] = ['operation_id' => $id, 'status' => 'uncertain']; $lookups = count(CloudEngineClient::$lookups); [$op, $lease] = $claim($op); Workflow::beforeApply($op, $lease, $tenantActor, false); $assert($phase($op) === 'paused' && count(CloudEngineClient::$lookups) === $lookups + 1 && end(CloudEngineClient::$lookups)[1]['is_admin'] === false && CloudEngineClient::$operations[$id]['status'] === 'uncertain', 'A tenant elevated its lookup or replaced an unreadable legacy child.'); // Both lookup and dispatch admission can be busy before a durable child row // exists. Retry with the same identity, without extending the command bound. foreach (['shutdown', 'start'] as $actionOffset => $action) { foreach (['lookup', 'dispatch'] as $stageOffset => $stage) { $op = $prepareCommand(1310 + $actionOffset * 2 + $stageOffset, $action); $failureField = $stage === 'lookup' ? 'lookupFailure' : 'postFailure'; CloudEngineClient::${$failureField} = new CloudError('busy-private-detail', 409, 'coordination_busy'); $count = count(CloudEngineClient::$commands); $advance($op); $state = Workflow::state($op->fresh()); $expectedPhase = $action === 'start' ? 'starting' : 'shutting_down'; $childId = $state[$action . '_id']; $deadline = $state[$action . '_deadline']; $assert($state['phase'] === $expectedPhase && $op->fresh()->status === 'waiting_observed' && $op->fresh()->lease_token === null && !isset(CloudEngineClient::$operations[$childId]) && $state['last_command_error'] === ['action' => $action, 'stage' => $stage, 'http_status' => 409, 'error_code' => 'coordination_busy'], 'Busy ' . $action . '/' . $stage . ' did not release for bounded polling.'); $assert(!str_contains($op->fresh()->power_workflow_json, 'busy-private-detail') && !array_key_exists('last_command_error', Workflow::publicState($op->fresh())), 'Private power diagnostics leaked message text or entered the public DTO.'); $advance($op); $state = Workflow::state($op->fresh()); $assert($state['phase'] === $expectedPhase && $state[$action . '_id'] === $childId && $state[$action . '_deadline'] === $deadline && count(CloudEngineClient::$commands) === $count, 'Busy polling changed the original command identity or deadline.'); CloudEngineClient::${$failureField} = null; $advance($op); $assert($phase($op) === ($action === 'start' ? 'starting' : 'waiting_stopped') && count(CloudEngineClient::$commands) === $count + 1 && end(CloudEngineClient::$commands)[0] === $childId, 'Cleared admission did not dispatch exactly the original command.'); if ($action === 'start') { $advance($op); $assert($phase($op) === 'completed' && count(CloudEngineClient::$commands) === $count + 1, 'Accepted start did not complete from running state readback.'); } } } foreach (['shutdown', 'start'] as $offset => $action) { $op = $prepareCommand(1314 + $offset, $action); CloudEngineClient::$postFailure = new CloudError('busy', 409, 'coordination_busy'); $advance($op); $state = Workflow::state($op->fresh()); $state[$action . '_deadline'] = time() - 1; $op->power_workflow_json = json_encode($state); $op->save(); CloudEngineClient::$postFailure = null; $attempts = count(CloudEngineClient::$postAttempts); $advance($op); $assert($phase($op) === 'paused' && $op->fresh()->lease_token === null && Workflow::state($op->fresh())['error_code'] === 'vpc_power_command_rejected' && count(CloudEngineClient::$postAttempts) === $attempts, 'Expired ' . $action . ' admission dispatched a late command.'); } foreach (['shutdown', 'start'] as $offset => $action) { $op = $prepareCommand(1343 + $offset, $action); CloudEngineClient::$postFailure = new CloudError('busy', 409, 'coordination_busy'); $advance($op); CloudEngineClient::$postFailure = null; $state = Workflow::state($op->fresh()); $state[$action . '_deadline'] = time() - 1; $op->power_workflow_json = json_encode($state); $op->save(); $id = $state[$action . '_id']; CloudEngineClient::$operations[$id] = ['operation_id' => $id, 'status' => 'succeeded', 'result' => ['uuid' => $state['uuid'], 'action' => $action, 'command_accepted' => true]]; CloudEngineClient::$operationOwners[$id] = $actor['user_id']; if ($action === 'start') { CloudEngineClient::$power = 'running'; } $attempts = count(CloudEngineClient::$postAttempts); $advance($op); if ($action === 'start') { $advance($op); } $assert($phase($op) === ($action === 'start' ? 'completed' : 'waiting_stopped') && count(CloudEngineClient::$postAttempts) === $attempts, 'The deadline discarded an exact accepted ' . $action . ' receipt.'); } foreach ([[409, 'coordination_uncertain'], [503, 'coordination_busy'], [503, 'coordination_unavailable'], [502, 'engine_transport_uncertain'], [502, null], [403, null], [409, 'private-secret-code']] as $offset => [$status, $code]) { $op = $prepareCommand(1316 + $offset, 'shutdown'); CloudEngineClient::$postFailure = new CloudError('private-secret-message', $status, $code); $advance($op); $state = Workflow::state($op->fresh()); $retryable = ($code === 'coordination_busy' && $status === 409) || ($code === 'engine_transport_uncertain' && $status === 502); $assert($phase($op) === ($retryable ? 'shutting_down' : 'paused') && $op->fresh()->status === ($retryable ? 'waiting_observed' : 'paused') && $op->fresh()->lease_token === null && $state['error_code'] === ($retryable ? null : 'vpc_power_result_uncertain') && $state['last_command_error']['http_status'] === $status && !str_contains($op->fresh()->power_workflow_json, 'private-secret'), 'A command transport failure was misclassified or leaked upstream diagnostics.'); CloudEngineClient::$postFailure = null; } // End-to-end service/worker lane with the existing strict host/member proofs. // Quiesce unrelated legacy fixture operations so the worker selects this intent. Capsule::table('cloud_vpc_operations')->update(['status' => 'superseded', 'lease_until' => 0]); CloudEngineClient::$vpcRespond = static function (array $payload) use ($activeHostEvidence): array { $hosts = []; foreach ($payload['hosts'] as $host) { $id = (string) $host['host_id']; if ($host['members'] === []) { $hosts[$id] = [ 'vpc_id' => $payload['vpc_id'], 'host_id' => $id, 'desired_revision' => $payload['desired_revision'], 'desired_digest' => $payload['desired_digest'], 'observed_revision' => $payload['desired_revision'], 'observed_digest' => $payload['desired_digest'], 'state' => 'active', 'transport_verified' => true, 'private_dataplane_verified' => true, 'removed' => true, 'bridge' => null, 'bridge_owner' => null, 'nft_table' => null, 'rollback' => 'none', 'readback' => 'confirmed', 'member_nics' => [], 'routes' => [], 'wg_allowed_ips' => [], 'proxy_ips' => [], ]; } else { $item = $activeHostEvidence($payload['vpc_id'], (int) $id, $payload['desired_revision'], $payload['desired_digest'], $payload['cidr'], $payload['members']); if (CloudEngineClient::$power === 'shutoff') { $item['state'] = 'degraded'; $item['private_dataplane_verified'] = false; $item['last_error_code'] = 'vpc_private_guest_stopped'; foreach ($item['member_nics'] as &$nic) { $nic['domain_running'] = false; $nic['guest_network_verified'] = false; $nic['guest_reason_code'] = 'vpc_private_guest_stopped'; } unset($nic); } $hosts[$id] = $item; } } $active = $hosts !== []; foreach ($hosts as $host) { $active = $active && $host['state'] === 'active' && $host['private_dataplane_verified'] === true; } return [ 'vpc_id' => $payload['vpc_id'], 'operation_id' => $payload['operation_id'], 'desired_revision' => $payload['desired_revision'], 'desired_digest' => $payload['desired_digest'], 'observed_revision' => $payload['desired_revision'], 'observed_digest' => $payload['desired_digest'], 'state' => $active ? 'active' : 'degraded', 'next_stage_allowed' => $active, 'private_dataplane_verified' => $active, 'hosts' => array_values($hosts), 'observed' => ['evidence_version' => CloudVpcService::OBSERVATION_EVIDENCE_VERSION, 'revision' => $payload['desired_revision'], 'digest' => $payload['desired_digest'], 'state' => $active ? 'active' : 'degraded', 'next_stage_allowed' => $active, 'private_dataplane_verified' => $active, 'fabric_gate' => ['active' => true], 'hosts' => $hosts], ]; }; $tick = static function (CloudVpcOperation $operation) use ($assert): void { Capsule::table('cloud_vpc_operations')->where('id', $operation->id) ->update(['updated_at' => time() - 31, 'lease_until' => 0]); $assert((new CloudVpcWorkerService())->tick(), 'Independent VPC worker did not select current workflow.'); }; $op = $make(1210); $tick($op); $assert($phase($op) === 'waiting_stopped' && CloudEngineClient::$applies === 0, 'VPC applied configuration before stopping the guest.'); CloudEngineClient::$power = 'shutoff'; $tick($op); $assert($phase($op) === 'starting' && CloudEngineClient::$power === 'running', 'Accepted automatic start was not retained for independent state readback.'); $tick($op); $assert($phase($op) === 'completed' && $op->fresh()->status === 'succeeded' && CloudEngineClient::$power === 'running', 'Service/worker did not apply and restore a running member.'); $root = CloudVpc::find($op->vpc_id); $member = CloudVpcMember::where('instance_id', 121000)->first(); $assert($root->state === 'active' && $root->sync_state === 'applied' && $root->last_error_code === null && $member->state === 'stopped' && $member->last_error_code === 'vpc_private_guest_stopped', 'Guest-only stopped evidence leaked into the applied root configuration.'); $commands = count(CloudEngineClient::$commands); $applies = CloudEngineClient::$applies; $attempts = (int) $op->fresh()->attempts; // A completed, config-exact workflow is terminal for the apply worker. A // later independent guest-health receipt may update only member/IP runtime. Capsule::table('cloud_vpc_operations')->where('id', '<>', $op->id) ->update(['status' => 'superseded', 'lease_until' => 0]); Capsule::table('cloud_vpc_operations')->where('id', $op->id) ->update(['updated_at' => time() - 31, 'lease_until' => 0]); $assert((new CloudVpcWorkerService())->tick() === false && (int) $op->fresh()->attempts === $attempts && CloudEngineClient::$applies === $applies && count(CloudEngineClient::$commands) === $commands, 'The worker replayed a succeeded operation solely for guest health.'); $root->refresh(); $member->refresh(); $private = \App\Models\CloudVpcPrivateIp::find($member->private_ip_id); $assert($phase($op) === 'completed' && $op->fresh()->status === 'succeeded' && (int) $op->fresh()->attempts === $attempts && $root->state === 'active' && $root->sync_state === 'applied' && $root->last_error_code === null && (int) $root->observed_revision === (int) $root->desired_revision && $member->state === 'stopped' && $member->last_error_code === 'vpc_private_guest_stopped' && $private->state === 'stopped' && $private->last_error_code === 'vpc_private_guest_stopped' && CloudEngineClient::$applies === $applies && count(CloudEngineClient::$commands) === $commands, 'A guest-only stopped state was not retained without replaying apply.'); $detach = $legacyDetach($root, $member, 'auto-detach-1210'); $tick($detach); CloudEngineClient::$power = 'shutoff'; // Losing the start response keeps the fixed child in automatic receipt/state // recovery after exact last-member teardown released membership/IP rows. CloudEngineClient::$loseReceipt = true; $tick($detach); $assert($phase($detach) === 'starting' && $detach->fresh()->status === 'waiting_observed' && CloudVpcMember::where('instance_id', 121000)->count() === 0, 'Last-member teardown did not preserve restoration workflow after removal.'); $other = $seedVpc(1211, '10.121.211.0/24', '10.121.211.0/24', 10); $expectCloudError(static fn () => CloudVpcService::attachMember($other['vpc_key'], $actor, [ 'instance_id' => 121000, 'request_key' => 'cross-attach-121000', ]), 409); $count = count(CloudEngineClient::$commands); $tick($detach); $tick($detach); $assert($phase($detach) === 'completed' && $detach->fresh()->status === 'succeeded' && count(CloudEngineClient::$commands) === $count, 'Last-member paused start could not recover its original receipt.'); // The departing host can finish while a surviving host is still converging. // Keep that exact accepted target in the GET lane until the complete proof // arrives; a VM must not remain off merely because the first receipt is partial. $ordinaryVpcRespond = CloudEngineClient::$vpcRespond; $runningVpcProof = static function (array $payload) use ($ordinaryVpcRespond, $activeHostEvidence): array { $result = $ordinaryVpcRespond($payload); $hosts = []; foreach ($result['hosts'] as $host) { $hostId = (string) $host['host_id']; $hosts[$hostId] = ($host['removed'] ?? false) === true ? $host : $activeHostEvidence($payload['vpc_id'], (int) $hostId, $payload['desired_revision'], $payload['desired_digest'], $payload['cidr'], $payload['members']); } $result['state'] = $result['observed']['state'] = 'active'; $result['next_stage_allowed'] = $result['observed']['next_stage_allowed'] = true; $result['private_dataplane_verified'] = $result['observed']['private_dataplane_verified'] = true; $result['hosts'] = array_values($hosts); $result['observed']['hosts'] = $hosts; return $result; }; // A multi-host attach may return a bounded recovery snapshot while one host // revalidates after an engine restart. Exact target/operation/revision/digest // and the complete host closure keep it in the same read-only window. Capsule::table('cloud_vpc_operations')->update(['status' => 'superseded', 'lease_until' => 0]); $seedInstance(128000, 20); $multiHostVpc = $seedVpc(1280, '10.128.0.0/24', '10.128.0.0/24', 20, 'active', 'active', [ 'member_row_id' => 928000, 'instance_id' => 128000, 'private_ip' => '10.128.0.1', ]); $seedPair(928000, 938000, $multiHostVpc, 128000, 20, '10.128.0.1', 'active'); $seedInstance(128001, 10); CloudEngineClient::$power = 'running'; CloudVpcService::attachMember($multiHostVpc['vpc_key'], $actor, [ 'instance_id' => 128001, 'request_key' => 'multi-host-auto-attach-128001', ]); $multiAttach = CloudVpcOperation::where('vpc_id', 1280) ->orderByDesc('desired_revision')->first(); Workflow::initialize( $multiAttach, CloudInstance::find(128001), CloudVpcMember::where('instance_id', 128001)->first() ); $multiAttach->refresh(); $tick($multiAttach); CloudEngineClient::$power = 'shutoff'; $multiPayload = json_decode($multiAttach->payload_json, true); $multiUncertain = $ordinaryVpcRespond($multiPayload); $multiUncertain['state'] = $multiUncertain['observed']['state'] = 'uncertain'; $multiUncertain['next_stage_allowed'] = $multiUncertain['observed']['next_stage_allowed'] = false; $multiUncertain['private_dataplane_verified'] = $multiUncertain['observed']['private_dataplane_verified'] = false; $multiUncertain['observed']['last_error_code'] = 'stale_pre_restart_failure'; $multiUncertain['observed']['fabric_gate'] = [ 'active' => false, 'available' => true, 'error_code' => 'vpc_recovery_pending', ]; $recoveringHost = $multiUncertain['observed']['hosts']['20']; $recoveringHost['state'] = 'uncertain'; $recoveringHost['transport_verified'] = false; $recoveringHost['private_dataplane_verified'] = false; $recoveringHost['config_verified'] = false; $recoveringHost['observed_revision'] = 0; $recoveringHost['observed_digest'] = null; $recoveringHost['config_observed_revision'] = 0; $recoveringHost['config_observed_digest'] = null; // VpcCoordinator.recover() closes the root Fabric gate and marks each host // uncertain, but deliberately does not manufacture a host-level error code. $recoveringHost['last_error_code'] = null; $multiUncertain['observed']['hosts']['20'] = $recoveringHost; $multiUncertain['hosts'] = array_values($multiUncertain['observed']['hosts']); CloudEngineClient::$vpcRespond = static fn () => $multiUncertain; $applies = CloudEngineClient::$applies; $commands = count(CloudEngineClient::$commands); $tick($multiAttach); $state = Workflow::state($multiAttach->fresh()); $deadline = $state['configuration_deadline'] ?? 0; $assert($state['phase'] === 'applying' && !$state['config_confirmed'] && $deadline > time() && $multiAttach->fresh()->status === 'waiting_observed' && CloudEngineClient::$applies === $applies + 1 && count(CloudEngineClient::$commands) === $commands, 'Reviewed multi-host attach recovery evidence paused the automatic workflow.'); CloudEngineClient::$vpcRead = static fn () => $multiUncertain; $tick($multiAttach); $state = Workflow::state($multiAttach->fresh()); $assert($state['phase'] === 'applying' && $state['configuration_deadline'] === $deadline && CloudEngineClient::$applies === $applies + 1 && count(CloudEngineClient::$commands) === $commands, 'Multi-host attach recovery resent apply or extended its read-only deadline.'); $multiExact = $ordinaryVpcRespond($multiPayload); CloudEngineClient::$vpcRead = static fn () => $multiExact; $tick($multiAttach); $tick($multiAttach); $assert($phase($multiAttach) === 'completed' && CloudEngineClient::$power === 'running' && CloudEngineClient::$applies === $applies + 1 && count(CloudEngineClient::$commands) === $commands + 1, 'Multi-host attach did not converge from exact later readback and restore power once.'); CloudEngineClient::$vpcRespond = $ordinaryVpcRespond; CloudEngineClient::$vpcRead = null; $partialDetach = static function (int $id, bool $sameHost = false, ?string $fault = null) use ( $make, $tick, $actor, $seedInstance, $ordinaryVpcRespond, $runningVpcProof, $legacyDetach, $assert ): array { Capsule::table('cloud_vpc_operations')->update(['status' => 'superseded', 'lease_until' => 0]); CloudEngineClient::$vpcRespond = $ordinaryVpcRespond; CloudEngineClient::$vpcRead = null; $attach = $make($id); $tick($attach); CloudEngineClient::$power = 'shutoff'; $tick($attach); $tick($attach); $vpc = CloudVpc::find($attach->vpc_id); foreach ([1, 2] as $offset) { $seedInstance($id * 100 + $offset, $sameHost ? 10 : 20); CloudVpcService::attachMember($vpc->vpc_key, $actor, [ 'instance_id' => $id * 100 + $offset, 'request_key' => 'retained-sibling-' . $id . '-' . $offset, ]); CloudVpcService::reconcile($vpc->vpc_key, $actor, true, true); } $target = CloudVpcMember::where('instance_id', $id * 100)->first(); $operation = $legacyDetach($vpc, $target, 'partial-detach-' . $id); $tick($operation); CloudEngineClient::$power = 'shutoff'; $full = $runningVpcProof(json_decode($operation->payload_json, true)); $partial = $full; $partial['state'] = $partial['observed']['state'] = 'degraded'; $partial['next_stage_allowed'] = $partial['observed']['next_stage_allowed'] = false; $partial['private_dataplane_verified'] = $partial['observed']['private_dataplane_verified'] = false; $partial['observed']['fabric_gate'] = ['active' => false, 'available' => true]; foreach ($partial['hosts'] as &$host) { if (($host['removed'] ?? false) === true) { continue; } $host['transport_verified'] = $host['config_verified'] = $host['private_dataplane_verified'] = false; $host['observed_revision'] = $host['config_observed_revision'] = 0; $host['observed_digest'] = $host['config_observed_digest'] = null; $host['last_error_code'] = 'vpc_private_dataplane_unverified'; $host['wg_allowed_ips'] = []; } unset($host); $partial['observed']['hosts'] = []; if ($fault === 'missing_host') { array_pop($partial['hosts']); } elseif ($fault === 'host_digest') { $partial['hosts'][count($partial['hosts']) - 1]['desired_digest'] = str_repeat('f', 64); } elseif ($fault === 'host_failed') { $partial['hosts'][count($partial['hosts']) - 1]['last_error_code'] = 'vpc_private_dataplane_apply_failed'; } elseif ($fault === 'host_uncertain_unknown') { $partial['hosts'][count($partial['hosts']) - 1]['state'] = 'uncertain'; $partial['hosts'][count($partial['hosts']) - 1]['last_error_code'] = 'unreviewed_host_failure'; } elseif ($fault === 'root_uncertain_unknown') { $partial['state'] = $partial['observed']['state'] = 'uncertain'; $partial['observed']['fabric_gate']['error_code'] = 'unreviewed_root_failure'; } CloudEngineClient::$vpcRespond = static fn () => $partial; $tick($operation); return [$operation, $partial, $full]; }; [$op, $partial, $full] = $partialDetach(1260); $state = Workflow::state($op->fresh()); $deadline = $state['configuration_deadline'] ?? 0; $applies = CloudEngineClient::$applies; $count = count(CloudEngineClient::$commands); $assert($state['phase'] === 'applying' && !$state['config_confirmed'] && $deadline > time() && $op->fresh()->status === 'waiting_observed' && $op->fresh()->lease_token === null && CloudVpcMember::where('instance_id', 126000)->count() === 1, 'The partial two-host detach paused or released an unconfirmed member.'); foreach (['partial', 'missing', 'older', 'offline'] as $mode) { CloudEngineClient::$vpcRead = static function () use ($partial, $mode): array { if ($mode === 'missing') { throw new CloudError('missing', 404); } if ($mode === 'offline') { throw new CloudError('offline', 502); } if ($mode === 'older') { $partial['desired_revision']--; } return $partial; }; $tick($op); $state = Workflow::state($op->fresh()); $assert($state['phase'] === 'applying' && !$state['config_confirmed'] && $state['configuration_deadline'] === $deadline && $op->fresh()->status === 'waiting_observed' && CloudEngineClient::$applies === $applies && count(CloudEngineClient::$commands) === $count, 'Accepted configuration was resubmitted, restarted or extended on ' . $mode); } CloudEngineClient::$vpcRead = static fn () => $full; $tick($op); $assert($phase($op) === 'starting', 'Exact detach proof did not retain accepted start for state readback.'); $tick($op); $assert($phase($op) === 'completed' && CloudEngineClient::$power === 'running' && CloudVpcMember::where('instance_id', 126000)->count() === 0 && CloudVpcMember::whereIn('instance_id', [126001, 126002])->where('state', 'active')->count() === 2 && CloudEngineClient::$applies === $applies && count(CloudEngineClient::$commands) === $count + 1, 'Later exact detach proof did not release only the target and restore it once.'); // The same mechanism also preserves siblings when the departing host stays live. [$op, $partial, $full] = $partialDetach(1261, true); $applies = CloudEngineClient::$applies; CloudEngineClient::$vpcRead = static fn () => $full; $tick($op); $tick($op); $assert($phase($op) === 'completed' && CloudVpcMember::where('instance_id', 126100)->count() === 0 && CloudVpcMember::whereIn('instance_id', [126101, 126102])->where('state', 'active')->count() === 2 && CloudEngineClient::$applies === $applies, 'Same-host detach required removing the host or lost its retained siblings.'); foreach (['partial', 'missing', 'exact'] as $offset => $mode) { [$op, $partial, $full] = $partialDetach(1262 + $offset); $state = Workflow::state($op->fresh()); $state['configuration_deadline'] = time() - 1; $op->power_workflow_json = json_encode($state); $op->save(); $applies = CloudEngineClient::$applies; $count = count(CloudEngineClient::$commands); CloudEngineClient::$vpcRead = static function () use ($mode, $partial, $full): array { if ($mode === 'missing') { throw new CloudError('missing', 404); } return $mode === 'exact' ? $full : $partial; }; $tick($op); if ($mode === 'exact') { $tick($op); } $assert($phase($op) === ($mode === 'exact' ? 'completed' : 'paused') && CloudEngineClient::$applies === $applies && count(CloudEngineClient::$commands) === $count + ($mode === 'exact' ? 1 : 0), 'Configuration deadline extended, resent a POST or discarded exact proof: ' . $mode); if ($mode !== 'exact') { $assert(Workflow::state($op->fresh())['configuration_deadline'] === $state['configuration_deadline'], 'A deadline was refreshed by an ordinary worker poll.'); CloudEngineClient::$vpcRead = static fn () => $partial; CloudVpcService::reconcile(CloudVpc::find($op->vpc_id)->vpc_key, $actor, true, true); $assert($phase($op) === 'applying' && Workflow::state($op->fresh())['configuration_deadline'] > time() && CloudEngineClient::$applies === $applies, 'Explicit retry could not open a bounded read-only recovery window.'); } } [$op, $partial, $full] = $partialDetach(1265); $instance = CloudInstance::find(126500); $instance->getConnection()->transaction(static function () use ($instance): void { Workflow::lockForInstance($instance); Workflow::manualIntentLocked($instance, 'action', 'start'); }); $applies = CloudEngineClient::$applies; $count = count(CloudEngineClient::$commands); CloudEngineClient::$vpcRead = static fn () => $full; $tick($op); $assert($phase($op) === 'completed' && Workflow::state($op->fresh())['resume_cancelled'] && CloudEngineClient::$applies === $applies && count(CloudEngineClient::$commands) === $count, 'Configuration polling overrode a newer manual power intent.'); foreach ([[1266, 'missing_host'], [1267, 'host_digest'], [1268, 'host_failed'], [1269, 'host_uncertain_unknown'], [1290, 'root_uncertain_unknown']] as [$caseId, $fault]) { [$op] = $partialDetach($caseId, false, $fault); $state = Workflow::state($op->fresh()); $assert($state['phase'] === 'paused' && !$state['config_confirmed'] && !isset($state['configuration_deadline']) && CloudEngineClient::$power === 'shutoff', 'Invalid partial proof opened an automatic waiting window: ' . $fault); } CloudEngineClient::$vpcRespond = $ordinaryVpcRespond; CloudEngineClient::$vpcRead = null; // Recover an interrupted apply from the exact durable engine receipt before // the stopped-guest mutation precondition. A manual start needs no second stop. $interrupted = static function (int $id, bool $paused = true) use ($make): CloudVpcOperation { $operation = $make($id, 'running'); $state = Workflow::state($operation); $state['phase'] = $paused ? 'paused' : 'applying'; $state['paused_from'] = 'applying'; $state['original_running'] = false; $state['resume_cancelled'] = true; $state['error_code'] = 'vpc_power_manual_override'; $operation->power_workflow_json = json_encode($state); $operation->status = $paused ? 'paused' : 'waiting_observed'; $operation->lease_token = null; $operation->lease_until = 0; $operation->save(); return $operation; }; $receipt = static function (CloudVpcOperation $operation): array { $result = (CloudEngineClient::$vpcRespond)(json_decode($operation->payload_json, true)); $result['observed']['state'] = 'active'; $result['observed']['next_stage_allowed'] = true; $result['observed']['private_dataplane_verified'] = true; // Exercise the engine's real compact observed + full top-level hosts DTO. $result['hosts'] = $result['observed']['hosts']; $result['observed']['hosts'] = []; return $result; }; foreach ([true, false] as $offset => $paused) { $op = $interrupted(1220 + $offset, $paused); $proof = $receipt($op); CloudEngineClient::$vpcRead = static fn () => $proof; $applies = CloudEngineClient::$applies; $reads = CloudEngineClient::$readbacks; $count = count(CloudEngineClient::$commands); CloudEngineClient::$failState = true; CloudVpcService::reconcile(CloudVpc::find($op->vpc_id)->vpc_key, $actor, true, $paused); CloudEngineClient::$failState = false; $root = CloudVpc::find($op->vpc_id); $assert($phase($op) === 'completed' && $op->fresh()->status === 'succeeded' && $root->state === 'active' && $root->observed_revision === $root->desired_revision && CloudEngineClient::$readbacks === $reads + 1 && CloudEngineClient::$applies === $applies && count(CloudEngineClient::$commands) === $count && Workflow::state($op->fresh())['error_code'] === null, 'Exact interrupted apply did not recover without another mutation/power query.'); } // A read failure/mismatch must not corrupt paused_from or trigger a new apply. foreach (['missing', 'older', 'offline', 'vpc', 'operation', 'digest', 'newer', 'invalid_revision', 'host', 'nic', 'acl', 'fractional_revision'] as $offset => $fault) { $op = $interrupted(1230 + $offset); $proof = $receipt($op); switch ($fault) { case 'vpc': $proof['vpc_id'] = str_repeat('f', 32); break; case 'operation': $proof['operation_id'] = str_repeat('f', 32); break; case 'digest': $proof['desired_digest'] = str_repeat('f', 64); break; case 'older': $proof['desired_revision']--; break; case 'newer': $proof['desired_revision']++; break; case 'invalid_revision': $proof['desired_revision'] = 'garbage'; break; case 'fractional_revision': $proof['desired_revision'] += 0.9; break; case 'host': $proof['hosts'] = []; break; case 'nic': $proof['hosts']['10']['member_nics'] = []; break; case 'acl': $proof['hosts']['10']['acl_revision'] = null; break; } CloudEngineClient::$vpcRead = static function () use ($proof, $fault): array { if (in_array($fault, ['missing', 'offline'], true)) { throw new CloudError('read failed', $fault === 'missing' ? 404 : 502); } return $proof; }; $applies = CloudEngineClient::$applies; $count = count(CloudEngineClient::$commands); CloudVpcService::reconcile(CloudVpc::find($op->vpc_id)->vpc_key, $actor, true, true); $state = Workflow::state($op->fresh()); $assert($phase($op) === 'paused' && $state['paused_from'] === 'applying' && !$state['config_confirmed'] && CloudEngineClient::$applies === $applies && count(CloudEngineClient::$commands) === $count, 'Invalid/missing receipt bypassed stopped guest gate or corrupted phase: ' . $fault); } // The same absent receipt may proceed through the original mutation lane only // after the guest is stopped. Unverified running proof never authorizes apply. $op = $interrupted(1251); CloudEngineClient::$power = 'shutoff'; CloudEngineClient::$vpcRead = null; $applies = CloudEngineClient::$applies; CloudVpcService::reconcile(CloudVpc::find($op->vpc_id)->vpc_key, $actor, true, true); $assert($phase($op) === 'completed' && CloudEngineClient::$applies === $applies + 1, 'Missing receipt prevented legitimate stopped-guest configuration.'); // A rejected first submission has no accepted partial receipt. Preserve the // original explicit-retry lane instead of fencing it behind a permanent GET. $op = $interrupted(1270); CloudEngineClient::$power = 'shutoff'; CloudEngineClient::$vpcRead = null; CloudEngineClient::$vpcRespond = static function (): array { throw new CloudError('rejected before apply', 422); }; $applies = CloudEngineClient::$applies; CloudVpcService::reconcile(CloudVpc::find($op->vpc_id)->vpc_key, $actor, true, true); $assert($phase($op) === 'paused' && !isset(Workflow::state($op->fresh())['configuration_deadline']) && CloudEngineClient::$applies === $applies + 1, 'A rejected first POST acquired an accepted-configuration fence.'); CloudEngineClient::$vpcRespond = $ordinaryVpcRespond; CloudVpcService::reconcile(CloudVpc::find($op->vpc_id)->vpc_key, $actor, true, true); $assert($phase($op) === 'completed' && CloudEngineClient::$applies === $applies + 2, 'Explicit retry could not resubmit after a definite first-POST rejection.'); // New manual intent during the read wins even if the original guest ran. $op = $interrupted(1252); $state = Workflow::state($op); $state['original_running'] = true; $state['resume_cancelled'] = false; $op->power_workflow_json = json_encode($state); $op->save(); $proof = $receipt($op); CloudEngineClient::$vpcRead = static function () use ($op, $proof): array { $instance = CloudInstance::find(125200); $instance->getConnection()->transaction(static function () use ($instance): void { Workflow::lockForInstance($instance); Workflow::manualIntentLocked($instance, 'action', 'start'); }); return $proof; }; $count = count(CloudEngineClient::$commands); CloudVpcService::reconcile(CloudVpc::find($op->vpc_id)->vpc_key, $actor, true, true); $assert($phase($op) === 'completed' && count(CloudEngineClient::$commands) === $count && Workflow::state($op->fresh())['resume_cancelled'], 'Receipt recovery overrode concurrent manual intent.'); // A late GET cannot publish evidence after ownership or instance identity moves. foreach (['lease', 'revision', 'instance'] as $offset => $change) { $op = $interrupted(1253 + $offset); $proof = $receipt($op); $root = CloudVpc::find($op->vpc_id); $previous = [$root->observed_revision, $root->observed_digest]; CloudEngineClient::$vpcRead = static function () use ($op, $proof, $change): array { if ($change === 'lease') { Capsule::table('cloud_vpc_operations')->where('id', $op->id)->update(['lease_token' => str_repeat('f', 32)]); } elseif ($change === 'revision') { Capsule::table('cloud_vpcs')->where('id', $op->vpc_id)->increment('desired_revision'); } else { Capsule::table('cloud_instances')->where('id', $op->power_instance_id)->update(['host_id' => 11]); } return $proof; }; $applies = CloudEngineClient::$applies; $count = count(CloudEngineClient::$commands); try { CloudVpcService::reconcile($root->vpc_key, $actor, true, true); } catch (CloudError $error) { /* Stale owner is deliberately unable to release its successor's lease. */ } $root->refresh(); $assert([$root->observed_revision, $root->observed_digest] === $previous && !Workflow::state($op->fresh())['config_confirmed'] && CloudEngineClient::$applies === $applies && count(CloudEngineClient::$commands) === $count, 'Late receipt updated evidence after a changed ' . $change); } CloudEngineClient::$vpcRead = null; // An installed pre-workflow schema continues to admit ordinary power, but // cannot accept a new automatic maintenance request. $instance = CloudInstance::find(121000); $index = 'cloud_vpc_operations_power_instance_id_index'; $assert($schema->hasIndex('cloud_vpc_operations', ['power_instance_id']), 'Workflow lookup has no index to bound its locking read.'); $schema->table('cloud_vpc_operations', static function ($table) use ($index): void { $table->dropIndex($index); }); $expectCloudError(static fn () => Workflow::lockForInstance($instance), 503); $schema->table('cloud_vpc_operations', static function ($table): void { $table->index('power_instance_id'); }); $detach->refresh(); $detach->power_instance_id = 999; $expectCloudError(static fn () => Workflow::state($detach), 503); $detach->refresh(); $schema->table('cloud_vpc_operations', static function ($table) use ($index): void { $table->dropIndex($index); $table->dropColumn('power_instance_id'); }); $expectCloudError(static fn () => Workflow::lockForInstance($instance), 503); $schema->table('cloud_vpc_operations', static function ($table): void { $table->dropColumn('power_workflow_json'); }); Workflow::lockForInstance($instance); Workflow::manualIntentLocked($instance, 'action', 'poweroff'); $expectCloudError(static fn () => Workflow::initialize($detach, $instance, $member), 503); echo "VPC automatic power workflow contracts passed.\n"; }