You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
386 lines
19 KiB
PHP
386 lines
19 KiB
PHP
<?php
|
|
|
|
namespace App\Services\Cloud {
|
|
final class CloudEngineClient
|
|
{
|
|
public static string $power = 'running';
|
|
public static array $commands = [];
|
|
public static array $operations = [];
|
|
public static bool $loseReceipt = false;
|
|
public static bool $failState = false;
|
|
public static $vpcRespond;
|
|
public static int $applies = 0;
|
|
public function __construct(...$args) {}
|
|
public function host(int $host, string $method, string $path, array $actor,
|
|
?array $body = null, ?string $operationId = null): array
|
|
{
|
|
$uuid = explode('/', $path)[1];
|
|
if ($method === 'GET') {
|
|
if (self::$failState) {
|
|
throw new CloudError('offline', 502);
|
|
}
|
|
return ['uuid' => $uuid, 'state' => self::$power];
|
|
}
|
|
self::$commands[] = [$operationId, $body['action'], $uuid];
|
|
$result = ['operation_id' => $operationId, 'status' => 'succeeded',
|
|
'result' => ['uuid' => $uuid, 'action' => $body['action'], 'command_accepted' => true]];
|
|
self::$operations[$operationId] = $result;
|
|
if ($body['action'] === 'start') { self::$power = 'running'; }
|
|
if (self::$loseReceipt) {
|
|
self::$loseReceipt = false;
|
|
throw new CloudError('lost response', 502);
|
|
}
|
|
return $result;
|
|
}
|
|
public function request(string $method, string $path, array $actor,
|
|
?array $body = null, ?string $operationId = null): array
|
|
{
|
|
return self::$operations[basename($path)]
|
|
?? throw new CloudError('missing', 404);
|
|
}
|
|
public function vpc(string $method, string $path, array $actor,
|
|
?array $body, ?string $operationId): array
|
|
{
|
|
self::$applies++;
|
|
return (self::$vpcRespond)($body);
|
|
}
|
|
}
|
|
}
|
|
|
|
namespace {
|
|
use App\Models\CloudInstance;
|
|
use App\Models\CloudVpc;
|
|
use App\Models\CloudVpcMember;
|
|
use App\Models\CloudVpcOperation;
|
|
use App\Services\Cloud\CloudEngineClient;
|
|
use App\Services\Cloud\CloudError;
|
|
use App\Services\Cloud\CloudVpcPowerWorkflow as Workflow;
|
|
use App\Services\Cloud\CloudVpcService;
|
|
use App\Services\Cloud\CloudVpcWorkerService;
|
|
use Illuminate\Database\Capsule\Manager as Capsule;
|
|
|
|
require __DIR__ . '/CloudVpcMemberReuseTest.php';
|
|
if (!isset($capsule)) { return; }
|
|
|
|
$make = static function (int $id, string $power = 'running') use ($seedVpc, $seedInstance, $actor): CloudVpcOperation {
|
|
$vpc = $seedVpc($id, '10.121.' . ($id % 250) . '.0/24',
|
|
'10.121.' . ($id % 250) . '.0/24', 10);
|
|
$seedInstance($id * 100, 10);
|
|
CloudEngineClient::$power = $power;
|
|
CloudVpcService::attachMember($vpc['vpc_key'], $actor, [
|
|
'instance_id' => $id * 100, 'request_key' => 'auto-member-' . $id,
|
|
'auto_power' => true,
|
|
]);
|
|
return CloudVpcOperation::where('vpc_id', $id)->orderByDesc('desired_revision')->first();
|
|
};
|
|
$claim = static function (CloudVpcOperation $operation): array {
|
|
$operation = $operation->fresh();
|
|
$operation->lease_token = bin2hex(random_bytes(16));
|
|
$operation->lease_until = time() + 120;
|
|
$operation->status = 'running';
|
|
$operation->save();
|
|
return [$operation, $operation->lease_token];
|
|
};
|
|
$observed = static function (CloudVpcOperation $operation): void {
|
|
Workflow::configurationObserved($operation->fresh(), true);
|
|
};
|
|
$phase = static fn (CloudVpcOperation $operation): string => Workflow::state($operation->fresh())['phase'];
|
|
|
|
// Normal running guest: command acknowledgement, separate state observation,
|
|
// exact configuration gate, then one automatic restart.
|
|
$op = $make(1201);
|
|
[$op, $lease] = $claim($op);
|
|
$assert(Workflow::beforeApply($op, $lease, $actor, false) === false
|
|
&& $phase($op) === 'waiting_stopped', 'Shutdown must finish only the command step.');
|
|
$assert($op->fresh()->lease_token === null, 'Waiting for guest shutdown held its lease.');
|
|
$count = count(CloudEngineClient::$commands);
|
|
[$op, $lease] = $claim($op);
|
|
$assert(!Workflow::beforeApply($op, $lease, $actor, false)
|
|
&& count(CloudEngineClient::$commands) === $count, 'Polling resent shutdown.');
|
|
CloudEngineClient::$power = 'shutoff';
|
|
[$op, $lease] = $claim($op);
|
|
$assert(Workflow::beforeApply($op, $lease, $actor, false), 'Stopped guest did not enter apply.');
|
|
Workflow::afterApply($op, $lease, $actor);
|
|
$assert(count(CloudEngineClient::$commands) === $count, 'Guest started before exact configuration proof.');
|
|
[$op, $lease] = $claim($op);
|
|
$assert(Workflow::beforeApply($op, $lease, $actor, true), 'Explicit retry did not resume configuration.');
|
|
$observed($op);
|
|
Workflow::afterApply($op, $lease, $actor);
|
|
$assert($phase($op) === 'completed' && count(CloudEngineClient::$commands) === $count + 1
|
|
&& end(CloudEngineClient::$commands)[1] === 'start', 'Originally running guest was not restored.');
|
|
|
|
// Immutable API replay includes the explicit maintenance acknowledgement.
|
|
$vpc = CloudVpc::find($op->vpc_id);
|
|
$expectCloudError(static fn () => CloudVpcService::attachMember($vpc->vpc_key, $actor, [
|
|
'instance_id' => 120100, 'request_key' => 'auto-member-1201', 'auto_power' => false,
|
|
]), 409);
|
|
CloudVpcService::attachMember($vpc->vpc_key, $actor, [
|
|
'instance_id' => 120100, 'request_key' => 'auto-member-1201', 'auto_power' => true,
|
|
]);
|
|
$expectCloudError(static fn () => CloudVpcService::attachMember($vpc->vpc_key, $actor, [
|
|
'instance_id' => 120100, 'request_key' => 'auto-member-1201', 'auto_power' => true,
|
|
'subnet_key' => str_repeat('a', 32),
|
|
]), 422);
|
|
foreach (['true', 1, null] as $badFlag) {
|
|
$expectCloudError(static fn () => Workflow::requested(['auto_power' => $badFlag]), 422);
|
|
}
|
|
|
|
// A previously stopped or already shutting-down guest must never be restarted.
|
|
foreach (['shutoff', 'shutdown'] as $offset => $initial) {
|
|
$op = $make(1202 + $offset, $initial);
|
|
$count = count(CloudEngineClient::$commands);
|
|
[$op, $lease] = $claim($op);
|
|
$ready = Workflow::beforeApply($op, $lease, $actor, false);
|
|
if (!$ready) {
|
|
CloudEngineClient::$power = 'shutoff';
|
|
[$op, $lease] = $claim($op);
|
|
$assert(Workflow::beforeApply($op, $lease, $actor, false), 'Existing shutdown did not converge.');
|
|
}
|
|
$observed($op);
|
|
Workflow::afterApply($op, $lease, $actor);
|
|
$assert($phase($op) === 'completed' && count(CloudEngineClient::$commands) === $count,
|
|
'A pre-existing stopped/shutdown state generated power commands.');
|
|
}
|
|
|
|
// Lost receipt: repeat uses the same durable engine operation, no second POST.
|
|
$op = $make(1204);
|
|
CloudEngineClient::$loseReceipt = true;
|
|
[$op, $lease] = $claim($op);
|
|
Workflow::beforeApply($op, $lease, $actor, false);
|
|
$assert($phase($op) === 'paused', 'Lost receipt did not pause.');
|
|
$count = count(CloudEngineClient::$commands);
|
|
[$op, $lease] = $claim($op);
|
|
Workflow::beforeApply($op, $lease, $actor, true);
|
|
$assert($phase($op) === 'waiting_stopped' && count(CloudEngineClient::$commands) === $count,
|
|
'Lost accepted command was submitted again.');
|
|
|
|
// Graceful timeout pauses, permits force-off, and cancels automatic restoration.
|
|
$state = Workflow::state($op->fresh());
|
|
$state['shutdown_deadline'] = time() - 1;
|
|
$op->power_workflow_json = json_encode($state); $op->save();
|
|
[$op, $lease] = $claim($op);
|
|
Workflow::beforeApply($op, $lease, $actor, false);
|
|
$assert($phase($op) === 'paused' && $op->fresh()->status === 'paused', 'Timeout became an uncertain mutation.');
|
|
$instance = CloudInstance::find(120400);
|
|
$instance->getConnection()->transaction(static function () use ($instance): void {
|
|
Workflow::lockForInstance($instance);
|
|
$locked = CloudInstance::where('id', $instance->id)->lockForUpdate()->first();
|
|
Workflow::manualIntentLocked($locked, 'action', 'poweroff');
|
|
});
|
|
CloudEngineClient::$power = 'shutoff';
|
|
[$op, $lease] = $claim($op);
|
|
$assert(Workflow::beforeApply($op, $lease, $actor, true), 'Manual force-off cannot resume configuration.');
|
|
$observed($op); Workflow::afterApply($op, $lease, $actor);
|
|
$assert($phase($op) === 'completed' && count(CloudEngineClient::$commands) === $count,
|
|
'Automation undid newer manual power intent.');
|
|
|
|
// Only the active revision can continue; unfinished workflows forbid replacing
|
|
// their VPC plan, including paused steps and last-member removal.
|
|
$op = $make(1205, 'shutoff');
|
|
$vpc = CloudVpc::find($op->vpc_id);
|
|
$expectCloudError(static fn () => Workflow::assertMutationAllowed($vpc), 409);
|
|
[$op, $lease] = $claim($op);
|
|
Capsule::table('cloud_vpcs')->where('id', $vpc->id)->update(['desired_revision' => 999]);
|
|
$count = count(CloudEngineClient::$commands);
|
|
$expectCloudError(static fn () => Workflow::beforeApply($op, $lease, $actor, false), 409);
|
|
$assert(count(CloudEngineClient::$commands) === $count, 'Stale revision sent a power command.');
|
|
|
|
// Read-only failures pause without a child action or a lingering task lease.
|
|
$op = $make(1206);
|
|
CloudEngineClient::$failState = true;
|
|
[$op, $lease] = $claim($op);
|
|
try { Workflow::beforeApply($op, $lease, $actor, false); }
|
|
catch (CloudError $error) { Workflow::handleFailure($op, $lease, $error); }
|
|
CloudEngineClient::$failState = false;
|
|
$assert($op->fresh()->status === 'paused' && $op->fresh()->lease_token === null,
|
|
'Read-only preflight failure retained a lease.');
|
|
|
|
// Crash during apply has a bounded lease, so later manual power stays possible.
|
|
$op = $make(1207, 'shutoff');
|
|
[$op, $lease] = $claim($op); Workflow::beforeApply($op, $lease, $actor, false);
|
|
$op = $op->fresh(); $op->lease_until = time() - 1; $op->save();
|
|
$instance = CloudInstance::find(120700);
|
|
$instance->getConnection()->transaction(static function () use ($instance): void {
|
|
Workflow::lockForInstance($instance);
|
|
Workflow::manualIntentLocked($instance, 'action', 'poweroff');
|
|
});
|
|
$assert(Workflow::state($op->fresh())['resume_cancelled'] === true,
|
|
'Expired apply worker permanently blocked ordinary power.');
|
|
|
|
// Corrupt child identity cannot be used to query or send an unrelated command.
|
|
$op = $make(1208);
|
|
$state = Workflow::state($op); $state['shutdown_id'] = str_repeat('f', 32);
|
|
$op->power_workflow_json = json_encode($state); $op->save();
|
|
$expectCloudError(static fn () => Workflow::state($op), 503);
|
|
|
|
// A crash before the first shutdown dispatch cannot overrule a newer start.
|
|
$op = $make(1209);
|
|
$state = Workflow::state($op);
|
|
$state['phase'] = 'shutting_down'; $state['original_running'] = true;
|
|
$state['shutdown_deadline'] = time() + 120;
|
|
$op->power_workflow_json = json_encode($state); $op->lease_until = 0; $op->save();
|
|
$instance = CloudInstance::find(120900);
|
|
$instance->getConnection()->transaction(static function () use ($instance): void {
|
|
Workflow::lockForInstance($instance);
|
|
Workflow::manualIntentLocked($instance, 'action', 'start');
|
|
});
|
|
$count = count(CloudEngineClient::$commands);
|
|
[$op, $lease] = $claim($op);
|
|
Workflow::beforeApply($op, $lease, $actor, false);
|
|
$assert($phase($op) === 'waiting_stopped' && count(CloudEngineClient::$commands) === $count,
|
|
'Recovered undispatched shutdown overruled a newer manual start.');
|
|
|
|
// A definitely undispatched rejection can proceed after manual stop; an
|
|
// uncertain/in-flight child is never replaced by a fresh command identifier.
|
|
$op = $make(1212);
|
|
$state = Workflow::state($op);
|
|
CloudEngineClient::$operations[$state['shutdown_id']] = [
|
|
'operation_id' => $state['shutdown_id'], 'status' => 'failed',
|
|
'execution_outcome' => 'not_dispatched', 'mutation_started' => false,
|
|
];
|
|
[$op, $lease] = $claim($op); Workflow::beforeApply($op, $lease, $actor, false);
|
|
$assert($phase($op) === 'paused' && Workflow::state($op->fresh())['error_code'] === 'vpc_power_command_rejected',
|
|
'Definitive rejection was mislabeled as unknown.');
|
|
CloudEngineClient::$power = 'shutoff';
|
|
[$op, $lease] = $claim($op); Workflow::beforeApply($op, $lease, $actor, true);
|
|
$assert($phase($op) === 'applying', 'Known undispatched shutdown could not proceed after manual stop.');
|
|
|
|
$op = $make(1213);
|
|
$state = Workflow::state($op);
|
|
CloudEngineClient::$operations[$state['shutdown_id']] = [
|
|
'operation_id' => $state['shutdown_id'], 'status' => 'running',
|
|
];
|
|
[$op, $lease] = $claim($op); Workflow::beforeApply($op, $lease, $actor, false);
|
|
$assert($op->fresh()->status === 'waiting_observed' && $phase($op) === 'shutting_down',
|
|
'In-flight power acknowledgement requires automatic polling, not a manual pause.');
|
|
CloudEngineClient::$operations[$state['shutdown_id']]['status'] = 'uncertain';
|
|
[$op, $lease] = $claim($op); Workflow::beforeApply($op, $lease, $actor, false);
|
|
$assert($phase($op) === 'paused' && count(CloudEngineClient::$commands) === $count,
|
|
'Unknown shutdown outcome was replayed.');
|
|
|
|
$op = $make(1214);
|
|
[$op, $lease] = $claim($op); Workflow::beforeApply($op, $lease, $actor, false);
|
|
CloudEngineClient::$power = 'shutoff';
|
|
[$op, $lease] = $claim($op); Workflow::beforeApply($op, $lease, $actor, false);
|
|
$observed($op); $state = Workflow::state($op->fresh());
|
|
CloudEngineClient::$operations[$state['start_id']] = [
|
|
'operation_id' => $state['start_id'], 'status' => 'failed',
|
|
'execution_outcome' => 'not_dispatched', 'mutation_started' => false,
|
|
];
|
|
Workflow::afterApply($op, $lease, $actor);
|
|
$instance = CloudInstance::find(121400);
|
|
$instance->getConnection()->transaction(static function () use ($instance): void {
|
|
Workflow::lockForInstance($instance);
|
|
Workflow::manualIntentLocked($instance, 'action', 'start');
|
|
});
|
|
[$op, $lease] = $claim($op); Workflow::beforeApply($op, $lease, $actor, true);
|
|
$assert($phase($op) === 'completed', 'Manual start could not finish rejected automatic restoration.');
|
|
|
|
// End-to-end service/worker lane with the existing strict host/member proofs.
|
|
// Quiesce unrelated legacy fixture operations so the worker selects this intent.
|
|
Capsule::table('cloud_vpc_operations')->update(['status' => 'superseded', 'lease_until' => 0]);
|
|
CloudEngineClient::$vpcRespond = static function (array $payload) use ($activeHostEvidence): array {
|
|
$hosts = [];
|
|
foreach ($payload['hosts'] as $host) {
|
|
$id = (string) $host['host_id'];
|
|
if ($host['members'] === []) {
|
|
$hosts[$id] = [
|
|
'vpc_id' => $payload['vpc_id'], 'host_id' => $id,
|
|
'desired_revision' => $payload['desired_revision'], 'desired_digest' => $payload['desired_digest'],
|
|
'observed_revision' => $payload['desired_revision'], 'observed_digest' => $payload['desired_digest'],
|
|
'state' => 'active', 'transport_verified' => true,
|
|
'private_dataplane_verified' => true, 'removed' => true,
|
|
'bridge' => null, 'bridge_owner' => null, 'nft_table' => null,
|
|
'rollback' => 'none', 'readback' => 'confirmed',
|
|
'member_nics' => [], 'routes' => [], 'wg_allowed_ips' => [], 'proxy_ips' => [],
|
|
];
|
|
} else {
|
|
$item = $activeHostEvidence($payload['vpc_id'], (int) $id,
|
|
$payload['desired_revision'], $payload['desired_digest'], $payload['cidr'], $payload['members']);
|
|
if (CloudEngineClient::$power === 'shutoff') {
|
|
$item['state'] = 'degraded';
|
|
$item['private_dataplane_verified'] = false;
|
|
$item['last_error_code'] = 'vpc_private_guest_stopped';
|
|
foreach ($item['member_nics'] as &$nic) {
|
|
$nic['domain_running'] = false;
|
|
$nic['guest_network_verified'] = false;
|
|
$nic['guest_reason_code'] = 'vpc_private_guest_stopped';
|
|
}
|
|
unset($nic);
|
|
}
|
|
$hosts[$id] = $item;
|
|
}
|
|
}
|
|
return [
|
|
'vpc_id' => $payload['vpc_id'], 'operation_id' => $payload['operation_id'],
|
|
'desired_revision' => $payload['desired_revision'], 'desired_digest' => $payload['desired_digest'],
|
|
'observed' => ['evidence_version' => CloudVpcService::OBSERVATION_EVIDENCE_VERSION,
|
|
'revision' => $payload['desired_revision'], 'digest' => $payload['desired_digest'],
|
|
'state' => 'degraded', 'next_stage_allowed' => false,
|
|
'private_dataplane_verified' => false, 'fabric_gate' => ['active' => true], 'hosts' => $hosts],
|
|
];
|
|
};
|
|
$tick = static function (CloudVpcOperation $operation) use ($assert): void {
|
|
Capsule::table('cloud_vpc_operations')->where('id', $operation->id)
|
|
->update(['updated_at' => time() - 31, 'lease_until' => 0]);
|
|
$assert((new CloudVpcWorkerService())->tick(), 'Independent VPC worker did not select current workflow.');
|
|
};
|
|
$op = $make(1210);
|
|
$tick($op);
|
|
$assert($phase($op) === 'waiting_stopped' && CloudEngineClient::$applies === 0,
|
|
'VPC applied configuration before stopping the guest.');
|
|
CloudEngineClient::$power = 'shutoff';
|
|
$tick($op);
|
|
$assert($phase($op) === 'completed' && $op->fresh()->status === 'succeeded'
|
|
&& CloudEngineClient::$power === 'running',
|
|
'Service/worker did not apply and restore a running member.');
|
|
$root = CloudVpc::find($op->vpc_id);
|
|
$member = CloudVpcMember::where('instance_id', 121000)->first();
|
|
CloudVpcService::detachMember($root->vpc_key, $member->member_key, $actor, [
|
|
'request_key' => 'auto-detach-1210', 'auto_power' => true,
|
|
]);
|
|
$detach = CloudVpcOperation::where('vpc_id', $root->id)->orderByDesc('desired_revision')->first();
|
|
$tick($detach);
|
|
CloudEngineClient::$power = 'shutoff';
|
|
// Losing the start response leaves a durable paused workflow after exact
|
|
// last-member teardown released the membership/IP rows.
|
|
CloudEngineClient::$loseReceipt = true;
|
|
$tick($detach);
|
|
$assert($phase($detach) === 'paused'
|
|
&& CloudVpcMember::where('instance_id', 121000)->count() === 0,
|
|
'Last-member teardown did not preserve restoration workflow after removal.');
|
|
$other = $seedVpc(1211, '10.121.211.0/24', '10.121.211.0/24', 10);
|
|
$expectCloudError(static fn () => CloudVpcService::attachMember($other['vpc_key'], $actor, [
|
|
'instance_id' => 121000, 'request_key' => 'cross-attach-121000', 'auto_power' => true,
|
|
]), 409);
|
|
$count = count(CloudEngineClient::$commands);
|
|
CloudVpcService::reconcile($root->vpc_key, $actor, true, true);
|
|
$assert($phase($detach) === 'completed' && $detach->fresh()->status === 'succeeded'
|
|
&& count(CloudEngineClient::$commands) === $count,
|
|
'Last-member paused start could not recover its original receipt.');
|
|
|
|
// An installed pre-workflow schema continues to admit ordinary power, but
|
|
// cannot accept a new automatic maintenance request.
|
|
$instance = CloudInstance::find(121000);
|
|
$index = 'cloud_vpc_operations_power_instance_id_index';
|
|
$assert($schema->hasIndex('cloud_vpc_operations', ['power_instance_id']),
|
|
'Workflow lookup has no index to bound its locking read.');
|
|
$schema->table('cloud_vpc_operations', static function ($table) use ($index): void { $table->dropIndex($index); });
|
|
$expectCloudError(static fn () => Workflow::lockForInstance($instance), 503);
|
|
$schema->table('cloud_vpc_operations', static function ($table): void { $table->index('power_instance_id'); });
|
|
$detach->refresh();
|
|
$detach->power_instance_id = 999;
|
|
$expectCloudError(static fn () => Workflow::state($detach), 503);
|
|
$detach->refresh();
|
|
$schema->table('cloud_vpc_operations', static function ($table) use ($index): void {
|
|
$table->dropIndex($index); $table->dropColumn('power_instance_id');
|
|
});
|
|
$expectCloudError(static fn () => Workflow::lockForInstance($instance), 503);
|
|
$schema->table('cloud_vpc_operations', static function ($table): void { $table->dropColumn('power_workflow_json'); });
|
|
Workflow::lockForInstance($instance);
|
|
Workflow::manualIntentLocked($instance, 'action', 'poweroff');
|
|
$expectCloudError(static fn () => Workflow::initialize($detach, $instance, $member), 503);
|
|
|
|
echo "VPC automatic power workflow contracts passed.\n";
|
|
}
|