feat(vpc): automate member power transitions and seed private ACLs
parent
387237a748
commit
d8debc5f3e
@ -0,0 +1,59 @@
|
||||
<?php
|
||||
|
||||
namespace App\Services\Cloud;
|
||||
|
||||
use App\Models\CloudVpc;
|
||||
use App\Models\CloudVpcAclRule;
|
||||
|
||||
/** Editable initial policy; never an instruction to backfill existing VPCs. */
|
||||
final class CloudVpcDefaultAcl
|
||||
{
|
||||
public const PRIORITIES = ['ingress' => 32760, 'egress' => 32761];
|
||||
|
||||
/**
|
||||
* An empty VPC may reserve the exact initial policy without a host. This
|
||||
* does not attest that an ACL has been installed: its rows remain pending
|
||||
* until an exact host receipt. Older empty-policy reservations still pass.
|
||||
*/
|
||||
public static function matchesEmptyReservation(CloudVpc $vpc): bool
|
||||
{
|
||||
$rules = CloudVpcAclRule::where('vpc_id', $vpc->id)
|
||||
->whereNotIn('state', ['deleting', 'deleted'])->get();
|
||||
if ($rules->isEmpty()) {
|
||||
return true;
|
||||
}
|
||||
$cidr = (string) $vpc->cidr;
|
||||
if ($rules->count() !== 2 || !str_ends_with($cidr, '/24')) {
|
||||
return false;
|
||||
}
|
||||
$address = substr($cidr, 0, -3);
|
||||
$network = ip2long($address);
|
||||
if ($network === false || ($network & 255) !== 0) {
|
||||
return false;
|
||||
}
|
||||
$seen = [];
|
||||
foreach ($rules as $rule) {
|
||||
$direction = (string) $rule->direction;
|
||||
if (isset($seen[$direction])
|
||||
|| (self::PRIORITIES[$direction] ?? 0) !== (int) $rule->priority
|
||||
|| preg_match('/^[0-9a-f]{32}$/D', (string) $rule->rule_key) !== 1
|
||||
|| (string) $rule->action !== 'allow'
|
||||
|| (string) $rule->protocol !== 'all'
|
||||
|| $rule->port_from !== null || $rule->port_to !== null
|
||||
|| (int) $rule->desired_revision !== (int) $vpc->desired_revision
|
||||
|| !hash_equals((string) $vpc->desired_digest, (string) $rule->desired_digest)) {
|
||||
return false;
|
||||
}
|
||||
foreach (['source', 'destination'] as $side) {
|
||||
if ((string) $rule->{$side . '_cidr'} !== $cidr
|
||||
|| (int) $rule->{$side . '_int'} !== $network
|
||||
|| (int) $rule->{$side . '_broadcast_int'} !== ($network + 255)
|
||||
|| (int) $rule->{$side . '_prefix_length'} !== 24) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
$seen[$direction] = true;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,507 @@
|
||||
<?php
|
||||
|
||||
namespace App\Services\Cloud;
|
||||
|
||||
use App\Models\CloudInstance;
|
||||
use App\Models\CloudJob;
|
||||
use App\Models\CloudVpc;
|
||||
use App\Models\CloudVpcMember;
|
||||
use App\Models\CloudVpcOperation;
|
||||
use Illuminate\Database\QueryException;
|
||||
|
||||
/** Explicit member maintenance. Its power intent never enters the VPC digest. */
|
||||
final class CloudVpcPowerWorkflow
|
||||
{
|
||||
public const SHUTDOWN_SECONDS = 120;
|
||||
private static array $lockedRootIds = [];
|
||||
|
||||
public static function requested(array $input): bool
|
||||
{
|
||||
if (array_key_exists('auto_power', $input) && !is_bool($input['auto_power'])) {
|
||||
throw new CloudError('自动关机设置必须为布尔值', 422);
|
||||
}
|
||||
return ($input['auto_power'] ?? false) === true;
|
||||
}
|
||||
|
||||
public static function initialize(CloudVpcOperation $operation,
|
||||
CloudInstance $instance, CloudVpcMember $member): void
|
||||
{
|
||||
self::requireSchema();
|
||||
$operation->power_instance_id = (int) $instance->id;
|
||||
$operation->power_workflow_json = json_encode([
|
||||
'version' => 1, 'instance_id' => (int) $instance->id,
|
||||
'host_id' => (int) $instance->host_id, 'uuid' => (string) $instance->uuid,
|
||||
'member_key' => (string) $member->member_key,
|
||||
'phase' => 'preflight', 'original_running' => null,
|
||||
'resume_cancelled' => false, 'config_confirmed' => false,
|
||||
'baseline_job_id' => (int) CloudJob::where('instance_id', $instance->id)->max('id'),
|
||||
'shutdown_id' => substr(hash('sha256', $operation->operation_id . ':vpc:shutdown'), 0, 32),
|
||||
'start_id' => substr(hash('sha256', $operation->operation_id . ':vpc:start'), 0, 32),
|
||||
'error_code' => null,
|
||||
], JSON_THROW_ON_ERROR);
|
||||
$operation->save();
|
||||
}
|
||||
|
||||
public static function state(CloudVpcOperation $operation): ?array
|
||||
{
|
||||
if ($operation->power_workflow_json === null) {
|
||||
return null;
|
||||
}
|
||||
$state = json_decode((string) $operation->power_workflow_json, true);
|
||||
if (!is_array($state) || ($state['version'] ?? null) !== 1
|
||||
|| !is_int($state['instance_id'] ?? null)
|
||||
|| $state['instance_id'] < 1
|
||||
|| (int) $operation->power_instance_id !== $state['instance_id']
|
||||
|| !in_array((string) $operation->kind, ['member_attach', 'member_detach'], true)
|
||||
|| !is_int($state['host_id'] ?? null) || $state['host_id'] < 1
|
||||
|| !is_string($state['uuid'] ?? null)
|
||||
|| preg_match('/^[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}$/D', $state['uuid']) !== 1
|
||||
|| !is_string($state['member_key'] ?? null)
|
||||
|| preg_match('/^[a-f0-9]{32}$/D', $state['member_key']) !== 1
|
||||
|| !is_int($state['baseline_job_id'] ?? null) || $state['baseline_job_id'] < 0
|
||||
|| !is_bool($state['resume_cancelled'] ?? null)
|
||||
|| !is_bool($state['config_confirmed'] ?? null)
|
||||
|| !array_key_exists('original_running', $state)
|
||||
|| ($state['original_running'] !== null && !is_bool($state['original_running']))
|
||||
|| ($state['shutdown_id'] ?? null) !== substr(hash('sha256', $operation->operation_id . ':vpc:shutdown'), 0, 32)
|
||||
|| ($state['start_id'] ?? null) !== substr(hash('sha256', $operation->operation_id . ':vpc:start'), 0, 32)
|
||||
|| (isset($state['shutdown_deadline']) && (!is_int($state['shutdown_deadline']) || $state['shutdown_deadline'] < 1))
|
||||
|| (isset($state['start_deadline']) && (!is_int($state['start_deadline']) || $state['start_deadline'] < 1))
|
||||
|| (($state['phase'] ?? null) === 'paused' && !in_array($state['paused_from'] ?? null,
|
||||
['preflight', 'shutting_down', 'waiting_stopped', 'applying', 'configured', 'starting'], true))
|
||||
|| !in_array($state['phase'] ?? null, ['preflight', 'shutting_down',
|
||||
'waiting_stopped', 'applying', 'configured', 'starting', 'paused', 'completed'], true)) {
|
||||
throw new CloudError('VPC 自动电源任务记录无效', 503, 'vpc_power_record_invalid');
|
||||
}
|
||||
return $state;
|
||||
}
|
||||
|
||||
public static function publicState(CloudVpcOperation $operation): ?array
|
||||
{
|
||||
$state = self::state($operation);
|
||||
return $state === null ? null : array_intersect_key($state, array_flip([
|
||||
'instance_id', 'phase', 'original_running', 'resume_cancelled', 'error_code',
|
||||
]));
|
||||
}
|
||||
|
||||
public static function unfinished(CloudVpcOperation $operation): bool
|
||||
{
|
||||
$state = self::state($operation);
|
||||
return $state !== null && $state['phase'] !== 'completed';
|
||||
}
|
||||
|
||||
private static function requireSchema(): void
|
||||
{
|
||||
$schema = (new CloudVpcOperation())->getConnection()->getSchemaBuilder();
|
||||
if (!$schema->hasColumn('cloud_vpc_operations', 'power_workflow_json')
|
||||
|| !$schema->hasColumn('cloud_vpc_operations', 'power_instance_id')
|
||||
|| !$schema->hasIndex('cloud_vpc_operations', ['power_instance_id'])) {
|
||||
throw new CloudError('VPC 数据库需要升级,请先执行数据库更新', 503,
|
||||
'vpc_power_schema_required');
|
||||
}
|
||||
}
|
||||
|
||||
private static function instanceOperations(int $instanceId)
|
||||
{
|
||||
self::requireSchema();
|
||||
// The indexed instance predicate bounds InnoDB's locking read to this
|
||||
// VM. JSON is verified only after loading; filtering JSON would scan
|
||||
// and potentially lock unrelated operations under REPEATABLE READ.
|
||||
return CloudVpcOperation::select('cloud_vpc_operations.*')
|
||||
->join('cloud_vpcs as workflow_root', static function ($join): void {
|
||||
$join->on('workflow_root.id', '=', 'cloud_vpc_operations.vpc_id')
|
||||
->on('workflow_root.last_operation_id', '=', 'cloud_vpc_operations.operation_id');
|
||||
})
|
||||
->where('cloud_vpc_operations.power_instance_id', $instanceId)
|
||||
->whereNotNull('power_workflow_json');
|
||||
}
|
||||
|
||||
/** Call before locking the instance row in every manual job transaction. */
|
||||
public static function lockForInstance(CloudInstance $instance): void
|
||||
{
|
||||
if (!$instance->getConnection()->getSchemaBuilder()->hasColumn('cloud_vpc_operations', 'power_workflow_json')) {
|
||||
return; // Existing non-VPC installations have no workflow to fence.
|
||||
}
|
||||
$ids = self::instanceOperations((int) $instance->id)->orderBy('vpc_id')
|
||||
->pluck('vpc_id')->all();
|
||||
if ($ids !== []) {
|
||||
CloudVpc::whereIn('id', $ids)->orderBy('id')->lockForUpdate()->get();
|
||||
}
|
||||
self::$lockedRootIds[(int) $instance->id] = array_map('intval', $ids);
|
||||
}
|
||||
|
||||
/** Root and instance locks have already been taken by lockForInstance. */
|
||||
public static function manualIntentLocked(CloudInstance $instance,
|
||||
string $kind, ?string $action): void
|
||||
{
|
||||
if (!$instance->getConnection()->getSchemaBuilder()->hasColumn('cloud_vpc_operations', 'power_workflow_json')) {
|
||||
return;
|
||||
}
|
||||
$candidates = self::currentInstanceOperations($instance);
|
||||
foreach ($candidates as $candidate) {
|
||||
if (!in_array((int) $candidate->vpc_id, self::$lockedRootIds[(int) $instance->id] ?? [], true)) {
|
||||
throw new CloudError('实例刚开始 VPC 配置,请刷新后重试', 409,
|
||||
'vpc_power_workflow_busy');
|
||||
}
|
||||
}
|
||||
// A new attachment must acquire this same instance lock; after the
|
||||
// comparison no new current workflow can appear before commit.
|
||||
foreach ($candidates as $operation) {
|
||||
$state = self::state($operation);
|
||||
if ($state === null || $state['phase'] === 'completed') {
|
||||
continue;
|
||||
}
|
||||
if ($kind !== 'action' || !CloudOperationCatalog::isPowerAction((string) $action)
|
||||
|| (in_array($state['phase'], ['shutting_down', 'applying', 'starting'], true)
|
||||
&& (int) $operation->lease_until > time())) {
|
||||
throw new CloudError('VPC 正在配置该实例,请等待当前步骤完成', 409,
|
||||
'vpc_power_workflow_busy');
|
||||
}
|
||||
// No later worker may undo this newer human intent by starting
|
||||
// the guest. Keep the workflow retryable after a manual force-off.
|
||||
$state['resume_cancelled'] = true;
|
||||
if ($state['phase'] === 'preflight') {
|
||||
$state['phase'] = 'paused';
|
||||
$state['paused_from'] = 'preflight';
|
||||
$state['error_code'] = 'vpc_power_manual_override';
|
||||
$operation->status = 'paused';
|
||||
}
|
||||
self::saveState($operation, $state);
|
||||
}
|
||||
}
|
||||
|
||||
/** A current read that cannot deadlock with root -> operation -> instance. */
|
||||
private static function currentInstanceOperations(CloudInstance $instance)
|
||||
{
|
||||
$lock = match ($instance->getConnection()->getDriverName()) {
|
||||
'mysql', 'mariadb' => 'FOR UPDATE NOWAIT',
|
||||
'sqlite' => true,
|
||||
default => throw new CloudError('无法核对 VPC 自动电源任务', 503, 'vpc_power_state_unavailable'),
|
||||
};
|
||||
try {
|
||||
return self::instanceOperations((int) $instance->id)->lock($lock)->get();
|
||||
} catch (QueryException $error) {
|
||||
$driverCode = (int) ($error->errorInfo[1] ?? 0);
|
||||
if (in_array($driverCode, [1205, 1213, 3572], true)) {
|
||||
throw new CloudError('VPC 正在处理实例,请稍后重试', 409, 'vpc_power_workflow_busy');
|
||||
}
|
||||
throw new CloudError('VPC 自动电源任务暂不可核对', 503, 'vpc_power_state_unavailable');
|
||||
}
|
||||
}
|
||||
|
||||
/** Also covers a detached member whose NIC/IP ledger is already removed. */
|
||||
public static function assertAttachAllowedLocked(CloudInstance $instance): void
|
||||
{
|
||||
if (!$instance->getConnection()->getSchemaBuilder()->hasColumn('cloud_vpc_operations', 'power_workflow_json')) {
|
||||
return;
|
||||
}
|
||||
foreach (self::currentInstanceOperations($instance) as $operation) {
|
||||
if (self::unfinished($operation)) {
|
||||
throw new CloudError('该实例仍有 VPC 自动配置任务,请先核对完成', 409,
|
||||
'vpc_power_workflow_busy');
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public static function assertMutationAllowed(CloudVpc $vpc): void
|
||||
{
|
||||
if (!$vpc->last_operation_id) {
|
||||
return;
|
||||
}
|
||||
$operation = CloudVpcOperation::where('operation_id', $vpc->last_operation_id)->first();
|
||||
if ($operation !== null && self::unfinished($operation)) {
|
||||
throw new CloudError('请先完成或核对当前 VPC 实例自动配置任务', 409,
|
||||
'vpc_power_workflow_busy');
|
||||
}
|
||||
}
|
||||
|
||||
/** Release the PHP worker lease between polls; never wait inside a VM lock. */
|
||||
private static function release(CloudVpcOperation $operation, array $state): void
|
||||
{
|
||||
$done = $state['phase'] === 'completed';
|
||||
$operation->status = $done ? 'succeeded'
|
||||
: ($state['phase'] === 'paused' ? 'paused' : 'waiting_observed');
|
||||
$operation->error_code = $state['error_code'] ?? null;
|
||||
$operation->finished_at = $done ? time() : null;
|
||||
$operation->lease_token = null;
|
||||
$operation->lease_until = 0;
|
||||
self::saveState($operation, $state);
|
||||
}
|
||||
|
||||
private static function saveState(CloudVpcOperation $operation, array $state): void
|
||||
{
|
||||
$operation->power_workflow_json = json_encode($state, JSON_THROW_ON_ERROR);
|
||||
$operation->updated_at = time();
|
||||
$operation->save();
|
||||
}
|
||||
|
||||
/** The root prevents supersession; the instance serializes manual intent. */
|
||||
private static function locked(CloudVpcOperation $claimed, string $lease, callable $callback)
|
||||
{
|
||||
return $claimed->getConnection()->transaction(static function () use ($claimed, $lease, $callback) {
|
||||
$root = CloudVpc::where('id', $claimed->vpc_id)->lockForUpdate()->first();
|
||||
$operation = CloudVpcOperation::where('id', $claimed->id)->lockForUpdate()->first();
|
||||
if ($root === null || $operation === null
|
||||
|| (string) $root->last_operation_id !== (string) $claimed->operation_id
|
||||
|| (int) $root->desired_revision !== (int) $claimed->desired_revision
|
||||
|| (string) $operation->lease_token !== $lease) {
|
||||
throw new CloudError('VPC 自动配置任务已变化,请重新核对', 409, 'vpc_power_stale');
|
||||
}
|
||||
$state = self::state($operation);
|
||||
if ($state === null) {
|
||||
return true;
|
||||
}
|
||||
$instance = CloudInstance::where('id', $state['instance_id'])->lockForUpdate()->first();
|
||||
if ($instance === null || (string) $instance->uuid !== $state['uuid']
|
||||
|| (int) $instance->host_id !== $state['host_id']) {
|
||||
throw new CloudError('VPC 实例身份已变化', 409, 'vpc_power_target_changed');
|
||||
}
|
||||
if (CloudJob::where('instance_id', $instance->id)
|
||||
->where('id', '>', $state['baseline_job_id'])->exists()) {
|
||||
$state['resume_cancelled'] = true;
|
||||
}
|
||||
return $callback($root, $operation, $instance, $state);
|
||||
}); // Never replay an HTTP side effect by retrying a DB transaction.
|
||||
}
|
||||
|
||||
private static function powerState(CloudEngineClient $client, array $actor, array $state): string
|
||||
{
|
||||
$result = $client->host($state['host_id'], 'GET', 'vms/' . $state['uuid'] . '/state', $actor);
|
||||
if (($result['uuid'] ?? null) !== $state['uuid']
|
||||
|| !in_array($result['state'] ?? null, ['running', 'shutoff', 'shutdown'], true)) {
|
||||
throw new CloudError('无法安全核对实例电源状态', 409, 'vpc_power_state_unavailable');
|
||||
}
|
||||
return $result['state'];
|
||||
}
|
||||
|
||||
private static function pause(CloudVpc $root, CloudVpcOperation $operation, array $state,
|
||||
string $code): void
|
||||
{
|
||||
$state['paused_from'] = $state['phase'];
|
||||
$state['phase'] = 'paused';
|
||||
$state['error_code'] = $code;
|
||||
self::release($operation, $state);
|
||||
$root->state = 'degraded';
|
||||
$root->sync_state = 'waiting';
|
||||
$root->last_error_code = $code;
|
||||
$root->save();
|
||||
}
|
||||
|
||||
/** Return true only when a stopped guest may enter the existing apply lane. */
|
||||
public static function beforeApply(CloudVpcOperation $operation, string $lease,
|
||||
array $actor, bool $explicitRetry, ?CloudEngineClient $client = null): bool
|
||||
{
|
||||
if (!self::unfinished($operation)) {
|
||||
return true;
|
||||
}
|
||||
$client ??= new CloudEngineClient();
|
||||
$phase = self::locked($operation, $lease, static function ($root, $row, $instance, $state)
|
||||
use ($client, $actor, $explicitRetry): string {
|
||||
if ($state['phase'] === 'paused') {
|
||||
if (!$explicitRetry) {
|
||||
self::release($row, $state);
|
||||
return 'paused';
|
||||
}
|
||||
$state['phase'] = $state['paused_from'] ?? 'waiting_stopped';
|
||||
$state['error_code'] = null;
|
||||
$state['shutdown_deadline'] = time() + self::SHUTDOWN_SECONDS;
|
||||
if ($state['phase'] === 'starting') {
|
||||
$state['start_deadline'] = time() + self::SHUTDOWN_SECONDS;
|
||||
}
|
||||
}
|
||||
if ((string) $instance->status !== 'active' || !$instance->serviceAvailable()
|
||||
|| CloudJob::where('instance_id', $instance->id)
|
||||
->whereIn('status', ['queued', 'running', 'uncertain'])->exists()) {
|
||||
self::pause($root, $row, $state, 'vpc_power_instance_busy');
|
||||
return 'paused';
|
||||
}
|
||||
if ($state['phase'] === 'preflight') {
|
||||
$power = self::powerState($client, $actor, $state);
|
||||
$state['original_running'] = $power === 'running';
|
||||
$state['phase'] = $power === 'shutoff' ? 'applying'
|
||||
: ($power === 'shutdown' || $state['resume_cancelled'] === true
|
||||
? 'waiting_stopped' : 'shutting_down');
|
||||
$state['shutdown_deadline'] = time() + self::SHUTDOWN_SECONDS;
|
||||
}
|
||||
if ($state['phase'] === 'waiting_stopped') {
|
||||
if (self::powerState($client, $actor, $state) === 'shutoff') {
|
||||
$state['phase'] = 'applying';
|
||||
} elseif (time() >= (int) $state['shutdown_deadline']) {
|
||||
self::pause($root, $row, $state, 'vpc_power_shutdown_timeout');
|
||||
return 'paused';
|
||||
} else {
|
||||
self::release($row, $state);
|
||||
return 'waiting_stopped';
|
||||
}
|
||||
}
|
||||
if ($state['phase'] === 'applying'
|
||||
&& self::powerState($client, $actor, $state) !== 'shutoff') {
|
||||
self::pause($root, $row, $state, 'vpc_power_manual_override');
|
||||
return 'paused';
|
||||
}
|
||||
self::saveState($row, $state);
|
||||
return $state['phase'];
|
||||
});
|
||||
if ($phase === 'shutting_down') {
|
||||
self::command($operation, $lease, $actor, 'shutdown', $client);
|
||||
return false;
|
||||
}
|
||||
if (in_array($phase, ['configured', 'starting'], true)) {
|
||||
self::afterApply($operation, $lease, $actor, $client);
|
||||
return false;
|
||||
}
|
||||
return $phase === 'applying' || $phase === 'completed';
|
||||
}
|
||||
|
||||
/** Called under recordObserved's root/operation locks after strict proof. */
|
||||
public static function configurationObserved(CloudVpcOperation $operation, bool $confirmed): bool
|
||||
{
|
||||
$state = self::state($operation);
|
||||
if ($state === null || $state['phase'] === 'completed') {
|
||||
return false;
|
||||
}
|
||||
if ($confirmed && $state['phase'] === 'applying') {
|
||||
$state['config_confirmed'] = true;
|
||||
$state['phase'] = 'configured';
|
||||
}
|
||||
self::saveState($operation, $state);
|
||||
return true;
|
||||
}
|
||||
|
||||
/** Preflight failures have no remote mutation uncertainty of their own. */
|
||||
public static function handleFailure(CloudVpcOperation $operation, string $lease,
|
||||
CloudError $error): bool
|
||||
{
|
||||
if (!self::unfinished($operation)) {
|
||||
return false;
|
||||
}
|
||||
self::locked($operation, $lease, static function ($root, $row, $instance, $state)
|
||||
use ($error): void {
|
||||
self::pause($root, $row, $state,
|
||||
in_array($state['phase'], ['preflight', 'waiting_stopped'], true)
|
||||
? 'vpc_power_state_unavailable'
|
||||
: (in_array($error->errorCode, ['vpc_power_state_unavailable', 'vpc_power_instance_busy'], true)
|
||||
? $error->errorCode : 'vpc_power_result_uncertain'));
|
||||
});
|
||||
return true;
|
||||
}
|
||||
|
||||
public static function afterApply(CloudVpcOperation $operation, string $lease,
|
||||
array $actor, ?CloudEngineClient $client = null): void
|
||||
{
|
||||
if (!self::unfinished($operation)) {
|
||||
return;
|
||||
}
|
||||
$client ??= new CloudEngineClient();
|
||||
$start = self::locked($operation, $lease, static function ($root, $row, $instance, $state): bool {
|
||||
if (!($state['config_confirmed'] ?? false)) {
|
||||
self::pause($root, $row, $state, 'vpc_power_configuration_unconfirmed');
|
||||
return false;
|
||||
}
|
||||
if ($state['original_running'] !== true || $state['resume_cancelled'] === true
|
||||
|| (string) $instance->status !== 'active' || !$instance->serviceAvailable()
|
||||
|| in_array((string) $instance->network_status,
|
||||
['blocking', 'blocking_expired', 'blocked_expired', 'restore_pending', 'restoring'], true)) {
|
||||
$state['phase'] = 'completed';
|
||||
self::release($row, $state);
|
||||
return false;
|
||||
}
|
||||
if ($state['phase'] !== 'starting') {
|
||||
$state['start_deadline'] = time() + self::SHUTDOWN_SECONDS;
|
||||
$state['phase'] = 'starting';
|
||||
}
|
||||
self::saveState($row, $state);
|
||||
return true;
|
||||
});
|
||||
if ($start) {
|
||||
self::command($operation, $lease, $actor, 'start', $client);
|
||||
}
|
||||
}
|
||||
|
||||
/** The fixed child ID and committed phase precede every network dispatch. */
|
||||
private static function command(CloudVpcOperation $operation, string $lease,
|
||||
array $actor, string $action, CloudEngineClient $client): void
|
||||
{
|
||||
self::locked($operation, $lease, static function ($root, $row, $instance, $state)
|
||||
use ($actor, $action, $client): void {
|
||||
$id = $state[$action === 'start' ? 'start_id' : 'shutdown_id'];
|
||||
$commandActor = array_replace($actor, ['operation_id' => $id]);
|
||||
try {
|
||||
try {
|
||||
$result = $client->request('GET', '/internal/v1/operations/' . $id,
|
||||
$commandActor, null, $id);
|
||||
} catch (CloudError $error) {
|
||||
if ($error->httpStatus !== 404) {
|
||||
throw $error;
|
||||
}
|
||||
if ($action === 'start' && $state['resume_cancelled'] === true) {
|
||||
$state['phase'] = 'completed';
|
||||
self::release($row, $state);
|
||||
return;
|
||||
}
|
||||
if ($action === 'shutdown' && $state['resume_cancelled'] === true) {
|
||||
// The old worker may have died after persisting intent
|
||||
// but before sending it. A newer manual start wins.
|
||||
$state['phase'] = self::powerState($client, $actor, $state) === 'shutoff'
|
||||
? 'applying' : 'waiting_stopped';
|
||||
self::release($row, $state);
|
||||
return;
|
||||
}
|
||||
if ((string) $instance->status !== 'active' || !$instance->serviceAvailable()
|
||||
|| CloudJob::where('instance_id', $instance->id)
|
||||
->whereIn('status', ['queued', 'running', 'uncertain'])->exists()) {
|
||||
self::pause($root, $row, $state, 'vpc_power_instance_busy');
|
||||
return;
|
||||
}
|
||||
$result = $client->host($state['host_id'], 'POST',
|
||||
'vms/' . $state['uuid'] . '/actions', $commandActor,
|
||||
['action' => $action], $id);
|
||||
}
|
||||
$receipt = $result['result'] ?? null;
|
||||
if (($result['operation_id'] ?? null) === $id
|
||||
&& in_array($result['status'] ?? null, ['queued', 'running', 'pending'], true)) {
|
||||
if (time() >= (int) ($state[$action === 'start' ? 'start_deadline' : 'shutdown_deadline'] ?? 0)) {
|
||||
self::pause($root, $row, $state, 'vpc_power_result_uncertain');
|
||||
} else {
|
||||
self::release($row, $state);
|
||||
}
|
||||
return;
|
||||
}
|
||||
if (($result['operation_id'] ?? null) === $id
|
||||
&& ($result['status'] ?? null) === 'failed'
|
||||
&& (($result['execution_outcome'] ?? null) === 'not_dispatched'
|
||||
|| ($result['mutation_started'] ?? null) === false)) {
|
||||
if ($action === 'shutdown'
|
||||
&& self::powerState($client, $actor, $state) === 'shutoff') {
|
||||
$state['phase'] = 'applying';
|
||||
$state['error_code'] = null;
|
||||
self::release($row, $state);
|
||||
return;
|
||||
}
|
||||
if ($action === 'start' && $state['resume_cancelled'] === true) {
|
||||
$state['phase'] = 'completed';
|
||||
$state['error_code'] = null;
|
||||
self::release($row, $state);
|
||||
return;
|
||||
}
|
||||
self::pause($root, $row, $state, 'vpc_power_command_rejected');
|
||||
return;
|
||||
}
|
||||
if (($result['operation_id'] ?? null) !== $id
|
||||
|| ($result['status'] ?? null) !== 'succeeded'
|
||||
|| !is_array($receipt) || ($receipt['command_accepted'] ?? null) !== true
|
||||
|| ($receipt['uuid'] ?? null) !== $state['uuid']
|
||||
|| ($receipt['action'] ?? null) !== $action) {
|
||||
self::pause($root, $row, $state, 'vpc_power_result_uncertain');
|
||||
return;
|
||||
}
|
||||
$state[$action . '_accepted'] = true;
|
||||
$state['phase'] = $action === 'start' ? 'completed' : 'waiting_stopped';
|
||||
$state['error_code'] = null;
|
||||
self::release($row, $state);
|
||||
} catch (CloudError $error) {
|
||||
self::pause($root, $row, $state, 'vpc_power_result_uncertain');
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,385 @@
|
||||
<?php
|
||||
|
||||
namespace App\Services\Cloud {
|
||||
final class CloudEngineClient
|
||||
{
|
||||
public static string $power = 'running';
|
||||
public static array $commands = [];
|
||||
public static array $operations = [];
|
||||
public static bool $loseReceipt = false;
|
||||
public static bool $failState = false;
|
||||
public static $vpcRespond;
|
||||
public static int $applies = 0;
|
||||
public function __construct(...$args) {}
|
||||
public function host(int $host, string $method, string $path, array $actor,
|
||||
?array $body = null, ?string $operationId = null): array
|
||||
{
|
||||
$uuid = explode('/', $path)[1];
|
||||
if ($method === 'GET') {
|
||||
if (self::$failState) {
|
||||
throw new CloudError('offline', 502);
|
||||
}
|
||||
return ['uuid' => $uuid, 'state' => self::$power];
|
||||
}
|
||||
self::$commands[] = [$operationId, $body['action'], $uuid];
|
||||
$result = ['operation_id' => $operationId, 'status' => 'succeeded',
|
||||
'result' => ['uuid' => $uuid, 'action' => $body['action'], 'command_accepted' => true]];
|
||||
self::$operations[$operationId] = $result;
|
||||
if ($body['action'] === 'start') { self::$power = 'running'; }
|
||||
if (self::$loseReceipt) {
|
||||
self::$loseReceipt = false;
|
||||
throw new CloudError('lost response', 502);
|
||||
}
|
||||
return $result;
|
||||
}
|
||||
public function request(string $method, string $path, array $actor,
|
||||
?array $body = null, ?string $operationId = null): array
|
||||
{
|
||||
return self::$operations[basename($path)]
|
||||
?? throw new CloudError('missing', 404);
|
||||
}
|
||||
public function vpc(string $method, string $path, array $actor,
|
||||
?array $body, ?string $operationId): array
|
||||
{
|
||||
self::$applies++;
|
||||
return (self::$vpcRespond)($body);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
namespace {
|
||||
use App\Models\CloudInstance;
|
||||
use App\Models\CloudVpc;
|
||||
use App\Models\CloudVpcMember;
|
||||
use App\Models\CloudVpcOperation;
|
||||
use App\Services\Cloud\CloudEngineClient;
|
||||
use App\Services\Cloud\CloudError;
|
||||
use App\Services\Cloud\CloudVpcPowerWorkflow as Workflow;
|
||||
use App\Services\Cloud\CloudVpcService;
|
||||
use App\Services\Cloud\CloudVpcWorkerService;
|
||||
use Illuminate\Database\Capsule\Manager as Capsule;
|
||||
|
||||
require __DIR__ . '/CloudVpcMemberReuseTest.php';
|
||||
if (!isset($capsule)) { return; }
|
||||
|
||||
$make = static function (int $id, string $power = 'running') use ($seedVpc, $seedInstance, $actor): CloudVpcOperation {
|
||||
$vpc = $seedVpc($id, '10.121.' . ($id % 250) . '.0/24',
|
||||
'10.121.' . ($id % 250) . '.0/24', 10);
|
||||
$seedInstance($id * 100, 10);
|
||||
CloudEngineClient::$power = $power;
|
||||
CloudVpcService::attachMember($vpc['vpc_key'], $actor, [
|
||||
'instance_id' => $id * 100, 'request_key' => 'auto-member-' . $id,
|
||||
'auto_power' => true,
|
||||
]);
|
||||
return CloudVpcOperation::where('vpc_id', $id)->orderByDesc('desired_revision')->first();
|
||||
};
|
||||
$claim = static function (CloudVpcOperation $operation): array {
|
||||
$operation = $operation->fresh();
|
||||
$operation->lease_token = bin2hex(random_bytes(16));
|
||||
$operation->lease_until = time() + 120;
|
||||
$operation->status = 'running';
|
||||
$operation->save();
|
||||
return [$operation, $operation->lease_token];
|
||||
};
|
||||
$observed = static function (CloudVpcOperation $operation): void {
|
||||
Workflow::configurationObserved($operation->fresh(), true);
|
||||
};
|
||||
$phase = static fn (CloudVpcOperation $operation): string => Workflow::state($operation->fresh())['phase'];
|
||||
|
||||
// Normal running guest: command acknowledgement, separate state observation,
|
||||
// exact configuration gate, then one automatic restart.
|
||||
$op = $make(1201);
|
||||
[$op, $lease] = $claim($op);
|
||||
$assert(Workflow::beforeApply($op, $lease, $actor, false) === false
|
||||
&& $phase($op) === 'waiting_stopped', 'Shutdown must finish only the command step.');
|
||||
$assert($op->fresh()->lease_token === null, 'Waiting for guest shutdown held its lease.');
|
||||
$count = count(CloudEngineClient::$commands);
|
||||
[$op, $lease] = $claim($op);
|
||||
$assert(!Workflow::beforeApply($op, $lease, $actor, false)
|
||||
&& count(CloudEngineClient::$commands) === $count, 'Polling resent shutdown.');
|
||||
CloudEngineClient::$power = 'shutoff';
|
||||
[$op, $lease] = $claim($op);
|
||||
$assert(Workflow::beforeApply($op, $lease, $actor, false), 'Stopped guest did not enter apply.');
|
||||
Workflow::afterApply($op, $lease, $actor);
|
||||
$assert(count(CloudEngineClient::$commands) === $count, 'Guest started before exact configuration proof.');
|
||||
[$op, $lease] = $claim($op);
|
||||
$assert(Workflow::beforeApply($op, $lease, $actor, true), 'Explicit retry did not resume configuration.');
|
||||
$observed($op);
|
||||
Workflow::afterApply($op, $lease, $actor);
|
||||
$assert($phase($op) === 'completed' && count(CloudEngineClient::$commands) === $count + 1
|
||||
&& end(CloudEngineClient::$commands)[1] === 'start', 'Originally running guest was not restored.');
|
||||
|
||||
// Immutable API replay includes the explicit maintenance acknowledgement.
|
||||
$vpc = CloudVpc::find($op->vpc_id);
|
||||
$expectCloudError(static fn () => CloudVpcService::attachMember($vpc->vpc_key, $actor, [
|
||||
'instance_id' => 120100, 'request_key' => 'auto-member-1201', 'auto_power' => false,
|
||||
]), 409);
|
||||
CloudVpcService::attachMember($vpc->vpc_key, $actor, [
|
||||
'instance_id' => 120100, 'request_key' => 'auto-member-1201', 'auto_power' => true,
|
||||
]);
|
||||
$expectCloudError(static fn () => CloudVpcService::attachMember($vpc->vpc_key, $actor, [
|
||||
'instance_id' => 120100, 'request_key' => 'auto-member-1201', 'auto_power' => true,
|
||||
'subnet_key' => str_repeat('a', 32),
|
||||
]), 422);
|
||||
foreach (['true', 1, null] as $badFlag) {
|
||||
$expectCloudError(static fn () => Workflow::requested(['auto_power' => $badFlag]), 422);
|
||||
}
|
||||
|
||||
// A previously stopped or already shutting-down guest must never be restarted.
|
||||
foreach (['shutoff', 'shutdown'] as $offset => $initial) {
|
||||
$op = $make(1202 + $offset, $initial);
|
||||
$count = count(CloudEngineClient::$commands);
|
||||
[$op, $lease] = $claim($op);
|
||||
$ready = Workflow::beforeApply($op, $lease, $actor, false);
|
||||
if (!$ready) {
|
||||
CloudEngineClient::$power = 'shutoff';
|
||||
[$op, $lease] = $claim($op);
|
||||
$assert(Workflow::beforeApply($op, $lease, $actor, false), 'Existing shutdown did not converge.');
|
||||
}
|
||||
$observed($op);
|
||||
Workflow::afterApply($op, $lease, $actor);
|
||||
$assert($phase($op) === 'completed' && count(CloudEngineClient::$commands) === $count,
|
||||
'A pre-existing stopped/shutdown state generated power commands.');
|
||||
}
|
||||
|
||||
// Lost receipt: repeat uses the same durable engine operation, no second POST.
|
||||
$op = $make(1204);
|
||||
CloudEngineClient::$loseReceipt = true;
|
||||
[$op, $lease] = $claim($op);
|
||||
Workflow::beforeApply($op, $lease, $actor, false);
|
||||
$assert($phase($op) === 'paused', 'Lost receipt did not pause.');
|
||||
$count = count(CloudEngineClient::$commands);
|
||||
[$op, $lease] = $claim($op);
|
||||
Workflow::beforeApply($op, $lease, $actor, true);
|
||||
$assert($phase($op) === 'waiting_stopped' && count(CloudEngineClient::$commands) === $count,
|
||||
'Lost accepted command was submitted again.');
|
||||
|
||||
// Graceful timeout pauses, permits force-off, and cancels automatic restoration.
|
||||
$state = Workflow::state($op->fresh());
|
||||
$state['shutdown_deadline'] = time() - 1;
|
||||
$op->power_workflow_json = json_encode($state); $op->save();
|
||||
[$op, $lease] = $claim($op);
|
||||
Workflow::beforeApply($op, $lease, $actor, false);
|
||||
$assert($phase($op) === 'paused' && $op->fresh()->status === 'paused', 'Timeout became an uncertain mutation.');
|
||||
$instance = CloudInstance::find(120400);
|
||||
$instance->getConnection()->transaction(static function () use ($instance): void {
|
||||
Workflow::lockForInstance($instance);
|
||||
$locked = CloudInstance::where('id', $instance->id)->lockForUpdate()->first();
|
||||
Workflow::manualIntentLocked($locked, 'action', 'poweroff');
|
||||
});
|
||||
CloudEngineClient::$power = 'shutoff';
|
||||
[$op, $lease] = $claim($op);
|
||||
$assert(Workflow::beforeApply($op, $lease, $actor, true), 'Manual force-off cannot resume configuration.');
|
||||
$observed($op); Workflow::afterApply($op, $lease, $actor);
|
||||
$assert($phase($op) === 'completed' && count(CloudEngineClient::$commands) === $count,
|
||||
'Automation undid newer manual power intent.');
|
||||
|
||||
// Only the active revision can continue; unfinished workflows forbid replacing
|
||||
// their VPC plan, including paused steps and last-member removal.
|
||||
$op = $make(1205, 'shutoff');
|
||||
$vpc = CloudVpc::find($op->vpc_id);
|
||||
$expectCloudError(static fn () => Workflow::assertMutationAllowed($vpc), 409);
|
||||
[$op, $lease] = $claim($op);
|
||||
Capsule::table('cloud_vpcs')->where('id', $vpc->id)->update(['desired_revision' => 999]);
|
||||
$count = count(CloudEngineClient::$commands);
|
||||
$expectCloudError(static fn () => Workflow::beforeApply($op, $lease, $actor, false), 409);
|
||||
$assert(count(CloudEngineClient::$commands) === $count, 'Stale revision sent a power command.');
|
||||
|
||||
// Read-only failures pause without a child action or a lingering task lease.
|
||||
$op = $make(1206);
|
||||
CloudEngineClient::$failState = true;
|
||||
[$op, $lease] = $claim($op);
|
||||
try { Workflow::beforeApply($op, $lease, $actor, false); }
|
||||
catch (CloudError $error) { Workflow::handleFailure($op, $lease, $error); }
|
||||
CloudEngineClient::$failState = false;
|
||||
$assert($op->fresh()->status === 'paused' && $op->fresh()->lease_token === null,
|
||||
'Read-only preflight failure retained a lease.');
|
||||
|
||||
// Crash during apply has a bounded lease, so later manual power stays possible.
|
||||
$op = $make(1207, 'shutoff');
|
||||
[$op, $lease] = $claim($op); Workflow::beforeApply($op, $lease, $actor, false);
|
||||
$op = $op->fresh(); $op->lease_until = time() - 1; $op->save();
|
||||
$instance = CloudInstance::find(120700);
|
||||
$instance->getConnection()->transaction(static function () use ($instance): void {
|
||||
Workflow::lockForInstance($instance);
|
||||
Workflow::manualIntentLocked($instance, 'action', 'poweroff');
|
||||
});
|
||||
$assert(Workflow::state($op->fresh())['resume_cancelled'] === true,
|
||||
'Expired apply worker permanently blocked ordinary power.');
|
||||
|
||||
// Corrupt child identity cannot be used to query or send an unrelated command.
|
||||
$op = $make(1208);
|
||||
$state = Workflow::state($op); $state['shutdown_id'] = str_repeat('f', 32);
|
||||
$op->power_workflow_json = json_encode($state); $op->save();
|
||||
$expectCloudError(static fn () => Workflow::state($op), 503);
|
||||
|
||||
// A crash before the first shutdown dispatch cannot overrule a newer start.
|
||||
$op = $make(1209);
|
||||
$state = Workflow::state($op);
|
||||
$state['phase'] = 'shutting_down'; $state['original_running'] = true;
|
||||
$state['shutdown_deadline'] = time() + 120;
|
||||
$op->power_workflow_json = json_encode($state); $op->lease_until = 0; $op->save();
|
||||
$instance = CloudInstance::find(120900);
|
||||
$instance->getConnection()->transaction(static function () use ($instance): void {
|
||||
Workflow::lockForInstance($instance);
|
||||
Workflow::manualIntentLocked($instance, 'action', 'start');
|
||||
});
|
||||
$count = count(CloudEngineClient::$commands);
|
||||
[$op, $lease] = $claim($op);
|
||||
Workflow::beforeApply($op, $lease, $actor, false);
|
||||
$assert($phase($op) === 'waiting_stopped' && count(CloudEngineClient::$commands) === $count,
|
||||
'Recovered undispatched shutdown overruled a newer manual start.');
|
||||
|
||||
// A definitely undispatched rejection can proceed after manual stop; an
|
||||
// uncertain/in-flight child is never replaced by a fresh command identifier.
|
||||
$op = $make(1212);
|
||||
$state = Workflow::state($op);
|
||||
CloudEngineClient::$operations[$state['shutdown_id']] = [
|
||||
'operation_id' => $state['shutdown_id'], 'status' => 'failed',
|
||||
'execution_outcome' => 'not_dispatched', 'mutation_started' => false,
|
||||
];
|
||||
[$op, $lease] = $claim($op); Workflow::beforeApply($op, $lease, $actor, false);
|
||||
$assert($phase($op) === 'paused' && Workflow::state($op->fresh())['error_code'] === 'vpc_power_command_rejected',
|
||||
'Definitive rejection was mislabeled as unknown.');
|
||||
CloudEngineClient::$power = 'shutoff';
|
||||
[$op, $lease] = $claim($op); Workflow::beforeApply($op, $lease, $actor, true);
|
||||
$assert($phase($op) === 'applying', 'Known undispatched shutdown could not proceed after manual stop.');
|
||||
|
||||
$op = $make(1213);
|
||||
$state = Workflow::state($op);
|
||||
CloudEngineClient::$operations[$state['shutdown_id']] = [
|
||||
'operation_id' => $state['shutdown_id'], 'status' => 'running',
|
||||
];
|
||||
[$op, $lease] = $claim($op); Workflow::beforeApply($op, $lease, $actor, false);
|
||||
$assert($op->fresh()->status === 'waiting_observed' && $phase($op) === 'shutting_down',
|
||||
'In-flight power acknowledgement requires automatic polling, not a manual pause.');
|
||||
CloudEngineClient::$operations[$state['shutdown_id']]['status'] = 'uncertain';
|
||||
[$op, $lease] = $claim($op); Workflow::beforeApply($op, $lease, $actor, false);
|
||||
$assert($phase($op) === 'paused' && count(CloudEngineClient::$commands) === $count,
|
||||
'Unknown shutdown outcome was replayed.');
|
||||
|
||||
$op = $make(1214);
|
||||
[$op, $lease] = $claim($op); Workflow::beforeApply($op, $lease, $actor, false);
|
||||
CloudEngineClient::$power = 'shutoff';
|
||||
[$op, $lease] = $claim($op); Workflow::beforeApply($op, $lease, $actor, false);
|
||||
$observed($op); $state = Workflow::state($op->fresh());
|
||||
CloudEngineClient::$operations[$state['start_id']] = [
|
||||
'operation_id' => $state['start_id'], 'status' => 'failed',
|
||||
'execution_outcome' => 'not_dispatched', 'mutation_started' => false,
|
||||
];
|
||||
Workflow::afterApply($op, $lease, $actor);
|
||||
$instance = CloudInstance::find(121400);
|
||||
$instance->getConnection()->transaction(static function () use ($instance): void {
|
||||
Workflow::lockForInstance($instance);
|
||||
Workflow::manualIntentLocked($instance, 'action', 'start');
|
||||
});
|
||||
[$op, $lease] = $claim($op); Workflow::beforeApply($op, $lease, $actor, true);
|
||||
$assert($phase($op) === 'completed', 'Manual start could not finish rejected automatic restoration.');
|
||||
|
||||
// End-to-end service/worker lane with the existing strict host/member proofs.
|
||||
// Quiesce unrelated legacy fixture operations so the worker selects this intent.
|
||||
Capsule::table('cloud_vpc_operations')->update(['status' => 'superseded', 'lease_until' => 0]);
|
||||
CloudEngineClient::$vpcRespond = static function (array $payload) use ($activeHostEvidence): array {
|
||||
$hosts = [];
|
||||
foreach ($payload['hosts'] as $host) {
|
||||
$id = (string) $host['host_id'];
|
||||
if ($host['members'] === []) {
|
||||
$hosts[$id] = [
|
||||
'vpc_id' => $payload['vpc_id'], 'host_id' => $id,
|
||||
'desired_revision' => $payload['desired_revision'], 'desired_digest' => $payload['desired_digest'],
|
||||
'observed_revision' => $payload['desired_revision'], 'observed_digest' => $payload['desired_digest'],
|
||||
'state' => 'active', 'transport_verified' => true,
|
||||
'private_dataplane_verified' => true, 'removed' => true,
|
||||
'bridge' => null, 'bridge_owner' => null, 'nft_table' => null,
|
||||
'rollback' => 'none', 'readback' => 'confirmed',
|
||||
'member_nics' => [], 'routes' => [], 'wg_allowed_ips' => [], 'proxy_ips' => [],
|
||||
];
|
||||
} else {
|
||||
$item = $activeHostEvidence($payload['vpc_id'], (int) $id,
|
||||
$payload['desired_revision'], $payload['desired_digest'], $payload['cidr'], $payload['members']);
|
||||
if (CloudEngineClient::$power === 'shutoff') {
|
||||
$item['state'] = 'degraded';
|
||||
$item['private_dataplane_verified'] = false;
|
||||
$item['last_error_code'] = 'vpc_private_guest_stopped';
|
||||
foreach ($item['member_nics'] as &$nic) {
|
||||
$nic['domain_running'] = false;
|
||||
$nic['guest_network_verified'] = false;
|
||||
$nic['guest_reason_code'] = 'vpc_private_guest_stopped';
|
||||
}
|
||||
unset($nic);
|
||||
}
|
||||
$hosts[$id] = $item;
|
||||
}
|
||||
}
|
||||
return [
|
||||
'vpc_id' => $payload['vpc_id'], 'operation_id' => $payload['operation_id'],
|
||||
'desired_revision' => $payload['desired_revision'], 'desired_digest' => $payload['desired_digest'],
|
||||
'observed' => ['evidence_version' => CloudVpcService::OBSERVATION_EVIDENCE_VERSION,
|
||||
'revision' => $payload['desired_revision'], 'digest' => $payload['desired_digest'],
|
||||
'state' => 'degraded', 'next_stage_allowed' => false,
|
||||
'private_dataplane_verified' => false, 'fabric_gate' => ['active' => true], 'hosts' => $hosts],
|
||||
];
|
||||
};
|
||||
$tick = static function (CloudVpcOperation $operation) use ($assert): void {
|
||||
Capsule::table('cloud_vpc_operations')->where('id', $operation->id)
|
||||
->update(['updated_at' => time() - 31, 'lease_until' => 0]);
|
||||
$assert((new CloudVpcWorkerService())->tick(), 'Independent VPC worker did not select current workflow.');
|
||||
};
|
||||
$op = $make(1210);
|
||||
$tick($op);
|
||||
$assert($phase($op) === 'waiting_stopped' && CloudEngineClient::$applies === 0,
|
||||
'VPC applied configuration before stopping the guest.');
|
||||
CloudEngineClient::$power = 'shutoff';
|
||||
$tick($op);
|
||||
$assert($phase($op) === 'completed' && $op->fresh()->status === 'succeeded'
|
||||
&& CloudEngineClient::$power === 'running',
|
||||
'Service/worker did not apply and restore a running member.');
|
||||
$root = CloudVpc::find($op->vpc_id);
|
||||
$member = CloudVpcMember::where('instance_id', 121000)->first();
|
||||
CloudVpcService::detachMember($root->vpc_key, $member->member_key, $actor, [
|
||||
'request_key' => 'auto-detach-1210', 'auto_power' => true,
|
||||
]);
|
||||
$detach = CloudVpcOperation::where('vpc_id', $root->id)->orderByDesc('desired_revision')->first();
|
||||
$tick($detach);
|
||||
CloudEngineClient::$power = 'shutoff';
|
||||
// Losing the start response leaves a durable paused workflow after exact
|
||||
// last-member teardown released the membership/IP rows.
|
||||
CloudEngineClient::$loseReceipt = true;
|
||||
$tick($detach);
|
||||
$assert($phase($detach) === 'paused'
|
||||
&& CloudVpcMember::where('instance_id', 121000)->count() === 0,
|
||||
'Last-member teardown did not preserve restoration workflow after removal.');
|
||||
$other = $seedVpc(1211, '10.121.211.0/24', '10.121.211.0/24', 10);
|
||||
$expectCloudError(static fn () => CloudVpcService::attachMember($other['vpc_key'], $actor, [
|
||||
'instance_id' => 121000, 'request_key' => 'cross-attach-121000', 'auto_power' => true,
|
||||
]), 409);
|
||||
$count = count(CloudEngineClient::$commands);
|
||||
CloudVpcService::reconcile($root->vpc_key, $actor, true, true);
|
||||
$assert($phase($detach) === 'completed' && $detach->fresh()->status === 'succeeded'
|
||||
&& count(CloudEngineClient::$commands) === $count,
|
||||
'Last-member paused start could not recover its original receipt.');
|
||||
|
||||
// An installed pre-workflow schema continues to admit ordinary power, but
|
||||
// cannot accept a new automatic maintenance request.
|
||||
$instance = CloudInstance::find(121000);
|
||||
$index = 'cloud_vpc_operations_power_instance_id_index';
|
||||
$assert($schema->hasIndex('cloud_vpc_operations', ['power_instance_id']),
|
||||
'Workflow lookup has no index to bound its locking read.');
|
||||
$schema->table('cloud_vpc_operations', static function ($table) use ($index): void { $table->dropIndex($index); });
|
||||
$expectCloudError(static fn () => Workflow::lockForInstance($instance), 503);
|
||||
$schema->table('cloud_vpc_operations', static function ($table): void { $table->index('power_instance_id'); });
|
||||
$detach->refresh();
|
||||
$detach->power_instance_id = 999;
|
||||
$expectCloudError(static fn () => Workflow::state($detach), 503);
|
||||
$detach->refresh();
|
||||
$schema->table('cloud_vpc_operations', static function ($table) use ($index): void {
|
||||
$table->dropIndex($index); $table->dropColumn('power_instance_id');
|
||||
});
|
||||
$expectCloudError(static fn () => Workflow::lockForInstance($instance), 503);
|
||||
$schema->table('cloud_vpc_operations', static function ($table): void { $table->dropColumn('power_workflow_json'); });
|
||||
Workflow::lockForInstance($instance);
|
||||
Workflow::manualIntentLocked($instance, 'action', 'poweroff');
|
||||
$expectCloudError(static fn () => Workflow::initialize($detach, $instance, $member), 503);
|
||||
|
||||
echo "VPC automatic power workflow contracts passed.\n";
|
||||
}
|
||||
Loading…
Reference in New Issue