|
|
|
|
@ -647,6 +647,169 @@ $assert($phase($detach) === 'completed' && $detach->fresh()->status === 'succeed
|
|
|
|
|
&& count(CloudEngineClient::$commands) === $count,
|
|
|
|
|
'Last-member paused start could not recover its original receipt.');
|
|
|
|
|
|
|
|
|
|
// The departing host can finish while a surviving host is still converging.
|
|
|
|
|
// Keep that exact accepted target in the GET lane until the complete proof
|
|
|
|
|
// arrives; a VM must not remain off merely because the first receipt is partial.
|
|
|
|
|
$ordinaryVpcRespond = CloudEngineClient::$vpcRespond;
|
|
|
|
|
$runningVpcProof = static function (array $payload) use ($ordinaryVpcRespond, $activeHostEvidence): array {
|
|
|
|
|
$result = $ordinaryVpcRespond($payload);
|
|
|
|
|
$hosts = [];
|
|
|
|
|
foreach ($result['hosts'] as $host) {
|
|
|
|
|
$hostId = (string) $host['host_id'];
|
|
|
|
|
$hosts[$hostId] = ($host['removed'] ?? false) === true ? $host
|
|
|
|
|
: $activeHostEvidence($payload['vpc_id'], (int) $hostId,
|
|
|
|
|
$payload['desired_revision'], $payload['desired_digest'],
|
|
|
|
|
$payload['cidr'], $payload['members']);
|
|
|
|
|
}
|
|
|
|
|
$result['state'] = $result['observed']['state'] = 'active';
|
|
|
|
|
$result['next_stage_allowed'] = $result['observed']['next_stage_allowed'] = true;
|
|
|
|
|
$result['private_dataplane_verified'] = $result['observed']['private_dataplane_verified'] = true;
|
|
|
|
|
$result['hosts'] = array_values($hosts);
|
|
|
|
|
$result['observed']['hosts'] = $hosts;
|
|
|
|
|
return $result;
|
|
|
|
|
};
|
|
|
|
|
$partialDetach = static function (int $id, bool $sameHost = false, ?string $fault = null) use (
|
|
|
|
|
$make, $tick, $actor, $seedInstance, $ordinaryVpcRespond, $runningVpcProof, $assert
|
|
|
|
|
): array {
|
|
|
|
|
Capsule::table('cloud_vpc_operations')->update(['status' => 'superseded', 'lease_until' => 0]);
|
|
|
|
|
CloudEngineClient::$vpcRespond = $ordinaryVpcRespond;
|
|
|
|
|
CloudEngineClient::$vpcRead = null;
|
|
|
|
|
$attach = $make($id);
|
|
|
|
|
$tick($attach);
|
|
|
|
|
CloudEngineClient::$power = 'shutoff';
|
|
|
|
|
$tick($attach); $tick($attach);
|
|
|
|
|
$vpc = CloudVpc::find($attach->vpc_id);
|
|
|
|
|
foreach ([1, 2] as $offset) {
|
|
|
|
|
$seedInstance($id * 100 + $offset, $sameHost ? 10 : 20);
|
|
|
|
|
CloudVpcService::attachMember($vpc->vpc_key, $actor, [
|
|
|
|
|
'instance_id' => $id * 100 + $offset,
|
|
|
|
|
'request_key' => 'retained-sibling-' . $id . '-' . $offset,
|
|
|
|
|
]);
|
|
|
|
|
CloudVpcService::reconcile($vpc->vpc_key, $actor, true, true);
|
|
|
|
|
}
|
|
|
|
|
$target = CloudVpcMember::where('instance_id', $id * 100)->first();
|
|
|
|
|
CloudVpcService::detachMember($vpc->vpc_key, $target->member_key, $actor, [
|
|
|
|
|
'request_key' => 'partial-detach-' . $id, 'auto_power' => true,
|
|
|
|
|
]);
|
|
|
|
|
$operation = CloudVpcOperation::where('vpc_id', $vpc->id)->orderByDesc('desired_revision')->first();
|
|
|
|
|
$tick($operation);
|
|
|
|
|
CloudEngineClient::$power = 'shutoff';
|
|
|
|
|
$full = $runningVpcProof(json_decode($operation->payload_json, true));
|
|
|
|
|
$partial = $full;
|
|
|
|
|
$partial['state'] = $partial['observed']['state'] = 'degraded';
|
|
|
|
|
$partial['next_stage_allowed'] = $partial['observed']['next_stage_allowed'] = false;
|
|
|
|
|
$partial['private_dataplane_verified'] = $partial['observed']['private_dataplane_verified'] = false;
|
|
|
|
|
$partial['observed']['fabric_gate'] = ['active' => false, 'available' => true];
|
|
|
|
|
foreach ($partial['hosts'] as &$host) {
|
|
|
|
|
if (($host['removed'] ?? false) === true) { continue; }
|
|
|
|
|
$host['transport_verified'] = $host['config_verified'] = $host['private_dataplane_verified'] = false;
|
|
|
|
|
$host['observed_revision'] = $host['config_observed_revision'] = 0;
|
|
|
|
|
$host['observed_digest'] = $host['config_observed_digest'] = null;
|
|
|
|
|
$host['last_error_code'] = 'vpc_private_dataplane_unverified';
|
|
|
|
|
$host['wg_allowed_ips'] = [];
|
|
|
|
|
}
|
|
|
|
|
unset($host);
|
|
|
|
|
$partial['observed']['hosts'] = [];
|
|
|
|
|
if ($fault === 'missing_host') {
|
|
|
|
|
array_pop($partial['hosts']);
|
|
|
|
|
} elseif ($fault === 'host_digest') {
|
|
|
|
|
$partial['hosts'][count($partial['hosts']) - 1]['desired_digest'] = str_repeat('f', 64);
|
|
|
|
|
} elseif ($fault === 'host_failed') {
|
|
|
|
|
$partial['hosts'][count($partial['hosts']) - 1]['last_error_code'] = 'vpc_private_dataplane_apply_failed';
|
|
|
|
|
}
|
|
|
|
|
CloudEngineClient::$vpcRespond = static fn () => $partial;
|
|
|
|
|
$tick($operation);
|
|
|
|
|
return [$operation, $partial, $full];
|
|
|
|
|
};
|
|
|
|
|
[$op, $partial, $full] = $partialDetach(1260);
|
|
|
|
|
$state = Workflow::state($op->fresh());
|
|
|
|
|
$deadline = $state['configuration_deadline'] ?? 0;
|
|
|
|
|
$applies = CloudEngineClient::$applies; $count = count(CloudEngineClient::$commands);
|
|
|
|
|
$assert($state['phase'] === 'applying' && !$state['config_confirmed'] && $deadline > time()
|
|
|
|
|
&& $op->fresh()->status === 'waiting_observed' && $op->fresh()->lease_token === null
|
|
|
|
|
&& CloudVpcMember::where('instance_id', 126000)->count() === 1,
|
|
|
|
|
'The partial two-host detach paused or released an unconfirmed member.');
|
|
|
|
|
foreach (['partial', 'missing', 'older'] as $mode) {
|
|
|
|
|
CloudEngineClient::$vpcRead = static function () use ($partial, $mode): array {
|
|
|
|
|
if ($mode === 'missing') { throw new CloudError('missing', 404); }
|
|
|
|
|
if ($mode === 'older') { $partial['desired_revision']--; }
|
|
|
|
|
return $partial;
|
|
|
|
|
};
|
|
|
|
|
$tick($op);
|
|
|
|
|
$state = Workflow::state($op->fresh());
|
|
|
|
|
$assert($state['phase'] === 'applying' && !$state['config_confirmed']
|
|
|
|
|
&& $state['configuration_deadline'] === $deadline
|
|
|
|
|
&& $op->fresh()->status === 'waiting_observed'
|
|
|
|
|
&& CloudEngineClient::$applies === $applies && count(CloudEngineClient::$commands) === $count,
|
|
|
|
|
'Accepted configuration was resubmitted, restarted or extended on ' . $mode);
|
|
|
|
|
}
|
|
|
|
|
CloudEngineClient::$vpcRead = static fn () => $full;
|
|
|
|
|
$tick($op);
|
|
|
|
|
$assert($phase($op) === 'completed' && CloudEngineClient::$power === 'running'
|
|
|
|
|
&& CloudVpcMember::where('instance_id', 126000)->count() === 0
|
|
|
|
|
&& CloudVpcMember::whereIn('instance_id', [126001, 126002])->where('state', 'active')->count() === 2
|
|
|
|
|
&& CloudEngineClient::$applies === $applies && count(CloudEngineClient::$commands) === $count + 1,
|
|
|
|
|
'Later exact detach proof did not release only the target and restore it once.');
|
|
|
|
|
|
|
|
|
|
// The same mechanism also preserves siblings when the departing host stays live.
|
|
|
|
|
[$op, $partial, $full] = $partialDetach(1261, true);
|
|
|
|
|
$applies = CloudEngineClient::$applies;
|
|
|
|
|
CloudEngineClient::$vpcRead = static fn () => $full;
|
|
|
|
|
$tick($op);
|
|
|
|
|
$assert($phase($op) === 'completed'
|
|
|
|
|
&& CloudVpcMember::where('instance_id', 126100)->count() === 0
|
|
|
|
|
&& CloudVpcMember::whereIn('instance_id', [126101, 126102])->where('state', 'active')->count() === 2
|
|
|
|
|
&& CloudEngineClient::$applies === $applies,
|
|
|
|
|
'Same-host detach required removing the host or lost its retained siblings.');
|
|
|
|
|
|
|
|
|
|
foreach (['partial', 'missing', 'exact'] as $offset => $mode) {
|
|
|
|
|
[$op, $partial, $full] = $partialDetach(1262 + $offset);
|
|
|
|
|
$state = Workflow::state($op->fresh());
|
|
|
|
|
$state['configuration_deadline'] = time() - 1;
|
|
|
|
|
$op->power_workflow_json = json_encode($state); $op->save();
|
|
|
|
|
$applies = CloudEngineClient::$applies; $count = count(CloudEngineClient::$commands);
|
|
|
|
|
CloudEngineClient::$vpcRead = static function () use ($mode, $partial, $full): array {
|
|
|
|
|
if ($mode === 'missing') { throw new CloudError('missing', 404); }
|
|
|
|
|
return $mode === 'exact' ? $full : $partial;
|
|
|
|
|
};
|
|
|
|
|
$tick($op);
|
|
|
|
|
$assert($phase($op) === ($mode === 'exact' ? 'completed' : 'paused')
|
|
|
|
|
&& CloudEngineClient::$applies === $applies
|
|
|
|
|
&& count(CloudEngineClient::$commands) === $count + ($mode === 'exact' ? 1 : 0),
|
|
|
|
|
'Configuration deadline extended, resent a POST or discarded exact proof: ' . $mode);
|
|
|
|
|
if ($mode !== 'exact') {
|
|
|
|
|
$assert(Workflow::state($op->fresh())['configuration_deadline'] === $state['configuration_deadline'],
|
|
|
|
|
'A deadline was refreshed by an ordinary worker poll.');
|
|
|
|
|
CloudEngineClient::$vpcRead = static fn () => $partial;
|
|
|
|
|
CloudVpcService::reconcile(CloudVpc::find($op->vpc_id)->vpc_key, $actor, true, true);
|
|
|
|
|
$assert($phase($op) === 'applying'
|
|
|
|
|
&& Workflow::state($op->fresh())['configuration_deadline'] > time()
|
|
|
|
|
&& CloudEngineClient::$applies === $applies,
|
|
|
|
|
'Explicit retry could not open a bounded read-only recovery window.');
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
[$op, $partial, $full] = $partialDetach(1265);
|
|
|
|
|
$instance = CloudInstance::find(126500);
|
|
|
|
|
$instance->getConnection()->transaction(static function () use ($instance): void {
|
|
|
|
|
Workflow::lockForInstance($instance);
|
|
|
|
|
Workflow::manualIntentLocked($instance, 'action', 'start');
|
|
|
|
|
});
|
|
|
|
|
$applies = CloudEngineClient::$applies; $count = count(CloudEngineClient::$commands);
|
|
|
|
|
CloudEngineClient::$vpcRead = static fn () => $full;
|
|
|
|
|
$tick($op);
|
|
|
|
|
$assert($phase($op) === 'completed' && Workflow::state($op->fresh())['resume_cancelled']
|
|
|
|
|
&& CloudEngineClient::$applies === $applies && count(CloudEngineClient::$commands) === $count,
|
|
|
|
|
'Configuration polling overrode a newer manual power intent.');
|
|
|
|
|
foreach (['missing_host', 'host_digest', 'host_failed'] as $offset => $fault) {
|
|
|
|
|
[$op] = $partialDetach(1266 + $offset, false, $fault);
|
|
|
|
|
$state = Workflow::state($op->fresh());
|
|
|
|
|
$assert($state['phase'] === 'paused' && !$state['config_confirmed']
|
|
|
|
|
&& !isset($state['configuration_deadline']) && CloudEngineClient::$power === 'shutoff',
|
|
|
|
|
'Invalid partial proof opened an automatic waiting window: ' . $fault);
|
|
|
|
|
}
|
|
|
|
|
CloudEngineClient::$vpcRespond = $ordinaryVpcRespond;
|
|
|
|
|
CloudEngineClient::$vpcRead = null;
|
|
|
|
|
|
|
|
|
|
// Recover an interrupted apply from the exact durable engine receipt before
|
|
|
|
|
// the stopped-guest mutation precondition. A manual start needs no second stop.
|
|
|
|
|
$interrupted = static function (int $id, bool $paused = true) use ($make): CloudVpcOperation {
|
|
|
|
|
@ -733,6 +896,25 @@ CloudVpcService::reconcile(CloudVpc::find($op->vpc_id)->vpc_key, $actor, true, t
|
|
|
|
|
$assert($phase($op) === 'completed' && CloudEngineClient::$applies === $applies + 1,
|
|
|
|
|
'Missing receipt prevented legitimate stopped-guest configuration.');
|
|
|
|
|
|
|
|
|
|
// A rejected first submission has no accepted partial receipt. Preserve the
|
|
|
|
|
// original explicit-retry lane instead of fencing it behind a permanent GET.
|
|
|
|
|
$op = $interrupted(1270);
|
|
|
|
|
CloudEngineClient::$power = 'shutoff';
|
|
|
|
|
CloudEngineClient::$vpcRead = null;
|
|
|
|
|
CloudEngineClient::$vpcRespond = static function (): array {
|
|
|
|
|
throw new CloudError('rejected before apply', 422);
|
|
|
|
|
};
|
|
|
|
|
$applies = CloudEngineClient::$applies;
|
|
|
|
|
CloudVpcService::reconcile(CloudVpc::find($op->vpc_id)->vpc_key, $actor, true, true);
|
|
|
|
|
$assert($phase($op) === 'paused'
|
|
|
|
|
&& !isset(Workflow::state($op->fresh())['configuration_deadline'])
|
|
|
|
|
&& CloudEngineClient::$applies === $applies + 1,
|
|
|
|
|
'A rejected first POST acquired an accepted-configuration fence.');
|
|
|
|
|
CloudEngineClient::$vpcRespond = $ordinaryVpcRespond;
|
|
|
|
|
CloudVpcService::reconcile(CloudVpc::find($op->vpc_id)->vpc_key, $actor, true, true);
|
|
|
|
|
$assert($phase($op) === 'completed' && CloudEngineClient::$applies === $applies + 2,
|
|
|
|
|
'Explicit retry could not resubmit after a definite first-POST rejection.');
|
|
|
|
|
|
|
|
|
|
// New manual intent during the read wins even if the original guest ran.
|
|
|
|
|
$op = $interrupted(1252);
|
|
|
|
|
$state = Workflow::state($op); $state['original_running'] = true; $state['resume_cancelled'] = false;
|
|
|
|
|
|