fix: keep VPC power flow waiting for configuration proof

venix
admin 1 month ago
parent eb38f9debd
commit df53cea203

@ -1,5 +1,41 @@
# 本版验证记录
## 2026-10-14:退出已关机后因部分配置回执提前暂停
- 现场 `vm-zf99er` 的退出操作 13
`0e715a8a98a8babb0192724a5538b0c5`、revision 9 在正常关机后
暂停为 `vpc_power_configuration_unconfirmed`,`paused_from=applying`、
`config_confirmed=false`,没有更新的手动电源任务。
- 暂停时的原始回执中,目标宿主 2 已 `removed=true` 且精确确认 revision 9;
宿主 1 的两个保留成员 NIC/来宾证明完整,但聚合的配置、传输证明尚未齐全。
后续引擎自行恢复 `active`、9/9、两个宿主配置验证通过;PHP 任务却仍 paused。
这与 Fabric 回读期间撤销旧 gate、随后重新验证的中间状态一致,不能据此认定
宿主故障。无远端成员时 `wg_allowed_ips=[]` 是合法配置。
- 根因是 `afterApply()` 收到一次未齐的配置证明就终止自动流程,后台 worker
不领取 paused 任务,因此引擎随后完成核对也无法触发原恢复开机步骤。
- 在修改部署前,确认当前完整配置证明及无更新手动电源意图后,通过页面对原任务
执行一次“核对 / 重试”。原退出已完成,页面为 9/9、`applied`、错误为空,
保留另两成员。真实宿主确认目标实例 `running`,运行及持久配置仅公网 `br0`
网卡,原私网桥不存在;QGA 只返回公网地址。没有重新提交加入/退出或直接写库。
本次现场恢复使用既有显式核对路径,不能算作新补丁的自动恢复验收。
- 最终原任务为 `succeeded/completed`、`config_confirmed=true`、
`resume_cancelled=false`;正常关机子操作
`8a6a2aea44c6b353722b79fb23afeb7d` 与恢复开机子操作
`3e685cc7a888ae75586ee367b2fd5f69` 均 accepted,保持原固定 ID。
- 修复仅将身份、版本、摘要及宿主集合匹配的合法中间回执留在
`applying/waiting_observed`;后台 GET 继续核对原目标,不重复 POST。
未确认时自动等待上限为 120 秒,普通轮询不延长期限;到期仍缺证明则暂停。
完整原目标证明即使迟到,仍可经既有身份、手动电源意图和全宿主配置检查恢复。
已接收目标后的 404/旧 revision 只等待;尚未接收回执的原提交异常恢复逻辑不变。
- PHP 8.5 WASM/SQLite 自动电源、失败核对、控制面、成员清理收敛及页面契约
**5 项脚本通过**。新增真实 service/worker 回归覆盖双宿主首轮 partial 后
多轮 GET 到完整证明再开机、同宿主保留两个成员、缺回执、固定期限、超时、
显式恢复、更新的手动电源操作,以及错误宿主集合/摘要不得进入自动等待。
首次 POST 明确拒绝后仍可显式核对并按原契约重新提交。
- 隔离副本仅把两处业务 PHP 恢复为 `eb38f9d`,新增双宿主回归在“部分回执
被提前暂停”断言失败;补丁版本通过。未改变 Engine、数据库结构或 active
判据,没有为通过测试移除原有严格配置检查。
## 2026-10-14:自动开机后的 VPC 回读延迟
- 现场 `vm-zf99er` 加入 VPC `xx` 的 revision 7 操作

@ -13,6 +13,7 @@ use Illuminate\Database\QueryException;
final class CloudVpcPowerWorkflow
{
public const SHUTDOWN_SECONDS = 120;
public const CONFIGURATION_SECONDS = 120;
private static array $lockedRootIds = [];
public static function requested(array $input): bool
@ -67,6 +68,9 @@ final class CloudVpcPowerWorkflow
|| ($state['start_id'] ?? null) !== substr(hash('sha256', $operation->operation_id . ':vpc:start'), 0, 32)
|| (isset($state['shutdown_deadline']) && (!is_int($state['shutdown_deadline']) || $state['shutdown_deadline'] < 1))
|| (isset($state['start_deadline']) && (!is_int($state['start_deadline']) || $state['start_deadline'] < 1))
|| (isset($state['configuration_deadline'])
&& (!is_int($state['configuration_deadline']) || $state['configuration_deadline'] < 1))
|| (isset($state['configuration_pending']) && !is_bool($state['configuration_pending']))
|| (($state['phase'] ?? null) === 'paused' && !in_array($state['paused_from'] ?? null,
['preflight', 'shutting_down', 'waiting_stopped', 'applying', 'configured', 'starting'], true))
|| !in_array($state['phase'] ?? null, ['preflight', 'shutting_down',
@ -414,7 +418,7 @@ final class CloudVpcPowerWorkflow
/** Called under recordObserved's root/operation locks after strict proof. */
public static function configurationObserved(CloudVpcOperation $operation, bool $confirmed,
bool $recoverPausedApply = false): bool
bool $recoverPausedApply = false, bool $pending = false): bool
{
$state = self::state($operation);
if ($state === null || $state['phase'] === 'completed') {
@ -426,18 +430,47 @@ final class CloudVpcPowerWorkflow
// recordObserved's transaction rolls back all evidence on mismatch.
self::lockInstance($state);
}
if ($confirmed && ($state['phase'] === 'applying'
if ($state['phase'] === 'applying'
|| ($recoverPausedApply && $state['phase'] === 'paused'
&& $state['paused_from'] === 'applying'))) {
$state['config_confirmed'] = true;
$state['phase'] = 'configured';
$state['error_code'] = null;
unset($state['paused_from']);
&& $state['paused_from'] === 'applying')) {
$state['configuration_pending'] = !$confirmed && $pending;
if ($confirmed || $pending) {
if ($pending && (!isset($state['configuration_deadline'])
|| $state['phase'] === 'paused')) {
$state['configuration_deadline'] = time() + self::CONFIGURATION_SECONDS;
}
$state['config_confirmed'] = $confirmed;
$state['phase'] = $confirmed ? 'configured' : 'applying';
$state['error_code'] = null;
unset($state['paused_from']);
}
}
self::saveState($operation, $state);
return true;
}
/** A previously accepted target must never become a second POST on GET 404. */
public static function missingConfigurationReceipt(CloudVpcOperation $operation,
string $lease, bool $explicitRetry): bool
{
return self::locked($operation, $lease,
static function ($root, $row, $instance, $state) use ($explicitRetry): bool {
if (!isset($state['configuration_deadline'])) {
return false;
}
if ($explicitRetry && $state['phase'] === 'paused'
&& $state['paused_from'] === 'applying') {
$state['phase'] = 'applying';
$state['configuration_deadline'] = time() + self::CONFIGURATION_SECONDS;
$state['error_code'] = null;
unset($state['paused_from']);
}
$state['configuration_pending'] = true;
self::saveState($row, $state);
return true;
});
}
/** Preflight failures have no remote mutation uncertainty of their own. */
public static function handleFailure(CloudVpcOperation $operation, string $lease,
CloudError $error): bool
@ -465,6 +498,12 @@ final class CloudVpcPowerWorkflow
$client ??= new CloudEngineClient();
$start = self::locked($operation, $lease, static function ($root, $row, $instance, $state): bool {
if (!($state['config_confirmed'] ?? false)) {
if ($state['phase'] === 'applying'
&& ($state['configuration_pending'] ?? false) === true
&& time() < (int) ($state['configuration_deadline'] ?? 0)) {
self::release($row, $state);
return false;
}
self::pause($root, $row, $state, 'vpc_power_configuration_unconfirmed');
return false;
}

@ -1443,7 +1443,9 @@ final class CloudVpcService
);
} catch (CloudError $error) {
if ($error->httpStatus === 404) {
return false; // No receipt: the original stopped-guest gate applies.
return CloudVpcPowerWorkflow::missingConfigurationReceipt(
$operation, $lease, $explicitRetry
);
}
throw $error;
}
@ -1451,14 +1453,51 @@ final class CloudVpcService
&& is_int($result['desired_revision'] ?? null)
&& $result['desired_revision'] > 0
&& $result['desired_revision'] < $payload['desired_revision']) {
return false; // This target has not reached the engine yet.
return CloudVpcPowerWorkflow::missingConfigurationReceipt(
$operation, $lease, $explicitRetry
);
}
$observed = self::applyObservation($operation, $payload, $result);
self::recordObserved($payload['vpc_id'], $payload['desired_revision'],
$payload['desired_digest'], $observed, (string) $operation->operation_id,
$lease, true);
$operation->refresh();
return (CloudVpcPowerWorkflow::state($operation)['config_confirmed'] ?? false) === true;
// Even a partial current receipt consumes this turn. Falling through
// would POST the same configuration again instead of waiting for its
// durable engine observation to finish.
return true;
}
/** Partial progress is retryable, never a substitute for strict proof. */
private static function pendingPowerConfiguration(array $observed, array $hosts,
CloudVpcOperation $operation, string $vpcKey, int $revision, string $digest): bool
{
if (($observed['evidence_version'] ?? null) !== self::OBSERVATION_EVIDENCE_VERSION
|| !in_array($observed['state'] ?? null, ['applying', 'degraded'], true)
|| ($observed['next_stage_allowed'] ?? null) !== false
|| ($observed['private_dataplane_verified'] ?? null) !== false) {
return false;
}
$payload = json_decode((string) $operation->payload_json, true);
$expected = is_array($payload['hosts'] ?? null) ? $payload['hosts'] : [];
if ($expected === [] || count($expected) !== count($hosts)) {
return false;
}
foreach ($expected as $target) {
$hostId = (string) ($target['host_id'] ?? '');
$item = $hosts[$hostId] ?? null;
$error = is_array($item) ? ($item['last_error_code'] ?? null) : null;
if (!is_array($item) || ($item['vpc_id'] ?? null) !== $vpcKey
|| (string) ($item['host_id'] ?? '') !== $hostId
|| ($item['desired_revision'] ?? null) !== $revision
|| !self::digestMatches($digest, (string) ($item['desired_digest'] ?? ''))
|| !in_array($item['state'] ?? null, ['active', 'applying', 'degraded'], true)
|| ($error !== null && $error !== 'vpc_private_dataplane_unverified'
&& self::deferredEvidenceError($error) === null)) {
return false;
}
}
return true;
}
/** GET recovery and POST apply must consume the same exact evidence. */
@ -1567,6 +1606,11 @@ final class CloudVpcService
$hostKey = (string) ($item['host_id'] ?? $key);
$hosts[$hostKey] = $item;
}
// Classify progress before validation replaces diagnostic fields.
// An active envelope with malformed proof remains an error.
$powerConfigurationPending = self::pendingPowerConfiguration(
$observed, $hosts, $operation, $vpcKey, $revision, $digest
);
$evidenceVersion = max(0, (int) ($observed['evidence_version']
?? $observed['observation_version'] ?? 0));
$evidenceContractCurrent =
@ -2448,7 +2492,8 @@ final class CloudVpcService
$operation->finished_at = ($operationSucceeded || $dataPlaneDeferred || $terminalFailure)
? time() : null;
$powerPending = CloudVpcPowerWorkflow::configurationObserved(
$operation, $operationSucceeded, $recoverPowerConfiguration
$operation, $operationSucceeded, $recoverPowerConfiguration,
$powerConfigurationPending && $desiredContractValid && !$terminalGate
);
if ($powerPending && $leaseToken !== null) {
$operation->status = 'running';

@ -647,6 +647,169 @@ $assert($phase($detach) === 'completed' && $detach->fresh()->status === 'succeed
&& count(CloudEngineClient::$commands) === $count,
'Last-member paused start could not recover its original receipt.');
// The departing host can finish while a surviving host is still converging.
// Keep that exact accepted target in the GET lane until the complete proof
// arrives; a VM must not remain off merely because the first receipt is partial.
$ordinaryVpcRespond = CloudEngineClient::$vpcRespond;
$runningVpcProof = static function (array $payload) use ($ordinaryVpcRespond, $activeHostEvidence): array {
$result = $ordinaryVpcRespond($payload);
$hosts = [];
foreach ($result['hosts'] as $host) {
$hostId = (string) $host['host_id'];
$hosts[$hostId] = ($host['removed'] ?? false) === true ? $host
: $activeHostEvidence($payload['vpc_id'], (int) $hostId,
$payload['desired_revision'], $payload['desired_digest'],
$payload['cidr'], $payload['members']);
}
$result['state'] = $result['observed']['state'] = 'active';
$result['next_stage_allowed'] = $result['observed']['next_stage_allowed'] = true;
$result['private_dataplane_verified'] = $result['observed']['private_dataplane_verified'] = true;
$result['hosts'] = array_values($hosts);
$result['observed']['hosts'] = $hosts;
return $result;
};
$partialDetach = static function (int $id, bool $sameHost = false, ?string $fault = null) use (
$make, $tick, $actor, $seedInstance, $ordinaryVpcRespond, $runningVpcProof, $assert
): array {
Capsule::table('cloud_vpc_operations')->update(['status' => 'superseded', 'lease_until' => 0]);
CloudEngineClient::$vpcRespond = $ordinaryVpcRespond;
CloudEngineClient::$vpcRead = null;
$attach = $make($id);
$tick($attach);
CloudEngineClient::$power = 'shutoff';
$tick($attach); $tick($attach);
$vpc = CloudVpc::find($attach->vpc_id);
foreach ([1, 2] as $offset) {
$seedInstance($id * 100 + $offset, $sameHost ? 10 : 20);
CloudVpcService::attachMember($vpc->vpc_key, $actor, [
'instance_id' => $id * 100 + $offset,
'request_key' => 'retained-sibling-' . $id . '-' . $offset,
]);
CloudVpcService::reconcile($vpc->vpc_key, $actor, true, true);
}
$target = CloudVpcMember::where('instance_id', $id * 100)->first();
CloudVpcService::detachMember($vpc->vpc_key, $target->member_key, $actor, [
'request_key' => 'partial-detach-' . $id, 'auto_power' => true,
]);
$operation = CloudVpcOperation::where('vpc_id', $vpc->id)->orderByDesc('desired_revision')->first();
$tick($operation);
CloudEngineClient::$power = 'shutoff';
$full = $runningVpcProof(json_decode($operation->payload_json, true));
$partial = $full;
$partial['state'] = $partial['observed']['state'] = 'degraded';
$partial['next_stage_allowed'] = $partial['observed']['next_stage_allowed'] = false;
$partial['private_dataplane_verified'] = $partial['observed']['private_dataplane_verified'] = false;
$partial['observed']['fabric_gate'] = ['active' => false, 'available' => true];
foreach ($partial['hosts'] as &$host) {
if (($host['removed'] ?? false) === true) { continue; }
$host['transport_verified'] = $host['config_verified'] = $host['private_dataplane_verified'] = false;
$host['observed_revision'] = $host['config_observed_revision'] = 0;
$host['observed_digest'] = $host['config_observed_digest'] = null;
$host['last_error_code'] = 'vpc_private_dataplane_unverified';
$host['wg_allowed_ips'] = [];
}
unset($host);
$partial['observed']['hosts'] = [];
if ($fault === 'missing_host') {
array_pop($partial['hosts']);
} elseif ($fault === 'host_digest') {
$partial['hosts'][count($partial['hosts']) - 1]['desired_digest'] = str_repeat('f', 64);
} elseif ($fault === 'host_failed') {
$partial['hosts'][count($partial['hosts']) - 1]['last_error_code'] = 'vpc_private_dataplane_apply_failed';
}
CloudEngineClient::$vpcRespond = static fn () => $partial;
$tick($operation);
return [$operation, $partial, $full];
};
[$op, $partial, $full] = $partialDetach(1260);
$state = Workflow::state($op->fresh());
$deadline = $state['configuration_deadline'] ?? 0;
$applies = CloudEngineClient::$applies; $count = count(CloudEngineClient::$commands);
$assert($state['phase'] === 'applying' && !$state['config_confirmed'] && $deadline > time()
&& $op->fresh()->status === 'waiting_observed' && $op->fresh()->lease_token === null
&& CloudVpcMember::where('instance_id', 126000)->count() === 1,
'The partial two-host detach paused or released an unconfirmed member.');
foreach (['partial', 'missing', 'older'] as $mode) {
CloudEngineClient::$vpcRead = static function () use ($partial, $mode): array {
if ($mode === 'missing') { throw new CloudError('missing', 404); }
if ($mode === 'older') { $partial['desired_revision']--; }
return $partial;
};
$tick($op);
$state = Workflow::state($op->fresh());
$assert($state['phase'] === 'applying' && !$state['config_confirmed']
&& $state['configuration_deadline'] === $deadline
&& $op->fresh()->status === 'waiting_observed'
&& CloudEngineClient::$applies === $applies && count(CloudEngineClient::$commands) === $count,
'Accepted configuration was resubmitted, restarted or extended on ' . $mode);
}
CloudEngineClient::$vpcRead = static fn () => $full;
$tick($op);
$assert($phase($op) === 'completed' && CloudEngineClient::$power === 'running'
&& CloudVpcMember::where('instance_id', 126000)->count() === 0
&& CloudVpcMember::whereIn('instance_id', [126001, 126002])->where('state', 'active')->count() === 2
&& CloudEngineClient::$applies === $applies && count(CloudEngineClient::$commands) === $count + 1,
'Later exact detach proof did not release only the target and restore it once.');
// The same mechanism also preserves siblings when the departing host stays live.
[$op, $partial, $full] = $partialDetach(1261, true);
$applies = CloudEngineClient::$applies;
CloudEngineClient::$vpcRead = static fn () => $full;
$tick($op);
$assert($phase($op) === 'completed'
&& CloudVpcMember::where('instance_id', 126100)->count() === 0
&& CloudVpcMember::whereIn('instance_id', [126101, 126102])->where('state', 'active')->count() === 2
&& CloudEngineClient::$applies === $applies,
'Same-host detach required removing the host or lost its retained siblings.');
foreach (['partial', 'missing', 'exact'] as $offset => $mode) {
[$op, $partial, $full] = $partialDetach(1262 + $offset);
$state = Workflow::state($op->fresh());
$state['configuration_deadline'] = time() - 1;
$op->power_workflow_json = json_encode($state); $op->save();
$applies = CloudEngineClient::$applies; $count = count(CloudEngineClient::$commands);
CloudEngineClient::$vpcRead = static function () use ($mode, $partial, $full): array {
if ($mode === 'missing') { throw new CloudError('missing', 404); }
return $mode === 'exact' ? $full : $partial;
};
$tick($op);
$assert($phase($op) === ($mode === 'exact' ? 'completed' : 'paused')
&& CloudEngineClient::$applies === $applies
&& count(CloudEngineClient::$commands) === $count + ($mode === 'exact' ? 1 : 0),
'Configuration deadline extended, resent a POST or discarded exact proof: ' . $mode);
if ($mode !== 'exact') {
$assert(Workflow::state($op->fresh())['configuration_deadline'] === $state['configuration_deadline'],
'A deadline was refreshed by an ordinary worker poll.');
CloudEngineClient::$vpcRead = static fn () => $partial;
CloudVpcService::reconcile(CloudVpc::find($op->vpc_id)->vpc_key, $actor, true, true);
$assert($phase($op) === 'applying'
&& Workflow::state($op->fresh())['configuration_deadline'] > time()
&& CloudEngineClient::$applies === $applies,
'Explicit retry could not open a bounded read-only recovery window.');
}
}
[$op, $partial, $full] = $partialDetach(1265);
$instance = CloudInstance::find(126500);
$instance->getConnection()->transaction(static function () use ($instance): void {
Workflow::lockForInstance($instance);
Workflow::manualIntentLocked($instance, 'action', 'start');
});
$applies = CloudEngineClient::$applies; $count = count(CloudEngineClient::$commands);
CloudEngineClient::$vpcRead = static fn () => $full;
$tick($op);
$assert($phase($op) === 'completed' && Workflow::state($op->fresh())['resume_cancelled']
&& CloudEngineClient::$applies === $applies && count(CloudEngineClient::$commands) === $count,
'Configuration polling overrode a newer manual power intent.');
foreach (['missing_host', 'host_digest', 'host_failed'] as $offset => $fault) {
[$op] = $partialDetach(1266 + $offset, false, $fault);
$state = Workflow::state($op->fresh());
$assert($state['phase'] === 'paused' && !$state['config_confirmed']
&& !isset($state['configuration_deadline']) && CloudEngineClient::$power === 'shutoff',
'Invalid partial proof opened an automatic waiting window: ' . $fault);
}
CloudEngineClient::$vpcRespond = $ordinaryVpcRespond;
CloudEngineClient::$vpcRead = null;
// Recover an interrupted apply from the exact durable engine receipt before
// the stopped-guest mutation precondition. A manual start needs no second stop.
$interrupted = static function (int $id, bool $paused = true) use ($make): CloudVpcOperation {
@ -733,6 +896,25 @@ CloudVpcService::reconcile(CloudVpc::find($op->vpc_id)->vpc_key, $actor, true, t
$assert($phase($op) === 'completed' && CloudEngineClient::$applies === $applies + 1,
'Missing receipt prevented legitimate stopped-guest configuration.');
// A rejected first submission has no accepted partial receipt. Preserve the
// original explicit-retry lane instead of fencing it behind a permanent GET.
$op = $interrupted(1270);
CloudEngineClient::$power = 'shutoff';
CloudEngineClient::$vpcRead = null;
CloudEngineClient::$vpcRespond = static function (): array {
throw new CloudError('rejected before apply', 422);
};
$applies = CloudEngineClient::$applies;
CloudVpcService::reconcile(CloudVpc::find($op->vpc_id)->vpc_key, $actor, true, true);
$assert($phase($op) === 'paused'
&& !isset(Workflow::state($op->fresh())['configuration_deadline'])
&& CloudEngineClient::$applies === $applies + 1,
'A rejected first POST acquired an accepted-configuration fence.');
CloudEngineClient::$vpcRespond = $ordinaryVpcRespond;
CloudVpcService::reconcile(CloudVpc::find($op->vpc_id)->vpc_key, $actor, true, true);
$assert($phase($op) === 'completed' && CloudEngineClient::$applies === $applies + 2,
'Explicit retry could not resubmit after a definite first-POST rejection.');
// New manual intent during the read wins even if the original guest ran.
$op = $interrupted(1252);
$state = Workflow::state($op); $state['original_running'] = true; $state['resume_cancelled'] = false;

Loading…
Cancel
Save