From df53cea203ad174486706754becb461267a7e2e4 Mon Sep 17 00:00:00 2001 From: admin Date: Wed, 14 Oct 2026 04:04:01 +0800 Subject: [PATCH] fix: keep VPC power flow waiting for configuration proof --- docs/VALIDATION.md | 36 ++++ .../Services/Cloud/CloudVpcPowerWorkflow.php | 53 ++++- web/app/Services/Cloud/CloudVpcService.php | 53 ++++- web/tests/CloudVpcPowerWorkflowTest.php | 182 ++++++++++++++++++ 4 files changed, 313 insertions(+), 11 deletions(-) diff --git a/docs/VALIDATION.md b/docs/VALIDATION.md index a6940c86..5e64c137 100644 --- a/docs/VALIDATION.md +++ b/docs/VALIDATION.md @@ -1,5 +1,41 @@ # 本版验证记录 +## 2026-10-14:退出已关机后因部分配置回执提前暂停 + +- 现场 `vm-zf99er` 的退出操作 13 + `0e715a8a98a8babb0192724a5538b0c5`、revision 9 在正常关机后 + 暂停为 `vpc_power_configuration_unconfirmed`,`paused_from=applying`、 + `config_confirmed=false`,没有更新的手动电源任务。 +- 暂停时的原始回执中,目标宿主 2 已 `removed=true` 且精确确认 revision 9; + 宿主 1 的两个保留成员 NIC/来宾证明完整,但聚合的配置、传输证明尚未齐全。 + 后续引擎自行恢复 `active`、9/9、两个宿主配置验证通过;PHP 任务却仍 paused。 + 这与 Fabric 回读期间撤销旧 gate、随后重新验证的中间状态一致,不能据此认定 + 宿主故障。无远端成员时 `wg_allowed_ips=[]` 是合法配置。 +- 根因是 `afterApply()` 收到一次未齐的配置证明就终止自动流程,后台 worker + 不领取 paused 任务,因此引擎随后完成核对也无法触发原恢复开机步骤。 +- 在修改部署前,确认当前完整配置证明及无更新手动电源意图后,通过页面对原任务 + 执行一次“核对 / 重试”。原退出已完成,页面为 9/9、`applied`、错误为空, + 保留另两成员。真实宿主确认目标实例 `running`,运行及持久配置仅公网 `br0` + 网卡,原私网桥不存在;QGA 只返回公网地址。没有重新提交加入/退出或直接写库。 + 本次现场恢复使用既有显式核对路径,不能算作新补丁的自动恢复验收。 +- 最终原任务为 `succeeded/completed`、`config_confirmed=true`、 + `resume_cancelled=false`;正常关机子操作 + `8a6a2aea44c6b353722b79fb23afeb7d` 与恢复开机子操作 + `3e685cc7a888ae75586ee367b2fd5f69` 均 accepted,保持原固定 ID。 +- 修复仅将身份、版本、摘要及宿主集合匹配的合法中间回执留在 + `applying/waiting_observed`;后台 GET 继续核对原目标,不重复 POST。 + 未确认时自动等待上限为 120 秒,普通轮询不延长期限;到期仍缺证明则暂停。 + 完整原目标证明即使迟到,仍可经既有身份、手动电源意图和全宿主配置检查恢复。 + 已接收目标后的 404/旧 revision 只等待;尚未接收回执的原提交异常恢复逻辑不变。 +- PHP 8.5 WASM/SQLite 自动电源、失败核对、控制面、成员清理收敛及页面契约 + **5 项脚本通过**。新增真实 service/worker 回归覆盖双宿主首轮 partial 后 + 多轮 GET 到完整证明再开机、同宿主保留两个成员、缺回执、固定期限、超时、 + 显式恢复、更新的手动电源操作,以及错误宿主集合/摘要不得进入自动等待。 + 首次 POST 明确拒绝后仍可显式核对并按原契约重新提交。 +- 隔离副本仅把两处业务 PHP 恢复为 `eb38f9d`,新增双宿主回归在“部分回执 + 被提前暂停”断言失败;补丁版本通过。未改变 Engine、数据库结构或 active + 判据,没有为通过测试移除原有严格配置检查。 + ## 2026-10-14:自动开机后的 VPC 回读延迟 - 现场 `vm-zf99er` 加入 VPC `xx` 的 revision 7 操作 diff --git a/web/app/Services/Cloud/CloudVpcPowerWorkflow.php b/web/app/Services/Cloud/CloudVpcPowerWorkflow.php index 8005ca20..e0c47f57 100644 --- a/web/app/Services/Cloud/CloudVpcPowerWorkflow.php +++ b/web/app/Services/Cloud/CloudVpcPowerWorkflow.php @@ -13,6 +13,7 @@ use Illuminate\Database\QueryException; final class CloudVpcPowerWorkflow { public const SHUTDOWN_SECONDS = 120; + public const CONFIGURATION_SECONDS = 120; private static array $lockedRootIds = []; public static function requested(array $input): bool @@ -67,6 +68,9 @@ final class CloudVpcPowerWorkflow || ($state['start_id'] ?? null) !== substr(hash('sha256', $operation->operation_id . ':vpc:start'), 0, 32) || (isset($state['shutdown_deadline']) && (!is_int($state['shutdown_deadline']) || $state['shutdown_deadline'] < 1)) || (isset($state['start_deadline']) && (!is_int($state['start_deadline']) || $state['start_deadline'] < 1)) + || (isset($state['configuration_deadline']) + && (!is_int($state['configuration_deadline']) || $state['configuration_deadline'] < 1)) + || (isset($state['configuration_pending']) && !is_bool($state['configuration_pending'])) || (($state['phase'] ?? null) === 'paused' && !in_array($state['paused_from'] ?? null, ['preflight', 'shutting_down', 'waiting_stopped', 'applying', 'configured', 'starting'], true)) || !in_array($state['phase'] ?? null, ['preflight', 'shutting_down', @@ -414,7 +418,7 @@ final class CloudVpcPowerWorkflow /** Called under recordObserved's root/operation locks after strict proof. */ public static function configurationObserved(CloudVpcOperation $operation, bool $confirmed, - bool $recoverPausedApply = false): bool + bool $recoverPausedApply = false, bool $pending = false): bool { $state = self::state($operation); if ($state === null || $state['phase'] === 'completed') { @@ -426,18 +430,47 @@ final class CloudVpcPowerWorkflow // recordObserved's transaction rolls back all evidence on mismatch. self::lockInstance($state); } - if ($confirmed && ($state['phase'] === 'applying' + if ($state['phase'] === 'applying' || ($recoverPausedApply && $state['phase'] === 'paused' - && $state['paused_from'] === 'applying'))) { - $state['config_confirmed'] = true; - $state['phase'] = 'configured'; - $state['error_code'] = null; - unset($state['paused_from']); + && $state['paused_from'] === 'applying')) { + $state['configuration_pending'] = !$confirmed && $pending; + if ($confirmed || $pending) { + if ($pending && (!isset($state['configuration_deadline']) + || $state['phase'] === 'paused')) { + $state['configuration_deadline'] = time() + self::CONFIGURATION_SECONDS; + } + $state['config_confirmed'] = $confirmed; + $state['phase'] = $confirmed ? 'configured' : 'applying'; + $state['error_code'] = null; + unset($state['paused_from']); + } } self::saveState($operation, $state); return true; } + /** A previously accepted target must never become a second POST on GET 404. */ + public static function missingConfigurationReceipt(CloudVpcOperation $operation, + string $lease, bool $explicitRetry): bool + { + return self::locked($operation, $lease, + static function ($root, $row, $instance, $state) use ($explicitRetry): bool { + if (!isset($state['configuration_deadline'])) { + return false; + } + if ($explicitRetry && $state['phase'] === 'paused' + && $state['paused_from'] === 'applying') { + $state['phase'] = 'applying'; + $state['configuration_deadline'] = time() + self::CONFIGURATION_SECONDS; + $state['error_code'] = null; + unset($state['paused_from']); + } + $state['configuration_pending'] = true; + self::saveState($row, $state); + return true; + }); + } + /** Preflight failures have no remote mutation uncertainty of their own. */ public static function handleFailure(CloudVpcOperation $operation, string $lease, CloudError $error): bool @@ -465,6 +498,12 @@ final class CloudVpcPowerWorkflow $client ??= new CloudEngineClient(); $start = self::locked($operation, $lease, static function ($root, $row, $instance, $state): bool { if (!($state['config_confirmed'] ?? false)) { + if ($state['phase'] === 'applying' + && ($state['configuration_pending'] ?? false) === true + && time() < (int) ($state['configuration_deadline'] ?? 0)) { + self::release($row, $state); + return false; + } self::pause($root, $row, $state, 'vpc_power_configuration_unconfirmed'); return false; } diff --git a/web/app/Services/Cloud/CloudVpcService.php b/web/app/Services/Cloud/CloudVpcService.php index 7a8c7c1b..fe756c2a 100644 --- a/web/app/Services/Cloud/CloudVpcService.php +++ b/web/app/Services/Cloud/CloudVpcService.php @@ -1443,7 +1443,9 @@ final class CloudVpcService ); } catch (CloudError $error) { if ($error->httpStatus === 404) { - return false; // No receipt: the original stopped-guest gate applies. + return CloudVpcPowerWorkflow::missingConfigurationReceipt( + $operation, $lease, $explicitRetry + ); } throw $error; } @@ -1451,14 +1453,51 @@ final class CloudVpcService && is_int($result['desired_revision'] ?? null) && $result['desired_revision'] > 0 && $result['desired_revision'] < $payload['desired_revision']) { - return false; // This target has not reached the engine yet. + return CloudVpcPowerWorkflow::missingConfigurationReceipt( + $operation, $lease, $explicitRetry + ); } $observed = self::applyObservation($operation, $payload, $result); self::recordObserved($payload['vpc_id'], $payload['desired_revision'], $payload['desired_digest'], $observed, (string) $operation->operation_id, $lease, true); $operation->refresh(); - return (CloudVpcPowerWorkflow::state($operation)['config_confirmed'] ?? false) === true; + // Even a partial current receipt consumes this turn. Falling through + // would POST the same configuration again instead of waiting for its + // durable engine observation to finish. + return true; + } + + /** Partial progress is retryable, never a substitute for strict proof. */ + private static function pendingPowerConfiguration(array $observed, array $hosts, + CloudVpcOperation $operation, string $vpcKey, int $revision, string $digest): bool + { + if (($observed['evidence_version'] ?? null) !== self::OBSERVATION_EVIDENCE_VERSION + || !in_array($observed['state'] ?? null, ['applying', 'degraded'], true) + || ($observed['next_stage_allowed'] ?? null) !== false + || ($observed['private_dataplane_verified'] ?? null) !== false) { + return false; + } + $payload = json_decode((string) $operation->payload_json, true); + $expected = is_array($payload['hosts'] ?? null) ? $payload['hosts'] : []; + if ($expected === [] || count($expected) !== count($hosts)) { + return false; + } + foreach ($expected as $target) { + $hostId = (string) ($target['host_id'] ?? ''); + $item = $hosts[$hostId] ?? null; + $error = is_array($item) ? ($item['last_error_code'] ?? null) : null; + if (!is_array($item) || ($item['vpc_id'] ?? null) !== $vpcKey + || (string) ($item['host_id'] ?? '') !== $hostId + || ($item['desired_revision'] ?? null) !== $revision + || !self::digestMatches($digest, (string) ($item['desired_digest'] ?? '')) + || !in_array($item['state'] ?? null, ['active', 'applying', 'degraded'], true) + || ($error !== null && $error !== 'vpc_private_dataplane_unverified' + && self::deferredEvidenceError($error) === null)) { + return false; + } + } + return true; } /** GET recovery and POST apply must consume the same exact evidence. */ @@ -1567,6 +1606,11 @@ final class CloudVpcService $hostKey = (string) ($item['host_id'] ?? $key); $hosts[$hostKey] = $item; } + // Classify progress before validation replaces diagnostic fields. + // An active envelope with malformed proof remains an error. + $powerConfigurationPending = self::pendingPowerConfiguration( + $observed, $hosts, $operation, $vpcKey, $revision, $digest + ); $evidenceVersion = max(0, (int) ($observed['evidence_version'] ?? $observed['observation_version'] ?? 0)); $evidenceContractCurrent = @@ -2448,7 +2492,8 @@ final class CloudVpcService $operation->finished_at = ($operationSucceeded || $dataPlaneDeferred || $terminalFailure) ? time() : null; $powerPending = CloudVpcPowerWorkflow::configurationObserved( - $operation, $operationSucceeded, $recoverPowerConfiguration + $operation, $operationSucceeded, $recoverPowerConfiguration, + $powerConfigurationPending && $desiredContractValid && !$terminalGate ); if ($powerPending && $leaseToken !== null) { $operation->status = 'running'; diff --git a/web/tests/CloudVpcPowerWorkflowTest.php b/web/tests/CloudVpcPowerWorkflowTest.php index b7816f38..fd1f4ffc 100644 --- a/web/tests/CloudVpcPowerWorkflowTest.php +++ b/web/tests/CloudVpcPowerWorkflowTest.php @@ -647,6 +647,169 @@ $assert($phase($detach) === 'completed' && $detach->fresh()->status === 'succeed && count(CloudEngineClient::$commands) === $count, 'Last-member paused start could not recover its original receipt.'); +// The departing host can finish while a surviving host is still converging. +// Keep that exact accepted target in the GET lane until the complete proof +// arrives; a VM must not remain off merely because the first receipt is partial. +$ordinaryVpcRespond = CloudEngineClient::$vpcRespond; +$runningVpcProof = static function (array $payload) use ($ordinaryVpcRespond, $activeHostEvidence): array { + $result = $ordinaryVpcRespond($payload); + $hosts = []; + foreach ($result['hosts'] as $host) { + $hostId = (string) $host['host_id']; + $hosts[$hostId] = ($host['removed'] ?? false) === true ? $host + : $activeHostEvidence($payload['vpc_id'], (int) $hostId, + $payload['desired_revision'], $payload['desired_digest'], + $payload['cidr'], $payload['members']); + } + $result['state'] = $result['observed']['state'] = 'active'; + $result['next_stage_allowed'] = $result['observed']['next_stage_allowed'] = true; + $result['private_dataplane_verified'] = $result['observed']['private_dataplane_verified'] = true; + $result['hosts'] = array_values($hosts); + $result['observed']['hosts'] = $hosts; + return $result; +}; +$partialDetach = static function (int $id, bool $sameHost = false, ?string $fault = null) use ( + $make, $tick, $actor, $seedInstance, $ordinaryVpcRespond, $runningVpcProof, $assert +): array { + Capsule::table('cloud_vpc_operations')->update(['status' => 'superseded', 'lease_until' => 0]); + CloudEngineClient::$vpcRespond = $ordinaryVpcRespond; + CloudEngineClient::$vpcRead = null; + $attach = $make($id); + $tick($attach); + CloudEngineClient::$power = 'shutoff'; + $tick($attach); $tick($attach); + $vpc = CloudVpc::find($attach->vpc_id); + foreach ([1, 2] as $offset) { + $seedInstance($id * 100 + $offset, $sameHost ? 10 : 20); + CloudVpcService::attachMember($vpc->vpc_key, $actor, [ + 'instance_id' => $id * 100 + $offset, + 'request_key' => 'retained-sibling-' . $id . '-' . $offset, + ]); + CloudVpcService::reconcile($vpc->vpc_key, $actor, true, true); + } + $target = CloudVpcMember::where('instance_id', $id * 100)->first(); + CloudVpcService::detachMember($vpc->vpc_key, $target->member_key, $actor, [ + 'request_key' => 'partial-detach-' . $id, 'auto_power' => true, + ]); + $operation = CloudVpcOperation::where('vpc_id', $vpc->id)->orderByDesc('desired_revision')->first(); + $tick($operation); + CloudEngineClient::$power = 'shutoff'; + $full = $runningVpcProof(json_decode($operation->payload_json, true)); + $partial = $full; + $partial['state'] = $partial['observed']['state'] = 'degraded'; + $partial['next_stage_allowed'] = $partial['observed']['next_stage_allowed'] = false; + $partial['private_dataplane_verified'] = $partial['observed']['private_dataplane_verified'] = false; + $partial['observed']['fabric_gate'] = ['active' => false, 'available' => true]; + foreach ($partial['hosts'] as &$host) { + if (($host['removed'] ?? false) === true) { continue; } + $host['transport_verified'] = $host['config_verified'] = $host['private_dataplane_verified'] = false; + $host['observed_revision'] = $host['config_observed_revision'] = 0; + $host['observed_digest'] = $host['config_observed_digest'] = null; + $host['last_error_code'] = 'vpc_private_dataplane_unverified'; + $host['wg_allowed_ips'] = []; + } + unset($host); + $partial['observed']['hosts'] = []; + if ($fault === 'missing_host') { + array_pop($partial['hosts']); + } elseif ($fault === 'host_digest') { + $partial['hosts'][count($partial['hosts']) - 1]['desired_digest'] = str_repeat('f', 64); + } elseif ($fault === 'host_failed') { + $partial['hosts'][count($partial['hosts']) - 1]['last_error_code'] = 'vpc_private_dataplane_apply_failed'; + } + CloudEngineClient::$vpcRespond = static fn () => $partial; + $tick($operation); + return [$operation, $partial, $full]; +}; +[$op, $partial, $full] = $partialDetach(1260); +$state = Workflow::state($op->fresh()); +$deadline = $state['configuration_deadline'] ?? 0; +$applies = CloudEngineClient::$applies; $count = count(CloudEngineClient::$commands); +$assert($state['phase'] === 'applying' && !$state['config_confirmed'] && $deadline > time() + && $op->fresh()->status === 'waiting_observed' && $op->fresh()->lease_token === null + && CloudVpcMember::where('instance_id', 126000)->count() === 1, + 'The partial two-host detach paused or released an unconfirmed member.'); +foreach (['partial', 'missing', 'older'] as $mode) { + CloudEngineClient::$vpcRead = static function () use ($partial, $mode): array { + if ($mode === 'missing') { throw new CloudError('missing', 404); } + if ($mode === 'older') { $partial['desired_revision']--; } + return $partial; + }; + $tick($op); + $state = Workflow::state($op->fresh()); + $assert($state['phase'] === 'applying' && !$state['config_confirmed'] + && $state['configuration_deadline'] === $deadline + && $op->fresh()->status === 'waiting_observed' + && CloudEngineClient::$applies === $applies && count(CloudEngineClient::$commands) === $count, + 'Accepted configuration was resubmitted, restarted or extended on ' . $mode); +} +CloudEngineClient::$vpcRead = static fn () => $full; +$tick($op); +$assert($phase($op) === 'completed' && CloudEngineClient::$power === 'running' + && CloudVpcMember::where('instance_id', 126000)->count() === 0 + && CloudVpcMember::whereIn('instance_id', [126001, 126002])->where('state', 'active')->count() === 2 + && CloudEngineClient::$applies === $applies && count(CloudEngineClient::$commands) === $count + 1, + 'Later exact detach proof did not release only the target and restore it once.'); + +// The same mechanism also preserves siblings when the departing host stays live. +[$op, $partial, $full] = $partialDetach(1261, true); +$applies = CloudEngineClient::$applies; +CloudEngineClient::$vpcRead = static fn () => $full; +$tick($op); +$assert($phase($op) === 'completed' + && CloudVpcMember::where('instance_id', 126100)->count() === 0 + && CloudVpcMember::whereIn('instance_id', [126101, 126102])->where('state', 'active')->count() === 2 + && CloudEngineClient::$applies === $applies, + 'Same-host detach required removing the host or lost its retained siblings.'); + +foreach (['partial', 'missing', 'exact'] as $offset => $mode) { + [$op, $partial, $full] = $partialDetach(1262 + $offset); + $state = Workflow::state($op->fresh()); + $state['configuration_deadline'] = time() - 1; + $op->power_workflow_json = json_encode($state); $op->save(); + $applies = CloudEngineClient::$applies; $count = count(CloudEngineClient::$commands); + CloudEngineClient::$vpcRead = static function () use ($mode, $partial, $full): array { + if ($mode === 'missing') { throw new CloudError('missing', 404); } + return $mode === 'exact' ? $full : $partial; + }; + $tick($op); + $assert($phase($op) === ($mode === 'exact' ? 'completed' : 'paused') + && CloudEngineClient::$applies === $applies + && count(CloudEngineClient::$commands) === $count + ($mode === 'exact' ? 1 : 0), + 'Configuration deadline extended, resent a POST or discarded exact proof: ' . $mode); + if ($mode !== 'exact') { + $assert(Workflow::state($op->fresh())['configuration_deadline'] === $state['configuration_deadline'], + 'A deadline was refreshed by an ordinary worker poll.'); + CloudEngineClient::$vpcRead = static fn () => $partial; + CloudVpcService::reconcile(CloudVpc::find($op->vpc_id)->vpc_key, $actor, true, true); + $assert($phase($op) === 'applying' + && Workflow::state($op->fresh())['configuration_deadline'] > time() + && CloudEngineClient::$applies === $applies, + 'Explicit retry could not open a bounded read-only recovery window.'); + } +} +[$op, $partial, $full] = $partialDetach(1265); +$instance = CloudInstance::find(126500); +$instance->getConnection()->transaction(static function () use ($instance): void { + Workflow::lockForInstance($instance); + Workflow::manualIntentLocked($instance, 'action', 'start'); +}); +$applies = CloudEngineClient::$applies; $count = count(CloudEngineClient::$commands); +CloudEngineClient::$vpcRead = static fn () => $full; +$tick($op); +$assert($phase($op) === 'completed' && Workflow::state($op->fresh())['resume_cancelled'] + && CloudEngineClient::$applies === $applies && count(CloudEngineClient::$commands) === $count, + 'Configuration polling overrode a newer manual power intent.'); +foreach (['missing_host', 'host_digest', 'host_failed'] as $offset => $fault) { + [$op] = $partialDetach(1266 + $offset, false, $fault); + $state = Workflow::state($op->fresh()); + $assert($state['phase'] === 'paused' && !$state['config_confirmed'] + && !isset($state['configuration_deadline']) && CloudEngineClient::$power === 'shutoff', + 'Invalid partial proof opened an automatic waiting window: ' . $fault); +} +CloudEngineClient::$vpcRespond = $ordinaryVpcRespond; +CloudEngineClient::$vpcRead = null; + // Recover an interrupted apply from the exact durable engine receipt before // the stopped-guest mutation precondition. A manual start needs no second stop. $interrupted = static function (int $id, bool $paused = true) use ($make): CloudVpcOperation { @@ -733,6 +896,25 @@ CloudVpcService::reconcile(CloudVpc::find($op->vpc_id)->vpc_key, $actor, true, t $assert($phase($op) === 'completed' && CloudEngineClient::$applies === $applies + 1, 'Missing receipt prevented legitimate stopped-guest configuration.'); +// A rejected first submission has no accepted partial receipt. Preserve the +// original explicit-retry lane instead of fencing it behind a permanent GET. +$op = $interrupted(1270); +CloudEngineClient::$power = 'shutoff'; +CloudEngineClient::$vpcRead = null; +CloudEngineClient::$vpcRespond = static function (): array { + throw new CloudError('rejected before apply', 422); +}; +$applies = CloudEngineClient::$applies; +CloudVpcService::reconcile(CloudVpc::find($op->vpc_id)->vpc_key, $actor, true, true); +$assert($phase($op) === 'paused' + && !isset(Workflow::state($op->fresh())['configuration_deadline']) + && CloudEngineClient::$applies === $applies + 1, + 'A rejected first POST acquired an accepted-configuration fence.'); +CloudEngineClient::$vpcRespond = $ordinaryVpcRespond; +CloudVpcService::reconcile(CloudVpc::find($op->vpc_id)->vpc_key, $actor, true, true); +$assert($phase($op) === 'completed' && CloudEngineClient::$applies === $applies + 2, + 'Explicit retry could not resubmit after a definite first-POST rejection.'); + // New manual intent during the read wins even if the original guest ran. $op = $interrupted(1252); $state = Workflow::state($op); $state['original_running'] = true; $state['resume_cancelled'] = false;