From 48ecd03f5a20e96119ce447d60f44f3694ba18e8 Mon Sep 17 00:00:00 2001 From: admin Date: Wed, 14 Oct 2026 01:45:08 +0800 Subject: [PATCH] fix(vpc): recover completed member applies before power preflight --- docs/VALIDATION.md | 25 +++ docs/VPC_AUTOMATIC_MEMBERSHIP.md | 10 ++ engine/tests/test_update_worker.py | 51 ++++-- engine/tests/test_vpc_dataplane.py | 54 ++++++ engine/virsh_panel/update_worker.py | 1 + engine/virsh_panel/vpc_dataplane.py | 4 +- .../Services/Cloud/CloudVpcPowerWorkflow.php | 44 +++-- web/app/Services/Cloud/CloudVpcService.php | 155 ++++++++++++------ web/tests/CloudVpcPowerWorkflowTest.php | 140 +++++++++++++++- 9 files changed, 404 insertions(+), 80 deletions(-) diff --git a/docs/VALIDATION.md b/docs/VALIDATION.md index 06437fb2..05abbc8e 100644 --- a/docs/VALIDATION.md +++ b/docs/VALIDATION.md @@ -1,5 +1,30 @@ # 本版验证记录 +## 2026-10-14:VPC 加入暂停后的回执恢复 + +- 修复运行中的成员已在引擎完成配置,但 PHP `applying`/暂停步骤在读取回执前 + 要求再次停机的问题。仅对已进入 apply 的工作流先 GET 原 VPC 回执;严格匹配 + operation、revision、digest,并复用完整宿主/成员证据校验。404 或合法旧版本 + 才回到原停机门;不完整证明不完成流程,错误目标/摘要/新版本不重新派发。 +- 回执恢复保留 root、operation、instance 身份与租约保护,以及后续手动电源意图。 + 读回执失败时保留原 `paused_from`,避免重复暂停产生非法步骤。配置证明确认后 + 才进入既有开机恢复逻辑;原本停机或已手动接管的实例不发送电源命令。 +- 宿主完整配置已验证但来宾停止时,Python 继续输出已验证的 `acl_revision`; + 运行可用性仍等待来宾证据。旧 revision、错误 digest、缺少 nft 证明仍拒绝确认。 +- 在线更新器纳入 `virt-cloud-vpc-worker.service`,随已安装且正在运行的服务停止, + 引擎健康后恢复,避免源码更新后独立 PHP daemon 继续运行旧类。 +- Python 3.12:`test_vpc.py`、`test_vpc_dataplane.py`、`test_update_worker.py`、 + `test_update_preflight_coordination.py`、`test_updater.py` 合计 **192 passed、 + 20 subtests passed**,24 条已有警告。新增 ACL/更新服务测试已在旧实现复现失败。 +- PHP 8.5 WASM/SQLite:`CloudVpcPowerWorkflowTest`、`CloudVpcFailedReconcileTest`、 + `CloudVpcRootDeletionTest`、`CloudVpcTombstoneConvergenceTest`、 + `CloudVpcMutationAdmissionTest`、`CloudPowerReceiptWorkflowTest`、 + `CloudVpcControlPlaneContractTest`、`CloudVpcMemberCandidateRecoveryTest` + **8 项脚本通过**。新增暂停回执测试在 `d8debc5` 的独立源码快照失败。 + 覆盖无额外 apply/电源调用、证据缺失、读失败、租约/版本/实例变化和手动接管。 +- 本地测试不替代 MariaDB 并发、真实宿主加入/退出或跨宿主流量验收。 + 现场恢复应保留原操作,以完整 readback 收敛;不得直接把数据库状态改为成功。 + ## 2026-10-14:VPC 自动加入/退出及默认组内 ACL - 新版页面确认短暂中断后提交 `auto_power: true`。VPC 操作独立保存电源进度: diff --git a/docs/VPC_AUTOMATIC_MEMBERSHIP.md b/docs/VPC_AUTOMATIC_MEMBERSHIP.md index 057a7ecd..f03c6438 100644 --- a/docs/VPC_AUTOMATIC_MEMBERSHIP.md +++ b/docs/VPC_AUTOMATIC_MEMBERSHIP.md @@ -24,6 +24,12 @@ PHP 的 VPC 工作流负责保存进度,Python 引擎仍负责 libvirt 调用 来宾地址和路由核对继续走既有观测流程,不把来宾尚未就绪变成普通电源任务的锁。 退出时只有原有严格删除证据成立,才释放成员和私网 IP;不能仅凭关机回执释放。 +若流程已经进入应用配置步骤,但回执丢失或因手动电源操作暂停,“核对 / 重试” +先只读查询原 VPC 的精确回执。只有相同操作编号、目标版本、摘要和完整宿主/ +成员证据成立,才能直接完成配置步骤,不要求已经开机的实例再次关机。证据不全 +仍需走原停机保护;用户后续的手动电源意图始终优先。已暂停步骤的读取失败 +保留原恢复位置,不改变实例电源。 + ## 默认私网互通 每个新 VPC 在分配 `/24` 的同一事务中创建两条可编辑 ACL: @@ -49,6 +55,10 @@ PHP 的 VPC 工作流负责保存进度,Python 引擎仍负责 libvirt 调用 索引和短事务用于缩小数据库锁的范围;具体部署仍需检查执行计划与并发行为, 参见 [InnoDB 锁定读取与扫描范围](https://dev.mysql.com/doc/refman/8.0/en/innodb-locks-set.html)。 +在线更新必须同时停止并重新加载已运行的 `virt-cloud-vpc-worker.service`; +这个 PHP 常驻进程不会自动重新加载更新后的类。新版更新器已将其纳入服务管理, +并在引擎健康后恢复。手动部署也需确认其启动时间晚于源码更新。 + 本地回归和模拟引擎验证不能代替真实宿主验收。部署后应分别验收运行和停止 实例的加入、退出,同宿主和跨宿主的 `/24` 双向通信,正常关机超时后的手动 强制断电,以及 worker 重启后继续核对原操作。还需确认公网配置保持不变、 diff --git a/engine/tests/test_update_worker.py b/engine/tests/test_update_worker.py index 38ae796c..60a901e4 100644 --- a/engine/tests/test_update_worker.py +++ b/engine/tests/test_update_worker.py @@ -47,14 +47,7 @@ class UpdateWorkerTests(unittest.TestCase): deployment = object.__new__(update_worker.DeploymentTransaction) deployment.repo_dir = Path("/tmp") deployment.config = {"service_secret": "unused"} - deployment.units = ( - "virt-cloud-engine.service", - "virt-cloud-worker.service", - "virt-cloud-lifecycle.service", - "virt-cloud-policy-worker@1.service", - "virt-cloud-policy-worker@2.service", - "php-fpm.service", - ) + deployment.units = update_worker._service_units(deployment.config) def start(units, *, message): calls.append(("start", units, message)) @@ -71,6 +64,7 @@ class UpdateWorkerTests(unittest.TestCase): engine = ("virt-cloud-engine.service",) consumers = ( "virt-cloud-worker.service", + "virt-cloud-vpc-worker.service", "virt-cloud-lifecycle.service", "virt-cloud-policy-worker@1.service", "virt-cloud-policy-worker@2.service", @@ -92,14 +86,7 @@ class UpdateWorkerTests(unittest.TestCase): deployment = object.__new__(update_worker.DeploymentTransaction) deployment.repo_dir = Path("/tmp") deployment.config = {"service_secret": "unused"} - deployment.units = ( - "virt-cloud-engine.service", - "virt-cloud-worker.service", - "virt-cloud-lifecycle.service", - "virt-cloud-policy-worker@1.service", - "virt-cloud-policy-worker@2.service", - "php-fpm.service", - ) + deployment.units = update_worker._service_units(deployment.config) def start(units, *, message): calls.append(("start", units, message)) @@ -121,6 +108,7 @@ class UpdateWorkerTests(unittest.TestCase): self.assertEqual(calls[4], ("health", deployment.config)) self.assertEqual(calls[5][:2], ("start", ( "virt-cloud-worker.service", + "virt-cloud-vpc-worker.service", "virt-cloud-lifecycle.service", "virt-cloud-policy-worker@1.service", "virt-cloud-policy-worker@2.service", @@ -303,6 +291,7 @@ class UpdateWorkerTests(unittest.TestCase): states = { "virt-cloud-engine.service": ("loaded", "active"), "virt-cloud-worker.service": ("loaded", "active"), + "virt-cloud-vpc-worker.service": ("not-found", "inactive"), "virt-cloud-lifecycle.service": ("not-found", "inactive"), "virt-cloud-policy-worker@1.service": ("not-found", "inactive"), "virt-cloud-policy-worker@2.service": ("not-found", "inactive"), @@ -325,6 +314,35 @@ class UpdateWorkerTests(unittest.TestCase): "php-fpm.service", )) + def test_normal_update_quiesces_only_a_running_vpc_worker(self): + vpc_unit = "virt-cloud-vpc-worker.service" + for vpc_state in ("active", "inactive"): + with self.subTest(vpc_state=vpc_state): + inspected = [] + + def systemctl(argv, **_kwargs): + unit = argv[-1] + inspected.append(unit) + active = vpc_state if unit == vpc_unit else "active" + return SimpleNamespace( + returncode=0, + stdout=f"LoadState=loaded\nActiveState={active}\n", + ) + + with patch.object(update_worker, "_executable", return_value="/usr/bin/systemctl"), \ + patch.object(update_worker.subprocess, "run", side_effect=systemctl), \ + patch.object(update_worker, "_run") as run: + deployment = update_worker.DeploymentTransaction(Path("/tmp"), {}) + deployment.quiesce() + + self.assertIn(vpc_unit, inspected) + self.assertEqual(vpc_unit in deployment.units, vpc_state == "active") + run.assert_called_once() + command = run.call_args.args[0] + self.assertEqual(command[:2], ["/usr/bin/systemctl", "stop"]) + self.assertEqual(command[2:], list(deployment.units)) + self.assertEqual(vpc_unit in command, vpc_state == "active") + def test_recovery_starts_installed_units_even_when_they_are_stopped(self): states = { unit: ("loaded", "inactive") for unit in update_worker._service_units({}) @@ -341,6 +359,7 @@ class UpdateWorkerTests(unittest.TestCase): patch.object(update_worker.subprocess, "run", side_effect=systemctl): units = update_worker._managed_service_units(Path("/tmp"), {}, recovery=True) self.assertEqual(units, update_worker._service_units({})) + self.assertIn("virt-cloud-vpc-worker.service", units) def test_update_refuses_before_stopping_when_required_service_is_missing(self): def systemctl(argv, **_kwargs): diff --git a/engine/tests/test_vpc_dataplane.py b/engine/tests/test_vpc_dataplane.py index e125a747..a066013a 100644 --- a/engine/tests/test_vpc_dataplane.py +++ b/engine/tests/test_vpc_dataplane.py @@ -1286,6 +1286,59 @@ def _active_observation_markers(plan): return markers +@pytest.mark.parametrize("acl_proof", ("exact", "old_revision", "wrong_digest", "missing_nft")) +def test_stopped_host_reports_acl_revision_only_after_exact_configuration_proof(acl_proof): + plan = _plan() + member = plan.local_members[0] + routes = private_guest_routes(plan.vpc, member) + persistent = patch_private_interface_xml( + """ + + + """, + member=member, resources=plan.resources, + guest_network_generation=private_guest_network_generation(member, routes), + vpc_id=plan.vpc.vpc_id, desired_revision=plan.desired_revision, + desired_digest=plan.desired_digest, + ) + markers = _active_observation_markers(plan) + invalid_markers = { + "old_revision": ("__VC_VPC_ACL_REVISION__=", "0"), + "wrong_digest": ("__VC_VPC_ACL_DIGEST__=", "00" * 32), + "missing_nft": ("__VC_VPC_NFT_TABLE__=", "missing"), + } + if acl_proof in invalid_markers: + prefix, value = invalid_markers[acl_proof] + markers = [prefix + value if item.startswith(prefix) else item for item in markers] + adapter = VpcPrivateHostAdapter(object(), {}) + adapter._deployment_contract = lambda: ("networkmanager", "", "", "") + adapter._run = lambda _script: "\n".join(markers) + "\n" + + observed = adapter.observe(plan, xml_by_instance={ + member.instance_uuid: { + "inactive": persistent, "live": None, "seed_verified": True, + "guest_network": { + "state": "stopped", "verified": False, + "reason_code": "vpc_private_guest_stopped", + }, + }, + }) + + configuration_verified = acl_proof == "exact" + assert observed["config_verified"] is configuration_verified + assert observed["config_observed_revision"] == ( + plan.desired_revision if configuration_verified else 0) + assert observed["acl_revision"] == ( + plan.desired_revision if configuration_verified else None) + assert observed["state"] == "applying" + assert observed["observed_revision"] == 0 + assert observed["observed_digest"] is None + assert observed["readback"] == "pending_guest" + assert observed["member_nics"][0]["domain_running"] is False + assert observed["member_nics"][0]["config_verified"] is True + assert observed["member_nics"][0]["guest_network_verified"] is False + + def test_observer_does_not_activate_unavailable_or_mismatched_guest(): plan = _plan() member = plan.local_members[0] @@ -1322,6 +1375,7 @@ def test_observer_does_not_activate_unavailable_or_mismatched_guest(): }, }) assert observed["config_verified"] is True + assert observed["acl_revision"] == plan.desired_revision assert observed["state"] == "applying" assert observed["observed_revision"] == 0 assert observed["observed_digest"] is None diff --git a/engine/virsh_panel/update_worker.py b/engine/virsh_panel/update_worker.py index cdac405d..51965bec 100644 --- a/engine/virsh_panel/update_worker.py +++ b/engine/virsh_panel/update_worker.py @@ -36,6 +36,7 @@ _ENGINE_SERVICE_UNIT = "virt-cloud-engine.service" _CORE_SERVICE_UNITS = ( _ENGINE_SERVICE_UNIT, "virt-cloud-worker.service", + "virt-cloud-vpc-worker.service", "virt-cloud-lifecycle.service", "virt-cloud-policy-worker@1.service", "virt-cloud-policy-worker@2.service", diff --git a/engine/virsh_panel/vpc_dataplane.py b/engine/virsh_panel/vpc_dataplane.py index 8091e142..7b3bd8df 100644 --- a/engine/virsh_panel/vpc_dataplane.py +++ b/engine/virsh_panel/vpc_dataplane.py @@ -5151,7 +5151,9 @@ class VpcPrivateHostAdapter: plan.resources.bridge, parsed.get("bridge_owner"), members, tuple(parsed.get("routes", [])), projected_wg, parsed.get("nft_table"), - plan.desired_revision if active else None, + # ACL installation is host configuration; a stopped or unready + # guest must not erase its exact revision/digest proof. + plan.desired_revision if host_config_verified else None, parsed.get("anti_spoof") == "yes", rp_filter, proxy_arp, proxy_arp_pvlan, tuple(sorted(actual_proxy_ips)), diff --git a/web/app/Services/Cloud/CloudVpcPowerWorkflow.php b/web/app/Services/Cloud/CloudVpcPowerWorkflow.php index 1fbf60dc..e06f4e49 100644 --- a/web/app/Services/Cloud/CloudVpcPowerWorkflow.php +++ b/web/app/Services/Cloud/CloudVpcPowerWorkflow.php @@ -251,19 +251,26 @@ final class CloudVpcPowerWorkflow if ($state === null) { return true; } - $instance = CloudInstance::where('id', $state['instance_id'])->lockForUpdate()->first(); - if ($instance === null || (string) $instance->uuid !== $state['uuid'] - || (int) $instance->host_id !== $state['host_id']) { - throw new CloudError('VPC 实例身份已变化', 409, 'vpc_power_target_changed'); - } - if (CloudJob::where('instance_id', $instance->id) - ->where('id', '>', $state['baseline_job_id'])->exists()) { - $state['resume_cancelled'] = true; - } + $instance = self::lockInstance($state); return $callback($root, $operation, $instance, $state); }); // Never replay an HTTP side effect by retrying a DB transaction. } + /** Caller already holds the root and operation locks. */ + private static function lockInstance(array &$state): CloudInstance + { + $instance = CloudInstance::where('id', $state['instance_id'])->lockForUpdate()->first(); + if ($instance === null || (string) $instance->uuid !== $state['uuid'] + || (int) $instance->host_id !== $state['host_id']) { + throw new CloudError('VPC 实例身份已变化', 409, 'vpc_power_target_changed'); + } + if (CloudJob::where('instance_id', $instance->id) + ->where('id', '>', $state['baseline_job_id'])->exists()) { + $state['resume_cancelled'] = true; + } + return $instance; + } + private static function powerState(CloudEngineClient $client, array $actor, array $state): string { $result = $client->host($state['host_id'], 'GET', 'vms/' . $state['uuid'] . '/state', $actor); @@ -277,7 +284,9 @@ final class CloudVpcPowerWorkflow private static function pause(CloudVpc $root, CloudVpcOperation $operation, array $state, string $code): void { - $state['paused_from'] = $state['phase']; + if ($state['phase'] !== 'paused') { + $state['paused_from'] = $state['phase']; + } $state['phase'] = 'paused'; $state['error_code'] = $code; self::release($operation, $state); @@ -354,15 +363,26 @@ final class CloudVpcPowerWorkflow } /** Called under recordObserved's root/operation locks after strict proof. */ - public static function configurationObserved(CloudVpcOperation $operation, bool $confirmed): bool + public static function configurationObserved(CloudVpcOperation $operation, bool $confirmed, + bool $recoverPausedApply = false): bool { $state = self::state($operation); if ($state === null || $state['phase'] === 'completed') { return false; } - if ($confirmed && $state['phase'] === 'applying') { + if ($recoverPausedApply) { + // Receipt recovery bypasses the stopped-guest preflight, but must + // retain its identity lock and every newer manual power intent. + // recordObserved's transaction rolls back all evidence on mismatch. + self::lockInstance($state); + } + if ($confirmed && ($state['phase'] === 'applying' + || ($recoverPausedApply && $state['phase'] === 'paused' + && $state['paused_from'] === 'applying'))) { $state['config_confirmed'] = true; $state['phase'] = 'configured'; + $state['error_code'] = null; + unset($state['paused_from']); } self::saveState($operation, $state); return true; diff --git a/web/app/Services/Cloud/CloudVpcService.php b/web/app/Services/Cloud/CloudVpcService.php index 1ffc902e..7a8c7c1b 100644 --- a/web/app/Services/Cloud/CloudVpcService.php +++ b/web/app/Services/Cloud/CloudVpcService.php @@ -1092,6 +1092,15 @@ final class CloudVpcService [$operation, $payload, $runtimeProofRefresh, $previousOutcomePending] = $claim; $leaseToken = (string) $operation->lease_token; try { + // A previous apply may have finished while its PHP receipt was + // lost. Recover that exact proof before requiring a stopped guest + // for another mutation; a newer manual start must remain intact. + if (self::recoverPowerConfiguration( + $operation, $payload, $leaseToken, $actor, $allowFailedRetry + )) { + CloudVpcPowerWorkflow::afterApply($operation, $leaseToken, $actor); + return self::show($vpcKey, $actor); + } if (!CloudVpcPowerWorkflow::beforeApply( $operation, $leaseToken, $actor, $allowFailedRetry )) { @@ -1131,53 +1140,7 @@ final class CloudVpcService null, self::ENGINE_TIMEOUT_SECONDS ))->vpc('POST', '/internal/v1/vpcs/apply', $actor, $payload, (string) $operation->operation_id); - $expectedRevision = (int) ($payload['desired_revision'] ?? 0); - $expectedDigest = (string) ($payload['desired_digest'] ?? ''); - if (!is_array($result) - || (string) ($result['operation_id'] ?? '') - !== (string) $operation->operation_id - || (string) ($result['vpc_id'] ?? '') - !== (string) ($payload['vpc_id'] ?? '') - || (int) ($result['desired_revision'] ?? 0) !== $expectedRevision - || !hash_equals($expectedDigest, (string) ($result['desired_digest'] ?? ''))) { - throw new CloudError('引擎 VPC 结果与操作编号不一致,正在核对', 502); - } - $observed = is_array($result['observed'] ?? null) ? $result['observed'] : $result; - // Some engine versions expose a compact `observed` projection and - // the full typed host readback beside it. Merge the latter into - // the bounded projection before validating; never discard - // member_nics/routes/WireGuard evidence merely because the - // transport envelope used a compact view. - if (is_array($result['hosts'] ?? null)) { - if (!is_array($observed['hosts'] ?? null)) { - $observed['hosts'] = []; - } - foreach ($result['hosts'] as $hostKey => $hostItem) { - if (!is_array($hostItem)) { - continue; - } - $hostKey = (string) ($hostItem['host_id'] ?? $hostKey); - $compact = is_array($observed['hosts'][$hostKey] ?? null) - ? $observed['hosts'][$hostKey] : []; - $observed['hosts'][$hostKey] = array_replace($compact, $hostItem); - } - } - if (isset($observed['operation_id']) - && (string) $observed['operation_id'] !== (string) $operation->operation_id) { - throw new CloudError('引擎 VPC 观测与操作编号不一致,正在核对', 502); - } - if (isset($observed['vpc_id']) - && (string) $observed['vpc_id'] !== (string) ($payload['vpc_id'] ?? '') ) { - throw new CloudError('引擎 VPC 观测目标不一致,正在核对', 502); - } - if (isset($observed['desired_revision']) - && (int) $observed['desired_revision'] !== $expectedRevision) { - throw new CloudError('引擎 VPC 观测版本不一致,正在核对', 502); - } - if (isset($observed['desired_digest']) - && !hash_equals($expectedDigest, (string) $observed['desired_digest'])) { - throw new CloudError('引擎 VPC 观测摘要不一致,正在核对', 502); - } + $observed = self::applyObservation($operation, $payload, $result); self::recordObserved($vpcKey, (int) ($payload['desired_revision'] ?? 0), (string) ($payload['desired_digest'] ?? ''), $observed, (string) $operation->operation_id, $leaseToken); @@ -1464,9 +1427,98 @@ final class CloudVpcService }, 3); } + /** Read only the immutable target of an interrupted member apply. */ + private static function recoverPowerConfiguration(CloudVpcOperation $operation, + array $payload, string $lease, array $actor, bool $explicitRetry): bool + { + $state = CloudVpcPowerWorkflow::state($operation); + if ($state === null || !($state['phase'] === 'applying' + || ($explicitRetry && $state['phase'] === 'paused' + && $state['paused_from'] === 'applying'))) { + return false; + } + try { + $result = (new CloudEngineClient(null, 20))->vpc( + 'GET', '/internal/v1/vpcs/' . $payload['vpc_id'], $actor + ); + } catch (CloudError $error) { + if ($error->httpStatus === 404) { + return false; // No receipt: the original stopped-guest gate applies. + } + throw $error; + } + if (($result['vpc_id'] ?? null) === $payload['vpc_id'] + && is_int($result['desired_revision'] ?? null) + && $result['desired_revision'] > 0 + && $result['desired_revision'] < $payload['desired_revision']) { + return false; // This target has not reached the engine yet. + } + $observed = self::applyObservation($operation, $payload, $result); + self::recordObserved($payload['vpc_id'], $payload['desired_revision'], + $payload['desired_digest'], $observed, (string) $operation->operation_id, + $lease, true); + $operation->refresh(); + return (CloudVpcPowerWorkflow::state($operation)['config_confirmed'] ?? false) === true; + } + + /** GET recovery and POST apply must consume the same exact evidence. */ + private static function applyObservation(CloudVpcOperation $operation, + array $payload, array $result): array + { + $expectedRevision = (int) ($payload['desired_revision'] ?? 0); + $expectedDigest = (string) ($payload['desired_digest'] ?? ''); + if (!is_array($result) + || (string) ($result['operation_id'] ?? '') + !== (string) $operation->operation_id + || (string) ($result['vpc_id'] ?? '') + !== (string) ($payload['vpc_id'] ?? '') + || ($result['desired_revision'] ?? null) !== $expectedRevision + || !hash_equals($expectedDigest, (string) ($result['desired_digest'] ?? ''))) { + throw new CloudError('引擎 VPC 结果与操作编号不一致,正在核对', 502); + } + $observed = is_array($result['observed'] ?? null) ? $result['observed'] : $result; + // Some engine versions expose a compact `observed` projection and + // the full typed host readback beside it. Merge the latter into + // the bounded projection before validating; never discard + // member_nics/routes/WireGuard evidence merely because the + // transport envelope used a compact view. + if (is_array($result['hosts'] ?? null)) { + if (!is_array($observed['hosts'] ?? null)) { + $observed['hosts'] = []; + } + foreach ($result['hosts'] as $hostKey => $hostItem) { + if (!is_array($hostItem)) { + continue; + } + $hostKey = (string) ($hostItem['host_id'] ?? $hostKey); + $compact = is_array($observed['hosts'][$hostKey] ?? null) + ? $observed['hosts'][$hostKey] : []; + $observed['hosts'][$hostKey] = array_replace($compact, $hostItem); + } + } + if (isset($observed['operation_id']) + && (string) $observed['operation_id'] !== (string) $operation->operation_id) { + throw new CloudError('引擎 VPC 观测与操作编号不一致,正在核对', 502); + } + if (isset($observed['vpc_id']) + && (string) $observed['vpc_id'] !== (string) ($payload['vpc_id'] ?? '') ) { + throw new CloudError('引擎 VPC 观测目标不一致,正在核对', 502); + } + if (isset($observed['desired_revision']) + && $observed['desired_revision'] !== $expectedRevision) { + throw new CloudError('引擎 VPC 观测版本不一致,正在核对', 502); + } + if (isset($observed['desired_digest']) + && !hash_equals($expectedDigest, (string) $observed['desired_digest'])) { + throw new CloudError('引擎 VPC 观测摘要不一致,正在核对', 502); + } + return $observed; + } + /** Apply an exact engine observation and enforce the all-host gate. */ public static function recordObserved(string $vpcKey, int $revision, string $digest, - array $observed, string $operationId, ?string $leaseToken = null): void + array $observed, string $operationId, ?string $leaseToken = null, + bool $recoverPowerConfiguration = false): void { self::assertKey($vpcKey, 'VPC'); if ($revision < 1 || preg_match('/^[a-f0-9]{64}$/D', $digest) !== 1 @@ -1475,7 +1527,8 @@ final class CloudVpcService } $connection = (new CloudVpc())->getConnection(); $connection->transaction(function () use ( - $vpcKey, $revision, $digest, $observed, $operationId, $leaseToken + $vpcKey, $revision, $digest, $observed, $operationId, $leaseToken, + $recoverPowerConfiguration ): void { $vpc = CloudVpc::where('vpc_key', $vpcKey)->lockForUpdate()->first(); if ($vpc === null) { @@ -2394,7 +2447,9 @@ final class CloudVpcService $operation->updated_at = time(); $operation->finished_at = ($operationSucceeded || $dataPlaneDeferred || $terminalFailure) ? time() : null; - $powerPending = CloudVpcPowerWorkflow::configurationObserved($operation, $operationSucceeded); + $powerPending = CloudVpcPowerWorkflow::configurationObserved( + $operation, $operationSucceeded, $recoverPowerConfiguration + ); if ($powerPending && $leaseToken !== null) { $operation->status = 'running'; $operation->finished_at = null; diff --git a/web/tests/CloudVpcPowerWorkflowTest.php b/web/tests/CloudVpcPowerWorkflowTest.php index 994b7f93..52ee2eda 100644 --- a/web/tests/CloudVpcPowerWorkflowTest.php +++ b/web/tests/CloudVpcPowerWorkflowTest.php @@ -10,6 +10,9 @@ namespace App\Services\Cloud { public static bool $failState = false; public static $vpcRespond; public static int $applies = 0; + public static int $readbacks = 0; + public static $vpcRead; + public function __construct(...$args) {} public function host(int $host, string $method, string $path, array $actor, ?array $body = null, ?string $operationId = null): array @@ -39,8 +42,13 @@ namespace App\Services\Cloud { ?? throw new CloudError('missing', 404); } public function vpc(string $method, string $path, array $actor, - ?array $body, ?string $operationId): array + ?array $body = null, ?string $operationId = null): array { + if ($method === 'GET') { + self::$readbacks++; + return self::$vpcRead !== null ? (self::$vpcRead)($path) + : throw new CloudError('missing', 404); + } self::$applies++; return (self::$vpcRespond)($body); } @@ -359,6 +367,136 @@ $assert($phase($detach) === 'completed' && $detach->fresh()->status === 'succeed && count(CloudEngineClient::$commands) === $count, 'Last-member paused start could not recover its original receipt.'); +// Recover an interrupted apply from the exact durable engine receipt before +// the stopped-guest mutation precondition. A manual start needs no second stop. +$interrupted = static function (int $id, bool $paused = true) use ($make): CloudVpcOperation { + $operation = $make($id, 'running'); + $state = Workflow::state($operation); + $state['phase'] = $paused ? 'paused' : 'applying'; + $state['paused_from'] = 'applying'; + $state['original_running'] = false; + $state['resume_cancelled'] = true; + $state['error_code'] = 'vpc_power_manual_override'; + $operation->power_workflow_json = json_encode($state); + $operation->status = $paused ? 'paused' : 'waiting_observed'; + $operation->lease_token = null; $operation->lease_until = 0; + $operation->save(); + return $operation; +}; +$receipt = static function (CloudVpcOperation $operation): array { + $result = (CloudEngineClient::$vpcRespond)(json_decode($operation->payload_json, true)); + $result['observed']['state'] = 'active'; + $result['observed']['next_stage_allowed'] = true; + $result['observed']['private_dataplane_verified'] = true; + // Exercise the engine's real compact observed + full top-level hosts DTO. + $result['hosts'] = $result['observed']['hosts']; + $result['observed']['hosts'] = []; + return $result; +}; +foreach ([true, false] as $offset => $paused) { + $op = $interrupted(1220 + $offset, $paused); + $proof = $receipt($op); + CloudEngineClient::$vpcRead = static fn () => $proof; + $applies = CloudEngineClient::$applies; $reads = CloudEngineClient::$readbacks; + $count = count(CloudEngineClient::$commands); + CloudEngineClient::$failState = true; + CloudVpcService::reconcile(CloudVpc::find($op->vpc_id)->vpc_key, $actor, true, $paused); + CloudEngineClient::$failState = false; + $root = CloudVpc::find($op->vpc_id); + $assert($phase($op) === 'completed' && $op->fresh()->status === 'succeeded' + && $root->state === 'active' && $root->observed_revision === $root->desired_revision + && CloudEngineClient::$readbacks === $reads + 1 + && CloudEngineClient::$applies === $applies && count(CloudEngineClient::$commands) === $count + && Workflow::state($op->fresh())['error_code'] === null, + 'Exact interrupted apply did not recover without another mutation/power query.'); +} + +// A read failure/mismatch must not corrupt paused_from or trigger a new apply. +foreach (['missing', 'older', 'offline', 'vpc', 'operation', 'digest', 'newer', 'invalid_revision', + 'host', 'nic', 'acl', 'fractional_revision'] as $offset => $fault) { + $op = $interrupted(1230 + $offset); + $proof = $receipt($op); + switch ($fault) { + case 'vpc': $proof['vpc_id'] = str_repeat('f', 32); break; + case 'operation': $proof['operation_id'] = str_repeat('f', 32); break; + case 'digest': $proof['desired_digest'] = str_repeat('f', 64); break; + case 'older': $proof['desired_revision']--; break; + case 'newer': $proof['desired_revision']++; break; + case 'invalid_revision': $proof['desired_revision'] = 'garbage'; break; + case 'fractional_revision': $proof['desired_revision'] += 0.9; break; + case 'host': $proof['hosts'] = []; break; + case 'nic': $proof['hosts']['10']['member_nics'] = []; break; + case 'acl': $proof['hosts']['10']['acl_revision'] = null; break; + } + CloudEngineClient::$vpcRead = static function () use ($proof, $fault): array { + if (in_array($fault, ['missing', 'offline'], true)) { + throw new CloudError('read failed', $fault === 'missing' ? 404 : 502); + } + return $proof; + }; + $applies = CloudEngineClient::$applies; $count = count(CloudEngineClient::$commands); + CloudVpcService::reconcile(CloudVpc::find($op->vpc_id)->vpc_key, $actor, true, true); + $state = Workflow::state($op->fresh()); + $assert($phase($op) === 'paused' && $state['paused_from'] === 'applying' + && !$state['config_confirmed'] && CloudEngineClient::$applies === $applies + && count(CloudEngineClient::$commands) === $count, + 'Invalid/missing receipt bypassed stopped guest gate or corrupted phase: ' . $fault); +} + +// The same absent receipt may proceed through the original mutation lane only +// after the guest is stopped. Unverified running proof never authorizes apply. +$op = $interrupted(1251); +CloudEngineClient::$power = 'shutoff'; +CloudEngineClient::$vpcRead = null; +$applies = CloudEngineClient::$applies; +CloudVpcService::reconcile(CloudVpc::find($op->vpc_id)->vpc_key, $actor, true, true); +$assert($phase($op) === 'completed' && CloudEngineClient::$applies === $applies + 1, + 'Missing receipt prevented legitimate stopped-guest configuration.'); + +// New manual intent during the read wins even if the original guest ran. +$op = $interrupted(1252); +$state = Workflow::state($op); $state['original_running'] = true; $state['resume_cancelled'] = false; +$op->power_workflow_json = json_encode($state); $op->save(); +$proof = $receipt($op); +CloudEngineClient::$vpcRead = static function () use ($op, $proof): array { + $instance = CloudInstance::find(125200); + $instance->getConnection()->transaction(static function () use ($instance): void { + Workflow::lockForInstance($instance); + Workflow::manualIntentLocked($instance, 'action', 'start'); + }); + return $proof; +}; +$count = count(CloudEngineClient::$commands); +CloudVpcService::reconcile(CloudVpc::find($op->vpc_id)->vpc_key, $actor, true, true); +$assert($phase($op) === 'completed' && count(CloudEngineClient::$commands) === $count + && Workflow::state($op->fresh())['resume_cancelled'], 'Receipt recovery overrode concurrent manual intent.'); + +// A late GET cannot publish evidence after ownership or instance identity moves. +foreach (['lease', 'revision', 'instance'] as $offset => $change) { + $op = $interrupted(1253 + $offset); + $proof = $receipt($op); $root = CloudVpc::find($op->vpc_id); + $previous = [$root->observed_revision, $root->observed_digest]; + CloudEngineClient::$vpcRead = static function () use ($op, $proof, $change): array { + if ($change === 'lease') { + Capsule::table('cloud_vpc_operations')->where('id', $op->id)->update(['lease_token' => str_repeat('f', 32)]); + } elseif ($change === 'revision') { + Capsule::table('cloud_vpcs')->where('id', $op->vpc_id)->increment('desired_revision'); + } else { + Capsule::table('cloud_instances')->where('id', $op->power_instance_id)->update(['host_id' => 11]); + } + return $proof; + }; + $applies = CloudEngineClient::$applies; $count = count(CloudEngineClient::$commands); + try { CloudVpcService::reconcile($root->vpc_key, $actor, true, true); } + catch (CloudError $error) { /* Stale owner is deliberately unable to release its successor's lease. */ } + $root->refresh(); + $assert([$root->observed_revision, $root->observed_digest] === $previous + && !Workflow::state($op->fresh())['config_confirmed'] + && CloudEngineClient::$applies === $applies && count(CloudEngineClient::$commands) === $count, + 'Late receipt updated evidence after a changed ' . $change); +} +CloudEngineClient::$vpcRead = null; + // An installed pre-workflow schema continues to admit ordinary power, but // cannot accept a new automatic maintenance request. $instance = CloudInstance::find(121000);