fix(vpc): recover completed member applies before power preflight

venix
admin 1 month ago
parent d8debc5f3e
commit 48ecd03f5a

@ -1,5 +1,30 @@
# 本版验证记录
## 2026-10-14:VPC 加入暂停后的回执恢复
- 修复运行中的成员已在引擎完成配置,但 PHP `applying`/暂停步骤在读取回执前
要求再次停机的问题。仅对已进入 apply 的工作流先 GET 原 VPC 回执;严格匹配
operation、revision、digest,并复用完整宿主/成员证据校验。404 或合法旧版本
才回到原停机门;不完整证明不完成流程,错误目标/摘要/新版本不重新派发。
- 回执恢复保留 root、operation、instance 身份与租约保护,以及后续手动电源意图。
读回执失败时保留原 `paused_from`,避免重复暂停产生非法步骤。配置证明确认后
才进入既有开机恢复逻辑;原本停机或已手动接管的实例不发送电源命令。
- 宿主完整配置已验证但来宾停止时,Python 继续输出已验证的 `acl_revision`;
运行可用性仍等待来宾证据。旧 revision、错误 digest、缺少 nft 证明仍拒绝确认。
- 在线更新器纳入 `virt-cloud-vpc-worker.service`,随已安装且正在运行的服务停止,
引擎健康后恢复,避免源码更新后独立 PHP daemon 继续运行旧类。
- Python 3.12:`test_vpc.py`、`test_vpc_dataplane.py`、`test_update_worker.py`、
`test_update_preflight_coordination.py`、`test_updater.py` 合计 **192 passed、
20 subtests passed**,24 条已有警告。新增 ACL/更新服务测试已在旧实现复现失败。
- PHP 8.5 WASM/SQLite:`CloudVpcPowerWorkflowTest`、`CloudVpcFailedReconcileTest`、
`CloudVpcRootDeletionTest`、`CloudVpcTombstoneConvergenceTest`、
`CloudVpcMutationAdmissionTest`、`CloudPowerReceiptWorkflowTest`、
`CloudVpcControlPlaneContractTest`、`CloudVpcMemberCandidateRecoveryTest`
**8 项脚本通过**。新增暂停回执测试在 `d8debc5` 的独立源码快照失败。
覆盖无额外 apply/电源调用、证据缺失、读失败、租约/版本/实例变化和手动接管。
- 本地测试不替代 MariaDB 并发、真实宿主加入/退出或跨宿主流量验收。
现场恢复应保留原操作,以完整 readback 收敛;不得直接把数据库状态改为成功。
## 2026-10-14:VPC 自动加入/退出及默认组内 ACL
- 新版页面确认短暂中断后提交 `auto_power: true`。VPC 操作独立保存电源进度:

@ -24,6 +24,12 @@ PHP 的 VPC 工作流负责保存进度,Python 引擎仍负责 libvirt 调用
来宾地址和路由核对继续走既有观测流程,不把来宾尚未就绪变成普通电源任务的锁。
退出时只有原有严格删除证据成立,才释放成员和私网 IP;不能仅凭关机回执释放。
若流程已经进入应用配置步骤,但回执丢失或因手动电源操作暂停,“核对 / 重试”
先只读查询原 VPC 的精确回执。只有相同操作编号、目标版本、摘要和完整宿主/
成员证据成立,才能直接完成配置步骤,不要求已经开机的实例再次关机。证据不全
仍需走原停机保护;用户后续的手动电源意图始终优先。已暂停步骤的读取失败
保留原恢复位置,不改变实例电源。
## 默认私网互通
每个新 VPC 在分配 `/24` 的同一事务中创建两条可编辑 ACL:
@ -49,6 +55,10 @@ PHP 的 VPC 工作流负责保存进度,Python 引擎仍负责 libvirt 调用
索引和短事务用于缩小数据库锁的范围;具体部署仍需检查执行计划与并发行为,
参见 [InnoDB 锁定读取与扫描范围](https://dev.mysql.com/doc/refman/8.0/en/innodb-locks-set.html)。
在线更新必须同时停止并重新加载已运行的 `virt-cloud-vpc-worker.service`;
这个 PHP 常驻进程不会自动重新加载更新后的类。新版更新器已将其纳入服务管理,
并在引擎健康后恢复。手动部署也需确认其启动时间晚于源码更新。
本地回归和模拟引擎验证不能代替真实宿主验收。部署后应分别验收运行和停止
实例的加入、退出,同宿主和跨宿主的 `/24` 双向通信,正常关机超时后的手动
强制断电,以及 worker 重启后继续核对原操作。还需确认公网配置保持不变、

@ -47,14 +47,7 @@ class UpdateWorkerTests(unittest.TestCase):
deployment = object.__new__(update_worker.DeploymentTransaction)
deployment.repo_dir = Path("/tmp")
deployment.config = {"service_secret": "unused"}
deployment.units = (
"virt-cloud-engine.service",
"virt-cloud-worker.service",
"virt-cloud-lifecycle.service",
"virt-cloud-policy-worker@1.service",
"virt-cloud-policy-worker@2.service",
"php-fpm.service",
)
deployment.units = update_worker._service_units(deployment.config)
def start(units, *, message):
calls.append(("start", units, message))
@ -71,6 +64,7 @@ class UpdateWorkerTests(unittest.TestCase):
engine = ("virt-cloud-engine.service",)
consumers = (
"virt-cloud-worker.service",
"virt-cloud-vpc-worker.service",
"virt-cloud-lifecycle.service",
"virt-cloud-policy-worker@1.service",
"virt-cloud-policy-worker@2.service",
@ -92,14 +86,7 @@ class UpdateWorkerTests(unittest.TestCase):
deployment = object.__new__(update_worker.DeploymentTransaction)
deployment.repo_dir = Path("/tmp")
deployment.config = {"service_secret": "unused"}
deployment.units = (
"virt-cloud-engine.service",
"virt-cloud-worker.service",
"virt-cloud-lifecycle.service",
"virt-cloud-policy-worker@1.service",
"virt-cloud-policy-worker@2.service",
"php-fpm.service",
)
deployment.units = update_worker._service_units(deployment.config)
def start(units, *, message):
calls.append(("start", units, message))
@ -121,6 +108,7 @@ class UpdateWorkerTests(unittest.TestCase):
self.assertEqual(calls[4], ("health", deployment.config))
self.assertEqual(calls[5][:2], ("start", (
"virt-cloud-worker.service",
"virt-cloud-vpc-worker.service",
"virt-cloud-lifecycle.service",
"virt-cloud-policy-worker@1.service",
"virt-cloud-policy-worker@2.service",
@ -303,6 +291,7 @@ class UpdateWorkerTests(unittest.TestCase):
states = {
"virt-cloud-engine.service": ("loaded", "active"),
"virt-cloud-worker.service": ("loaded", "active"),
"virt-cloud-vpc-worker.service": ("not-found", "inactive"),
"virt-cloud-lifecycle.service": ("not-found", "inactive"),
"virt-cloud-policy-worker@1.service": ("not-found", "inactive"),
"virt-cloud-policy-worker@2.service": ("not-found", "inactive"),
@ -325,6 +314,35 @@ class UpdateWorkerTests(unittest.TestCase):
"php-fpm.service",
))
def test_normal_update_quiesces_only_a_running_vpc_worker(self):
vpc_unit = "virt-cloud-vpc-worker.service"
for vpc_state in ("active", "inactive"):
with self.subTest(vpc_state=vpc_state):
inspected = []
def systemctl(argv, **_kwargs):
unit = argv[-1]
inspected.append(unit)
active = vpc_state if unit == vpc_unit else "active"
return SimpleNamespace(
returncode=0,
stdout=f"LoadState=loaded\nActiveState={active}\n",
)
with patch.object(update_worker, "_executable", return_value="/usr/bin/systemctl"), \
patch.object(update_worker.subprocess, "run", side_effect=systemctl), \
patch.object(update_worker, "_run") as run:
deployment = update_worker.DeploymentTransaction(Path("/tmp"), {})
deployment.quiesce()
self.assertIn(vpc_unit, inspected)
self.assertEqual(vpc_unit in deployment.units, vpc_state == "active")
run.assert_called_once()
command = run.call_args.args[0]
self.assertEqual(command[:2], ["/usr/bin/systemctl", "stop"])
self.assertEqual(command[2:], list(deployment.units))
self.assertEqual(vpc_unit in command, vpc_state == "active")
def test_recovery_starts_installed_units_even_when_they_are_stopped(self):
states = {
unit: ("loaded", "inactive") for unit in update_worker._service_units({})
@ -341,6 +359,7 @@ class UpdateWorkerTests(unittest.TestCase):
patch.object(update_worker.subprocess, "run", side_effect=systemctl):
units = update_worker._managed_service_units(Path("/tmp"), {}, recovery=True)
self.assertEqual(units, update_worker._service_units({}))
self.assertIn("virt-cloud-vpc-worker.service", units)
def test_update_refuses_before_stopping_when_required_service_is_missing(self):
def systemctl(argv, **_kwargs):

@ -1286,6 +1286,59 @@ def _active_observation_markers(plan):
return markers
@pytest.mark.parametrize("acl_proof", ("exact", "old_revision", "wrong_digest", "missing_nft"))
def test_stopped_host_reports_acl_revision_only_after_exact_configuration_proof(acl_proof):
plan = _plan()
member = plan.local_members[0]
routes = private_guest_routes(plan.vpc, member)
persistent = patch_private_interface_xml(
"""<domain><metadata/><devices>
<interface type='bridge'><mac address='52:54:00:00:00:01'/>
<alias name='ua-public0'/><source bridge='br0'/></interface>
</devices></domain>""",
member=member, resources=plan.resources,
guest_network_generation=private_guest_network_generation(member, routes),
vpc_id=plan.vpc.vpc_id, desired_revision=plan.desired_revision,
desired_digest=plan.desired_digest,
)
markers = _active_observation_markers(plan)
invalid_markers = {
"old_revision": ("__VC_VPC_ACL_REVISION__=", "0"),
"wrong_digest": ("__VC_VPC_ACL_DIGEST__=", "00" * 32),
"missing_nft": ("__VC_VPC_NFT_TABLE__=", "missing"),
}
if acl_proof in invalid_markers:
prefix, value = invalid_markers[acl_proof]
markers = [prefix + value if item.startswith(prefix) else item for item in markers]
adapter = VpcPrivateHostAdapter(object(), {})
adapter._deployment_contract = lambda: ("networkmanager", "", "", "")
adapter._run = lambda _script: "\n".join(markers) + "\n"
observed = adapter.observe(plan, xml_by_instance={
member.instance_uuid: {
"inactive": persistent, "live": None, "seed_verified": True,
"guest_network": {
"state": "stopped", "verified": False,
"reason_code": "vpc_private_guest_stopped",
},
},
})
configuration_verified = acl_proof == "exact"
assert observed["config_verified"] is configuration_verified
assert observed["config_observed_revision"] == (
plan.desired_revision if configuration_verified else 0)
assert observed["acl_revision"] == (
plan.desired_revision if configuration_verified else None)
assert observed["state"] == "applying"
assert observed["observed_revision"] == 0
assert observed["observed_digest"] is None
assert observed["readback"] == "pending_guest"
assert observed["member_nics"][0]["domain_running"] is False
assert observed["member_nics"][0]["config_verified"] is True
assert observed["member_nics"][0]["guest_network_verified"] is False
def test_observer_does_not_activate_unavailable_or_mismatched_guest():
plan = _plan()
member = plan.local_members[0]
@ -1322,6 +1375,7 @@ def test_observer_does_not_activate_unavailable_or_mismatched_guest():
},
})
assert observed["config_verified"] is True
assert observed["acl_revision"] == plan.desired_revision
assert observed["state"] == "applying"
assert observed["observed_revision"] == 0
assert observed["observed_digest"] is None

@ -36,6 +36,7 @@ _ENGINE_SERVICE_UNIT = "virt-cloud-engine.service"
_CORE_SERVICE_UNITS = (
_ENGINE_SERVICE_UNIT,
"virt-cloud-worker.service",
"virt-cloud-vpc-worker.service",
"virt-cloud-lifecycle.service",
"virt-cloud-policy-worker@1.service",
"virt-cloud-policy-worker@2.service",

@ -5151,7 +5151,9 @@ class VpcPrivateHostAdapter:
plan.resources.bridge,
parsed.get("bridge_owner"), members, tuple(parsed.get("routes", [])),
projected_wg, parsed.get("nft_table"),
plan.desired_revision if active else None,
# ACL installation is host configuration; a stopped or unready
# guest must not erase its exact revision/digest proof.
plan.desired_revision if host_config_verified else None,
parsed.get("anti_spoof") == "yes", rp_filter, proxy_arp,
proxy_arp_pvlan,
tuple(sorted(actual_proxy_ips)),

@ -251,19 +251,26 @@ final class CloudVpcPowerWorkflow
if ($state === null) {
return true;
}
$instance = CloudInstance::where('id', $state['instance_id'])->lockForUpdate()->first();
if ($instance === null || (string) $instance->uuid !== $state['uuid']
|| (int) $instance->host_id !== $state['host_id']) {
throw new CloudError('VPC 实例身份已变化', 409, 'vpc_power_target_changed');
}
if (CloudJob::where('instance_id', $instance->id)
->where('id', '>', $state['baseline_job_id'])->exists()) {
$state['resume_cancelled'] = true;
}
$instance = self::lockInstance($state);
return $callback($root, $operation, $instance, $state);
}); // Never replay an HTTP side effect by retrying a DB transaction.
}
/** Caller already holds the root and operation locks. */
private static function lockInstance(array &$state): CloudInstance
{
$instance = CloudInstance::where('id', $state['instance_id'])->lockForUpdate()->first();
if ($instance === null || (string) $instance->uuid !== $state['uuid']
|| (int) $instance->host_id !== $state['host_id']) {
throw new CloudError('VPC 实例身份已变化', 409, 'vpc_power_target_changed');
}
if (CloudJob::where('instance_id', $instance->id)
->where('id', '>', $state['baseline_job_id'])->exists()) {
$state['resume_cancelled'] = true;
}
return $instance;
}
private static function powerState(CloudEngineClient $client, array $actor, array $state): string
{
$result = $client->host($state['host_id'], 'GET', 'vms/' . $state['uuid'] . '/state', $actor);
@ -277,7 +284,9 @@ final class CloudVpcPowerWorkflow
private static function pause(CloudVpc $root, CloudVpcOperation $operation, array $state,
string $code): void
{
$state['paused_from'] = $state['phase'];
if ($state['phase'] !== 'paused') {
$state['paused_from'] = $state['phase'];
}
$state['phase'] = 'paused';
$state['error_code'] = $code;
self::release($operation, $state);
@ -354,15 +363,26 @@ final class CloudVpcPowerWorkflow
}
/** Called under recordObserved's root/operation locks after strict proof. */
public static function configurationObserved(CloudVpcOperation $operation, bool $confirmed): bool
public static function configurationObserved(CloudVpcOperation $operation, bool $confirmed,
bool $recoverPausedApply = false): bool
{
$state = self::state($operation);
if ($state === null || $state['phase'] === 'completed') {
return false;
}
if ($confirmed && $state['phase'] === 'applying') {
if ($recoverPausedApply) {
// Receipt recovery bypasses the stopped-guest preflight, but must
// retain its identity lock and every newer manual power intent.
// recordObserved's transaction rolls back all evidence on mismatch.
self::lockInstance($state);
}
if ($confirmed && ($state['phase'] === 'applying'
|| ($recoverPausedApply && $state['phase'] === 'paused'
&& $state['paused_from'] === 'applying'))) {
$state['config_confirmed'] = true;
$state['phase'] = 'configured';
$state['error_code'] = null;
unset($state['paused_from']);
}
self::saveState($operation, $state);
return true;

@ -1092,6 +1092,15 @@ final class CloudVpcService
[$operation, $payload, $runtimeProofRefresh, $previousOutcomePending] = $claim;
$leaseToken = (string) $operation->lease_token;
try {
// A previous apply may have finished while its PHP receipt was
// lost. Recover that exact proof before requiring a stopped guest
// for another mutation; a newer manual start must remain intact.
if (self::recoverPowerConfiguration(
$operation, $payload, $leaseToken, $actor, $allowFailedRetry
)) {
CloudVpcPowerWorkflow::afterApply($operation, $leaseToken, $actor);
return self::show($vpcKey, $actor);
}
if (!CloudVpcPowerWorkflow::beforeApply(
$operation, $leaseToken, $actor, $allowFailedRetry
)) {
@ -1131,53 +1140,7 @@ final class CloudVpcService
null, self::ENGINE_TIMEOUT_SECONDS
))->vpc('POST', '/internal/v1/vpcs/apply',
$actor, $payload, (string) $operation->operation_id);
$expectedRevision = (int) ($payload['desired_revision'] ?? 0);
$expectedDigest = (string) ($payload['desired_digest'] ?? '');
if (!is_array($result)
|| (string) ($result['operation_id'] ?? '')
!== (string) $operation->operation_id
|| (string) ($result['vpc_id'] ?? '')
!== (string) ($payload['vpc_id'] ?? '')
|| (int) ($result['desired_revision'] ?? 0) !== $expectedRevision
|| !hash_equals($expectedDigest, (string) ($result['desired_digest'] ?? ''))) {
throw new CloudError('引擎 VPC 结果与操作编号不一致,正在核对', 502);
}
$observed = is_array($result['observed'] ?? null) ? $result['observed'] : $result;
// Some engine versions expose a compact `observed` projection and
// the full typed host readback beside it. Merge the latter into
// the bounded projection before validating; never discard
// member_nics/routes/WireGuard evidence merely because the
// transport envelope used a compact view.
if (is_array($result['hosts'] ?? null)) {
if (!is_array($observed['hosts'] ?? null)) {
$observed['hosts'] = [];
}
foreach ($result['hosts'] as $hostKey => $hostItem) {
if (!is_array($hostItem)) {
continue;
}
$hostKey = (string) ($hostItem['host_id'] ?? $hostKey);
$compact = is_array($observed['hosts'][$hostKey] ?? null)
? $observed['hosts'][$hostKey] : [];
$observed['hosts'][$hostKey] = array_replace($compact, $hostItem);
}
}
if (isset($observed['operation_id'])
&& (string) $observed['operation_id'] !== (string) $operation->operation_id) {
throw new CloudError('引擎 VPC 观测与操作编号不一致,正在核对', 502);
}
if (isset($observed['vpc_id'])
&& (string) $observed['vpc_id'] !== (string) ($payload['vpc_id'] ?? '') ) {
throw new CloudError('引擎 VPC 观测目标不一致,正在核对', 502);
}
if (isset($observed['desired_revision'])
&& (int) $observed['desired_revision'] !== $expectedRevision) {
throw new CloudError('引擎 VPC 观测版本不一致,正在核对', 502);
}
if (isset($observed['desired_digest'])
&& !hash_equals($expectedDigest, (string) $observed['desired_digest'])) {
throw new CloudError('引擎 VPC 观测摘要不一致,正在核对', 502);
}
$observed = self::applyObservation($operation, $payload, $result);
self::recordObserved($vpcKey, (int) ($payload['desired_revision'] ?? 0),
(string) ($payload['desired_digest'] ?? ''), $observed,
(string) $operation->operation_id, $leaseToken);
@ -1464,9 +1427,98 @@ final class CloudVpcService
}, 3);
}
/** Read only the immutable target of an interrupted member apply. */
private static function recoverPowerConfiguration(CloudVpcOperation $operation,
array $payload, string $lease, array $actor, bool $explicitRetry): bool
{
$state = CloudVpcPowerWorkflow::state($operation);
if ($state === null || !($state['phase'] === 'applying'
|| ($explicitRetry && $state['phase'] === 'paused'
&& $state['paused_from'] === 'applying'))) {
return false;
}
try {
$result = (new CloudEngineClient(null, 20))->vpc(
'GET', '/internal/v1/vpcs/' . $payload['vpc_id'], $actor
);
} catch (CloudError $error) {
if ($error->httpStatus === 404) {
return false; // No receipt: the original stopped-guest gate applies.
}
throw $error;
}
if (($result['vpc_id'] ?? null) === $payload['vpc_id']
&& is_int($result['desired_revision'] ?? null)
&& $result['desired_revision'] > 0
&& $result['desired_revision'] < $payload['desired_revision']) {
return false; // This target has not reached the engine yet.
}
$observed = self::applyObservation($operation, $payload, $result);
self::recordObserved($payload['vpc_id'], $payload['desired_revision'],
$payload['desired_digest'], $observed, (string) $operation->operation_id,
$lease, true);
$operation->refresh();
return (CloudVpcPowerWorkflow::state($operation)['config_confirmed'] ?? false) === true;
}
/** GET recovery and POST apply must consume the same exact evidence. */
private static function applyObservation(CloudVpcOperation $operation,
array $payload, array $result): array
{
$expectedRevision = (int) ($payload['desired_revision'] ?? 0);
$expectedDigest = (string) ($payload['desired_digest'] ?? '');
if (!is_array($result)
|| (string) ($result['operation_id'] ?? '')
!== (string) $operation->operation_id
|| (string) ($result['vpc_id'] ?? '')
!== (string) ($payload['vpc_id'] ?? '')
|| ($result['desired_revision'] ?? null) !== $expectedRevision
|| !hash_equals($expectedDigest, (string) ($result['desired_digest'] ?? ''))) {
throw new CloudError('引擎 VPC 结果与操作编号不一致,正在核对', 502);
}
$observed = is_array($result['observed'] ?? null) ? $result['observed'] : $result;
// Some engine versions expose a compact `observed` projection and
// the full typed host readback beside it. Merge the latter into
// the bounded projection before validating; never discard
// member_nics/routes/WireGuard evidence merely because the
// transport envelope used a compact view.
if (is_array($result['hosts'] ?? null)) {
if (!is_array($observed['hosts'] ?? null)) {
$observed['hosts'] = [];
}
foreach ($result['hosts'] as $hostKey => $hostItem) {
if (!is_array($hostItem)) {
continue;
}
$hostKey = (string) ($hostItem['host_id'] ?? $hostKey);
$compact = is_array($observed['hosts'][$hostKey] ?? null)
? $observed['hosts'][$hostKey] : [];
$observed['hosts'][$hostKey] = array_replace($compact, $hostItem);
}
}
if (isset($observed['operation_id'])
&& (string) $observed['operation_id'] !== (string) $operation->operation_id) {
throw new CloudError('引擎 VPC 观测与操作编号不一致,正在核对', 502);
}
if (isset($observed['vpc_id'])
&& (string) $observed['vpc_id'] !== (string) ($payload['vpc_id'] ?? '') ) {
throw new CloudError('引擎 VPC 观测目标不一致,正在核对', 502);
}
if (isset($observed['desired_revision'])
&& $observed['desired_revision'] !== $expectedRevision) {
throw new CloudError('引擎 VPC 观测版本不一致,正在核对', 502);
}
if (isset($observed['desired_digest'])
&& !hash_equals($expectedDigest, (string) $observed['desired_digest'])) {
throw new CloudError('引擎 VPC 观测摘要不一致,正在核对', 502);
}
return $observed;
}
/** Apply an exact engine observation and enforce the all-host gate. */
public static function recordObserved(string $vpcKey, int $revision, string $digest,
array $observed, string $operationId, ?string $leaseToken = null): void
array $observed, string $operationId, ?string $leaseToken = null,
bool $recoverPowerConfiguration = false): void
{
self::assertKey($vpcKey, 'VPC');
if ($revision < 1 || preg_match('/^[a-f0-9]{64}$/D', $digest) !== 1
@ -1475,7 +1527,8 @@ final class CloudVpcService
}
$connection = (new CloudVpc())->getConnection();
$connection->transaction(function () use (
$vpcKey, $revision, $digest, $observed, $operationId, $leaseToken
$vpcKey, $revision, $digest, $observed, $operationId, $leaseToken,
$recoverPowerConfiguration
): void {
$vpc = CloudVpc::where('vpc_key', $vpcKey)->lockForUpdate()->first();
if ($vpc === null) {
@ -2394,7 +2447,9 @@ final class CloudVpcService
$operation->updated_at = time();
$operation->finished_at = ($operationSucceeded || $dataPlaneDeferred || $terminalFailure)
? time() : null;
$powerPending = CloudVpcPowerWorkflow::configurationObserved($operation, $operationSucceeded);
$powerPending = CloudVpcPowerWorkflow::configurationObserved(
$operation, $operationSucceeded, $recoverPowerConfiguration
);
if ($powerPending && $leaseToken !== null) {
$operation->status = 'running';
$operation->finished_at = null;

@ -10,6 +10,9 @@ namespace App\Services\Cloud {
public static bool $failState = false;
public static $vpcRespond;
public static int $applies = 0;
public static int $readbacks = 0;
public static $vpcRead;
public function __construct(...$args) {}
public function host(int $host, string $method, string $path, array $actor,
?array $body = null, ?string $operationId = null): array
@ -39,8 +42,13 @@ namespace App\Services\Cloud {
?? throw new CloudError('missing', 404);
}
public function vpc(string $method, string $path, array $actor,
?array $body, ?string $operationId): array
?array $body = null, ?string $operationId = null): array
{
if ($method === 'GET') {
self::$readbacks++;
return self::$vpcRead !== null ? (self::$vpcRead)($path)
: throw new CloudError('missing', 404);
}
self::$applies++;
return (self::$vpcRespond)($body);
}
@ -359,6 +367,136 @@ $assert($phase($detach) === 'completed' && $detach->fresh()->status === 'succeed
&& count(CloudEngineClient::$commands) === $count,
'Last-member paused start could not recover its original receipt.');
// Recover an interrupted apply from the exact durable engine receipt before
// the stopped-guest mutation precondition. A manual start needs no second stop.
$interrupted = static function (int $id, bool $paused = true) use ($make): CloudVpcOperation {
$operation = $make($id, 'running');
$state = Workflow::state($operation);
$state['phase'] = $paused ? 'paused' : 'applying';
$state['paused_from'] = 'applying';
$state['original_running'] = false;
$state['resume_cancelled'] = true;
$state['error_code'] = 'vpc_power_manual_override';
$operation->power_workflow_json = json_encode($state);
$operation->status = $paused ? 'paused' : 'waiting_observed';
$operation->lease_token = null; $operation->lease_until = 0;
$operation->save();
return $operation;
};
$receipt = static function (CloudVpcOperation $operation): array {
$result = (CloudEngineClient::$vpcRespond)(json_decode($operation->payload_json, true));
$result['observed']['state'] = 'active';
$result['observed']['next_stage_allowed'] = true;
$result['observed']['private_dataplane_verified'] = true;
// Exercise the engine's real compact observed + full top-level hosts DTO.
$result['hosts'] = $result['observed']['hosts'];
$result['observed']['hosts'] = [];
return $result;
};
foreach ([true, false] as $offset => $paused) {
$op = $interrupted(1220 + $offset, $paused);
$proof = $receipt($op);
CloudEngineClient::$vpcRead = static fn () => $proof;
$applies = CloudEngineClient::$applies; $reads = CloudEngineClient::$readbacks;
$count = count(CloudEngineClient::$commands);
CloudEngineClient::$failState = true;
CloudVpcService::reconcile(CloudVpc::find($op->vpc_id)->vpc_key, $actor, true, $paused);
CloudEngineClient::$failState = false;
$root = CloudVpc::find($op->vpc_id);
$assert($phase($op) === 'completed' && $op->fresh()->status === 'succeeded'
&& $root->state === 'active' && $root->observed_revision === $root->desired_revision
&& CloudEngineClient::$readbacks === $reads + 1
&& CloudEngineClient::$applies === $applies && count(CloudEngineClient::$commands) === $count
&& Workflow::state($op->fresh())['error_code'] === null,
'Exact interrupted apply did not recover without another mutation/power query.');
}
// A read failure/mismatch must not corrupt paused_from or trigger a new apply.
foreach (['missing', 'older', 'offline', 'vpc', 'operation', 'digest', 'newer', 'invalid_revision',
'host', 'nic', 'acl', 'fractional_revision'] as $offset => $fault) {
$op = $interrupted(1230 + $offset);
$proof = $receipt($op);
switch ($fault) {
case 'vpc': $proof['vpc_id'] = str_repeat('f', 32); break;
case 'operation': $proof['operation_id'] = str_repeat('f', 32); break;
case 'digest': $proof['desired_digest'] = str_repeat('f', 64); break;
case 'older': $proof['desired_revision']--; break;
case 'newer': $proof['desired_revision']++; break;
case 'invalid_revision': $proof['desired_revision'] = 'garbage'; break;
case 'fractional_revision': $proof['desired_revision'] += 0.9; break;
case 'host': $proof['hosts'] = []; break;
case 'nic': $proof['hosts']['10']['member_nics'] = []; break;
case 'acl': $proof['hosts']['10']['acl_revision'] = null; break;
}
CloudEngineClient::$vpcRead = static function () use ($proof, $fault): array {
if (in_array($fault, ['missing', 'offline'], true)) {
throw new CloudError('read failed', $fault === 'missing' ? 404 : 502);
}
return $proof;
};
$applies = CloudEngineClient::$applies; $count = count(CloudEngineClient::$commands);
CloudVpcService::reconcile(CloudVpc::find($op->vpc_id)->vpc_key, $actor, true, true);
$state = Workflow::state($op->fresh());
$assert($phase($op) === 'paused' && $state['paused_from'] === 'applying'
&& !$state['config_confirmed'] && CloudEngineClient::$applies === $applies
&& count(CloudEngineClient::$commands) === $count,
'Invalid/missing receipt bypassed stopped guest gate or corrupted phase: ' . $fault);
}
// The same absent receipt may proceed through the original mutation lane only
// after the guest is stopped. Unverified running proof never authorizes apply.
$op = $interrupted(1251);
CloudEngineClient::$power = 'shutoff';
CloudEngineClient::$vpcRead = null;
$applies = CloudEngineClient::$applies;
CloudVpcService::reconcile(CloudVpc::find($op->vpc_id)->vpc_key, $actor, true, true);
$assert($phase($op) === 'completed' && CloudEngineClient::$applies === $applies + 1,
'Missing receipt prevented legitimate stopped-guest configuration.');
// New manual intent during the read wins even if the original guest ran.
$op = $interrupted(1252);
$state = Workflow::state($op); $state['original_running'] = true; $state['resume_cancelled'] = false;
$op->power_workflow_json = json_encode($state); $op->save();
$proof = $receipt($op);
CloudEngineClient::$vpcRead = static function () use ($op, $proof): array {
$instance = CloudInstance::find(125200);
$instance->getConnection()->transaction(static function () use ($instance): void {
Workflow::lockForInstance($instance);
Workflow::manualIntentLocked($instance, 'action', 'start');
});
return $proof;
};
$count = count(CloudEngineClient::$commands);
CloudVpcService::reconcile(CloudVpc::find($op->vpc_id)->vpc_key, $actor, true, true);
$assert($phase($op) === 'completed' && count(CloudEngineClient::$commands) === $count
&& Workflow::state($op->fresh())['resume_cancelled'], 'Receipt recovery overrode concurrent manual intent.');
// A late GET cannot publish evidence after ownership or instance identity moves.
foreach (['lease', 'revision', 'instance'] as $offset => $change) {
$op = $interrupted(1253 + $offset);
$proof = $receipt($op); $root = CloudVpc::find($op->vpc_id);
$previous = [$root->observed_revision, $root->observed_digest];
CloudEngineClient::$vpcRead = static function () use ($op, $proof, $change): array {
if ($change === 'lease') {
Capsule::table('cloud_vpc_operations')->where('id', $op->id)->update(['lease_token' => str_repeat('f', 32)]);
} elseif ($change === 'revision') {
Capsule::table('cloud_vpcs')->where('id', $op->vpc_id)->increment('desired_revision');
} else {
Capsule::table('cloud_instances')->where('id', $op->power_instance_id)->update(['host_id' => 11]);
}
return $proof;
};
$applies = CloudEngineClient::$applies; $count = count(CloudEngineClient::$commands);
try { CloudVpcService::reconcile($root->vpc_key, $actor, true, true); }
catch (CloudError $error) { /* Stale owner is deliberately unable to release its successor's lease. */ }
$root->refresh();
$assert([$root->observed_revision, $root->observed_digest] === $previous
&& !Workflow::state($op->fresh())['config_confirmed']
&& CloudEngineClient::$applies === $applies && count(CloudEngineClient::$commands) === $count,
'Late receipt updated evidence after a changed ' . $change);
}
CloudEngineClient::$vpcRead = null;
// An installed pre-workflow schema continues to admit ordinary power, but
// cannot accept a new automatic maintenance request.
$instance = CloudInstance::find(121000);

Loading…
Cancel
Save