diff --git a/web/app/Controllers/CloudAdminOperationsController.php b/web/app/Controllers/CloudAdminOperationsController.php index 41a56879..5580defe 100644 --- a/web/app/Controllers/CloudAdminOperationsController.php +++ b/web/app/Controllers/CloudAdminOperationsController.php @@ -101,4 +101,16 @@ final class CloudAdminOperationsController 202 )->withHeader('Cache-Control', 'private, no-store'); } + + public function retireVpcHost($request, $response, $args) + { + return $response->withJson( + CloudAdminOperationRecoveryService::retireVpcHost( + $request->getAttribute('cloud_actor') + ?? CloudAccess::actor(false), + (string) ($args['id'] ?? ''), + CloudInput::object($request) + ) + )->withHeader('Cache-Control', 'private, no-store'); + } } diff --git a/web/app/Services/Cloud/CloudAdminOperationRecoveryService.php b/web/app/Services/Cloud/CloudAdminOperationRecoveryService.php index 7f130079..9f8e53b0 100644 --- a/web/app/Services/Cloud/CloudAdminOperationRecoveryService.php +++ b/web/app/Services/Cloud/CloudAdminOperationRecoveryService.php @@ -563,6 +563,30 @@ final class CloudAdminOperationRecoveryService ]; } + /** Resolve one exact VPC removal after a physical host is retired. */ + public static function retireVpcHost( + array $actor, + string $id, + array $input + ): array { + CloudInput::only($input, [ + 'acknowledge_permanent_retirement', 'reason', + ]); + if (($input['acknowledge_permanent_retirement'] ?? null) !== true) { + throw new CloudError('必须明确确认宿主机已永久退役', 409); + } + $reason = $input['reason'] ?? null; + if (!is_string($reason)) { + throw new CloudError('必须填写可审计的宿主机退役依据'); + } + if (preg_match('/^[1-9][0-9]{0,17}$/D', $id) !== 1) { + throw new CloudError('编号无效', 400); + } + return CloudVpcHostRetirementService::retire( + $actor, (int) $id, $reason + ); + } + /** @return array{0:CloudJob,1:CloudAgentClient} */ private static function context( array $actor, diff --git a/web/app/Services/Cloud/CloudAdminOperationsService.php b/web/app/Services/Cloud/CloudAdminOperationsService.php index b098ec15..efcebe73 100644 --- a/web/app/Services/Cloud/CloudAdminOperationsService.php +++ b/web/app/Services/Cloud/CloudAdminOperationsService.php @@ -444,6 +444,8 @@ final class CloudAdminOperationsService [$canRepairVpc, $repairVpcReason] = self::vpcRepairable( $job, $canManage ); + [$canRetireVpcHost, $retireVpcHostReason] = + self::vpcHostRetireable($job, $canManage); $recovery = CloudJobService::recoveryPresentation($job, true); $effectScope = $recovery === null ? CloudJobService::effectScope($job) : null; @@ -483,6 +485,8 @@ final class CloudAdminOperationsService 'force_delete_reason' => $forceDeleteReason, 'can_repair_vpc' => $canRepairVpc, 'repair_vpc_reason' => $repairVpcReason, + 'can_retire_vpc_host' => $canRetireVpcHost, + 'retire_vpc_host_reason' => $retireVpcHostReason, 'supersedes_job_id' => $job->supersedes_job_id === null ? null : (string) $job->supersedes_job_id, 'dispatch_attempts' => $job->dispatch_attempts === null @@ -711,13 +715,44 @@ final class CloudAdminOperationsService || (string) $job->status !== 'failed' || !(bool) $job->effect_may_have_applied || (bool) $job->safe_to_resubmit - || (string) $job->effect_resolution !== 'unknown') { + || (string) $job->effect_resolution !== 'unknown' + || !CloudDispatchLedger::isDispatched($job) + || (int) ($job->dispatched_at ?? 0) < 1 + || CloudJobService::hasTerminalRecoveryStateConflict($job)) { return [false, 'VPC 任务不处于可修复的影响未知状态']; } return [true, '服务端会先读取宿主机新鲜证据,再创建具有新操作标识的精确修复任务']; } + /** @return array{0:bool,1:string} */ + private static function vpcHostRetireable($job, bool $canManage): array + { + if (!$canManage) { + return [false, '只有超级管理员可以确认宿主机永久退役']; + } + if ((string) $job->kind !== 'vpc_host_remove') { + return [false, '当前任务不是 VPC 宿主机清理']; + } + if (CloudDispatchLedger::isInvalid($job) + || CloudDispatchLedger::hasRepairIncident($job) + || (bool) ($job->finalization_incident ?? false) + || (string) $job->execution_mode !== 'agent' + || (string) $job->phase !== 'terminal' + || (string) $job->outcome !== 'failed' + || (string) $job->status !== 'failed' + || !(bool) $job->effect_may_have_applied + || (bool) $job->safe_to_resubmit + || (string) $job->effect_resolution !== 'unknown' + || !CloudDispatchLedger::isDispatched($job) + || (int) ($job->dispatched_at ?? 0) < 1 + || CloudJobService::hasTerminalRecoveryStateConflict($job)) { + return [false, 'VPC 宿主机清理不处于可退役状态']; + } + return [true, + '仅当宿主机、Agent 和调度均已停用,且实例、资源、迁移和 VPC 网卡全部清空时才会解除该清理栅栏']; + } + private static function timestamp($value): ?int { return $value !== null && (int) $value > 0 ? (int) $value : null; diff --git a/web/app/Services/Cloud/CloudVpcHostRetirementService.php b/web/app/Services/Cloud/CloudVpcHostRetirementService.php new file mode 100644 index 00000000..4f1427a2 --- /dev/null +++ b/web/app/Services/Cloud/CloudVpcHostRetirementService.php @@ -0,0 +1,347 @@ +where('id', $jobId)->first(); + if ($reference === null) { + throw new CloudError('任务不存在', 404); + } + self::assertReference($reference); + + return (new CloudJob())->getConnection()->transaction( + static function () use ($actor, $reference, $reason): array { + $hostInstanceIds = CloudInstance::query() + ->where('host_id', (int) $reference->host_id) + ->where('status', '<>', 'deleted')->pluck('id') + ->map(static fn ($id): int => (int) $id)->all(); + $instanceIds = array_values(array_unique(array_merge( + $hostInstanceIds, + [(int) $reference->instance_id] + ))); + $instances = CloudVpcLockOrder::instances($instanceIds); + $instance = $instances->get((int) $reference->instance_id); + $vpc = CloudVpcLockOrder::vpcById( + (int) $reference->vpc_id + ); + $port = CloudVpcLockOrder::portById( + (int) $reference->vpc_port_id + ); + $state = CloudVpcHostState::query() + ->where('vpc_id', (int) $reference->vpc_id) + ->where('host_id', (int) $reference->host_id) + ->lockForUpdate()->first(); + if ($instance === null || $vpc === null || $port === null + || $state === null) { + throw new CloudError('VPC 宿主机退役账本不完整', 409); + } + + $host = CloudHost::query() + ->where('id', (int) $reference->host_id) + ->lockForUpdate()->first(); + if ($host === null) { + throw new CloudError('宿主机不存在', 404); + } + self::assertHostEmptyLocked($host); + + // Coordination rows are last in the canonical VPC lock order. + // The complete root catalog is acquired in ascending id order + // so a retirement cannot invert a concurrent repair lock. + $chain = self::repairChainLocked($reference); + $job = $chain[0]; + self::assertCurrent($job, $instance, $vpc, $port, $state); + $now = time(); + $auditReason = substr( + 'host_permanently_retired:' . $reason, + 0, + self::MAX_REASON_LENGTH + ); + foreach ($chain as $generation) { + $generation->effect_resolution = 'applied'; + $generation->effect_resolved_at = $now; + $generation->effect_resolved_by = + (int) ($actor['user_id'] ?? 0); + $generation->effect_resolution_reason = $auditReason; + $generation->available_at = 0; + $generation->read_at = null; + $generation->updated_at = $now; + $generation->save(); + } + + $state->delete(); + CloudVpcPortService::finishRetiredHostRemovalLocked( + $job, $vpc, $port, $instance + ); + return [ + 'retired' => true, + 'task_id' => (string) $job->id, + 'host_id' => (string) $host->id, + 'vpc_id' => (string) $vpc->id, + 'resolved_generations' => count($chain), + ]; + }, + 3 + ); + } + + private static function authorize(array $actor): void + { + if (!($actor['is_admin'] ?? false) + || !($actor['is_super_admin'] ?? false) + || !($actor['can_manage'] ?? false) + || (int) ($actor['user_id'] ?? 0) < 1) { + throw new CloudError( + '只有超级管理员可以确认宿主机永久退役', 403 + ); + } + CloudAccess::manage($actor); + } + + private static function reason(string $reason): string + { + $reason = trim($reason); + $length = mb_strlen($reason); + if ($length < self::MIN_REASON_LENGTH + || $length > self::MAX_REASON_LENGTH) { + throw new CloudError('退役依据需为 20 至 500 个字符'); + } + return $reason; + } + + private static function assertReference(CloudJob $job): void + { + if ((string) $job->kind !== 'vpc_host_remove' + || (int) ($job->host_id ?? 0) < 1 + || (int) ($job->instance_id ?? 0) < 1 + || (int) ($job->vpc_id ?? 0) < 1 + || (int) ($job->vpc_port_id ?? 0) < 1) { + throw new CloudError('该任务不是可退役的 VPC 宿主机清理', 409); + } + } + + private static function assertCurrent( + CloudJob $job, + CloudInstance $instance, + CloudVpc $vpc, + CloudVpcPort $port, + CloudVpcHostState $state + ): void { + self::assertReference($job); + if ((string) $job->execution_mode !== 'agent' + || (string) $job->phase !== 'terminal' + || (string) $job->outcome !== 'failed' + || (string) $job->status !== 'failed' + || !(bool) $job->effect_may_have_applied + || (bool) $job->safe_to_resubmit + || (string) $job->effect_resolution !== 'unknown' + || !CloudDispatchLedger::isDispatched($job) + || (int) ($job->dispatched_at ?? 0) < 1 + || CloudJobService::hasTerminalRecoveryStateConflict($job) + || (bool) $job->finalization_incident + || CloudDispatchLedger::isInvalid($job) + || CloudDispatchLedger::hasRepairIncident($job) + || (int) $job->instance_id !== (int) $instance->id + || (int) $job->vpc_id !== (int) $vpc->id + || (int) $job->vpc_port_id !== (int) $port->id + || (int) $job->host_id !== (int) $state->host_id + || (int) $port->vpc_id !== (int) $vpc->id + || !(bool) $port->active_claim + || (string) $port->state !== 'release_pending' + || !hash_equals( + (string) $state->last_operation_id, + (string) $job->operation_id + ) + || !in_array((string) $state->sync_state, + ['failed', 'unknown'], true)) { + throw new CloudError('VPC 宿主机清理代次已变化或不可退役', 409); + } + } + + private static function assertHostEmptyLocked( + CloudHost $host + ): void { + $hostId = (int) $host->id; + if ((bool) $host->enabled) { + throw new CloudError('请先停用宿主机,再确认永久退役', 409); + } + $agent = CloudHostAgent::query()->where('host_id', $hostId) + ->lockForUpdate()->first(); + if ($agent !== null && (bool) $agent->enabled) { + throw new CloudError('请先停用该宿主机的 Agent 配置', 409); + } + $memberships = CloudHostResourcePoolMembership::query() + ->where('host_id', $hostId)->orderBy('id') + ->lockForUpdate()->get(); + if ($memberships->contains(static fn ($row): bool => + (string) $row->scheduler_state !== 'disabled')) { + throw new CloudError('请先停用宿主机的全部资源池调度', 409); + } + // Placement serializes on the host row. This second current/locking + // read therefore catches a placement that committed after the + // discovery read but before this transaction acquired the host. + $currentInstances = CloudInstance::query() + ->where('host_id', $hostId) + ->where('status', '<>', 'deleted')->orderBy('id') + ->lockForUpdate()->get(); + if (!$currentInstances->isEmpty()) { + throw new CloudError('宿主机仍有未删除实例,不能确认永久退役', 409); + } + $reservations = CloudResourceReservation::query() + ->where('host_id', $hostId) + ->whereIn('state', ['held', 'committed', 'uncertain']) + ->orderBy('id')->lockForUpdate()->get(); + if (!$reservations->isEmpty()) { + throw new CloudError('宿主机仍占用计算资源,不能确认永久退役', 409); + } + $migrations = CloudInstanceMigration::query() + ->where(static function ($query) use ($hostId): void { + $query->where('source_host_id', $hostId) + ->orWhere('target_host_id', $hostId); + })->orderBy('id')->lockForUpdate()->get(); + if ($migrations->contains(static fn ($row): bool => + CloudInstanceMigrationWorkflow::blocksNewMigration( + (string) $row->state + ))) { + throw new CloudError('宿主机仍有关联维护迁移,不能确认永久退役', 409); + } + $ports = CloudVpcPort::query() + ->leftJoin( + 'cloud_instances as retirement_instance', + 'retirement_instance.id', '=', 'cloud_vpc_ports.instance_id' + ) + ->where('cloud_vpc_ports.active_claim', 1) + ->where('cloud_vpc_ports.state', '<>', 'release_pending') + ->where(static function ($query) use ($hostId): void { + $query->where( + 'cloud_vpc_ports.migration_host_id', $hostId + )->orWhere('retirement_instance.host_id', $hostId); + }) + ->orderBy('cloud_vpc_ports.id') + ->lockForUpdate()->get(['cloud_vpc_ports.id']); + if (!$ports->isEmpty()) { + throw new CloudError('宿主机仍承载 VPC 网卡,不能确认永久退役', 409); + } + } + + /** @return list current generation first */ + private static function repairChainLocked(CloudJob $reference): array + { + $chain = []; + $seen = []; + $rootId = (int) ($reference->vpc_repair_root_job_id + ?? $reference->id); + if ($rootId < 1) { + throw new CloudError('VPC 宿主机清理继任链缺少根任务', 409); + } + $catalog = CloudJob::query() + ->where(static function ($query) use ($rootId): void { + $query->where('id', $rootId) + ->orWhere('vpc_repair_root_job_id', $rootId); + })->orderBy('id')->lockForUpdate()->get()->keyBy('id'); + $current = $catalog->get((int) $reference->id); + if (!$current instanceof CloudJob) { + throw new CloudError('VPC 宿主机清理当前代次已丢失', 409); + } + for ($depth = 0; $depth < self::MAX_SUPERSESSION_DEPTH; $depth++) { + $id = (int) $current->id; + if ($id < 1 || isset($seen[$id])) { + throw new CloudError('VPC 宿主机清理继任链包含循环', 409); + } + $seen[$id] = true; + if ((string) $current->kind !== 'vpc_host_remove' + || (int) $current->host_id < 1 + || (string) $current->phase !== 'terminal' + || (string) $current->outcome !== 'failed' + || !(bool) $current->effect_may_have_applied + || (bool) $current->safe_to_resubmit + || (string) $current->effect_resolution !== 'unknown' + || !CloudDispatchLedger::isDispatched($current) + || (int) ($current->dispatched_at ?? 0) < 1 + || CloudJobService::hasTerminalRecoveryStateConflict( + $current + ) + || (bool) $current->finalization_incident + || CloudDispatchLedger::isInvalid($current) + || CloudDispatchLedger::hasRepairIncident($current) + || (int) ($current->vpc_repair_root_job_id + ?? $current->id) !== $rootId) { + throw new CloudError('VPC 宿主机清理继任链已变化', 409); + } + if ($chain !== []) { + $tip = $chain[0]; + if ((int) $current->host_id !== (int) $tip->host_id + || (int) $current->instance_id + !== (int) $tip->instance_id + || (int) $current->vpc_id !== (int) $tip->vpc_id + || (int) $current->vpc_port_id + !== (int) $tip->vpc_port_id) { + throw new CloudError('VPC 宿主机清理继任链资源不一致', 409); + } + } + $chain[] = $current; + $priorId = (int) ($current->supersedes_job_id ?? 0); + if ($priorId < 1) { + if ((int) $current->id !== $rootId) { + throw new CloudError( + 'VPC 宿主机清理根任务身份不一致', 409 + ); + } + foreach ($catalog as $candidate) { + if ((int) $candidate->instance_id + !== (int) $reference->instance_id + || (int) $candidate->vpc_id + !== (int) $reference->vpc_id + || (int) $candidate->vpc_port_id + !== (int) $reference->vpc_port_id + || (int) $candidate->host_id + !== (int) $reference->host_id + || (string) $candidate->kind !== 'vpc_host_remove' + || (string) $candidate->phase !== 'terminal' + || ((string) $candidate->effect_resolution + === 'unknown' + && !isset($seen[(int) $candidate->id]))) { + throw new CloudError( + 'VPC 宿主机清理根链仍有其他活动代次', 409 + ); + } + } + return $chain; + } + $prior = $catalog->get($priorId); + if ($prior === null) { + throw new CloudError('VPC 宿主机清理前序任务已丢失', 409); + } + $current = $prior; + } + throw new CloudError('VPC 宿主机清理继任链超过安全上限', 409); + } +} diff --git a/web/app/Services/Cloud/CloudVpcMembershipBatchService.php b/web/app/Services/Cloud/CloudVpcMembershipBatchService.php index 1614942b..d704d356 100644 --- a/web/app/Services/Cloud/CloudVpcMembershipBatchService.php +++ b/web/app/Services/Cloud/CloudVpcMembershipBatchService.php @@ -335,6 +335,40 @@ final class CloudVpcMembershipBatchService $batch->save(); } + /** + * Treat one exact host-removal child as converged only after the physical + * host-retirement service has proved the disabled node owns no resources. + */ + public static function finishRetiredHostRemovalLocked( + CloudJob $job, + CloudVpc $vpc + ): void { + if ((string) $job->kind !== 'vpc_host_remove' + || $job->vpc_batch_id === null + || (string) $job->effect_resolution !== 'applied') { + throw new CloudError('VPC 批次退役子任务无效', 409); + } + $batch = CloudVpcMembershipBatch::query() + ->where('id', (int) $job->vpc_batch_id) + ->lockForUpdate()->first(); + if ($batch === null || (int) $batch->vpc_id !== (int) $vpc->id + || (string) $batch->state !== 'blocked' + || in_array((string) $batch->state, + self::TERMINAL_STATES, true)) { + throw new CloudError('VPC 批次退役账本不一致', 409); + } + $parent = CloudJob::query() + ->where('id', (int) $batch->parent_job_id) + ->lockForUpdate()->first(); + if ($parent === null || (string) $parent->kind !== self::PARENT_KIND + || (string) $parent->phase === 'terminal') { + throw new CloudError('VPC 批次退役父任务账本不一致', 409); + } + self::advanceHostStageLocked( + $batch, $parent, $job, 'succeeded', '', $vpc + ); + } + private static function startAttachLocked(CloudVpcMembershipBatch $batch, CloudVpc $vpc, $instances, array $instanceIds): void { diff --git a/web/app/Services/Cloud/CloudVpcPortService.php b/web/app/Services/Cloud/CloudVpcPortService.php index d3493fb4..756cf587 100644 --- a/web/app/Services/Cloud/CloudVpcPortService.php +++ b/web/app/Services/Cloud/CloudVpcPortService.php @@ -2204,6 +2204,36 @@ final class CloudVpcPortService return self::hostReady($state); } + /** + * Continue panel orchestration after a super administrator proves that an + * empty, disabled compute node has been permanently retired. The caller + * has already removed the exact host-state row and resolved every unknown + * predecessor generation; no Agent request is made here. + */ + public static function finishRetiredHostRemovalLocked( + CloudJob $job, + CloudVpc $vpc, + CloudVpcPort $port, + CloudInstance $instance + ): void { + if ((string) $job->kind !== 'vpc_host_remove' + || (int) $job->vpc_id !== (int) $vpc->id + || (int) $job->vpc_port_id !== (int) $port->id + || (int) $job->instance_id !== (int) $instance->id + || CloudVpcHostState::query() + ->where('vpc_id', (int) $vpc->id) + ->where('host_id', (int) $job->host_id)->exists()) { + throw new CloudError('VPC 宿主机退役收尾账本不一致', 409); + } + if ($job->vpc_batch_id !== null) { + CloudVpcMembershipBatchService::finishRetiredHostRemovalLocked( + $job, $vpc + ); + return; + } + self::advancePortLocked($instance, $vpc, $port); + } + private static function sealJob(CloudJob $job, CloudVpc $vpc, CloudVpcPort $port, string $method, string $target, array $body, array $extra = []): void diff --git a/web/config/routes_cloud.php b/web/config/routes_cloud.php index 9994a5b3..8209aad4 100644 --- a/web/config/routes_cloud.php +++ b/web/config/routes_cloud.php @@ -144,6 +144,8 @@ $app->group('/api/v1', function () { $this->post('/admin/operations/jobs/{id:[0-9]+}/force-delete', 'App\Controllers\CloudAdminOperationsController:forceDelete'); $this->post('/admin/operations/jobs/{id:[0-9]+}/repair-vpc', 'App\Controllers\CloudAdminOperationsController:repairVpc'); + $this->post('/admin/operations/jobs/{id:[0-9]+}/retire-vpc-host', + 'App\Controllers\CloudAdminOperationsController:retireVpcHost'); $this->get('/admin/billing/invoices', 'App\Controllers\CloudAdminBillingController:invoices'); $this->get('/admin/billing/invoices/{id:[0-9]+}', diff --git a/web/public/cloud/admin-operations.js b/web/public/cloud/admin-operations.js index b6a7378a..1ac2e060 100644 --- a/web/public/cloud/admin-operations.js +++ b/web/public/cloud/admin-operations.js @@ -25,6 +25,7 @@ const state = { forceDeleteBlocked: false, forceDeleteAcceptedIds: new Set(), forceDeleteBlockedIds: new Set(), forceDeleteObservers: new Map(), vpcRepairAccepted: false, vpcRepairAcceptedIds: new Set(), + vpcHostRetired: false, vpcHostRetiredIds: new Set(), inspection: null, effectResolved: false, }; @@ -400,6 +401,13 @@ function renderDetailActions() { repairVpc.disabled = busy || state.vpcRepairAccepted; repairVpc.textContent = state.vpcRepairAccepted ? '修复任务已接收' : busy && state.actionRequest?.kind === 'repair-vpc' ? '正在创建…' : '创建 VPC 修复任务'; + const retireVpcHost = byId('operation-detail-retire-vpc-host'); + retireVpcHost.hidden = !canManage() + || state.detailJob?.can_retire_vpc_host !== true; + retireVpcHost.disabled = busy || state.vpcHostRetired; + retireVpcHost.textContent = state.vpcHostRetired ? '退役账本已收敛' + : busy && state.actionRequest?.kind === 'retire-vpc-host' + ? '正在核对…' : '确认宿主机永久退役'; const force = byId('operation-detail-force-delete'); force.hidden = !canManage() || state.detailJob?.can_force_delete !== true; force.disabled = busy || state.forceDeleteAccepted || state.forceDeleteBlocked; @@ -635,6 +643,51 @@ async function repairVpc(id) { } } +async function retireVpcHost(id) { + if (!canManage() || state.detailId !== String(id) + || state.detailJob?.can_retire_vpc_host !== true || state.actionRequest + || state.vpcHostRetired) return; + const reason = window.prompt('请输入宿主机永久退役依据(20–500 字符)。\n\n系统只会在宿主机、Agent 与调度均已停用,且实例、资源预留、维护迁移和 VPC 网卡全部清空时解除这一个清理栅栏。'); + if (reason === null) return; + const normalized = reason.trim(); + if (Array.from(normalized).length < 20 || Array.from(normalized).length > 500) { + detailNotice('退役依据需为 20 至 500 个字符。', true); + return; + } + if (!window.confirm('这是物理宿主机永久退役确认,不会联系 Agent,也不会重试原操作。\n\n确认该宿主机已永久退出服务且不会带着原本地 VPC 状态重新上线?')) return; + const controller = new AbortController(); + controller.kind = 'retire-vpc-host'; state.actionRequest = controller; + renderDetailActions(); detailNotice('正在核对宿主机资源账本…'); + try { + const payload = await api(`/api/v1/admin/operations/jobs/${encodeURIComponent(id)}/retire-vpc-host`, controller, 45000, { + method: 'POST', + body: JSON.stringify({ + acknowledge_permanent_retirement: true, + reason: normalized, + }), + }); + if (payload?.retired !== true || String(payload.task_id || '') !== String(id)) { + throw new Error('退役响应与当前任务不一致。'); + } + state.vpcHostRetired = true; + state.vpcHostRetiredIds.add(String(id)); + await loadPage(listUrl(), 'none'); + if (byId('operation-detail').open) await openDetail(String(id), null); + notice(`宿主机 #${payload.host_id} 的 VPC 清理栅栏已按永久退役事实收敛。`); + if (byId('operation-detail').open) { + detailNotice('宿主机永久退役已记录;原失败任务和审计证据保持不变。'); + } + } catch (error) { + if (controller.signal.aborted && controller.signal.reason !== 'timeout') return; + detailNotice(`宿主机退役未确认:${error.message}`, true); + notice(`宿主机退役未确认:${error.message}`, true); + } finally { + if (state.actionRequest === controller) { + state.actionRequest = null; renderDetailActions(); + } + } +} + function finishDetailClose(id) { state.detailClosing = false; document.documentElement.classList.remove('admin-instance-detail-open'); @@ -655,6 +708,7 @@ function closeDetail() { state.detailId = null; state.detailJob = null; state.forceDeleteAccepted = false; state.forceDeleteBlocked = false; state.vpcRepairAccepted = false; + state.vpcHostRetired = false; state.inspection = null; state.effectResolved = false; state.detailClosing = true; void window.VenixDrawerMotion.closeDrawerWithMotion(byId('operation-detail')).then(closed => { @@ -669,12 +723,14 @@ async function openDetail(id, opener) { state.detailId = id; state.detailJob = null; state.opener = opener; state.forceDeleteAccepted = state.forceDeleteAcceptedIds.has(String(id)); state.forceDeleteBlocked = state.forceDeleteBlockedIds.has(String(id)); state.vpcRepairAccepted = state.vpcRepairAcceptedIds.has(String(id)); + state.vpcHostRetired = state.vpcHostRetiredIds.has(String(id)); state.inspection = null; state.effectResolved = false; const controller = new AbortController(); state.detailRequest = controller; byId('operation-detail-title').textContent = `任务 #${id}`; byId('operation-detail-body').replaceChildren(); byId('operation-detail-links').replaceChildren(); byId('operation-detail-inspect').hidden = true; byId('operation-detail-repair-vpc').hidden = true; + byId('operation-detail-retire-vpc-host').hidden = true; byId('operation-detail-force-delete').hidden = true; byId('operation-effect-resolution').hidden = true; byId('operation-effect-value').value = ''; @@ -730,6 +786,9 @@ if (root) { byId('operation-detail-repair-vpc').addEventListener('click', event => { event.preventDefault(); if (state.detailId) void repairVpc(state.detailId); }); + byId('operation-detail-retire-vpc-host').addEventListener('click', event => { + event.preventDefault(); if (state.detailId) void retireVpcHost(state.detailId); + }); byId('operation-effect-submit').addEventListener('click', event => { event.preventDefault(); if (state.detailId) void resolveEffect(state.detailId); }); diff --git a/web/resources/views/venix/admin/cloud/operations.tpl b/web/resources/views/venix/admin/cloud/operations.tpl index 0719fece..8730cdf9 100644 --- a/web/resources/views/venix/admin/cloud/operations.tpl +++ b/web/resources/views/venix/admin/cloud/operations.tpl @@ -39,7 +39,7 @@

任务详情

任务

正在读取任务详情…

-
失败任务不会自动重提。详情标为持续自动核对时,只需修复所示通信或数据异常;系统会继续收敛。
+
失败任务不会自动重提。详情标为持续自动核对时,只需修复所示通信或数据异常;系统会继续收敛。
diff --git a/web/resources/views/venix/layout_assets.tpl b/web/resources/views/venix/layout_assets.tpl index 50d3bf1b..e8a7174e 100644 --- a/web/resources/views/venix/layout_assets.tpl +++ b/web/resources/views/venix/layout_assets.tpl @@ -1,2 +1,2 @@ {* Shared Venix CSS/JS cache key. Increment whenever a shared browser asset changes. *} -{assign var=venix_asset_version value='2026102907' scope='global'} +{assign var=venix_asset_version value='2026102908' scope='global'} diff --git a/web/tests/AdminOperationsViewTest.php b/web/tests/AdminOperationsViewTest.php index 0a1c2a72..d3a9c074 100644 --- a/web/tests/AdminOperationsViewTest.php +++ b/web/tests/AdminOperationsViewTest.php @@ -159,6 +159,7 @@ $assert($xpath->query('//*[@id="operations-health-body"]//p')->length === 1 $assert($xpath->query('//button[contains(@id,"retry") or contains(@id,"refund") or contains(@id,"stop-auto") or contains(@id,"abort-job") or contains(@id,"cleanup")]')->length === 0 && $xpath->query('//button[@id="operation-detail-inspect" and @hidden]')->length === 1 && $xpath->query('//button[@id="operation-detail-repair-vpc" and @hidden]')->length === 1 + && $xpath->query('//button[@id="operation-detail-retire-vpc-host" and @hidden]')->length === 1 && $xpath->query('//button[@id="operation-detail-force-delete" and @hidden]')->length === 1 && $xpath->query('//*[@id="operation-effect-resolution" and @hidden]')->length === 1, 'The task list must expose no replay/stop mutation and keep audited fence resolution inside detail.'); diff --git a/web/tests/CloudAdminOperationsServiceTest.php b/web/tests/CloudAdminOperationsServiceTest.php index 8913559d..4a018b8f 100644 --- a/web/tests/CloudAdminOperationsServiceTest.php +++ b/web/tests/CloudAdminOperationsServiceTest.php @@ -505,6 +505,24 @@ namespace { ), 'Administrator display mislabeled a mixed aggregate as all-applied.'); + $insert(17, [ + 'kind' => 'vpc_host_remove', + 'outcome' => 'failed', 'status' => 'failed', + 'effect_may_have_applied' => true, + 'safe_to_resubmit' => false, + 'effect_resolution' => 'unknown', + 'error_code' => 'operation_effect_unknown', + ]); + $retireableVpcHost = Operations::detail($admin, '17')['job']; + $readonlyVpcHost = Operations::detail($readonly, '17')['job']; + $assert($retireableVpcHost['can_retire_vpc_host'] === true + && str_contains( + (string) $retireableVpcHost['retire_vpc_host_reason'], + '实例、资源、迁移和 VPC 网卡全部清空' + ) + && $readonlyVpcHost['can_retire_vpc_host'] === false, + 'Permanent VPC host-retirement exit was hidden from superadmins or exposed to read-only admins.'); + $reason = '已在宿主机只读核对虚拟机与磁盘,确认原操作效果已经发生。'; $operationId = sprintf('%032x', 3); $freshInspection = [ diff --git a/web/tests/CloudVpcHostRetirementServiceTest.php b/web/tests/CloudVpcHostRetirementServiceTest.php new file mode 100644 index 00000000..e7903c0e --- /dev/null +++ b/web/tests/CloudVpcHostRetirementServiceTest.php @@ -0,0 +1,580 @@ +httpStatus === 409, + $message . ': HTTP ' . $error->httpStatus); + return; + } + throw new RuntimeException($message . ': request was accepted'); +}; + +$capsule = new Capsule(); +$capsule->addConnection([ + 'driver' => 'sqlite', 'database' => ':memory:', +], 'default'); +$capsule->setAsGlobal(); +$capsule->bootEloquent(); +$schema = $capsule->schema('default'); + +$schema->create('cloud_hosts', static function (Blueprint $table): void { + $table->increments('id'); + $table->string('name'); + $table->string('region')->default(''); + $table->boolean('enabled')->default(false); + $table->integer('created_at'); + $table->integer('updated_at'); +}); +$schema->create('cloud_host_agents', static function (Blueprint $table): void { + $table->integer('host_id')->primary(); + $table->boolean('enabled')->default(false); +}); +$schema->create('cloud_host_resource_pool_memberships', static function ( + Blueprint $table +): void { + $table->increments('id'); + $table->integer('host_id'); + $table->integer('resource_pool_id'); + $table->string('scheduler_state'); + $table->integer('priority')->default(0); + $table->integer('weight')->default(100); + $table->integer('created_at'); + $table->integer('updated_at'); +}); +$schema->create('cloud_instances', static function (Blueprint $table): void { + $table->increments('id'); + $table->string('uuid'); + $table->integer('host_id'); + $table->integer('user_id'); + $table->string('name'); + $table->string('status'); + $table->string('service_status'); + $table->string('network_status'); + $table->text('spec_json'); + $table->integer('created_at'); + $table->integer('updated_at'); +}); +$schema->create('cloud_resource_reservations', static function ( + Blueprint $table +): void { + $table->increments('id'); + $table->integer('host_id'); + $table->string('state'); +}); +$schema->create('cloud_instance_migrations', static function ( + Blueprint $table +): void { + $table->increments('id'); + $table->integer('instance_id'); + $table->integer('source_host_id'); + $table->integer('target_host_id'); + $table->string('state'); +}); +$schema->create('cloud_vpcs', static function (Blueprint $table): void { + $table->increments('id'); + $table->string('vpc_key'); + $table->integer('user_id'); + $table->integer('region_id'); + $table->integer('cidr_pool_id'); + $table->string('name'); + $table->string('cidr'); + $table->integer('prefix_length'); + $table->string('gateway'); + $table->integer('active_claim')->nullable(); + $table->string('state'); + $table->integer('desired_revision'); + $table->integer('deleted_at')->nullable(); + $table->integer('created_at'); + $table->integer('updated_at'); +}); +$schema->create('cloud_vpc_ports', static function (Blueprint $table): void { + $table->increments('id'); + $table->string('port_key'); + $table->integer('vpc_id'); + $table->integer('instance_id'); + $table->integer('migration_host_id')->nullable(); + $table->integer('user_id'); + $table->string('private_ip'); + $table->string('mac_address'); + $table->integer('active_claim')->nullable(); + $table->string('state'); + $table->integer('desired_revision'); + $table->integer('applied_revision'); + $table->string('last_operation_id')->nullable(); + $table->string('last_error_code')->default(''); + $table->integer('attached_at')->nullable(); + $table->integer('released_at')->nullable(); + $table->integer('created_at'); + $table->integer('updated_at'); +}); +$schema->create('cloud_vpc_host_states', static function ( + Blueprint $table +): void { + $table->increments('id'); + $table->integer('vpc_id'); + $table->integer('host_id'); + $table->integer('desired_revision'); + $table->integer('applied_revision'); + $table->string('desired_digest'); + $table->string('applied_digest')->nullable(); + $table->string('sync_state'); + $table->string('last_operation_id')->nullable(); + $table->string('last_error_code')->default(''); + $table->integer('last_success_at')->default(0); + $table->integer('last_failure_at')->default(0); + $table->integer('created_at'); + $table->integer('updated_at'); +}); +$schema->create('cloud_vpc_membership_batches', static function ( + Blueprint $table +): void { + $table->increments('id'); + $table->string('batch_key'); + $table->integer('vpc_id'); + $table->integer('user_id'); + $table->integer('parent_job_id'); + $table->string('kind'); + $table->string('state'); + $table->integer('desired_revision'); + $table->integer('active_claim')->nullable(); + $table->string('last_error_code')->default(''); + $table->integer('terminal_at')->nullable(); + $table->integer('created_at'); + $table->integer('updated_at'); +}); +$schema->create('cloud_vpc_membership_batch_items', static function ( + Blueprint $table +): void { + $table->increments('id'); + $table->integer('batch_id'); + $table->integer('user_id'); + $table->integer('instance_id'); + $table->integer('vpc_port_id'); + $table->integer('host_id'); + $table->integer('child_job_id')->nullable(); + $table->string('state'); + $table->string('last_error_code')->default(''); + $table->integer('created_at'); + $table->integer('updated_at'); +}); +$schema->create('cloud_jobs', static function (Blueprint $table): void { + $table->increments('id'); + $table->string('operation_id'); + $table->string('request_idempotency_key'); + $table->string('request_fingerprint'); + $table->integer('parent_job_id')->nullable(); + $table->integer('supersedes_job_id')->nullable(); + $table->integer('vpc_repair_root_job_id')->nullable(); + $table->integer('order_id')->nullable(); + $table->integer('invoice_id')->nullable(); + $table->integer('instance_id')->nullable(); + $table->integer('snapshot_id')->nullable(); + $table->integer('backup_id')->nullable(); + $table->integer('backup_restore_id')->nullable(); + $table->integer('backup_recovery_id')->nullable(); + $table->integer('vpc_id')->nullable(); + $table->integer('vpc_port_id')->nullable(); + $table->integer('vpc_batch_id')->nullable(); + $table->integer('user_id'); + $table->integer('host_id')->nullable(); + $table->string('kind'); + $table->string('execution_mode'); + $table->string('phase'); + $table->string('outcome')->nullable(); + $table->string('status'); + $table->text('sealed_payload'); + $table->integer('attempts'); + $table->integer('dispatch_attempts'); + $table->integer('dispatch_deadline_at'); + $table->integer('dispatch_started_at')->nullable(); + $table->integer('agent_accepted_at')->nullable(); + $table->integer('receipt_deadline_at')->nullable(); + $table->integer('terminal_at')->nullable(); + $table->boolean('effect_may_have_applied')->nullable(); + $table->boolean('safe_to_resubmit')->nullable(); + $table->string('effect_resolution'); + $table->integer('effect_resolved_at')->nullable(); + $table->integer('effect_resolved_by')->nullable(); + $table->text('effect_resolution_reason')->default(''); + $table->boolean('finalization_incident')->default(false); + $table->string('finalization_error_code')->nullable(); + $table->string('completion_event_id')->nullable(); + $table->integer('read_at')->nullable(); + $table->integer('dispatched_at')->nullable(); + $table->string('lease_token')->nullable(); + $table->integer('lease_until')->default(0); + $table->integer('available_at'); + $table->string('error_code')->nullable(); + $table->text('result_json')->nullable(); + $table->integer('created_at'); + $table->integer('updated_at'); +}); + +$now = time(); +$actor = [ + 'user_id' => 99, 'is_admin' => true, + 'is_super_admin' => true, 'can_manage' => true, +]; +Capsule::table('cloud_hosts')->insert([ + ['id' => 1, 'name' => 'retiring-host', 'region' => 'hk', + 'enabled' => 1, 'created_at' => $now, 'updated_at' => $now], + ['id' => 2, 'name' => 'surviving-host', 'region' => 'hk', + 'enabled' => 1, 'created_at' => $now, 'updated_at' => $now], +]); +Capsule::table('cloud_host_agents')->insert([ + 'host_id' => 1, 'enabled' => 1, +]); +Capsule::table('cloud_host_resource_pool_memberships')->insert([ + 'id' => 1, 'host_id' => 1, 'resource_pool_id' => 1, + 'scheduler_state' => 'active', 'priority' => 0, 'weight' => 100, + 'created_at' => $now, 'updated_at' => $now, +]); +$instanceRows = [ + ['id' => 10, 'uuid' => '11111111-1111-4111-8111-111111111110', + 'host_id' => 1, 'user_id' => 7, 'name' => 'retired-reference', + 'status' => 'deleted'], + ['id' => 11, 'uuid' => '11111111-1111-4111-8111-111111111111', + 'host_id' => 1, 'user_id' => 7, 'name' => 'host-blocker', + 'status' => 'active'], +]; +foreach ($instanceRows as &$row) { + $row += ['service_status' => 'active', 'network_status' => 'active', + 'spec_json' => '{}', 'created_at' => $now, 'updated_at' => $now]; +} +unset($row); +Capsule::table('cloud_instances')->insert($instanceRows); +Capsule::table('cloud_resource_reservations')->insert([ + 'id' => 1, 'host_id' => 1, 'state' => 'held', +]); +Capsule::table('cloud_instance_migrations')->insert([ + 'id' => 1, 'instance_id' => 11, 'source_host_id' => 1, + 'target_host_id' => 2, 'state' => 'preparing_source', +]); +Capsule::table('cloud_vpcs')->insert([ + 'id' => 20, 'vpc_key' => str_repeat('a', 32), 'user_id' => 7, + 'region_id' => 1, 'cidr_pool_id' => 1, 'name' => 'retire-test', + 'cidr' => '10.10.0.0/24', 'prefix_length' => 24, + 'gateway' => '10.10.0.1', 'active_claim' => 1, + 'state' => 'provisioning', 'desired_revision' => 3, + 'deleted_at' => null, 'created_at' => $now, 'updated_at' => $now, +]); +Capsule::table('cloud_vpc_ports')->insert([ + [ + 'id' => 30, 'port_key' => str_repeat('b', 32), 'vpc_id' => 20, + 'instance_id' => 10, 'migration_host_id' => null, 'user_id' => 7, + 'private_ip' => '10.10.0.2', 'mac_address' => '02:00:00:00:00:02', + 'active_claim' => 1, 'state' => 'release_pending', + 'desired_revision' => 3, 'applied_revision' => 2, + 'last_operation_id' => null, 'last_error_code' => '', + 'attached_at' => $now - 100, 'released_at' => null, + 'created_at' => $now, 'updated_at' => $now, + ], + [ + 'id' => 31, 'port_key' => str_repeat('c', 32), 'vpc_id' => 20, + 'instance_id' => 11, 'migration_host_id' => null, 'user_id' => 7, + 'private_ip' => '10.10.0.3', 'mac_address' => '02:00:00:00:00:03', + 'active_claim' => 1, 'state' => 'attached', + 'desired_revision' => 2, 'applied_revision' => 2, + 'last_operation_id' => null, 'last_error_code' => '', + 'attached_at' => $now - 100, 'released_at' => null, + 'created_at' => $now, 'updated_at' => $now, + ], +]); +$rootOperation = str_repeat('d', 32); +$tipOperation = str_repeat('e', 32); +Capsule::table('cloud_vpc_host_states')->insert([ + 'id' => 40, 'vpc_id' => 20, 'host_id' => 1, + 'desired_revision' => 3, 'applied_revision' => 2, + 'desired_digest' => str_repeat('f', 64), + 'applied_digest' => str_repeat('1', 64), 'sync_state' => 'unknown', + 'last_operation_id' => $tipOperation, + 'last_error_code' => 'operation_effect_unknown', + 'last_success_at' => $now - 100, 'last_failure_at' => $now - 10, + 'created_at' => $now, 'updated_at' => $now, +]); +$job = static function ( + int $id, + string $operationId, + ?int $supersedes, + ?int $root +) use ($now): array { + return [ + 'id' => $id, 'operation_id' => $operationId, + 'request_idempotency_key' => $operationId, + 'request_fingerprint' => hash('sha256', $operationId), + 'parent_job_id' => null, 'supersedes_job_id' => $supersedes, + 'vpc_repair_root_job_id' => $root, 'order_id' => null, + 'invoice_id' => null, 'instance_id' => 10, 'snapshot_id' => null, + 'backup_id' => null, 'backup_restore_id' => null, + 'backup_recovery_id' => null, 'vpc_id' => 20, + 'vpc_port_id' => 30, 'vpc_batch_id' => null, + 'user_id' => 7, 'host_id' => 1, 'kind' => 'vpc_host_remove', + 'execution_mode' => 'agent', 'phase' => 'terminal', + 'outcome' => 'failed', 'status' => 'failed', 'sealed_payload' => '{}', + 'attempts' => 1, 'dispatch_attempts' => 1, + 'dispatch_deadline_at' => $now - 200, + 'dispatch_started_at' => $now - 190, + 'agent_accepted_at' => $now - 185, + 'receipt_deadline_at' => $now - 100, 'terminal_at' => $now - 20, + 'effect_may_have_applied' => 1, 'safe_to_resubmit' => 0, + 'effect_resolution' => 'unknown', 'effect_resolved_at' => null, + 'effect_resolved_by' => null, 'effect_resolution_reason' => '', + 'finalization_incident' => 0, 'finalization_error_code' => null, + 'completion_event_id' => null, 'read_at' => null, + 'dispatched_at' => $now - 185, 'lease_token' => null, + 'lease_until' => 0, 'available_at' => 0, + 'error_code' => 'operation_effect_unknown', 'result_json' => null, + 'created_at' => $now - 200, 'updated_at' => $now - 20, + ]; +}; +Capsule::table('cloud_jobs')->insert([ + $job(100, $rootOperation, null, null), + $job(101, $tipOperation, 100, 100), + array_replace( + $job(102, str_repeat('9', 32), 101, 100), + [ + 'effect_may_have_applied' => 0, + 'safe_to_resubmit' => 1, + 'effect_resolution' => 'not_applied', + 'error_code' => 'agent_rejected', + ] + ), +]); + +$retire = static fn (): array => CloudVpcHostRetirementService::retire( + $actor, + 101, + 'The physical node was decommissioned after all tenant data was removed.' +); + +$reject($retire, 'enabled host must block permanent retirement'); +Capsule::table('cloud_hosts')->where('id', 1)->update(['enabled' => 0]); +$reject($retire, 'enabled Agent must block permanent retirement'); +Capsule::table('cloud_host_agents')->where('host_id', 1) + ->update(['enabled' => 0]); +$reject($retire, 'active scheduler membership must block retirement'); +Capsule::table('cloud_host_resource_pool_memberships')->where('id', 1) + ->update(['scheduler_state' => 'disabled']); +$reject($retire, 'nondeleted instance must block permanent retirement'); +Capsule::table('cloud_instances')->where('id', 11) + ->update(['status' => 'deleted']); +$reject($retire, 'active resource reservation must block retirement'); +Capsule::table('cloud_resource_reservations')->where('id', 1) + ->update(['state' => 'released']); +$reject($retire, 'active maintenance migration must block retirement'); +Capsule::table('cloud_instance_migrations')->where('id', 1) + ->update(['state' => 'completed']); +$reject($retire, 'attached VPC port must block permanent retirement'); +Capsule::table('cloud_vpc_ports')->where('id', 31)->update([ + 'active_claim' => null, 'state' => 'released', 'released_at' => $now, +]); + +Capsule::table('cloud_jobs')->where('id', 102)->update([ + 'effect_may_have_applied' => 1, 'safe_to_resubmit' => 0, + 'effect_resolution' => 'unknown', +]); +$reject($retire, 'parallel unknown repair generation must block retirement'); +Capsule::table('cloud_jobs')->where('id', 102)->update([ + 'effect_may_have_applied' => 0, 'safe_to_resubmit' => 1, + 'effect_resolution' => 'not_applied', +]); + +Capsule::table('cloud_jobs')->where('id', 101)->update([ + 'effect_may_have_applied' => 0, 'safe_to_resubmit' => 1, + 'effect_resolution' => 'not_applied', +]); +$reject($retire, 'known no-effect job must not be rewritten as applied'); +Capsule::table('cloud_jobs')->where('id', 101)->update([ + 'effect_may_have_applied' => 1, 'safe_to_resubmit' => 0, + 'effect_resolution' => 'unknown', +]); + +$result = $retire(); +$assert($result['retired'] === true && $result['task_id'] === '101' + && $result['host_id'] === '1' && $result['vpc_id'] === '20' + && $result['resolved_generations'] === 2, + 'retirement result must identify the exact converged chain'); +$assert(Capsule::table('cloud_vpc_host_states')->where('id', 40) + ->doesntExist(), 'retired host projection must be deleted'); + +$resolved = CloudJob::query()->whereIn('id', [100, 101]) + ->orderBy('id')->get(); +$assert($resolved->count() === 2, 'repair chain audit rows must be retained'); +foreach ($resolved as $generation) { + $assert((string) $generation->outcome === 'failed' + && (string) $generation->status === 'failed' + && (string) $generation->effect_resolution === 'applied' + && (int) $generation->effect_resolved_by === 99 + && str_starts_with( + (string) $generation->effect_resolution_reason, + 'host_permanently_retired:' + ), 'each failed generation must retain outcome and gain audit proof'); +} +$knownNoEffectSibling = CloudJob::find(102); +$assert($knownNoEffectSibling !== null + && (string) $knownNoEffectSibling->effect_resolution === 'not_applied' + && $knownNoEffectSibling->effect_resolved_by === null, + 'known no-effect repair sibling must retain its original conclusion'); +$port = Capsule::table('cloud_vpc_ports')->where('id', 30)->first(); +$vpc = Capsule::table('cloud_vpcs')->where('id', 20)->first(); +$assert($port !== null && $port->active_claim === null + && $port->state === 'released' && (int) $port->released_at > 0, + 'reference VPC port must finish its pending release'); +$assert($vpc !== null && $vpc->state === 'active', + 'VPC must converge after its last host projection is retired'); + +Capsule::table('cloud_instances')->insert([ + 'id' => 12, 'uuid' => '11111111-1111-4111-8111-111111111112', + 'host_id' => 1, 'user_id' => 7, 'name' => 'batch-reference', + 'status' => 'deleted', 'service_status' => 'active', + 'network_status' => 'active', 'spec_json' => '{}', + 'created_at' => $now, 'updated_at' => $now, +]); +Capsule::table('cloud_vpcs')->insert([ + 'id' => 21, 'vpc_key' => str_repeat('4', 32), 'user_id' => 7, + 'region_id' => 1, 'cidr_pool_id' => 1, 'name' => 'batch-retire-test', + 'cidr' => '10.20.0.0/24', 'prefix_length' => 24, + 'gateway' => '10.20.0.1', 'active_claim' => 1, + 'state' => 'degraded', 'desired_revision' => 4, + 'deleted_at' => null, 'created_at' => $now, 'updated_at' => $now, +]); +Capsule::table('cloud_vpc_ports')->insert([ + 'id' => 32, 'port_key' => str_repeat('5', 32), 'vpc_id' => 21, + 'instance_id' => 12, 'migration_host_id' => null, 'user_id' => 7, + 'private_ip' => '10.20.0.2', 'mac_address' => '02:00:00:00:00:04', + 'active_claim' => 1, 'state' => 'release_pending', + 'desired_revision' => 4, 'applied_revision' => 3, + 'last_operation_id' => null, 'last_error_code' => '', + 'attached_at' => $now - 100, 'released_at' => null, + 'created_at' => $now, 'updated_at' => $now, +]); +$batchOperation = str_repeat('6', 32); +Capsule::table('cloud_vpc_host_states')->insert([ + 'id' => 41, 'vpc_id' => 21, 'host_id' => 1, + 'desired_revision' => 4, 'applied_revision' => 3, + 'desired_digest' => str_repeat('7', 64), + 'applied_digest' => str_repeat('8', 64), 'sync_state' => 'unknown', + 'last_operation_id' => $batchOperation, + 'last_error_code' => 'operation_effect_unknown', + 'last_success_at' => $now - 100, 'last_failure_at' => $now - 10, + 'created_at' => $now, 'updated_at' => $now, +]); +$batchPrefix = CloudVpcPortService::batchProjectionRequestPrefix(50, 4); +$parent = array_replace($job(200, str_repeat('2', 32), null, null), [ + 'request_idempotency_key' => str_repeat('2', 32), + 'instance_id' => null, 'vpc_id' => 21, 'vpc_port_id' => null, + 'vpc_batch_id' => null, 'host_id' => null, + 'kind' => 'vpc_membership_batch', 'execution_mode' => 'local', + 'phase' => 'accepted', 'outcome' => null, 'status' => 'queued', + 'attempts' => 0, 'dispatch_attempts' => 0, + 'dispatch_deadline_at' => $now + 3600, + 'dispatch_started_at' => null, 'agent_accepted_at' => null, + 'receipt_deadline_at' => null, 'terminal_at' => null, + 'effect_may_have_applied' => null, 'safe_to_resubmit' => null, + 'effect_resolution' => 'not_applicable', 'dispatched_at' => null, + 'error_code' => null, +]); +$batchChild = array_replace( + $job(201, $batchOperation, null, null), + [ + 'request_idempotency_key' => $batchPrefix + . 'vpc_host_remove-1-' . $batchOperation, + 'instance_id' => 12, 'vpc_id' => 21, 'vpc_port_id' => 32, + 'vpc_batch_id' => 50, + ] +); +Capsule::table('cloud_jobs')->insert([$parent, $batchChild]); +Capsule::table('cloud_vpc_membership_batches')->insert([ + 'id' => 50, 'batch_key' => str_repeat('3', 32), 'vpc_id' => 21, + 'user_id' => 7, 'parent_job_id' => 200, 'kind' => 'detach', + 'state' => 'blocked', 'desired_revision' => 4, 'active_claim' => 1, + 'last_error_code' => 'vpc_result_uncertain', 'terminal_at' => null, + 'created_at' => $now, 'updated_at' => $now, +]); +Capsule::table('cloud_vpc_membership_batch_items')->insert([ + 'id' => 60, 'batch_id' => 50, 'user_id' => 7, 'instance_id' => 12, + 'vpc_port_id' => 32, 'host_id' => 1, 'child_job_id' => null, + 'state' => 'succeeded', 'last_error_code' => '', + 'created_at' => $now, 'updated_at' => $now, +]); +$batchResult = CloudVpcHostRetirementService::retire( + $actor, + 201, + 'The second physical node projection was retired after the batch was audited.' +); +$retiredBatch = Capsule::table('cloud_vpc_membership_batches') + ->where('id', 50)->first(); +$retiredParent = CloudJob::find(200); +$retiredBatchPort = Capsule::table('cloud_vpc_ports') + ->where('id', 32)->first(); +$assert($batchResult['retired'] === true + && $batchResult['resolved_generations'] === 1, + 'blocked batch host retirement must resolve one exact child generation'); +$assert($retiredBatch !== null && $retiredBatch->state === 'succeeded' + && $retiredBatch->active_claim === null + && (int) $retiredBatch->terminal_at > 0, + 'blocked membership batch must reach a terminal success state'); +$assert($retiredParent !== null + && (string) $retiredParent->phase === 'terminal' + && (string) $retiredParent->outcome === 'succeeded', + 'membership batch parent task must converge with the retired child'); +$assert($retiredBatchPort !== null + && $retiredBatchPort->active_claim === null + && $retiredBatchPort->state === 'released', + 'batch release-pending port must be released after host retirement'); + +$root = dirname(__DIR__); +$routes = file_get_contents($root . '/config/routes_cloud.php'); +$controller = file_get_contents( + $root . '/app/Controllers/CloudAdminOperationsController.php' +); +$recovery = file_get_contents( + $root . '/app/Services/Cloud/CloudAdminOperationRecoveryService.php' +); +$script = file_get_contents($root . '/public/cloud/admin-operations.js'); +$view = file_get_contents( + $root . '/resources/views/venix/admin/cloud/operations.tpl' +); +$assert(is_string($routes) && is_string($controller) + && is_string($recovery) && is_string($script) && is_string($view) + && str_contains($routes, '/retire-vpc-host') + && str_contains($controller, 'function retireVpcHost(') + && str_contains($recovery, 'function retireVpcHost(') + && str_contains($script, 'async function retireVpcHost(') + && str_contains($view, 'operation-detail-retire-vpc-host'), + 'permanent host-retirement route, controller, UI, or service wiring is incomplete'); + +echo "CloudVpcHostRetirementServiceTest passed\n"; +}