diff --git a/agent/api/server_test.go b/agent/api/server_test.go index a690c7dc..e73575ab 100644 --- a/agent/api/server_test.go +++ b/agent/api/server_test.go @@ -337,13 +337,20 @@ func TestVPCPortMutationBindsGuestAndHostProjectionIdentity(t *testing.T) { t.Fatalf("VPC port operation record = %#v", record) } for _, required := range []string{ - "catalog:vpc-host/state", "catalog:vpc/" + vpcKey, "catalog:vpc-port/" + portKey, "domain:" + uuid, } { if !slices.Contains(record.LockKeys, required) { t.Fatalf("VPC port locks missing %q: %#v", required, record.LockKeys) } } + for _, global := range []string{ + "catalog:vpc-host/state", "catalog:vpc/" + vpcKey, + } { + if slices.Contains(record.LockKeys, global) { + t.Fatalf("VPC port operation retained global lock %q: %#v", + global, record.LockKeys) + } + } } func TestExitRescueRequiresDataLossAcknowledgement(t *testing.T) { diff --git a/agent/hypervisor/admission.go b/agent/hypervisor/admission.go index 1d0ad047..675456cc 100644 --- a/agent/hypervisor/admission.go +++ b/agent/hypervisor/admission.go @@ -39,8 +39,6 @@ func VPCNetworkAdmission(vpcKey string) MutationAdmission { func VPCPortAdmission(uuid string, request VPCPortRequest) MutationAdmission { uuid = strings.ToLower(uuid) return MutationAdmission{LockKeys: []string{ - "catalog:vpc-host/state", - catalogLockKey("vpc", request.VPCKey), catalogLockKey("vpc-port", request.PortKey), domainLockKey(uuid), volumeLockKey(seedStoragePoolName, seedVolumeName(uuid)), diff --git a/agent/hypervisor/admission_test.go b/agent/hypervisor/admission_test.go index 0ffd1e4e..53a81a3e 100644 --- a/agent/hypervisor/admission_test.go +++ b/agent/hypervisor/admission_test.go @@ -39,7 +39,7 @@ func TestVPCNetworkAdmissionSerializesHostProjectionAndOneVPC(t *testing.T) { } } -func TestVPCPortAdmissionSerializesDomainSeedAndHostProjection(t *testing.T) { +func TestVPCPortAdmissionAllowsIndependentPortsToRunConcurrently(t *testing.T) { const uuid = "123e4567-e89b-42d3-a456-426614174000" request := VPCPortRequest{ PortKey: "1123456789abcdef0123456789abcdef", @@ -48,7 +48,6 @@ func TestVPCPortAdmissionSerializesDomainSeedAndHostProjection(t *testing.T) { } admission := VPCPortAdmission(uuid, request) for _, required := range []string{ - "catalog:vpc-host/state", "catalog:vpc/" + request.VPCKey, "catalog:vpc-port/" + request.PortKey, domainLockKey(uuid), volumeLockKey(seedStoragePoolName, seedVolumeName(uuid)), networkIdentityLockKey(request.PrivateIP), networkIdentityLockKey(request.MAC), @@ -57,6 +56,13 @@ func TestVPCPortAdmissionSerializesDomainSeedAndHostProjection(t *testing.T) { t.Fatalf("VPC port admission missing %q: %#v", required, admission) } } + for _, global := range []string{ + "catalog:vpc-host/state", "catalog:vpc/" + request.VPCKey, + } { + if slices.Contains(admission.LockKeys, global) { + t.Fatalf("VPC port admission still has global lock %q: %#v", global, admission) + } + } } func TestPeerMigrationReceiveAdmissionIncludesVPCIdentity(t *testing.T) { diff --git a/agent/hypervisor/libvirt_vpc_port.go b/agent/hypervisor/libvirt_vpc_port.go index 188022dc..161a336a 100644 --- a/agent/hypervisor/libvirt_vpc_port.go +++ b/agent/hypervisor/libvirt_vpc_port.go @@ -46,12 +46,14 @@ func (l *Libvirt) changeVPCPort(ctx context.Context, uuid string, if l.vpcRouter == nil { return VPCPortResult{}, NewError(ErrUnavailable, "vpc_router_unavailable", "VPC host router is unavailable", nil) } - if _, err := l.vpcRouter.AuthorizeLocalPort( + _, routeLease, err := l.vpcRouter.AcquireLocalPortLease( request.VPCKey, request.HostRevision, request.HostDigest, request.CIDR, request.Gateway, request.PrivateIP, request.MAC, - ); err != nil { + ) + if err != nil { return VPCPortResult{}, NewError(ErrConflict, "vpc_host_projection_not_ready", "VPC host routing must contain this exact local port before the VM interface changes", err) } + defer routeLease.Release() port := vpcPortMetadata(request) if present { conflict, err := l.vpcPortIdentityConflict(ctx, uuid, port) diff --git a/agent/vpcrouting/manager.go b/agent/vpcrouting/manager.go index 698d9346..3f0cc46b 100644 --- a/agent/vpcrouting/manager.go +++ b/agent/vpcrouting/manager.go @@ -10,6 +10,7 @@ import ( "path/filepath" "sort" "strings" + "sync" "time" "git.luoli.live/root/virt/agent/operation" @@ -48,6 +49,32 @@ type hostDriver interface { type Manager struct { statePath string driver hostDriver + gate sync.RWMutex +} + +// PortLease pins the exact committed projection while a VM seed and domain +// definition are changed. Multiple unrelated VM ports may hold read leases at +// once; Apply/Remove require the writer side and therefore cannot invalidate +// their authorization midway through a cold restart. +type PortLease struct { + manager *Manager + file *os.File + once sync.Once +} + +func (lease *PortLease) Release() { + if lease == nil { + return + } + lease.once.Do(func() { + if lease.file != nil { + _ = unix.Flock(int(lease.file.Fd()), unix.LOCK_UN) + _ = lease.file.Close() + } + if lease.manager != nil { + lease.manager.gate.RUnlock() + } + }) } func Open(statePath string) (*Manager, error) { @@ -243,31 +270,58 @@ func (m *Manager) Inspect(operationID, vpcKey string, revision uint64, digest st // the NIC, and reverse that order during detach. func (m *Manager) AuthorizeLocalPort(vpcKey string, revision uint64, digest, cidr, gateway, ip, mac string) (Network, error) { + authorized, lease, err := m.AcquireLocalPortLease( + vpcKey, revision, digest, cidr, gateway, ip, mac, + ) + if lease != nil { + lease.Release() + } + return authorized, err +} + +// AcquireLocalPortLease authorizes one exact port and keeps the shared state +// lease until the caller has durably completed or rolled back its VM mutation. +func (m *Manager) AcquireLocalPortLease(vpcKey string, revision uint64, digest, + cidr, gateway, ip, mac string) (Network, *PortLease, error) { if !vpcKeyPattern.MatchString(vpcKey) || revision == 0 || !lowerHexDigest.MatchString(digest) || !canonicalMACPattern.MatchString(mac) { - return Network{}, invalid("invalid_vpc_port_authorization", "VPC port authorization identity is invalid") + return Network{}, nil, invalid("invalid_vpc_port_authorization", "VPC port authorization identity is invalid") + } + if m == nil { + return Network{}, nil, unavailable("vpc_router_unavailable", "VPC host router is unavailable", nil) + } + m.gate.RLock() + file, err := m.lockFile(unix.LOCK_SH) + if err != nil { + m.gate.RUnlock() + return Network{}, nil, err } + lease := &PortLease{manager: m, file: file} var authorized Network - err := m.withLock(func() error { - document, loadErr := m.load() - if loadErr != nil { - return loadErr - } + document, err := m.load() + if err == nil { entry, exists := document.network(vpcKey) if !exists || entry.Network.DesiredRevision != revision || entry.Digest != digest || entry.Network.CIDR != cidr || entry.Network.Gateway != gateway { - return unavailable("vpc_host_projection_not_ready", "VPC host routing projection is not ready for this port", nil) - } - for _, port := range entry.Network.LocalPorts { - if port.IP == ip && port.MAC == mac { - authorized = entry.Network - return nil + err = unavailable("vpc_host_projection_not_ready", "VPC host routing projection is not ready for this port", nil) + } else { + for _, port := range entry.Network.LocalPorts { + if port.IP == ip && port.MAC == mac { + authorized = entry.Network + break + } + } + if authorized.VPCKey == "" { + err = unavailable("vpc_host_projection_not_ready", "VPC port is absent from the committed host routing projection", nil) } } - return unavailable("vpc_host_projection_not_ready", "VPC port is absent from the committed host routing projection", nil) - }) - return authorized, err + } + if err != nil { + lease.Release() + return Network{}, nil, err + } + return authorized, lease, nil } func (m *Manager) rollback(before, after map[string]Network, cause error) error { @@ -288,18 +342,33 @@ func (m *Manager) withLock(call func() error) error { if m == nil { return unavailable("vpc_router_unavailable", "VPC host router is unavailable", nil) } + m.gate.Lock() + defer m.gate.Unlock() + file, err := m.lockFile(unix.LOCK_EX) + if err != nil { + return err + } + defer file.Close() + defer unix.Flock(int(file.Fd()), unix.LOCK_UN) //nolint:errcheck + return call() +} + +func (m *Manager) lockFile(mode int) (*os.File, error) { lockPath := m.statePath + ".lock" descriptor, err := unix.Open(lockPath, unix.O_CREAT|unix.O_RDWR|unix.O_CLOEXEC|unix.O_NOFOLLOW, 0o600) if err != nil { - return unavailable("vpc_state_unavailable", "VPC host state lock could not be opened", err) + return nil, unavailable("vpc_state_unavailable", "VPC host state lock could not be opened", err) } file := os.NewFile(uintptr(descriptor), lockPath) - defer file.Close() - if err := unix.Flock(descriptor, unix.LOCK_EX); err != nil { - return unavailable("vpc_state_unavailable", "VPC host state lock could not be acquired", err) + if file == nil { + _ = unix.Close(descriptor) + return nil, unavailable("vpc_state_unavailable", "VPC host state lock could not be opened", nil) } - defer unix.Flock(descriptor, unix.LOCK_UN) //nolint:errcheck - return call() + if err := unix.Flock(descriptor, mode); err != nil { + _ = file.Close() + return nil, unavailable("vpc_state_unavailable", "VPC host state lock could not be acquired", err) + } + return file, nil } func (m *Manager) load() (stateDocument, error) { diff --git a/agent/vpcrouting/types_test.go b/agent/vpcrouting/types_test.go index 88b7b717..7a1e07d8 100644 --- a/agent/vpcrouting/types_test.go +++ b/agent/vpcrouting/types_test.go @@ -5,6 +5,7 @@ import ( "errors" "reflect" "testing" + "time" "git.luoli.live/root/virt/agent/operation" ) @@ -184,6 +185,63 @@ func TestManagerAuthorizesOnlyExactCommittedLocalPort(t *testing.T) { } } +func TestPortLeasesShareReadSideAndBlockProjectionWriter(t *testing.T) { + manager := testManager(t, &fakeDriver{}) + initial := applyRequest(t, 1, "operation-vpc-lease-source") + if _, err := manager.Apply(t.Context(), initial); err != nil { + t.Fatal(err) + } + _, first, err := manager.AcquireLocalPortLease( + testVPCKey, 1, initial.DesiredDigest, initial.Network.CIDR, + initial.Network.Gateway, "10.60.0.2", "02:00:00:00:00:02", + ) + if err != nil { + t.Fatal(err) + } + defer first.Release() + _, second, err := manager.AcquireLocalPortLease( + testVPCKey, 1, initial.DesiredDigest, initial.Network.CIDR, + initial.Network.Gateway, "10.60.0.9", "02:00:00:00:00:09", + ) + if err != nil { + t.Fatal(err) + } + defer second.Release() + + started := make(chan struct{}) + done := make(chan error, 1) + writer := applyRequest(t, 2, "operation-vpc-lease-writer") + go func() { + close(started) + _, applyErr := manager.Apply( + context.Background(), + writer, + ) + done <- applyErr + }() + <-started + select { + case err := <-done: + t.Fatalf("projection writer crossed active port leases: %v", err) + case <-time.After(50 * time.Millisecond): + } + first.Release() + select { + case err := <-done: + t.Fatalf("one remaining port lease did not fence writer: %v", err) + case <-time.After(50 * time.Millisecond): + } + second.Release() + select { + case err := <-done: + if err != nil { + t.Fatalf("projection writer failed after leases released: %v", err) + } + case <-time.After(2 * time.Second): + t.Fatal("projection writer stayed blocked after all port leases released") + } +} + func TestManagerRollsBackOrFencesPartialHostMutation(t *testing.T) { applyFailure := errors.New("injected apply failure") driver := &fakeDriver{errors: []error{applyFailure, nil}} diff --git a/deploy/virt-cloud-vpc-worker@.service b/deploy/virt-cloud-vpc-worker@.service new file mode 100644 index 00000000..b4ac8783 --- /dev/null +++ b/deploy/virt-cloud-vpc-worker@.service @@ -0,0 +1,24 @@ +[Unit] +Description=Virt Cloud VPC batch worker %i +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +User=apache +Group=apache +WorkingDirectory=/home/virt-cloud/web +ExecStart=/usr/bin/php /home/virt-cloud/web/xcat cloudWorker --vpc-batch-only +Restart=on-failure +RestartSec=3 +UMask=0077 +NoNewPrivileges=true +PrivateTmp=true +ProtectHome=false +ProtectSystem=strict +ReadWritePaths=-/home/virt-cloud/runtime/php +ReadWritePaths=-/home/virt-cloud/web/storage +TimeoutStopSec=30 + +[Install] +WantedBy=multi-user.target diff --git a/docs/ADMIN_SYSTEM_LOGS.md b/docs/ADMIN_SYSTEM_LOGS.md index 77402373..d9b1a27b 100644 --- a/docs/ADMIN_SYSTEM_LOGS.md +++ b/docs/ADMIN_SYSTEM_LOGS.md @@ -10,6 +10,7 @@ - `virt-cloud-policy-worker.service` - `virt-cloud-policy-worker@1.service` - `virt-cloud-policy-worker@2.service` +- `virt-cloud-vpc-worker@1.service` 到 `virt-cloud-vpc-worker@8.service` - `virt-cloud-agent-gateway.service` 接口使用参数数组启动 `/usr/bin/journalctl`(系统没有该路径时使用 `/bin/journalctl`),不会经过 shell,也不接受任意 unit、命令或文件路径。单次最多读取 200 条,使用 `--cursor` 加 `--reverse` 加载更早记录;journalctl 会从游标向更早记录移动,返回结果通常不包含边界,服务端也会防御性移除意外返回的边界记录。这样可以避开部分旧 systemd 版本在 `--after-cursor` 与 unit 筛选组合下漏掉首条记录的问题。带时间下界的游标页会在 PHP 中继续应用下界,因为 systemd 不允许 `--since` 与游标同时出现。消息预览、详情字段、输出、stderr 和子进程运行时间都有上限。每条列表记录保留 journal cursor、服务和 UTC 微秒时间戳,点击后通过 `GET /api/v1/admin/system-logs/detail?cursor=...&unit=...×tamp=...` 在该时间戳窗口内重新读取,再校验返回 cursor,避免不同 systemd 版本的 cursor seek 差异影响详情;日志轮转或游标不存在时返回 404。命令行、可执行文件和 core-dump 字段不会返回,名称明显表示密码、令牌、密钥、凭据或授权信息的结构化字段会隐藏值,消息中的常见凭据形式也会遮罩。日志内容可能包含运行时上下文,因此接口仅对管理员开放,并设置 `private, no-store`。 diff --git a/docs/DEPLOYMENT.md b/docs/DEPLOYMENT.md index 2e50afaa..3d91fcd8 100644 --- a/docs/DEPLOYMENT.md +++ b/docs/DEPLOYMENT.md @@ -190,14 +190,18 @@ PHP workers 不依赖本机虚拟化服务;它们先持久化任务,再通 cp /home/virt-cloud/deploy/virt-cloud-worker.service /etc/systemd/system/ cp /home/virt-cloud/deploy/virt-cloud-lifecycle.service /etc/systemd/system/ cp /home/virt-cloud/deploy/virt-cloud-policy-worker@.service /etc/systemd/system/ +cp /home/virt-cloud/deploy/virt-cloud-vpc-worker@.service /etc/systemd/system/ cp /home/virt-cloud/deploy/virt-cloud-recovery-worker.service /etc/systemd/system/ systemctl daemon-reload systemctl enable --now virt-cloud-worker.service \ virt-cloud-recovery-worker.service virt-cloud-lifecycle.service \ - virt-cloud-policy-worker@1.service virt-cloud-policy-worker@2.service + virt-cloud-policy-worker@1.service virt-cloud-policy-worker@2.service \ + virt-cloud-vpc-worker@{1..8}.service ``` -普通 worker 负责购买、创建、重装、实例升级、邮件和最终删除;至少两个策略 worker 处理到期断网、网络恢复与删除前策略;lifecycle 生成续费账单并推进账期、流量、宽限和删除状态。`virt-cloud-recovery-worker.service` 是必需的独立 operation 恢复进程,覆盖策略表中的实例电源、重装/救援、公网 IP、实例设置、实例升级、快照、防火墙、删除、生命周期网络、模板与 ISO Agent 叶子任务。它只做 operation-bound inspection/栅栏审计、持久晚到回执的 PHP 收敛或聚合父任务的本地汇总,不投递邮件,也绝不重放原 mutation。离线、身份冲突或账本异常按固定低频间隔持续核对,没有次数或总时长上限。它与普通队列隔离,避免某个离线 Agent 的核对占住正常 worker。 +普通 worker 负责购买、创建、重装、实例升级、邮件和最终删除;至少两个策略 worker 处理到期断网、网络恢复与删除前策略;lifecycle 生成续费账单并推进账期、流量、宽限和删除状态。VPC 批量加入/移出没有实例数量上限:面板会在一个数据库事务中释放同一批次的全部 VM 子任务,实际同一时刻运行多少台由 `virt-cloud-vpc-worker@N.service` 的实例数和各宿主机 Agent 的安全 admission 决定,超出 worker 并发度的任务仍已持久接收并等待空闲进程,不依赖浏览器。上例启用 8 个专用 worker;可按面板 CPU、数据库连接和宿主机冷启动能力增减实例数,而不是在页面设置固定的“最多 4 台”。未安装专用 unit 时普通 worker 仍会安全消费这些任务,但吞吐量较低。 + +`virt-cloud-recovery-worker.service` 是必需的独立 operation 恢复进程,覆盖策略表中的实例电源、重装/救援、公网 IP、实例设置、实例升级、快照、防火墙、删除、生命周期网络、模板与 ISO Agent 叶子任务。它只做 operation-bound inspection/栅栏审计、持久晚到回执的 PHP 收敛或聚合父任务的本地汇总,不投递邮件,也绝不重放原 mutation。离线、身份冲突或账本异常按固定低频间隔持续核对,没有次数或总时长上限。它与普通队列隔离,避免某个离线 Agent 的核对占住正常 worker。 恢复进程也覆盖已派发开通任务的失败收尾:完整缺失证据回执可直接完成本地退款和资源释放;回执不足时,只读检查原 operation,确认实例及托管资源均不存在后才收尾。升级此能力需同时更新面板代码与宿主机 Agent,并重启已有 worker/recovery worker 使其加载新代码;无需数据库迁移或新增 service。旧 Agent 拒绝开通自动检查时会保留订单占用并低频重试,不会绕过授权退款。已有失败且效果未知的开通订单会被后台领取,不必重发开通请求。 @@ -383,7 +387,8 @@ location = /agent/console { journalctl -u virt-cloud-agent.service -n 200 --no-pager journalctl -u virt-cloud-worker.service -u virt-cloud-recovery-worker.service \ -u virt-cloud-lifecycle.service \ - -u 'virt-cloud-policy-worker@*' -n 200 --no-pager + -u 'virt-cloud-policy-worker@*' -u 'virt-cloud-vpc-worker@*' \ + -n 200 --no-pager journalctl -u virt-cloud-agent-gateway.service -n 200 --no-pager ``` diff --git a/web/app/Command/CloudSetup.php b/web/app/Command/CloudSetup.php index a2b07df3..3ef1b61e 100644 --- a/web/app/Command/CloudSetup.php +++ b/web/app/Command/CloudSetup.php @@ -8,7 +8,7 @@ use App\Services\Config; /** Initializes only the clean-break PHP business and Agent log schemas. */ final class CloudSetup { - private const SCHEMA_VERSION = 'go-agent-wss-v13-vpc-l3-v3'; + private const SCHEMA_VERSION = 'go-agent-wss-v13-vpc-l3-v4-batch'; private const MAIN_TABLES = [ 'cloud_regions', 'cloud_network_lines', 'cloud_resource_pools', 'cloud_hosts', 'cloud_host_agents', @@ -21,7 +21,8 @@ final class CloudSetup 'cloud_host_vpc_transports', 'cloud_vpc_cidr_pools', 'cloud_host_capacity_policies', 'cloud_public_ips', 'cloud_offers', 'cloud_instances', 'cloud_vpcs', 'cloud_vpc_ports', - 'cloud_vpc_host_states', 'cloud_instance_firewalls', + 'cloud_vpc_host_states', 'cloud_vpc_membership_batches', + 'cloud_vpc_membership_batch_items', 'cloud_instance_firewalls', 'cloud_instance_firewall_versions', 'cloud_instance_firewall_rules', 'cloud_platform_test_policy_state', 'cloud_platform_test_policy_rules', 'cloud_instance_snapshots', @@ -54,7 +55,7 @@ final class CloudSetup 'id', 'operation_id', 'request_idempotency_key', 'request_fingerprint', 'parent_job_id', 'user_id', 'host_id', 'backup_id', 'backup_restore_id', 'backup_recovery_id', - 'vpc_id', 'vpc_port_id', + 'vpc_id', 'vpc_port_id', 'vpc_batch_id', 'kind', 'execution_mode', 'phase', 'outcome', 'status', 'attempts', 'dispatch_attempts', 'dispatch_deadline_at', 'dispatch_started_at', 'agent_accepted_at', @@ -96,6 +97,16 @@ final class CloudSetup 'sync_state', 'last_operation_id', 'last_error_code', 'last_success_at', 'last_failure_at', 'created_at', 'updated_at', ], + 'cloud_vpc_membership_batches' => [ + 'id', 'batch_key', 'vpc_id', 'user_id', 'parent_job_id', + 'kind', 'state', 'desired_revision', 'active_claim', + 'last_error_code', 'terminal_at', 'created_at', 'updated_at', + ], + 'cloud_vpc_membership_batch_items' => [ + 'id', 'batch_id', 'user_id', 'instance_id', 'vpc_port_id', + 'host_id', 'child_job_id', 'state', 'last_error_code', + 'created_at', 'updated_at', + ], 'cloud_agent_completion_events' => [ 'event_id', 'host_id', 'operation_id', 'payload_sha256', 'outcome', 'effect_may_have_applied', 'safe_to_resubmit', @@ -229,6 +240,14 @@ final class CloudSetup 'cloud_vpc_host_states' => [ 'chk_cloud_vpc_host_state', 'chk_cloud_vpc_host_revision', ], + 'cloud_vpc_membership_batches' => [ + 'chk_cloud_vpc_membership_batch_kind', + 'chk_cloud_vpc_membership_batch_state', + 'chk_cloud_vpc_membership_batch_claim', + ], + 'cloud_vpc_membership_batch_items' => [ + 'chk_cloud_vpc_membership_batch_item_state', + ], 'cloud_backup_subscriptions' => [ 'chk_cloud_backup_subscription_frequency', 'chk_cloud_backup_subscription_weekday', @@ -284,13 +303,26 @@ final class CloudSetup 'PRIMARY', 'cloud_vpc_host_identity', 'cloud_vpc_host_operation', 'cloud_vpc_host_sync', ], + 'cloud_vpc_membership_batches' => [ + 'PRIMARY', 'cloud_vpc_membership_batch_key', + 'cloud_vpc_membership_batch_job', + 'cloud_vpc_membership_batch_active', + 'cloud_vpc_membership_batch_owner', + ], + 'cloud_vpc_membership_batch_items' => [ + 'PRIMARY', 'cloud_vpc_membership_batch_instance', + 'cloud_vpc_membership_batch_port', + 'cloud_vpc_membership_batch_child', + 'cloud_vpc_membership_batch_item_owner', + 'cloud_vpc_membership_batch_item_state', + ], 'cloud_jobs' => [ 'PRIMARY', 'uniq_cloud_operation', 'uniq_cloud_job_request', 'uniq_cloud_job_completion_event', 'uniq_cloud_provision_order', 'cloud_job_poll', 'cloud_job_receipt_deadline', 'cloud_job_backup', 'cloud_job_backup_restore', 'cloud_job_backup_recovery', - 'cloud_job_vpc', 'cloud_job_vpc_port', + 'cloud_job_vpc', 'cloud_job_vpc_port', 'cloud_job_vpc_batch', ], 'cloud_agent_completion_events' => [ 'PRIMARY', 'uniq_cloud_completion_operation', diff --git a/web/app/Command/CloudWorker.php b/web/app/Command/CloudWorker.php index 99fb1bd6..25110fcd 100644 --- a/web/app/Command/CloudWorker.php +++ b/web/app/Command/CloudWorker.php @@ -12,20 +12,23 @@ final class CloudWorker { $once = in_array('--once', $argv, true); $policyOnly = in_array('--policy-only', $argv, true); + $vpcBatchOnly = in_array('--vpc-batch-only', $argv, true); $recoveryOnly = in_array('--recovery-only', $argv, true); - if ($policyOnly && $recoveryOnly) { + if (($policyOnly ? 1 : 0) + ($vpcBatchOnly ? 1 : 0) + + ($recoveryOnly ? 1 : 0) > 1) { throw new \InvalidArgumentException( - '--policy-only 与 --recovery-only 不能同时使用。' + 'worker 专用模式不能同时使用。' ); } $heartbeat ??= CloudRuntimeHeartbeat::start( - $recoveryOnly ? 'recovery' : ($policyOnly ? 'policy' : 'normal'), + $recoveryOnly ? 'recovery' : ($policyOnly ? 'policy' + : ($vpcBatchOnly ? 'vpc' : 'normal')), $once ); try { $worker = $recoveryOnly ? new CloudAutomaticOperationRecoveryService() - : new CloudWorkerService(null, $policyOnly); + : new CloudWorkerService(null, $policyOnly, $vpcBatchOnly); do { $errors = []; $heartbeat->busy( @@ -51,7 +54,7 @@ final class CloudWorker $heartbeat->error($errors[0]); } // SMTP remains isolated from lifecycle and policy workers. - if (!$policyOnly && !$recoveryOnly) { + if (!$policyOnly && !$vpcBatchOnly && !$recoveryOnly) { $errors = array_merge($errors, self::deliverNotifications($heartbeat)); } if ($errors === []) { diff --git a/web/app/Command/XCat.php b/web/app/Command/XCat.php index f46437e2..c192201e 100644 --- a/web/app/Command/XCat.php +++ b/web/app/Command/XCat.php @@ -98,7 +98,7 @@ class XCat echo "Virt Cloud CLI\n" . " php xcat createAdmin [name]\n" . " php xcat cloudInit\n" - . " php xcat cloudWorker [--once] [--policy-only|--recovery-only]\n" + . " php xcat cloudWorker [--once] [--policy-only|--vpc-batch-only|--recovery-only]\n" . " php xcat cloudLifecycle [--once]\n" . " php xcat cloudRepairInstanceMetadata [--apply] [--limit=N] [--after-id=N]\n" . " php xcat setTelegram\n" diff --git a/web/app/Controllers/CloudVpcController.php b/web/app/Controllers/CloudVpcController.php index 60fd9741..f1058b89 100644 --- a/web/app/Controllers/CloudVpcController.php +++ b/web/app/Controllers/CloudVpcController.php @@ -6,6 +6,7 @@ use App\Services\Cloud\CloudAccess; use App\Services\Cloud\CloudError; use App\Services\Cloud\CloudInput; use App\Services\Cloud\CloudVpcFeature; +use App\Services\Cloud\CloudVpcMembershipBatchService; use App\Services\Cloud\CloudVpcPortService; use App\Services\Cloud\CloudVpcService; @@ -52,6 +53,21 @@ final class CloudVpcController ))->withHeader('Cache-Control', 'private, no-store'); } + /** Submit one durable server-side attach/detach batch. */ + public function membershipBatch($request, $response, $args) + { + $actor = $this->actor($request, true); + $result = CloudVpcMembershipBatchService::start( + $actor, + (string) ($args['vpc_key'] ?? ''), + CloudInput::object($request) + ); + return $response->withJson([ + 'task_id' => (string) $result['job']->id, + 'status_url' => '/api/v1/tasks/' . (int) $result['job']->id, + ], 202)->withHeader('Cache-Control', 'private, no-store'); + } + /** Read, attach, or detach the one optional VPC port on an instance. */ public function port($request, $response, $args) { diff --git a/web/app/Models/CloudJob.php b/web/app/Models/CloudJob.php index e8e6e2be..2c8d2120 100644 --- a/web/app/Models/CloudJob.php +++ b/web/app/Models/CloudJob.php @@ -13,6 +13,7 @@ final class CloudJob extends Model 'backup_recovery_id' => 'int', 'vpc_id' => 'int', 'vpc_port_id' => 'int', + 'vpc_batch_id' => 'int', ]; /** diff --git a/web/app/Models/CloudVpcMembershipBatch.php b/web/app/Models/CloudVpcMembershipBatch.php new file mode 100644 index 00000000..ef93818b --- /dev/null +++ b/web/app/Models/CloudVpcMembershipBatch.php @@ -0,0 +1,21 @@ + 'int', + 'vpc_id' => 'int', + 'user_id' => 'int', + 'parent_job_id' => 'int', + 'desired_revision' => 'int', + 'active_claim' => 'int', + 'terminal_at' => 'int', + 'created_at' => 'int', + 'updated_at' => 'int', + ]; +} diff --git a/web/app/Models/CloudVpcMembershipBatchItem.php b/web/app/Models/CloudVpcMembershipBatchItem.php new file mode 100644 index 00000000..46f15712 --- /dev/null +++ b/web/app/Models/CloudVpcMembershipBatchItem.php @@ -0,0 +1,21 @@ + 'int', + 'batch_id' => 'int', + 'user_id' => 'int', + 'instance_id' => 'int', + 'vpc_port_id' => 'int', + 'host_id' => 'int', + 'child_job_id' => 'int', + 'created_at' => 'int', + 'updated_at' => 'int', + ]; +} diff --git a/web/app/Services/Cloud/CloudJobService.php b/web/app/Services/Cloud/CloudJobService.php index 1ea59302..a6874379 100644 --- a/web/app/Services/Cloud/CloudJobService.php +++ b/web/app/Services/Cloud/CloudJobService.php @@ -3995,7 +3995,7 @@ final class CloudJobService }; // Per-host child jobs are implementation details of one public batch. // Only their parent appears in the task center. - $base = CloudJob::query()->where( + $base = CloudJob::query()->whereNull('vpc_batch_id')->where( static function ($public): void { $public->whereNull('parent_job_id') ->orWhere(static function ($currentCatalog): void { @@ -4050,6 +4050,7 @@ final class CloudJobService public static function unreadCount(array $actor): int { $query = CloudJob::where('phase', 'terminal')->whereNull('read_at') + ->whereNull('vpc_batch_id') ->where('user_id', (int) ($actor['user_id'] ?? 0)) ->where(static function ($public): void { $public->whereNull('parent_job_id') diff --git a/web/app/Services/Cloud/CloudOperationCatalog.php b/web/app/Services/Cloud/CloudOperationCatalog.php index abdab230..90d08b89 100644 --- a/web/app/Services/Cloud/CloudOperationCatalog.php +++ b/web/app/Services/Cloud/CloudOperationCatalog.php @@ -39,6 +39,7 @@ final class CloudOperationCatalog 'vpc_host_remove' => '清理 VPC 宿主机路由', 'vpc_port_attach' => '连接 VPC 私网', 'vpc_port_detach' => '断开 VPC 私网', + 'vpc_membership_batch' => '批量调整 VPC 实例', 'service_suspend' => '到期停用', 'network_suspend' => '限制网络', 'network_resume' => '恢复网络', diff --git a/web/app/Services/Cloud/CloudOperationErrorCatalog.php b/web/app/Services/Cloud/CloudOperationErrorCatalog.php index 73c9497c..3ff51e7b 100644 --- a/web/app/Services/Cloud/CloudOperationErrorCatalog.php +++ b/web/app/Services/Cloud/CloudOperationErrorCatalog.php @@ -116,6 +116,9 @@ final class CloudOperationErrorCatalog 'vpc_port_attach_failed' => 'VPC 私网连接失败;宿主机路由保持不变,可核对后重试', 'vpc_port_detach_failed' => 'VPC 私网断开失败;当前端口账本保持锁定,可核对后重试', 'vpc_result_uncertain' => 'VPC 操作结果暂时无法确认;系统不会重复执行,资源保持锁定并等待核对', + 'vpc_batch_attach_failed' => '部分实例未能加入 VPC;已成功实例保持生效,失败实例可在状态核对后重试', + 'vpc_batch_detach_failed' => '所选实例均未能移出 VPC;现有私网连接保持不变', + 'vpc_batch_detach_partial' => '部分实例已移出 VPC;其余实例保持连接,可在状态核对后重试', 'force_required' => '虚拟机未在正常关机等待期内停止;必须由超级管理员明确确认数据丢失风险后发起独立强制删除任务', 'force_delete_queued' => '已创建独立强制删除任务,原正常关机任务不再自动重试', 'force_delete_completed' => '原正常关机任务未完成,后续独立强制删除任务已完成', @@ -589,6 +592,9 @@ final class CloudOperationErrorCatalog if (array_is_list($value)) { return []; } + if (!$nested && $kind === 'vpc_membership_batch') { + return self::vpcMembershipBatchResult($value); + } $result = []; foreach ($value as $key => $item) { $name = self::normalizeKey((string) $key); @@ -648,6 +654,34 @@ final class CloudOperationErrorCatalog return $result; } + /** + * Return the complete public result of one durable VPC membership batch. + * + * The batch key is deliberately excluded: the task id is the public + * identity, while this DTO only reports the bounded operation outcome. + */ + private static function vpcMembershipBatchResult(array $value): array + { + $action = $value['action'] ?? null; + $total = $value['total'] ?? null; + $succeeded = $value['succeeded'] ?? null; + $failed = $value['failed'] ?? null; + if (!in_array($action, ['attach', 'detach'], true) + || !is_int($total) || $total < 1 + || !is_int($succeeded) || $succeeded < 0 + || !is_int($failed) || $failed < 0 + || $succeeded > $total || $failed > $total + || $succeeded + $failed !== $total) { + return []; + } + return [ + 'action' => $action, + 'total' => $total, + 'succeeded' => $succeeded, + 'failed' => $failed, + ]; + } + private static function safeResultScalar(string $key, $value) { if (is_bool($value)) { diff --git a/web/app/Services/Cloud/CloudRuntimeHealthService.php b/web/app/Services/Cloud/CloudRuntimeHealthService.php index a8e95824..82f8a72f 100644 --- a/web/app/Services/Cloud/CloudRuntimeHealthService.php +++ b/web/app/Services/Cloud/CloudRuntimeHealthService.php @@ -92,6 +92,7 @@ final class CloudRuntimeHealthService $heartbeats = []; } foreach (['worker' => '普通任务处理', 'policy' => '策略任务处理', + 'vpc' => 'VPC 批量任务处理', 'recovery' => '任务安全收敛', 'lifecycle' => '到期与流量调度'] as $key => $label) { $found = null; diff --git a/web/app/Services/Cloud/CloudRuntimeHeartbeat.php b/web/app/Services/Cloud/CloudRuntimeHeartbeat.php index 698eae30..03ac38b7 100644 --- a/web/app/Services/Cloud/CloudRuntimeHeartbeat.php +++ b/web/app/Services/Cloud/CloudRuntimeHeartbeat.php @@ -6,6 +6,7 @@ final class CloudRuntimeHeartbeat { private const KINDS = ['normal' => ['worker', '普通任务 worker'], 'policy' => ['policy', '策略任务 worker'], + 'vpc' => ['vpc', 'VPC 批量任务 worker'], 'recovery' => ['recovery', '任务安全收敛 worker'], 'lifecycle' => ['lifecycle', '生命周期调度']]; private const PHASES = ['startup' => '启动', 'worker_tick' => '任务消费或核对', @@ -212,7 +213,7 @@ final class CloudRuntimeHeartbeat if (++$scanned > self::SCAN_LIMIT || $removed >= 32) { break; } - if (preg_match('/^(normal|policy|recovery|lifecycle)-[1-9][0-9]*-[a-f0-9]{24}\.json(?:\.tmp)?$/D', + if (preg_match('/^(normal|policy|vpc|recovery|lifecycle)-[1-9][0-9]*-[a-f0-9]{24}\.json(?:\.tmp)?$/D', $entry->getFilename()) !== 1) { continue; } @@ -302,7 +303,7 @@ final class CloudRuntimeHeartbeat $incomplete = true; break; } - if (preg_match('/^(normal|policy|recovery|lifecycle)-[1-9][0-9]*-[a-f0-9]{24}\.json$/D', + if (preg_match('/^(normal|policy|vpc|recovery|lifecycle)-[1-9][0-9]*-[a-f0-9]{24}\.json$/D', $entry->getFilename(), $match) !== 1) { continue; } diff --git a/web/app/Services/Cloud/CloudSystemJournalService.php b/web/app/Services/Cloud/CloudSystemJournalService.php index bd5b52da..de02a872 100644 --- a/web/app/Services/Cloud/CloudSystemJournalService.php +++ b/web/app/Services/Cloud/CloudSystemJournalService.php @@ -34,6 +34,14 @@ final class CloudSystemJournalService 'virt-cloud-policy-worker.service', 'virt-cloud-policy-worker@1.service', 'virt-cloud-policy-worker@2.service', + 'virt-cloud-vpc-worker@1.service', + 'virt-cloud-vpc-worker@2.service', + 'virt-cloud-vpc-worker@3.service', + 'virt-cloud-vpc-worker@4.service', + 'virt-cloud-vpc-worker@5.service', + 'virt-cloud-vpc-worker@6.service', + 'virt-cloud-vpc-worker@7.service', + 'virt-cloud-vpc-worker@8.service', 'virt-cloud-agent-gateway.service', ]; diff --git a/web/app/Services/Cloud/CloudVpcMembershipBatchService.php b/web/app/Services/Cloud/CloudVpcMembershipBatchService.php new file mode 100644 index 00000000..8f4e3042 --- /dev/null +++ b/web/app/Services/Cloud/CloudVpcMembershipBatchService.php @@ -0,0 +1,735 @@ + all port jobs; detach = all port jobs -> + * one route generation. There is no selection-count limit in this domain. + */ +final class CloudVpcMembershipBatchService +{ + private const PARENT_KIND = 'vpc_membership_batch'; + private const TERMINAL_STATES = ['succeeded', 'failed']; + + public static function start(array $actor, string $vpcKey, + array $input): array + { + CloudInput::only($input, [ + 'action', 'instance_ids', 'confirm_restart', + ]); + if (preg_match('/^[a-f0-9]{32}$/D', $vpcKey) !== 1) { + throw new CloudError('VPC 标识无效', 404); + } + $action = $input['action'] ?? null; + if (!in_array($action, ['attach', 'detach'], true)) { + throw new CloudError('VPC 批量操作类型无效'); + } + if (($input['confirm_restart'] ?? null) !== true) { + throw new CloudError('请确认所选实例会自动关机并冷启动', 409); + } + $instanceIds = self::instanceIds($input['instance_ids'] ?? null); + $userId = (int) ($actor['user_id'] ?? 0); + $requestKey = $actor['request_idempotency_key'] ?? null; + if ($userId < 1 || !is_string($requestKey) + || preg_match('/^[A-Za-z0-9_.-]{16,128}$/D', $requestKey) !== 1) { + throw new CloudError('请求幂等标识无效', 400); + } + $fingerprint = hash('sha256', json_encode([ + 'kind' => self::PARENT_KIND, + 'user_id' => $userId, + 'vpc_key' => $vpcKey, + 'action' => $action, + 'instance_ids' => $instanceIds, + ], JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES + | JSON_UNESCAPED_UNICODE)); + $batchKey = substr(hash('sha256', "vpc-membership-batch\0" + . $userId . "\0" . $vpcKey . "\0" . $requestKey), 0, 32); + + return (new CloudVpcMembershipBatch())->getConnection()->transaction( + static function () use ( + $actor, $userId, $requestKey, $fingerprint, $batchKey, + $vpcKey, $action, $instanceIds + ): array { + $vpc = CloudVpc::query()->where('vpc_key', $vpcKey) + ->lockForUpdate()->first(); + if ($vpc === null || (int) $vpc->user_id !== $userId + || $vpc->active_claim === null) { + throw new CloudError('VPC 不存在或当前不可用', 404); + } + $prior = CloudJob::query()->where('user_id', $userId) + ->where('request_idempotency_key', $requestKey) + ->lockForUpdate()->first(); + if ($prior !== null) { + if ((string) $prior->kind !== self::PARENT_KIND + || !hash_equals((string) $prior->request_fingerprint, + $fingerprint)) { + throw new CloudError( + '同一请求标识不能用于不同的批量操作', 409 + ); + } + $batch = CloudVpcMembershipBatch::query() + ->where('parent_job_id', (int) $prior->id)->first(); + if ($batch === null || (int) $batch->vpc_id !== (int) $vpc->id) { + throw new CloudError('VPC 批次幂等账本不完整', 409); + } + return ['job' => $prior, 'batch' => self::dto($batch), + 'unchanged' => true]; + } + if (CloudVpcMembershipBatch::query() + ->where('vpc_id', (int) $vpc->id) + ->where('active_claim', 1)->lockForUpdate()->exists()) { + throw new CloudError('该 VPC 已有批量操作正在执行', 409); + } + CloudVpcPortService::assertBatchVpcSettledLocked($vpc); + + $instances = CloudInstance::query()->whereIn('id', $instanceIds) + ->orderBy('id')->lockForUpdate()->get()->keyBy('id'); + if ($instances->count() !== count($instanceIds)) { + throw new CloudError('部分实例不存在或状态已变化', 409); + } + foreach ($instanceIds as $instanceId) { + $instance = $instances->get($instanceId); + CloudVpcPortService::assertBatchInstanceLocked( + $instance, $actor + ); + if (CloudVpcPortService::batchInstanceRegionId($instance) + !== (int) $vpc->region_id) { + throw new CloudError('所选实例必须与 VPC 位于同一地区', 409); + } + } + + $parent = self::newParentJob( + $userId, $requestKey, $fingerprint, $vpc, $action, + $instanceIds + ); + $now = time(); + $batch = new CloudVpcMembershipBatch(); + $batch->batch_key = $batchKey; + $batch->vpc_id = (int) $vpc->id; + $batch->user_id = $userId; + $batch->parent_job_id = (int) $parent->id; + $batch->kind = $action; + $batch->state = $action === 'attach' ? 'routing' : 'ports'; + $batch->desired_revision = (int) $vpc->desired_revision; + $batch->active_claim = 1; + $batch->last_error_code = ''; + $batch->terminal_at = null; + $batch->created_at = $batch->updated_at = $now; + $batch->save(); + + if ($action === 'attach') { + self::startAttachLocked( + $batch, $vpc, $instances, $instanceIds + ); + } else { + self::startDetachLocked( + $batch, $vpc, $instances, $instanceIds + ); + } + return ['job' => $parent, 'batch' => self::dto($batch), + 'unchanged' => false]; + }, + 3 + ); + } + + /** @return list> */ + public static function activeForOwner(int $userId): array + { + if ($userId < 1) { + return []; + } + $batches = CloudVpcMembershipBatch::query() + ->where('user_id', $userId)->where('active_claim', 1) + ->orderBy('id')->get(); + return $batches->map(static fn ($batch): array => self::dto($batch)) + ->all(); + } + + /** Called inside the worker's child-finalization transaction. */ + public static function finishChildLocked(CloudJob $job, string $status, + $result, array $operation, CloudVpc $vpc, CloudVpcPort $port, + CloudInstance $instance): void + { + $batch = CloudVpcMembershipBatch::query() + ->where('id', (int) $job->vpc_batch_id) + ->lockForUpdate()->first(); + if ($batch === null || (int) $batch->vpc_id !== (int) $vpc->id + || (int) $batch->user_id !== (int) $job->user_id) { + throw new CloudError('VPC 批次与子任务账本不一致', 409); + } + if (in_array((string) $batch->state, self::TERMINAL_STATES, true)) { + return; + } + $parent = CloudJob::query()->where('id', (int) $batch->parent_job_id) + ->lockForUpdate()->first(); + if ($parent === null || (string) $parent->kind !== self::PARENT_KIND + || (string) $parent->phase === 'terminal') { + throw new CloudError('VPC 批次父任务账本不一致', 409); + } + + if (in_array((string) $job->kind, + ['vpc_host_apply', 'vpc_host_remove'], true)) { + $code = CloudVpcPortService::settleBatchHostJobLocked( + $job, $status, $result, $operation, $vpc + ); + self::advanceHostStageLocked( + $batch, $parent, $job, $status, $code, $vpc + ); + return; + } + self::settlePortChildLocked( + $batch, $parent, $job, $status, $result, $operation, + $vpc, $port, $instance + ); + } + + /** Preserve the batch fence when a child is terminalized as unknown. */ + public static function markChildUncertainLocked(CloudJob $job): void + { + if ($job->vpc_batch_id === null) { + return; + } + $batch = CloudVpcMembershipBatch::query() + ->where('id', (int) $job->vpc_batch_id) + ->lockForUpdate()->first(); + if ($batch === null + || in_array((string) $batch->state, self::TERMINAL_STATES, true)) { + return; + } + if (in_array((string) $job->kind, + ['vpc_port_attach', 'vpc_port_detach'], true)) { + CloudVpcMembershipBatchItem::query() + ->where('batch_id', (int) $batch->id) + ->where('child_job_id', (int) $job->id)->update([ + 'state' => 'unknown', + 'last_error_code' => 'vpc_result_uncertain', + 'updated_at' => time(), + ]); + } + $batch->state = 'blocked'; + $batch->last_error_code = 'vpc_result_uncertain'; + $batch->updated_at = time(); + $batch->save(); + } + + private static function startAttachLocked(CloudVpcMembershipBatch $batch, + CloudVpc $vpc, $instances, array $instanceIds): void + { + $revision = (int) $vpc->desired_revision + 1; + $focus = null; + foreach ($instanceIds as $instanceId) { + $instance = $instances->get($instanceId); + $port = CloudVpcPortService::reserveBatchPortLocked( + $instance, $vpc, (string) $batch->batch_key, $revision + ); + self::newItem($batch, $instance, $port, 'waiting_route'); + $focus ??= $port; + } + if ($focus === null) { + throw new CloudError('VPC 批次没有可执行实例', 409); + } + $vpc->desired_revision = $revision; + $vpc->state = 'provisioning'; + $vpc->updated_at = time(); + $vpc->save(); + $batch->desired_revision = $revision; + $batch->updated_at = time(); + $batch->save(); + if (CloudVpcPortService::queueBatchProjectionLocked( + $vpc, $focus, $revision, (int) $batch->id + ) === []) { + throw new CloudError('VPC 路由投影没有可执行宿主机', 503); + } + } + + private static function startDetachLocked(CloudVpcMembershipBatch $batch, + CloudVpc $vpc, $instances, array $instanceIds): void + { + $ports = CloudVpcPort::query()->whereIn('instance_id', $instanceIds) + ->where('active_claim', 1)->orderBy('instance_id') + ->lockForUpdate()->get()->keyBy('instance_id'); + if ($ports->count() !== count($instanceIds)) { + throw new CloudError('部分实例未连接该 VPC', 409); + } + foreach ($instanceIds as $instanceId) { + $instance = $instances->get($instanceId); + $port = $ports->get($instanceId); + if ((int) $port->vpc_id !== (int) $vpc->id + || (int) $port->user_id !== (int) $vpc->user_id + || (string) $port->state !== 'attached') { + throw new CloudError('所选实例的 VPC 连接状态已变化', 409); + } + $state = CloudVpcHostState::query() + ->where('vpc_id', (int) $vpc->id) + ->where('host_id', (int) $instance->host_id) + ->lockForUpdate()->first(); + if (!CloudVpcPortService::batchHostReady($state)) { + throw new CloudError('实例所在宿主机的 VPC 路由尚未收敛', 409); + } + $port->state = 'detaching'; + $port->last_error_code = ''; + $port->updated_at = time(); + $port->save(); + $item = self::newItem($batch, $instance, $port, 'queued'); + $job = CloudVpcPortService::queueBatchPortJobLocked( + 'vpc_port_detach', $instance, $vpc, $port, $state, + (int) $batch->id + ); + $item->child_job_id = (int) $job->id; + $item->updated_at = time(); + $item->save(); + } + $vpc->state = 'provisioning'; + $vpc->updated_at = time(); + $vpc->save(); + } + + private static function settlePortChildLocked( + CloudVpcMembershipBatch $batch, + CloudJob $parent, + CloudJob $job, + string $status, + $result, + array $operation, + CloudVpc $vpc, + CloudVpcPort $port, + CloudInstance $instance + ): void { + $item = CloudVpcMembershipBatchItem::query() + ->where('batch_id', (int) $batch->id) + ->where('instance_id', (int) $instance->id) + ->where('vpc_port_id', (int) $port->id) + ->lockForUpdate()->first(); + if ($item === null || (int) ($item->child_job_id ?? 0) !== (int) $job->id) { + throw new CloudError('VPC 批次实例明细与子任务不一致', 409); + } + $now = time(); + if ($status === 'succeeded') { + CloudVpcPortService::assertBatchPortResult( + $job, $vpc, $port, $instance, $result + ); + if ((string) $batch->kind === 'attach') { + $port->state = 'attached'; + $port->applied_revision = (int) $port->desired_revision; + $port->attached_at = $now; + } else { + $port->state = 'release_pending'; + } + $port->last_error_code = ''; + $port->updated_at = $now; + $port->save(); + $item->state = 'succeeded'; + $item->last_error_code = ''; + } elseif ($status === 'uncertain') { + $item->state = 'unknown'; + $item->last_error_code = 'vpc_result_uncertain'; + $port->last_error_code = 'vpc_result_uncertain'; + $port->updated_at = $now; + $port->save(); + } else { + $code = CloudOperationErrorCatalog::workerSummary( + (string) $job->kind, 'failed', $operation + ) ?? 'vpc_port_change_failed'; + if ((string) $batch->kind === 'detach') { + $port->state = 'attached'; + } + $port->last_error_code = substr($code, 0, 64); + $port->updated_at = $now; + $port->save(); + $item->state = 'failed'; + $item->last_error_code = substr($code, 0, 64); + } + $item->updated_at = $now; + $item->save(); + + $items = self::itemsLocked($batch); + if ($items->contains(static fn ($row): bool => + (string) $row->state === 'queued')) { + return; + } + if ($items->contains(static fn ($row): bool => + (string) $row->state === 'unknown')) { + self::blockLocked($batch, $vpc, 'vpc_result_uncertain'); + return; + } + if ((string) $batch->kind === 'attach') { + self::finishAttachPortsLocked($batch, $parent, $vpc, $items); + return; + } + self::beginDetachCleanupLocked($batch, $parent, $vpc, $items); + } + + private static function advanceHostStageLocked( + CloudVpcMembershipBatch $batch, + CloudJob $parent, + CloudJob $current, + string $currentStatus, + string $code, + CloudVpc $vpc + ): void { + $outcomes = self::hostStageOutcomes( + $batch, $current, $currentStatus + ); + if ($outcomes['pending'] > 0) { + return; + } + if ($outcomes['unknown'] > 0) { + self::blockLocked($batch, $vpc, + $code !== '' ? $code : 'vpc_result_uncertain'); + return; + } + if ($outcomes['failed'] > 0) { + if ((string) $batch->kind === 'attach') { + CloudVpcMembershipBatchItem::query() + ->where('batch_id', (int) $batch->id) + ->where('state', 'waiting_route')->update([ + 'state' => 'failed', + 'last_error_code' => $code !== '' + ? $code : 'vpc_host_apply_failed', + 'updated_at' => time(), + ]); + } + $vpc->state = 'degraded'; + $vpc->updated_at = time(); + $vpc->save(); + self::finishLocked($batch, $parent, false, + $outcomes['succeeded'] > 0, + $outcomes['succeeded'] === 0, + $code !== '' ? $code : 'vpc_host_apply_failed'); + return; + } + if ((string) $batch->kind === 'attach') { + self::releaseAttachPortJobsLocked($batch, $vpc); + return; + } + self::finishDetachCleanupLocked($batch, $parent, $vpc); + } + + private static function releaseAttachPortJobsLocked( + CloudVpcMembershipBatch $batch, + CloudVpc $vpc + ): void { + $items = self::itemsLocked($batch); + foreach ($items as $item) { + if ((string) $item->state !== 'waiting_route') { + throw new CloudError('VPC 加入批次明细状态不一致', 409); + } + $instance = CloudInstance::query() + ->where('id', (int) $item->instance_id) + ->lockForUpdate()->first(); + $port = CloudVpcPort::query() + ->where('id', (int) $item->vpc_port_id) + ->where('active_claim', 1)->lockForUpdate()->first(); + $state = $instance === null ? null : CloudVpcHostState::query() + ->where('vpc_id', (int) $vpc->id) + ->where('host_id', (int) $instance->host_id) + ->lockForUpdate()->first(); + if ($instance === null || $port === null + || !CloudVpcPortService::batchHostReady($state)) { + throw new CloudError('VPC 加入批次路由屏障未收敛', 409); + } + $job = CloudVpcPortService::queueBatchPortJobLocked( + 'vpc_port_attach', $instance, $vpc, $port, $state, + (int) $batch->id + ); + $item->child_job_id = (int) $job->id; + $item->state = 'queued'; + $item->updated_at = time(); + $item->save(); + } + $batch->state = 'ports'; + $batch->last_error_code = ''; + $batch->updated_at = time(); + $batch->save(); + } + + private static function finishAttachPortsLocked( + CloudVpcMembershipBatch $batch, + CloudJob $parent, + CloudVpc $vpc, + $items + ): void { + $failed = $items->where('state', 'failed')->count(); + $succeeded = $items->where('state', 'succeeded')->count(); + $vpc->state = $failed === 0 ? 'active' : 'degraded'; + $vpc->updated_at = time(); + $vpc->save(); + if ($failed === 0) { + self::finishLocked($batch, $parent, true, false, false, ''); + return; + } + self::finishLocked($batch, $parent, false, $succeeded > 0, + $succeeded === 0, 'vpc_batch_attach_failed'); + } + + private static function beginDetachCleanupLocked( + CloudVpcMembershipBatch $batch, + CloudJob $parent, + CloudVpc $vpc, + $items + ): void { + $succeeded = $items->where('state', 'succeeded'); + if ($succeeded->isEmpty()) { + $vpc->state = 'degraded'; + $vpc->updated_at = time(); + $vpc->save(); + self::finishLocked($batch, $parent, false, false, true, + 'vpc_batch_detach_failed'); + return; + } + $revision = (int) $vpc->desired_revision + 1; + CloudVpcPort::query()->whereIn('id', $succeeded->pluck('vpc_port_id')) + ->update(['desired_revision' => $revision, 'updated_at' => time()]); + $vpc->desired_revision = $revision; + $vpc->state = 'provisioning'; + $vpc->updated_at = time(); + $vpc->save(); + $batch->state = 'cleanup'; + $batch->desired_revision = $revision; + $batch->updated_at = time(); + $batch->save(); + $focus = CloudVpcPort::query() + ->where('id', (int) $succeeded->first()->vpc_port_id) + ->lockForUpdate()->first(); + if ($focus === null) { + throw new CloudError('VPC 移出批次清理端口缺失', 409); + } + $jobs = CloudVpcPortService::queueBatchProjectionLocked( + $vpc, $focus, $revision, (int) $batch->id + ); + if ($jobs === []) { + self::finishDetachCleanupLocked($batch, $parent, $vpc); + } + } + + private static function finishDetachCleanupLocked( + CloudVpcMembershipBatch $batch, + CloudJob $parent, + CloudVpc $vpc + ): void { + $items = self::itemsLocked($batch); + $now = time(); + foreach ($items->where('state', 'succeeded') as $item) { + $port = CloudVpcPort::query() + ->where('id', (int) $item->vpc_port_id) + ->where('active_claim', 1)->lockForUpdate()->first(); + if ($port === null || (string) $port->state !== 'release_pending') { + throw new CloudError('VPC 移出批次端口释放账本不一致', 409); + } + $port->state = 'released'; + $port->active_claim = null; + $port->last_operation_id = null; + $port->last_error_code = ''; + $port->released_at = $now; + $port->updated_at = $now; + $port->save(); + } + $failed = $items->where('state', 'failed')->count(); + $vpc->state = $failed === 0 ? 'active' : 'degraded'; + $vpc->updated_at = $now; + $vpc->save(); + if ($failed === 0) { + self::finishLocked($batch, $parent, true, false, false, ''); + } else { + self::finishLocked($batch, $parent, false, true, false, + 'vpc_batch_detach_partial'); + } + } + + private static function finishLocked(CloudVpcMembershipBatch $batch, + CloudJob $parent, bool $success, bool $effect, bool $safe, + string $code): void + { + $items = self::itemsLocked($batch); + $now = time(); + $batch->state = $success ? 'succeeded' : 'failed'; + $batch->active_claim = null; + $batch->last_error_code = $success ? '' : substr($code, 0, 64); + $batch->terminal_at = $now; + $batch->updated_at = $now; + $batch->save(); + CloudJobService::completeAggregateLocked( + $parent, + $success ? 'succeeded' : 'failed', + $effect, + $safe, + $success ? '' : $code, + [ + 'batch_key' => (string) $batch->batch_key, + 'action' => (string) $batch->kind, + 'total' => $items->count(), + 'succeeded' => $items->where('state', 'succeeded')->count(), + 'failed' => $items->where('state', 'failed')->count(), + ] + ); + // A partial batch failure is a known mixed result, not an ambiguous + // dispatch. Successful children are already reflected in the port and + // route ledgers, while every failed child is proven not applied. + if (!$success && $effect) { + $parent->effect_resolution = 'applied'; + $parent->effect_resolved_at = $now; + $parent->effect_resolution_reason = + 'Known partial VPC membership batch result'; + $parent->save(); + } + } + + private static function blockLocked(CloudVpcMembershipBatch $batch, + CloudVpc $vpc, string $code): void + { + $batch->state = 'blocked'; + $batch->last_error_code = substr($code, 0, 64); + $batch->updated_at = time(); + $batch->save(); + $vpc->state = 'degraded'; + $vpc->updated_at = time(); + $vpc->save(); + } + + /** @return array{pending:int,succeeded:int,failed:int,unknown:int} */ + private static function hostStageOutcomes(CloudVpcMembershipBatch $batch, + CloudJob $current, string $currentStatus): array + { + $counts = ['pending' => 0, 'succeeded' => 0, 'failed' => 0, + 'unknown' => 0]; + $jobs = CloudJob::query()->where('vpc_batch_id', (int) $batch->id) + ->whereIn('kind', ['vpc_host_apply', 'vpc_host_remove']) + ->orderBy('id')->get(); + if ($jobs->isEmpty()) { + throw new CloudError('VPC 批次宿主机子任务账本缺失', 409); + } + foreach ($jobs as $job) { + if ((int) $job->id === (int) $current->id) { + $bucket = match ($currentStatus) { + 'succeeded' => 'succeeded', + 'failed' => 'failed', + default => 'unknown', + }; + } elseif ((string) $job->phase !== 'terminal') { + $bucket = 'pending'; + } elseif ((bool) $job->finalization_incident + || (string) $job->effect_resolution === 'unknown') { + $bucket = 'unknown'; + } elseif ((string) $job->effect_resolution === 'applied') { + $bucket = 'succeeded'; + } else { + $bucket = 'failed'; + } + $counts[$bucket]++; + } + return $counts; + } + + private static function newParentJob(int $userId, string $requestKey, + string $fingerprint, CloudVpc $vpc, string $action, + array $instanceIds): CloudJob + { + $now = time(); + $job = new CloudJob(); + $job->operation_id = bin2hex(random_bytes(16)); + $job->request_idempotency_key = $requestKey; + $job->request_fingerprint = $fingerprint; + $job->vpc_id = (int) $vpc->id; + $job->user_id = $userId; + $job->host_id = null; + $job->kind = self::PARENT_KIND; + $job->execution_mode = 'local'; + $job->sealed_payload = CloudSecrets::encrypt([ + 'local_vpc_membership_batch' => true, + 'vpc_key' => (string) $vpc->vpc_key, + 'action' => $action, + 'instance_ids' => $instanceIds, + ], 'job-payload'); + // This local coordination row is completed only by child finalizers; + // it never crosses an Agent dispatch deadline. + $job->dispatch_deadline_at = $now + 315360000; + $job->available_at = 0; + $job->created_at = $job->updated_at = $now; + $job->save(); + return $job; + } + + private static function newItem(CloudVpcMembershipBatch $batch, + CloudInstance $instance, CloudVpcPort $port, + string $state): CloudVpcMembershipBatchItem + { + $now = time(); + $item = new CloudVpcMembershipBatchItem(); + $item->batch_id = (int) $batch->id; + $item->user_id = (int) $batch->user_id; + $item->instance_id = (int) $instance->id; + $item->vpc_port_id = (int) $port->id; + $item->host_id = (int) $instance->host_id; + $item->child_job_id = null; + $item->state = $state; + $item->last_error_code = ''; + $item->created_at = $item->updated_at = $now; + $item->save(); + return $item; + } + + private static function itemsLocked(CloudVpcMembershipBatch $batch) + { + return CloudVpcMembershipBatchItem::query() + ->where('batch_id', (int) $batch->id)->orderBy('id') + ->lockForUpdate()->get(); + } + + private static function dto(CloudVpcMembershipBatch $batch): array + { + $items = CloudVpcMembershipBatchItem::query() + ->where('batch_id', (int) $batch->id)->get(); + $vpc = CloudVpc::find((int) $batch->vpc_id); + return [ + 'batch_key' => (string) $batch->batch_key, + 'vpc_id' => (string) $batch->vpc_id, + 'vpc_key' => $vpc === null ? null : (string) $vpc->vpc_key, + 'action' => (string) $batch->kind, + 'state' => (string) $batch->state, + 'total' => $items->count(), + 'succeeded' => $items->where('state', 'succeeded')->count(), + 'failed' => $items->where('state', 'failed')->count(), + 'unknown' => $items->where('state', 'unknown')->count(), + 'task_id' => (string) $batch->parent_job_id, + 'status_url' => '/api/v1/tasks/' . (int) $batch->parent_job_id, + 'created_at' => (int) $batch->created_at, + ]; + } + + /** @return list */ + private static function instanceIds($value): array + { + if (!is_array($value) || !array_is_list($value) || $value === []) { + throw new CloudError('请选择至少一台实例'); + } + $ids = []; + foreach ($value as $candidate) { + if (is_bool($candidate) + || (!is_int($candidate) + && (!is_string($candidate) + || preg_match('/^[1-9][0-9]*$/D', $candidate) !== 1))) { + throw new CloudError('实例列表包含无效标识'); + } + $id = (int) $candidate; + if ($id < 1) { + throw new CloudError('实例列表包含无效标识'); + } + $ids[$id] = $id; + } + $ids = array_values($ids); + sort($ids, SORT_NUMERIC); + return $ids; + } +} diff --git a/web/app/Services/Cloud/CloudVpcPortService.php b/web/app/Services/Cloud/CloudVpcPortService.php index 405fdaa1..7989a26b 100644 --- a/web/app/Services/Cloud/CloudVpcPortService.php +++ b/web/app/Services/Cloud/CloudVpcPortService.php @@ -885,6 +885,12 @@ final class CloudVpcPortService if ($port === null || $vpc === null || $instance === null) { throw new CloudError('VPC 任务资源账本已丢失', 409); } + if ($job->vpc_batch_id !== null) { + CloudVpcMembershipBatchService::finishChildLocked( + $job, $status, $result, $operation, $vpc, $port, $instance + ); + return; + } if (in_array($kind, ['vpc_host_apply', 'vpc_host_remove'], true)) { self::finishHostJobLocked( $job, $status, $result, $operation, $vpc, $port, $instance @@ -927,6 +933,7 @@ final class CloudVpcPortService CloudVpc::query()->where('id', (int) $job->vpc_id)->update([ 'state' => 'degraded', 'updated_at' => $now, ]); + CloudVpcMembershipBatchService::markChildUncertainLocked($job); if (!is_string($job->sealed_payload) || $job->sealed_payload === '') { return; @@ -1441,7 +1448,11 @@ final class CloudVpcPortService CloudVpcPort $port, CloudVpcHostState $state, ?array $network, array $jobContext = []): CloudJob { - $job = self::baseJob($kind, $vpc, $port, (int) $state->host_id); + $batchId = isset($jobContext['vpc_batch_id']) + ? (int) $jobContext['vpc_batch_id'] : null; + $job = self::baseJob( + $kind, $vpc, $port, (int) $state->host_id, $batchId + ); $body = $kind === 'vpc_host_apply' ? [ 'operation_id' => (string) $job->operation_id, 'desired_digest' => (string) $state->desired_digest, @@ -1471,12 +1482,14 @@ final class CloudVpcPortService private static function queuePortJobLocked(string $kind, CloudInstance $instance, CloudVpc $vpc, CloudVpcPort $port, - ?CloudVpcHostState $state): CloudJob + ?CloudVpcHostState $state, ?int $batchId = null): CloudJob { if (!self::hostReady($state)) { throw new CloudError('实例宿主机的 VPC 路由尚未收敛', 409); } - $job = self::baseJob($kind, $vpc, $port, (int) $instance->host_id); + $job = self::baseJob( + $kind, $vpc, $port, (int) $instance->host_id, $batchId + ); $body = [ 'operation_id' => (string) $job->operation_id, 'port_key' => (string) $port->port_key, @@ -1492,7 +1505,9 @@ final class CloudVpcPortService self::sealJob($job, $vpc, $port, $kind === 'vpc_port_attach' ? 'POST' : 'DELETE', '/v1/vms/' . rawurlencode(strtolower((string) $instance->uuid)) - . '/vpc-port', $body); + . '/vpc-port', $body, $batchId === null ? [] : [ + 'vpc_batch_id' => $batchId, + ]); $port->last_operation_id = (string) $job->operation_id; $port->last_error_code = ''; $port->updated_at = time(); @@ -1501,7 +1516,7 @@ final class CloudVpcPortService } private static function baseJob(string $kind, CloudVpc $vpc, - CloudVpcPort $port, int $hostId): CloudJob + CloudVpcPort $port, int $hostId, ?int $batchId = null): CloudJob { $attempt = CloudJob::query()->where('kind', $kind) ->where('vpc_port_id', (int) $port->id) @@ -1514,6 +1529,9 @@ final class CloudVpcPortService $job->instance_id = (int) $port->instance_id; $job->vpc_id = (int) $vpc->id; $job->vpc_port_id = (int) $port->id; + if ($batchId !== null) { + $job->vpc_batch_id = $batchId; + } $job->user_id = (int) $port->user_id; $job->host_id = $hostId; $job->kind = $kind; @@ -1522,6 +1540,154 @@ final class CloudVpcPortService return $job; } + /** + * Internal batch boundary. The batch service owns orchestration while this + * service remains the single authority for port allocation and Agent wire + * contracts. + */ + public static function reserveBatchPortLocked( + CloudInstance $instance, + CloudVpc $vpc, + string $batchKey, + int $revision + ): CloudVpcPort { + self::assertOwnerAndInstance($instance, [ + 'user_id' => (int) $vpc->user_id, + ], true); + if (self::instanceRegionId($instance) !== (int) $vpc->region_id) { + throw new CloudError('实例只能加入同一地区的 VPC', 409); + } + self::assertNoInstanceJobLocked($instance); + if (CloudVpcPort::query()->where('instance_id', (int) $instance->id) + ->where('active_claim', 1)->lockForUpdate()->exists()) { + throw new CloudError('实例已连接 VPC 或正在执行连接任务', 409); + } + $now = time(); + $port = new CloudVpcPort(); + $port->port_key = substr(hash('sha256', "vpc-batch-port\0" + . $batchKey . "\0" . (int) $instance->id), 0, 32); + $port->vpc_id = (int) $vpc->id; + $port->instance_id = (int) $instance->id; + $port->user_id = (int) $vpc->user_id; + $port->private_ip = self::allocateIpLocked($vpc); + $port->mac_address = self::allocateMacLocked((string) $port->port_key); + $port->active_claim = 1; + $port->state = 'preparing'; + $port->desired_revision = $revision; + $port->applied_revision = 0; + $port->last_operation_id = null; + $port->last_error_code = ''; + $port->attached_at = null; + $port->released_at = null; + $port->created_at = $port->updated_at = $now; + $port->save(); + return $port; + } + + public static function assertBatchInstanceLocked( + CloudInstance $instance, + array $actor + ): void { + self::assertOwnerAndInstance($instance, $actor, true); + self::assertNoInstanceJobLocked($instance); + } + + public static function batchInstanceRegionId(CloudInstance $instance): int + { + return self::instanceRegionId($instance); + } + + public static function assertBatchVpcSettledLocked(CloudVpc $vpc): void + { + self::assertVpcSettledLocked($vpc); + } + + /** @return list */ + public static function queueBatchProjectionLocked( + CloudVpc $vpc, + CloudVpcPort $focusPort, + int $revision, + int $batchId + ): array { + return self::queueProjectionLocked($vpc, $focusPort, $revision, [ + 'vpc_batch_id' => $batchId, + ]); + } + + public static function queueBatchPortJobLocked( + string $kind, + CloudInstance $instance, + CloudVpc $vpc, + CloudVpcPort $port, + CloudVpcHostState $state, + int $batchId + ): CloudJob { + return self::queuePortJobLocked( + $kind, $instance, $vpc, $port, $state, $batchId + ); + } + + /** Update only the host projection ledger; batch advancement stays outside. */ + public static function settleBatchHostJobLocked( + CloudJob $job, + string $status, + $result, + array $operation, + CloudVpc $vpc + ): string { + $state = CloudVpcHostState::query() + ->where('vpc_id', (int) $vpc->id) + ->where('host_id', (int) $job->host_id) + ->lockForUpdate()->first(); + if ($state === null + || !hash_equals((string) $state->last_operation_id, + (string) $job->operation_id)) { + throw new CloudError('VPC 批次宿主机结果与当前投影不一致', 409); + } + $now = time(); + if ($status === 'succeeded') { + self::assertHostResult($job, $vpc, $state, $result); + if ((string) $job->kind === 'vpc_host_remove') { + $state->delete(); + } else { + $state->applied_revision = (int) $state->desired_revision; + $state->applied_digest = (string) $state->desired_digest; + $state->sync_state = 'ready'; + $state->last_error_code = ''; + $state->last_success_at = $now; + $state->updated_at = $now; + $state->save(); + } + return ''; + } + $uncertain = $status === 'uncertain'; + $code = $uncertain ? 'vpc_result_uncertain' + : (CloudOperationErrorCatalog::workerSummary( + (string) $job->kind, 'failed', $operation + ) ?? 'vpc_host_apply_failed'); + $state->sync_state = $uncertain ? 'unknown' : 'failed'; + $state->last_error_code = substr($code, 0, 64); + $state->last_failure_at = $now; + $state->updated_at = $now; + $state->save(); + return substr($code, 0, 64); + } + + public static function assertBatchPortResult( + CloudJob $job, + CloudVpc $vpc, + CloudVpcPort $port, + CloudInstance $instance, + $result + ): void { + self::assertPortResult($job, $vpc, $port, $instance, $result); + } + + public static function batchHostReady(?CloudVpcHostState $state): bool + { + return self::hostReady($state); + } + private static function sealJob(CloudJob $job, CloudVpc $vpc, CloudVpcPort $port, string $method, string $target, array $body, array $extra = []): void diff --git a/web/app/Services/Cloud/CloudVpcService.php b/web/app/Services/Cloud/CloudVpcService.php index f57c5edb..81d07d14 100644 --- a/web/app/Services/Cloud/CloudVpcService.php +++ b/web/app/Services/Cloud/CloudVpcService.php @@ -35,6 +35,8 @@ final class CloudVpcService return [ 'items' => $items, 'instances' => CloudVpcPortService::managementListing($actor), + 'membership_batches' => + CloudVpcMembershipBatchService::activeForOwner($userId), 'regions' => self::regionCatalog($regions), 'maximum' => self::MAX_VPCS_PER_USER, 'used' => count($items), diff --git a/web/app/Services/Cloud/CloudWorkerService.php b/web/app/Services/Cloud/CloudWorkerService.php index 90958e74..9ed16786 100644 --- a/web/app/Services/Cloud/CloudWorkerService.php +++ b/web/app/Services/Cloud/CloudWorkerService.php @@ -18,11 +18,16 @@ final class CloudWorkerService private const LEASE_SECONDS = 930; private const POLICY_KINDS = ['service_suspend', 'network_suspend', 'network_resume']; + private const VPC_BATCH_KINDS = [ + 'vpc_host_apply', 'vpc_host_remove', + 'vpc_port_attach', 'vpc_port_detach', + ]; private const REINSTALL_REVIEW_CODES = [ 'reinstall_cleanup_pending', ]; private bool $policyOnly; + private bool $vpcBatchOnly; private array $observedErrors = []; public function observationErrors(): array @@ -31,9 +36,15 @@ final class CloudWorkerService } public function __construct(private ?CloudAgentClient $agent = null, - bool $policyOnly = false) + bool $policyOnly = false, bool $vpcBatchOnly = false) { + if ($policyOnly && $vpcBatchOnly) { + throw new \InvalidArgumentException( + 'Policy-only and VPC-batch-only modes are mutually exclusive' + ); + } $this->policyOnly = $policyOnly; + $this->vpcBatchOnly = $vpcBatchOnly; // A cloud-image clone can legitimately take minutes on remote storage. // The layered deadlines are Agent API 840s, Agent WSS control 855s, // Gateway 870s, PHP 900s and worker lease 930s. Each outer layer has @@ -223,6 +234,9 @@ final class CloudWorkerService ->where('lease_until', '<=', $now); if ($this->policyOnly) { $query->whereIn('kind', self::POLICY_KINDS); + } elseif ($this->vpcBatchOnly) { + $query->whereNotNull('vpc_batch_id') + ->whereIn('kind', self::VPC_BATCH_KINDS); } else { // The regular worker remains a fail-safe policy consumer when // the dedicated workers are not installed, but always claims @@ -722,6 +736,12 @@ final class CloudWorkerService }); }); }); + })->where(static function ($claimable): void { + // A VPC batch parent is a PHP-local coordination ledger. It never + // crosses an Agent dispatch deadline; only its durable child + // finalizers may complete it. + $claimable->where('kind', '<>', 'vpc_membership_batch') + ->orWhere('execution_mode', '<>', 'local'); }); if ($userId !== null) { // Child jobs are not publicly listed, but the owner-scoped diff --git a/web/app/Services/UserDataDeletionService.php b/web/app/Services/UserDataDeletionService.php index 133601ed..3bc41a2e 100644 --- a/web/app/Services/UserDataDeletionService.php +++ b/web/app/Services/UserDataDeletionService.php @@ -18,7 +18,10 @@ class UserDataDeletionService // Delete cloud history from dependants to parents after the active VM, // order and payment guards above have proved the account is safe to // remove. cloud_jobs may contain encrypted provisioning credentials. - ['cloud_jobs', 'user_id'], ['cloud_instance_snapshots', 'user_id'], + ['cloud_vpc_membership_batch_items', 'user_id'], + ['cloud_vpc_membership_batches', 'user_id'], + ['cloud_jobs', 'user_id'], + ['cloud_instance_snapshots', 'user_id'], ['cloud_instance_firewalls', 'user_id'], ['cloud_backup_period_entitlements', 'user_id'], ['cloud_service_periods', 'user_id'], diff --git a/web/config/routes_cloud.php b/web/config/routes_cloud.php index 099644f2..2afb0f63 100644 --- a/web/config/routes_cloud.php +++ b/web/config/routes_cloud.php @@ -30,6 +30,8 @@ $app->group('/api/v1', function () { 'App\Controllers\CloudVpcController:index'); $this->delete('/vpcs/{vpc_key:[0-9a-f]{32}}', 'App\Controllers\CloudVpcController:delete'); + $this->post('/vpcs/{vpc_key:[0-9a-f]{32}}/ports/batch', + 'App\Controllers\CloudVpcController:membershipBatch'); $this->map(['GET', 'POST'], '/cloud-snapshots', 'App\Controllers\CloudApiController:cloudSnapshots'); $this->delete('/cloud-snapshots/{snapshot_key:[0-9a-f]{32}}', diff --git a/web/public/cloud/styles.css b/web/public/cloud/styles.css index dc52e246..00fa1b2d 100644 --- a/web/public/cloud/styles.css +++ b/web/public/cloud/styles.css @@ -333,3 +333,8 @@ body.vu-body .ssh-key-picker-manage:hover { .cloud-vpc-batch-toolbar .button{min-width:112px} @media(max-width:740px){.cloud-vpc-candidate-list{grid-template-columns:1fr}} @media(max-width:510px){.cloud-vpc-disclosure{padding:0}.cloud-vpc-card-summary{padding:15px 43px 15px 15px}.cloud-vpc-card-summary::after{right:16px}.cloud-vpc-card-body{gap:15px;padding:0 15px 15px}.cloud-vpc-instance-heading small{font-size:12px}.cloud-vpc-instance-row{gap:12px;padding:14px}.cloud-vpc-instance-identity strong{font-size:14px}.cloud-vpc-instance-identity span{font-size:12px}.cloud-vpc-candidate-identity strong{font-size:13px}.cloud-vpc-candidate-identity small{font-size:12px}.cloud-vpc-batch-status{align-items:flex-start;flex-direction:column}.cloud-vpc-batch-status .button{width:100%}.cloud-vpc-batch-toolbar{align-items:stretch;flex-direction:column}.cloud-vpc-batch-toolbar>span{margin:0}.cloud-vpc-batch-toolbar .button{width:100%}} +.cloud-vpc-instance-row{grid-template-columns:auto minmax(0,1fr) auto auto;transition:border-color .15s ease,background-color .15s ease} +.cloud-vpc-instance-row.selected{border-color:#67c2c7;background:#effafa} +.cloud-vpc-instance-row.disabled{opacity:.82} +.cloud-vpc-instance-checkbox{width:17px;height:17px;margin:0;accent-color:#18a6b0} +@media(max-width:740px){.cloud-vpc-instance-row{grid-template-columns:auto minmax(0,1fr) auto}.cloud-vpc-instance-actions{grid-column:2/-1}.cloud-vpc-instance-actions:empty{display:none}} diff --git a/web/public/cloud/vpcs.js b/web/public/cloud/vpcs.js index 72a552df..fc8a723b 100644 --- a/web/public/cloud/vpcs.js +++ b/web/public/cloud/vpcs.js @@ -33,9 +33,10 @@ const state = { loading: false, controller: null, creating: false, initialized: false, createKey: '', createFingerprint: '', userKey: 'unknown', mutatingVpcs: new Set(), mutatingInstances: new Set(), + mutatingBatches: new Set(), membershipBatches: [], + batchRefreshAt: 0, observers: new Map(), terminalTasks: new Set(), expandedVpcs: new Set(), selections: new Map(), - batchPlan: null, batchAdvancing: false, batchAdvanceScheduled: false, }; function element(tag, className = '', content) { @@ -84,12 +85,14 @@ function setPageStatus(message = '', type = '') { function apply(payload) { state.items = Array.isArray(payload?.items) ? payload.items : []; state.instances = Array.isArray(payload?.instances) ? payload.instances : []; + state.membershipBatches = Array.isArray(payload?.membership_batches) + ? payload.membership_batches : []; state.regions = Array.isArray(payload?.regions) ? payload.regions : []; state.maximum = Math.max(1, Number(payload?.maximum) || 16); state.used = Math.max(0, Number(payload?.used) || state.items.length); render(); observeActiveTasks(); - scheduleBatchAdvance(); + observeActiveBatches(); } function renderRegions() { @@ -171,88 +174,62 @@ function publicIpv4(instance) { return String(instance?.public_ipv4 || '').trim() || '未分配'; } -function selectionForVpc(vpc) { - const key = vpcKey(vpc); +function selectionForVpc(vpc, action) { + const key = `${vpcKey(vpc)}:${action}`; if (!state.selections.has(key)) state.selections.set(key, new Set()); return state.selections.get(key); } -function batchStorageKey() { - return `virt-cloud:vpc-attach-batch:${state.userKey}`; +function activeBatchForVpc(vpc) { + const key = vpcKey(vpc); + return state.membershipBatches.find(batch => String(batch?.vpc_key || '') === key) + || null; } -function persistBatchPlan() { - try { - if (state.batchPlan) { - sessionStorage.setItem(batchStorageKey(), JSON.stringify(state.batchPlan)); - } else { - sessionStorage.removeItem(batchStorageKey()); - } - } catch {} +function batchPendingStorageKey(vpc, action) { + return `virt-cloud:pending-vpc-batch:${state.userKey}:${vpcKey(vpc)}:${action}`; } -function restoreBatchPlan() { +function pendingBatchKey(vpc, action, ids) { + const fingerprint = `${action}:${[...ids].map(String).sort((a, b) => Number(a) - Number(b)).join(',')}`; + const storageKey = batchPendingStorageKey(vpc, action); let saved = null; - try { saved = JSON.parse(sessionStorage.getItem(batchStorageKey()) || 'null'); } - catch {} - const key = String(saved?.vpcKey || ''); - const ids = Array.isArray(saved?.ids) - ? [...new Set(saved.ids.map(String).filter(id => /^[0-9]+$/.test(id)))] : []; - if (!/^[a-f0-9]{32}$/.test(key) || !ids.length - || !['running', 'failed', 'uncertain'].includes(String(saved?.status || ''))) { - state.batchPlan = null; - persistBatchPlan(); - return; - } - const completed = Array.isArray(saved?.completed) - ? [...new Set(saved.completed.map(String).filter(id => ids.includes(id)))] : []; - const current = ids.includes(String(saved?.current || '')) - ? String(saved.current) : ''; - state.batchPlan = { - vpcKey: key, - ids, - completed, - current, - currentState: current ? String(saved?.currentState || 'submitting') : '', - status: String(saved.status), - failedMessage: String(saved?.failedMessage || ''), - }; - state.expandedVpcs.add(key); -} - -function clearBatchPlan() { - state.batchPlan = null; - persistBatchPlan(); - setPageStatus(); - render(); -} - -function failBatchPlan(status, message) { - if (!state.batchPlan) return; - state.batchPlan.status = status; - state.batchPlan.failedMessage = message; - persistBatchPlan(); - setPageStatus(message, status === 'uncertain' ? 'warning' : 'error'); - render(); + try { saved = JSON.parse(sessionStorage.getItem(storageKey) || 'null'); } catch {} + const key = saved?.fingerprint === fingerprint + && /^[A-Za-z0-9_.-]{16,128}$/.test(String(saved?.key || '')) + ? String(saved.key) : idempotencyKey(); + try { sessionStorage.setItem(storageKey, JSON.stringify({ fingerprint, key })); } catch {} + return key; } -function scheduleBatchAdvance() { - if (state.batchAdvanceScheduled || state.batchAdvancing || state.loading - || state.batchPlan?.status !== 'running') return; - state.batchAdvanceScheduled = true; - queueMicrotask(() => { - state.batchAdvanceScheduled = false; - void advanceBatchPlan(); - }); +function clearPendingBatchKey(vpc, action) { + try { sessionStorage.removeItem(batchPendingStorageKey(vpc, action)); } catch {} } -function renderInstanceRow(instance, vpc) { +function renderInstanceRow(instance, vpc, selection, batchBlocked, onSelectionChange) { const port = instance?.port || {}; const [label, className] = PORT_STATE_LABELS[String(port.state || '')] || [String(port.state || '状态未知'), 'other']; const row = element('div', 'cloud-vpc-instance-row'); + const id = instanceId(instance); + const checkbox = document.createElement('input'); + checkbox.type = 'checkbox'; + checkbox.className = 'cloud-vpc-instance-checkbox'; + checkbox.value = id; + checkbox.checked = selection.has(id); + checkbox.disabled = batchBlocked || instance.can_detach !== true + || state.mutatingInstances.has(id) || Boolean(taskId(instance?.active_task)); + checkbox.setAttribute('aria-label', `选择移出 ${instance.name || `实例 #${id}`}`); + row.classList.toggle('selected', checkbox.checked); + row.classList.toggle('disabled', checkbox.disabled); + checkbox.addEventListener('change', () => { + if (checkbox.checked) selection.add(id); + else selection.delete(id); + row.classList.toggle('selected', checkbox.checked); + onSelectionChange(); + }); const identity = element('div', 'cloud-vpc-instance-identity'); - identity.append(element('strong', '', instance.name || `实例 #${instanceId(instance)}`)); + identity.append(element('strong', '', instance.name || `实例 #${id}`)); identity.append(element('span', '', `公网 IPv4 ${publicIpv4(instance)}`)); identity.append(element('span', '', @@ -265,7 +242,7 @@ function renderInstanceRow(instance, vpc) { const controls = element('div', 'cloud-vpc-instance-actions'); const action = portAction(instance); const activeTask = taskId(instance?.active_task); - const mutating = state.mutatingInstances.has(instanceId(instance)); + const mutating = state.mutatingInstances.has(id); let button = null; if (action === 'retry_attach') { button = element('button', 'button button-outline small', '重试连接'); @@ -273,45 +250,42 @@ function renderInstanceRow(instance, vpc) { } else if (action === 'retry_detach') { button = element('button', 'button button-outline small danger', '重试清理'); button.addEventListener('click', () => void mutatePort(instance, vpc, 'detach')); - } else if (action === 'detach') { - button = element('button', 'button button-outline small danger', '移出 VPC'); - button.addEventListener('click', () => void mutatePort(instance, vpc, 'detach')); } else if (activeTask) { button = element('button', 'button button-outline small', '任务执行中'); } if (button) { button.type = 'button'; - button.disabled = mutating || Boolean(activeTask) || !action - || Boolean(state.batchPlan); + button.disabled = mutating || Boolean(activeTask) || !action || batchBlocked; controls.append(button); } - row.append(identity, status, controls); + row.append(checkbox, identity, status, controls); return row; } function renderBatchStatus(vpc) { - const plan = state.batchPlan; - if (!plan || plan.vpcKey !== vpcKey(vpc)) return null; - const box = element('div', `cloud-vpc-batch-status ${plan.status}`); + const batch = activeBatchForVpc(vpc); + if (!batch) return null; + const batchState = String(batch.state || ''); + const blocked = batchState === 'blocked'; + const box = element('div', `cloud-vpc-batch-status${blocked ? ' uncertain' : ''}`); const content = element('div'); - const completed = plan.completed.length; - const current = state.instances.find(item => instanceId(item) === plan.current); - let title = `已完成 ${completed} / ${plan.ids.length} 台`; - let message = plan.failedMessage; - if (plan.status === 'running') { - title = `正在按顺序应用 ${completed} / ${plan.ids.length} 台`; - message = plan.current - ? `当前处理:${current?.name || `实例 #${plan.current}`}。完成后会自动提交下一台。` - : '正在核对下一台实例的连接资格。'; - } + const total = Math.max(0, Number(batch.total) || 0); + const succeeded = Math.max(0, Number(batch.succeeded) || 0); + const failed = Math.max(0, Number(batch.failed) || 0); + const action = batch.action === 'detach' ? '移出' : '加入'; + const phase = { + routing: '正在一次同步所需路由;完成后会同时释放全部实例任务。', + ports: `全部实例任务已释放,由工作进程并发执行关机、网卡变更和冷启动。`, + cleanup: '实例网卡已处理,正在统一清理并确认宿主机路由。', + blocked: '有子任务结果无法确认,系统已停止重放并保留资源锁,请在任务中心交由管理员核对。', + }[batchState] || '服务端正在收敛批次状态。'; + const progress = failed > 0 + ? `${succeeded} 成功,${failed} 失败,共 ${total} 台` + : `${succeeded} / ${total} 台已完成`; + const title = `${action} VPC 批次 · ${progress}`; + const message = phase; content.append(element('strong', '', title), element('small', '', message)); box.append(content); - if (plan.status !== 'running') { - const clear = element('button', 'button button-outline small', '清除队列'); - clear.type = 'button'; - clear.addEventListener('click', clearBatchPlan); - box.append(clear); - } return box; } @@ -321,26 +295,84 @@ function renderInstanceManager(vpc) { const title = element('div'); title.append(element('strong', '', '实例连接'), element('small', '', - '可一次选择多台;系统会逐台正常关机、应用网卡并冷启动,请在队列完成前保持页面打开。')); + '可选择任意数量实例一次应用。任务由服务端持久执行,页面关闭不影响;路由就绪后会同时释放所选实例任务,并由工作进程并发处理。')); heading.append(title); section.append(heading); const batchStatus = renderBatchStatus(vpc); if (batchStatus) section.append(batchStatus); + const batch = activeBatchForVpc(vpc); + const batchBlocked = Boolean(batch) || state.mutatingBatches.has(vpcKey(vpc)); const connected = instancesForVpc(vpc); + const detachSelection = selectionForVpc(vpc, 'detach'); + const eligibleDetachIds = new Set(connected + .filter(instance => instance.can_detach === true).map(instanceId)); + for (const id of [...detachSelection]) { + if (!eligibleDetachIds.has(id)) detachSelection.delete(id); + } const rows = element('div', 'cloud-vpc-instance-list'); + let updateDetachControls = () => {}; if (!connected.length) { rows.append(element('p', 'cloud-vpc-instance-empty', '尚未加入实例。')); } else { - for (const instance of connected) rows.append(renderInstanceRow(instance, vpc)); + for (const instance of connected) { + rows.append(renderInstanceRow( + instance, vpc, detachSelection, batchBlocked, + () => updateDetachControls() + )); + } } section.append(rows); + if (connected.length) { + const detachToolbar = element('div', 'cloud-vpc-batch-toolbar'); + const detachCount = element('span', '', '已选择 0 台'); + const detachAll = element('button', 'button button-outline small', '全选可移出'); + detachAll.type = 'button'; + const detachSubmit = element('button', 'button button-outline small danger', '移出所选实例'); + detachSubmit.type = 'button'; + const connectedRows = [...rows.querySelectorAll('.cloud-vpc-instance-row')]; + const detachCheckboxes = connectedRows.map(row => ({ + row, checkbox: row.querySelector('.cloud-vpc-instance-checkbox'), + })).filter(item => item.checkbox); + updateDetachControls = () => { + const count = [...detachSelection].filter(id => eligibleDetachIds.has(id)).length; + detachCount.textContent = `已选择 ${count} 台`; + detachSubmit.textContent = count ? `移出 ${count} 台实例` : '移出所选实例'; + detachSubmit.disabled = batchBlocked || count < 1; + const selectable = detachCheckboxes.filter(item => !item.checkbox.disabled); + const allSelected = selectable.length > 0 + && selectable.every(item => detachSelection.has(item.checkbox.value)); + detachAll.textContent = allSelected ? '取消全选' : '全选可移出'; + detachAll.disabled = batchBlocked || selectable.length < 1; + }; + detachAll.addEventListener('click', () => { + const selectable = detachCheckboxes.filter(item => !item.checkbox.disabled); + const allSelected = selectable.length > 0 + && selectable.every(item => detachSelection.has(item.checkbox.value)); + for (const item of selectable) { + item.checkbox.checked = !allSelected; + item.row.classList.toggle('selected', !allSelected); + if (allSelected) detachSelection.delete(item.checkbox.value); + else detachSelection.add(item.checkbox.value); + } + updateDetachControls(); + }); + detachSubmit.addEventListener('click', () => { + const selected = connected.filter(instance => + detachSelection.has(instanceId(instance)) && instance.can_detach === true); + void submitMembershipBatch(vpc, 'detach', selected); + }); + detachToolbar.append(detachCount, detachAll, detachSubmit); + section.append(detachToolbar); + updateDetachControls(); + } + const candidates = candidatesForVpc(vpc); const eligibleCandidateIds = new Set(candidates .filter(instance => instance.can_attach === true).map(instanceId)); - const selection = selectionForVpc(vpc); + const selection = selectionForVpc(vpc, 'attach'); for (const id of [...selection]) { if (!eligibleCandidateIds.has(id)) selection.delete(id); } @@ -350,7 +382,6 @@ function renderInstanceManager(vpc) { picker.append(legend); const list = element('div', 'cloud-vpc-candidate-list'); const checkboxes = []; - const batchBlocked = Boolean(state.batchPlan); for (const instance of candidates) { const reason = String(instance.action_unavailable_reason || ''); const id = instanceId(instance); @@ -416,7 +447,7 @@ function renderInstanceManager(vpc) { }); form.addEventListener('submit', event => { event.preventDefault(); - void startBatchAttach(vpc, candidates.filter(instance => + void submitMembershipBatch(vpc, 'attach', candidates.filter(instance => selection.has(instanceId(instance)) && instance.can_attach === true)); }); toolbar.append(selectionCount, selectAll, submit); @@ -471,7 +502,7 @@ function render() { const remove = element('button', 'button button-outline small danger', '删除 VPC'); remove.type = 'button'; remove.disabled = state.mutatingVpcs.has(String(vpc.vpc_key)) - || vpc.can_delete !== true || Boolean(state.batchPlan); + || vpc.can_delete !== true || Boolean(activeBatchForVpc(vpc)); remove.title = vpc.port_count > 0 ? '请先移出该 VPC 中的全部实例。' : ''; remove.addEventListener('click', () => void removeVpc(vpc)); actions.append(remove); @@ -483,7 +514,7 @@ function render() { byId('cloud-vpcs-quota').textContent = `${state.used} / ${state.maximum}`; byId('cloud-vpcs-refresh').disabled = state.loading || state.creating || state.mutatingInstances.size > 0 || state.mutatingVpcs.size > 0 - || state.batchPlan?.status === 'running'; + || state.mutatingBatches.size > 0; } function taskFailureMessage(task) { @@ -509,24 +540,12 @@ function observeReceipt(instance, receipt, accepted = false) { state.observers.delete(id); state.terminalTasks.add(id); state.mutatingInstances.delete(instanceId(instance)); - const batchPlanAtTerminal = state.batchPlan; - const belongsToBatch = batchPlanAtTerminal?.status === 'running' - && batchPlanAtTerminal.current === instanceId(instance); void load(true).then(() => { const refreshed = state.instances.find(item => instanceId(item) === instanceId(instance)); const nextTask = taskId(refreshed?.active_task); const portState = String(refreshed?.port?.state || ''); const settled = !refreshed?.port || portState === 'attached'; - if (belongsToBatch) { - if (state.batchPlan !== batchPlanAtTerminal - || state.batchPlan?.status !== 'running') return; - setPageStatus(taskSucceededWithoutRecovery(task) - ? '批量 VPC 队列正在核对当前实例,收敛后会自动处理下一台。' - : taskFailureMessage(task), - taskSucceededWithoutRecovery(task) ? '' : 'error'); - return; - } if (nextTask && nextTask !== id) { setPageStatus(taskSucceededWithoutRecovery(task) ? 'VPC 编排正在继续执行下一阶段,请勿重复提交。' @@ -558,11 +577,67 @@ function observeReceipt(instance, receipt, accepted = false) { } function observeActiveTasks() { + const batchVpcKeys = new Set(state.membershipBatches + .map(batch => String(batch?.vpc_key || '')).filter(Boolean)); for (const instance of state.instances) { - if (instance?.active_task) observeReceipt(instance, instance.active_task, false); + if (instance?.active_task + && !batchVpcKeys.has(String(instance?.port?.vpc_key || ''))) { + observeReceipt(instance, instance.active_task, false); + } } } +function observeBatch(batch, accepted = false) { + const receipt = { + task_id: String(batch?.task_id || ''), + status_url: String(batch?.status_url || ''), + }; + const id = taskId(receipt); + const key = String(batch?.vpc_key || ''); + if (!id || !key || String(batch?.state || '') === 'blocked' + || state.observers.has(id) || state.terminalTasks.has(id)) return null; + if (accepted) announceTaskAccepted(receipt, { title: '批量调整 VPC 实例' }); + const handle = observeTask(receipt, { + onUpdate: () => { + setPageStatus('VPC 批次正在服务端执行;关闭页面不会中断任务。'); + const now = Date.now(); + if (!state.loading && now - state.batchRefreshAt >= 2000) { + state.batchRefreshAt = now; + void load(true); + } + }, + onTerminal: task => { + state.observers.delete(id); + state.terminalTasks.add(id); + state.mutatingBatches.delete(key); + void load(true).then(() => { + if (taskSucceededWithoutRecovery(task)) { + setPageStatus('所选实例的 VPC 批量变更已完成。', 'success'); + toast('VPC 批量操作已完成。', 'success'); + } else { + setPageStatus(taskFailureMessage(task), 'error'); + } + }); + }, + onStop: reason => { + state.observers.delete(id); + state.mutatingBatches.delete(key); + if (reason !== 'pagehide') { + setPageStatus('批次观察已停止,请在任务中心核对结果。', 'warning'); + void load(true); + } + }, + }); + if (!handle.url) return null; + state.observers.set(id, handle); + void handle.done.finally(() => state.observers.delete(id)); + return handle; +} + +function observeActiveBatches() { + for (const batch of state.membershipBatches) observeBatch(batch, false); +} + function stopObservers(reason = 'pagehide') { for (const observer of state.observers.values()) observer.stop(reason); state.observers.clear(); @@ -589,141 +664,71 @@ async function load(force = false) { state.controller = null; state.loading = false; render(); - scheduleBatchAdvance(); } } } -async function startBatchAttach(vpc, instances) { - if (state.batchPlan || !instances.length) return; +async function submitMembershipBatch(vpc, action, instances) { + const key = vpcKey(vpc); + if (!key || !instances.length || activeBatchForVpc(vpc) + || state.mutatingBatches.has(key)) return; + const attach = action === 'attach'; + const ids = [...new Set(instances.map(instance => Number(instanceId(instance))))] + .filter(id => Number.isInteger(id) && id > 0); + if (!ids.length) return; const confirmed = await requestUserConfirmation({ - title: `确认将 ${instances.length} 台实例加入 VPC`, - message: `系统会按顺序将所选实例加入“${vpc.name}”。每台实例都会正常关机、应用私网网卡并冷启动;前一台完全收敛后才会提交下一台。`, - confirmText: `应用 ${instances.length} 台实例`, - tone: 'default', + title: `确认${attach ? '加入' : '移出'} ${ids.length} 台实例`, + message: attach + ? `系统会先一次同步“${vpc.name}”所需路由,然后同时释放全部实例任务。每台实例会正常关机、添加私网网卡并冷启动;任务由服务端持续执行。` + : `系统会同时释放全部实例任务。每台实例会正常关机、移除私网网卡并冷启动;全部结果确认后再统一清理路由并释放私网 IP。`, + confirmText: `${attach ? '加入' : '移出'} ${ids.length} 台实例`, + tone: attach ? 'default' : 'danger', }); - if (!confirmed || state.batchPlan) return; - const key = vpcKey(vpc); - state.batchPlan = { - vpcKey: key, - ids: instances.map(instanceId), - completed: [], - current: '', - currentState: '', - status: 'running', - failedMessage: '', - }; - state.selections.get(key)?.clear(); + if (!confirmed || activeBatchForVpc(vpc) || state.mutatingBatches.has(key)) return; + const requestKey = pendingBatchKey(vpc, action, ids); + state.mutatingBatches.add(key); state.expandedVpcs.add(key); - persistBatchPlan(); - setPageStatus(`已接收 ${instances.length} 台实例,将按顺序应用 VPC 私网。`); + setPageStatus(`正在提交 ${ids.length} 台实例的 VPC 批次…`); render(); - scheduleBatchAdvance(); -} - -async function advanceBatchPlan() { - const plan = state.batchPlan; - if (!plan || plan.status !== 'running' || state.batchAdvancing - || state.loading) return; - state.batchAdvancing = true; try { - const vpc = state.items.find(item => vpcKey(item) === plan.vpcKey); - if (!vpc) { - failBatchPlan('failed', '批量加入已停止:目标 VPC 不存在或已不可见。'); - return; - } - - while (state.batchPlan === plan && plan.status === 'running') { - if (plan.current) { - const current = state.instances.find(item => instanceId(item) === plan.current); - if (!current) { - failBatchPlan('failed', `批量加入已停止:实例 #${plan.current} 已不可见。`); - return; - } - const attached = vpcKey(current?.port) === plan.vpcKey - && String(current?.port?.state || '') === 'attached'; - if (attached) { - if (!plan.completed.includes(plan.current)) plan.completed.push(plan.current); - plan.current = ''; - plan.currentState = ''; - persistBatchPlan(); - render(); - continue; - } - if (taskId(current?.active_task) - || state.mutatingInstances.has(instanceId(current))) return; - if (current?.port) { - const message = current.can_retry_attach === true - ? `实例“${current.name}”的连接阶段需要人工重试,后续实例尚未提交。` - : `实例“${current.name}”未收敛到已连接状态,后续实例尚未提交。`; - failBatchPlan('failed', message); - return; - } - if (plan.currentState === 'submitting' || plan.currentState === 'accepted') { - failBatchPlan('uncertain', - `实例“${current.name}”的提交结果无法从服务端确认,已停止后续队列,请先在任务中心核对。`); - return; - } - failBatchPlan('failed', `实例“${current.name}”未能加入 VPC,后续实例尚未提交。`); - return; - } - - const nextId = plan.ids.find(id => !plan.completed.includes(id)); - if (!nextId) { - const total = plan.ids.length; - state.batchPlan = null; - persistBatchPlan(); - render(); - setPageStatus(`${total} 台实例已全部加入 VPC。`, 'success'); - toast(`${total} 台实例已全部加入 VPC。`, 'success'); - return; - } - const next = state.instances.find(item => instanceId(item) === nextId); - if (!next) { - failBatchPlan('failed', `批量加入已停止:实例 #${nextId} 已不可见。`); - return; - } - if (vpcKey(next?.port) === plan.vpcKey - && String(next?.port?.state || '') === 'attached') { - plan.completed.push(nextId); - persistBatchPlan(); - continue; - } - if (next.can_attach !== true) { - failBatchPlan('failed', - `实例“${next.name}”当前不可加入:${next.action_unavailable_reason || '实例状态已变化'}。`); - return; - } - - plan.current = nextId; - plan.currentState = 'submitting'; - persistBatchPlan(); - render(); - const outcome = await mutatePort(next, vpc, 'attach', { - confirmed: true, - quiet: true, - }); - if (state.batchPlan !== plan) return; - if (outcome.status === 'accepted') { - plan.currentState = 'accepted'; - persistBatchPlan(); - render(); - const refreshed = state.instances.find(item => instanceId(item) === nextId); - if (vpcKey(refreshed?.port) === plan.vpcKey - && String(refreshed?.port?.state || '') === 'attached') continue; - return; - } - if (outcome.status === 'uncertain') { - failBatchPlan('uncertain', outcome.message - || `实例“${next.name}”的提交结果暂时无法确认,已停止后续队列。`); - return; - } - failBatchPlan('failed', outcome.message - || `实例“${next.name}”提交失败,后续实例尚未提交。`); - return; + const receipt = await api(`/api/v1/vpcs/${encodeURIComponent(key)}/ports/batch`, { + method: 'POST', + headers: { 'Idempotency-Key': requestKey }, + body: { action, instance_ids: ids, confirm_restart: true }, + expectedStatus: 202, + requireTaskReceipt: true, + }); + clearPendingBatchKey(vpc, action); + const batch = receipt?.batch || { + task_id: receipt.task_id, + status_url: receipt.status_url, + vpc_key: key, + action, + state: attach ? 'routing' : 'ports', + total: ids.length, + succeeded: 0, + failed: 0, + }; + state.membershipBatches = state.membershipBatches + .filter(item => String(item?.vpc_key || '') !== key); + state.membershipBatches.push(batch); + selectionForVpc(vpc, action).clear(); + observeBatch(batch, true); + setPageStatus(`已接收 ${ids.length} 台实例;任务将在服务端并发处理。`); + toast('VPC 批量任务已接收。', 'success'); + await load(true); + } catch (error) { + await load(true); + if (transportUncertain(error)) { + setPageStatus(error?.message + || '批次提交结果暂时无法确认,已保留同一请求标识,请勿重复提交。', 'warning'); + } else { + clearPendingBatchKey(vpc, action); + setPageStatus(error?.message || 'VPC 批次提交失败。', 'error'); } } finally { - state.batchAdvancing = false; + state.mutatingBatches.delete(key); + render(); } } @@ -863,7 +868,6 @@ export async function initialize() { const bootstrap = readBootstrap('cloud-vpcs-bootstrap'); if (!await initializeSession('vpcs', bootstrap?.user)) return; state.userKey = String(bootstrap?.user?.id || bootstrap?.user?.user_id || 'unknown'); - restoreBatchPlan(); if (Array.isArray(bootstrap?.items) && Array.isArray(bootstrap?.instances) && Array.isArray(bootstrap?.regions)) { apply(bootstrap); diff --git a/web/resources/views/venix/layout_assets.tpl b/web/resources/views/venix/layout_assets.tpl index 4e9861ea..5cec11d5 100644 --- a/web/resources/views/venix/layout_assets.tpl +++ b/web/resources/views/venix/layout_assets.tpl @@ -1,2 +1,2 @@ {* Shared Venix CSS/JS cache key. Increment whenever a shared browser asset changes. *} -{assign var=venix_asset_version value='2026102904' scope='global'} +{assign var=venix_asset_version value='2026102905' scope='global'} diff --git a/web/sql/cloud.sql b/web/sql/cloud.sql index e874b138..397eb93e 100644 --- a/web/sql/cloud.sql +++ b/web/sql/cloud.sql @@ -590,6 +590,64 @@ CREATE TABLE cloud_vpc_host_states ( applied_revision <= desired_revision ) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; + +-- A user-visible batch is one durable orchestration boundary. It reserves all +-- selected ports under one VPC revision, then releases every per-instance +-- mutation together after the route projection has converged. active_claim +-- serializes batches per VPC without deleting their audit history. +CREATE TABLE cloud_vpc_membership_batches ( + id BIGINT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY, + batch_key CHAR(32) CHARACTER SET ascii COLLATE ascii_bin NOT NULL, + vpc_id BIGINT UNSIGNED NOT NULL, + user_id INT NOT NULL, + parent_job_id BIGINT UNSIGNED NOT NULL, + kind VARCHAR(16) CHARACTER SET ascii COLLATE ascii_bin NOT NULL, + state VARCHAR(24) CHARACTER SET ascii COLLATE ascii_bin NOT NULL, + desired_revision BIGINT UNSIGNED NOT NULL, + active_claim TINYINT UNSIGNED DEFAULT 1, + last_error_code VARCHAR(64) CHARACTER SET ascii COLLATE ascii_bin NOT NULL DEFAULT '', + terminal_at BIGINT UNSIGNED DEFAULT NULL, + created_at BIGINT UNSIGNED NOT NULL, + updated_at BIGINT UNSIGNED NOT NULL, + UNIQUE KEY cloud_vpc_membership_batch_key (batch_key), + UNIQUE KEY cloud_vpc_membership_batch_job (parent_job_id), + UNIQUE KEY cloud_vpc_membership_batch_active (vpc_id, active_claim), + KEY cloud_vpc_membership_batch_owner (user_id, id), + CONSTRAINT chk_cloud_vpc_membership_batch_kind CHECK ( + kind IN ('attach','detach') + ), + CONSTRAINT chk_cloud_vpc_membership_batch_state CHECK ( + state IN ('routing','ports','cleanup','blocked','succeeded','failed') + ), + CONSTRAINT chk_cloud_vpc_membership_batch_claim CHECK ( + (state IN ('succeeded','failed') AND active_claim IS NULL + AND terminal_at IS NOT NULL) + OR (state NOT IN ('succeeded','failed') AND active_claim = 1 + AND terminal_at IS NULL) + ) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; + +CREATE TABLE cloud_vpc_membership_batch_items ( + id BIGINT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY, + batch_id BIGINT UNSIGNED NOT NULL, + user_id INT NOT NULL, + instance_id BIGINT UNSIGNED NOT NULL, + vpc_port_id BIGINT UNSIGNED NOT NULL, + host_id BIGINT UNSIGNED NOT NULL, + child_job_id BIGINT UNSIGNED DEFAULT NULL, + state VARCHAR(24) CHARACTER SET ascii COLLATE ascii_bin NOT NULL, + last_error_code VARCHAR(64) CHARACTER SET ascii COLLATE ascii_bin NOT NULL DEFAULT '', + created_at BIGINT UNSIGNED NOT NULL, + updated_at BIGINT UNSIGNED NOT NULL, + UNIQUE KEY cloud_vpc_membership_batch_instance (batch_id, instance_id), + UNIQUE KEY cloud_vpc_membership_batch_port (batch_id, vpc_port_id), + UNIQUE KEY cloud_vpc_membership_batch_child (child_job_id), + KEY cloud_vpc_membership_batch_item_owner (user_id, id), + KEY cloud_vpc_membership_batch_item_state (batch_id, state, id), + CONSTRAINT chk_cloud_vpc_membership_batch_item_state CHECK ( + state IN ('waiting_route','queued','succeeded','failed','unknown') + ) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; -- Desired/applied host-enforced firewall state. The policy row is the current -- tenant view, immutable versions retain idempotency/audit evidence, and rules -- contain only the typed public DTO (never hypervisor XML or host commands). @@ -1212,6 +1270,7 @@ CREATE TABLE cloud_jobs ( backup_recovery_id BIGINT UNSIGNED DEFAULT NULL, vpc_id BIGINT UNSIGNED DEFAULT NULL, vpc_port_id BIGINT UNSIGNED DEFAULT NULL, + vpc_batch_id BIGINT UNSIGNED DEFAULT NULL, user_id INT NOT NULL, host_id BIGINT UNSIGNED DEFAULT NULL, kind VARCHAR(32) NOT NULL, @@ -1306,7 +1365,8 @@ CREATE TABLE cloud_jobs ( CONSTRAINT chk_cloud_job_target_shape CHECK ( (execution_mode = 'agent' AND host_id IS NOT NULL) OR (execution_mode = 'local' AND host_id IS NULL - AND parent_job_id IS NULL AND kind = 'payment_create') + AND parent_job_id IS NULL + AND kind IN ('payment_create','vpc_membership_batch')) OR (execution_mode = 'aggregate' AND host_id IS NULL AND parent_job_id IS NULL AND kind IN ('template_import','template_metadata_sync','template_delete', @@ -1329,6 +1389,7 @@ CREATE TABLE cloud_jobs ( KEY cloud_job_backup_recovery (backup_recovery_id, id), KEY cloud_job_vpc (vpc_id, id), KEY cloud_job_vpc_port (vpc_port_id, id), + KEY cloud_job_vpc_batch (vpc_batch_id, id), KEY cloud_job_overview_state (kind, order_id, phase, outcome, terminal_at, created_at), KEY cloud_job_owner (user_id, id), KEY cloud_job_parent (parent_job_id, id) diff --git a/web/tests/AdminSystemLogsContractTest.php b/web/tests/AdminSystemLogsContractTest.php index e4a24e0f..51ffcee0 100644 --- a/web/tests/AdminSystemLogsContractTest.php +++ b/web/tests/AdminSystemLogsContractTest.php @@ -131,6 +131,7 @@ $assert(str_contains($service, 'private const UNITS = [') && str_contains($service, 'virt-cloud-worker.service') && str_contains($service, 'virt-cloud-recovery-worker.service') && str_contains($service, 'virt-cloud-policy-worker.service') + && str_contains($service, 'virt-cloud-vpc-worker@1.service') && str_contains($service, 'virt-cloud-agent-gateway.service') && !str_contains($service, 'virt-cloud-engine.service') && str_contains($service, 'MAX_FETCH_LINES') diff --git a/web/tests/CloudFreshSchemaContractTest.php b/web/tests/CloudFreshSchemaContractTest.php index af4a577f..6f2687bd 100644 --- a/web/tests/CloudFreshSchemaContractTest.php +++ b/web/tests/CloudFreshSchemaContractTest.php @@ -22,7 +22,7 @@ $assert(stripos($schema, 'compatibility import') === false $assert(!preg_match('/\bengine_[A-Za-z0-9_]*\b/', $schema), 'cloud.sql retained an engine-prefixed identifier.'); $assert(str_contains($setup, - "SCHEMA_VERSION = 'go-agent-wss-v13-vpc-l3-v3'") + "SCHEMA_VERSION = 'go-agent-wss-v13-vpc-l3-v4-batch'") && !str_contains($setup, "SCHEMA_VERSION = 'go-v1'"), 'Outbound Agent schema must reject the incompatible earlier Go database.'); $assert(str_contains($schema, @@ -40,7 +40,9 @@ foreach (['cloud_host_agents', 'cloud_templates', 'cloud_hosts', 'cloud_instance 'cloud_backup_recoveries', 'cloud_order_items', 'cloud_invoice_items', 'cloud_instance_migrations', 'cloud_peer_migration_artifacts', 'cloud_host_vpc_transports', 'cloud_vpc_cidr_pools', 'cloud_vpcs', - 'cloud_vpc_ports', 'cloud_vpc_host_states'] as $table) { + 'cloud_vpc_ports', 'cloud_vpc_host_states', + 'cloud_vpc_membership_batches', + 'cloud_vpc_membership_batch_items'] as $table) { $assert(str_contains($schema, 'CREATE TABLE ' . $table . ' ('), 'Missing fresh-install table: ' . $table); } diff --git a/web/tests/CloudOperationErrorCatalogTest.php b/web/tests/CloudOperationErrorCatalogTest.php index 6ad77d4d..dcaca8c0 100644 --- a/web/tests/CloudOperationErrorCatalogTest.php +++ b/web/tests/CloudOperationErrorCatalogTest.php @@ -203,6 +203,34 @@ $assert(CloudOperationErrorCatalog::workerSummary('delete_force', 'failed', [ '资源保持安全锁定'), 'A failed independent force-delete task lost its reviewed terminal code.'); +$vpcBatchResult = CloudOperationErrorCatalog::result([ + 'batch_key' => 'private-batch-identity', + 'action' => 'attach', + 'total' => 6, + 'succeeded' => 5, + 'failed' => 1, + 'message' => 'host-internal diagnostic', +], false, 'vpc_membership_batch', 'failed'); +$assert($vpcBatchResult === [ + 'action' => 'attach', + 'total' => 6, + 'succeeded' => 5, + 'failed' => 1, + ] + && CloudOperationErrorCatalog::result([ + 'action' => 'attach', + 'total' => 6, + 'succeeded' => 6, + 'failed' => 1, + ], false, 'vpc_membership_batch', 'failed') === [] + && CloudOperationErrorCatalog::result([ + 'action' => 'attach', + 'total' => 6, + 'succeeded' => 5, + 'failed' => 1, + ], false, 'action', 'failed') === [], + 'VPC membership batch result was hidden, inconsistent, or leaked into other task kinds.'); + $publicSpec = CloudInstanceMetadata::publicSpec([ 'plan_name' => 'Basic', 'public_ip' => '203.0.113.10', 'region' => str_repeat('x', 500), 'memory_mib' => 'not-a-number', diff --git a/web/tests/CloudRuntimeHealthTest.php b/web/tests/CloudRuntimeHealthTest.php index b10b8505..64cd65c2 100644 --- a/web/tests/CloudRuntimeHealthTest.php +++ b/web/tests/CloudRuntimeHealthTest.php @@ -50,9 +50,11 @@ foreach ([['user_id' => 42, 'is_admin' => false], ['user_id' => 0, 'is_admin' => $assert($calls === [], 'Unauthorized requests performed probes.'); $payload = $service->snapshot($actor); $checks = array_column($payload['checks'], null, 'key'); -$assert(count($checks) === 8 && $calls === ['default', 'log', 'agents', 'redis'], 'Missing or duplicate probes.'); +$assert(count($checks) === 9 && $calls === ['default', 'log', 'agents', 'redis'], 'Missing or duplicate probes.'); $assert($checks['agents']['status'] === 'ok' && $checks['worker']['status'] === 'busy' - && $checks['policy']['status'] === 'unknown' && $checks['lifecycle']['status'] === 'warning', + && $checks['policy']['status'] === 'unknown' + && $checks['vpc']['status'] === 'unknown' + && $checks['lifecycle']['status'] === 'warning', 'Health or heartbeat semantics changed.'); $assert(!str_contains(json_encode($payload), 'MUST_NOT_EXPOSE'), 'Heartbeat internal fields leaked.'); $assert(str_contains($checks['agents']['message'], '不代表宿主机持续健康'), @@ -176,7 +178,7 @@ $response = (new CloudAdminRuntimeController())->index($request, new Response()) $assert($response->getStatusCode() === 200 && $response->getHeaderLine('Cache-Control') === 'private, no-store', 'Runtime response cached or unavailable.'); $decoded = json_decode((string) $response->getBody(), true); -$assert(count($decoded['checks']) === 8 +$assert(count($decoded['checks']) === 9 && array_column($decoded['checks'], 'status', 'key')['agents'] === 'error', 'Partial failure lost other checks or fabricated Agent configuration health.'); $routes = file_get_contents(dirname(__DIR__) . '/config/routes_cloud.php'); diff --git a/web/tests/CloudRuntimeHeartbeatTest.php b/web/tests/CloudRuntimeHeartbeatTest.php index b52080c0..d28b2231 100644 --- a/web/tests/CloudRuntimeHeartbeatTest.php +++ b/web/tests/CloudRuntimeHeartbeatTest.php @@ -46,7 +46,7 @@ $removeTree = static function (string $directory) use (&$removeTree): void { try { $missing = $testRoot . '/not-created'; $empty = $checks($missing); - $assert(array_keys($empty) === ['worker', 'policy', 'recovery', 'lifecycle'] + $assert(array_keys($empty) === ['worker', 'policy', 'vpc', 'recovery', 'lifecycle'] && array_unique(array_column($empty, 'status')) === ['unknown'] && !file_exists($missing), 'Missing snapshot must be unknown, cover all kinds, and never create runtime files.'); @@ -257,7 +257,8 @@ try { foreach (['dispatched_at', 'order_id', 'invoice_id', 'instance_id', 'snapshot_id', 'parent_job_id', 'dispatch_started_at', 'agent_accepted_at', 'receipt_deadline_at', 'terminal_at', - 'effect_resolved_at', 'effect_resolved_by', 'read_at'] as $column) { + 'effect_resolved_at', 'effect_resolved_by', 'read_at', + 'vpc_batch_id'] as $column) { $table->integer($column)->nullable(); } $table->integer('dispatch_attempts')->default(0); @@ -390,14 +391,16 @@ try { $table->integer('user_id')->default(0); $table->integer('created_at')->default(0); }); - foreach (['normal', 'policy', 'recovery', 'lifecycle'] as $kind) { + foreach (['normal', 'policy', 'vpc', 'recovery', 'lifecycle'] as $kind) { $commandDirectory = $testRoot . '/command-' . $kind; $heartbeat = CloudRuntimeHeartbeat::start($kind, true, $commandDirectory, $clock); $status = $kind === 'lifecycle' ? CloudLifecycle::run(['--once'], $heartbeat) : CloudWorker::run($kind === 'policy' ? ['--once', '--policy-only'] - : ($kind === 'recovery' - ? ['--once', '--recovery-only'] : ['--once']), $heartbeat); + : ($kind === 'vpc' + ? ['--once', '--vpc-batch-only'] + : ($kind === 'recovery' + ? ['--once', '--recovery-only'] : ['--once'])), $heartbeat); $record = $read($commandDirectory)[0]; if ($kind === 'lifecycle') { $assert($status === 0 && $record['state'] === 'stopped' @@ -429,8 +432,9 @@ try { $assert(str_contains($workerCommand, "'--recovery-only'") && str_contains($workerCommand, 'CloudAutomaticOperationRecoveryService') && str_contains($workerCommand, - "if (!\$policyOnly && !\$recoveryOnly)") - && str_contains($workerCommand, '!$policyOnly && !$recoveryOnly') + "if (!\$policyOnly && !\$vpcBatchOnly && !\$recoveryOnly)") + && str_contains($workerCommand, + '!$policyOnly && !$vpcBatchOnly && !$recoveryOnly') && str_contains($xcat, '--recovery-only') && str_contains($workerUnit, 'cloudWorker --recovery-only'), 'The dedicated recovery worker is missing, sends mail, or is not deployable.'); diff --git a/web/tests/CloudTaskRecoveryPresentationTest.php b/web/tests/CloudTaskRecoveryPresentationTest.php index 11cd4728..239f3302 100644 --- a/web/tests/CloudTaskRecoveryPresentationTest.php +++ b/web/tests/CloudTaskRecoveryPresentationTest.php @@ -29,6 +29,7 @@ $schema->create('cloud_jobs', static function (Blueprint $table): void { $table->integer('invoice_id')->nullable(); $table->integer('instance_id')->nullable(); $table->integer('snapshot_id')->nullable(); + $table->integer('vpc_batch_id')->nullable(); $table->integer('user_id'); $table->integer('host_id')->nullable(); $table->string('kind'); @@ -221,6 +222,18 @@ $insert(7, [ 'kind' => 'template_import_replica', 'finalization_error_code' => 'recovery_receipt_invalid', ]); +$insert(8000, [ + 'kind' => 'vpc_port_attach', + 'vpc_batch_id' => 44, + 'outcome' => 'succeeded', + 'status' => 'succeeded', + 'effect_may_have_applied' => 0, + 'safe_to_resubmit' => 0, + 'effect_resolution' => 'applied', + 'finalization_incident' => 0, + 'finalization_error_code' => null, + 'available_at' => 0, +]); $tasks = CloudJobService::tasks(['user_id' => 7, 'is_admin' => false], 'completed'); diff --git a/web/tests/CloudVpcPortWorkflowTest.php b/web/tests/CloudVpcPortWorkflowTest.php index 5d205873..778d5c53 100644 --- a/web/tests/CloudVpcPortWorkflowTest.php +++ b/web/tests/CloudVpcPortWorkflowTest.php @@ -25,8 +25,10 @@ namespace { use App\Models\CloudJob; use App\Models\CloudVpc; use App\Models\CloudVpcHostState; + use App\Models\CloudVpcMembershipBatch; use App\Models\CloudVpcPort; use App\Services\Cloud\CloudSecrets; + use App\Services\Cloud\CloudVpcMembershipBatchService; use App\Services\Cloud\CloudVpcPortService; use App\Services\Cloud\CloudVpcProjection; use Illuminate\Database\Capsule\Manager as Capsule; @@ -197,6 +199,7 @@ namespace { $table->integer('backup_recovery_id')->nullable(); $table->integer('vpc_id')->nullable(); $table->integer('vpc_port_id')->nullable(); + $table->integer('vpc_batch_id')->nullable(); $table->integer('user_id'); $table->integer('host_id')->nullable(); $table->string('kind'); $table->string('execution_mode'); $table->string('phase'); $table->string('outcome')->nullable(); @@ -225,6 +228,32 @@ namespace { $table->integer('created_at'); $table->integer('updated_at'); $table->unique(['user_id', 'request_idempotency_key']); }); + $schema->create('cloud_vpc_membership_batches', static function ( + Blueprint $table + ): void { + $table->increments('id'); $table->string('batch_key')->unique(); + $table->integer('vpc_id'); $table->integer('user_id'); + $table->integer('parent_job_id')->unique(); $table->string('kind'); + $table->string('state'); $table->integer('desired_revision'); + $table->integer('active_claim')->nullable(); + $table->string('last_error_code')->default(''); + $table->integer('terminal_at')->nullable(); + $table->integer('created_at'); $table->integer('updated_at'); + $table->unique(['vpc_id', 'active_claim']); + }); + $schema->create('cloud_vpc_membership_batch_items', static function ( + Blueprint $table + ): void { + $table->increments('id'); $table->integer('batch_id'); + $table->integer('user_id'); $table->integer('instance_id'); + $table->integer('vpc_port_id'); $table->integer('host_id'); + $table->integer('child_job_id')->nullable(); $table->string('state'); + $table->string('last_error_code')->default(''); + $table->integer('created_at'); $table->integer('updated_at'); + $table->unique(['batch_id', 'instance_id']); + $table->unique(['batch_id', 'vpc_port_id']); + $table->unique('child_job_id'); + }); Capsule::table('cloud_network_lines')->insert([ 'id' => 1, 'region_id' => 1, @@ -717,6 +746,184 @@ namespace { && CloudVpcHostState::where('vpc_id', 1)->count() === 0, 'Deleted instance VPC cleanup did not release the address after convergence.'); + // A membership batch has no four-instance ceiling. All attach mutations + // remain fenced behind one route generation, then become claimable in the + // same finalizer transaction. Detach mutations are claimable immediately + // and route cleanup is deferred until every VM result is known. + $batchInstances = []; + foreach (range(10, 15) as $id) { + $batchInstances[] = $newInstance($id, $id % 2 === 0 ? 1 : 2); + } + $attachBatch = CloudVpcMembershipBatchService::start([ + 'user_id' => 7, + 'request_idempotency_key' => 'vpc-batch-attach-more-than-four-0001', + ], (string) $vpc->vpc_key, [ + 'action' => 'attach', + 'instance_ids' => array_map( + static fn (CloudInstance $instance): int => (int) $instance->id, + $batchInstances + ), + 'confirm_restart' => true, + ]); + $attachReplay = CloudVpcMembershipBatchService::start([ + 'user_id' => 7, + 'request_idempotency_key' => 'vpc-batch-attach-more-than-four-0001', + ], (string) $vpc->vpc_key, [ + 'action' => 'attach', + 'instance_ids' => [15, 14, 13, 12, 11, 10], + 'confirm_restart' => true, + ]); + $assert((int) $attachReplay['job']->id === (int) $attachBatch['job']->id + && ($attachReplay['unchanged'] ?? false) === true, + 'Batch retry did not return the same durable parent task.'); + $attachBatchId = (int) CloudVpcMembershipBatch::where( + 'parent_job_id', (int) $attachBatch['job']->id + )->value('id'); + $attachRouteJobs = CloudJob::where('vpc_batch_id', $attachBatchId) + ->where('kind', 'vpc_host_apply')->orderBy('id')->get(); + $assert($attachBatchId > 0 && $attachRouteJobs->count() === 2 + && CloudJob::where('vpc_batch_id', $attachBatchId) + ->where('kind', 'vpc_port_attach')->count() === 0, + 'Batch attach did not fence more than four instances behind one route generation.'); + foreach ($attachRouteJobs as $index => $routeJob) { + $payload = CloudSecrets::decrypt( + (string) $routeJob->sealed_payload, 'job-payload' + ); + $hostState = CloudVpcHostState::where('vpc_id', 1) + ->where('host_id', (int) $routeJob->host_id)->first(); + CloudVpcPortService::finishJobLocked($routeJob, 'succeeded', [ + 'vpc_key' => (string) $vpc->vpc_key, + 'desired_revision' => (int) $hostState->desired_revision, + 'desired_digest' => (string) $hostState->desired_digest, + 'bridge' => CloudVpcProjection::bridgeName((string) $vpc->vpc_key), + 'applied' => true, + ], $payload, ['operation_id' => (string) $routeJob->operation_id]); + $terminalize($routeJob); + $releasedCount = CloudJob::where('vpc_batch_id', $attachBatchId) + ->where('kind', 'vpc_port_attach')->count(); + $assert($releasedCount === ($index + 1 === $attachRouteJobs->count() + ? count($batchInstances) : 0), + 'Batch attach released VM jobs before or after the exact route barrier.'); + } + $attachPortJobs = CloudJob::where('vpc_batch_id', $attachBatchId) + ->where('kind', 'vpc_port_attach')->get(); + $assert($attachPortJobs->count() === 6 + && $attachPortJobs->every(static fn (CloudJob $job): bool => + (string) $job->phase === 'accepted' + && (int) $job->dispatch_attempts === 0), + 'Batch attach did not release every selected instance together.'); + + foreach ($attachPortJobs as $index => $attachPortJob) { + $payload = CloudSecrets::decrypt( + (string) $attachPortJob->sealed_payload, 'job-payload' + ); + $body = $payload['body']; + $instance = CloudInstance::find((int) $attachPortJob->instance_id); + $batchPort = CloudVpcPort::find((int) $attachPortJob->vpc_port_id); + CloudVpcPortService::finishJobLocked($attachPortJob, 'succeeded', [ + 'uuid' => (string) $instance->uuid, + 'port_key' => (string) $batchPort->port_key, + 'vpc_key' => (string) $vpc->vpc_key, + 'vpc_port' => [ + 'port_key' => (string) $batchPort->port_key, + 'vpc_key' => (string) $vpc->vpc_key, + 'private_ip' => (string) $batchPort->private_ip, + 'cidr' => (string) $vpc->cidr, + 'gateway' => (string) $vpc->gateway, + 'mac' => (string) $batchPort->mac_address, + 'bridge' => CloudVpcProjection::bridgeName( + (string) $vpc->vpc_key + ), + 'host_revision' => (int) $body['host_revision'], + 'host_digest' => (string) $body['host_digest'], + ], + 'status' => 'running', 'attached' => true, + ], $payload, [ + 'operation_id' => (string) $attachPortJob->operation_id, + ]); + $terminalize($attachPortJob); + if ($index + 1 < $attachPortJobs->count()) { + $assert((string) $attachBatch['job']->fresh()->phase + !== 'terminal', + 'Batch attach parent completed before every VM result.'); + } + } + $assert((string) $attachBatch['job']->fresh()->outcome === 'succeeded' + && CloudVpcPort::where('vpc_id', 1) + ->where('active_claim', 1)->where('state', 'attached')->count() === 6, + 'Batch attach did not converge its parent and all selected ports.'); + + $detachBatch = CloudVpcMembershipBatchService::start([ + 'user_id' => 7, + 'request_idempotency_key' => 'vpc-batch-detach-more-than-four-0001', + ], (string) $vpc->vpc_key, [ + 'action' => 'detach', + 'instance_ids' => array_map( + static fn (CloudInstance $instance): int => (int) $instance->id, + $batchInstances + ), + 'confirm_restart' => true, + ]); + $detachBatchId = (int) CloudVpcMembershipBatch::where( + 'parent_job_id', (int) $detachBatch['job']->id + )->value('id'); + $detachPortJobs = CloudJob::where('vpc_batch_id', $detachBatchId) + ->where('kind', 'vpc_port_detach')->get(); + $assert($detachBatchId > 0 && $detachPortJobs->count() === 6 + && $detachPortJobs->every(static fn (CloudJob $job): bool => + (string) $job->phase === 'accepted' + && (int) $job->dispatch_attempts === 0) + && CloudJob::where('vpc_batch_id', $detachBatchId) + ->whereIn('kind', ['vpc_host_apply', 'vpc_host_remove']) + ->count() === 0, + 'Batch detach did not release every selected instance before route cleanup.'); + foreach ($detachPortJobs as $index => $detachPortJob) { + $payload = CloudSecrets::decrypt( + (string) $detachPortJob->sealed_payload, 'job-payload' + ); + $instance = CloudInstance::find((int) $detachPortJob->instance_id); + $batchPort = CloudVpcPort::find((int) $detachPortJob->vpc_port_id); + CloudVpcPortService::finishJobLocked($detachPortJob, 'succeeded', [ + 'uuid' => (string) $instance->uuid, + 'port_key' => (string) $batchPort->port_key, + 'vpc_key' => (string) $vpc->vpc_key, + 'status' => 'running', 'removed' => true, + ], $payload, [ + 'operation_id' => (string) $detachPortJob->operation_id, + ]); + $terminalize($detachPortJob); + $cleanupCount = CloudJob::where('vpc_batch_id', $detachBatchId) + ->whereIn('kind', ['vpc_host_apply', 'vpc_host_remove'])->count(); + $assert($cleanupCount === ($index + 1 === $detachPortJobs->count() + ? 2 : 0), + 'Batch detach started route cleanup before or after the exact VM barrier.'); + } + $detachRouteJobs = CloudJob::where('vpc_batch_id', $detachBatchId) + ->whereIn('kind', ['vpc_host_apply', 'vpc_host_remove']) + ->orderBy('id')->get(); + foreach ($detachRouteJobs as $routeJob) { + $payload = CloudSecrets::decrypt( + (string) $routeJob->sealed_payload, 'job-payload' + ); + $hostState = CloudVpcHostState::where('vpc_id', 1) + ->where('host_id', (int) $routeJob->host_id)->first(); + $result = [ + 'vpc_key' => (string) $vpc->vpc_key, + 'desired_revision' => (int) $hostState->applied_revision, + 'desired_digest' => (string) $hostState->applied_digest, + 'removed' => true, + ]; + CloudVpcPortService::finishJobLocked($routeJob, 'succeeded', + $result, $payload, + ['operation_id' => (string) $routeJob->operation_id]); + $terminalize($routeJob); + } + $assert((string) $detachBatch['job']->fresh()->outcome === 'succeeded' + && CloudVpcPort::where('vpc_id', 1) + ->where('active_claim', 1)->count() === 0 + && CloudVpcHostState::where('vpc_id', 1)->count() === 0, + 'Batch detach did not converge route cleanup before releasing ports.'); + $worker = file_get_contents( dirname(__DIR__) . '/app/Services/Cloud/CloudWorkerService.php' ); diff --git a/web/tests/CloudVpcSchemaContractTest.php b/web/tests/CloudVpcSchemaContractTest.php index a63f5701..d6d04d3e 100644 --- a/web/tests/CloudVpcSchemaContractTest.php +++ b/web/tests/CloudVpcSchemaContractTest.php @@ -21,6 +21,8 @@ foreach ([ 'cloud_vpcs', 'cloud_vpc_ports', 'cloud_vpc_host_states', + 'cloud_vpc_membership_batches', + 'cloud_vpc_membership_batch_items', ] as $table) { $assert(str_contains($schema, 'CREATE TABLE ' . $table . ' ('), 'Missing VPC table: ' . $table); @@ -85,15 +87,18 @@ $jobs = $jobsStart === false || $jobsEnd === false ? '' : substr($schema, $jobsStart, $jobsEnd - $jobsStart); $assert(str_contains($jobs, 'vpc_id BIGINT UNSIGNED DEFAULT NULL') && str_contains($jobs, 'vpc_port_id BIGINT UNSIGNED DEFAULT NULL') + && str_contains($jobs, 'vpc_batch_id BIGINT UNSIGNED DEFAULT NULL') && str_contains($jobs, 'KEY cloud_job_vpc (vpc_id, id)') - && str_contains($jobs, 'KEY cloud_job_vpc_port (vpc_port_id, id)'), + && str_contains($jobs, 'KEY cloud_job_vpc_port (vpc_port_id, id)') + && str_contains($jobs, 'KEY cloud_job_vpc_batch (vpc_batch_id, id)'), 'Cloud jobs cannot be correlated to VPC or VPC-port work.'); $assert(str_contains($setup, - "private const SCHEMA_VERSION = 'go-agent-wss-v13-vpc-l3-v3'") + "private const SCHEMA_VERSION = 'go-agent-wss-v13-vpc-l3-v4-batch'") && str_contains($setup, "'name' => 'cloud_vpc_enabled'") && str_contains($setup, "'value' => 'false'") - && str_contains($setup, "'cloud_job_vpc', 'cloud_job_vpc_port'") + && str_contains($setup, + "'cloud_job_vpc', 'cloud_job_vpc_port', 'cloud_job_vpc_batch'") && str_contains($setup, "'chk_cloud_vpc_port_release'"), 'cloudInit does not publish and validate the disabled VPC schema.'); @@ -110,6 +115,8 @@ $models = [ 'CloudVpc' => 'cloud_vpcs', 'CloudVpcPort' => 'cloud_vpc_ports', 'CloudVpcHostState' => 'cloud_vpc_host_states', + 'CloudVpcMembershipBatch' => 'cloud_vpc_membership_batches', + 'CloudVpcMembershipBatchItem' => 'cloud_vpc_membership_batch_items', ]; foreach ($models as $class => $table) { $source = file_get_contents($root . '/app/Models/' . $class . '.php'); @@ -122,7 +129,8 @@ foreach ($models as $class => $table) { $jobModel = file_get_contents($root . '/app/Models/CloudJob.php'); $assert(str_contains($jobModel, "'vpc_id' => 'int'") - && str_contains($jobModel, "'vpc_port_id' => 'int'"), + && str_contains($jobModel, "'vpc_port_id' => 'int'") + && str_contains($jobModel, "'vpc_batch_id' => 'int'"), 'CloudJob does not cast VPC correlations.'); echo "VPC schema, model and feature-switch contracts passed.\n"; diff --git a/web/tests/CloudVpcServiceTest.php b/web/tests/CloudVpcServiceTest.php index b0eea471..89209a79 100644 --- a/web/tests/CloudVpcServiceTest.php +++ b/web/tests/CloudVpcServiceTest.php @@ -77,6 +77,26 @@ $schema->create('cloud_vpc_host_states', static function (Blueprint $table): void { $table->increments('id'); $table->integer('vpc_id'); }); +$schema->create('cloud_vpc_membership_batches', + static function (Blueprint $table): void { + $table->increments('id'); $table->string('batch_key'); + $table->integer('vpc_id'); $table->integer('user_id'); + $table->integer('parent_job_id'); $table->string('kind'); + $table->string('state'); $table->integer('desired_revision'); + $table->integer('active_claim')->nullable(); + $table->string('last_error_code')->default(''); + $table->integer('terminal_at')->nullable(); + $table->integer('created_at'); $table->integer('updated_at'); + }); +$schema->create('cloud_vpc_membership_batch_items', + static function (Blueprint $table): void { + $table->increments('id'); $table->integer('batch_id'); + $table->integer('user_id'); $table->integer('instance_id'); + $table->integer('vpc_port_id'); $table->integer('host_id'); + $table->integer('child_job_id')->nullable(); $table->string('state'); + $table->string('last_error_code')->default(''); + $table->integer('created_at'); $table->integer('updated_at'); + }); $schema->create('cloud_instances', static function (Blueprint $table): void { $table->increments('id'); $table->string('uuid'); $table->integer('host_id'); $table->integer('user_id'); diff --git a/web/tests/CloudVpcUserContractTest.php b/web/tests/CloudVpcUserContractTest.php index 70edbc3f..de5b690b 100644 --- a/web/tests/CloudVpcUserContractTest.php +++ b/web/tests/CloudVpcUserContractTest.php @@ -12,6 +12,9 @@ $service = file_get_contents( $portService = file_get_contents( $root . '/app/Services/Cloud/CloudVpcPortService.php' ); +$batchService = file_get_contents( + $root . '/app/Services/Cloud/CloudVpcMembershipBatchService.php' +); $pageController = file_get_contents( $root . '/app/Controllers/CloudPageController.php' ); @@ -45,10 +48,12 @@ $assert(str_contains($routes, "'/cloud/vpcs'") $assert(str_contains($apiRoutes, "'/vpcs'") && str_contains($apiRoutes, "'/vpcs/{vpc_key:") && str_contains($apiRoutes, "'/vms/{id:[0-9]+}/vpc'") + && str_contains($apiRoutes, "'/vpcs/{vpc_key:[0-9a-f]{32}}/ports/batch'") && str_contains($controller, "getHeaderLine('Idempotency-Key')") && str_contains($controller, 'CloudVpcPortService::listing') && str_contains($controller, 'CloudVpcPortService::attach') && str_contains($controller, 'CloudVpcPortService::detach') + && str_contains($controller, 'CloudVpcMembershipBatchService::start') && str_contains($controller, "'Cache-Control', 'private, no-store'"), 'VPC user API lacks port lifecycle, idempotency or private-cache boundaries.'); $assert(str_contains($sidebar, "panel_cloud_active_key == 'vpcs'") @@ -83,9 +88,14 @@ $assert(str_contains($template, 'class="cloud-vpcs-stack"') && str_contains($styles, '.cloud-vpc-candidate-list') && str_contains($javascript, 'function renderInstanceManager(') && str_contains($javascript, "element('details', 'cloud-vpc-card cloud-vpc-disclosure')") - && str_contains($javascript, 'function startBatchAttach(') - && str_contains($javascript, 'async function advanceBatchPlan(') - && str_contains($javascript, 'sessionStorage.setItem(batchStorageKey()') + && str_contains($javascript, 'async function submitMembershipBatch(') + && str_contains($javascript, "action, instance_ids: ids") + && str_contains($javascript, 'function observeActiveBatches(') + && !str_contains($javascript, 'async function advanceBatchPlan(') + && !str_contains($javascript, 'vpc-attach-batch') + && str_contains($batchService, 'There is no selection-count limit') + && str_contains($batchService, 'releaseAttachPortJobsLocked(') + && str_contains($batchService, 'beginDetachCleanupLocked(') && str_contains($javascript, 'public_ipv4') && str_contains($portService, "'public_ipv4' => \$publicIpv4") && !str_contains($javascript, '可分配 ${region.available} 个子网'), @@ -109,6 +119,7 @@ $assert(!str_contains($detailTemplate, 'id="vm-settings-nav-vpc"') 'The retired instance-detail VPC card or its static resources remain.'); $assert(str_contains($controller, "'task_id' => (string) \$job->id") && str_contains($controller, "'status_url' => '/api/v1/tasks/'") + && !str_contains($controller, "'batch' => \$result['batch']") && !str_contains($controller, "'kind' => (string) \$job->kind"), 'VPC mutations must return the exact public task receipt contract.');