diff --git a/README.md b/README.md index 1bd66559..cb150ea5 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,7 @@ Virt Cloud 是 PHP 8.5 面板与宿主机 Go Agent 组成的虚拟机销售平 - 每个云镜像可配置仅首次安装执行的启动脚本。创建任务会快照脚本版本;脚本成功、失败或中断后都不会自动重放,租户不能读取或覆盖脚本正文。 - 用户购买套餐开通一台 VM;每次购买或续费生成独立预付费账期及流量额度。额度用尽只断网、不关机;下一已付费账期恢复网络。最终到期只断网;未支付续费账单进入三天宽限期,之后永久删除 VM 定义和面板管理的磁盘。 - 用户页面及登录页保持原虚拟机面板样式,管理员页面保留 venix 主题。普通用户不获得宿主机管理、Agent 设置或退役节点管理入口。 -- 任务只有执行中、成功和失败三种公开状态,终态只有 `succeeded` 或 `failed`;所有任务都有绝对截止时间。派发后超时或断线直接终态失败,并标记“效果可能已发生、不可重提”;任务不会停在“待核对”,也不会自动重试、退款或再次删除磁盘。不能可靠受 Go context 中止的 libvirt mutation 在独立子进程中执行,截止后由父 Agent 回收整个进程组。 +- 任务只有执行中、成功和失败三种公开状态,终态只有 `succeeded` 或 `failed`;所有任务都有绝对截止时间。派发后超时或断线直接终态失败,并标记“效果可能已发生、不可重放原 operation ID”;任务不会停在“待核对”,也不会自动重试、退款或再次删除磁盘。不能可靠受 Go context 中止的 libvirt mutation 在独立子进程中执行,截止后由父 Agent 回收整个进程组。固定存储池配置是非破坏性的幂等例外:失败任务仍终止且不重放,但不保留跨任务资源栅栏;已定义的固定 pool 不做失败回滚,管理员可显式提交新任务核验或补全同路径固定池,绝不改绑、迁移或删除数据。 ## 目录 diff --git a/agent/cmd/virt-agent/main.go b/agent/cmd/virt-agent/main.go index 444da768..8dea4839 100644 --- a/agent/cmd/virt-agent/main.go +++ b/agent/cmd/virt-agent/main.go @@ -107,9 +107,10 @@ func main() { log.Fatalf("initialize request authentication: %v", err) } local := hypervisor.LocalConfig{ - StoragePathRoots: append([]string(nil), cfg.StoragePathRoots...), - FirmwareLoader: cfg.FirmwareLoader, - FirmwareNVRAMTemplate: cfg.FirmwareNVRAMTemplate, NVRAMDirectory: cfg.NVRAMDirectory, + StoragePathRoots: append([]string(nil), cfg.StoragePathRoots...), + StorageConfigurationLockPath: cfg.JournalPath + ".storage-config.lock", + FirmwareLoader: cfg.FirmwareLoader, + FirmwareNVRAMTemplate: cfg.FirmwareNVRAMTemplate, NVRAMDirectory: cfg.NVRAMDirectory, TrafficStatePath: cfg.JournalPath + ".traffic.json", GuestAgentTimeoutSeconds: int(cfg.GuestAgentTimeoutSec), } @@ -244,8 +245,9 @@ func runOperationWorker(configPath, journalPath, receiptPath string) error { return fmt.Errorf("initialize worker authentication: %w", err) } local := hypervisor.LocalConfig{ - StoragePathRoots: append([]string(nil), cfg.StoragePathRoots...), - FirmwareLoader: cfg.FirmwareLoader, FirmwareNVRAMTemplate: cfg.FirmwareNVRAMTemplate, + StoragePathRoots: append([]string(nil), cfg.StoragePathRoots...), + StorageConfigurationLockPath: cfg.JournalPath + ".storage-config.lock", + FirmwareLoader: cfg.FirmwareLoader, FirmwareNVRAMTemplate: cfg.FirmwareNVRAMTemplate, NVRAMDirectory: cfg.NVRAMDirectory, TrafficStatePath: cfg.JournalPath + ".traffic.json", GuestAgentTimeoutSeconds: int(cfg.GuestAgentTimeoutSec), } diff --git a/agent/hypervisor/libvirt_storage_configuration.go b/agent/hypervisor/libvirt_storage_configuration.go index cf2970d3..787f647a 100644 --- a/agent/hypervisor/libvirt_storage_configuration.go +++ b/agent/hypervisor/libvirt_storage_configuration.go @@ -6,7 +6,9 @@ import ( "context" "encoding/json" "errors" + "os" "path/filepath" + "syscall" "git.luoli.live/root/virt/agent/operation" "libvirt.org/go/libvirt" @@ -33,63 +35,77 @@ func (l *Libvirt) storagePoolName(role string) string { func (l *Libvirt) ConfigureStorage(ctx context.Context, request StorageConfigurationRequest) (StorageConfigurationResult, error) { if err := validateStorageDirectories(request, l.local.StoragePathRoots); err != nil { + return StorageConfigurationResult{}, storageConfigurationNotApplied(err) + } + release, err := acquireStorageConfigurationLock(l.local.StorageConfigurationLockPath) + if err != nil { + return StorageConfigurationResult{}, storageConfigurationNotApplied(err) + } + defer release() + if err := configureStorageRoles(ctx, request, func(role, path string) error { + _, err := l.resolveStoragePool(role, path) + return err + }); err != nil { return StorageConfigurationResult{}, err } - createdNames := make([]string, 0, len(storageRoles())) - rollback := func(cause error) error { - if cleanupErr := l.rollbackDefinedStoragePools(createdNames); cleanupErr != nil { - return operation.MarkUncertain(errors.Join(cause, cleanupErr)) - } - return cause + return StorageConfigurationResult{Configured: true}, nil +} + +func acquireStorageConfigurationLock(path string) (func(), error) { + if path == "" || !filepath.IsAbs(path) || filepath.Clean(path) != path { + return nil, NewError(ErrUnavailable, "storage_configuration_lock_unavailable", "storage configuration lock path is invalid", nil) } - for _, desired := range storageRolePaths(request) { - if err := ctx.Err(); err != nil { - return StorageConfigurationResult{}, rollback(err) - } - binding, created, err := l.resolveStoragePool(desired.Role, desired.Path) - if err != nil { - return StorageConfigurationResult{}, rollback(err) - } - if created { - createdNames = append(createdNames, binding.PoolName) + lock, err := os.OpenFile(path, os.O_CREATE|os.O_RDWR, 0o600) + if err != nil { + return nil, NewError(ErrUnavailable, "storage_configuration_lock_unavailable", "storage configuration lock could not be opened", err) + } + if err := syscall.Flock(int(lock.Fd()), syscall.LOCK_EX|syscall.LOCK_NB); err != nil { + _ = lock.Close() + if errors.Is(err, syscall.EWOULDBLOCK) || errors.Is(err, syscall.EAGAIN) { + return nil, NewError(ErrConflict, "storage_configuration_busy", "another storage configuration helper is still active", nil) } + return nil, NewError(ErrUnavailable, "storage_configuration_lock_unavailable", "storage configuration lock could not be acquired", err) } - return StorageConfigurationResult{Configured: true}, nil + return func() { + _ = syscall.Flock(int(lock.Fd()), syscall.LOCK_UN) + _ = lock.Close() + }, nil } -func (l *Libvirt) resolveStoragePool(role, directory string) (storagePoolBinding, bool, error) { +// resolveStoragePool is deliberately monotonic. Once a fixed pool has been +// defined, a later failure leaves that definition in place so another +// operation can validate and finish it. Rolling pools back is unsafe after a +// timeout because the old helper may resume after a newer operation succeeds. +func (l *Libvirt) resolveStoragePool(role, directory string) (storagePoolBinding, error) { name := l.storagePoolName(role) binding := storagePoolBinding{Role: role, Directory: directory, PoolName: name} existing, err := l.conn.LookupStoragePoolByName(name) if err == nil { defer existing.Free() if err := l.ensureStoragePoolReady(existing, binding); err != nil { - return storagePoolBinding{}, false, err + return storagePoolBinding{}, err } - return binding, false, nil + return binding, nil } if !isNoStoragePool(err) { - return storagePoolBinding{}, false, translateLibvirtError("lookup fixed storage pool", err) + return storagePoolBinding{}, translateLibvirtError("lookup fixed storage pool", err) } document, err := (&libvirtxml.StoragePool{ Type: "dir", Name: name, Target: &libvirtxml.StoragePoolTarget{Path: directory}, }).Marshal() if err != nil { - return storagePoolBinding{}, false, NewError(ErrInvalid, "storage_pool_definition_invalid", "storage pool definition could not be generated", err) + return storagePoolBinding{}, NewError(ErrInvalid, "storage_pool_definition_invalid", "storage pool definition could not be generated", err) } pool, err := l.conn.StoragePoolDefineXML(document, 0) if err != nil { - return storagePoolBinding{}, false, mutationLibvirtError("define storage pool", err) + return storagePoolBinding{}, mutationLibvirtError("define storage pool", err) } defer pool.Free() if err := l.ensureStoragePoolReady(pool, binding); err != nil { - if cleanupErr := l.rollbackDefinedStoragePools([]string{name}); cleanupErr != nil { - return storagePoolBinding{}, false, operation.MarkUncertain(errors.Join(err, cleanupErr)) - } - return storagePoolBinding{}, false, err + return storagePoolBinding{}, err } - return binding, true, nil + return binding, nil } func storagePoolDefinition(pool storagePoolReader) (libvirtxml.StoragePool, error) { @@ -232,33 +248,6 @@ func (l *Libvirt) storagePoolHandleReady(pool storagePoolReader, binding storage return persistent && ready && storagePoolTargetMatches(definition, binding.Directory), nil } -func (l *Libvirt) rollbackDefinedStoragePools(names []string) error { - var failures []error - for index := len(names) - 1; index >= 0; index-- { - pool, err := l.conn.LookupStoragePoolByName(names[index]) - if isNoStoragePool(err) { - continue - } - if err != nil { - failures = append(failures, translateLibvirtError("lookup storage pool during rollback", err)) - continue - } - active, activeErr := pool.IsActive() - if activeErr != nil { - failures = append(failures, translateLibvirtError("inspect storage pool during rollback", activeErr)) - } else if active { - if destroyErr := pool.Destroy(); destroyErr != nil { - failures = append(failures, mutationLibvirtError("stop storage pool during rollback", destroyErr)) - } - } - if undefineErr := pool.Undefine(); undefineErr != nil { - failures = append(failures, mutationLibvirtError("undefine storage pool during rollback", undefineErr)) - } - pool.Free() - } - return errors.Join(failures...) -} - func isNoStoragePool(err error) bool { var libvirtErr libvirt.Error return errors.As(err, &libvirtErr) && libvirtErr.Code == libvirt.ERR_NO_STORAGE_POOL diff --git a/agent/hypervisor/libvirt_storage_lock_test.go b/agent/hypervisor/libvirt_storage_lock_test.go new file mode 100644 index 00000000..140d1168 --- /dev/null +++ b/agent/hypervisor/libvirt_storage_lock_test.go @@ -0,0 +1,53 @@ +//go:build libvirt + +package hypervisor + +import ( + "errors" + "path/filepath" + "testing" +) + +func TestStorageConfigurationLockFailsFastAndReleases(t *testing.T) { + path := filepath.Join(t.TempDir(), "storage-config.lock") + release, err := acquireStorageConfigurationLock(path) + if err != nil { + t.Fatal(err) + } + + secondRelease, err := acquireStorageConfigurationLock(path) + if err == nil { + secondRelease() + release() + t.Fatal("concurrent storage configuration acquired the host lock") + } + var public interface{ PublicCode() string } + if !errors.As(err, &public) || public.PublicCode() != "storage_configuration_busy" { + release() + t.Fatalf("concurrent lock error = %v", err) + } + wrapped := storageConfigurationNotApplied(err) + var semantics interface{ FailureSemantics() (bool, bool) } + if !errors.As(wrapped, &semantics) { + release() + t.Fatal("pre-mutation storage failure did not expose failure semantics") + } + effect, safe := semantics.FailureSemantics() + if effect || !safe { + release() + t.Fatalf("pre-mutation storage failure semantics = effect:%v safe:%v", effect, safe) + } + + release() + thirdRelease, err := acquireStorageConfigurationLock(path) + if err != nil { + t.Fatalf("storage configuration lock remained held after release: %v", err) + } + thirdRelease() +} + +func TestStorageConfigurationLockRejectsInvalidPath(t *testing.T) { + if _, err := acquireStorageConfigurationLock("relative.lock"); err == nil { + t.Fatal("relative storage configuration lock path was accepted") + } +} diff --git a/agent/hypervisor/storage_configuration.go b/agent/hypervisor/storage_configuration.go index 8f0e2483..daa4a0ef 100644 --- a/agent/hypervisor/storage_configuration.go +++ b/agent/hypervisor/storage_configuration.go @@ -1,6 +1,7 @@ package hypervisor import ( + "context" "crypto/sha256" "encoding/hex" "errors" @@ -24,6 +25,21 @@ type storageRolePath struct { Path string } +type storageConfigurationNotAppliedError struct{ cause error } + +func (e *storageConfigurationNotAppliedError) Error() string { return e.cause.Error() } +func (e *storageConfigurationNotAppliedError) Unwrap() error { return e.cause } +func (e *storageConfigurationNotAppliedError) FailureSemantics() (bool, bool) { + return false, true +} + +func storageConfigurationNotApplied(err error) error { + if err == nil { + return nil + } + return &storageConfigurationNotAppliedError{cause: err} +} + func storageRoles() []string { return []string{StorageRoleManaged, StorageRoleTemplate, StorageRoleSeed, StorageRoleUserISO, StorageRoleRescue} } @@ -128,6 +144,21 @@ func storageRolePaths(request StorageConfigurationRequest) []storageRolePath { } } +// configureStorageRoles deliberately retains all completed role mutations if +// a later role fails. A new operation can then reuse and finish the same fixed +// pool mapping without any rollback racing a late worker. +func configureStorageRoles(ctx context.Context, request StorageConfigurationRequest, resolve func(string, string) error) error { + for _, desired := range storageRolePaths(request) { + if err := ctx.Err(); err != nil { + return err + } + if err := resolve(desired.Role, desired.Path); err != nil { + return err + } + } + return nil +} + func ValidateStorageConfigurationRequest(request StorageConfigurationRequest) error { paths := storageRolePaths(request) for _, item := range paths { diff --git a/agent/hypervisor/storage_configuration_test.go b/agent/hypervisor/storage_configuration_test.go index 19646bf4..5e2abf87 100644 --- a/agent/hypervisor/storage_configuration_test.go +++ b/agent/hypervisor/storage_configuration_test.go @@ -1,6 +1,11 @@ package hypervisor import ( + "context" + "errors" + "go/ast" + "go/parser" + "go/token" "os" "path/filepath" "strings" @@ -109,3 +114,72 @@ func TestStorageConfigurationPathDigestsDoNotExposePaths(t *testing.T) { t.Fatalf("path digests = %#v", digests) } } + +func TestStorageConfigurationNeverRollsBackFixedPools(t *testing.T) { + file, err := parser.ParseFile(token.NewFileSet(), "libvirt_storage_configuration.go", nil, 0) + if err != nil { + t.Fatal(err) + } + ast.Inspect(file, func(node ast.Node) bool { + switch value := node.(type) { + case *ast.FuncDecl: + if value.Name.Name == "rollbackDefinedStoragePools" { + t.Error("fixed storage pool rollback helper must not exist") + } + case *ast.CallExpr: + selector, ok := value.Fun.(*ast.SelectorExpr) + if ok && (selector.Sel.Name == "Destroy" || selector.Sel.Name == "Undefine") { + t.Errorf("storage configuration must not call %s on a fixed pool", selector.Sel.Name) + } + } + return true + }) +} + +func TestConfigureStorageRolesRetainsPartialProgressForSuccessor(t *testing.T) { + request := validStorageConfigurationRequest() + retained := make(map[string]string) + failure := errors.New("seed readiness failed") + err := configureStorageRoles(context.Background(), request, func(role, path string) error { + if role == StorageRoleSeed { + return failure + } + retained[role] = path + return nil + }) + if !errors.Is(err, failure) { + t.Fatalf("first attempt error = %v", err) + } + if len(retained) != 2 || retained[StorageRoleManaged] != request.ManagedDirectory || + retained[StorageRoleTemplate] != request.TemplateDirectory { + t.Fatalf("partial fixed-pool progress was not retained: %#v", retained) + } + + err = configureStorageRoles(context.Background(), request, func(role, path string) error { + if existing, ok := retained[role]; ok && existing != path { + return errors.New("fixed pool path was rebound") + } + retained[role] = path + return nil + }) + if err != nil { + t.Fatalf("same-path successor did not converge: %v", err) + } + if len(retained) != len(storageRoles()) { + t.Fatalf("successor completed %d fixed roles, want %d", len(retained), len(storageRoles())) + } +} + +func TestConfigureStorageRolesStopsBetweenRolesOnCancellation(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + var calls []string + err := configureStorageRoles(ctx, validStorageConfigurationRequest(), func(role, _ string) error { + calls = append(calls, role) + cancel() + return nil + }) + if !errors.Is(err, context.Canceled) || len(calls) != 1 || calls[0] != StorageRoleManaged { + t.Fatalf("canceled role traversal calls=%v err=%v", calls, err) + } +} diff --git a/agent/hypervisor/types.go b/agent/hypervisor/types.go index c6ec291b..0d5c9b0c 100644 --- a/agent/hypervisor/types.go +++ b/agent/hypervisor/types.go @@ -226,12 +226,13 @@ type DomainDefinition struct { } type LocalConfig struct { - StoragePathRoots []string - FirmwareLoader string - FirmwareNVRAMTemplate string - NVRAMDirectory string - TrafficStatePath string - GuestAgentTimeoutSeconds int + StoragePathRoots []string + StorageConfigurationLockPath string + FirmwareLoader string + FirmwareNVRAMTemplate string + NVRAMDirectory string + TrafficStatePath string + GuestAgentTimeoutSeconds int } type LocalNetworkSource struct { diff --git a/agent/operation/journal.go b/agent/operation/journal.go index a38dc9cf..4956db58 100644 --- a/agent/operation/journal.go +++ b/agent/operation/journal.go @@ -1238,6 +1238,16 @@ func (j *Journal) initialize() error { return fmt.Errorf("create operation resource blocker bucket: %w", err) } } + // resource_blockers is a derived admission index. Host storage + // configuration keeps immutable failure evidence, but a terminal attempt + // must never prevent an administrator from submitting a new, independently + // identified configuration request. Remove entries written by the earlier + // schema-v6 policy before this process admits any request. The surrounding + // Bolt transaction makes the repair atomic, and the operation plus its + // immutable completion receipt remain untouched. + if err := removeStorageConfigurationBlockers(operations, blockers); err != nil { + return err + } now := j.currentTime() var recovered []Record var stale [][]byte @@ -1652,10 +1662,79 @@ func blockerEntryPrefix(lockKey string) []byte { return []byte(lockKey + "\x00") } -func blocksResources(record Record) bool { +func effectfulTerminalFailure(record Record) bool { return !record.Tombstone && record.State == StateFailed && record.EffectMayHaveApplied } +func blocksResources(record Record) bool { + // Fixed storage-pool registration is monotonic: a successor validates and + // reuses matching pools, and never deletes or rebinds an existing pool. Keep + // the immutable failed operation as evidence, but do not let it fence a new, + // explicitly identified configure-storage request. + return effectfulTerminalFailure(record) && record.Action != "configure-storage" +} + +// removeStorageConfigurationBlockers removes only derived blocker-index +// entries owned by host storage configuration operations. Every reference is +// validated before any key is deleted so a missing, malformed, or stale +// non-storage reference still fails Agent startup instead of silently dropping +// a safety fence. +func removeStorageConfigurationBlockers(operations, blockers *bolt.Bucket) error { + if operations == nil || blockers == nil { + return errors.New("operation resource blocker index is unavailable") + } + removable := make([][]byte, 0) + cursor := blockers.Cursor() + for key, _ := cursor.First(); key != nil; key, _ = cursor.Next() { + separator := bytes.IndexByte(key, 0) + if separator <= 0 || separator != bytes.LastIndexByte(key, 0) || separator == len(key)-1 { + return fmt.Errorf("invalid operation resource blocker key %q", key) + } + lockKey := string(key[:separator]) + operationID := string(key[separator+1:]) + if err := validateLockKey(lockKey); err != nil || !operationIDPattern.MatchString(operationID) { + return fmt.Errorf("invalid operation resource blocker key %q", key) + } + encoded := operations.Get([]byte(operationID)) + if encoded == nil { + return fmt.Errorf("resource blocker index references missing operation %q", operationID) + } + record, err := decodeRecord(encoded) + if err != nil { + return fmt.Errorf("decode resource blocker %q: %w", operationID, err) + } + if record.OperationID != operationID { + return fmt.Errorf("resource blocker index references mismatched operation %q", operationID) + } + matchingKey := false + for _, expected := range effectiveLockKeys(record.OperationID, record.ResourceID, record.Action, record.LockKeys) { + if expected == lockKey { + matchingKey = true + break + } + } + if !matchingKey { + return fmt.Errorf("resource blocker index references unexpected lock %q for operation %q", lockKey, operationID) + } + if record.Action == "configure-storage" { + if !effectfulTerminalFailure(record) { + return fmt.Errorf("resource blocker index references non-blocking operation %q", operationID) + } + removable = append(removable, append([]byte(nil), key...)) + continue + } + if !blocksResources(record) { + return fmt.Errorf("resource blocker index references resolved operation %q", operationID) + } + } + for _, key := range removable { + if err := blockers.Delete(key); err != nil { + return fmt.Errorf("delete retired storage configuration blocker %q: %w", key, err) + } + } + return nil +} + func syncResourceBlockers(blockers *bolt.Bucket, previous, current *Record) error { if blockers == nil { return errors.New("operation resource blocker bucket is unavailable") diff --git a/agent/operation/journal_test.go b/agent/operation/journal_test.go index c34b97d2..749c79c6 100644 --- a/agent/operation/journal_test.go +++ b/agent/operation/journal_test.go @@ -50,6 +50,18 @@ func testSpec(id, resource string) Spec { } } +func storageConfigurationSpec(id string) Spec { + return Spec{ + OperationID: id, + ResourceID: "host-storage", + Action: "configure-storage", + Fingerprint: Fingerprint("POST", "/v1/storage/configuration", []byte(id)), + Destructive: true, + LockKeys: []string{"storage-config"}, + HeavyIO: true, + } +} + func TestDefaultResourceLockKeyUsesCurrentNamespace(t *testing.T) { key := defaultResourceLockKey("operation-resource-key", "opaque-resource", "delete-vm") if len(key) != len("resource-hash:")+64 || key[:len("resource-hash:")] != "resource-hash:" { @@ -63,6 +75,104 @@ func TestDefaultResourceLockKeyUsesCurrentNamespace(t *testing.T) { } } +func TestStorageConfigurationTerminalFailureDoesNotFenceNewOperation(t *testing.T) { + for _, test := range []struct { + name string + executionFenced bool + }{ + {name: "effect unknown"}, + {name: "execution fenced", executionFenced: true}, + } { + t.Run(test.name, func(t *testing.T) { + j := openTestJournal(t) + first := storageConfigurationSpec("operation-storage-failed") + var calls atomic.Int32 + record, replayed, err := j.Execute(context.Background(), first, func(context.Context) (json.RawMessage, error) { + calls.Add(1) + return nil, testFailure{ + code: "storage_configuration_failed", message: "storage configuration failed", + effect: true, safe: false, fence: test.executionFenced, + } + }) + if err == nil || replayed || record.State != StateFailed || !record.EffectMayHaveApplied || + record.SafeToResubmit || record.ExecutionFenced != test.executionFenced { + t.Fatalf("first storage result = %#v replayed=%v err=%v", record, replayed, err) + } + + replayedRecord, replayed, err := j.Execute(context.Background(), first, func(context.Context) (json.RawMessage, error) { + calls.Add(1) + return nil, errors.New("same operation must not run again") + }) + if err != nil || !replayed || replayedRecord.State != StateFailed || calls.Load() != 1 { + t.Fatalf("same operation replay = %#v replayed=%v calls=%d err=%v", replayedRecord, replayed, calls.Load(), err) + } + + successor := storageConfigurationSpec("operation-storage-successor") + succeeded, replayed, err := j.Execute(context.Background(), successor, func(context.Context) (json.RawMessage, error) { + calls.Add(1) + return json.RawMessage(`{"configured":true}`), nil + }) + if err != nil || replayed || succeeded.State != StateSucceeded || calls.Load() != 2 { + t.Fatalf("independent successor = %#v replayed=%v calls=%d err=%v", succeeded, replayed, calls.Load(), err) + } + }) + } +} + +func TestStorageConfigurationAdmissionRemainsSerializedWhileRunning(t *testing.T) { + j := openTestJournal(t) + firstStarted := make(chan struct{}) + releaseFirst := make(chan struct{}) + secondStarted := make(chan struct{}) + firstDone := make(chan error, 1) + secondDone := make(chan error, 1) + + go func() { + _, _, err := j.Execute(context.Background(), storageConfigurationSpec("operation-storage-running"), func(context.Context) (json.RawMessage, error) { + close(firstStarted) + <-releaseFirst + return nil, testFailure{ + code: "storage_configuration_failed", message: "storage configuration failed", + effect: true, safe: false, + } + }) + firstDone <- err + }() + <-firstStarted + + go func() { + _, _, err := j.Execute(context.Background(), storageConfigurationSpec("operation-storage-waiting"), func(context.Context) (json.RawMessage, error) { + close(secondStarted) + return json.RawMessage(`{"configured":true}`), nil + }) + secondDone <- err + }() + deadline := time.Now().Add(time.Second) + for { + select { + case <-secondStarted: + t.Fatal("second storage mutation crossed admission while the first was running") + default: + } + record, err := j.Get("operation-storage-waiting") + if err == nil && record.State == StateReserved { + break + } + if time.Now().After(deadline) { + t.Fatalf("second storage operation did not reach serialized admission: %#v err=%v", record, err) + } + time.Sleep(time.Millisecond) + } + + close(releaseFirst) + if err := <-firstDone; err == nil { + t.Fatal("first storage mutation unexpectedly succeeded") + } + if err := <-secondDone; err != nil { + t.Fatalf("second storage mutation remained blocked after terminal failure: %v", err) + } +} + func TestOperationIDIsPermanentTombstoneAndMutationRunsOnce(t *testing.T) { j := openTestJournal(t) spec := testSpec("operation-single-dispatch", "vm-single") @@ -537,6 +647,168 @@ func TestSuccessfulForceSuccessorOnlyRemovesPredecessorBlockerIndex(t *testing.T } } +func TestInitializeRemovesLegacyStorageBlockerAndPreservesVMFence(t *testing.T) { + path := filepath.Join(t.TempDir(), "operations.db") + j, err := Open(path) + if err != nil { + t.Fatal(err) + } + storage := storageConfigurationSpec("operation-legacy-storage") + if _, _, err := j.Execute(context.Background(), storage, func(context.Context) (json.RawMessage, error) { + return nil, testFailure{ + code: "storage_configuration_failed", message: "storage effect unknown", effect: true, safe: false, + } + }); err == nil { + t.Fatal("legacy storage seed unexpectedly succeeded") + } + vm := testSpec("operation-legacy-vm-fence", "vm-legacy-fence") + if _, _, err := j.Execute(context.Background(), vm, func(context.Context) (json.RawMessage, error) { + return nil, testFailure{code: "vm_effect_unknown", message: "VM effect unknown", effect: true, safe: false} + }); err == nil { + t.Fatal("VM fence seed unexpectedly succeeded") + } + + storageBlockerKey := blockerEntryKey("storage-config", storage.OperationID) + vmLockKeys := effectiveLockKeys(vm.OperationID, vm.ResourceID, vm.Action, vm.LockKeys) + vmBlockerKey := blockerEntryKey(vmLockKeys[0], vm.OperationID) + storageReceiptKey := []byte(completionEventID(storage.OperationID)) + var operationBefore, receiptBefore []byte + if err := j.db.Update(func(tx *bolt.Tx) error { + operations := tx.Bucket(operationsBucket) + blockers := tx.Bucket(resourceBlockersBucket) + receipts := tx.Bucket(completionReceiptsBucket) + if err := blockers.Put(storageBlockerKey, nil); err != nil { + return err + } + operationBefore = append([]byte(nil), operations.Get([]byte(storage.OperationID))...) + receiptBefore = append([]byte(nil), receipts.Get(storageReceiptKey)...) + if len(operationBefore) == 0 || len(receiptBefore) == 0 { + return errors.New("storage operation evidence was not persisted") + } + return nil + }); err != nil { + t.Fatal(err) + } + if err := j.Close(); err != nil { + t.Fatal(err) + } + + j, err = Open(path) + if err != nil { + t.Fatal(err) + } + defer j.Close() + if err := j.db.View(func(tx *bolt.Tx) error { + operations := tx.Bucket(operationsBucket) + blockers := tx.Bucket(resourceBlockersBucket) + receipts := tx.Bucket(completionReceiptsBucket) + metadata := tx.Bucket(metadataBucket) + if string(metadata.Get(schemaVersionKey)) != string(schemaVersion) { + return fmt.Errorf("schema version changed during blocker repair") + } + cursor := blockers.Cursor() + if key, _ := cursor.Seek(storageBlockerKey); bytes.Equal(key, storageBlockerKey) { + return errors.New("legacy storage blocker survived restart repair") + } + if key, _ := cursor.Seek(vmBlockerKey); !bytes.Equal(key, vmBlockerKey) { + return errors.New("VM blocker was removed during storage repair") + } + if !bytes.Equal(operationBefore, operations.Get([]byte(storage.OperationID))) || + !bytes.Equal(receiptBefore, receipts.Get(storageReceiptKey)) { + return errors.New("storage blocker repair rewrote immutable operation evidence") + } + return nil + }); err != nil { + t.Fatal(err) + } + + successor := storageConfigurationSpec("operation-after-storage-repair") + if record, replayed, err := j.Execute(context.Background(), successor, func(context.Context) (json.RawMessage, error) { + return json.RawMessage(`{"configured":true}`), nil + }); err != nil || replayed || record.State != StateSucceeded { + t.Fatalf("storage successor after restart repair = %#v replayed=%v err=%v", record, replayed, err) + } + vmCalled := false + if _, _, err := j.Execute(context.Background(), testSpec("operation-after-vm-fence", vm.ResourceID), func(context.Context) (json.RawMessage, error) { + vmCalled = true + return nil, nil + }); !errors.Is(err, ErrResourceBlocked) || vmCalled { + t.Fatalf("VM fence changed during storage repair: called=%v err=%v", vmCalled, err) + } +} + +func TestInitializeRejectsBrokenResourceBlockerReferencesAtomically(t *testing.T) { + for _, test := range []struct { + name string + seed func(*bolt.Tx) error + }{ + { + name: "missing operation", + seed: func(tx *bolt.Tx) error { + return tx.Bucket(resourceBlockersBucket).Put( + blockerEntryKey("storage-config", "operation-zz-missing-reference"), nil, + ) + }, + }, + { + name: "damaged operation", + seed: func(tx *bolt.Tx) error { + operationID := "operation-zz-damaged-reference" + if err := tx.Bucket(operationsBucket).Put([]byte(operationID), []byte(`{"record":`)); err != nil { + return err + } + return tx.Bucket(resourceBlockersBucket).Put(blockerEntryKey("storage-config", operationID), nil) + }, + }, + } { + t.Run(test.name, func(t *testing.T) { + path := filepath.Join(t.TempDir(), "operations.db") + j, err := Open(path) + if err != nil { + t.Fatal(err) + } + storage := storageConfigurationSpec("operation-valid-storage-reference") + if _, _, err := j.Execute(context.Background(), storage, func(context.Context) (json.RawMessage, error) { + return nil, testFailure{code: "storage_configuration_failed", message: "unknown", effect: true} + }); err == nil { + t.Fatal("storage seed unexpectedly succeeded") + } + storageBlockerKey := blockerEntryKey("storage-config", storage.OperationID) + if err := j.db.Update(func(tx *bolt.Tx) error { + if err := tx.Bucket(resourceBlockersBucket).Put(storageBlockerKey, nil); err != nil { + return err + } + return test.seed(tx) + }); err != nil { + t.Fatal(err) + } + if err := j.Close(); err != nil { + t.Fatal(err) + } + + if reopened, err := Open(path); err == nil { + _ = reopened.Close() + t.Fatal("broken blocker reference was accepted") + } + db, err := bolt.Open(path, 0o600, nil) + if err != nil { + t.Fatal(err) + } + defer db.Close() + if err := db.View(func(tx *bolt.Tx) error { + cursor := tx.Bucket(resourceBlockersBucket).Cursor() + key, _ := cursor.Seek(storageBlockerKey) + if !bytes.Equal(key, storageBlockerKey) { + return errors.New("failed startup partially deleted the valid storage blocker") + } + return nil + }); err != nil { + t.Fatal(err) + } + }) + } +} + func TestFreshSchemaRejectsOldJournalWithoutMigration(t *testing.T) { path := filepath.Join(t.TempDir(), "operations.db") db, err := bolt.Open(path, 0o600, nil) diff --git a/docs/ADMIN_OPERATIONS.md b/docs/ADMIN_OPERATIONS.md index 818c53e9..7b3c593d 100644 --- a/docs/ADMIN_OPERATIONS.md +++ b/docs/ADMIN_OPERATIONS.md @@ -7,7 +7,7 @@ - 提交接口只在任务已经持久化后返回 HTTP `202 Accepted`。这表示“任务已接收”,不表示宿主机操作成功。 - 每个任务最多派发一次。派发前拒绝、派发后断线、Agent 超时、worker 退出和回执截止都会在固定期限内落为 `failed`,不会重新排队。 - 终态只有 `succeeded`、`failed`,一旦写入便不可改变。晚到回执只能得到幂等 ACK,不能翻转任务、退款或释放资源。 -- `effect_may_have_applied=true` 表示底层副作用可能已经发生。任务本身仍然已经结束,但相关资源继续被安全栅栏保护;新的冲突请求会立即返回错误,不会排队等待。 +- `effect_may_have_applied=true` 表示底层副作用可能已经发生。任务本身仍然已经结束;破坏性操作的相关资源继续被安全栅栏保护,新的冲突请求会立即返回错误,不会排队等待。固定存储池配置是非破坏性的例外:保留失败证据但不保留跨任务栅栏,可按已定义 pool 的同一路径,或只修正尚未定义的角色后显式创建新任务。 - 普通用户和管理员界面均不提供任务重试按钮。需要再次执行时,必须先完成只读核验和人工处置,再创建一个经过重新授权的新任务。 - 只有具备种类专用业务账本终结器的失败任务才能解除栅栏。开通只允许用新鲜缺失证据确认 `not_applied`,并原子退款与释放资源;支付根据签名回调或管理员已在外部平台核验的证据收敛本地账本。其他类型当前拒绝假解锁,但任务本身已是终态失败,不会继续占用 worker 或重试。 diff --git a/docs/DEPLOYMENT.md b/docs/DEPLOYMENT.md index 18d9292d..d33d2648 100644 --- a/docs/DEPLOYMENT.md +++ b/docs/DEPLOYMENT.md @@ -117,7 +117,7 @@ Agent 配置字段: 先在宿主机创建五个互相独立的真实目录,并为 libvirt/QEMU 配置正确的属主、权限、挂载和剩余空间;Agent 不会创建目录、挂载文件系统或删除其中的数据。保存并启用 Agent 连接后,在后台“虚拟机管理 -> 宿主机 -> 存储目录”填写系统盘与快照、云系统模板、cloud-init Seed、用户 ISO、救援 ISO 五类目录。Agent 校验目录是 `storage_path_roots` 的真实子目录后,分别注册固定名称的 persistent `dir` pool:`virt-cloud-managed`、`virt-cloud-templates`、`virt-cloud-seeds`、`virt-cloud-user-isos`、`virt-cloud-rescue-isos`,并启动、刷新和启用 autostart。若固定名称已被其他目录占用,Agent 会拒绝注册;不会扫描、复用或迁移其他 pool。 -libvirt 的持久 pool 定义就是 Agent 后续使用的映射,不另写存储状态文件。系统盘与快照、模板、Seed、用户 ISO 和救援 ISO 后续只使用各自固定 pool。当前实现不兼容旧存储布局,也不提供自动迁移或回退读取;已有数据必须在启用本功能前另行完成受控迁移。首次成功应用后网页不允许更换路径。若任务以失败终态结束,旧 CloudJob 和 operation ID 保留为审计记录且绝不重放;管理员可修正目录后显式提交一个新 CloudJob 和新 operation ID。这个新任务只会幂等地核验并补全固定 pool;既不更换已有 pool 的路径,也不迁移或删除数据。注册成功后回到 `/admin/agents` 重新执行连接测试;`storage_configuration` capability 可用前,宿主机不会进入调度。 +libvirt 的持久 pool 定义就是 Agent 后续使用的映射,不另写存储状态文件。系统盘与快照、模板、Seed、用户 ISO 和救援 ISO 后续只使用各自固定 pool。当前实现不兼容旧存储布局,也不提供自动迁移或回退读取;已有数据必须在启用本功能前另行完成受控迁移。首次成功应用后网页不允许更换路径。若任务以失败终态结束,旧 CloudJob 和 operation ID 保留为审计记录且绝不重放;管理员可修正尚未定义角色的目录,或按已经定义的同一路径显式提交一个新 CloudJob 和新 operation ID。固定存储池配置失败不会在 Agent Journal 中保留跨任务 `storage-config` 资源栅栏;Agent 启动时也会清理由旧版本生成的该类派生索引,但不会改写原 operation 或完成回执。配置过程是单调的:已经定义的固定 pool 即使后续步骤失败也不会被回滚、destroy 或 undefine,避免旧超时 helper 晚到后破坏新任务结果。新任务只会幂等地核验并补全同路径固定 pool;既不更换已有 pool 的路径,也不迁移或删除数据。宿主机同时还使用 `journal_path + ".storage-config.lock"` 的非阻塞进程锁;旧 helper 尚未退出时新任务立即以 `storage_configuration_busy` 失败,不等待、不自动重试,也不留下永久栅栏。注册成功后回到 `/admin/agents` 重新执行连接测试;`storage_configuration` capability 可用前,宿主机不会进入调度。 网络接入统一在宿主机页面填写“接入类型”和“网桥 / 网络名称”,新建默认 Linux 网桥 `br0`。面板保存为 `bridge:br0` 或 `network:default` 这样的 `default_network`;名称以字母或数字开头,后续只能包含字母、数字、点、下划线和短横线,bridge 名称为 1~15 字符,libvirt network 名称为 1~64 字符。宿主机上的网桥或 libvirt network 必须由运维预先准备。Agent 只选择、验证和使用现有网络,不创建网桥、不启动 libvirt network,也不修改宿主机系统网卡、路由或挂载。 @@ -129,7 +129,7 @@ Agent 的 systemd 单元保持 `ProtectSystem=strict`。`journal_path` 必须位 `journal_path` 是当前版本专用的 bbolt 数据库,不是可编辑 JSON;同目录的 `.traffic.json` 是流量状态,两者都要纳入一致性备份。Agent 会拒绝旧 schema,不执行升级或回填;全新安装必须使用空状态目录。Journal 在变更跨越派发边界前写入 operation tombstone,随后只允许这一次 worker 执行。Agent 重启若发现已启动但没有完整回执的 operation,会直接写入终态失败和执行栅栏,不会重新调用 libvirt。operation ID、指纹、资源、动作和终态永久保留用于防重放;完成回执的网络重发有固定次数和绝对截止时间,不会重新执行 mutation。 -每个变更由独立 helper 进程打开自己的 libvirt 连接。父 Agent 到达绝对截止后终止整个 helper 进程组并执行有界 `Wait`;只有确认 helper 已退出才释放运行时 admission。无法回收时持久执行栅栏继续阻止相同资源上的变更,但对应面板任务仍会在回执截止时显示终态失败,不会长期保持运行中。 +每个变更由独立 helper 进程打开自己的 libvirt 连接。父 Agent 到达绝对截止后终止整个 helper 进程组并执行有界 `Wait`;正常确认 helper 已退出后释放运行时 admission。若回收仍无法确认,Journal 会把任务终态化并释放进程内 admission:虚拟机和磁盘等破坏性操作继续由持久执行栅栏拒绝同资源的新 mutation;固定存储配置则由 helper 持有的宿主机非阻塞文件锁拒绝并发,旧进程真实退出时由内核自动释放。两类任务都不会长期保持运行中,固定存储任务也不会因旧失败留下永久栅栏。 配置必须存在、可读、是单个严格 JSON 对象并通过内容校验。Agent **不会**检查文件 owner、group 或 mode,也不会因 `0600`、`0640`、`0644` 等可读权限差异拒绝启动;文档中的收紧权限只是运维建议。操作系统本身无权读取、JSON 损坏、字段未知或配置值无效仍会明确启动失败;这些是可用性/内容错误,不是权限位审计。旧的 `listen_address`、`shared_secret`、`tls_cert_file` 和 `tls_key_file` 会作为未知字段被拒绝;Agent 不启动入站监听。 diff --git a/docs/INTEGRATION_PLAN.md b/docs/INTEGRATION_PLAN.md index f4268285..36b9f081 100644 --- a/docs/INTEGRATION_PLAN.md +++ b/docs/INTEGRATION_PLAN.md @@ -71,7 +71,7 @@ Unix 时间戳 4. 主 Agent 对 worker 使用绝对截止、进程组 TERM/KILL 和有界 `Wait`。只有确认 worker 已退出后才释放运行时资源锁。截止、崩溃或重启直接持久化 `failed/result_unconfirmed`,不扫描、不续作、不重新调用原 mutation。 5. Agent 通过完成 outbox 投递不可变回执。相同 `event_id` 可在固定次数和固定年龄内重复投递;PHP 按事件 ID、operation ID 与正文摘要幂等 ACK。回执传输截止后停止,不能形成无限请求。 6. PHP 任务的公开终态只有 `succeeded` 和 `failed`。`receipt_deadline_at` 到期时直接失败;晚到回执只被 ACK 和审计,不能翻转终态。 -7. 失败且 `effect_may_have_applied=true` 时,任务已经结束,但对应资源进入安全锁定。冲突请求立即失败而不是等待;系统不自动退款、释放 reservation 或再次执行。超级管理员完成独立只读检查及宿主机人工收敛后,只能通过审计动作解除资源锁,不能改变原任务结果。 +7. 失败且 `effect_may_have_applied=true` 时,任务已经结束,但破坏性操作对应的资源进入安全锁定。冲突请求立即失败而不是等待;系统不自动退款、释放 reservation 或再次执行。超级管理员完成独立只读检查及宿主机人工收敛后,只能通过审计动作解除资源锁,不能改变原任务结果。固定存储池配置不删除、不迁移也不改绑数据,因此失败只终止原 operation,不保留跨任务栅栏;后续只能按已定义 pool 的同一路径,或只修正尚未定义的角色后显式创建新任务,并继续由 `storage-config` 运行时准入及宿主机非阻塞文件锁串行。 同一 `operation_id` 和同一指纹只返回已存在的任务或终态;同一标识绑定不同请求必须拒绝。这个防重放规则不构成重试许可。 diff --git a/docs/TASK_EXECUTION.md b/docs/TASK_EXECUTION.md index f5c8eafe..47a32909 100644 --- a/docs/TASK_EXECUTION.md +++ b/docs/TASK_EXECUTION.md @@ -60,7 +60,9 @@ PHP 的 `receipt_deadline_at` 到期时直接把任务终态化为失败。之 - 没有专用终结器的类型必须拒绝“只改任务字段”的假解锁。它们的原任务仍已终态失败,但资金或资源保持 fenced,直到实现并验收对应的业务账本终结器; - 需要修复时创建一个经过人工判断的新操作,而不是复用或重放原任务。 -如果业务账本终结器本身因损坏数据或数据库异常失败,公开任务仍必须落为 `failed`,并写入 `finalization_incident`。该不可失败的最终路径清除密封请求载荷、将副作用标记为 unknown,保留资金与资源栅栏,且向管理端显示需要修复账本的事件;它绝不会为了让任务看起来成功而猜测退款、释放或重放 mutation。 +固定存储池配置是非破坏性的窄例外:失败回执仍如实保留 `effect_may_have_applied`,原 operation ID 仍永久终止且不能重放,但它不形成跨任务资源栅栏。存储配置只做单调补全,已定义的固定 pool 不因后续失败而被 destroy、undefine 或回滚。管理员可以按已定义的同一路径,或只修正尚未定义的角色后显式创建新任务;新 operation 仍通过 `storage-config` 运行时准入和宿主机非阻塞文件锁串行,并且只核验或补全固定池,不改绑、迁移或删除数据。旧 helper 尚未退出时,新任务必须立即失败而不是等待,且不得自动重试或留下新的永久栅栏。 + +如果业务账本终结器本身因损坏数据或数据库异常失败,公开任务仍必须落为 `failed`,并写入 `finalization_incident`。该不可失败的最终路径清除密封请求载荷、将副作用标记为 unknown;除上述固定存储池配置例外外,资金与资源栅栏继续保留,并向管理端显示需要修复账本的事件。它绝不会为了让任务看起来成功而猜测退款、释放或重放 mutation。 这一区分同时满足可用性和安全性:用户不会看到永久运行的任务,worker 不会永久占用槽位,而未知破坏性副作用也不会被自动重复执行。 diff --git a/docs/VALIDATION.md b/docs/VALIDATION.md index eb669a62..518222b6 100644 --- a/docs/VALIDATION.md +++ b/docs/VALIDATION.md @@ -57,11 +57,11 @@ jq empty deploy/*.json.example 3. helper 已启动、回执前崩溃:重启后终态失败并保留 durable fence,调用次数为 1。 4. libvirt 返回后丢失响应:任务在回执截止时失败,调用次数为 1,晚到结果不能翻转终态。 5. helper 超时:父进程 TERM/KILL 整个进程组并执行有界 `Wait`;确认回收后才释放 admission。 -6. helper 无法回收:任务仍终态失败,但执行栅栏继续立即拒绝相同资源的新 mutation。 +6. helper 无法回收:任务仍终态失败;VM、磁盘等破坏性操作的执行栅栏继续立即拒绝相同资源的新 mutation。固定存储池配置保留失败证据但不得形成跨任务栅栏。 7. Agent/Gateway/PHP worker 分别重启:任务不重派,数据库租约和浏览器 busy 状态均在固定截止内释放。 8. 完成 endpoint 连续失败:同一个不可变事件最多发送配置中的固定次数,并在绝对截止后停止。 -测试必须覆盖不相交资源并行、相同资源拒绝、重 I/O 容量限制、nonce 防重放、operation 指纹冲突、Journal schema 不匹配拒绝启动,以及 `.traffic.json` 跨 helper 进程的文件锁。 +测试必须覆盖不相交资源并行、相同资源拒绝、存储配置运行时串行但终态失败后允许新 operation、旧存储 blocker 派生索引的原子清理、存储配置非阻塞进程锁的 busy/释放行为、角色中途失败后同路径新任务单调补全且不回滚、重 I/O 容量限制、nonce 防重放、operation 指纹冲突、Journal schema 不匹配拒绝启动,以及 `.traffic.json` 跨 helper 进程的文件锁。 ## 4. libvirt 语义 @@ -73,7 +73,7 @@ jq empty deploy/*.json.example ## 5. 资源、资金与人工处置 -- 派发后失败默认 `effect_may_have_applied=true`、`safe_to_resubmit=false`;同 VM、卷、过滤器、地址、模板或账单的新冲突请求立即失败,不进入等待队列。 +- 派发后失败默认 `effect_may_have_applied=true`、`safe_to_resubmit=false`;同 VM、卷、过滤器、地址、模板或账单的新冲突请求立即失败,不进入等待队列。固定存储池配置保持同一 operation 不重放,但失败后不以旧任务阻断具有新 operation ID 的显式请求。 - 副作用可能发生时禁止自动退款、释放 reservation/IP、再次删除磁盘或创建替代任务。 - 管理员 `inspect` 只能读;不得改变 Journal、任务、Domain、卷、过滤器或 libvirt job。 - `resolve-effect` 只对已有种类专用终结器开放:开通任务用新鲜宿主机缺失证据确认 `not_applied`,先写一次不执行 libvirt 的 Agent 栅栏审计元数据,再原子退款并释放面板资源;支付任务只依据签名回调或管理员已核验的本地渠道账本收敛,绝不重新下单。其他任务类型必须拒绝假解锁。任何路径都不能翻转原任务的 `failed` 终态;解除栅栏后若需新操作,必须由用户另行显式提交。 diff --git a/web/app/Services/Cloud/CloudAdminOperationsService.php b/web/app/Services/Cloud/CloudAdminOperationsService.php index 89ba6dfb..3304c8d6 100644 --- a/web/app/Services/Cloud/CloudAdminOperationsService.php +++ b/web/app/Services/Cloud/CloudAdminOperationsService.php @@ -30,7 +30,7 @@ final class CloudAdminOperationsService . "COALESCE(SUM(CASE WHEN cloud_jobs.phase <> 'terminal' THEN 1 ELSE 0 END), 0) AS active, " . "COALESCE(SUM(CASE WHEN cloud_jobs.phase = 'terminal' AND cloud_jobs.outcome = 'succeeded' THEN 1 ELSE 0 END), 0) AS succeeded, " . "COALESCE(SUM(CASE WHEN cloud_jobs.phase = 'terminal' AND cloud_jobs.outcome = 'failed' THEN 1 ELSE 0 END), 0) AS failed, " - . "COALESCE(SUM(CASE WHEN cloud_jobs.phase = 'terminal' AND cloud_jobs.outcome = 'failed' AND cloud_jobs.effect_resolution = 'unknown' THEN 1 ELSE 0 END), 0) AS unresolved_effect" + . "COALESCE(SUM(CASE WHEN cloud_jobs.phase = 'terminal' AND cloud_jobs.outcome = 'failed' AND cloud_jobs.effect_resolution = 'unknown' AND cloud_jobs.kind <> 'host_storage_configure' THEN 1 ELSE 0 END), 0) AS unresolved_effect" )->first(); $summary = []; foreach (['total', 'active', 'succeeded', 'failed', @@ -88,6 +88,14 @@ final class CloudAdminOperationsService if ($job === null) { throw new CloudError('任务不存在', 404); } + $storageConfiguration = (string) $job->kind + === 'host_storage_configure'; + $effectNote = $storageConfiguration + ? '存储配置失败不会保留跨任务资源栅栏;旧 operation ID 仍永久终止且绝不重放,管理员可按已有固定池的同一路径用新 operation ID 显式提交独立任务。' + : '副作用可能已发生时,资源安全栅栏保持生效;只有超级管理员完成新鲜只读检查并记录审计结论后,才会显式解除面板与 Agent 栅栏。'; + $ledgerNote = $storageConfiguration + ? '新的存储配置任务只会核验或补全路径一致的固定池;不会更换已有池路径,也不会迁移或删除数据。' + : '其他类型若还没有专用业务账本终结器,系统会拒绝假解锁;任务已终态失败,资源栅栏保持安全锁定。'; return [ 'job' => self::row($job, self::canManage($actor)), 'generated_at' => time(), @@ -95,10 +103,10 @@ final class CloudAdminOperationsService '任务状态来自数据库,不会因打开页面而联系 Agent。', '任务最多派发一次;失败后不会自动核对、重新排队或重放宿主机操作。', '宿主机只读检查不改变任务终态,也不执行清理或续作。', - '副作用可能已发生时,资源安全栅栏保持生效;只有超级管理员完成新鲜只读检查并记录审计结论后,才会显式解除面板与 Agent 栅栏。', + $effectNote, '开通任务只有在宿主机明确确认虚拟机和管理资源均不存在时,才能按未应用原子退款并释放库存、公网 IP 与资源预留;只读检查不能把开通任务确认成已应用。', '支付创建结果不明时,签名回调可自动确认已应用;人工结论只收敛本地支付账本,绝不重新向渠道下单。', - '其他类型若还没有专用业务账本终结器,系统会拒绝假解锁;任务已终态失败,资源栅栏保持安全锁定。', + $ledgerNote, '正常关机明确返回 force_required 时,强制删除会创建具有新操作标识的独立任务。', ], ]; @@ -493,7 +501,9 @@ final class CloudAdminOperationsService 'action' => $labels['action_label'], 'status' => self::STATUSES[$status], 'effect' => (string) $job->effect_resolution === 'unknown' - ? '副作用状态未知;任务已失败,资源已安全锁定' + ? ((string) $job->kind === 'host_storage_configure' + ? '部分固定存储池可能已应用;旧任务已终止且不阻断新的独立任务' + : '副作用状态未知;任务已失败,资源已安全锁定') : '无任务级安全栅栏', ], ]; @@ -539,6 +549,10 @@ final class CloudAdminOperationsService return [true, '需要先在外部支付平台核实原订单;本操作不会重新下单']; } + if ((string) $job->kind === 'host_storage_configure') { + return [false, + '存储配置终态失败不保留跨任务资源栅栏;请按已有固定池的同一路径,或只修正尚未定义的角色后提交新任务']; + } return [false, '该任务类型尚无可原子收敛业务账本的解除流程;任务已失败终止,资源保持安全锁定']; } diff --git a/web/app/Services/Cloud/CloudAgentClient.php b/web/app/Services/Cloud/CloudAgentClient.php index 63796764..5f6f71fb 100644 --- a/web/app/Services/Cloud/CloudAgentClient.php +++ b/web/app/Services/Cloud/CloudAgentClient.php @@ -301,6 +301,14 @@ final class CloudAgentClient $operation['result'] = $record['result']; } $causeCode = self::operationCauseCode($record['cause_code'] ?? null); + if ($causeCode === null && !array_key_exists('cause_code', $record)) { + // Admission failures such as resource_operation_blocked are + // represented by the authenticated outer envelope while the + // embedded journal record retains its own terminal error. Keep + // both facts instead of replacing the record error with the + // envelope code. + $causeCode = self::operationCauseCode($error['code'] ?? null); + } if ($causeCode !== null) { $operation['cause_code'] = $causeCode; } diff --git a/web/app/Services/Cloud/CloudOperationErrorCatalog.php b/web/app/Services/Cloud/CloudOperationErrorCatalog.php index 8abb2c0b..9068bfe0 100644 --- a/web/app/Services/Cloud/CloudOperationErrorCatalog.php +++ b/web/app/Services/Cloud/CloudOperationErrorCatalog.php @@ -126,6 +126,16 @@ final class CloudOperationErrorCatalog 'managed_disk_shared' => '目标受管磁盘仍被另一台虚拟机的当前配置、快照或保存状态引用;请先解除引用,再立即核对', 'managed_filter_shared' => '目标网络过滤器仍被另一台虚拟机或过滤器引用;请先解除引用,再立即核对', 'graceful_shutdown_ambiguous' => '正常关机请求结果无法确认;该任务不会授权强制删除,请立即核对后重新发起正常关机操作', + 'resource_operation_blocked' => '本次请求因当时存在的资源栅栏未执行;固定存储配置请先更新并重启 Agent 后创建新任务,其他资源仍需核对旧任务', + 'storage_path_root_unavailable' => 'Agent 无法访问 storage_path_roots 允许根目录;请检查目录存在性、权限和服务沙箱', + 'storage_directory_invalid' => '存储目录不存在、不是目录或路径中包含符号链接;请修正宿主机目录', + 'storage_path_not_allowed' => '存储目录不在 Agent storage_path_roots 允许范围内;请核对 Agent 配置', + 'storage_pool_path_conflict' => '固定存储池已存在,但类型或目标路径与请求不一致;为保护现有磁盘,Agent 不会改绑', + 'storage_pool_definition_invalid' => '固定存储池定义无效或无法生成;请检查 libvirt 存储池定义', + 'storage_pool_configuration_failed' => '固定存储池未达到持久、活动且自动启动状态;请检查 libvirt 存储池状态', + 'storage_configuration_busy' => '另一存储配置 helper 尚未退出;本次任务已立即失败且不会重试,请在 helper 退出后创建新任务', + 'storage_configuration_lock_unavailable' => 'Agent 无法使用存储配置进程锁;请检查 journal_path 所在目录的权限和文件系统锁支持', + 'libvirt_unavailable' => 'Agent 无法连接 libvirt;请检查 libvirt 服务和 Agent 权限', ]; /** diff --git a/web/public/cloud/admin-operations.js b/web/public/cloud/admin-operations.js index 267c44c9..269c5b05 100644 --- a/web/public/cloud/admin-operations.js +++ b/web/public/cloud/admin-operations.js @@ -376,6 +376,7 @@ function renderDetail(payload) { ['宿主机', `${text(job.host_name)}${job.host_id ? ` #${job.host_id}` : ''}`], ['关联实例', job.instance_id ? `${text(job.instance_name)} #${job.instance_id}` : '无关联实例'], ['实例 UUID', job.instance_uuid], ['关联订单', job.order_id ? `#${job.order_id}` : '无关联订单'], + ['Agent 原因', job.agent_cause_label || job.agent_cause_code], ['创建时间', date(job.created_at)], ['更新时间', date(job.updated_at)], ])); const failure = failureText(job, view); diff --git a/web/public/cloud/admin.js b/web/public/cloud/admin.js index 7df0d8c2..067f71f6 100644 --- a/web/public/cloud/admin.js +++ b/web/public/cloud/admin.js @@ -188,7 +188,7 @@ function renderHostStorage(payload) { for (const [field, id] of Object.entries(storageFields)) { const input = byId(id); input.value = payload[field] || ''; input.readOnly = locked; input.disabled = !canManage; } - const labels = { unconfigured: '尚未配置', applying: '正在等待 Agent 完成', failed: '上一次任务已失败且不会自动重试;可修正目录后提交新的独立任务', applied: '已应用' }; + const labels = { unconfigured: '尚未配置', applying: '正在等待 Agent 完成', failed: '上一次任务已失败且不会自动重试;新任务须沿用已定义固定池的路径,只可修正尚未定义的角色', applied: '已应用' }; let message = labels[payload.state] || '状态协议无效,资源已安全锁定,需管理员人工处理'; if (payload.state === 'failed' && !canSubmit) { message = '应用失败,但任务账本状态不完整,请联系管理员核查'; @@ -248,7 +248,7 @@ async function saveHostStorage(event) { if (!currentHost()) return; renderHostStorage(payload); const next = payload.state === 'failed' && payload.can_submit === true - ? ' 旧任务已终止且不会重放;修正目录后可提交新的独立任务。' + ? ' 旧任务已终止且不会重放;新任务须沿用已定义固定池的路径,只可修正尚未定义的角色。' : ' 当前任务账本尚未进入可提交状态,请联系管理员核查。'; hostStorageNotice(`${failure}${partial}${next}`, true); }) diff --git a/web/resources/views/venix/admin/cloud/operations.tpl b/web/resources/views/venix/admin/cloud/operations.tpl index 54be1537..73f7120b 100644 --- a/web/resources/views/venix/admin/cloud/operations.tpl +++ b/web/resources/views/venix/admin/cloud/operations.tpl @@ -5,7 +5,7 @@ {assign var=operations_statuses value=['all'=>'全部状态','active'=>'执行中','succeeded'=>'已成功','failed'=>'已失败']}
-

系统运行与任务记录

任务只展示执行中、成功或失败。失败且效果可能已发生时,资源保持安全锁定并由管理员人工处理。

+

系统运行与任务记录

任务只展示执行中、成功或失败。失败且效果可能已发生时,资源保持安全锁定并由管理员人工处理;固定存储池配置失败不保留跨任务栅栏。

{if $operations_error|default:''}{$operations_error|escape}{else}已读取 {$operations_page.pagination.total|default:0|escape} 条任务{/if}
diff --git a/web/resources/views/venix/admin/cloud/operations_rows.tpl b/web/resources/views/venix/admin/cloud/operations_rows.tpl index 03c79dee..c64dfcc7 100644 --- a/web/resources/views/venix/admin/cloud/operations_rows.tpl +++ b/web/resources/views/venix/admin/cloud/operations_rows.tpl @@ -4,7 +4,7 @@ {$job.instance_name|default:'无关联实例'|escape}{if $job.instance_id}实例 #{$job.instance_id|escape}{/if}{if $job.order_id}订单 #{$job.order_id|escape}{/if} 操作人:{$job.actor_name|default:'未记录'|escape}{if $job.actor_id} #{$job.actor_id|escape}{/if}{if $job.owner_id}所属:{$job.owner_name|default:'未记录'|escape} #{$job.owner_id|escape}{/if} {$job.host_name|default:'未记录'|escape}{$job.region|default:'地区未记录'|escape} - {if isset($job.completed) && !$job.completed && $job.status == 'active'}执行中{elseif isset($job.completed) && $job.completed && in_array($job.status,['succeeded','failed'])}{if $job.status == 'succeeded'}已成功{else}已失败{/if}{if $job.status == 'failed' && $job.effect_may_have_applied|default:false && $job.effect_resolution == 'unknown'}操作效果可能已发生;资源已安全锁定,需管理员人工处理。{elseif $job.status == 'failed' && $job.effect_may_have_applied|default:false && in_array($job.effect_resolution,['applied','not_applied'])}任务仍为失败;人工审计已解除任务级安全栅栏。{/if}{else}协议错误资源按可能已生效处理并保持锁定。{/if} + {if isset($job.completed) && !$job.completed && $job.status == 'active'}执行中{elseif isset($job.completed) && $job.completed && in_array($job.status,['succeeded','failed'])}{if $job.status == 'succeeded'}已成功{else}已失败{/if}{if $job.status == 'failed' && $job.kind == 'host_storage_configure'}旧存储任务已终止且不会重放;后续任务须沿用已定义固定池的路径。{elseif $job.status == 'failed' && $job.effect_may_have_applied|default:false && $job.effect_resolution == 'unknown'}操作效果可能已发生;资源已安全锁定,需管理员人工处理。{elseif $job.status == 'failed' && $job.effect_may_have_applied|default:false && in_array($job.effect_resolution,['applied','not_applied'])}任务仍为失败;人工审计已解除任务级安全栅栏。{/if}{else}协议错误资源按可能已生效处理并保持锁定。{/if} {$job.operation_id|default:'无操作标识'|escape}{$job.phase|default:'未记录阶段'|escape} {if $job.updated_at}{$job.updated_at|date_format:'%Y-%m-%d %H:%M:%S'|escape}{else}更新时间未记录{/if}{if $job.error}{$job.error|escape}{else}未记录错误{/if}
diff --git a/web/resources/views/venix/layout_assets.tpl b/web/resources/views/venix/layout_assets.tpl index 140c005e..e9027a9f 100644 --- a/web/resources/views/venix/layout_assets.tpl +++ b/web/resources/views/venix/layout_assets.tpl @@ -1,2 +1,2 @@ {* Shared Venix CSS/JS cache key. Increment whenever a shared browser asset changes. *} -{assign var=venix_asset_version value='2026101901' scope='global'} +{assign var=venix_asset_version value='2026101902' scope='global'} diff --git a/web/tests/AdminOperationsCancelBehaviorTest.mjs b/web/tests/AdminOperationsCancelBehaviorTest.mjs index bd2c3494..bbde6afb 100644 --- a/web/tests/AdminOperationsCancelBehaviorTest.mjs +++ b/web/tests/AdminOperationsCancelBehaviorTest.mjs @@ -27,6 +27,7 @@ assert.match(source, /可到宿主机存储目录页修正后提交新的独立 assert.match(source, /storageFailure/); assert.match(source, /收尾事件仅保留审计且不会重放/); assert.match(source, /不会影响存储目录页提交新的独立任务/); +assert.match(source, /\['Agent 原因', job\.agent_cause_label \|\| job\.agent_cause_code\]/); // The one retained high-risk action creates a distinct task once. The UI // validates the fresh receipt, immediately acknowledges it, and observes the diff --git a/web/tests/AdminOperationsViewTest.php b/web/tests/AdminOperationsViewTest.php index 870718bc..faef307e 100644 --- a/web/tests/AdminOperationsViewTest.php +++ b/web/tests/AdminOperationsViewTest.php @@ -49,11 +49,16 @@ $items = [ 'instance_id' => null, 'instance_name' => null, 'instance_uuid' => null, 'instance_status' => null, 'host_id' => null, 'host_name' => '', 'region' => '', 'links' => ['instance' => null, 'billing' => null], 'display' => ['kind' => '到期清理', 'status' => '执行中', 'attention' => '', 'lease' => '未持有执行租约']]), + array_replace($common, ['id' => '44', 'kind' => 'host_storage_configure', + 'kind_label' => '配置宿主机存储', 'instance_id' => null, + 'instance_name' => null, 'instance_uuid' => null, + 'instance_status' => null, 'owner_id' => null, 'owner_name' => null, + 'owner_email' => null]), ]; $page = [ 'items' => $items, - 'pagination' => ['page' => 1, 'per_page' => 20, 'total' => 43, 'total_pages' => 3, 'from' => 1, 'to' => 3], - 'summary' => ['total' => 43, 'active' => 1, 'succeeded' => 20, + 'pagination' => ['page' => 1, 'per_page' => 20, 'total' => 44, 'total_pages' => 3, 'from' => 1, 'to' => 4], + 'summary' => ['total' => 44, 'active' => 1, 'succeeded' => 20, 'failed' => 22, 'unresolved_effect' => 1], 'filters' => ['status' => 'all', 'kind' => '', 'host_id' => null, 'q' => '', 'page' => 1, 'per_page' => 20], 'hosts' => [['id' => '7', 'name' => '香港宿主机 A', 'region' => 'HongKong']], @@ -114,9 +119,10 @@ $dom = new DOMDocument(); libxml_use_internal_errors(true); $dom->loadHTML('' . $html); $xpath = new DOMXPath($dom); -$assert($xpath->query('//tbody[@id="operations-body"]/tr')->length === 3, 'SSR must include all task rows.'); +$assert($xpath->query('//tbody[@id="operations-body"]/tr')->length === 4, 'SSR must include all task rows.'); $assert(str_contains($dom->textContent, 'vm-demo') && str_contains($dom->textContent, '系统策略') && str_contains($dom->textContent, '操作效果可能已发生') + && str_contains($dom->textContent, '后续任务须沿用已定义固定池的路径') && str_contains($dom->textContent, '执行中'), 'Useful task records and terminal safety state are absent before JavaScript.'); $assert($xpath->query('//select[@name="status"]/option[@value="all" and @selected]')->length === 1, diff --git a/web/tests/CloudAdminOperationsServiceTest.php b/web/tests/CloudAdminOperationsServiceTest.php index 9caf24a8..173b7ed2 100644 --- a/web/tests/CloudAdminOperationsServiceTest.php +++ b/web/tests/CloudAdminOperationsServiceTest.php @@ -173,6 +173,13 @@ namespace { 'effect_may_have_applied' => false, 'safe_to_resubmit' => false, 'error_code' => 'validation_failed', ]); + $insert(6, [ + 'order_id' => null, 'instance_id' => null, + 'kind' => 'host_storage_configure', + 'outcome' => 'failed', 'status' => 'failed', + 'effect_may_have_applied' => true, 'safe_to_resubmit' => false, + 'effect_resolution' => 'unknown', 'error_code' => 'operation_failed', + ]); $admin = ['user_id' => 99, 'is_admin' => true, 'is_super_admin' => true, 'can_manage' => true]; @@ -182,14 +189,14 @@ namespace { 'is_super_admin' => false, 'can_manage' => false]; $all = Operations::page($admin, ['status' => 'all']); - $assert($ids($all['items']) === [5, 4, 3, 2, 1] + $assert($ids($all['items']) === [6, 5, 4, 3, 2, 1] && $all['summary'] === [ - 'total' => 5, 'active' => 1, 'succeeded' => 1, - 'failed' => 3, 'unresolved_effect' => 1, + 'total' => 6, 'active' => 1, 'succeeded' => 1, + 'failed' => 4, 'unresolved_effect' => 1, ], 'Operations inventory did not expose active and terminal states exactly.'); foreach ([ 'active' => [1], 'succeeded' => [2], - 'failed' => [5, 4, 3], 'attention' => [5, 4, 3], + 'failed' => [6, 5, 4, 3], 'attention' => [6, 5, 4, 3], ] as $status => $expected) { $page = Operations::page($admin, ['status' => $status]); $assert($ids($page['items']) === $expected, @@ -221,6 +228,17 @@ namespace { 'Accepted task was exposed as a fabricated terminal result.'); $assert(Operations::detail($readonly, '3')['job']['can_resolve_effect'] === false, 'Read-only administrator received effect-resolution authority.'); + $storageDetail = Operations::detail($admin, '6'); + $assert($storageDetail['job']['can_resolve_effect'] === false + && str_contains($storageDetail['job']['resolve_effect_reason'], + '不保留跨任务资源栅栏') + && str_contains($storageDetail['job']['display']['effect'], + '不阻断新的独立任务') + && str_contains(implode(' ', $storageDetail['notes']), + '旧 operation ID 仍永久终止且绝不重放') + && str_contains(implode(' ', $storageDetail['notes']), + '不会更换已有池路径'), + 'Storage failure was still presented as a durable resource blocker.'); $reason = '已在宿主机只读核对虚拟机与磁盘,确认原操作效果已经发生。'; $operationId = sprintf('%032x', 3); diff --git a/web/tests/CloudAgentProtocolTest.php b/web/tests/CloudAgentProtocolTest.php index fbe52e7f..0aab4dc0 100644 --- a/web/tests/CloudAgentProtocolTest.php +++ b/web/tests/CloudAgentProtocolTest.php @@ -203,6 +203,17 @@ $handler = new MockHandler([ 'error' => ['code' => 'creation_absent'], ], ]], JSON_THROW_ON_ERROR)), + new Response(409, ['Content-Type' => 'application/json'], + json_encode(['error' => [ + 'code' => 'resource_operation_blocked', + 'message' => 'private blocker detail', + 'retryable' => false, + 'operation' => [ + 'operation_id' => 'operation_12345678', + 'status' => 'failed', + 'error' => ['code' => 'operation_not_dispatched'], + ], + ]], JSON_THROW_ON_ERROR)), ]); $stack = HandlerStack::create($handler); $stack->push(Middleware::history($history)); @@ -341,5 +352,12 @@ $malformedCause = $client->request('POST', '/v1/vms', $actor, $assert(($malformedCause['status'] ?? null) === 'failed' && !isset($malformedCause['cause_code']), 'A malformed durable Agent cause code crossed the PHP trust boundary.'); +$outerCause = $client->request('POST', '/v1/vms', $actor, + ['operation_id' => 'operation_12345678']); +$assert(($outerCause['status'] ?? null) === 'failed' + && ($outerCause['cause_code'] ?? null) === 'resource_operation_blocked' + && ($outerCause['error']['code'] ?? null) === 'operation_not_dispatched' + && !isset($outerCause['message']), + 'A stable outer Agent cause was lost or replaced the embedded terminal error.'); echo "Agent outbound gateway and nested HMAC protocol contract passed.\n"; diff --git a/web/tests/CloudOperationErrorCatalogTest.php b/web/tests/CloudOperationErrorCatalogTest.php index 420018b6..cf78692a 100644 --- a/web/tests/CloudOperationErrorCatalogTest.php +++ b/web/tests/CloudOperationErrorCatalogTest.php @@ -81,6 +81,24 @@ $assert(str_contains((string) CloudOperationErrorCatalog::agentCauseLabel( && CloudOperationErrorCatalog::agentCauseLabel('storage_clone_failed') === null && CloudOperationErrorCatalog::agentCauseLabel('bad/path') === null, 'Reviewed Agent cause labels exposed raw or unknown diagnostic codes.'); +$storageCauseLabels = [ + 'resource_operation_blocked' => '旧任务', + 'storage_path_root_unavailable' => 'storage_path_roots', + 'storage_directory_invalid' => '符号链接', + 'storage_path_not_allowed' => '允许范围', + 'storage_pool_path_conflict' => '不会改绑', + 'storage_pool_definition_invalid' => '存储池定义', + 'storage_pool_configuration_failed' => '自动启动状态', + 'storage_configuration_busy' => '立即失败', + 'storage_configuration_lock_unavailable' => '进程锁', + 'libvirt_unavailable' => 'libvirt', +]; +foreach ($storageCauseLabels as $causeCode => $expectedLabelText) { + $label = CloudOperationErrorCatalog::agentCauseLabel($causeCode); + $assert(is_string($label) && str_contains($label, $expectedLabelText), + 'Storage Agent cause did not receive a reviewed administrator label: ' + . $causeCode); +} $assert(CloudOperationErrorCatalog::workerSummary('create', 'failed', [ 'cause_code' => 'storage_clone_failed',