fix: prevent failed storage tasks from blocking

go
codex 1 month ago
parent 01e3683cb2
commit 683d950725

@ -12,7 +12,7 @@ Virt Cloud 是 PHP 8.5 面板与宿主机 Go Agent 组成的虚拟机销售平
- 每个云镜像可配置仅首次安装执行的启动脚本。创建任务会快照脚本版本;脚本成功、失败或中断后都不会自动重放,租户不能读取或覆盖脚本正文。
- 用户购买套餐开通一台 VM;每次购买或续费生成独立预付费账期及流量额度。额度用尽只断网、不关机;下一已付费账期恢复网络。最终到期只断网;未支付续费账单进入三天宽限期,之后永久删除 VM 定义和面板管理的磁盘。
- 用户页面及登录页保持原虚拟机面板样式,管理员页面保留 venix 主题。普通用户不获得宿主机管理、Agent 设置或退役节点管理入口。
- 任务只有执行中、成功和失败三种公开状态,终态只有 `succeeded` 或 `failed`;所有任务都有绝对截止时间。派发后超时或断线直接终态失败,并标记“效果可能已发生、不可重提”;任务不会停在“待核对”,也不会自动重试、退款或再次删除磁盘。不能可靠受 Go context 中止的 libvirt mutation 在独立子进程中执行,截止后由父 Agent 回收整个进程组。
- 任务只有执行中、成功和失败三种公开状态,终态只有 `succeeded` 或 `failed`;所有任务都有绝对截止时间。派发后超时或断线直接终态失败,并标记“效果可能已发生、不可重放原 operation ID”;任务不会停在“待核对”,也不会自动重试、退款或再次删除磁盘。不能可靠受 Go context 中止的 libvirt mutation 在独立子进程中执行,截止后由父 Agent 回收整个进程组。固定存储池配置是非破坏性的幂等例外:失败任务仍终止且不重放,但不保留跨任务资源栅栏;已定义的固定 pool 不做失败回滚,管理员可显式提交新任务核验或补全同路径固定池,绝不改绑、迁移或删除数据。
## 目录

@ -107,9 +107,10 @@ func main() {
log.Fatalf("initialize request authentication: %v", err)
}
local := hypervisor.LocalConfig{
StoragePathRoots: append([]string(nil), cfg.StoragePathRoots...),
FirmwareLoader: cfg.FirmwareLoader,
FirmwareNVRAMTemplate: cfg.FirmwareNVRAMTemplate, NVRAMDirectory: cfg.NVRAMDirectory,
StoragePathRoots: append([]string(nil), cfg.StoragePathRoots...),
StorageConfigurationLockPath: cfg.JournalPath + ".storage-config.lock",
FirmwareLoader: cfg.FirmwareLoader,
FirmwareNVRAMTemplate: cfg.FirmwareNVRAMTemplate, NVRAMDirectory: cfg.NVRAMDirectory,
TrafficStatePath: cfg.JournalPath + ".traffic.json",
GuestAgentTimeoutSeconds: int(cfg.GuestAgentTimeoutSec),
}
@ -244,8 +245,9 @@ func runOperationWorker(configPath, journalPath, receiptPath string) error {
return fmt.Errorf("initialize worker authentication: %w", err)
}
local := hypervisor.LocalConfig{
StoragePathRoots: append([]string(nil), cfg.StoragePathRoots...),
FirmwareLoader: cfg.FirmwareLoader, FirmwareNVRAMTemplate: cfg.FirmwareNVRAMTemplate,
StoragePathRoots: append([]string(nil), cfg.StoragePathRoots...),
StorageConfigurationLockPath: cfg.JournalPath + ".storage-config.lock",
FirmwareLoader: cfg.FirmwareLoader, FirmwareNVRAMTemplate: cfg.FirmwareNVRAMTemplate,
NVRAMDirectory: cfg.NVRAMDirectory, TrafficStatePath: cfg.JournalPath + ".traffic.json",
GuestAgentTimeoutSeconds: int(cfg.GuestAgentTimeoutSec),
}

@ -6,7 +6,9 @@ import (
"context"
"encoding/json"
"errors"
"os"
"path/filepath"
"syscall"
"git.luoli.live/root/virt/agent/operation"
"libvirt.org/go/libvirt"
@ -33,63 +35,77 @@ func (l *Libvirt) storagePoolName(role string) string {
func (l *Libvirt) ConfigureStorage(ctx context.Context, request StorageConfigurationRequest) (StorageConfigurationResult, error) {
if err := validateStorageDirectories(request, l.local.StoragePathRoots); err != nil {
return StorageConfigurationResult{}, storageConfigurationNotApplied(err)
}
release, err := acquireStorageConfigurationLock(l.local.StorageConfigurationLockPath)
if err != nil {
return StorageConfigurationResult{}, storageConfigurationNotApplied(err)
}
defer release()
if err := configureStorageRoles(ctx, request, func(role, path string) error {
_, err := l.resolveStoragePool(role, path)
return err
}); err != nil {
return StorageConfigurationResult{}, err
}
createdNames := make([]string, 0, len(storageRoles()))
rollback := func(cause error) error {
if cleanupErr := l.rollbackDefinedStoragePools(createdNames); cleanupErr != nil {
return operation.MarkUncertain(errors.Join(cause, cleanupErr))
}
return cause
return StorageConfigurationResult{Configured: true}, nil
}
func acquireStorageConfigurationLock(path string) (func(), error) {
if path == "" || !filepath.IsAbs(path) || filepath.Clean(path) != path {
return nil, NewError(ErrUnavailable, "storage_configuration_lock_unavailable", "storage configuration lock path is invalid", nil)
}
for _, desired := range storageRolePaths(request) {
if err := ctx.Err(); err != nil {
return StorageConfigurationResult{}, rollback(err)
}
binding, created, err := l.resolveStoragePool(desired.Role, desired.Path)
if err != nil {
return StorageConfigurationResult{}, rollback(err)
}
if created {
createdNames = append(createdNames, binding.PoolName)
lock, err := os.OpenFile(path, os.O_CREATE|os.O_RDWR, 0o600)
if err != nil {
return nil, NewError(ErrUnavailable, "storage_configuration_lock_unavailable", "storage configuration lock could not be opened", err)
}
if err := syscall.Flock(int(lock.Fd()), syscall.LOCK_EX|syscall.LOCK_NB); err != nil {
_ = lock.Close()
if errors.Is(err, syscall.EWOULDBLOCK) || errors.Is(err, syscall.EAGAIN) {
return nil, NewError(ErrConflict, "storage_configuration_busy", "another storage configuration helper is still active", nil)
}
return nil, NewError(ErrUnavailable, "storage_configuration_lock_unavailable", "storage configuration lock could not be acquired", err)
}
return StorageConfigurationResult{Configured: true}, nil
return func() {
_ = syscall.Flock(int(lock.Fd()), syscall.LOCK_UN)
_ = lock.Close()
}, nil
}
func (l *Libvirt) resolveStoragePool(role, directory string) (storagePoolBinding, bool, error) {
// resolveStoragePool is deliberately monotonic. Once a fixed pool has been
// defined, a later failure leaves that definition in place so another
// operation can validate and finish it. Rolling pools back is unsafe after a
// timeout because the old helper may resume after a newer operation succeeds.
func (l *Libvirt) resolveStoragePool(role, directory string) (storagePoolBinding, error) {
name := l.storagePoolName(role)
binding := storagePoolBinding{Role: role, Directory: directory, PoolName: name}
existing, err := l.conn.LookupStoragePoolByName(name)
if err == nil {
defer existing.Free()
if err := l.ensureStoragePoolReady(existing, binding); err != nil {
return storagePoolBinding{}, false, err
return storagePoolBinding{}, err
}
return binding, false, nil
return binding, nil
}
if !isNoStoragePool(err) {
return storagePoolBinding{}, false, translateLibvirtError("lookup fixed storage pool", err)
return storagePoolBinding{}, translateLibvirtError("lookup fixed storage pool", err)
}
document, err := (&libvirtxml.StoragePool{
Type: "dir", Name: name, Target: &libvirtxml.StoragePoolTarget{Path: directory},
}).Marshal()
if err != nil {
return storagePoolBinding{}, false, NewError(ErrInvalid, "storage_pool_definition_invalid", "storage pool definition could not be generated", err)
return storagePoolBinding{}, NewError(ErrInvalid, "storage_pool_definition_invalid", "storage pool definition could not be generated", err)
}
pool, err := l.conn.StoragePoolDefineXML(document, 0)
if err != nil {
return storagePoolBinding{}, false, mutationLibvirtError("define storage pool", err)
return storagePoolBinding{}, mutationLibvirtError("define storage pool", err)
}
defer pool.Free()
if err := l.ensureStoragePoolReady(pool, binding); err != nil {
if cleanupErr := l.rollbackDefinedStoragePools([]string{name}); cleanupErr != nil {
return storagePoolBinding{}, false, operation.MarkUncertain(errors.Join(err, cleanupErr))
}
return storagePoolBinding{}, false, err
return storagePoolBinding{}, err
}
return binding, true, nil
return binding, nil
}
func storagePoolDefinition(pool storagePoolReader) (libvirtxml.StoragePool, error) {
@ -232,33 +248,6 @@ func (l *Libvirt) storagePoolHandleReady(pool storagePoolReader, binding storage
return persistent && ready && storagePoolTargetMatches(definition, binding.Directory), nil
}
func (l *Libvirt) rollbackDefinedStoragePools(names []string) error {
var failures []error
for index := len(names) - 1; index >= 0; index-- {
pool, err := l.conn.LookupStoragePoolByName(names[index])
if isNoStoragePool(err) {
continue
}
if err != nil {
failures = append(failures, translateLibvirtError("lookup storage pool during rollback", err))
continue
}
active, activeErr := pool.IsActive()
if activeErr != nil {
failures = append(failures, translateLibvirtError("inspect storage pool during rollback", activeErr))
} else if active {
if destroyErr := pool.Destroy(); destroyErr != nil {
failures = append(failures, mutationLibvirtError("stop storage pool during rollback", destroyErr))
}
}
if undefineErr := pool.Undefine(); undefineErr != nil {
failures = append(failures, mutationLibvirtError("undefine storage pool during rollback", undefineErr))
}
pool.Free()
}
return errors.Join(failures...)
}
func isNoStoragePool(err error) bool {
var libvirtErr libvirt.Error
return errors.As(err, &libvirtErr) && libvirtErr.Code == libvirt.ERR_NO_STORAGE_POOL

@ -0,0 +1,53 @@
//go:build libvirt
package hypervisor
import (
"errors"
"path/filepath"
"testing"
)
func TestStorageConfigurationLockFailsFastAndReleases(t *testing.T) {
path := filepath.Join(t.TempDir(), "storage-config.lock")
release, err := acquireStorageConfigurationLock(path)
if err != nil {
t.Fatal(err)
}
secondRelease, err := acquireStorageConfigurationLock(path)
if err == nil {
secondRelease()
release()
t.Fatal("concurrent storage configuration acquired the host lock")
}
var public interface{ PublicCode() string }
if !errors.As(err, &public) || public.PublicCode() != "storage_configuration_busy" {
release()
t.Fatalf("concurrent lock error = %v", err)
}
wrapped := storageConfigurationNotApplied(err)
var semantics interface{ FailureSemantics() (bool, bool) }
if !errors.As(wrapped, &semantics) {
release()
t.Fatal("pre-mutation storage failure did not expose failure semantics")
}
effect, safe := semantics.FailureSemantics()
if effect || !safe {
release()
t.Fatalf("pre-mutation storage failure semantics = effect:%v safe:%v", effect, safe)
}
release()
thirdRelease, err := acquireStorageConfigurationLock(path)
if err != nil {
t.Fatalf("storage configuration lock remained held after release: %v", err)
}
thirdRelease()
}
func TestStorageConfigurationLockRejectsInvalidPath(t *testing.T) {
if _, err := acquireStorageConfigurationLock("relative.lock"); err == nil {
t.Fatal("relative storage configuration lock path was accepted")
}
}

@ -1,6 +1,7 @@
package hypervisor
import (
"context"
"crypto/sha256"
"encoding/hex"
"errors"
@ -24,6 +25,21 @@ type storageRolePath struct {
Path string
}
type storageConfigurationNotAppliedError struct{ cause error }
func (e *storageConfigurationNotAppliedError) Error() string { return e.cause.Error() }
func (e *storageConfigurationNotAppliedError) Unwrap() error { return e.cause }
func (e *storageConfigurationNotAppliedError) FailureSemantics() (bool, bool) {
return false, true
}
func storageConfigurationNotApplied(err error) error {
if err == nil {
return nil
}
return &storageConfigurationNotAppliedError{cause: err}
}
func storageRoles() []string {
return []string{StorageRoleManaged, StorageRoleTemplate, StorageRoleSeed, StorageRoleUserISO, StorageRoleRescue}
}
@ -128,6 +144,21 @@ func storageRolePaths(request StorageConfigurationRequest) []storageRolePath {
}
}
// configureStorageRoles deliberately retains all completed role mutations if
// a later role fails. A new operation can then reuse and finish the same fixed
// pool mapping without any rollback racing a late worker.
func configureStorageRoles(ctx context.Context, request StorageConfigurationRequest, resolve func(string, string) error) error {
for _, desired := range storageRolePaths(request) {
if err := ctx.Err(); err != nil {
return err
}
if err := resolve(desired.Role, desired.Path); err != nil {
return err
}
}
return nil
}
func ValidateStorageConfigurationRequest(request StorageConfigurationRequest) error {
paths := storageRolePaths(request)
for _, item := range paths {

@ -1,6 +1,11 @@
package hypervisor
import (
"context"
"errors"
"go/ast"
"go/parser"
"go/token"
"os"
"path/filepath"
"strings"
@ -109,3 +114,72 @@ func TestStorageConfigurationPathDigestsDoNotExposePaths(t *testing.T) {
t.Fatalf("path digests = %#v", digests)
}
}
func TestStorageConfigurationNeverRollsBackFixedPools(t *testing.T) {
file, err := parser.ParseFile(token.NewFileSet(), "libvirt_storage_configuration.go", nil, 0)
if err != nil {
t.Fatal(err)
}
ast.Inspect(file, func(node ast.Node) bool {
switch value := node.(type) {
case *ast.FuncDecl:
if value.Name.Name == "rollbackDefinedStoragePools" {
t.Error("fixed storage pool rollback helper must not exist")
}
case *ast.CallExpr:
selector, ok := value.Fun.(*ast.SelectorExpr)
if ok && (selector.Sel.Name == "Destroy" || selector.Sel.Name == "Undefine") {
t.Errorf("storage configuration must not call %s on a fixed pool", selector.Sel.Name)
}
}
return true
})
}
func TestConfigureStorageRolesRetainsPartialProgressForSuccessor(t *testing.T) {
request := validStorageConfigurationRequest()
retained := make(map[string]string)
failure := errors.New("seed readiness failed")
err := configureStorageRoles(context.Background(), request, func(role, path string) error {
if role == StorageRoleSeed {
return failure
}
retained[role] = path
return nil
})
if !errors.Is(err, failure) {
t.Fatalf("first attempt error = %v", err)
}
if len(retained) != 2 || retained[StorageRoleManaged] != request.ManagedDirectory ||
retained[StorageRoleTemplate] != request.TemplateDirectory {
t.Fatalf("partial fixed-pool progress was not retained: %#v", retained)
}
err = configureStorageRoles(context.Background(), request, func(role, path string) error {
if existing, ok := retained[role]; ok && existing != path {
return errors.New("fixed pool path was rebound")
}
retained[role] = path
return nil
})
if err != nil {
t.Fatalf("same-path successor did not converge: %v", err)
}
if len(retained) != len(storageRoles()) {
t.Fatalf("successor completed %d fixed roles, want %d", len(retained), len(storageRoles()))
}
}
func TestConfigureStorageRolesStopsBetweenRolesOnCancellation(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
var calls []string
err := configureStorageRoles(ctx, validStorageConfigurationRequest(), func(role, _ string) error {
calls = append(calls, role)
cancel()
return nil
})
if !errors.Is(err, context.Canceled) || len(calls) != 1 || calls[0] != StorageRoleManaged {
t.Fatalf("canceled role traversal calls=%v err=%v", calls, err)
}
}

@ -226,12 +226,13 @@ type DomainDefinition struct {
}
type LocalConfig struct {
StoragePathRoots []string
FirmwareLoader string
FirmwareNVRAMTemplate string
NVRAMDirectory string
TrafficStatePath string
GuestAgentTimeoutSeconds int
StoragePathRoots []string
StorageConfigurationLockPath string
FirmwareLoader string
FirmwareNVRAMTemplate string
NVRAMDirectory string
TrafficStatePath string
GuestAgentTimeoutSeconds int
}
type LocalNetworkSource struct {

@ -1238,6 +1238,16 @@ func (j *Journal) initialize() error {
return fmt.Errorf("create operation resource blocker bucket: %w", err)
}
}
// resource_blockers is a derived admission index. Host storage
// configuration keeps immutable failure evidence, but a terminal attempt
// must never prevent an administrator from submitting a new, independently
// identified configuration request. Remove entries written by the earlier
// schema-v6 policy before this process admits any request. The surrounding
// Bolt transaction makes the repair atomic, and the operation plus its
// immutable completion receipt remain untouched.
if err := removeStorageConfigurationBlockers(operations, blockers); err != nil {
return err
}
now := j.currentTime()
var recovered []Record
var stale [][]byte
@ -1652,10 +1662,79 @@ func blockerEntryPrefix(lockKey string) []byte {
return []byte(lockKey + "\x00")
}
func blocksResources(record Record) bool {
func effectfulTerminalFailure(record Record) bool {
return !record.Tombstone && record.State == StateFailed && record.EffectMayHaveApplied
}
func blocksResources(record Record) bool {
// Fixed storage-pool registration is monotonic: a successor validates and
// reuses matching pools, and never deletes or rebinds an existing pool. Keep
// the immutable failed operation as evidence, but do not let it fence a new,
// explicitly identified configure-storage request.
return effectfulTerminalFailure(record) && record.Action != "configure-storage"
}
// removeStorageConfigurationBlockers removes only derived blocker-index
// entries owned by host storage configuration operations. Every reference is
// validated before any key is deleted so a missing, malformed, or stale
// non-storage reference still fails Agent startup instead of silently dropping
// a safety fence.
func removeStorageConfigurationBlockers(operations, blockers *bolt.Bucket) error {
if operations == nil || blockers == nil {
return errors.New("operation resource blocker index is unavailable")
}
removable := make([][]byte, 0)
cursor := blockers.Cursor()
for key, _ := cursor.First(); key != nil; key, _ = cursor.Next() {
separator := bytes.IndexByte(key, 0)
if separator <= 0 || separator != bytes.LastIndexByte(key, 0) || separator == len(key)-1 {
return fmt.Errorf("invalid operation resource blocker key %q", key)
}
lockKey := string(key[:separator])
operationID := string(key[separator+1:])
if err := validateLockKey(lockKey); err != nil || !operationIDPattern.MatchString(operationID) {
return fmt.Errorf("invalid operation resource blocker key %q", key)
}
encoded := operations.Get([]byte(operationID))
if encoded == nil {
return fmt.Errorf("resource blocker index references missing operation %q", operationID)
}
record, err := decodeRecord(encoded)
if err != nil {
return fmt.Errorf("decode resource blocker %q: %w", operationID, err)
}
if record.OperationID != operationID {
return fmt.Errorf("resource blocker index references mismatched operation %q", operationID)
}
matchingKey := false
for _, expected := range effectiveLockKeys(record.OperationID, record.ResourceID, record.Action, record.LockKeys) {
if expected == lockKey {
matchingKey = true
break
}
}
if !matchingKey {
return fmt.Errorf("resource blocker index references unexpected lock %q for operation %q", lockKey, operationID)
}
if record.Action == "configure-storage" {
if !effectfulTerminalFailure(record) {
return fmt.Errorf("resource blocker index references non-blocking operation %q", operationID)
}
removable = append(removable, append([]byte(nil), key...))
continue
}
if !blocksResources(record) {
return fmt.Errorf("resource blocker index references resolved operation %q", operationID)
}
}
for _, key := range removable {
if err := blockers.Delete(key); err != nil {
return fmt.Errorf("delete retired storage configuration blocker %q: %w", key, err)
}
}
return nil
}
func syncResourceBlockers(blockers *bolt.Bucket, previous, current *Record) error {
if blockers == nil {
return errors.New("operation resource blocker bucket is unavailable")

@ -50,6 +50,18 @@ func testSpec(id, resource string) Spec {
}
}
func storageConfigurationSpec(id string) Spec {
return Spec{
OperationID: id,
ResourceID: "host-storage",
Action: "configure-storage",
Fingerprint: Fingerprint("POST", "/v1/storage/configuration", []byte(id)),
Destructive: true,
LockKeys: []string{"storage-config"},
HeavyIO: true,
}
}
func TestDefaultResourceLockKeyUsesCurrentNamespace(t *testing.T) {
key := defaultResourceLockKey("operation-resource-key", "opaque-resource", "delete-vm")
if len(key) != len("resource-hash:")+64 || key[:len("resource-hash:")] != "resource-hash:" {
@ -63,6 +75,104 @@ func TestDefaultResourceLockKeyUsesCurrentNamespace(t *testing.T) {
}
}
func TestStorageConfigurationTerminalFailureDoesNotFenceNewOperation(t *testing.T) {
for _, test := range []struct {
name string
executionFenced bool
}{
{name: "effect unknown"},
{name: "execution fenced", executionFenced: true},
} {
t.Run(test.name, func(t *testing.T) {
j := openTestJournal(t)
first := storageConfigurationSpec("operation-storage-failed")
var calls atomic.Int32
record, replayed, err := j.Execute(context.Background(), first, func(context.Context) (json.RawMessage, error) {
calls.Add(1)
return nil, testFailure{
code: "storage_configuration_failed", message: "storage configuration failed",
effect: true, safe: false, fence: test.executionFenced,
}
})
if err == nil || replayed || record.State != StateFailed || !record.EffectMayHaveApplied ||
record.SafeToResubmit || record.ExecutionFenced != test.executionFenced {
t.Fatalf("first storage result = %#v replayed=%v err=%v", record, replayed, err)
}
replayedRecord, replayed, err := j.Execute(context.Background(), first, func(context.Context) (json.RawMessage, error) {
calls.Add(1)
return nil, errors.New("same operation must not run again")
})
if err != nil || !replayed || replayedRecord.State != StateFailed || calls.Load() != 1 {
t.Fatalf("same operation replay = %#v replayed=%v calls=%d err=%v", replayedRecord, replayed, calls.Load(), err)
}
successor := storageConfigurationSpec("operation-storage-successor")
succeeded, replayed, err := j.Execute(context.Background(), successor, func(context.Context) (json.RawMessage, error) {
calls.Add(1)
return json.RawMessage(`{"configured":true}`), nil
})
if err != nil || replayed || succeeded.State != StateSucceeded || calls.Load() != 2 {
t.Fatalf("independent successor = %#v replayed=%v calls=%d err=%v", succeeded, replayed, calls.Load(), err)
}
})
}
}
func TestStorageConfigurationAdmissionRemainsSerializedWhileRunning(t *testing.T) {
j := openTestJournal(t)
firstStarted := make(chan struct{})
releaseFirst := make(chan struct{})
secondStarted := make(chan struct{})
firstDone := make(chan error, 1)
secondDone := make(chan error, 1)
go func() {
_, _, err := j.Execute(context.Background(), storageConfigurationSpec("operation-storage-running"), func(context.Context) (json.RawMessage, error) {
close(firstStarted)
<-releaseFirst
return nil, testFailure{
code: "storage_configuration_failed", message: "storage configuration failed",
effect: true, safe: false,
}
})
firstDone <- err
}()
<-firstStarted
go func() {
_, _, err := j.Execute(context.Background(), storageConfigurationSpec("operation-storage-waiting"), func(context.Context) (json.RawMessage, error) {
close(secondStarted)
return json.RawMessage(`{"configured":true}`), nil
})
secondDone <- err
}()
deadline := time.Now().Add(time.Second)
for {
select {
case <-secondStarted:
t.Fatal("second storage mutation crossed admission while the first was running")
default:
}
record, err := j.Get("operation-storage-waiting")
if err == nil && record.State == StateReserved {
break
}
if time.Now().After(deadline) {
t.Fatalf("second storage operation did not reach serialized admission: %#v err=%v", record, err)
}
time.Sleep(time.Millisecond)
}
close(releaseFirst)
if err := <-firstDone; err == nil {
t.Fatal("first storage mutation unexpectedly succeeded")
}
if err := <-secondDone; err != nil {
t.Fatalf("second storage mutation remained blocked after terminal failure: %v", err)
}
}
func TestOperationIDIsPermanentTombstoneAndMutationRunsOnce(t *testing.T) {
j := openTestJournal(t)
spec := testSpec("operation-single-dispatch", "vm-single")
@ -537,6 +647,168 @@ func TestSuccessfulForceSuccessorOnlyRemovesPredecessorBlockerIndex(t *testing.T
}
}
func TestInitializeRemovesLegacyStorageBlockerAndPreservesVMFence(t *testing.T) {
path := filepath.Join(t.TempDir(), "operations.db")
j, err := Open(path)
if err != nil {
t.Fatal(err)
}
storage := storageConfigurationSpec("operation-legacy-storage")
if _, _, err := j.Execute(context.Background(), storage, func(context.Context) (json.RawMessage, error) {
return nil, testFailure{
code: "storage_configuration_failed", message: "storage effect unknown", effect: true, safe: false,
}
}); err == nil {
t.Fatal("legacy storage seed unexpectedly succeeded")
}
vm := testSpec("operation-legacy-vm-fence", "vm-legacy-fence")
if _, _, err := j.Execute(context.Background(), vm, func(context.Context) (json.RawMessage, error) {
return nil, testFailure{code: "vm_effect_unknown", message: "VM effect unknown", effect: true, safe: false}
}); err == nil {
t.Fatal("VM fence seed unexpectedly succeeded")
}
storageBlockerKey := blockerEntryKey("storage-config", storage.OperationID)
vmLockKeys := effectiveLockKeys(vm.OperationID, vm.ResourceID, vm.Action, vm.LockKeys)
vmBlockerKey := blockerEntryKey(vmLockKeys[0], vm.OperationID)
storageReceiptKey := []byte(completionEventID(storage.OperationID))
var operationBefore, receiptBefore []byte
if err := j.db.Update(func(tx *bolt.Tx) error {
operations := tx.Bucket(operationsBucket)
blockers := tx.Bucket(resourceBlockersBucket)
receipts := tx.Bucket(completionReceiptsBucket)
if err := blockers.Put(storageBlockerKey, nil); err != nil {
return err
}
operationBefore = append([]byte(nil), operations.Get([]byte(storage.OperationID))...)
receiptBefore = append([]byte(nil), receipts.Get(storageReceiptKey)...)
if len(operationBefore) == 0 || len(receiptBefore) == 0 {
return errors.New("storage operation evidence was not persisted")
}
return nil
}); err != nil {
t.Fatal(err)
}
if err := j.Close(); err != nil {
t.Fatal(err)
}
j, err = Open(path)
if err != nil {
t.Fatal(err)
}
defer j.Close()
if err := j.db.View(func(tx *bolt.Tx) error {
operations := tx.Bucket(operationsBucket)
blockers := tx.Bucket(resourceBlockersBucket)
receipts := tx.Bucket(completionReceiptsBucket)
metadata := tx.Bucket(metadataBucket)
if string(metadata.Get(schemaVersionKey)) != string(schemaVersion) {
return fmt.Errorf("schema version changed during blocker repair")
}
cursor := blockers.Cursor()
if key, _ := cursor.Seek(storageBlockerKey); bytes.Equal(key, storageBlockerKey) {
return errors.New("legacy storage blocker survived restart repair")
}
if key, _ := cursor.Seek(vmBlockerKey); !bytes.Equal(key, vmBlockerKey) {
return errors.New("VM blocker was removed during storage repair")
}
if !bytes.Equal(operationBefore, operations.Get([]byte(storage.OperationID))) ||
!bytes.Equal(receiptBefore, receipts.Get(storageReceiptKey)) {
return errors.New("storage blocker repair rewrote immutable operation evidence")
}
return nil
}); err != nil {
t.Fatal(err)
}
successor := storageConfigurationSpec("operation-after-storage-repair")
if record, replayed, err := j.Execute(context.Background(), successor, func(context.Context) (json.RawMessage, error) {
return json.RawMessage(`{"configured":true}`), nil
}); err != nil || replayed || record.State != StateSucceeded {
t.Fatalf("storage successor after restart repair = %#v replayed=%v err=%v", record, replayed, err)
}
vmCalled := false
if _, _, err := j.Execute(context.Background(), testSpec("operation-after-vm-fence", vm.ResourceID), func(context.Context) (json.RawMessage, error) {
vmCalled = true
return nil, nil
}); !errors.Is(err, ErrResourceBlocked) || vmCalled {
t.Fatalf("VM fence changed during storage repair: called=%v err=%v", vmCalled, err)
}
}
func TestInitializeRejectsBrokenResourceBlockerReferencesAtomically(t *testing.T) {
for _, test := range []struct {
name string
seed func(*bolt.Tx) error
}{
{
name: "missing operation",
seed: func(tx *bolt.Tx) error {
return tx.Bucket(resourceBlockersBucket).Put(
blockerEntryKey("storage-config", "operation-zz-missing-reference"), nil,
)
},
},
{
name: "damaged operation",
seed: func(tx *bolt.Tx) error {
operationID := "operation-zz-damaged-reference"
if err := tx.Bucket(operationsBucket).Put([]byte(operationID), []byte(`{"record":`)); err != nil {
return err
}
return tx.Bucket(resourceBlockersBucket).Put(blockerEntryKey("storage-config", operationID), nil)
},
},
} {
t.Run(test.name, func(t *testing.T) {
path := filepath.Join(t.TempDir(), "operations.db")
j, err := Open(path)
if err != nil {
t.Fatal(err)
}
storage := storageConfigurationSpec("operation-valid-storage-reference")
if _, _, err := j.Execute(context.Background(), storage, func(context.Context) (json.RawMessage, error) {
return nil, testFailure{code: "storage_configuration_failed", message: "unknown", effect: true}
}); err == nil {
t.Fatal("storage seed unexpectedly succeeded")
}
storageBlockerKey := blockerEntryKey("storage-config", storage.OperationID)
if err := j.db.Update(func(tx *bolt.Tx) error {
if err := tx.Bucket(resourceBlockersBucket).Put(storageBlockerKey, nil); err != nil {
return err
}
return test.seed(tx)
}); err != nil {
t.Fatal(err)
}
if err := j.Close(); err != nil {
t.Fatal(err)
}
if reopened, err := Open(path); err == nil {
_ = reopened.Close()
t.Fatal("broken blocker reference was accepted")
}
db, err := bolt.Open(path, 0o600, nil)
if err != nil {
t.Fatal(err)
}
defer db.Close()
if err := db.View(func(tx *bolt.Tx) error {
cursor := tx.Bucket(resourceBlockersBucket).Cursor()
key, _ := cursor.Seek(storageBlockerKey)
if !bytes.Equal(key, storageBlockerKey) {
return errors.New("failed startup partially deleted the valid storage blocker")
}
return nil
}); err != nil {
t.Fatal(err)
}
})
}
}
func TestFreshSchemaRejectsOldJournalWithoutMigration(t *testing.T) {
path := filepath.Join(t.TempDir(), "operations.db")
db, err := bolt.Open(path, 0o600, nil)

@ -7,7 +7,7 @@
- 提交接口只在任务已经持久化后返回 HTTP `202 Accepted`。这表示“任务已接收”,不表示宿主机操作成功。
- 每个任务最多派发一次。派发前拒绝、派发后断线、Agent 超时、worker 退出和回执截止都会在固定期限内落为 `failed`,不会重新排队。
- 终态只有 `succeeded`、`failed`,一旦写入便不可改变。晚到回执只能得到幂等 ACK,不能翻转任务、退款或释放资源。
- `effect_may_have_applied=true` 表示底层副作用可能已经发生。任务本身仍然已经结束,但相关资源继续被安全栅栏保护;新的冲突请求会立即返回错误,不会排队等待。
- `effect_may_have_applied=true` 表示底层副作用可能已经发生。任务本身仍然已经结束;破坏性操作的相关资源继续被安全栅栏保护,新的冲突请求会立即返回错误,不会排队等待。固定存储池配置是非破坏性的例外:保留失败证据但不保留跨任务栅栏,可按已定义 pool 的同一路径,或只修正尚未定义的角色后显式创建新任务。
- 普通用户和管理员界面均不提供任务重试按钮。需要再次执行时,必须先完成只读核验和人工处置,再创建一个经过重新授权的新任务。
- 只有具备种类专用业务账本终结器的失败任务才能解除栅栏。开通只允许用新鲜缺失证据确认 `not_applied`,并原子退款与释放资源;支付根据签名回调或管理员已在外部平台核验的证据收敛本地账本。其他类型当前拒绝假解锁,但任务本身已是终态失败,不会继续占用 worker 或重试。

@ -117,7 +117,7 @@ Agent 配置字段:
先在宿主机创建五个互相独立的真实目录,并为 libvirt/QEMU 配置正确的属主、权限、挂载和剩余空间;Agent 不会创建目录、挂载文件系统或删除其中的数据。保存并启用 Agent 连接后,在后台“虚拟机管理 -> 宿主机 -> 存储目录”填写系统盘与快照、云系统模板、cloud-init Seed、用户 ISO、救援 ISO 五类目录。Agent 校验目录是 `storage_path_roots` 的真实子目录后,分别注册固定名称的 persistent `dir` pool:`virt-cloud-managed`、`virt-cloud-templates`、`virt-cloud-seeds`、`virt-cloud-user-isos`、`virt-cloud-rescue-isos`,并启动、刷新和启用 autostart。若固定名称已被其他目录占用,Agent 会拒绝注册;不会扫描、复用或迁移其他 pool。
libvirt 的持久 pool 定义就是 Agent 后续使用的映射,不另写存储状态文件。系统盘与快照、模板、Seed、用户 ISO 和救援 ISO 后续只使用各自固定 pool。当前实现不兼容旧存储布局,也不提供自动迁移或回退读取;已有数据必须在启用本功能前另行完成受控迁移。首次成功应用后网页不允许更换路径。若任务以失败终态结束,旧 CloudJob 和 operation ID 保留为审计记录且绝不重放;管理员可修正目录后显式提交一个新 CloudJob 和新 operation ID。这个新任务只会幂等地核验并补全固定 pool;既不更换已有 pool 的路径,也不迁移或删除数据。注册成功后回到 `/admin/agents` 重新执行连接测试;`storage_configuration` capability 可用前,宿主机不会进入调度。
libvirt 的持久 pool 定义就是 Agent 后续使用的映射,不另写存储状态文件。系统盘与快照、模板、Seed、用户 ISO 和救援 ISO 后续只使用各自固定 pool。当前实现不兼容旧存储布局,也不提供自动迁移或回退读取;已有数据必须在启用本功能前另行完成受控迁移。首次成功应用后网页不允许更换路径。若任务以失败终态结束,旧 CloudJob 和 operation ID 保留为审计记录且绝不重放;管理员可修正尚未定义角色的目录,或按已经定义的同一路径显式提交一个新 CloudJob 和新 operation ID。固定存储池配置失败不会在 Agent Journal 中保留跨任务 `storage-config` 资源栅栏;Agent 启动时也会清理由旧版本生成的该类派生索引,但不会改写原 operation 或完成回执。配置过程是单调的:已经定义的固定 pool 即使后续步骤失败也不会被回滚、destroy 或 undefine,避免旧超时 helper 晚到后破坏新任务结果。新任务只会幂等地核验并补全同路径固定 pool;既不更换已有 pool 的路径,也不迁移或删除数据。宿主机同时还使用 `journal_path + ".storage-config.lock"` 的非阻塞进程锁;旧 helper 尚未退出时新任务立即以 `storage_configuration_busy` 失败,不等待、不自动重试,也不留下永久栅栏。注册成功后回到 `/admin/agents` 重新执行连接测试;`storage_configuration` capability 可用前,宿主机不会进入调度。
网络接入统一在宿主机页面填写“接入类型”和“网桥 / 网络名称”,新建默认 Linux 网桥 `br0`。面板保存为 `bridge:br0` 或 `network:default` 这样的 `default_network`;名称以字母或数字开头,后续只能包含字母、数字、点、下划线和短横线,bridge 名称为 1~15 字符,libvirt network 名称为 1~64 字符。宿主机上的网桥或 libvirt network 必须由运维预先准备。Agent 只选择、验证和使用现有网络,不创建网桥、不启动 libvirt network,也不修改宿主机系统网卡、路由或挂载。
@ -129,7 +129,7 @@ Agent 的 systemd 单元保持 `ProtectSystem=strict`。`journal_path` 必须位
`journal_path` 是当前版本专用的 bbolt 数据库,不是可编辑 JSON;同目录的 `.traffic.json` 是流量状态,两者都要纳入一致性备份。Agent 会拒绝旧 schema,不执行升级或回填;全新安装必须使用空状态目录。Journal 在变更跨越派发边界前写入 operation tombstone,随后只允许这一次 worker 执行。Agent 重启若发现已启动但没有完整回执的 operation,会直接写入终态失败和执行栅栏,不会重新调用 libvirt。operation ID、指纹、资源、动作和终态永久保留用于防重放;完成回执的网络重发有固定次数和绝对截止时间,不会重新执行 mutation。
每个变更由独立 helper 进程打开自己的 libvirt 连接。父 Agent 到达绝对截止后终止整个 helper 进程组并执行有界 `Wait`;只有确认 helper 已退出才释放运行时 admission。无法回收时持久执行栅栏继续阻止相同资源上的变更,但对应面板任务仍会在回执截止时显示终态失败,不会长期保持运行中。
每个变更由独立 helper 进程打开自己的 libvirt 连接。父 Agent 到达绝对截止后终止整个 helper 进程组并执行有界 `Wait`;正常确认 helper 已退出后释放运行时 admission。若回收仍无法确认,Journal 会把任务终态化并释放进程内 admission:虚拟机和磁盘等破坏性操作继续由持久执行栅栏拒绝同资源的新 mutation;固定存储配置则由 helper 持有的宿主机非阻塞文件锁拒绝并发,旧进程真实退出时由内核自动释放。两类任务都不会长期保持运行中,固定存储任务也不会因旧失败留下永久栅栏。
配置必须存在、可读、是单个严格 JSON 对象并通过内容校验。Agent **不会**检查文件 owner、group 或 mode,也不会因 `0600`、`0640`、`0644` 等可读权限差异拒绝启动;文档中的收紧权限只是运维建议。操作系统本身无权读取、JSON 损坏、字段未知或配置值无效仍会明确启动失败;这些是可用性/内容错误,不是权限位审计。旧的 `listen_address`、`shared_secret`、`tls_cert_file` 和 `tls_key_file` 会作为未知字段被拒绝;Agent 不启动入站监听。

@ -71,7 +71,7 @@ Unix 时间戳
4. 主 Agent 对 worker 使用绝对截止、进程组 TERM/KILL 和有界 `Wait`。只有确认 worker 已退出后才释放运行时资源锁。截止、崩溃或重启直接持久化 `failed/result_unconfirmed`,不扫描、不续作、不重新调用原 mutation。
5. Agent 通过完成 outbox 投递不可变回执。相同 `event_id` 可在固定次数和固定年龄内重复投递;PHP 按事件 ID、operation ID 与正文摘要幂等 ACK。回执传输截止后停止,不能形成无限请求。
6. PHP 任务的公开终态只有 `succeeded` 和 `failed`。`receipt_deadline_at` 到期时直接失败;晚到回执只被 ACK 和审计,不能翻转终态。
7. 失败且 `effect_may_have_applied=true` 时,任务已经结束,但对应资源进入安全锁定。冲突请求立即失败而不是等待;系统不自动退款、释放 reservation 或再次执行。超级管理员完成独立只读检查及宿主机人工收敛后,只能通过审计动作解除资源锁,不能改变原任务结果。
7. 失败且 `effect_may_have_applied=true` 时,任务已经结束,但破坏性操作对应的资源进入安全锁定。冲突请求立即失败而不是等待;系统不自动退款、释放 reservation 或再次执行。超级管理员完成独立只读检查及宿主机人工收敛后,只能通过审计动作解除资源锁,不能改变原任务结果。固定存储池配置不删除、不迁移也不改绑数据,因此失败只终止原 operation,不保留跨任务栅栏;后续只能按已定义 pool 的同一路径,或只修正尚未定义的角色后显式创建新任务,并继续由 `storage-config` 运行时准入及宿主机非阻塞文件锁串行。
同一 `operation_id` 和同一指纹只返回已存在的任务或终态;同一标识绑定不同请求必须拒绝。这个防重放规则不构成重试许可。

@ -60,7 +60,9 @@ PHP 的 `receipt_deadline_at` 到期时直接把任务终态化为失败。之
- 没有专用终结器的类型必须拒绝“只改任务字段”的假解锁。它们的原任务仍已终态失败,但资金或资源保持 fenced,直到实现并验收对应的业务账本终结器;
- 需要修复时创建一个经过人工判断的新操作,而不是复用或重放原任务。
如果业务账本终结器本身因损坏数据或数据库异常失败,公开任务仍必须落为 `failed`,并写入 `finalization_incident`。该不可失败的最终路径清除密封请求载荷、将副作用标记为 unknown,保留资金与资源栅栏,且向管理端显示需要修复账本的事件;它绝不会为了让任务看起来成功而猜测退款、释放或重放 mutation。
固定存储池配置是非破坏性的窄例外:失败回执仍如实保留 `effect_may_have_applied`,原 operation ID 仍永久终止且不能重放,但它不形成跨任务资源栅栏。存储配置只做单调补全,已定义的固定 pool 不因后续失败而被 destroy、undefine 或回滚。管理员可以按已定义的同一路径,或只修正尚未定义的角色后显式创建新任务;新 operation 仍通过 `storage-config` 运行时准入和宿主机非阻塞文件锁串行,并且只核验或补全固定池,不改绑、迁移或删除数据。旧 helper 尚未退出时,新任务必须立即失败而不是等待,且不得自动重试或留下新的永久栅栏。
如果业务账本终结器本身因损坏数据或数据库异常失败,公开任务仍必须落为 `failed`,并写入 `finalization_incident`。该不可失败的最终路径清除密封请求载荷、将副作用标记为 unknown;除上述固定存储池配置例外外,资金与资源栅栏继续保留,并向管理端显示需要修复账本的事件。它绝不会为了让任务看起来成功而猜测退款、释放或重放 mutation。
这一区分同时满足可用性和安全性:用户不会看到永久运行的任务,worker 不会永久占用槽位,而未知破坏性副作用也不会被自动重复执行。

@ -57,11 +57,11 @@ jq empty deploy/*.json.example
3. helper 已启动、回执前崩溃:重启后终态失败并保留 durable fence,调用次数为 1。
4. libvirt 返回后丢失响应:任务在回执截止时失败,调用次数为 1,晚到结果不能翻转终态。
5. helper 超时:父进程 TERM/KILL 整个进程组并执行有界 `Wait`;确认回收后才释放 admission。
6. helper 无法回收:任务仍终态失败,但执行栅栏继续立即拒绝相同资源的新 mutation。
6. helper 无法回收:任务仍终态失败;VM、磁盘等破坏性操作的执行栅栏继续立即拒绝相同资源的新 mutation。固定存储池配置保留失败证据但不得形成跨任务栅栏。
7. Agent/Gateway/PHP worker 分别重启:任务不重派,数据库租约和浏览器 busy 状态均在固定截止内释放。
8. 完成 endpoint 连续失败:同一个不可变事件最多发送配置中的固定次数,并在绝对截止后停止。
测试必须覆盖不相交资源并行、相同资源拒绝、重 I/O 容量限制、nonce 防重放、operation 指纹冲突、Journal schema 不匹配拒绝启动,以及 `.traffic.json` 跨 helper 进程的文件锁。
测试必须覆盖不相交资源并行、相同资源拒绝、存储配置运行时串行但终态失败后允许新 operation、旧存储 blocker 派生索引的原子清理、存储配置非阻塞进程锁的 busy/释放行为、角色中途失败后同路径新任务单调补全且不回滚、重 I/O 容量限制、nonce 防重放、operation 指纹冲突、Journal schema 不匹配拒绝启动,以及 `.traffic.json` 跨 helper 进程的文件锁。
## 4. libvirt 语义
@ -73,7 +73,7 @@ jq empty deploy/*.json.example
## 5. 资源、资金与人工处置
- 派发后失败默认 `effect_may_have_applied=true`、`safe_to_resubmit=false`;同 VM、卷、过滤器、地址、模板或账单的新冲突请求立即失败,不进入等待队列。
- 派发后失败默认 `effect_may_have_applied=true`、`safe_to_resubmit=false`;同 VM、卷、过滤器、地址、模板或账单的新冲突请求立即失败,不进入等待队列。固定存储池配置保持同一 operation 不重放,但失败后不以旧任务阻断具有新 operation ID 的显式请求。
- 副作用可能发生时禁止自动退款、释放 reservation/IP、再次删除磁盘或创建替代任务。
- 管理员 `inspect` 只能读;不得改变 Journal、任务、Domain、卷、过滤器或 libvirt job。
- `resolve-effect` 只对已有种类专用终结器开放:开通任务用新鲜宿主机缺失证据确认 `not_applied`,先写一次不执行 libvirt 的 Agent 栅栏审计元数据,再原子退款并释放面板资源;支付任务只依据签名回调或管理员已核验的本地渠道账本收敛,绝不重新下单。其他任务类型必须拒绝假解锁。任何路径都不能翻转原任务的 `failed` 终态;解除栅栏后若需新操作,必须由用户另行显式提交。

@ -30,7 +30,7 @@ final class CloudAdminOperationsService
. "COALESCE(SUM(CASE WHEN cloud_jobs.phase <> 'terminal' THEN 1 ELSE 0 END), 0) AS active, "
. "COALESCE(SUM(CASE WHEN cloud_jobs.phase = 'terminal' AND cloud_jobs.outcome = 'succeeded' THEN 1 ELSE 0 END), 0) AS succeeded, "
. "COALESCE(SUM(CASE WHEN cloud_jobs.phase = 'terminal' AND cloud_jobs.outcome = 'failed' THEN 1 ELSE 0 END), 0) AS failed, "
. "COALESCE(SUM(CASE WHEN cloud_jobs.phase = 'terminal' AND cloud_jobs.outcome = 'failed' AND cloud_jobs.effect_resolution = 'unknown' THEN 1 ELSE 0 END), 0) AS unresolved_effect"
. "COALESCE(SUM(CASE WHEN cloud_jobs.phase = 'terminal' AND cloud_jobs.outcome = 'failed' AND cloud_jobs.effect_resolution = 'unknown' AND cloud_jobs.kind <> 'host_storage_configure' THEN 1 ELSE 0 END), 0) AS unresolved_effect"
)->first();
$summary = [];
foreach (['total', 'active', 'succeeded', 'failed',
@ -88,6 +88,14 @@ final class CloudAdminOperationsService
if ($job === null) {
throw new CloudError('任务不存在', 404);
}
$storageConfiguration = (string) $job->kind
=== 'host_storage_configure';
$effectNote = $storageConfiguration
? '存储配置失败不会保留跨任务资源栅栏;旧 operation ID 仍永久终止且绝不重放,管理员可按已有固定池的同一路径用新 operation ID 显式提交独立任务。'
: '副作用可能已发生时,资源安全栅栏保持生效;只有超级管理员完成新鲜只读检查并记录审计结论后,才会显式解除面板与 Agent 栅栏。';
$ledgerNote = $storageConfiguration
? '新的存储配置任务只会核验或补全路径一致的固定池;不会更换已有池路径,也不会迁移或删除数据。'
: '其他类型若还没有专用业务账本终结器,系统会拒绝假解锁;任务已终态失败,资源栅栏保持安全锁定。';
return [
'job' => self::row($job, self::canManage($actor)),
'generated_at' => time(),
@ -95,10 +103,10 @@ final class CloudAdminOperationsService
'任务状态来自数据库,不会因打开页面而联系 Agent。',
'任务最多派发一次;失败后不会自动核对、重新排队或重放宿主机操作。',
'宿主机只读检查不改变任务终态,也不执行清理或续作。',
'副作用可能已发生时,资源安全栅栏保持生效;只有超级管理员完成新鲜只读检查并记录审计结论后,才会显式解除面板与 Agent 栅栏。',
$effectNote,
'开通任务只有在宿主机明确确认虚拟机和管理资源均不存在时,才能按未应用原子退款并释放库存、公网 IP 与资源预留;只读检查不能把开通任务确认成已应用。',
'支付创建结果不明时,签名回调可自动确认已应用;人工结论只收敛本地支付账本,绝不重新向渠道下单。',
'其他类型若还没有专用业务账本终结器,系统会拒绝假解锁;任务已终态失败,资源栅栏保持安全锁定。',
$ledgerNote,
'正常关机明确返回 force_required 时,强制删除会创建具有新操作标识的独立任务。',
],
];
@ -493,7 +501,9 @@ final class CloudAdminOperationsService
'action' => $labels['action_label'],
'status' => self::STATUSES[$status],
'effect' => (string) $job->effect_resolution === 'unknown'
? '副作用状态未知;任务已失败,资源已安全锁定'
? ((string) $job->kind === 'host_storage_configure'
? '部分固定存储池可能已应用;旧任务已终止且不阻断新的独立任务'
: '副作用状态未知;任务已失败,资源已安全锁定')
: '无任务级安全栅栏',
],
];
@ -539,6 +549,10 @@ final class CloudAdminOperationsService
return [true,
'需要先在外部支付平台核实原订单;本操作不会重新下单'];
}
if ((string) $job->kind === 'host_storage_configure') {
return [false,
'存储配置终态失败不保留跨任务资源栅栏;请按已有固定池的同一路径,或只修正尚未定义的角色后提交新任务'];
}
return [false,
'该任务类型尚无可原子收敛业务账本的解除流程;任务已失败终止,资源保持安全锁定'];
}

@ -301,6 +301,14 @@ final class CloudAgentClient
$operation['result'] = $record['result'];
}
$causeCode = self::operationCauseCode($record['cause_code'] ?? null);
if ($causeCode === null && !array_key_exists('cause_code', $record)) {
// Admission failures such as resource_operation_blocked are
// represented by the authenticated outer envelope while the
// embedded journal record retains its own terminal error. Keep
// both facts instead of replacing the record error with the
// envelope code.
$causeCode = self::operationCauseCode($error['code'] ?? null);
}
if ($causeCode !== null) {
$operation['cause_code'] = $causeCode;
}

@ -126,6 +126,16 @@ final class CloudOperationErrorCatalog
'managed_disk_shared' => '目标受管磁盘仍被另一台虚拟机的当前配置、快照或保存状态引用;请先解除引用,再立即核对',
'managed_filter_shared' => '目标网络过滤器仍被另一台虚拟机或过滤器引用;请先解除引用,再立即核对',
'graceful_shutdown_ambiguous' => '正常关机请求结果无法确认;该任务不会授权强制删除,请立即核对后重新发起正常关机操作',
'resource_operation_blocked' => '本次请求因当时存在的资源栅栏未执行;固定存储配置请先更新并重启 Agent 后创建新任务,其他资源仍需核对旧任务',
'storage_path_root_unavailable' => 'Agent 无法访问 storage_path_roots 允许根目录;请检查目录存在性、权限和服务沙箱',
'storage_directory_invalid' => '存储目录不存在、不是目录或路径中包含符号链接;请修正宿主机目录',
'storage_path_not_allowed' => '存储目录不在 Agent storage_path_roots 允许范围内;请核对 Agent 配置',
'storage_pool_path_conflict' => '固定存储池已存在,但类型或目标路径与请求不一致;为保护现有磁盘,Agent 不会改绑',
'storage_pool_definition_invalid' => '固定存储池定义无效或无法生成;请检查 libvirt 存储池定义',
'storage_pool_configuration_failed' => '固定存储池未达到持久、活动且自动启动状态;请检查 libvirt 存储池状态',
'storage_configuration_busy' => '另一存储配置 helper 尚未退出;本次任务已立即失败且不会重试,请在 helper 退出后创建新任务',
'storage_configuration_lock_unavailable' => 'Agent 无法使用存储配置进程锁;请检查 journal_path 所在目录的权限和文件系统锁支持',
'libvirt_unavailable' => 'Agent 无法连接 libvirt;请检查 libvirt 服务和 Agent 权限',
];
/**

@ -376,6 +376,7 @@ function renderDetail(payload) {
['宿主机', `${text(job.host_name)}${job.host_id ? ` #${job.host_id}` : ''}`],
['关联实例', job.instance_id ? `${text(job.instance_name)} #${job.instance_id}` : '无关联实例'],
['实例 UUID', job.instance_uuid], ['关联订单', job.order_id ? `#${job.order_id}` : '无关联订单'],
['Agent 原因', job.agent_cause_label || job.agent_cause_code],
['创建时间', date(job.created_at)], ['更新时间', date(job.updated_at)],
]));
const failure = failureText(job, view);

@ -188,7 +188,7 @@ function renderHostStorage(payload) {
for (const [field, id] of Object.entries(storageFields)) {
const input = byId(id); input.value = payload[field] || ''; input.readOnly = locked; input.disabled = !canManage;
}
const labels = { unconfigured: '尚未配置', applying: '正在等待 Agent 完成', failed: '上一次任务已失败且不会自动重试;可修正目录后提交新的独立任务', applied: '已应用' };
const labels = { unconfigured: '尚未配置', applying: '正在等待 Agent 完成', failed: '上一次任务已失败且不会自动重试;新任务须沿用已定义固定池的路径,只可修正尚未定义的角色', applied: '已应用' };
let message = labels[payload.state] || '状态协议无效,资源已安全锁定,需管理员人工处理';
if (payload.state === 'failed' && !canSubmit) {
message = '应用失败,但任务账本状态不完整,请联系管理员核查';
@ -248,7 +248,7 @@ async function saveHostStorage(event) {
if (!currentHost()) return;
renderHostStorage(payload);
const next = payload.state === 'failed' && payload.can_submit === true
? ' 旧任务已终止且不会重放;修正目录后可提交新的独立任务。'
? ' 旧任务已终止且不会重放;新任务须沿用已定义固定池的路径,只可修正尚未定义的角色。'
: ' 当前任务账本尚未进入可提交状态,请联系管理员核查。';
hostStorageNotice(`${failure}${partial}${next}`, true);
})

@ -5,7 +5,7 @@
{assign var=operations_statuses value=['all'=>'全部状态','active'=>'执行中','succeeded'=>'已成功','failed'=>'已失败']}
<main data-admin-operations data-can-manage="{if $admin_can_manage|default:false}1{else}0{/if}">
<div class="py-3 d-flex align-items-sm-center flex-sm-row flex-column gap-3">
<div class="flex-grow-1"><h1 class="fs-20 fw-semibold mb-1">系统运行与任务记录</h1><p class="text-muted mb-0">任务只展示执行中、成功或失败。失败且效果可能已发生时,资源保持安全锁定并由管理员人工处理。</p></div>
<div class="flex-grow-1"><h1 class="fs-20 fw-semibold mb-1">系统运行与任务记录</h1><p class="text-muted mb-0">任务只展示执行中、成功或失败。失败且效果可能已发生时,资源保持安全锁定并由管理员人工处理;固定存储池配置失败不保留跨任务栅栏。</p></div>
<div class="admin-checkout-actions"><span class="admin-config-status {if $operations_error|default:''}is-error{else}is-success{/if}" id="operations-notice" role="status" aria-live="polite">{if $operations_error|default:''}{$operations_error|escape}{else}已读取 {$operations_page.pagination.total|default:0|escape} 条任务{/if}</span><button class="btn btn-light" id="operations-refresh" type="button"><i class="mdi mdi-refresh me-1" aria-hidden="true"></i>刷新任务</button></div>
</div>
<section class="card mb-3" aria-labelledby="operations-health-title">

@ -4,7 +4,7 @@
<td data-label="关联实例"><span class="admin-checkout-user"><strong>{$job.instance_name|default:'无关联实例'|escape}</strong>{if $job.instance_id}<span>实例 #{$job.instance_id|escape}</span>{/if}{if $job.order_id}<span>订单 #{$job.order_id|escape}</span>{/if}</span></td>
<td data-label="用户"><span class="admin-checkout-user"><strong>操作人:{$job.actor_name|default:'未记录'|escape}{if $job.actor_id} #{$job.actor_id|escape}{/if}</strong>{if $job.owner_id}<span>所属:{$job.owner_name|default:'未记录'|escape} #{$job.owner_id|escape}</span>{/if}</span></td>
<td data-label="宿主机"><span class="admin-checkout-user"><strong>{$job.host_name|default:'未记录'|escape}</strong><span>{$job.region|default:'地区未记录'|escape}</span></span></td>
<td data-label="状态"><span class="admin-checkout-user">{if isset($job.completed) && !$job.completed && $job.status == 'active'}<strong class="text-primary">执行中</strong>{elseif isset($job.completed) && $job.completed && in_array($job.status,['succeeded','failed'])}<strong class="{if $job.status == 'succeeded'}text-success{else}text-danger{/if}">{if $job.status == 'succeeded'}已成功{else}已失败{/if}</strong>{if $job.status == 'failed' && $job.effect_may_have_applied|default:false && $job.effect_resolution == 'unknown'}<span class="text-danger">操作效果可能已发生;资源已安全锁定,需管理员人工处理。</span>{elseif $job.status == 'failed' && $job.effect_may_have_applied|default:false && in_array($job.effect_resolution,['applied','not_applied'])}<span>任务仍为失败;人工审计已解除任务级安全栅栏。</span>{/if}{else}<strong class="text-danger">协议错误</strong><span class="text-danger">资源按可能已生效处理并保持锁定。</span>{/if}</span></td>
<td data-label="状态"><span class="admin-checkout-user">{if isset($job.completed) && !$job.completed && $job.status == 'active'}<strong class="text-primary">执行中</strong>{elseif isset($job.completed) && $job.completed && in_array($job.status,['succeeded','failed'])}<strong class="{if $job.status == 'succeeded'}text-success{else}text-danger{/if}">{if $job.status == 'succeeded'}已成功{else}已失败{/if}</strong>{if $job.status == 'failed' && $job.kind == 'host_storage_configure'}<span class="text-danger">旧存储任务已终止且不会重放;后续任务须沿用已定义固定池的路径。</span>{elseif $job.status == 'failed' && $job.effect_may_have_applied|default:false && $job.effect_resolution == 'unknown'}<span class="text-danger">操作效果可能已发生;资源已安全锁定,需管理员人工处理。</span>{elseif $job.status == 'failed' && $job.effect_may_have_applied|default:false && in_array($job.effect_resolution,['applied','not_applied'])}<span>任务仍为失败;人工审计已解除任务级安全栅栏。</span>{/if}{else}<strong class="text-danger">协议错误</strong><span class="text-danger">资源按可能已生效处理并保持锁定。</span>{/if}</span></td>
<td data-label="执行信息"><span class="admin-checkout-user"><strong>{$job.operation_id|default:'无操作标识'|escape}</strong><span>{$job.phase|default:'未记录阶段'|escape}</span></span></td>
<td data-label="更新与异常"><span class="admin-checkout-user"><strong>{if $job.updated_at}{$job.updated_at|date_format:'%Y-%m-%d %H:%M:%S'|escape}{else}更新时间未记录{/if}</strong>{if $job.error}<span class="text-danger">{$job.error|escape}</span>{else}<span>未记录错误</span>{/if}</span></td>
<td data-label="操作"><div class="d-flex flex-wrap gap-1"><button class="btn btn-primary btn-sm" type="button" data-operation-detail="{$job.id|escape}" aria-label="查看任务 #{$job.id|escape} 的详情">查看详情</button></div></td>

@ -1,2 +1,2 @@
{* Shared Venix CSS/JS cache key. Increment whenever a shared browser asset changes. *}
{assign var=venix_asset_version value='2026101901' scope='global'}
{assign var=venix_asset_version value='2026101902' scope='global'}

@ -27,6 +27,7 @@ assert.match(source, /可到宿主机存储目录页修正后提交新的独立
assert.match(source, /storageFailure/);
assert.match(source, /收尾事件仅保留审计且不会重放/);
assert.match(source, /不会影响存储目录页提交新的独立任务/);
assert.match(source, /\['Agent 原因', job\.agent_cause_label \|\| job\.agent_cause_code\]/);
// The one retained high-risk action creates a distinct task once. The UI
// validates the fresh receipt, immediately acknowledges it, and observes the

@ -49,11 +49,16 @@ $items = [
'instance_id' => null, 'instance_name' => null, 'instance_uuid' => null, 'instance_status' => null,
'host_id' => null, 'host_name' => '', 'region' => '', 'links' => ['instance' => null, 'billing' => null],
'display' => ['kind' => '到期清理', 'status' => '执行中', 'attention' => '', 'lease' => '未持有执行租约']]),
array_replace($common, ['id' => '44', 'kind' => 'host_storage_configure',
'kind_label' => '配置宿主机存储', 'instance_id' => null,
'instance_name' => null, 'instance_uuid' => null,
'instance_status' => null, 'owner_id' => null, 'owner_name' => null,
'owner_email' => null]),
];
$page = [
'items' => $items,
'pagination' => ['page' => 1, 'per_page' => 20, 'total' => 43, 'total_pages' => 3, 'from' => 1, 'to' => 3],
'summary' => ['total' => 43, 'active' => 1, 'succeeded' => 20,
'pagination' => ['page' => 1, 'per_page' => 20, 'total' => 44, 'total_pages' => 3, 'from' => 1, 'to' => 4],
'summary' => ['total' => 44, 'active' => 1, 'succeeded' => 20,
'failed' => 22, 'unresolved_effect' => 1],
'filters' => ['status' => 'all', 'kind' => '', 'host_id' => null, 'q' => '', 'page' => 1, 'per_page' => 20],
'hosts' => [['id' => '7', 'name' => '香港宿主机 A', 'region' => 'HongKong']],
@ -114,9 +119,10 @@ $dom = new DOMDocument();
libxml_use_internal_errors(true);
$dom->loadHTML('<?xml encoding="UTF-8">' . $html);
$xpath = new DOMXPath($dom);
$assert($xpath->query('//tbody[@id="operations-body"]/tr')->length === 3, 'SSR must include all task rows.');
$assert($xpath->query('//tbody[@id="operations-body"]/tr')->length === 4, 'SSR must include all task rows.');
$assert(str_contains($dom->textContent, 'vm-demo') && str_contains($dom->textContent, '系统策略')
&& str_contains($dom->textContent, '操作效果可能已发生')
&& str_contains($dom->textContent, '后续任务须沿用已定义固定池的路径')
&& str_contains($dom->textContent, '执行中'),
'Useful task records and terminal safety state are absent before JavaScript.');
$assert($xpath->query('//select[@name="status"]/option[@value="all" and @selected]')->length === 1,

@ -173,6 +173,13 @@ namespace {
'effect_may_have_applied' => false, 'safe_to_resubmit' => false,
'error_code' => 'validation_failed',
]);
$insert(6, [
'order_id' => null, 'instance_id' => null,
'kind' => 'host_storage_configure',
'outcome' => 'failed', 'status' => 'failed',
'effect_may_have_applied' => true, 'safe_to_resubmit' => false,
'effect_resolution' => 'unknown', 'error_code' => 'operation_failed',
]);
$admin = ['user_id' => 99, 'is_admin' => true,
'is_super_admin' => true, 'can_manage' => true];
@ -182,14 +189,14 @@ namespace {
'is_super_admin' => false, 'can_manage' => false];
$all = Operations::page($admin, ['status' => 'all']);
$assert($ids($all['items']) === [5, 4, 3, 2, 1]
$assert($ids($all['items']) === [6, 5, 4, 3, 2, 1]
&& $all['summary'] === [
'total' => 5, 'active' => 1, 'succeeded' => 1,
'failed' => 3, 'unresolved_effect' => 1,
'total' => 6, 'active' => 1, 'succeeded' => 1,
'failed' => 4, 'unresolved_effect' => 1,
], 'Operations inventory did not expose active and terminal states exactly.');
foreach ([
'active' => [1], 'succeeded' => [2],
'failed' => [5, 4, 3], 'attention' => [5, 4, 3],
'failed' => [6, 5, 4, 3], 'attention' => [6, 5, 4, 3],
] as $status => $expected) {
$page = Operations::page($admin, ['status' => $status]);
$assert($ids($page['items']) === $expected,
@ -221,6 +228,17 @@ namespace {
'Accepted task was exposed as a fabricated terminal result.');
$assert(Operations::detail($readonly, '3')['job']['can_resolve_effect']
=== false, 'Read-only administrator received effect-resolution authority.');
$storageDetail = Operations::detail($admin, '6');
$assert($storageDetail['job']['can_resolve_effect'] === false
&& str_contains($storageDetail['job']['resolve_effect_reason'],
'不保留跨任务资源栅栏')
&& str_contains($storageDetail['job']['display']['effect'],
'不阻断新的独立任务')
&& str_contains(implode(' ', $storageDetail['notes']),
'旧 operation ID 仍永久终止且绝不重放')
&& str_contains(implode(' ', $storageDetail['notes']),
'不会更换已有池路径'),
'Storage failure was still presented as a durable resource blocker.');
$reason = '已在宿主机只读核对虚拟机与磁盘,确认原操作效果已经发生。';
$operationId = sprintf('%032x', 3);

@ -203,6 +203,17 @@ $handler = new MockHandler([
'error' => ['code' => 'creation_absent'],
],
]], JSON_THROW_ON_ERROR)),
new Response(409, ['Content-Type' => 'application/json'],
json_encode(['error' => [
'code' => 'resource_operation_blocked',
'message' => 'private blocker detail',
'retryable' => false,
'operation' => [
'operation_id' => 'operation_12345678',
'status' => 'failed',
'error' => ['code' => 'operation_not_dispatched'],
],
]], JSON_THROW_ON_ERROR)),
]);
$stack = HandlerStack::create($handler);
$stack->push(Middleware::history($history));
@ -341,5 +352,12 @@ $malformedCause = $client->request('POST', '/v1/vms', $actor,
$assert(($malformedCause['status'] ?? null) === 'failed'
&& !isset($malformedCause['cause_code']),
'A malformed durable Agent cause code crossed the PHP trust boundary.');
$outerCause = $client->request('POST', '/v1/vms', $actor,
['operation_id' => 'operation_12345678']);
$assert(($outerCause['status'] ?? null) === 'failed'
&& ($outerCause['cause_code'] ?? null) === 'resource_operation_blocked'
&& ($outerCause['error']['code'] ?? null) === 'operation_not_dispatched'
&& !isset($outerCause['message']),
'A stable outer Agent cause was lost or replaced the embedded terminal error.');
echo "Agent outbound gateway and nested HMAC protocol contract passed.\n";

@ -81,6 +81,24 @@ $assert(str_contains((string) CloudOperationErrorCatalog::agentCauseLabel(
&& CloudOperationErrorCatalog::agentCauseLabel('storage_clone_failed') === null
&& CloudOperationErrorCatalog::agentCauseLabel('bad/path') === null,
'Reviewed Agent cause labels exposed raw or unknown diagnostic codes.');
$storageCauseLabels = [
'resource_operation_blocked' => '旧任务',
'storage_path_root_unavailable' => 'storage_path_roots',
'storage_directory_invalid' => '符号链接',
'storage_path_not_allowed' => '允许范围',
'storage_pool_path_conflict' => '不会改绑',
'storage_pool_definition_invalid' => '存储池定义',
'storage_pool_configuration_failed' => '自动启动状态',
'storage_configuration_busy' => '立即失败',
'storage_configuration_lock_unavailable' => '进程锁',
'libvirt_unavailable' => 'libvirt',
];
foreach ($storageCauseLabels as $causeCode => $expectedLabelText) {
$label = CloudOperationErrorCatalog::agentCauseLabel($causeCode);
$assert(is_string($label) && str_contains($label, $expectedLabelText),
'Storage Agent cause did not receive a reviewed administrator label: '
. $causeCode);
}
$assert(CloudOperationErrorCatalog::workerSummary('create', 'failed', [
'cause_code' => 'storage_clone_failed',

Loading…
Cancel
Save