diff --git a/agent/api/server.go b/agent/api/server.go index 022bfefe..de314f07 100644 --- a/agent/api/server.go +++ b/agent/api/server.go @@ -380,7 +380,7 @@ func (s *Server) routeVMs(w http.ResponseWriter, r *http.Request, segments []str action := "create-backup:" + request.BackupKey + ":" + request.ConsistencyMode s.runAsyncMutationWithDeadline( w, r, body, actor, request.OperationID, uuid, action, false, - hypervisor.CreateBackupAdmission(uuid, request.BackupKey), backupOperationTimeout, + hypervisor.CreateBackupAdmission(uuid, request), backupOperationTimeout, func(ctx context.Context) (any, error) { return s.Hypervisor.CreateBackup(ctx, uuid, request) }, ) return diff --git a/agent/hypervisor/admission.go b/agent/hypervisor/admission.go index 9de4aec2..b3fe48ef 100644 --- a/agent/hypervisor/admission.go +++ b/agent/hypervisor/admission.go @@ -107,18 +107,27 @@ func CreateSnapshotAdmission(uuid, snapshotKey string) MutationAdmission { }} } -func CreateBackupAdmission(uuid, backupKey string) MutationAdmission { +func CreateBackupAdmission(uuid string, request BackupRequest) MutationAdmission { uuid = strings.ToLower(uuid) - return MutationAdmission{HeavyIO: true, LockKeys: []string{ + locks := []string{ domainLockKey(uuid), vmManagedVolumesLockKey(uuid), - volumeLockKey(managedStoragePoolName, snapshotVolumeName(uuid, backupKey)), - catalogLockKey("backup", backupKey), - }} + volumeLockKey(managedStoragePoolName, snapshotVolumeName(uuid, request.BackupKey)), + catalogLockKey("backup", request.BackupKey), + } + if request.MigrationBundle != nil { + for _, snapshot := range request.MigrationBundle.Snapshots { + locks = append(locks, volumeLockKey( + managedStoragePoolName, + snapshotVolumeName(uuid, snapshot.SnapshotKey), + )) + } + } + return MutationAdmission{HeavyIO: true, LockKeys: locks} } func RestoreBackupAdmission(request RestoreBackupRequest) MutationAdmission { uuid := strings.ToLower(request.UUID) - return MutationAdmission{HeavyIO: true, LockKeys: []string{ + locks := []string{ domainLockKey(uuid), vmManagedVolumesLockKey(uuid), volumeLockKey(managedStoragePoolName, rootVolumeName(uuid)), volumeLockKey(seedStoragePoolName, seedVolumeName(uuid)), @@ -126,7 +135,16 @@ func RestoreBackupAdmission(request RestoreBackupRequest) MutationAdmission { networkIdentityLockKey(request.PublicIPCIDR), networkIdentityLockKey(vmMACAddress(uuid)), catalogLockKey("backup", request.BackupKey), - }} + } + if request.MigrationBundle != nil { + for _, snapshot := range request.MigrationBundle.Snapshots { + locks = append(locks, volumeLockKey( + managedStoragePoolName, + snapshotVolumeName(uuid, snapshot.SnapshotKey), + )) + } + } + return MutationAdmission{HeavyIO: true, LockKeys: locks} } func DeleteSnapshotAdmission(uuid, snapshotKey string) MutationAdmission { diff --git a/agent/hypervisor/backup.go b/agent/hypervisor/backup.go index 243f6463..12a83427 100644 --- a/agent/hypervisor/backup.go +++ b/agent/hypervisor/backup.go @@ -29,6 +29,32 @@ func ValidateBackupRequest(request BackupRequest) error { if err := objecttransfer.ValidateMultipartPlan(request.ObjectUpload); err != nil { return NewError(ErrInvalid, "invalid_multipart_plan", "multipart upload plan is invalid", err) } + if request.MigrationBundle == nil { + if request.PrepareMaintenanceMigration { + return NewError(ErrInvalid, "invalid_migration_bundle", "maintenance migration requires a migration bundle", nil) + } + return nil + } + if !request.PrepareMaintenanceMigration || request.MigrationBundle.Version != 1 || + len(request.MigrationBundle.Snapshots) > 2 { + return NewError(ErrInvalid, "invalid_migration_bundle", "migration bundle version or snapshot count is invalid", nil) + } + seen := make(map[string]struct{}, len(request.MigrationBundle.Snapshots)) + for _, snapshot := range request.MigrationBundle.Snapshots { + if !catalogIDPattern.MatchString(snapshot.SnapshotKey) || snapshot.SnapshotKey == request.BackupKey { + return NewError(ErrInvalid, "invalid_migration_bundle", "migration bundle snapshot identity is invalid", nil) + } + if _, exists := seen[snapshot.SnapshotKey]; exists { + return NewError(ErrInvalid, "invalid_migration_bundle", "migration bundle snapshot identities must be unique", nil) + } + seen[snapshot.SnapshotKey] = struct{}{} + if snapshot.ObjectUpload.MaximumSizeBytes > maxBackupObjectBytes { + return NewError(ErrInvalid, "backup_object_too_large", "migration snapshot object size limit is too large", nil) + } + if err := objecttransfer.ValidateMultipartPlan(snapshot.ObjectUpload); err != nil { + return NewError(ErrInvalid, "invalid_multipart_plan", "migration snapshot upload plan is invalid", err) + } + } return nil } @@ -49,6 +75,33 @@ func ValidateRestoreBackupRequest(request RestoreBackupRequest) error { if err := objecttransfer.ValidateDownload(request.SourceURL, request.SourceSizeBytes, request.SourceSHA256); err != nil { return NewError(ErrInvalid, "invalid_backup_object", "backup object identity is invalid", err) } + if request.MigrationBundle != nil { + bundle := request.MigrationBundle + if bundle.Version != 1 || !bundle.StagePoweredOff || len(bundle.Snapshots) > 2 { + return NewError(ErrInvalid, "invalid_migration_bundle", "migration restore must stage a supported powered-off bundle", nil) + } + seen := make(map[string]struct{}, len(bundle.Snapshots)) + for _, snapshot := range bundle.Snapshots { + if !catalogIDPattern.MatchString(snapshot.SnapshotKey) || snapshot.SnapshotKey == request.BackupKey || + snapshot.Name == "" || len(snapshot.Name) > 64 || strings.ContainsRune(snapshot.Name, 0) || + (snapshot.Consistency != "filesystem" && snapshot.Consistency != "crash") || + snapshot.VirtualSizeBytes == 0 || + snapshot.VirtualSizeBytes > uint64(maxBackupObjectBytes) || + snapshot.CreatedAt < 1 { + return NewError(ErrInvalid, "invalid_migration_bundle", "migration snapshot metadata is invalid", nil) + } + if _, exists := seen[snapshot.SnapshotKey]; exists { + return NewError(ErrInvalid, "invalid_migration_bundle", "migration snapshot identities must be unique", nil) + } + seen[snapshot.SnapshotKey] = struct{}{} + if snapshot.SourceSizeBytes > maxBackupObjectBytes { + return NewError(ErrInvalid, "backup_object_too_large", "migration snapshot object is too large", nil) + } + if err := objecttransfer.ValidateDownload(snapshot.SourceURL, snapshot.SourceSizeBytes, snapshot.SourceSHA256); err != nil { + return NewError(ErrInvalid, "invalid_backup_object", "migration snapshot object identity is invalid", err) + } + } + } // Reuse the complete create request validation for resource, network, // hostname, policy, UUID, and derived-domain constraints. The placeholders diff --git a/agent/hypervisor/backup_test.go b/agent/hypervisor/backup_test.go index 5223099e..1f5d2e34 100644 --- a/agent/hypervisor/backup_test.go +++ b/agent/hypervisor/backup_test.go @@ -120,11 +120,12 @@ func TestValidateRestoreBackupReusesCreateIntentWithoutReplacingCredentials(t *t func TestBackupAdmissionsFenceSourceAndNewVMArtifacts(t *testing.T) { const uuid = "123e4567-e89b-42d3-a456-426614174000" - backup := CreateBackupAdmission(uuid, testBackupKey) + backupRequest := validBackupRequest() + backup := CreateBackupAdmission(uuid, backupRequest) for _, key := range []string{ domainLockKey(uuid), vmManagedVolumesLockKey(uuid), - volumeLockKey(managedStoragePoolName, snapshotVolumeName(uuid, testBackupKey)), - catalogLockKey("backup", testBackupKey), + volumeLockKey(managedStoragePoolName, snapshotVolumeName(uuid, backupRequest.BackupKey)), + catalogLockKey("backup", backupRequest.BackupKey), } { if !containsString(backup.LockKeys, key) || !backup.HeavyIO { t.Fatalf("backup admission missing %q: %#v", key, backup) @@ -142,3 +143,60 @@ func TestBackupAdmissionsFenceSourceAndNewVMArtifacts(t *testing.T) { } } } + +func TestMigrationBundleValidationAndAdmissionIncludeEverySnapshot(t *testing.T) { + const uuid = "123e4567-e89b-42d3-a456-426614174000" + request := validBackupRequest() + request.PrepareMaintenanceMigration = true + request.MigrationBundle = &MigrationBundleExport{ + Version: 1, + Snapshots: []MigrationBundleSnapshotUpload{{ + SnapshotKey: "fedcba9876543210fedcba9876543210", + ObjectUpload: request.ObjectUpload, + }}, + } + if err := ValidateBackupRequest(request); err != nil { + t.Fatalf("valid migration bundle rejected: %v", err) + } + admission := CreateBackupAdmission(uuid, request) + for _, key := range []string{ + volumeLockKey(managedStoragePoolName, snapshotVolumeName(uuid, request.BackupKey)), + volumeLockKey(managedStoragePoolName, snapshotVolumeName(uuid, request.MigrationBundle.Snapshots[0].SnapshotKey)), + } { + if !containsString(admission.LockKeys, key) { + t.Fatalf("migration bundle admission missing %q: %#v", key, admission) + } + } + + request.MigrationBundle.Snapshots = append(request.MigrationBundle.Snapshots, + request.MigrationBundle.Snapshots[0]) + if err := ValidateBackupRequest(request); err == nil { + t.Fatal("migration bundle accepted duplicate snapshot keys") + } +} + +func TestRestoreMigrationBundleRequiresStagedTargetAndImmutableSnapshotSources(t *testing.T) { + request := validRestoreBackupRequest(t) + request.MigrationBundle = &MigrationBundleRestore{ + Version: 1, + StagePoweredOff: true, + NetworkSuspended: true, + Snapshots: []MigrationBundleSnapshotSource{{ + SnapshotKey: "fedcba9876543210fedcba9876543210", + Name: "before-maintenance", + Consistency: "crash", + VirtualSizeBytes: 40 << 30, + SourceURL: "https://objects.example.test/snapshots/object?signature=secret", + SourceSizeBytes: 4 << 20, + SourceSHA256: strings.Repeat("c", 64), + CreatedAt: 1700000000, + }}, + } + if err := ValidateRestoreBackupRequest(request); err != nil { + t.Fatalf("valid migration bundle restore rejected: %v", err) + } + request.MigrationBundle.StagePoweredOff = false + if err := ValidateRestoreBackupRequest(request); err == nil { + t.Fatal("migration bundle restore accepted a target that could start before firewall convergence") + } +} diff --git a/agent/hypervisor/domain_xml.go b/agent/hypervisor/domain_xml.go index e64bdbe7..ee72f48a 100644 --- a/agent/hypervisor/domain_xml.go +++ b/agent/hypervisor/domain_xml.go @@ -168,6 +168,9 @@ func BuildDomainXML(definition DomainDefinition) (string, error) { if attachment.MAC != "" { iface.MAC = &libvirtxml.DomainInterfaceMAC{Address: strings.ToLower(attachment.MAC)} } + if attachment.LinkDown { + iface.Link = &libvirtxml.DomainInterfaceLink{State: "down"} + } if attachment.Bridge != "" { iface.Source.Bridge = &libvirtxml.DomainInterfaceSourceBridge{Bridge: attachment.Bridge} } else { @@ -300,6 +303,7 @@ func marshalDomainMetadata(definition DomainDefinition) (string, error) { DNSServers: append([]string(nil), definition.DNSServers...), NetworkSelector: definition.NetworkSelector, TemplateID: definition.TemplateID, RootVolumeName: definition.RootVolumeName, PlatformPolicyJSON: string(policyJSON), + Snapshots: append([]domainSnapshotMetadataXML(nil), definition.Snapshots...), }) if err != nil { return "", fmt.Errorf("marshal domain metadata: %w", err) diff --git a/agent/hypervisor/domain_xml_test.go b/agent/hypervisor/domain_xml_test.go index 4d0a6807..6eefa1d6 100644 --- a/agent/hypervisor/domain_xml_test.go +++ b/agent/hypervisor/domain_xml_test.go @@ -4,6 +4,7 @@ import ( "crypto/sha256" "encoding/hex" "encoding/json" + "encoding/xml" "errors" "strings" "testing" @@ -105,6 +106,32 @@ func TestBuildDomainXMLUsesQ35UEFIWithSecureBootDisabled(t *testing.T) { } } +func TestBuildDomainXMLCanStageMigrationOfflineWithSnapshotCatalog(t *testing.T) { + definition := validDomainDefinition() + definition.Network[0].LinkDown = true + definition.Snapshots = []domainSnapshotMetadataXML{{ + Key: "fedcba9876543210fedcba9876543210", Name: "before-maintenance", + Consistency: "crash", SizeBytes: 40 << 30, CreatedAt: 1700000000, + }} + document, err := BuildDomainXML(definition) + if err != nil { + t.Fatal(err) + } + var domain libvirtxml.Domain + if err := domain.Unmarshal(document); err != nil { + t.Fatal(err) + } + if len(domain.Devices.Interfaces) != 1 || domain.Devices.Interfaces[0].Link == nil || + domain.Devices.Interfaces[0].Link.State != "down" { + t.Fatalf("staged migration interface is not persistently down: %#v", domain.Devices.Interfaces) + } + var stored domainMetadataXML + if domain.Metadata == nil || xml.Unmarshal([]byte(domain.Metadata.XML), &stored) != nil || + len(stored.Snapshots) != 1 || stored.Snapshots[0].Key != definition.Snapshots[0].Key { + t.Fatalf("staged migration snapshot catalog missing: %#v", stored.Snapshots) + } +} + func TestBuildDomainXMLRejectsSecureBootAndUnsafePaths(t *testing.T) { request := validDomainDefinition() request.Firmware.SecureBoot = true diff --git a/agent/hypervisor/libvirt_adapter.go b/agent/hypervisor/libvirt_adapter.go index d907f7b8..400c78e4 100644 --- a/agent/hypervisor/libvirt_adapter.go +++ b/agent/hypervisor/libvirt_adapter.go @@ -270,6 +270,36 @@ func (l *Libvirt) CreateVM(ctx context.Context, request CreateVMRequest) (_ VMMe // fresh seed, MAC, public-IP filter, NVRAM, traffic baseline, and domain. func (l *Libvirt) createVMWithPreparedRoot(ctx context.Context, request CreateVMRequest, attachment NetworkAttachment, seed []byte, root *libvirt.StorageVol) (_ VMMetadata, resultErr error) { + return l.createVMWithPreparedRootOptions(ctx, request, attachment, seed, root, + preparedVMOptions{Autostart: true, Start: true}) +} + +type preparedVMOptions struct { + Autostart bool + Start bool + NetworkSuspended bool + Snapshots []domainSnapshotMetadataXML + SnapshotVolumes []*libvirt.StorageVol +} + +func (l *Libvirt) createVMWithPreparedRootOptions(ctx context.Context, + request CreateVMRequest, attachment NetworkAttachment, seed []byte, + root *libvirt.StorageVol, options preparedVMOptions) (_ VMMetadata, resultErr error) { + snapshotVolumesOwned := len(options.SnapshotVolumes) > 0 + defer func() { + for _, volume := range options.SnapshotVolumes { + if volume == nil { + continue + } + if snapshotVolumesOwned { + if cleanupErr := volume.Delete(libvirt.STORAGE_VOL_DELETE_NORMAL); cleanupErr != nil { + resultErr = operation.MarkUncertain(errors.Join(resultErr, + mutationLibvirtError("roll back restored migration snapshot", cleanupErr))) + } + } + _ = volume.Free() + } + }() rootPath, err := root.GetPath() if err != nil { cleanupErr := root.Delete(libvirt.STORAGE_VOL_DELETE_NORMAL) @@ -349,6 +379,7 @@ func (l *Libvirt) createVMWithPreparedRoot(ctx context.Context, request CreateVM } _ = filter.Free() }() + attachment.LinkDown = options.NetworkSuspended document, err := BuildDomainXML(DomainDefinition{ UUID: request.UUID, Name: request.DomainName, DisplayName: request.Name, Description: request.Description, @@ -365,6 +396,7 @@ func (l *Libvirt) createVMWithPreparedRoot(ctx context.Context, request CreateVM }, Network: []NetworkAttachment{attachment}, FilterName: filterName, PlatformNetworkPolicy: request.PlatformNetworkPolicy, + Snapshots: append([]domainSnapshotMetadataXML(nil), options.Snapshots...), }) if err != nil { return VMMetadata{}, createVMDomainDefineStage.wrap(err) @@ -391,6 +423,7 @@ func (l *Libvirt) createVMWithPreparedRoot(ctx context.Context, request CreateVM seedOwned = false filterOwned = false trafficOwned = false + snapshotVolumesOwned = false } domain, err := l.conn.DomainDefineXML(document) if err != nil { @@ -411,7 +444,7 @@ func (l *Libvirt) createVMWithPreparedRoot(ctx context.Context, request CreateVM return VMMetadata{}, createVMDomainDefineStage.wrap(mutationLibvirtError("define domain", err)) } defer domain.Free() - if err := domain.SetAutostart(true); err != nil { + if err := domain.SetAutostart(options.Autostart); err != nil { rollbackErr := domain.UndefineFlags(libvirt.DOMAIN_UNDEFINE_NVRAM) absent, inspectErr := l.deleteDomainAbsent(ctx, request.UUID) if inspectErr != nil || !absent { @@ -432,6 +465,25 @@ func (l *Libvirt) createVMWithPreparedRoot(ctx context.Context, request CreateVM mutationLibvirtError("enable domain autostart", err)) } preserveDefinedDomainArtifacts() + if !options.Start { + if err := l.verifyStagedMigrationDomain( + domain, request, filterName, attachment.MAC, + options.NetworkSuspended, options.Snapshots, + ); err != nil { + return VMMetadata{}, operation.MarkUncertain(err) + } + metadata, err := metadataFromXML(document) + if err != nil { + return VMMetadata{}, operation.MarkUncertain(err) + } + metadata.Status = VMStateShutoff + metadata.DiskGiB = request.DiskGiB + metadata.BandwidthMbps = request.BandwidthMbps + metadata.Hostname = request.Hostname + metadata.PublicIPCIDR = request.PublicIPCIDR + metadata.TemplateID = request.TemplateID + return metadata, nil + } // From this point onward Domain.Create may have taken effect even if its // response is lost. Retain the zero baseline so reconciliation and the // first successful sample account for all traffic since initial boot. @@ -739,6 +791,52 @@ func (l *Libvirt) verifyCreatedDomain(domain *libvirt.Domain, request CreateVMRe return l.verifyManagedFilter(filterName, storedPolicy) } +func (l *Libvirt) verifyStagedMigrationDomain(domain *libvirt.Domain, + request CreateVMRequest, filterName, mac string, networkSuspended bool, + snapshots []domainSnapshotMetadataXML) error { + autostart, err := domain.GetAutostart() + if err != nil || autostart { + return mutationLibvirtError("verify staged domain autostart", err) + } + active, err := domain.IsActive() + if err != nil || active { + return mutationLibvirtError("verify staged domain power state", err) + } + document, err := domain.GetXMLDesc(0) + if err != nil { + return translateLibvirtError("verify staged domain XML", err) + } + definition, metadata, err := parseManagedDomain(document) + if err != nil { + return err + } + if !strings.EqualFold(definition.UUID, request.UUID) || + definition.Name != request.DomainName || + metadata.DisplayName != request.Name || + metadata.PublicPoolKey != request.PublicPoolKey || + metadata.PublicIPCIDR != request.PublicIPCIDR || + definition.Devices == nil || len(definition.Devices.Interfaces) != 1 || + len(metadata.Snapshots) != len(snapshots) { + return NewError(ErrConflict, "migration_stage_observation_mismatch", + "staged migration domain identity or snapshot catalog did not match", nil) + } + for index := range snapshots { + if metadata.Snapshots[index] != snapshots[index] { + return NewError(ErrConflict, "migration_stage_observation_mismatch", + "staged migration snapshot catalog did not match", nil) + } + } + iface := definition.Devices.Interfaces[0] + linkDown := iface.Link != nil && iface.Link.State == "down" + if linkDown != networkSuspended || iface.MAC == nil || + !strings.EqualFold(iface.MAC.Address, mac) || + !managedInterfaceFilterIPMatches(&iface, filterName, request.PublicIPCIDR) { + return NewError(ErrConflict, "migration_stage_network_mismatch", + "staged migration network state did not match", nil) + } + return l.verifyManagedFilter(filterName, request.PlatformNetworkPolicy) +} + func (l *Libvirt) verifyManagedFilter(filterName string, policy PlatformNetworkPolicySnapshot) error { filter, err := l.conn.LookupNWFilterByName(filterName) if err != nil { diff --git a/agent/hypervisor/libvirt_backup.go b/agent/hypervisor/libvirt_backup.go index 0866b17c..a981cf5a 100644 --- a/agent/hypervisor/libvirt_backup.go +++ b/agent/hypervisor/libvirt_backup.go @@ -22,6 +22,13 @@ func (l *Libvirt) CreateBackup(ctx context.Context, uuid string, request BackupR if err := ValidateBackupRequest(request); err != nil { return BackupResult{}, operation.MarkNotApplied(err) } + if request.MigrationBundle != nil { + return l.createMigrationBundle(ctx, uuid, request) + } + return l.createPortableBackup(ctx, uuid, request) +} + +func (l *Libvirt) createPortableBackup(ctx context.Context, uuid string, request BackupRequest) (_ BackupResult, resultErr error) { client := objecttransfer.New() defer client.CloseIdleConnections() active, err := l.validateBackupSource(ctx, uuid, request.BackupKey) @@ -155,10 +162,10 @@ func (l *Libvirt) prepareMaintenanceMigrationSource(ctx context.Context, translateLibvirtError("read maintenance migration power state", err), ) } - if !autostart { + if !autostart && active { return maintenanceMigrationSourceState{}, operation.MarkNotApplied( - NewError(ErrConflict, "maintenance_source_autostart_required", - "maintenance migration requires a service-managed autostart domain", nil), + NewError(ErrConflict, "maintenance_source_service_state_invalid", + "maintenance migration source cannot be running while service autostart is disabled", nil), ) } state := maintenanceMigrationSourceState{wasRunning: active, autostart: autostart} @@ -542,6 +549,9 @@ func (l *Libvirt) RestoreBackup(ctx context.Context, request RestoreBackupReques if err != nil { return RestoreBackupResult{}, err } + if request.MigrationBundle != nil { + return l.restoreMigrationBundle(ctx, request, create, attachment, seed, root) + } metadata, err := l.createVMWithPreparedRoot(ctx, create, attachment, seed, root) if err != nil { return RestoreBackupResult{}, err diff --git a/agent/hypervisor/libvirt_migration_bundle.go b/agent/hypervisor/libvirt_migration_bundle.go new file mode 100644 index 00000000..232d900e --- /dev/null +++ b/agent/hypervisor/libvirt_migration_bundle.go @@ -0,0 +1,399 @@ +//go:build libvirt + +package hypervisor + +import ( + "context" + "errors" + "fmt" + "path/filepath" + "strings" + + "git.luoli.live/root/virt/agent/objecttransfer" + "git.luoli.live/root/virt/agent/operation" + "libvirt.org/go/libvirt" + "libvirt.org/go/libvirtxml" +) + +// createMigrationBundle exports the immutable local snapshot set first and +// stops the source only for the final system-disk capture. A partially +// completed multi-object upload is deliberately fenced as uncertain: the +// Agent has no object-store delete credential and must never pretend the +// already committed artifacts do not exist. +func (l *Libvirt) createMigrationBundle(ctx context.Context, uuid string, + request BackupRequest) (_ BackupResult, resultErr error) { + client := objecttransfer.New() + defer client.CloseIdleConnections() + + stored, err := l.validateMigrationBundleSource(ctx, uuid, request) + if err != nil { + return BackupResult{}, l.abortRejectedMigrationBundlePlans( + ctx, client, request, err, + ) + } + artifacts := make([]MigrationBundleArtifactResult, 0, len(stored)) + for index, metadata := range stored { + volume, lookupErr := l.lookupOptionalManagedVolume( + l.storagePoolName(StorageRoleManaged), + snapshotVolumeName(uuid, metadata.Key), + ) + if lookupErr != nil || volume == nil { + return BackupResult{}, l.migrationBundleExportFailure( + ctx, client, request, index, len(artifacts) > 0, + NewError(ErrUnavailable, "migration_snapshot_unavailable", + "migration snapshot volume is unavailable", lookupErr), + ) + } + info, inspectErr := validateStandaloneQCOW2Volume(volume, metadata.SizeBytes) + path, pathErr := volume.GetPath() + _ = volume.Free() + if inspectErr != nil || pathErr != nil || !filepath.IsAbs(path) { + cause := inspectErr + if cause == nil { + cause = translateOptionalLibvirtError( + "resolve migration snapshot path", pathErr, + ) + } + return BackupResult{}, l.migrationBundleExportFailure( + ctx, client, request, index, len(artifacts) > 0, cause, + ) + } + upload, uploadErr := client.UploadFile( + ctx, path, request.MigrationBundle.Snapshots[index].ObjectUpload, + ) + if uploadErr != nil { + return BackupResult{}, l.migrationBundleExportFailure( + ctx, client, request, index+1, len(artifacts) > 0, + translateObjectTransferError(uploadErr), + ) + } + if upload.SizeBytes <= 0 || upload.SHA256 == "" { + return BackupResult{}, operation.MarkUncertain(NewError( + ErrUnavailable, "migration_snapshot_upload_invalid", + "migration snapshot upload result is invalid", nil, + )) + } + artifacts = append(artifacts, MigrationBundleArtifactResult{ + SnapshotKey: metadata.Key, Name: metadata.Name, + Consistency: metadata.Consistency, SizeBytes: upload.SizeBytes, + VirtualSizeBytes: info.Capacity, SHA256: upload.SHA256, + CreatedAt: metadata.CreatedAt, + }) + } + + maintenanceState, err := l.prepareMaintenanceMigrationSource(ctx, uuid) + if err != nil { + return BackupResult{}, operation.MarkUncertain(err) + } + defer func() { + if resultErr == nil { + return + } + if rollbackErr := l.restoreMaintenanceMigrationSource( + context.WithoutCancel(ctx), uuid, maintenanceState, + ); rollbackErr != nil { + resultErr = operation.MarkUncertain(errors.Join(resultErr, rollbackErr)) + } + }() + + snapshot, err := l.createStoppedBackupSnapshot(ctx, uuid, SnapshotRequest{ + OperationID: request.OperationID, SnapshotKey: request.BackupKey, + Name: request.Name, ConsistencyMode: request.ConsistencyMode, + }) + if err != nil { + return BackupResult{}, operation.MarkUncertain(err) + } + volume, err := l.lookupOptionalManagedVolume( + l.storagePoolName(StorageRoleManaged), snapshotVolumeName(uuid, request.BackupKey), + ) + if err != nil || volume == nil { + return BackupResult{}, operation.MarkUncertain(NewError( + ErrUnavailable, "backup_snapshot_unavailable", + "local migration root snapshot is unavailable", err, + )) + } + info, inspectErr := validateStandaloneQCOW2Volume(volume, snapshot.SizeBytes) + path, pathErr := volume.GetPath() + _ = volume.Free() + if inspectErr != nil || pathErr != nil || !filepath.IsAbs(path) { + cause := inspectErr + if cause == nil { + cause = translateOptionalLibvirtError( + "resolve migration root snapshot path", pathErr, + ) + } + return BackupResult{}, operation.MarkUncertain(cause) + } + upload, err := client.UploadFile(ctx, path, request.ObjectUpload) + if err != nil { + return BackupResult{}, operation.MarkUncertain( + translateObjectTransferError(err), + ) + } + if upload.SizeBytes <= 0 || upload.SHA256 == "" { + return BackupResult{}, operation.MarkUncertain(NewError( + ErrUnavailable, "backup_upload_invalid", + "migration root upload result is invalid", nil, + )) + } + result := BackupResult{ + BackupKey: request.BackupKey, SizeBytes: upload.SizeBytes, + VirtualSizeBytes: info.Capacity, SHA256: upload.SHA256, + Consistency: snapshot.Consistency, SourceStopped: true, + SourceWasRunning: maintenanceState.wasRunning, + SourceAutostartEnabled: maintenanceState.autostart, + SourceAutostartDisabled: true, MigrationBundleVersion: 1, + SnapshotArtifacts: artifacts, + } + cleanupCtx, cleanupCancel := backupLocalCleanupContext(ctx) + _, cleanupErr := l.DeleteSnapshot(cleanupCtx, uuid, request.BackupKey, + SnapshotMutationRequest{OperationID: request.OperationID}) + cleanupCancel() + if cleanupErr != nil { + result.CleanupPending = true + } + return result, nil +} + +func (l *Libvirt) validateMigrationBundleSource(ctx context.Context, uuid string, + request BackupRequest) ([]domainSnapshotMetadataXML, error) { + if _, err := l.validateBackupSource(ctx, uuid, request.BackupKey); err != nil { + return nil, err + } + domain, err := l.domainByUUID(ctx, uuid) + if err != nil { + return nil, err + } + defer domain.Free() + document, err := domain.GetXMLDesc(0) + if err != nil { + return nil, translateLibvirtError("read migration source metadata", err) + } + _, metadata, err := parseManagedDomain(document) + if err != nil { + return nil, err + } + if len(metadata.Snapshots) != len(request.MigrationBundle.Snapshots) { + return nil, NewError(ErrConflict, "migration_snapshot_catalog_changed", + "migration bundle must include every current snapshot", nil) + } + byKey := make(map[string]domainSnapshotMetadataXML, len(metadata.Snapshots)) + for _, snapshot := range metadata.Snapshots { + byKey[snapshot.Key] = snapshot + } + ordered := make([]domainSnapshotMetadataXML, 0, len(metadata.Snapshots)) + for _, requested := range request.MigrationBundle.Snapshots { + snapshot, ok := byKey[requested.SnapshotKey] + if !ok { + return nil, NewError(ErrConflict, "migration_snapshot_catalog_changed", + "migration snapshot catalog changed before export", nil) + } + ordered = append(ordered, snapshot) + } + return ordered, nil +} + +func (l *Libvirt) abortRejectedMigrationBundlePlans(ctx context.Context, + client *objecttransfer.Client, request BackupRequest, rejection error) error { + errs := []error{rejection} + if err := client.AbortMultipart(ctx, request.ObjectUpload); err != nil { + errs = append(errs, err) + } + for _, snapshot := range request.MigrationBundle.Snapshots { + if err := client.AbortMultipart(ctx, snapshot.ObjectUpload); err != nil { + errs = append(errs, err) + } + } + return operation.MarkNotApplied(errors.Join(errs...)) +} + +// migrationBundleExportFailure aborts plans that have not started. Once an +// artifact completed, the overall multi-object effect can no longer be called +// not-applied even when a later upload fails deterministically. +func (l *Libvirt) migrationBundleExportFailure(ctx context.Context, + client *objecttransfer.Client, request BackupRequest, nextSnapshot int, + completed bool, cause error) error { + errs := []error{cause} + if err := client.AbortMultipart(ctx, request.ObjectUpload); err != nil { + errs = append(errs, err) + } + for index := nextSnapshot; index < len(request.MigrationBundle.Snapshots); index++ { + if err := client.AbortMultipart( + ctx, request.MigrationBundle.Snapshots[index].ObjectUpload, + ); err != nil { + errs = append(errs, err) + } + } + joined := errors.Join(errs...) + if completed || len(errs) > 1 || operation.IsUncertain(cause) { + return operation.MarkUncertain(joined) + } + return operation.MarkNotApplied(joined) +} + +func (l *Libvirt) restoreMigrationBundle(ctx context.Context, + request RestoreBackupRequest, create CreateVMRequest, + attachment NetworkAttachment, seed []byte, + root *libvirt.StorageVol) (_ RestoreBackupResult, resultErr error) { + volumes := make([]*libvirt.StorageVol, 0, + len(request.MigrationBundle.Snapshots)) + metadata := make([]domainSnapshotMetadataXML, 0, + len(request.MigrationBundle.Snapshots)) + artifacts := make([]MigrationBundleArtifactResult, 0, + len(request.MigrationBundle.Snapshots)) + for _, source := range request.MigrationBundle.Snapshots { + volume, err := l.createMigrationSnapshotVolume(ctx, request.UUID, source) + if err != nil { + for _, prior := range volumes { + if cleanupErr := prior.Delete(libvirt.STORAGE_VOL_DELETE_NORMAL); cleanupErr != nil { + resultErr = operation.MarkUncertain(errors.Join( + err, mutationLibvirtError( + "roll back restored migration snapshot", cleanupErr, + ), + )) + } + _ = prior.Free() + } + if cleanupErr := root.Delete(libvirt.STORAGE_VOL_DELETE_NORMAL); cleanupErr != nil { + resultErr = operation.MarkUncertain(errors.Join( + resultErr, err, mutationLibvirtError( + "roll back restored migration root", cleanupErr, + ), + )) + } + _ = root.Free() + if resultErr != nil { + return RestoreBackupResult{}, resultErr + } + return RestoreBackupResult{}, err + } + volumes = append(volumes, volume) + metadata = append(metadata, domainSnapshotMetadataXML{ + Key: source.SnapshotKey, Name: source.Name, + Consistency: source.Consistency, SizeBytes: source.VirtualSizeBytes, + CreatedAt: source.CreatedAt, + }) + artifacts = append(artifacts, MigrationBundleArtifactResult{ + SnapshotKey: source.SnapshotKey, Name: source.Name, + Consistency: source.Consistency, SizeBytes: source.SourceSizeBytes, + VirtualSizeBytes: source.VirtualSizeBytes, + SHA256: source.SourceSHA256, CreatedAt: source.CreatedAt, + }) + } + vm, err := l.createVMWithPreparedRootOptions( + ctx, create, attachment, seed, root, preparedVMOptions{ + Autostart: false, Start: false, + NetworkSuspended: request.MigrationBundle.NetworkSuspended, + Snapshots: metadata, SnapshotVolumes: volumes, + }, + ) + if err != nil { + return RestoreBackupResult{}, err + } + return RestoreBackupResult{ + VMMetadata: vm, BackupKey: request.BackupKey, + SHA256: request.SourceSHA256, MigrationBundleVersion: 1, + SnapshotArtifacts: artifacts, StagedPoweredOff: true, + }, nil +} + +func (l *Libvirt) createMigrationSnapshotVolume(ctx context.Context, + uuid string, source MigrationBundleSnapshotSource) (*libvirt.StorageVol, error) { + volumeName := snapshotVolumeName(uuid, source.SnapshotKey) + initialCapacity := source.VirtualSizeBytes + if objectBytes := uint64(source.SourceSizeBytes); objectBytes > initialCapacity { + initialCapacity = objectBytes + } + pool, err := l.conn.LookupStoragePoolByName( + l.storagePoolName(StorageRoleManaged), + ) + if err != nil { + return nil, createVMStorageCloneStage.wrap( + translateLibvirtError("lookup managed storage pool", err), + ) + } + defer pool.Free() + definition := libvirtxml.StorageVolume{ + Name: volumeName, + Capacity: &libvirtxml.StorageVolumeSize{ + Value: initialCapacity, Unit: "bytes", + }, + Target: &libvirtxml.StorageVolumeTarget{ + Format: &libvirtxml.StorageVolumeTargetFormat{Type: "qcow2"}, + }, + } + document, err := definition.Marshal() + if err != nil { + return nil, createVMStorageCloneStage.wrap( + fmt.Errorf("marshal restored migration snapshot XML: %w", err), + ) + } + volume, err := pool.StorageVolCreateXML(document, 0) + if err != nil { + absent, inspectErr := l.dirPoolVolumeAbsent(pool, volumeName) + return nil, createVMStorageCloneStage.wrap( + deterministicArtifactCreateFailure( + "create restored migration snapshot", err, inspectErr, !absent, + ), + ) + } + stream, err := l.conn.NewStream(libvirt.STREAM_NONBLOCK) + if err != nil { + return nil, createVMStorageCloneStage.wrap(l.rollbackUnpublishedVolume( + pool, volume, volumeName, err, + "roll back restored migration snapshot", + )) + } + transfer := &storageStreamTransfer{stream: stream} + if err := volume.Upload(stream, 0, uint64(source.SourceSizeBytes), 0); err != nil { + cause := transfer.abort(err) + return nil, createVMStorageCloneStage.wrap(l.rollbackRestoreStreamFailure( + pool, volume, volumeName, cause, + "roll back restored migration snapshot", + )) + } + client := objecttransfer.New() + defer client.CloseIdleConnections() + download, downloadErr := client.Download( + ctx, source.SourceURL, source.SourceSizeBytes, source.SourceSHA256, + &storageStreamWriter{ctx: ctx, transfer: transfer}, + ) + if downloadErr != nil { + cause := transfer.abort(translateObjectTransferError(downloadErr)) + return nil, createVMStorageCloneStage.wrap(l.rollbackRestoreStreamFailure( + pool, volume, volumeName, cause, + "roll back restored migration snapshot", + )) + } + if err := transfer.finish(ctx); err != nil { + return nil, createVMStorageCloneStage.wrap(l.rollbackRestoreStreamFailure( + pool, volume, volumeName, err, + "roll back restored migration snapshot", + )) + } + if download.SizeBytes != source.SourceSizeBytes || + !strings.EqualFold(download.SHA256, source.SourceSHA256) { + return nil, createVMStorageCloneStage.wrap(l.rollbackUnpublishedVolume( + pool, volume, volumeName, + NewError(ErrConflict, "backup_object_mismatch", + "downloaded migration snapshot identity does not match", nil), + "roll back restored migration snapshot", + )) + } + if err := pool.Refresh(0); err != nil { + return nil, createVMStorageCloneStage.wrap(l.rollbackUnpublishedVolume( + pool, volume, volumeName, + translateLibvirtError("refresh restored migration snapshot", err), + "roll back restored migration snapshot", + )) + } + if _, err := validateStandaloneQCOW2Volume( + volume, source.VirtualSizeBytes, + ); err != nil { + return nil, createVMStorageCloneStage.wrap(l.rollbackUnpublishedVolume( + pool, volume, volumeName, err, + "roll back restored migration snapshot", + )) + } + return volume, nil +} diff --git a/agent/hypervisor/libvirt_operation_admin.go b/agent/hypervisor/libvirt_operation_admin.go index 37e48d5b..648b1f83 100644 --- a/agent/hypervisor/libvirt_operation_admin.go +++ b/agent/hypervisor/libvirt_operation_admin.go @@ -1220,12 +1220,17 @@ func operationEffectProjection(record operation.Record, raw []byte) json.RawMess } projection = struct { operationVMEffectProjection - BackupKey string `json:"backup_key"` - SHA256 string `json:"sha256"` + BackupKey string `json:"backup_key"` + SHA256 string `json:"sha256"` + MigrationBundleVersion uint64 `json:"migration_bundle_version,omitempty"` + SnapshotArtifacts []MigrationBundleArtifactResult `json:"snapshot_artifacts,omitempty"` + StagedPoweredOff bool `json:"staged_powered_off,omitempty"` }{ operationVMEffectProjection: vmEffectProjection(value.VMMetadata), - BackupKey: value.BackupKey, - SHA256: value.SHA256, + BackupKey: value.BackupKey, SHA256: value.SHA256, + MigrationBundleVersion: value.MigrationBundleVersion, + SnapshotArtifacts: value.SnapshotArtifacts, + StagedPoweredOff: value.StagedPoweredOff, } case operation.IsCreateLikeAction(record.Action) || record.Action == "update-settings" || record.Action == "change-network" || record.Action == "reinstall-vm": diff --git a/agent/hypervisor/libvirt_safety_test.go b/agent/hypervisor/libvirt_safety_test.go index 93d11f53..a76fbb84 100644 --- a/agent/hypervisor/libvirt_safety_test.go +++ b/agent/hypervisor/libvirt_safety_test.go @@ -95,6 +95,13 @@ func TestMaintenanceMigrationRollbackRestoresSourceState(t *testing.T) { adapter := &Libvirt{} domain := &fakeServiceStateDomainControl{active: false} err := adapter.restoreMaintenanceMigrationDomain(domain, + maintenanceMigrationSourceState{wasRunning: false, autostart: false}) + if err != nil || domain.setValue || domain.createCalled { + t.Fatalf("expected powered-off unmanaged source state restoration, err=%v state=%+v", err, domain) + } + + domain = &fakeServiceStateDomainControl{active: false} + err = adapter.restoreMaintenanceMigrationDomain(domain, maintenanceMigrationSourceState{wasRunning: true, autostart: true}) if err != nil || !domain.setValue || !domain.createCalled { t.Fatalf("expected source state restoration, err=%v state=%+v", err, domain) diff --git a/agent/hypervisor/types.go b/agent/hypervisor/types.go index 81c5dfd9..1edfe374 100644 --- a/agent/hypervisor/types.go +++ b/agent/hypervisor/types.go @@ -166,9 +166,10 @@ type Firmware struct { } type NetworkAttachment struct { - Bridge string `json:"bridge,omitempty"` - Network string `json:"network,omitempty"` - MAC string `json:"mac,omitempty"` + Bridge string `json:"bridge,omitempty"` + Network string `json:"network,omitempty"` + MAC string `json:"mac,omitempty"` + LinkDown bool `json:"-"` } type CreateVMRequest struct { @@ -210,18 +211,44 @@ type BackupRequest struct { ConsistencyMode string `json:"consistency_mode"` PrepareMaintenanceMigration bool `json:"prepare_maintenance_migration,omitempty"` ObjectUpload objecttransfer.MultipartPlan `json:"object_upload"` + MigrationBundle *MigrationBundleExport `json:"migration_bundle,omitempty"` +} + +// MigrationBundleExport is present only for a host-maintenance move. Ordinary +// portable backups deliberately remain one system-disk object. +type MigrationBundleExport struct { + Version uint64 `json:"version"` + Snapshots []MigrationBundleSnapshotUpload `json:"snapshots"` +} + +type MigrationBundleSnapshotUpload struct { + SnapshotKey string `json:"snapshot_key"` + ObjectUpload objecttransfer.MultipartPlan `json:"object_upload"` +} + +type MigrationBundleArtifactResult struct { + SnapshotKey string `json:"snapshot_key"` + Name string `json:"name"` + Consistency string `json:"consistency"` + SizeBytes int64 `json:"size_bytes"` + VirtualSizeBytes uint64 `json:"virtual_size_bytes"` + SHA256 string `json:"sha256"` + CreatedAt int64 `json:"created_at"` } type BackupResult struct { - BackupKey string `json:"backup_key"` - SizeBytes int64 `json:"size_bytes"` - VirtualSizeBytes uint64 `json:"virtual_size_bytes"` - SHA256 string `json:"sha256"` - Consistency string `json:"consistency"` - CleanupPending bool `json:"cleanup_pending,omitempty"` - SourceStopped bool `json:"source_stopped,omitempty"` - SourceWasRunning bool `json:"source_was_running"` - SourceAutostartDisabled bool `json:"source_autostart_disabled,omitempty"` + BackupKey string `json:"backup_key"` + SizeBytes int64 `json:"size_bytes"` + VirtualSizeBytes uint64 `json:"virtual_size_bytes"` + SHA256 string `json:"sha256"` + Consistency string `json:"consistency"` + CleanupPending bool `json:"cleanup_pending,omitempty"` + SourceStopped bool `json:"source_stopped,omitempty"` + SourceWasRunning bool `json:"source_was_running"` + SourceAutostartEnabled bool `json:"source_autostart_enabled"` + SourceAutostartDisabled bool `json:"source_autostart_disabled,omitempty"` + MigrationBundleVersion uint64 `json:"migration_bundle_version,omitempty"` + SnapshotArtifacts []MigrationBundleArtifactResult `json:"snapshot_artifacts,omitempty"` } // RestoreBackupRequest is deliberately flat on the wire so it matches the @@ -239,13 +266,35 @@ type RestoreBackupRequest struct { // Debian12NetworkFix is mandatory backup-time template metadata. A pointer // distinguishes an explicit false value from an omitted, untrusted legacy // request without exposing the hint in the public restore result. - Debian12NetworkFix *bool `json:"debian_12_network_fix"` + Debian12NetworkFix *bool `json:"debian_12_network_fix"` + MigrationBundle *MigrationBundleRestore `json:"migration_bundle,omitempty"` +} + +type MigrationBundleRestore struct { + Version uint64 `json:"version"` + StagePoweredOff bool `json:"stage_powered_off"` + NetworkSuspended bool `json:"network_suspended"` + Snapshots []MigrationBundleSnapshotSource `json:"snapshots"` +} + +type MigrationBundleSnapshotSource struct { + SnapshotKey string `json:"snapshot_key"` + Name string `json:"name"` + Consistency string `json:"consistency"` + VirtualSizeBytes uint64 `json:"virtual_size_bytes"` + SourceURL string `json:"source_url"` + SourceSizeBytes int64 `json:"source_size_bytes"` + SourceSHA256 string `json:"source_sha256"` + CreatedAt int64 `json:"created_at"` } type RestoreBackupResult struct { VMMetadata - BackupKey string `json:"backup_key"` - SHA256 string `json:"sha256"` + BackupKey string `json:"backup_key"` + SHA256 string `json:"sha256"` + MigrationBundleVersion uint64 `json:"migration_bundle_version,omitempty"` + SnapshotArtifacts []MigrationBundleArtifactResult `json:"snapshot_artifacts,omitempty"` + StagedPoweredOff bool `json:"staged_powered_off,omitempty"` } // DomainDefinition is resolved entirely by the Agent. No host path in this @@ -275,6 +324,7 @@ type DomainDefinition struct { Network []NetworkAttachment FilterName string PlatformNetworkPolicy PlatformNetworkPolicySnapshot + Snapshots []domainSnapshotMetadataXML } type LocalConfig struct { diff --git a/agent/operation/recovery_contract.json b/agent/operation/recovery_contract.json index 9499fa0f..3f09a712 100644 --- a/agent/operation/recovery_contract.json +++ b/agent/operation/recovery_contract.json @@ -99,6 +99,19 @@ "not_applied" ] }, + { + "kind": "instance_migration_activate", + "recovery_mode": "generic", + "identity_scope": "instance", + "actions": [ + "service-resume" + ], + "action_patterns": [], + "system_resolutions": [ + "applied", + "not_applied" + ] + }, { "kind": "expiry_delete", "recovery_mode": "generic", diff --git a/docs/AGENT_API.md b/docs/AGENT_API.md index afcb8d31..846790b0 100644 --- a/docs/AGENT_API.md +++ b/docs/AGENT_API.md @@ -215,6 +215,14 @@ cleanup 的 mutation endpoint。 提供执行它的 mutation endpoint。 action 为 `create-backup:{backup_key}:{consistency}`。 +维护迁移在同一 endpoint 上额外携带 +`prepare_maintenance_migration=true` 与版本化 `migration_bundle`。bundle +必须精确列出 Domain metadata 中的全部当前快照,每个快照都有独立 multipart +能力;普通备份不得携带该字段。Agent 对每个快照键取得资源锁,先上传这些不可变 +卷,再停止源 Domain 并上传系统盘。成功结果同时返回源电源/autostart 状态和每个 +快照的键、名称、一致性、实际对象大小、虚拟容量、SHA-256 与 Agent 创建时间。 +多对象过程中只要已有对象提交,后续失败就保持 effect fence,不能伪装为整体未执行。 + `POST /v1/vms/{new_uuid}/restore-backup` 使用与新建 VM 相同的 CPU、内存、系统盘、 带宽、主机名及完整公网意图,并额外携带 `backup_key`、一次性 `source_url`、 `source_size_bytes`、`source_sha256`。Agent 严格核对实际字节数、SHA-256、独立 @@ -226,6 +234,13 @@ qcow2 格式和虚拟容量,并要求备份时冻结的严格布尔字段 受管创建工件均确认不存在后提交 `not_applied`,不能根据当前状态推断并提交 `applied`。备份和恢复均为 HeavyIO,helper 上限为 24 小时。 +维护迁移恢复额外携带同版本 `migration_bundle`、全部快照只读下载能力、 +`stage_powered_off=true` 和原网络限制状态。Agent 为每个快照创建独立 qcow2 卷, +重建 Domain 快照 metadata,并只定义 autostart=false 的关机目标;网络受限时持久 +interface XML 同时写入 `link state=down`。成功回执必须证明目标仍关机且每个快照 +对象身份完全匹配。PHP 在该回执之后才应用用户防火墙,并通过单独的 +`service/resume` operation 恢复原运行/关机状态,恢复请求本身绝不提前启动目标。 + ## 关键资源语义 - VM 内部 libvirt 名称由 actor user ID 和完整 UUID 派生;面板名称只写受控 metadata。客户端不能指定 Domain 名或提交 XML。 diff --git a/docs/PORTABLE_VM_BACKUPS.md b/docs/PORTABLE_VM_BACKUPS.md index 3976b5f7..6a5d77b3 100644 --- a/docs/PORTABLE_VM_BACKUPS.md +++ b/docs/PORTABLE_VM_BACKUPS.md @@ -26,9 +26,13 @@ ## 维护迁移 -维护迁移是独立恢复之外的管理员流程,只允许选择同一资源池内的其他宿主机。控制面先锁定目标容量和新公网 IP,再正常关闭源实例并关闭其 autostart,从稳定系统盘生成备份;目标宿主机恢复并验证新 UUID、新公网 IP、系统盘摘要和资源预留后,才切换原 `cloud_instances` 记录。迁移不会创建第二份账单实例,也不会改动原订单、套餐、服务期、到期时间、续费参数或当前流量周期。 +维护迁移是独立恢复之外的管理员流程,只允许选择同一资源池内的其他宿主机。它使用独立的 `MigrationBundle` 编排,不改变普通云备份的单系统盘格式:控制面先冻结当前全部已就绪快照目录,为系统盘和每个快照分别生成唯一对象键,并按系统盘加全部快照容量锁定目标资源和新公网 IP。Agent 先上传不可变快照卷,再正常关闭源实例、关闭其 autostart 并从稳定系统盘生成根对象;任何未就绪、目录漂移或回执不完整都会阻止切换。 -目标使用统一的 `bridge:br0` 网络和新公网 IPv4。原累计流量作为新 Agent 计数器世代的基数继续单调累计,不能因宿主机和 UUID 改变而清零或重复计费。若实例启用了用户防火墙,目标必须重新应用并确认同一策略后,流程才允许进入源副本清理。 +目标 Agent 下载系统盘与全部快照卷,重建快照元数据后只定义关机且禁用 autostart 的 Domain,不会在恢复请求中启动。控制面确认新 UUID、新公网 IP、系统盘摘要、每个快照回执和资源预留后,才切换原 `cloud_instances` 记录及快照宿主机归属。迁移不会创建第二份账单实例,也不会改动原订单、套餐、服务期、到期时间、续费参数或当前流量周期。 + +目标使用统一的 `bridge:br0` 网络和新公网 IPv4。原累计流量作为新 Agent 计数器世代的基数继续单调累计,不能因宿主机和 UUID 改变而清零或重复计费。修改过的来宾主机名从备份时冻结的 `guest_hostname` 生成新 seed,不回退为实例显示名。目标保持源实例的服务状态、网络状态和电源状态;源网络已因流量或到期被限制时,目标持久网卡从定义开始就是 `link down`。 + +目标在整个恢复和防火墙阶段保持关机。若实例启用了用户防火墙,控制面必须先在关机目标上重新应用并确认同一策略;没有策略或策略已禁用时也必须完成恢复终结。只有这些步骤完成后,独立的目标激活任务才恢复原 autostart 和运行/关机状态,随后才允许进入源副本清理。因此目标不会在用户防火墙生效前短暂联网启动。 管理员可在开始时选择两种源副本策略: @@ -36,7 +40,7 @@ - “保留源副本”会保持源 Domain 关机且禁用 autostart,并继续锁定原公网 IP 和源容量;管理员稍后可从同一页面显式发起清理。 - 在源副本清理或异常核对完成前,面板会阻止普通实例变更和套餐升级;到期策略仍可断网,但不会只删除目标实例而遗留源副本。 -任何删除响应超时、断连或回执丢失都进入待核对状态,系统不会重放源删除,也不会提前释放源资源。迁移备份与普通备份共用对象存储目录,但迁移目标恢复属于隐藏的编排步骤,不会生成一个免计费的独立业务实例。 +任何删除响应超时、断连或回执丢失都进入待核对状态,系统不会重放源删除,也不会提前释放源资源。迁移对象使用独立的 `vm-migrations/{bundle_key}/` 前缀,根对象为 `system.qcow2`,快照对象以快照键命名;普通云备份继续使用 `vm-backups/` 且不携带快照。迁移目录和目标恢复属于隐藏的编排步骤,不会生成一个免计费的独立业务实例。 ## 失败与不确定效果 diff --git a/docs/TASK_EXECUTION.md b/docs/TASK_EXECUTION.md index 9f5fd281..b3d3844c 100644 --- a/docs/TASK_EXECUTION.md +++ b/docs/TASK_EXECUTION.md @@ -30,7 +30,7 @@ accepted ──单次领取──> dispatching ──Agent 已接收──> awai 面板跨过本地派发边界,并不证明 HTTP 已送达 Gateway。对于固定回环地址,PHP 仅在 libcurl 明确连接失败或连接阶段超时,且完整统计证明连接未建立、HTTP 请求和上传/下载字节均为零、没有响应或重定向时,将该次失败记为 `not_applied`;如果存在多个底层传输尝试,每次都必须具有未发送证明。创建任务通过既有原子终结器退款并释放预留,其他类型通过各自终结器收尾,原任务均不重发。请求已发送后的超时、空响应、缺失统计和 Agent 查不到 operation 均不等于未执行,仍保留未知影响。该证明无法追溯补回历史任务未保存的传输证据。 -任务一旦终态即不可变。晚到的宿主机回执不能把 `failed` 改成 `succeeded`,派发账本异常也不能把既有 `succeeded` 改成 `failed`;两者都不能重写原任务的错误、结果、终态时间或完成事件编号。恢复异常通过独立的 `finalization_incident`、效果结论和资源锁表达,页面不能仅因原终态为成功就显示普通成功。只有列入种类策略表、且具备幂等业务账本终结器的精确同源回执,才可以在不重放 mutation 的前提下收敛其副作用账本。策略表覆盖实例电源、重装、救援、更换网络、实例设置、实例升级、快照、防火墙、删除/强制删除、生命周期网络控制,以及模板、救援模板和用户 ISO 的 Agent 叶子任务;聚合父任务只根据子任务重新汇总。精确成功回执收敛为 `applied`,明确 `effect_may_have_applied=false` 的精确失败回执收敛为 `not_applied`。两者都不会重放,且原任务终态保持不变。 +任务一旦终态即不可变。晚到的宿主机回执不能把 `failed` 改成 `succeeded`,派发账本异常也不能把既有 `succeeded` 改成 `failed`;两者都不能重写原任务的错误、结果、终态时间或完成事件编号。恢复异常通过独立的 `finalization_incident`、效果结论和资源锁表达,页面不能仅因原终态为成功就显示普通成功。只有列入种类策略表、且具备幂等业务账本终结器的精确同源回执,才可以在不重放 mutation 的前提下收敛其副作用账本。策略表覆盖实例电源、重装、救援、更换网络、实例设置、实例升级、快照、备份、防火墙、维护迁移目标状态恢复、删除/强制删除、生命周期网络控制,以及模板、救援模板和用户 ISO 的 Agent 叶子任务;聚合父任务只根据子任务重新汇总。精确成功回执收敛为 `applied`,明确 `effect_may_have_applied=false` 的精确失败回执收敛为 `not_applied`。两者都不会重放,且原任务终态保持不变。 ## Agent 执行边界 diff --git a/docs/VALIDATION.md b/docs/VALIDATION.md index ce85b938..91873092 100644 --- a/docs/VALIDATION.md +++ b/docs/VALIDATION.md @@ -130,6 +130,7 @@ jq empty deploy/*.json.example - 对象存储保存后必须处于未启用状态;只有当前配置版本的签名 `HeadBucket` 成功后才能就绪。修改或停用时旧验证立即失效;失败响应只显示稳定的本地错误,不泄露 Endpoint 响应正文、凭据或服务器路径。 - 使用受控的 S3/MinIO 测试 Bucket 分别验收路径寻址和虚拟主机寻址、错误 Region、错误凭据、缺少 `s3:ListBucket`、无效证书和连接超时;验证过程中 Bucket 对象数与内容不得变化。 - 可移植备份必须拒绝额外可写盘、backing chain、LUN 和活动 pivot/mirror;运行实例分别验证文件系统一致性与 Guest Agent 不可用时的崩溃一致性路径,关机实例验证稳定完整副本。 +- 维护迁移 `MigrationBundle` 必须验证当前全部快照都生成独立对象、任一快照缺失或回执漂移即拒绝切换、目标恢复后快照目录整体改绑。目标在恢复和用户防火墙收敛期间必须保持关机;源机的运行/关机、autostart、断网状态与修改后主机名必须在独立激活步骤后保持一致。普通云备份仍只能产生单系统盘对象。 - 恢复必须选择不同宿主机,目标使用 `bridge:br0` 和全新预留公网 IP;校验下载对象字节数与 SHA-256,新实例 UUID、MAC、seed、NVRAM、网络过滤器和流量状态均不得复用源实例,且不得继承订单、账单、服务期、到期时间或流量周期。 - 对备份/恢复断连、超时和回执丢失执行故障注入:同一 operation ID 只能查询收敛,不能再次派发。恢复失败只有在全部托管产物明确不存在时才能释放容量与 IP;对象删除响应未知时只能 HEAD 核对不存在,不能重发 DELETE。 diff --git a/web/app/Command/CloudSetup.php b/web/app/Command/CloudSetup.php index 9916bd4a..5281e1cc 100644 --- a/web/app/Command/CloudSetup.php +++ b/web/app/Command/CloudSetup.php @@ -8,7 +8,7 @@ use App\Services\Config; /** Initializes only the clean-break PHP business and Agent log schemas. */ final class CloudSetup { - private const SCHEMA_VERSION = 'go-agent-wss-v5-maintenance-migrations'; + private const SCHEMA_VERSION = 'go-agent-wss-v6-migration-bundles'; private const MAIN_TABLES = [ 'cloud_regions', 'cloud_network_lines', 'cloud_resource_pools', 'cloud_hosts', 'cloud_host_agents', @@ -24,7 +24,7 @@ final class CloudSetup 'cloud_platform_test_policy_state', 'cloud_platform_test_policy_rules', 'cloud_instance_snapshots', 'cloud_instance_backups', 'cloud_backup_restores', - 'cloud_instance_migrations', + 'cloud_instance_migrations', 'cloud_migration_bundle_artifacts', 'cloud_instance_runtime', 'cloud_orders', 'cloud_resource_reservations', 'cloud_service_periods', 'cloud_renewals', 'cloud_invoices', 'cloud_invoice_payments', @@ -64,7 +64,7 @@ final class CloudSetup ], 'cloud_instance_backups' => [ 'id', 'backup_key', 'source_instance_id', 'source_host_id', - 'user_id', 'source_uuid', 'name', 'state', 'consistency', + 'user_id', 'source_uuid', 'name', 'purpose', 'state', 'consistency', 'object_key', 'object_size_bytes', 'virtual_size_bytes', 'sha256', 'spec_json', 'sealed_storage_runtime', 'sealed_credential', 'storage_revision', 'active_job_id', @@ -82,10 +82,18 @@ final class CloudSetup 'source_host_id', 'source_uuid', 'source_reservation_id', 'target_host_id', 'target_uuid', 'target_reservation_id', 'backup_job_id', 'restore_job_id', 'firewall_job_id', - 'rollback_job_id', 'cleanup_job_id', - 'cleanup_mode', 'state', 'source_was_running', 'error_code', + 'activate_job_id', 'rollback_job_id', 'cleanup_job_id', + 'cleanup_mode', 'state', 'source_was_running', + 'source_autostart_enabled', + 'source_service_status', 'source_network_status', 'error_code', 'cutover_at', 'completed_at', 'created_at', 'updated_at', ], + 'cloud_migration_bundle_artifacts' => [ + 'id', 'migration_id', 'source_snapshot_id', 'snapshot_key', + 'name', 'consistency', 'object_key', 'object_size_bytes', + 'virtual_size_bytes', 'sha256', 'state', 'snapshot_created_at', + 'created_at', 'updated_at', + ], 'cloud_resource_reservations' => [ 'id', 'reservation_key', 'operation_id', 'order_id', 'backup_restore_id', 'offer_id', 'resource_pool_id', 'host_id', @@ -163,12 +171,18 @@ final class CloudSetup 'cloud_instance_migration_backup_job', 'cloud_instance_migration_restore_job', 'cloud_instance_migration_firewall_job', + 'cloud_instance_migration_activate_job', 'cloud_instance_migration_rollback_job', 'cloud_instance_migration_cleanup_job', 'cloud_instance_migration_instance', 'cloud_instance_migration_source', 'cloud_instance_migration_target', ], + 'cloud_migration_bundle_artifacts' => [ + 'PRIMARY', 'cloud_migration_bundle_snapshot', + 'cloud_migration_bundle_source_snapshot', + 'cloud_migration_bundle_object', 'cloud_migration_bundle_state', + ], 'cloud_resource_reservations' => [ 'PRIMARY', 'cloud_resource_reservation_key', 'cloud_resource_reservation_operation', diff --git a/web/app/Models/CloudInstanceMigration.php b/web/app/Models/CloudInstanceMigration.php index cbbc0635..912dccd6 100644 --- a/web/app/Models/CloudInstanceMigration.php +++ b/web/app/Models/CloudInstanceMigration.php @@ -19,9 +19,11 @@ final class CloudInstanceMigration extends Model 'backup_job_id' => 'int', 'restore_job_id' => 'int', 'firewall_job_id' => 'int', + 'activate_job_id' => 'int', 'rollback_job_id' => 'int', 'cleanup_job_id' => 'int', 'source_was_running' => 'bool', + 'source_autostart_enabled' => 'bool', 'cutover_at' => 'int', 'completed_at' => 'int', 'created_at' => 'int', diff --git a/web/app/Models/CloudMigrationBundleArtifact.php b/web/app/Models/CloudMigrationBundleArtifact.php new file mode 100644 index 00000000..3159197f --- /dev/null +++ b/web/app/Models/CloudMigrationBundleArtifact.php @@ -0,0 +1,20 @@ + 'int', + 'migration_id' => 'int', + 'source_snapshot_id' => 'int', + 'object_size_bytes' => 'int', + 'virtual_size_bytes' => 'int', + 'snapshot_created_at' => 'int', + 'created_at' => 'int', + 'updated_at' => 'int', + ]; +} diff --git a/web/app/Services/Cloud/CloudAdminOperationRecoveryService.php b/web/app/Services/Cloud/CloudAdminOperationRecoveryService.php index 3e5a6c0d..7dc74ed4 100644 --- a/web/app/Services/Cloud/CloudAdminOperationRecoveryService.php +++ b/web/app/Services/Cloud/CloudAdminOperationRecoveryService.php @@ -102,7 +102,10 @@ final class CloudAdminOperationRecoveryService 'cleanup_pending' => 'boolean', 'source_stopped' => 'boolean', 'source_was_running' => 'boolean', + 'source_autostart_enabled' => 'boolean', 'source_autostart_disabled' => 'boolean', + 'migration_bundle_version' => 'integer', + 'snapshot_artifacts' => 'migration_artifacts', ]; private const CATALOG_IMPORT_PROJECTION = [ 'id' => 'string', @@ -955,6 +958,9 @@ final class CloudAdminOperationRecoveryService return array_merge(self::VM_PROJECTION, [ 'backup_key' => 'string', 'sha256' => 'string', + 'migration_bundle_version' => 'integer', + 'snapshot_artifacts' => 'migration_artifacts', + 'staged_powered_off' => 'boolean', ]); } if ($kind === 'instance_upgrade') { @@ -985,6 +991,7 @@ final class CloudAdminOperationRecoveryService } if (in_array($kind, [ 'action', 'service_suspend', 'instance_migration_rollback', + 'instance_migration_activate', ], true)) { return ['uuid' => 'string', 'state' => 'string']; } @@ -1065,7 +1072,8 @@ final class CloudAdminOperationRecoveryService 'instance_migration_cleanup' => [ 'deleted', 'managed_artifacts_absent', 'forced', ], - 'action', 'service_suspend', 'instance_migration_rollback' => + 'action', 'service_suspend', 'instance_migration_rollback', + 'instance_migration_activate' => ['uuid', 'state'], 'network_suspend', 'network_resume' => ['network_enabled'], 'snapshot_create' => [ @@ -1112,6 +1120,7 @@ final class CloudAdminOperationRecoveryService && preg_match('/[\x00-\x1f\x7f]/', $value) !== 1, 'string_list' => self::validStringList($value), 'firewall_rules' => self::validFirewallRules($value), + 'migration_artifacts' => self::validMigrationArtifacts($value), default => false, }; } @@ -1130,6 +1139,46 @@ final class CloudAdminOperationRecoveryService return true; } + private static function validMigrationArtifacts(mixed $value): bool + { + if (!is_array($value) || !array_is_list($value) + || count($value) > CloudSnapshotService::MAX_PER_INSTANCE) { + return false; + } + $seen = []; + $fields = [ + 'snapshot_key', 'name', 'consistency', 'size_bytes', + 'virtual_size_bytes', 'sha256', 'created_at', + ]; + foreach ($value as $item) { + if (!is_array($item) || array_is_list($item) + || array_diff(array_keys($item), $fields) !== [] + || array_diff($fields, array_keys($item)) !== []) { + return false; + } + $key = $item['snapshot_key'] ?? null; + if (!is_string($key) + || preg_match('/^[a-f0-9]{32}$/D', $key) !== 1 + || isset($seen[$key]) + || !self::validProjectionValue($item['name'] ?? null, 'string') + || !in_array($item['consistency'] ?? null, + ['filesystem', 'crash'], true) + || !is_int($item['size_bytes'] ?? null) + || (int) $item['size_bytes'] < 1 + || !is_int($item['virtual_size_bytes'] ?? null) + || (int) $item['virtual_size_bytes'] < 1 + || !is_string($item['sha256'] ?? null) + || preg_match('/^[a-f0-9]{64}$/D', + $item['sha256']) !== 1 + || !is_int($item['created_at'] ?? null) + || (int) $item['created_at'] < 1) { + return false; + } + $seen[$key] = true; + } + return true; + } + private static function validFirewallRules(mixed $value): bool { if (!is_array($value) || !array_is_list($value) diff --git a/web/app/Services/Cloud/CloudBackupService.php b/web/app/Services/Cloud/CloudBackupService.php index b4dc2026..653d8e4b 100644 --- a/web/app/Services/Cloud/CloudBackupService.php +++ b/web/app/Services/Cloud/CloudBackupService.php @@ -43,7 +43,8 @@ final class CloudBackupService if (!in_array($perPage, [20, 50], true)) { throw new CloudError('备份库每页条数仅支持 20 或 50', 400); } - $query = CloudInstanceBackup::where('state', '<>', 'deleted'); + $query = CloudInstanceBackup::where('purpose', 'portable') + ->where('state', '<>', 'deleted'); $total = (int) (clone $query)->count('id'); $totalPages = max(1, (int) ceil($total / $perPage)); $page = min($page, $totalPages); @@ -162,7 +163,8 @@ final class CloudBackupService } $rows = CloudInstanceBackup::where( 'source_instance_id', (int) $instance->id - )->where('state', '<>', 'deleted')->orderByDesc('id')->get(); + )->where('purpose', 'portable')->where('state', '<>', 'deleted') + ->orderByDesc('id')->get(); $operationIds = []; foreach ($rows as $row) { if (preg_match('/^[a-f0-9]{32}$/D', @@ -243,24 +245,30 @@ final class CloudBackupService ], CloudPlacementService::maintenanceMigrationTargets( $instance, - self::resourcesFromSpec($spec) + self::resourcesFromSpec($spec), + CloudMigrationBundleService::advisoryAdditionalDiskGib( + $instance + ) ) ) : []; $migrationBlocks = $migration !== null && in_array( (string) $migration->state, ['backing_up', 'restoring', 'cutover', 'cleanup_pending', 'firewall_running', 'firewall_failed', 'firewall_uncertain', + 'activation_running', 'activation_failed', + 'activation_uncertain', 'rollback_running', 'rollback_failed', 'rollback_uncertain', 'cleanup_running', 'cleanup_uncertain', 'source_retained', 'uncertain'], true ); - $canCreate = $canManage + $canOperate = $canManage && $active === null && !$migrationBlocks && (string) $instance->status === 'active' && $supported - && (bool) ($storage['ready'] ?? false) + && (bool) ($storage['ready'] ?? false); + $canCreate = $canOperate && count(array_filter($backups, static fn (array $backup): bool => in_array($backup['state'], self::ACTIVE_BACKUP_STATES, true) )) < self::MAX_PER_INSTANCE; @@ -295,7 +303,7 @@ final class CloudBackupService 'max_backups' => self::MAX_PER_INSTANCE, 'can_create' => $canCreate, 'can_restore' => $canManage && $targetMap !== [], - 'can_migrate' => $canManage && $canCreate + 'can_migrate' => $canOperate && $migrationTargets !== [], 'create_block_reason' => $reason, 'active_job' => $active === null ? null : [ @@ -488,11 +496,12 @@ final class CloudBackupService $runtime = CloudObjectStorageConfig::runtime(true); $plan = null; $objectKey = null; + $bundlePlans = []; try { return $instance->getConnection()->transaction(function () use ( $instance, $actor, $name, $mode, $runtime, $requestTarget, $canonicalRequest, $fingerprint, $maintenance, - &$plan, &$objectKey + &$plan, &$objectKey, &$bundlePlans ): array { $locked = CloudInstance::where('id', (int) $instance->id) ->lockForUpdate()->first(); @@ -514,19 +523,27 @@ final class CloudBackupService } self::assertManagedSource($locked); self::assertNoActiveJob($locked); + $snapshots = []; if ($maintenance !== null) { + $snapshots = CloudMigrationBundleService:: + freezeSnapshotsLocked($locked); CloudInstanceMigrationService::assertStartAvailable( $locked, (int) $maintenance['target_host_id'], self::resourcesFromSpec(self::decodeSpec( (string) $locked->spec_json - )) + )), + CloudMigrationBundleService::additionalDiskGib( + $snapshots + ) ); } $used = CloudInstanceBackup::where( 'source_instance_id', (int) $locked->id - )->whereIn('state', self::ACTIVE_BACKUP_STATES)->count('id'); - if ((int) $used >= self::MAX_PER_INSTANCE) { + )->where('purpose', 'portable') + ->whereIn('state', self::ACTIVE_BACKUP_STATES)->count('id'); + if ($maintenance === null + && (int) $used >= self::MAX_PER_INSTANCE) { throw new CloudError('每台实例最多保留十个有效备份', 409); } $spec = self::freezeRestoreCompatibility( @@ -539,7 +556,8 @@ final class CloudBackupService $key = bin2hex(random_bytes(16)); $prefix = trim((string) $runtime['config']['prefix'], '/'); $objectKey = ($prefix === '' ? '' : $prefix . '/') - . 'vm-backups/' . $key . '/system.qcow2'; + . ($maintenance === null ? 'vm-backups/' : 'vm-migrations/') + . $key . '/system.qcow2'; $plan = (new CloudObjectStorageS3Client())->beginMultipartPlan( $runtime['config'], $runtime['credentials'], $objectKey, $maximumBytes, self::TRANSFER_URL_SECONDS @@ -552,6 +570,18 @@ final class CloudBackupService ]; if ($maintenance !== null) { $agentBody['prepare_maintenance_migration'] = true; + $bundle = CloudMigrationBundleService::beginExportPlans( + $snapshots, + $runtime, + $prefix, + $key, + $maximumBytes + ); + $bundlePlans = $bundle['prepared']; + $agentBody['migration_bundle'] = [ + 'version' => CloudMigrationBundleService::VERSION, + 'snapshots' => $bundle['body'], + ]; } $encodedAgentBody = json_encode( $agentBody, @@ -571,6 +601,8 @@ final class CloudBackupService $backup->user_id = (int) $locked->user_id; $backup->source_uuid = strtolower((string) $locked->uuid); $backup->name = $name; + $backup->purpose = $maintenance === null + ? 'portable' : 'migration_bundle'; $backup->state = 'creating'; $backup->consistency = $mode; $backup->object_key = $objectKey; @@ -627,7 +659,7 @@ final class CloudBackupService $backup->updated_at = time(); $backup->save(); if ($maintenance !== null) { - CloudInstanceMigrationService::createLedgerLocked( + $migration = CloudInstanceMigrationService::createLedgerLocked( $locked, $backup, $job, @@ -635,6 +667,10 @@ final class CloudBackupService (string) $maintenance['cleanup_mode'], $resources ); + CloudMigrationBundleService::persistArtifactsLocked( + $migration, + $bundlePlans + ); } return ['backup' => self::displayBackup($backup, $job), 'job' => $job]; @@ -651,6 +687,12 @@ final class CloudBackupService . get_class($cleanupError)); } } + if ($bundlePlans !== []) { + CloudMigrationBundleService::abortPreparedPlans( + $runtime, + $bundlePlans + ); + } throw $error; } } @@ -701,6 +743,9 @@ final class CloudBackupService return ['restore' => self::displayRestore($restore), 'job' => $prior]; } + if ((string) $backup->purpose !== 'portable') { + throw new CloudError('维护迁移包不能创建独立恢复实例', 409); + } if ((string) $backup->state !== 'ready' || (int) $backup->object_size_bytes < 1 || preg_match('/^[a-f0-9]{64}$/D', diff --git a/web/app/Services/Cloud/CloudInstanceMigrationService.php b/web/app/Services/Cloud/CloudInstanceMigrationService.php index 9808fce0..d49af984 100644 --- a/web/app/Services/Cloud/CloudInstanceMigrationService.php +++ b/web/app/Services/Cloud/CloudInstanceMigrationService.php @@ -20,6 +20,7 @@ final class CloudInstanceMigrationService private const OPEN_STATES = [ 'backing_up', 'restoring', 'cutover', 'cleanup_pending', 'firewall_running', 'firewall_failed', 'firewall_uncertain', + 'activation_running', 'activation_failed', 'activation_uncertain', 'rollback_running', 'rollback_failed', 'rollback_uncertain', 'cleanup_running', 'cleanup_uncertain', 'source_retained', 'uncertain', @@ -27,8 +28,18 @@ final class CloudInstanceMigrationService public static function assertStartAvailable(CloudInstance $instance, int $targetHostId, array $resources, + int $snapshotDiskGib = 0, ?int $excludeMigrationId = null): void { + if (!in_array((string) $instance->service_status, + ['active', 'exempt', 'expired_grace', 'suspended_expired'], true) + || !in_array((string) $instance->network_status, + ['active', 'blocked_traffic', 'blocked_expired'], true)) { + throw new CloudError( + '实例服务或网络状态正在变化,暂不能维护迁移', + 409 + ); + } if ($targetHostId === (int) $instance->host_id) { throw new CloudError('维护迁移必须选择其他宿主机', 409); } @@ -49,7 +60,8 @@ final class CloudInstanceMigrationService $allowed = array_filter( CloudPlacementService::maintenanceMigrationTargets( $instance, - $resources + $resources, + $snapshotDiskGib ), static fn (array $target): bool => (int) ($target['id'] ?? 0) === $targetHostId @@ -74,7 +86,7 @@ final class CloudInstanceMigrationService throw new \LogicException('Invalid maintenance migration ledger'); } $migration = new CloudInstanceMigration(); - $migration->migration_key = bin2hex(random_bytes(16)); + $migration->migration_key = (string) $backup->backup_key; $migration->instance_id = (int) $instance->id; $migration->backup_id = (int) $backup->id; $migration->restore_id = null; @@ -87,11 +99,15 @@ final class CloudInstanceMigrationService $migration->backup_job_id = (int) $job->id; $migration->restore_job_id = null; $migration->firewall_job_id = null; + $migration->activate_job_id = null; $migration->rollback_job_id = null; $migration->cleanup_job_id = null; $migration->cleanup_mode = $cleanupMode; $migration->state = 'backing_up'; $migration->source_was_running = null; + $migration->source_autostart_enabled = null; + $migration->source_service_status = (string) $instance->service_status; + $migration->source_network_status = (string) $instance->network_status; $migration->error_code = ''; $migration->cutover_at = null; $migration->completed_at = null; @@ -124,7 +140,10 @@ final class CloudInstanceMigrationService $restore, (int) $instance->user_id, $resources, - (int) $instance->resource_pool_id + (int) $instance->resource_pool_id, + CloudMigrationBundleService::additionalDiskGib( + CloudMigrationBundleService::freezeSnapshotsLocked($instance) + ) ); $reservation = $placement['reservation']; $reservation->instance_id = (int) $instance->id; @@ -157,6 +176,8 @@ final class CloudInstanceMigrationService ? null : (string) $migration->cleanup_job_id, 'firewall_task_id' => $migration->firewall_job_id === null ? null : (string) $migration->firewall_job_id, + 'activation_task_id' => $migration->activate_job_id === null + ? null : (string) $migration->activate_job_id, 'rollback_task_id' => $migration->rollback_job_id === null ? null : (string) $migration->rollback_job_id, 'source_retained' => in_array($state, @@ -260,6 +281,20 @@ final class CloudInstanceMigrationService } return; } + if ((string) $job->kind === 'instance_migration_activate') { + $migration = CloudInstanceMigration::where( + 'activate_job_id', (int) $job->id + )->lockForUpdate()->first(); + if ($migration !== null) { + self::finishActivationLocked( + $migration, + $job, + $status, + $result + ); + } + return; + } if ((string) $job->kind === 'instance_migration_rollback') { $migration = CloudInstanceMigration::where( 'rollback_job_id', (int) $job->id @@ -279,6 +314,7 @@ final class CloudInstanceMigrationService 'backup_create' => 'backup_job_id', 'backup_restore' => 'restore_job_id', 'firewall_apply' => 'firewall_job_id', + 'instance_migration_activate' => 'activate_job_id', 'instance_migration_rollback' => 'rollback_job_id', 'instance_migration_cleanup' => 'cleanup_job_id', default => null, @@ -292,6 +328,7 @@ final class CloudInstanceMigrationService 'state' => match ((string) $job->kind) { 'instance_migration_cleanup' => 'cleanup_uncertain', 'firewall_apply' => 'firewall_uncertain', + 'instance_migration_activate' => 'activation_uncertain', 'instance_migration_rollback' => 'rollback_uncertain', default => 'uncertain', }, @@ -358,10 +395,14 @@ final class CloudInstanceMigrationService if (!is_array($result) || ($result['source_stopped'] ?? null) !== true || ($result['source_autostart_disabled'] ?? null) !== true - || !is_bool($result['source_was_running'] ?? null)) { + || !is_bool($result['source_was_running'] ?? null) + || !is_bool($result['source_autostart_enabled'] ?? null)) { throw new CloudError('Agent 未确认迁移源实例已安全停止', 502); } $migration->source_was_running = $result['source_was_running']; + $migration->source_autostart_enabled = + $result['source_autostart_enabled']; + CloudMigrationBundleService::finishExportLocked($migration, $result); self::queueRestoreLocked($migration, $job); } @@ -424,7 +465,11 @@ final class CloudInstanceMigrationService || !hash_equals(strtolower((string) $instance->uuid), strtolower((string) $migration->source_uuid)) || (int) $instance->reservation_id - !== (int) $migration->source_reservation_id) { + !== (int) $migration->source_reservation_id + || !hash_equals((string) $migration->source_service_status, + (string) $instance->service_status) + || !hash_equals((string) $migration->source_network_status, + (string) $instance->network_status)) { throw new CloudRecoveryInstanceStateConflict( '维护迁移源实例在备份完成前发生变化', 409 @@ -466,6 +511,10 @@ final class CloudInstanceMigrationService (string) $backup->object_key, self::TRANSFER_URL_SECONDS ); + $snapshotSources = CloudMigrationBundleService::restoreSources( + $migration, + $runtime + ); $now = time(); $operationId = (string) $restore->operation_id; $policy = $spec['platform_network_policy_snapshot'] ?? []; @@ -486,7 +535,9 @@ final class CloudInstanceMigrationService 'template_revision' => 0, 'template_generation_token' => '', 'startup_script' => '', - 'hostname' => CloudInput::hostname((string) $instance->name), + 'hostname' => CloudInput::hostname((string) ( + $spec['guest_hostname'] ?? $instance->name + )), 'guest_ssh_public_key' => '', 'root_password' => '', 'public_pool_key' => (string) $network['public_pool_key'], @@ -501,6 +552,13 @@ final class CloudInstanceMigrationService 'source_url' => $sourceUrl, 'source_size_bytes' => (int) $backup->object_size_bytes, 'source_sha256' => (string) $backup->sha256, + 'migration_bundle' => [ + 'version' => CloudMigrationBundleService::VERSION, + 'stage_powered_off' => true, + 'network_suspended' => + (string) $migration->source_network_status !== 'active', + 'snapshots' => $snapshotSources, + ], ]; [$restoreJob, $created] = CloudJobService::reserveStandaloneAgentMutation( [ @@ -613,12 +671,22 @@ final class CloudInstanceMigrationService ); $restore->state = 'failed'; $restore->error_code = 'backup_restore_failed'; - self::queueRollbackLocked( - $migration, - $instance, - (int) $job->user_id, - 'migration_restore_failed' - ); + if ((bool) $migration->source_autostart_enabled) { + self::queueRollbackLocked( + $migration, + $instance, + (int) $job->user_id, + 'migration_restore_failed' + ); + } else { + self::setState( + $migration, + 'rolled_back', + 'migration_restore_failed' + ); + $migration->completed_at = $now; + $migration->save(); + } } elseif ($status === 'succeeded') { $vm = is_array($result['vm'] ?? null) ? $result['vm'] : $result; if (!is_array($result) || !is_array($vm) @@ -636,6 +704,51 @@ final class CloudInstanceMigrationService )) { throw new CloudError('Agent 迁移恢复结果与目标资源不一致', 502); } + CloudMigrationBundleService::finishRestoreLocked( + $migration, + $result + ); + $rootDiskGib = filter_var( + $payload['body']['disk_gib'] ?? null, + FILTER_VALIDATE_INT, + ['options' => ['min_range' => 1]] + ); + $reservation = CloudResourceReservation::where( + 'id', (int) $migration->target_reservation_id + )->lockForUpdate()->first(); + $sourceReservation = CloudResourceReservation::where( + 'id', (int) $migration->source_reservation_id + )->lockForUpdate()->first(); + $expectedClaim = $rootDiskGib === false ? 0 + : (int) $rootDiskGib + + CloudMigrationBundleService::artifactDiskGib($migration); + if ($reservation === null || $sourceReservation === null + || $rootDiskGib === false + || !in_array((string) $reservation->state, + ['held', 'uncertain'], true) + || (int) $reservation->disk_gib !== $expectedClaim + || (string) $sourceReservation->state !== 'committed' + || (int) $sourceReservation->host_id + !== (int) $migration->source_host_id + || (int) $sourceReservation->instance_id + !== (int) $instance->id + || (int) $sourceReservation->disk_gib + !== (int) $rootDiskGib) { + throw new CloudRecoveryInstanceStateConflict( + '维护迁移目标快照容量预留账本不一致', + 409 + ); + } + // Snapshot rows now account for their target-host volumes. Keep + // only the root disk in the ordinary instance reservation; the + // source duplicate remains claimed by restored bundle artifacts + // until source cleanup is proven. + $reservation->disk_gib = (int) $rootDiskGib; + $reservation->updated_at = $now; + $reservation->save(); + $sourceReservation->disk_gib = $expectedClaim; + $sourceReservation->updated_at = $now; + $sourceReservation->save(); CloudPlacementService::finishBackupRestore( $restore, (int) $instance->id, @@ -676,8 +789,8 @@ final class CloudInstanceMigrationService JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE ); - $instance->status = 'active'; - $instance->network_status = 'active'; + $instance->service_status = (string) $migration->source_service_status; + $instance->network_status = (string) $migration->source_network_status; // The target Agent starts a new durable counter generation. Its // first sample must be allowed to advance the carried lifetime // base even when the source host clock was ahead. @@ -903,6 +1016,143 @@ final class CloudInstanceMigrationService CloudInstanceMigration $migration, CloudInstance $instance, int $actorUserId + ): void { + if (!(bool) $migration->source_autostart_enabled) { + self::setState($migration, 'activated', ''); + self::advanceAfterActivationLocked( + $migration, + $instance, + $actorUserId + ); + return; + } + self::queueActivationLocked($migration, $instance, $actorUserId); + } + + private static function queueActivationLocked( + CloudInstanceMigration $migration, + CloudInstance $instance, + int $actorUserId + ): CloudJob { + if ($migration->activate_job_id !== null) { + $existing = CloudJob::find((int) $migration->activate_job_id); + if ($existing === null) { + throw new CloudRecoveryInstanceStateConflict( + '维护迁移目标激活任务目录缺失', + 409 + ); + } + return $existing; + } + [$job, $created] = CloudJobService::reserveStandaloneAgentMutation( + [ + 'user_id' => $actorUserId, + 'request_idempotency_key' => 'migration-activate-' + . (string) $migration->migration_key, + ], + (int) $migration->target_host_id, + 'instance_migration_activate', + 'POST', + '/v1/vms/' . rawurlencode((string) $migration->target_uuid) + . '/service/resume', + [ + 'reason' => 'maintenance_migration_activate', + 'start_if_suspended' => (bool) $migration->source_was_running, + ], + [ + 'owner_id' => (int) $instance->user_id, + 'can_manage' => true, + 'maintenance_migration_id' => (int) $migration->id, + ], + [ + 'migration_id' => (int) $migration->id, + 'target_host_id' => (int) $migration->target_host_id, + 'target_uuid' => (string) $migration->target_uuid, + 'source_was_running' => (bool) $migration->source_was_running, + ], + null, + '/v1/instance-maintenance-migrations/' + . (int) $migration->id . '/activate' + ); + if (!$created) { + throw new CloudRecoveryInstanceStateConflict( + '维护迁移目标激活任务幂等冲突', + 409 + ); + } + $job->instance_id = (int) $instance->id; + $job->backup_id = (int) $migration->backup_id; + $job->backup_restore_id = (int) $migration->restore_id; + $job->save(); + $migration->activate_job_id = (int) $job->id; + $migration->state = 'activation_running'; + $migration->error_code = ''; + $migration->updated_at = time(); + $migration->save(); + return $job; + } + + private static function finishActivationLocked( + CloudInstanceMigration $migration, + CloudJob $job, + string $status, + $result + ): void { + $instance = CloudInstance::where( + 'id', (int) $migration->instance_id + )->lockForUpdate()->first(); + if ($instance === null + || (int) $migration->activate_job_id !== (int) $job->id + || (int) $job->instance_id !== (int) $instance->id + || (int) $job->host_id !== (int) $migration->target_host_id + || (int) $instance->host_id !== (int) $migration->target_host_id + || !hash_equals(strtolower((string) $instance->uuid), + strtolower((string) $migration->target_uuid))) { + throw new CloudRecoveryInstanceStateConflict( + '维护迁移目标激活任务与目标实例不一致', + 409 + ); + } + if ($status === 'running') { + return; + } + if ($status === 'uncertain') { + self::setState( + $migration, + 'activation_uncertain', + 'operation_effect_unknown' + ); + return; + } + if ($status === 'failed') { + self::setState( + $migration, + 'activation_failed', + 'migration_target_activation_failed' + ); + return; + } + $state = is_array($result) ? ($result['state'] ?? null) : null; + $running = in_array($state, ['running', 'blocked'], true); + if (!is_array($result) + || !hash_equals(strtolower((string) $migration->target_uuid), + strtolower((string) ($result['uuid'] ?? ''))) + || ((bool) $migration->source_was_running && !$running) + || (!(bool) $migration->source_was_running && $running)) { + throw new CloudError('Agent 未确认迁移目标恢复原电源状态', 502); + } + self::setState($migration, 'activated', ''); + self::advanceAfterActivationLocked( + $migration, + $instance, + (int) $job->user_id + ); + } + + private static function advanceAfterActivationLocked( + CloudInstanceMigration $migration, + CloudInstance $instance, + int $actorUserId ): void { if ((string) $migration->cleanup_mode === 'delete') { self::queueCleanupLocked($migration, $instance, [ @@ -919,7 +1169,7 @@ final class CloudInstanceMigrationService CloudInstance $instance, array $actor): CloudJob { if (!in_array((string) $migration->state, - ['cutover', 'firewall_running', 'source_retained'], true)) { + ['activated', 'source_retained'], true)) { throw new CloudError('当前迁移状态不能清理源副本', 409); } [$job, $created] = CloudJobService::reserveStandaloneAgentMutation( diff --git a/web/app/Services/Cloud/CloudMigrationBundleService.php b/web/app/Services/Cloud/CloudMigrationBundleService.php new file mode 100644 index 00000000..e3dae9e4 --- /dev/null +++ b/web/app/Services/Cloud/CloudMigrationBundleService.php @@ -0,0 +1,332 @@ + $snapshots */ + public static function additionalDiskGib(array $snapshots): int + { + $bytes = 0; + foreach ($snapshots as $snapshot) { + $bytes += max(0, (int) ($snapshot->size_bytes ?? 0)); + } + return (int) ceil($bytes / self::GIB); + } + + /** Snapshot capacity frozen into one persisted migration bundle. */ + public static function artifactDiskGib( + CloudInstanceMigration $migration + ): int { + $bytes = (int) CloudMigrationBundleArtifact::where( + 'migration_id', (int) $migration->id + )->sum('virtual_size_bytes'); + return (int) ceil(max(0, $bytes) / self::GIB); + } + + public static function advisoryAdditionalDiskGib( + CloudInstance $instance + ): int { + return self::additionalDiskGib(CloudInstanceSnapshot::where( + 'instance_id', (int) $instance->id + )->where('host_id', (int) $instance->host_id) + ->where('user_id', (int) $instance->user_id) + ->where('state', 'ready')->get()->all()); + } + + /** @return array */ + public static function freezeSnapshotsLocked(CloudInstance $instance): array + { + $rows = CloudInstanceSnapshot::where( + 'instance_id', (int) $instance->id + )->where('user_id', (int) $instance->user_id) + ->whereIn('state', [ + 'creating', 'ready', 'restoring', 'deleting', 'uncertain', + ])->orderBy('id')->lockForUpdate()->get()->all(); + if (count($rows) > CloudSnapshotService::MAX_PER_INSTANCE) { + throw new CloudError('实例快照目录超过可迁移上限', 409); + } + foreach ($rows as $snapshot) { + if ((string) $snapshot->state !== 'ready' + || (int) $snapshot->host_id !== (int) $instance->host_id + || (int) ($snapshot->size_bytes ?? 0) < 1 + || preg_match('/^[a-f0-9]{32}$/D', + (string) $snapshot->snapshot_key) !== 1) { + throw new CloudError( + '实例存在未就绪或宿主机不一致的快照,不能维护迁移', + 409 + ); + } + } + return $rows; + } + + /** + * @param array $snapshots + * @return array{body:array>,prepared:array>} + */ + public static function beginExportPlans(array $snapshots, array $runtime, + string $prefix, string $bundleKey, int $maximumBytes): array + { + $client = new CloudObjectStorageS3Client(); + $body = []; + $prepared = []; + try { + foreach ($snapshots as $snapshot) { + $objectKey = ($prefix === '' ? '' : $prefix . '/') + . 'vm-migrations/' . $bundleKey . '/snapshots/' + . (string) $snapshot->snapshot_key . '.qcow2'; + $plan = $client->beginMultipartPlan( + $runtime['config'], $runtime['credentials'], $objectKey, + $maximumBytes, 604800 + ); + $prepared[] = [ + 'snapshot' => $snapshot, + 'object_key' => $objectKey, + 'plan' => $plan, + ]; + $body[] = [ + 'snapshot_key' => (string) $snapshot->snapshot_key, + 'object_upload' => self::publicTransferPlan($plan), + ]; + } + } catch (\Throwable $error) { + self::abortPreparedPlans($runtime, $prepared); + throw $error; + } + return ['body' => $body, 'prepared' => $prepared]; + } + + /** @param array> $prepared */ + public static function persistArtifactsLocked( + CloudInstanceMigration $migration, + array $prepared + ): void { + $now = time(); + foreach ($prepared as $item) { + $snapshot = $item['snapshot'] ?? null; + if (!$snapshot instanceof CloudInstanceSnapshot) { + throw new \LogicException('Invalid migration snapshot plan'); + } + $artifact = new CloudMigrationBundleArtifact(); + $artifact->migration_id = (int) $migration->id; + $artifact->source_snapshot_id = (int) $snapshot->id; + $artifact->snapshot_key = (string) $snapshot->snapshot_key; + $artifact->name = (string) $snapshot->name; + $artifact->consistency = (string) $snapshot->consistency; + $artifact->object_key = (string) $item['object_key']; + $artifact->object_size_bytes = 0; + $artifact->virtual_size_bytes = (int) $snapshot->size_bytes; + $artifact->sha256 = ''; + $artifact->state = 'creating'; + $artifact->snapshot_created_at = (int) $snapshot->created_at; + $artifact->created_at = $artifact->updated_at = $now; + $artifact->save(); + } + } + + /** Validate the complete Agent export receipt before target restore. */ + public static function finishExportLocked( + CloudInstanceMigration $migration, + array $result + ): void { + if ((int) ($result['migration_bundle_version'] ?? 0) + !== self::VERSION + || !is_array($result['snapshot_artifacts'] ?? null)) { + throw new CloudError('Agent 未返回完整的迁移快照包', 502); + } + $artifacts = CloudMigrationBundleArtifact::where( + 'migration_id', (int) $migration->id + )->orderBy('id')->lockForUpdate()->get(); + $received = []; + foreach ($result['snapshot_artifacts'] as $item) { + $key = is_array($item) ? ($item['snapshot_key'] ?? null) : null; + if (!is_string($key) || isset($received[$key])) { + throw new CloudError('Agent 迁移快照回执存在重复标识', 502); + } + $received[$key] = $item; + } + if (count($received) !== count($artifacts)) { + throw new CloudError('Agent 迁移快照回执数量不一致', 502); + } + $now = time(); + foreach ($artifacts as $artifact) { + $item = $received[(string) $artifact->snapshot_key] ?? null; + $size = filter_var($item['size_bytes'] ?? null, + FILTER_VALIDATE_INT, ['options' => ['min_range' => 1]]); + $virtual = filter_var($item['virtual_size_bytes'] ?? null, + FILTER_VALIDATE_INT, ['options' => ['min_range' => 1]]); + $digest = $item['sha256'] ?? null; + if (!is_array($item) + || !hash_equals((string) $artifact->name, + (string) ($item['name'] ?? '')) + || !hash_equals((string) $artifact->consistency, + (string) ($item['consistency'] ?? '')) + || $size === false || $virtual === false + || (int) $virtual !== (int) $artifact->virtual_size_bytes + || filter_var($item['created_at'] ?? null, + FILTER_VALIDATE_INT, + ['options' => ['min_range' => 1]]) === false + || !is_string($digest) + || preg_match('/^[a-f0-9]{64}$/D', $digest) !== 1) { + throw new CloudError('Agent 迁移快照回执与冻结目录不一致', 502); + } + $artifact->object_size_bytes = (int) $size; + $artifact->sha256 = $digest; + // The PHP row is created when the request is admitted, while the + // Agent metadata records the later instant the volume was actually + // frozen. Preserve the authoritative Agent timestamp inside the + // transport bundle without weakening the immutable key/name/size + // checks above. + $artifact->snapshot_created_at = (int) $item['created_at']; + $artifact->state = 'ready'; + $artifact->updated_at = $now; + $artifact->save(); + } + } + + /** @return array> */ + public static function restoreSources(CloudInstanceMigration $migration, + array $runtime): array + { + $rows = CloudMigrationBundleArtifact::where( + 'migration_id', (int) $migration->id + )->orderBy('id')->get(); + $client = new CloudObjectStorageS3Client(); + $sources = []; + foreach ($rows as $artifact) { + if ((string) $artifact->state !== 'ready' + || (int) $artifact->object_size_bytes < 1 + || (int) $artifact->virtual_size_bytes < 1 + || preg_match('/^[a-f0-9]{64}$/D', + (string) $artifact->sha256) !== 1) { + throw new CloudError('迁移快照包尚未就绪', 409); + } + $sources[] = [ + 'snapshot_key' => (string) $artifact->snapshot_key, + 'name' => (string) $artifact->name, + 'consistency' => (string) $artifact->consistency, + 'virtual_size_bytes' => (int) $artifact->virtual_size_bytes, + 'source_url' => $client->presignedGet( + $runtime['config'], $runtime['credentials'], + (string) $artifact->object_key, 604800 + ), + 'source_size_bytes' => (int) $artifact->object_size_bytes, + 'source_sha256' => (string) $artifact->sha256, + 'created_at' => (int) $artifact->snapshot_created_at, + ]; + } + return $sources; + } + + /** Prove target artifact receipts and atomically move the PHP catalog. */ + public static function finishRestoreLocked( + CloudInstanceMigration $migration, + array $result + ): void { + $vm = is_array($result['vm'] ?? null) ? $result['vm'] : $result; + if (($result['staged_powered_off'] ?? null) !== true + || (int) ($result['migration_bundle_version'] ?? 0) + !== self::VERSION + || !is_array($result['snapshot_artifacts'] ?? null) + || !is_array($vm) + || (string) ($vm['status'] ?? '') !== 'shutoff') { + throw new CloudError('Agent 未确认迁移目标保持关机且快照完整', 502); + } + $received = []; + foreach ($result['snapshot_artifacts'] as $item) { + $key = is_array($item) ? ($item['snapshot_key'] ?? null) : null; + if (!is_string($key) || isset($received[$key])) { + throw new CloudError('Agent 目标快照回执存在重复标识', 502); + } + $received[$key] = $item; + } + $artifacts = CloudMigrationBundleArtifact::where( + 'migration_id', (int) $migration->id + )->orderBy('id')->lockForUpdate()->get(); + if (count($received) !== count($artifacts)) { + throw new CloudError('Agent 目标快照回执数量不一致', 502); + } + foreach ($artifacts as $artifact) { + $item = $received[(string) $artifact->snapshot_key] ?? null; + if (!is_array($item) + || !hash_equals((string) $artifact->sha256, + (string) ($item['sha256'] ?? '')) + || (int) $artifact->object_size_bytes + !== (int) ($item['size_bytes'] ?? -1) + || (int) $artifact->virtual_size_bytes + !== (int) ($item['virtual_size_bytes'] ?? -1) + || (int) $artifact->snapshot_created_at + !== (int) ($item['created_at'] ?? -1)) { + throw new CloudError('Agent 目标快照与迁移包不一致', 502); + } + $snapshot = CloudInstanceSnapshot::where( + 'id', (int) $artifact->source_snapshot_id + )->lockForUpdate()->first(); + if ($snapshot === null + || (int) $snapshot->instance_id !== (int) $migration->instance_id + || (int) $snapshot->host_id !== (int) $migration->source_host_id + || (string) $snapshot->state !== 'ready' + || !hash_equals((string) $artifact->snapshot_key, + (string) $snapshot->snapshot_key)) { + throw new CloudRecoveryInstanceStateConflict( + '迁移期间实例快照目录已变化', + 409 + ); + } + $snapshot->host_id = (int) $migration->target_host_id; + $snapshot->updated_at = time(); + $snapshot->save(); + $artifact->state = 'restored'; + $artifact->updated_at = time(); + $artifact->save(); + } + } + + /** @param array> $prepared */ + public static function abortPreparedPlans(array $runtime, + array $prepared): void + { + $client = new CloudObjectStorageS3Client(); + foreach ($prepared as $item) { + try { + $client->abortMultipart( + $runtime['config'], $runtime['credentials'], + (string) ($item['object_key'] ?? ''), + (string) ($item['plan']['upload_id'] ?? '') + ); + } catch (\Throwable $error) { + error_log('migration bundle multipart cleanup unavailable: ' + . get_class($error)); + } + } + } + + private static function publicTransferPlan(array $plan): array + { + return [ + 'part_size_bytes' => (int) $plan['part_size_bytes'], + 'maximum_size_bytes' => (int) $plan['maximum_size_bytes'], + 'part_urls' => array_values($plan['part_urls']), + 'complete_url' => (string) $plan['complete_url'], + 'abort_url' => (string) $plan['abort_url'], + 'verify_url' => (string) $plan['verify_url'], + ]; + } +} diff --git a/web/app/Services/Cloud/CloudOperationCatalog.php b/web/app/Services/Cloud/CloudOperationCatalog.php index 7a76ebf6..3d144807 100644 --- a/web/app/Services/Cloud/CloudOperationCatalog.php +++ b/web/app/Services/Cloud/CloudOperationCatalog.php @@ -28,6 +28,7 @@ final class CloudOperationCatalog 'backup_restore' => '从备份创建实例', 'instance_migration_cleanup' => '清理维护迁移源副本', 'instance_migration_rollback' => '恢复维护迁移源实例', + 'instance_migration_activate' => '恢复维护迁移目标状态', 'firewall_apply' => '应用防火墙', 'service_suspend' => '到期停用', 'network_suspend' => '限制网络', @@ -72,6 +73,7 @@ final class CloudOperationCatalog 'backup_restore', 'instance_migration_cleanup', 'instance_migration_rollback', + 'instance_migration_activate', 'rescue', ]; diff --git a/web/app/Services/Cloud/CloudOperationRecoveryContract.php b/web/app/Services/Cloud/CloudOperationRecoveryContract.php index 9a38f943..f234755f 100644 --- a/web/app/Services/Cloud/CloudOperationRecoveryContract.php +++ b/web/app/Services/Cloud/CloudOperationRecoveryContract.php @@ -56,6 +56,13 @@ final class CloudOperationRecoveryContract 'action_patterns' => [], 'system_resolutions' => ['applied', 'not_applied'], ], + 'instance_migration_activate' => [ + 'recovery_mode' => 'generic', + 'identity_scope' => 'instance', + 'actions' => ['service-resume'], + 'action_patterns' => [], + 'system_resolutions' => ['applied', 'not_applied'], + ], 'expiry_delete' => [ 'recovery_mode' => 'generic', 'identity_scope' => 'instance', diff --git a/web/app/Services/Cloud/CloudOperationRecoveryIdentity.php b/web/app/Services/Cloud/CloudOperationRecoveryIdentity.php index 2f0b1d1b..681f1a54 100644 --- a/web/app/Services/Cloud/CloudOperationRecoveryIdentity.php +++ b/web/app/Services/Cloud/CloudOperationRecoveryIdentity.php @@ -7,6 +7,7 @@ use App\Models\CloudInstanceBackup; use App\Models\CloudInstanceMigration; use App\Models\CloudInstanceFirewallVersion; use App\Models\CloudInstanceSnapshot; +use App\Models\CloudMigrationBundleArtifact; use App\Models\CloudBackupRestore; use App\Models\CloudHostStorageConfiguration; use App\Models\CloudJob; @@ -259,7 +260,8 @@ final class CloudOperationRecoveryIdentity $kind = (string) $job->kind; if (in_array($kind, ['backup_restore', 'instance_migration_cleanup', - 'instance_migration_rollback'], true) + 'instance_migration_rollback', + 'instance_migration_activate'], true) && isset($payload['maintenance_migration_id'])) { return self::maintenanceMigrationIdentity( $job, @@ -405,6 +407,36 @@ final class CloudOperationRecoveryIdentity ); return [$action, $uuid]; } + if ((string) $job->kind === 'instance_migration_activate') { + if ((int) $migration->activate_job_id !== (int) $job->id + || (int) $migration->target_host_id !== (int) $job->host_id + || preg_match(self::UUID_PATTERN, + strtolower((string) $migration->target_uuid)) !== 1) { + throw new CloudRecoveryInstanceStateConflict( + '维护迁移目标激活身份不一致', + 409 + ); + } + $uuid = strtolower((string) $migration->target_uuid); + $base = '/v1/vms/' . rawurlencode($uuid); + self::expectRequest( + $method, + $target, + 'POST', + $base . '/service/resume' + ); + self::assertAllowedBody($body, [ + 'operation_id', 'reason', 'start_if_suspended', + ]); + if (($body['reason'] ?? null) + !== 'maintenance_migration_activate' + || !is_bool($body['start_if_suspended'] ?? null) + || $body['start_if_suspended'] + !== (bool) $migration->source_was_running) { + self::invalidPayload('维护迁移目标激活确认无效'); + } + return ['service-resume', $uuid]; + } if ((string) $job->kind === 'instance_migration_rollback') { if ((int) $migration->rollback_job_id !== (int) $job->id || (int) $migration->source_host_id !== (int) $job->host_id @@ -978,6 +1010,7 @@ final class CloudOperationRecoveryIdentity self::assertAllowedBody($body, [ 'operation_id', 'backup_key', 'name', 'consistency_mode', 'object_upload', 'prepare_maintenance_migration', + 'migration_bundle', ]); $backupId = filter_var($job->backup_id, FILTER_VALIDATE_INT, ['options' => ['min_range' => 1]]); @@ -1021,11 +1054,19 @@ final class CloudOperationRecoveryIdentity } if (($migration !== null && (($body['prepare_maintenance_migration'] ?? null) !== true - || (int) $migration->backup_job_id !== (int) $job->id)) + || (int) $migration->backup_job_id !== (int) $job->id + || (string) $backup->purpose !== 'migration_bundle')) || ($migration === null - && array_key_exists('prepare_maintenance_migration', $body))) { + && (array_key_exists('prepare_maintenance_migration', $body) + || array_key_exists('migration_bundle', $body)))) { self::invalidPayload('维护迁移冷备请求无效'); } + if ($migration !== null) { + self::assertMigrationBundleExportBody( + $body['migration_bundle'] ?? null, + $migration + ); + } foreach ($upload['part_urls'] as $url) { if (!is_string($url) || !str_starts_with($url, 'https://')) { self::invalidPayload('创建可移植备份请求无效'); @@ -1053,6 +1094,7 @@ final class CloudOperationRecoveryIdentity 'public_ip_cidr', 'gateway', 'dns_servers', 'platform_network_policy_snapshot', 'debian_12_network_fix', 'backup_key', 'source_url', 'source_size_bytes', 'source_sha256', + 'migration_bundle', ]); $restoreId = filter_var($job->backup_restore_id, FILTER_VALIDATE_INT, ['options' => ['min_range' => 1]]); @@ -1085,6 +1127,12 @@ final class CloudOperationRecoveryIdentity $expectedUuid = $migration === null ? strtolower((string) $instance->uuid) : strtolower((string) $migration->target_uuid); + $expectedReservationDisk = (int) ($body['disk_gib'] ?? 0); + if ($migration !== null && $reservation !== null + && (string) $reservation->state !== 'committed') { + $expectedReservationDisk += + CloudMigrationBundleService::artifactDiskGib($migration); + } if ($restore === null || $backup === null || $reservation === null || $ip === null || (int) $restore->backup_id !== (int) $backup->id @@ -1131,7 +1179,8 @@ final class CloudOperationRecoveryIdentity || (int) ($body['memory_mib'] ?? 0) !== (int) $reservation->memory_mib || (int) ($body['disk_gib'] ?? 0) - !== (int) $reservation->disk_gib + !== (int) ($backupSpec['disk_gib'] ?? 0) + || (int) $reservation->disk_gib !== $expectedReservationDisk || (int) ($body['bandwidth_mbps'] ?? 0) !== (int) ($instanceSpec['bandwidth_mbps'] ?? 0) || (int) ($body['vcpu'] ?? 0) @@ -1140,8 +1189,6 @@ final class CloudOperationRecoveryIdentity || (int) ($body['memory_mib'] ?? 0) !== (int) ($backupSpec['memory_mib'] ?? $backupSpec['ram_mib'] ?? 0) - || (int) ($body['disk_gib'] ?? 0) - !== (int) ($backupSpec['disk_gib'] ?? 0) || (int) ($body['bandwidth_mbps'] ?? 0) !== (int) ($backupSpec['bandwidth_mbps'] ?? 0) || !hash_equals( @@ -1159,6 +1206,8 @@ final class CloudOperationRecoveryIdentity $body['source_sha256']) !== 1 || !hash_equals((string) $backup->sha256, (string) $body['source_sha256']) + || ($migration === null + && array_key_exists('migration_bundle', $body)) || ($migration !== null && ((int) $migration->restore_id !== (int) $restore->id || (int) $migration->instance_id !== (int) $instance->id @@ -1169,9 +1218,111 @@ final class CloudOperationRecoveryIdentity )))) { self::invalidPayload('备份恢复请求无效'); } + if ($migration !== null) { + $expectedHostname = CloudInput::hostname((string) ( + $backupSpec['guest_hostname'] ?? $instance->name + )); + if (!hash_equals( + $expectedHostname, + (string) ($body['hostname'] ?? '') + )) { + self::invalidPayload('维护迁移目标主机名快照无效'); + } + self::assertMigrationBundleRestoreBody( + $body['migration_bundle'] ?? null, + $migration + ); + } return 'restore-backup:' . $key; } + private static function assertMigrationBundleExportBody( + mixed $value, + CloudInstanceMigration $migration + ): void { + if (!is_array($value) || array_is_list($value) + || (int) ($value['version'] ?? 0) + !== CloudMigrationBundleService::VERSION + || !is_array($value['snapshots'] ?? null) + || !array_is_list($value['snapshots'])) { + self::invalidPayload('维护迁移快照导出请求无效'); + } + $rows = CloudMigrationBundleArtifact::where( + 'migration_id', (int) $migration->id + )->orderBy('id')->get(); + if (count($value['snapshots']) !== count($rows)) { + self::invalidPayload('维护迁移快照导出数量无效'); + } + foreach ($rows as $index => $row) { + $item = $value['snapshots'][$index] ?? null; + $upload = is_array($item) ? ($item['object_upload'] ?? null) : null; + if (!is_array($item) || array_is_list($item) + || array_diff(array_keys($item), + ['snapshot_key', 'object_upload']) !== [] + || !hash_equals((string) $row->snapshot_key, + (string) ($item['snapshot_key'] ?? '')) + || !is_array($upload) || array_is_list($upload) + || !is_array($upload['part_urls'] ?? null) + || !array_is_list($upload['part_urls']) + || $upload['part_urls'] === []) { + self::invalidPayload('维护迁移快照导出身份无效'); + } + foreach (array_merge($upload['part_urls'], [ + $upload['complete_url'] ?? null, + $upload['abort_url'] ?? null, + $upload['verify_url'] ?? null, + ]) as $url) { + if (!is_string($url) || !str_starts_with($url, 'https://')) { + self::invalidPayload('维护迁移快照导出凭据无效'); + } + } + } + } + + private static function assertMigrationBundleRestoreBody( + mixed $value, + CloudInstanceMigration $migration + ): void { + if (!is_array($value) || array_is_list($value) + || (int) ($value['version'] ?? 0) + !== CloudMigrationBundleService::VERSION + || ($value['stage_powered_off'] ?? null) !== true + || ($value['network_suspended'] ?? null) + !== ((string) $migration->source_network_status !== 'active') + || !is_array($value['snapshots'] ?? null) + || !array_is_list($value['snapshots'])) { + self::invalidPayload('维护迁移快照恢复请求无效'); + } + $rows = CloudMigrationBundleArtifact::where( + 'migration_id', (int) $migration->id + )->where('state', 'ready')->orderBy('id')->get(); + if (count($value['snapshots']) !== count($rows)) { + self::invalidPayload('维护迁移快照恢复数量无效'); + } + foreach ($rows as $index => $row) { + $item = $value['snapshots'][$index] ?? null; + if (!is_array($item) || array_is_list($item) + || !hash_equals((string) $row->snapshot_key, + (string) ($item['snapshot_key'] ?? '')) + || !hash_equals((string) $row->name, + (string) ($item['name'] ?? '')) + || !hash_equals((string) $row->consistency, + (string) ($item['consistency'] ?? '')) + || (int) $row->object_size_bytes + !== (int) ($item['source_size_bytes'] ?? -1) + || (int) $row->virtual_size_bytes + !== (int) ($item['virtual_size_bytes'] ?? -1) + || !hash_equals((string) $row->sha256, + (string) ($item['source_sha256'] ?? '')) + || (int) $row->snapshot_created_at + !== (int) ($item['created_at'] ?? -1) + || !is_string($item['source_url'] ?? null) + || !str_starts_with($item['source_url'], 'https://')) { + self::invalidPayload('维护迁移快照恢复身份无效'); + } + } + } + private static function firewallAction( CloudJob $job, array $payload, diff --git a/web/app/Services/Cloud/CloudPanelUsageService.php b/web/app/Services/Cloud/CloudPanelUsageService.php index 509d5ab0..c9360d24 100644 --- a/web/app/Services/Cloud/CloudPanelUsageService.php +++ b/web/app/Services/Cloud/CloudPanelUsageService.php @@ -13,6 +13,8 @@ use App\Models\CloudResourceReservation; */ final class CloudPanelUsageService { + private const GIB = 1073741824; + /** * The panel deliberately does not persist an Agent management address in * cloud_hosts. Operators must put a management IPv4 in the public-pool @@ -49,10 +51,34 @@ final class CloudPanelUsageService } } - $instances = CloudInstance::where('host_id', $hostId) - ->where('status', '<>', 'deleted') - ->get(['id', 'spec_json']); + $instances = CloudInstance::where('cloud_instances.host_id', $hostId) + ->where('cloud_instances.status', '<>', 'deleted') + ->leftJoin( + 'cloud_instance_snapshots as capacity_snapshots', + static function ($join): void { + $join->on( + 'capacity_snapshots.instance_id', + '=', + 'cloud_instances.id' + )->whereColumn( + 'capacity_snapshots.host_id', + 'cloud_instances.host_id' + )->whereIn('capacity_snapshots.state', [ + 'creating', 'ready', 'restoring', + 'deleting', 'uncertain', + ])->whereNotNull('capacity_snapshots.size_bytes'); + } + )->groupBy('cloud_instances.id', 'cloud_instances.spec_json') + ->select([ + 'cloud_instances.id', + 'cloud_instances.spec_json', + ])->selectRaw( + 'COALESCE(SUM(capacity_snapshots.size_bytes), 0)' + . ' AS snapshot_bytes' + )->get(); + $snapshotBytes = 0; foreach ($instances as $instance) { + $snapshotBytes += max(0, (int) ($instance->snapshot_bytes ?? 0)); if (isset($coveredInstances[(int) $instance->id])) { continue; } @@ -66,6 +92,8 @@ final class CloudPanelUsageService $usage['memory']['allocated'] += $spec['memory']; $usage['disk']['allocated'] += $spec['disk']; } + $usage['disk']['allocated'] += + (int) ceil($snapshotBytes / self::GIB); $usage['cpu_sum'] = $usage['cpu']['allocated'] + $usage['cpu']['reserved']; $usage['memory_sum'] = $usage['memory']['allocated'] + $usage['memory']['reserved']; $usage['disk_sum'] = $usage['disk']['allocated'] + $usage['disk']['reserved']; diff --git a/web/app/Services/Cloud/CloudPlacementService.php b/web/app/Services/Cloud/CloudPlacementService.php index 56b63943..c179c581 100644 --- a/web/app/Services/Cloud/CloudPlacementService.php +++ b/web/app/Services/Cloud/CloudPlacementService.php @@ -443,14 +443,19 @@ final class CloudPlacementService /** Same-pool, different-host candidates for a cold maintenance migration. */ public static function maintenanceMigrationTargets(CloudInstance $instance, - array $resources): array + array $resources, int $additionalDiskGib = 0): array { $poolId = (int) ($instance->resource_pool_id ?? 0); if ($poolId < 1) { return []; } + $capacityResources = self::upgradeResources($resources); + $capacityResources['disk_gib'] += max(0, $additionalDiskGib); return array_values(array_filter( - self::backupRestoreTargets((int) $instance->host_id, $resources), + self::backupRestoreTargets( + (int) $instance->host_id, + $capacityResources + ), static fn (array $target): bool => (int) ($target['id'] ?? 0) !== (int) $instance->host_id && (int) ($target['resource_pool_id'] ?? 0) === $poolId @@ -466,9 +471,12 @@ final class CloudPlacementService */ public static function reserveBackupRestore(CloudBackupRestore $restore, int $userId, array $resources, - ?int $requiredResourcePoolId = null): array + ?int $requiredResourcePoolId = null, + int $additionalDiskGib = 0): array { $resources = self::upgradeResources($resources); + $capacityResources = $resources; + $capacityResources['disk_gib'] += max(0, $additionalDiskGib); $hostId = (int) $restore->target_host_id; $host = CloudHost::where('id', $hostId)->lockForUpdate()->first(); $memberships = CloudHostResourcePoolMembership::where('host_id', $hostId) @@ -531,7 +539,7 @@ final class CloudPlacementService if (!self::hasCapacity( $policy, CloudPanelUsageService::forHost($hostId, $reservations), - $resources + $capacityResources )) { throw new CloudError('目标宿主机没有足够的可用容量', 409); } @@ -581,7 +589,10 @@ final class CloudPlacementService $reservation->public_ip_id = (int) $ip->id; $reservation->cpu_cores = $resources['cpu_cores']; $reservation->memory_mib = $resources['memory_mib']; - $reservation->disk_gib = $resources['disk_gib']; + // A maintenance restore reserves the root disk plus every transported + // snapshot volume. The finalizer normalizes this temporary claim back + // to the root disk once the snapshot catalog is atomically rebound. + $reservation->disk_gib = $capacityResources['disk_gib']; $reservation->state = 'held'; $reservation->expires_at = null; $reservation->released_at = null; diff --git a/web/public/cloud/admin-instances.js b/web/public/cloud/admin-instances.js index 4fc7aa23..a72cabe6 100644 --- a/web/public/cloud/admin-instances.js +++ b/web/public/cloud/admin-instances.js @@ -951,6 +951,10 @@ function maintenanceMigrationSection(payload, vmId) { cutover: '正在切换实例账本', firewall_running: '正在恢复目标防火墙', firewall_failed: '目标防火墙恢复失败', firewall_uncertain: '目标防火墙结果待核对', + activation_running: '防火墙已就绪,正在恢复目标电源状态', + activation_failed: '目标电源状态恢复失败', + activation_uncertain: '目标电源状态恢复结果待核对', + activated: '目标状态已恢复', rollback_running: '目标恢复失败,正在恢复源实例', rollback_failed: '源实例自动恢复失败', rollback_uncertain: '源实例恢复结果待核对', @@ -970,6 +974,9 @@ function maintenanceMigrationSection(payload, vmId) { if (['firewall_failed', 'firewall_uncertain'].includes(migration.state)) { body.append(el('p', 'alert alert-danger fs-13', '目标实例已恢复,但原用户防火墙尚未确认生效;源副本不会自动删除,也不能开始新的迁移。请先在任务中心核对防火墙任务。')); + } else if (['activation_failed', 'activation_uncertain'].includes(migration.state)) { + body.append(el('p', 'alert alert-danger fs-13', + '目标防火墙已经完成,但原运行或关机状态尚未确认恢复;源副本不会自动删除。请先在任务中心核对目标状态恢复任务,系统不会自动重放。')); } else if (['rollback_failed', 'rollback_uncertain'].includes(migration.state)) { body.append(el('p', 'alert alert-danger fs-13', '目标恢复失败,源实例自动恢复尚未确认成功。请先在任务中心核对源实例恢复任务,系统不会自动重放。')); @@ -1037,7 +1044,7 @@ function maintenanceMigrationSection(payload, vmId) { feedback.dataset.migrationFeedback = ''; feedback.hidden = true; const blocked = payload?.can_migrate !== true; for (const control of [host, mode, submit, confirm]) control.disabled = blocked; - if (blocked && (!migration || !['backing_up', 'restoring', 'cutover', 'firewall_running', 'firewall_failed', 'firewall_uncertain', 'rollback_running', 'rollback_failed', 'rollback_uncertain', 'cleanup_running', 'source_retained', 'cleanup_uncertain', 'uncertain'].includes(migration.state))) { + if (blocked && (!migration || !['backing_up', 'restoring', 'cutover', 'firewall_running', 'firewall_failed', 'firewall_uncertain', 'activation_running', 'activation_failed', 'activation_uncertain', 'activated', 'rollback_running', 'rollback_failed', 'rollback_uncertain', 'cleanup_running', 'source_retained', 'cleanup_uncertain', 'uncertain'].includes(migration.state))) { feedback.hidden = false; feedback.className = 'text-warning fs-13 mb-0 mt-2'; feedback.textContent = targets.length ? text(payload?.create_block_reason, '当前实例不能开始维护迁移。') diff --git a/web/resources/views/venix/layout_assets.tpl b/web/resources/views/venix/layout_assets.tpl index 35279a64..65a85c60 100644 --- a/web/resources/views/venix/layout_assets.tpl +++ b/web/resources/views/venix/layout_assets.tpl @@ -1,2 +1,2 @@ {* Shared Venix CSS/JS cache key. Increment whenever a shared browser asset changes. *} -{assign var=venix_asset_version value='2026102505' scope='global'} +{assign var=venix_asset_version value='2026102506' scope='global'} diff --git a/web/sql/cloud.sql b/web/sql/cloud.sql index bbe8e247..1d497d69 100644 --- a/web/sql/cloud.sql +++ b/web/sql/cloud.sql @@ -528,6 +528,7 @@ CREATE TABLE cloud_instance_backups ( user_id BIGINT UNSIGNED NOT NULL, source_uuid CHAR(36) CHARACTER SET ascii COLLATE ascii_bin NOT NULL, name VARCHAR(80) NOT NULL, + purpose VARCHAR(24) CHARACTER SET ascii COLLATE ascii_bin NOT NULL DEFAULT 'portable', state VARCHAR(16) CHARACTER SET ascii COLLATE ascii_bin NOT NULL DEFAULT 'creating', consistency VARCHAR(16) CHARACTER SET ascii COLLATE ascii_bin NOT NULL, object_key VARCHAR(512) CHARACTER SET ascii COLLATE ascii_bin NOT NULL, @@ -596,11 +597,15 @@ CREATE TABLE cloud_instance_migrations ( backup_job_id BIGINT UNSIGNED NOT NULL, restore_job_id BIGINT UNSIGNED DEFAULT NULL, firewall_job_id BIGINT UNSIGNED DEFAULT NULL, + activate_job_id BIGINT UNSIGNED DEFAULT NULL, rollback_job_id BIGINT UNSIGNED DEFAULT NULL, cleanup_job_id BIGINT UNSIGNED DEFAULT NULL, cleanup_mode VARCHAR(16) CHARACTER SET ascii COLLATE ascii_bin NOT NULL, state VARCHAR(24) CHARACTER SET ascii COLLATE ascii_bin NOT NULL DEFAULT 'backing_up', source_was_running TINYINT(1) DEFAULT NULL, + source_autostart_enabled TINYINT(1) DEFAULT NULL, + source_service_status VARCHAR(24) CHARACTER SET ascii COLLATE ascii_bin NOT NULL, + source_network_status VARCHAR(24) CHARACTER SET ascii COLLATE ascii_bin NOT NULL, error_code VARCHAR(64) CHARACTER SET ascii COLLATE ascii_bin NOT NULL DEFAULT '', cutover_at BIGINT UNSIGNED DEFAULT NULL, completed_at BIGINT UNSIGNED DEFAULT NULL, @@ -612,12 +617,36 @@ CREATE TABLE cloud_instance_migrations ( UNIQUE KEY cloud_instance_migration_backup_job (backup_job_id), UNIQUE KEY cloud_instance_migration_restore_job (restore_job_id), UNIQUE KEY cloud_instance_migration_firewall_job (firewall_job_id), + UNIQUE KEY cloud_instance_migration_activate_job (activate_job_id), UNIQUE KEY cloud_instance_migration_rollback_job (rollback_job_id), UNIQUE KEY cloud_instance_migration_cleanup_job (cleanup_job_id), KEY cloud_instance_migration_instance (instance_id, state, id), KEY cloud_instance_migration_source (source_host_id, source_uuid), KEY cloud_instance_migration_target (target_host_id, target_uuid) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; +-- Immutable object-store artifacts that exist only for one maintenance +-- migration. Ordinary cloud backups remain a single system.qcow2 object and +-- never create rows in this table. +CREATE TABLE cloud_migration_bundle_artifacts ( + id BIGINT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY, + migration_id BIGINT UNSIGNED NOT NULL, + source_snapshot_id BIGINT UNSIGNED NOT NULL, + snapshot_key CHAR(32) CHARACTER SET ascii COLLATE ascii_bin NOT NULL, + name VARCHAR(64) NOT NULL, + consistency VARCHAR(16) CHARACTER SET ascii COLLATE ascii_bin NOT NULL, + object_key VARCHAR(512) CHARACTER SET ascii COLLATE ascii_bin NOT NULL, + object_size_bytes BIGINT UNSIGNED NOT NULL DEFAULT 0, + virtual_size_bytes BIGINT UNSIGNED NOT NULL DEFAULT 0, + sha256 CHAR(64) CHARACTER SET ascii COLLATE ascii_bin NOT NULL DEFAULT '', + state VARCHAR(16) CHARACTER SET ascii COLLATE ascii_bin NOT NULL DEFAULT 'creating', + snapshot_created_at BIGINT UNSIGNED NOT NULL, + created_at BIGINT UNSIGNED NOT NULL, + updated_at BIGINT UNSIGNED NOT NULL, + UNIQUE KEY cloud_migration_bundle_snapshot (migration_id, snapshot_key), + UNIQUE KEY cloud_migration_bundle_source_snapshot (migration_id, source_snapshot_id), + UNIQUE KEY cloud_migration_bundle_object (object_key), + KEY cloud_migration_bundle_state (migration_id, state, id) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; -- Last-known Agent power state for fast instance-list first paint. This is -- an observation cache only; cloud_instances remains the lifecycle source of -- truth and the host Agent remains authoritative for the current power state. diff --git a/web/tests/AdminHostNetworkPoolsContractTest.php b/web/tests/AdminHostNetworkPoolsContractTest.php index 6abbbbaa..74dfd79e 100644 --- a/web/tests/AdminHostNetworkPoolsContractTest.php +++ b/web/tests/AdminHostNetworkPoolsContractTest.php @@ -111,7 +111,9 @@ $assert(str_contains($service, 'assertNoOverlap(') && !str_contains($service, '->delete()'), 'Pool limits, overlap checks, occupied runtime locks and soft lifecycle states are required.'); $assert(str_contains($service, 'materializeIpLedger(') - && str_contains((string) $usage, "CloudInstance::where('host_id', \$hostId)") + && str_contains((string) $usage, + "CloudInstance::where('cloud_instances.host_id', \$hostId)") + && str_contains((string) $usage, 'capacity_snapshots') && str_contains((string) $usage, "'unknown'") && str_contains((string) $usage, 'spec_json') && str_contains((string) $usage, 'managementAddressFromValue'), diff --git a/web/tests/CloudFreshSchemaContractTest.php b/web/tests/CloudFreshSchemaContractTest.php index 866d9de8..1953d27f 100644 --- a/web/tests/CloudFreshSchemaContractTest.php +++ b/web/tests/CloudFreshSchemaContractTest.php @@ -22,7 +22,7 @@ $assert(stripos($schema, 'compatibility import') === false $assert(!preg_match('/\bengine_[A-Za-z0-9_]*\b/', $schema), 'cloud.sql retained an engine-prefixed identifier.'); $assert(str_contains($setup, - "SCHEMA_VERSION = 'go-agent-wss-v5-maintenance-migrations'") + "SCHEMA_VERSION = 'go-agent-wss-v6-migration-bundles'") && !str_contains($setup, "SCHEMA_VERSION = 'go-v1'"), 'Outbound Agent schema must reject the incompatible earlier Go database.'); @@ -30,7 +30,8 @@ foreach (['cloud_host_agents', 'cloud_templates', 'cloud_hosts', 'cloud_instance 'cloud_orders', 'cloud_invoices', 'cloud_jobs', 'cloud_agent_completion_events', 'cloud_console_tickets', 'cloud_rescue_template_imports', 'cloud_instance_backups', - 'cloud_backup_restores', 'cloud_instance_migrations'] as $table) { + 'cloud_backup_restores', 'cloud_instance_migrations', + 'cloud_migration_bundle_artifacts'] as $table) { $assert(str_contains($schema, 'CREATE TABLE ' . $table . ' ('), 'Missing fresh-install table: ' . $table); } @@ -72,6 +73,12 @@ $assert(str_contains($schema, "'cloud_job_backup', 'cloud_job_backup_restore'"), 'Fresh schema is missing portable-backup catalog or restore reservation constraints.'); $assert(str_contains($schema, 'CREATE TABLE cloud_instance_migrations (') + && str_contains($schema, 'purpose VARCHAR(24) CHARACTER SET ascii COLLATE ascii_bin NOT NULL DEFAULT \'portable\'') + && str_contains($schema, 'CREATE TABLE cloud_migration_bundle_artifacts (') + && str_contains($schema, 'activate_job_id BIGINT UNSIGNED DEFAULT NULL') + && str_contains($schema, 'source_autostart_enabled TINYINT(1) DEFAULT NULL') + && str_contains($schema, 'source_service_status VARCHAR(24)') + && str_contains($schema, 'source_network_status VARCHAR(24)') && str_contains($schema, 'firewall_job_id BIGINT UNSIGNED DEFAULT NULL') && str_contains($schema, 'rollback_job_id BIGINT UNSIGNED DEFAULT NULL') && str_contains($schema, @@ -83,6 +90,8 @@ $assert(str_contains($schema, 'CREATE TABLE cloud_instance_migrations (') && str_contains($schema, 'KEY cloud_instance_migration_instance (instance_id, state, id)') && str_contains($setup, "'cloud_instance_migrations'") + && str_contains($setup, "'cloud_migration_bundle_artifacts'") + && str_contains($setup, "'cloud_instance_migration_activate_job'") && str_contains($setup, "'cloud_instance_migration_rollback_job'") && str_contains($setup, "'cloud_instance_migration_cleanup_job'"), 'Fresh schema is missing the maintenance-migration orchestration ledger.'); diff --git a/web/tests/CloudMaintenanceMigrationFinalizerTest.php b/web/tests/CloudMaintenanceMigrationFinalizerTest.php index f589e3d3..dd22b9ce 100644 --- a/web/tests/CloudMaintenanceMigrationFinalizerTest.php +++ b/web/tests/CloudMaintenanceMigrationFinalizerTest.php @@ -117,11 +117,15 @@ $schema->create('cloud_instance_migrations', $table->integer('backup_job_id'); $table->integer('restore_job_id'); $table->integer('firewall_job_id')->nullable(); + $table->integer('activate_job_id')->nullable(); $table->integer('rollback_job_id')->nullable(); $table->integer('cleanup_job_id')->nullable(); $table->string('cleanup_mode'); $table->string('state'); $table->boolean('source_was_running')->nullable(); + $table->boolean('source_autostart_enabled')->nullable(); + $table->string('source_service_status'); + $table->string('source_network_status'); $table->string('error_code'); $table->integer('cutover_at')->nullable(); $table->integer('completed_at')->nullable(); @@ -161,7 +165,7 @@ $capsule->table('cloud_resource_reservations')->insert([ 'resource_pool_id' => 101, 'host_id' => 1, 'user_id' => 9, 'instance_id' => 11, 'network_attachment_id' => 81, 'public_ip_pool_id' => 91, 'public_ip_id' => 101, - 'cpu_cores' => 2, 'memory_mib' => 2048, 'disk_gib' => 40, + 'cpu_cores' => 2, 'memory_mib' => 2048, 'disk_gib' => 90, 'state' => 'committed', 'expires_at' => null, 'released_at' => null, 'release_reason' => '', 'created_at' => $now - 1000, 'updated_at' => $now, @@ -202,10 +206,13 @@ $capsule->table('cloud_instance_migrations')->insert([ 'source_reservation_id' => 61, 'target_host_id' => 2, 'target_uuid' => $targetUuid, 'target_reservation_id' => 62, 'backup_job_id' => 91, 'restore_job_id' => 92, - 'firewall_job_id' => null, 'rollback_job_id' => null, + 'firewall_job_id' => null, 'activate_job_id' => null, + 'rollback_job_id' => null, 'cleanup_job_id' => 93, 'cleanup_mode' => 'delete', 'state' => 'cleanup_running', - 'source_was_running' => true, 'error_code' => '', + 'source_was_running' => true, 'source_autostart_enabled' => true, + 'source_service_status' => 'active', + 'source_network_status' => 'active', 'error_code' => '', 'cutover_at' => $now - 10, 'completed_at' => null, 'created_at' => $now - 200, 'updated_at' => $now, ]); @@ -238,6 +245,7 @@ $assert((int) $instance->reservation_id === 61 && (int) $source->resource_pool_id === 102 && (int) $source->public_ip_id === 102 && (int) $source->network_attachment_id === 82 + && (int) $source->disk_gib === 40 && (string) $source->state === 'committed', 'The original paid reservation did not take ownership of target placement.'); $assert((string) $sourceIp->state === 'available' @@ -282,9 +290,13 @@ $capsule->table('cloud_instance_migrations')->insert([ 'source_reservation_id' => 63, 'target_host_id' => 2, 'target_uuid' => $targetUuid, 'target_reservation_id' => 64, 'backup_job_id' => 191, 'restore_job_id' => 192, - 'firewall_job_id' => null, 'rollback_job_id' => 200, + 'firewall_job_id' => null, 'activate_job_id' => null, + 'rollback_job_id' => 200, 'cleanup_job_id' => null, 'cleanup_mode' => 'delete', 'state' => 'rollback_running', 'source_was_running' => true, + 'source_autostart_enabled' => true, + 'source_service_status' => 'active', + 'source_network_status' => 'active', 'error_code' => 'migration_restore_failed', 'cutover_at' => null, 'completed_at' => null, 'created_at' => $now - 200, 'updated_at' => $now, @@ -311,4 +323,60 @@ $assert((string) $rolledBack->state === 'rolled_back' && (int) CloudInstance::findOrFail(12)->host_id === 1, 'A failed target restore did not confirm recovery of the original source service.'); +$capsule->table('cloud_instances')->insert([ + 'id' => 13, 'uuid' => $targetUuid, 'host_id' => 2, 'user_id' => 9, + 'order_id' => 23, 'plan_id' => 31, 'offer_id' => 41, + 'resource_pool_id' => 102, 'reservation_id' => 65, + 'current_period_id' => 73, 'expires_at' => $now + 86400, + 'renewal_price' => '19.95', 'renewal_days' => 30, + 'renewal_traffic_bytes' => 2_000_000, + 'traffic_quota_bytes' => 1_500_000, + 'lifetime_download_bytes' => 100, 'lifetime_upload_bytes' => 50, + 'name' => 'powered-off-service', 'status' => 'active', + 'service_status' => 'suspended_expired', + 'network_status' => 'blocked_expired', + 'spec_json' => '{"guest_hostname":"retained.example"}', + 'created_at' => $now - 1000, 'updated_at' => $now, +]); +$capsule->table('cloud_instance_migrations')->insert([ + 'id' => 3, 'migration_key' => str_repeat('1', 32), + 'instance_id' => 13, 'backup_id' => 43, 'restore_id' => 53, + 'source_host_id' => 1, 'source_uuid' => $sourceUuid, + 'source_reservation_id' => 66, 'target_host_id' => 2, + 'target_uuid' => $targetUuid, 'target_reservation_id' => 65, + 'backup_job_id' => 291, 'restore_job_id' => 292, + 'firewall_job_id' => null, 'activate_job_id' => 300, + 'rollback_job_id' => null, 'cleanup_job_id' => null, + 'cleanup_mode' => 'retain', 'state' => 'activation_running', + 'source_was_running' => false, 'source_autostart_enabled' => true, + 'source_service_status' => 'suspended_expired', + 'source_network_status' => 'blocked_expired', 'error_code' => '', + 'cutover_at' => $now - 10, 'completed_at' => null, + 'created_at' => $now - 200, 'updated_at' => $now, +]); +$activationJob = new CloudJob(); +$activationJob->id = 300; +$activationJob->instance_id = 13; +$activationJob->host_id = 2; +$activationJob->user_id = 9; +$finishActivation = new ReflectionMethod( + CloudInstanceMigrationService::class, + 'finishActivationLocked' +); +$finishActivation->invoke( + null, + CloudInstanceMigration::findOrFail(3), + $activationJob, + 'succeeded', + ['uuid' => $targetUuid, 'state' => 'shutoff'] +); +$poweredOff = CloudInstance::findOrFail(13); +$activated = CloudInstanceMigration::findOrFail(3); +$assert((string) $activated->state === 'source_retained' + && (string) $poweredOff->service_status === 'suspended_expired' + && (string) $poweredOff->network_status === 'blocked_expired' + && str_contains((string) $poweredOff->spec_json, + 'retained.example'), + 'Maintenance migration did not retain powered-off, disconnected, or hostname state.'); + echo "Maintenance migration logical-service and placement finalizer contracts passed.\n"; diff --git a/web/tests/CloudMigrationBundleBehaviorTest.php b/web/tests/CloudMigrationBundleBehaviorTest.php new file mode 100644 index 00000000..0264f50d --- /dev/null +++ b/web/tests/CloudMigrationBundleBehaviorTest.php @@ -0,0 +1,172 @@ +addConnection([ + 'driver' => 'sqlite', 'database' => ':memory:', +], 'default'); +$capsule->setAsGlobal(); +$capsule->bootEloquent(); +$schema = $capsule->schema('default'); + +$schema->create('cloud_instances', static function (Blueprint $table): void { + $table->increments('id'); + $table->integer('host_id'); + $table->integer('user_id'); +}); +$schema->create('cloud_instance_snapshots', + static function (Blueprint $table): void { + $table->increments('id'); + $table->string('snapshot_key')->unique(); + $table->integer('instance_id'); + $table->integer('user_id'); + $table->integer('host_id'); + $table->string('name'); + $table->string('state'); + $table->string('consistency'); + $table->integer('size_bytes')->nullable(); + $table->integer('created_at'); + $table->integer('updated_at'); + }); +$schema->create('cloud_instance_migrations', + static function (Blueprint $table): void { + $table->increments('id'); + $table->integer('instance_id'); + $table->integer('source_host_id'); + $table->integer('target_host_id'); + }); +$schema->create('cloud_migration_bundle_artifacts', + static function (Blueprint $table): void { + $table->increments('id'); + $table->integer('migration_id'); + $table->integer('source_snapshot_id'); + $table->string('snapshot_key'); + $table->string('name'); + $table->string('consistency'); + $table->string('object_key'); + $table->integer('object_size_bytes')->default(0); + $table->integer('virtual_size_bytes')->default(0); + $table->string('sha256')->default(''); + $table->string('state'); + $table->integer('snapshot_created_at'); + $table->integer('created_at'); + $table->integer('updated_at'); + }); + +$now = time(); +$instance = new CloudInstance(); +$instance->id = 10; +$instance->host_id = 1; +$instance->user_id = 90; +$instance->save(); +$snapshots = []; +foreach ([ + [11, str_repeat('a', 32), 'before-upgrade', 'filesystem', 20 * 1073741824], + [12, str_repeat('b', 32), 'after-upgrade', 'crash', 30 * 1073741824], +] as [$id, $key, $name, $consistency, $bytes]) { + $snapshot = new CloudInstanceSnapshot(); + $snapshot->id = $id; + $snapshot->snapshot_key = $key; + $snapshot->instance_id = 10; + $snapshot->user_id = 90; + $snapshot->host_id = 1; + $snapshot->name = $name; + $snapshot->state = 'ready'; + $snapshot->consistency = $consistency; + $snapshot->size_bytes = $bytes; + $snapshot->created_at = $now - $id; + $snapshot->updated_at = $now; + $snapshot->save(); + $snapshots[] = $snapshot; +} + +$frozen = CloudMigrationBundleService::freezeSnapshotsLocked($instance); +$assert(count($frozen) === 2 + && CloudMigrationBundleService::additionalDiskGib($frozen) === 50, + 'MigrationBundle did not freeze and reserve every ready snapshot.'); + +$migration = new CloudInstanceMigration(); +$migration->id = 20; +$migration->instance_id = 10; +$migration->source_host_id = 1; +$migration->target_host_id = 2; +$migration->save(); +$prepared = []; +foreach ($snapshots as $snapshot) { + $prepared[] = [ + 'snapshot' => $snapshot, + 'object_key' => 'virt/vm-migrations/bundle/snapshots/' + . $snapshot->snapshot_key . '.qcow2', + 'plan' => ['upload_id' => 'upload-' . $snapshot->id], + ]; +} +CloudMigrationBundleService::persistArtifactsLocked($migration, $prepared); + +$receipts = []; +foreach ($snapshots as $index => $snapshot) { + $receipts[] = [ + 'snapshot_key' => (string) $snapshot->snapshot_key, + 'name' => (string) $snapshot->name, + 'consistency' => (string) $snapshot->consistency, + 'size_bytes' => 1000 + $index, + 'virtual_size_bytes' => (int) $snapshot->size_bytes, + 'sha256' => str_repeat((string) ($index + 1), 64), + 'created_at' => (int) $snapshot->created_at - 30 - $index, + ]; +} +CloudMigrationBundleService::finishExportLocked($migration, [ + 'migration_bundle_version' => CloudMigrationBundleService::VERSION, + 'snapshot_artifacts' => $receipts, +]); +$assert(CloudMigrationBundleArtifact::where('migration_id', 20) + ->where('state', 'ready')->count() === 2 + && (int) CloudMigrationBundleArtifact::where('migration_id', 20) + ->orderBy('id')->first()->snapshot_created_at + === (int) $receipts[0]['created_at'], + 'MigrationBundle did not seal every exported snapshot receipt.'); + +try { + CloudMigrationBundleService::finishRestoreLocked($migration, [ + 'status' => 'running', + 'staged_powered_off' => false, + 'migration_bundle_version' => CloudMigrationBundleService::VERSION, + 'snapshot_artifacts' => $receipts, + ]); + throw new RuntimeException( + 'MigrationBundle accepted a target before powered-off staging was proven.' + ); +} catch (CloudError $error) { + $assert($error->httpStatus === 502, + 'MigrationBundle rejected unsafe staging with the wrong status.'); +} +$assert(CloudInstanceSnapshot::where('host_id', 1)->count() === 2, + 'Snapshot ownership moved before powered-off target staging was proven.'); + +CloudMigrationBundleService::finishRestoreLocked($migration, [ + 'status' => 'shutoff', + 'staged_powered_off' => true, + 'migration_bundle_version' => CloudMigrationBundleService::VERSION, + 'snapshot_artifacts' => $receipts, +]); +$assert(CloudInstanceSnapshot::where('host_id', 2)->count() === 2 + && CloudMigrationBundleArtifact::where('migration_id', 20) + ->where('state', 'restored')->count() === 2, + 'MigrationBundle did not atomically rebind the complete snapshot catalog.'); + +echo "Maintenance MigrationBundle snapshot transport behavior passed.\n"; diff --git a/web/tests/CloudMigrationBundleCapacityTest.php b/web/tests/CloudMigrationBundleCapacityTest.php new file mode 100644 index 00000000..1a40f207 --- /dev/null +++ b/web/tests/CloudMigrationBundleCapacityTest.php @@ -0,0 +1,144 @@ +addConnection([ + 'driver' => 'sqlite', 'database' => ':memory:', +], 'default'); +$capsule->setAsGlobal(); +$capsule->bootEloquent(); +$schema = $capsule->schema('default'); + +$schema->create('cloud_instances', static function (Blueprint $table): void { + $table->increments('id'); + $table->integer('host_id'); + $table->string('status'); + $table->text('spec_json'); +}); +$schema->create('cloud_resource_reservations', + static function (Blueprint $table): void { + $table->increments('id'); + $table->integer('host_id'); + $table->integer('instance_id')->nullable(); + $table->integer('cpu_cores'); + $table->integer('memory_mib'); + $table->integer('disk_gib'); + $table->string('state'); + }); +$schema->create('cloud_instance_snapshots', + static function (Blueprint $table): void { + $table->increments('id'); + $table->integer('instance_id'); + $table->integer('host_id'); + $table->string('state'); + $table->integer('size_bytes')->nullable(); + }); +$schema->create('cloud_instance_migrations', + static function (Blueprint $table): void { + $table->increments('id'); + $table->integer('source_host_id'); + $table->string('state'); + }); +$schema->create('cloud_migration_bundle_artifacts', + static function (Blueprint $table): void { + $table->increments('id'); + $table->integer('migration_id'); + $table->integer('virtual_size_bytes'); + $table->string('state'); + }); + +$gib = 1073741824; +$capsule->table('cloud_instances')->insert([ + 'id' => 10, + 'host_id' => 1, + 'status' => 'active', + 'spec_json' => json_encode([ + 'vcpu' => 2, 'memory_mib' => 2048, 'disk_gib' => 40, + ], JSON_THROW_ON_ERROR), +]); +$capsule->table('cloud_resource_reservations')->insert([ + [ + 'id' => 1, 'host_id' => 1, 'instance_id' => 10, + 'cpu_cores' => 2, 'memory_mib' => 2048, 'disk_gib' => 40, + 'state' => 'committed', + ], + [ + 'id' => 2, 'host_id' => 2, 'instance_id' => 10, + 'cpu_cores' => 2, 'memory_mib' => 2048, 'disk_gib' => 90, + 'state' => 'held', + ], +]); +$capsule->table('cloud_instance_snapshots')->insert([ + [ + 'id' => 11, 'instance_id' => 10, 'host_id' => 1, + 'state' => 'ready', 'size_bytes' => 20 * $gib, + ], + [ + 'id' => 12, 'instance_id' => 10, 'host_id' => 1, + 'state' => 'ready', 'size_bytes' => 30 * $gib, + ], +]); +$capsule->table('cloud_instance_migrations')->insert([ + 'id' => 20, 'source_host_id' => 1, 'state' => 'restoring', +]); +$capsule->table('cloud_migration_bundle_artifacts')->insert([ + [ + 'id' => 21, 'migration_id' => 20, + 'virtual_size_bytes' => 20 * $gib, 'state' => 'ready', + ], + [ + 'id' => 22, 'migration_id' => 20, + 'virtual_size_bytes' => 30 * $gib, 'state' => 'ready', + ], +]); + +$source = CloudPanelUsageService::forHost(1); +$target = CloudPanelUsageService::forHost(2); +$assert($source['disk_sum'] === 90, + 'Source capacity did not include its ready snapshot volumes.'); +$assert($target['disk_sum'] === 90, + 'Target capacity did not retain the complete MigrationBundle claim.'); + +// After target staging, the logical snapshot catalog moves to the target and +// the temporary target reservation returns to the root-disk size. The source +// copies still exist until their cleanup has been proven, so the source +// reservation temporarily retains the transported snapshot capacity. +$capsule->table('cloud_instance_snapshots')->update(['host_id' => 2]); +$capsule->table('cloud_migration_bundle_artifacts') + ->update(['state' => 'restored']); +$capsule->table('cloud_instance_migrations') + ->update(['state' => 'cutover']); +$capsule->table('cloud_resource_reservations')->where('id', 2) + ->update(['disk_gib' => 40, 'state' => 'committed']); +$capsule->table('cloud_resource_reservations')->where('id', 1) + ->update(['disk_gib' => 90]); +$capsule->table('cloud_instances')->where('id', 10) + ->update(['host_id' => 2]); + +$source = CloudPanelUsageService::forHost(1); +$target = CloudPanelUsageService::forHost(2); +$assert($source['disk_sum'] === 90, + 'Source capacity was released before duplicate snapshot cleanup.'); +$assert($target['disk_sum'] === 90, + 'Target snapshot volumes were double-counted or omitted after cutover.'); + +$capsule->table('cloud_instance_migrations') + ->update(['state' => 'completed']); +$capsule->table('cloud_resource_reservations')->where('id', 1) + ->update(['state' => 'released']); +$source = CloudPanelUsageService::forHost(1); +$assert($source['disk_sum'] === 0, + 'Source snapshot capacity remained reserved after proven cleanup.'); + +echo "Maintenance MigrationBundle capacity accounting passed.\n"; diff --git a/web/tests/CloudOperationRecoveryMatrixTest.php b/web/tests/CloudOperationRecoveryMatrixTest.php index e923fe6e..a67fa73b 100644 --- a/web/tests/CloudOperationRecoveryMatrixTest.php +++ b/web/tests/CloudOperationRecoveryMatrixTest.php @@ -184,6 +184,7 @@ namespace { $table->integer('user_id'); $table->string('source_uuid'); $table->string('name'); + $table->string('purpose')->default('portable'); $table->string('state'); $table->string('consistency'); $table->string('object_key'); @@ -235,11 +236,15 @@ namespace { $table->integer('backup_job_id'); $table->integer('restore_job_id')->nullable(); $table->integer('firewall_job_id')->nullable(); + $table->integer('activate_job_id')->nullable(); $table->integer('rollback_job_id')->nullable(); $table->integer('cleanup_job_id')->nullable(); $table->string('cleanup_mode'); $table->string('state'); $table->boolean('source_was_running')->nullable(); + $table->boolean('source_autostart_enabled')->nullable(); + $table->string('source_service_status')->default('active'); + $table->string('source_network_status')->default('active'); $table->string('error_code')->default(''); $table->integer('cutover_at')->nullable(); $table->integer('completed_at')->nullable(); @@ -692,6 +697,24 @@ namespace { ], $instanceId ); + $migrationTargetUuid = '123e4567-e89b-42d3-a456-426614174099'; + $migrationActivate = $insertJob( + 'instance_migration_activate', + static fn (int $id, string $operationId): array => [ + 'method' => 'POST', + 'request_target' => '/v1/vms/' + . rawurlencode($migrationTargetUuid) . '/service/resume', + 'body' => [ + 'operation_id' => $operationId, + 'reason' => 'maintenance_migration_activate', + 'start_if_suspended' => true, + ], + 'owner_id' => $ownerId, + 'can_manage' => true, + 'maintenance_migration_id' => 1, + ], + $instanceId + ); Capsule::table('cloud_instance_migrations')->insert([ 'id' => 1, 'migration_key' => str_repeat('e', 32), @@ -701,17 +724,21 @@ namespace { 'source_host_id' => $hostId, 'source_uuid' => $instanceUuid, 'source_reservation_id' => 9003, - 'target_host_id' => $hostId + 1, - 'target_uuid' => '123e4567-e89b-42d3-a456-426614174099', + 'target_host_id' => $hostId, + 'target_uuid' => $migrationTargetUuid, 'target_reservation_id' => 9004, 'backup_job_id' => 9005, 'restore_job_id' => 9006, 'firewall_job_id' => null, + 'activate_job_id' => $migrationActivate['id'], 'rollback_job_id' => $migrationRollback['id'], 'cleanup_job_id' => $migrationCleanup['id'], 'cleanup_mode' => 'delete', 'state' => 'cleanup_uncertain', 'source_was_running' => true, + 'source_autostart_enabled' => true, + 'source_service_status' => 'active', + 'source_network_status' => 'active', 'error_code' => 'operation_effect_unknown', 'cutover_at' => $now - 60, 'completed_at' => null, @@ -734,6 +761,14 @@ namespace { $instanceUuid, ['uuid' => $instanceUuid, 'state' => 'running'] ); + $addFixture( + 'instance_migration_activate', + 'target-activate', + $migrationActivate, + 'service-resume', + $migrationTargetUuid, + ['uuid' => $migrationTargetUuid, 'state' => 'running'] + ); $instanceFixture( 'network_change', diff --git a/web/tests/CloudOperationRecoveryPolicyTest.php b/web/tests/CloudOperationRecoveryPolicyTest.php index 764955af..46b07f03 100644 --- a/web/tests/CloudOperationRecoveryPolicyTest.php +++ b/web/tests/CloudOperationRecoveryPolicyTest.php @@ -55,6 +55,7 @@ namespace { 'delete_force', 'instance_migration_cleanup', 'instance_migration_rollback', + 'instance_migration_activate', 'expiry_delete', 'network_change', 'instance_settings', @@ -77,8 +78,8 @@ namespace { ]; $assert(CloudOperationRecoveryPolicy::kinds() === $expectedKinds, 'The Agent leaf recovery policy changed without updating its contract.'); - $assert(count(CloudOperationRecoveryPolicy::kinds()) === 26, - 'The recovery policy contract must contain exactly 26 Agent leaf kinds.'); + $assert(count(CloudOperationRecoveryPolicy::kinds()) === 27, + 'The recovery policy contract must contain exactly 27 Agent leaf kinds.'); foreach ($expectedKinds as $kind) { $assert(CloudOperationRecoveryPolicy::supports($kind), 'The recovery policy omitted ' . $kind . '.'); diff --git a/web/tests/CloudPlanUpgradeServiceTest.php b/web/tests/CloudPlanUpgradeServiceTest.php index 15c3c1a7..b98f83d2 100644 --- a/web/tests/CloudPlanUpgradeServiceTest.php +++ b/web/tests/CloudPlanUpgradeServiceTest.php @@ -278,6 +278,7 @@ namespace { $table->integer('user_id'); $table->integer('host_id'); $table->string('state'); + $table->integer('size_bytes')->nullable(); }); $schema->create('cloud_invoices', static function (Blueprint $table): void { $table->increments('id'); diff --git a/web/tests/CloudPortableBackupAdminUiContractTest.php b/web/tests/CloudPortableBackupAdminUiContractTest.php index 90f125dd..bf36799d 100644 --- a/web/tests/CloudPortableBackupAdminUiContractTest.php +++ b/web/tests/CloudPortableBackupAdminUiContractTest.php @@ -44,6 +44,7 @@ $assert(str_contains($routes, 'Administrator portable backup routes must use the durable backup service and strict task receipt.'); $migration = $read('app/Services/Cloud/CloudInstanceMigrationService.php'); +$bundle = $read('app/Services/Cloud/CloudMigrationBundleService.php'); $assert(str_contains($runtime, 'function maintenanceMigrationSection(') && str_contains($runtime, 'confirm_maintenance_migration: true') && str_contains($runtime, 'confirm_source_cleanup: true') @@ -64,6 +65,19 @@ $assert(str_contains($runtime, 'function maintenanceMigrationSection(') && str_contains($migration, "['retain', 'delete']"), 'Maintenance migration must pre-reserve its target, preserve one service ledger and fence source cleanup.'); +$assert(str_contains($bundle, 'final class CloudMigrationBundleService') + && str_contains($bundle, 'freezeSnapshotsLocked(') + && str_contains($bundle, 'finishRestoreLocked(') + && str_contains($migration, "'stage_powered_off' => true") + && str_contains($migration, 'queueMaintenanceMigrationReapplyLocked(') + && str_contains($migration, 'queueActivationLocked(') + && strpos($migration, 'queueMaintenanceMigrationReapplyLocked(') + < strpos($migration, 'queueActivationLocked(') + && str_contains($migration, "\$spec['guest_hostname'] ?? \$instance->name") + && str_contains($runtime, "activation_running: '防火墙已就绪,正在恢复目标电源状态'") + && str_contains($runtime, "'activation_failed', 'activation_uncertain'"), + 'MigrationBundle must restore every snapshot offline, converge firewall first, and expose target activation failures.'); + $assert(str_contains($template, 'id="instances-backup-library"') && str_contains($template, 'id="backup-library"') && str_contains($runtime, 'async function loadBackupLibrary(') diff --git a/web/tests/CloudPortableBackupDeletionTest.php b/web/tests/CloudPortableBackupDeletionTest.php index 55a57c83..9c5290c6 100644 --- a/web/tests/CloudPortableBackupDeletionTest.php +++ b/web/tests/CloudPortableBackupDeletionTest.php @@ -69,6 +69,7 @@ namespace { $table->integer('user_id'); $table->string('source_uuid'); $table->string('name'); + $table->string('purpose')->default('portable'); $table->string('state'); $table->string('consistency'); $table->string('object_key')->unique(); diff --git a/web/tests/CloudPortableBackupFinalizerTest.php b/web/tests/CloudPortableBackupFinalizerTest.php index a94c98c3..2b52a35d 100644 --- a/web/tests/CloudPortableBackupFinalizerTest.php +++ b/web/tests/CloudPortableBackupFinalizerTest.php @@ -33,6 +33,7 @@ $schema->create('cloud_instance_backups', $table->integer('user_id'); $table->string('source_uuid'); $table->string('name'); + $table->string('purpose')->default('portable'); $table->string('state'); $table->string('consistency'); $table->string('object_key'); diff --git a/web/tests/CloudPortableBackupLibraryTest.php b/web/tests/CloudPortableBackupLibraryTest.php index 07037073..1025c710 100644 --- a/web/tests/CloudPortableBackupLibraryTest.php +++ b/web/tests/CloudPortableBackupLibraryTest.php @@ -42,6 +42,7 @@ $schema->create('cloud_instance_backups', $table->integer('user_id'); $table->string('source_uuid'); $table->string('name'); + $table->string('purpose')->default('portable'); $table->string('state'); $table->string('consistency'); $table->string('object_key'); @@ -94,6 +95,30 @@ for ($index = 1; $index <= 21; $index++) { $backup->created_at = $backup->updated_at = 1700000000 + $index; $backup->save(); } +$migrationBundle = new CloudInstanceBackup(); +$migrationBundle->backup_key = str_repeat('f', 32); +$migrationBundle->source_instance_id = (int) $source->id; +$migrationBundle->source_host_id = (int) $host->id; +$migrationBundle->user_id = 77; +$migrationBundle->source_uuid = '11111111-1111-4111-8111-111111111111'; +$migrationBundle->name = 'hidden-maintenance-bundle'; +$migrationBundle->purpose = 'migration_bundle'; +$migrationBundle->state = 'ready'; +$migrationBundle->consistency = 'crash'; +$migrationBundle->object_key = 'virt/vm-migrations/hidden/system.qcow2'; +$migrationBundle->object_size_bytes = 1024; +$migrationBundle->virtual_size_bytes = 2048; +$migrationBundle->sha256 = str_repeat('a', 64); +$migrationBundle->spec_json = '{}'; +$migrationBundle->sealed_storage_runtime = 'sealed'; +$migrationBundle->sealed_credential = null; +$migrationBundle->storage_revision = 1; +$migrationBundle->active_job_id = null; +$migrationBundle->last_operation_id = null; +$migrationBundle->error_code = ''; +$migrationBundle->ready_at = 1700000100; +$migrationBundle->created_at = $migrationBundle->updated_at = 1700000100; +$migrationBundle->save(); $readOnly = [ 'user_id' => 9, @@ -107,7 +132,7 @@ $assert(count($first['items']) === 20 && $first['pagination']['total'] === 21 && $first['pagination']['total_pages'] === 2 && $first['can_manage'] === false, - 'Global backup library pagination is not bounded or read-only.'); + 'Global backup library exposed a MigrationBundle or lost bounded pagination.'); $item = $first['items'][0]; $assert(($item['source']['deleted'] ?? null) === true && ($item['source']['status'] ?? null) === 'deleted' diff --git a/web/tests/CloudPortableBackupSameHostRestoreTest.php b/web/tests/CloudPortableBackupSameHostRestoreTest.php index 72e256e5..497e0eae 100644 --- a/web/tests/CloudPortableBackupSameHostRestoreTest.php +++ b/web/tests/CloudPortableBackupSameHostRestoreTest.php @@ -153,6 +153,14 @@ $schema->create('cloud_instances', static function (Blueprint $table): void { $table->string('status'); $table->text('spec_json'); }); +$schema->create('cloud_instance_snapshots', + static function (Blueprint $table): void { + $table->increments('id'); + $table->integer('instance_id'); + $table->integer('host_id'); + $table->string('state'); + $table->integer('size_bytes')->nullable(); + }); $capsule->table('cloud_regions')->insert([ 'id' => 1, 'name' => 'Hong Kong', 'country_code' => 'HK', diff --git a/web/tests/CloudSnapshotWorkflowTest.php b/web/tests/CloudSnapshotWorkflowTest.php index e636d7b2..b66a431a 100644 --- a/web/tests/CloudSnapshotWorkflowTest.php +++ b/web/tests/CloudSnapshotWorkflowTest.php @@ -634,7 +634,7 @@ namespace { 'instance_upgrade', 'firewall_apply', 'snapshot_create', 'snapshot_restore', 'snapshot_delete', 'backup_create', 'backup_restore', 'instance_migration_cleanup', - 'instance_migration_rollback', 'rescue', + 'instance_migration_rollback', 'instance_migration_activate', 'rescue', ], 'The instance operation log does not cover every settings operation.'); $assert(in_array((string) $settingsAfterDelete->id, $operationLogIds, true), 'An instance-settings task was omitted from the instance operation log.'); diff --git a/web/tests/LegalPagesContractTest.php b/web/tests/LegalPagesContractTest.php index 153d26e1..7a72aef4 100644 --- a/web/tests/LegalPagesContractTest.php +++ b/web/tests/LegalPagesContractTest.php @@ -73,7 +73,7 @@ $assert(str_contains($auth, "Config::get('tos_reg') === 'true'") 'Legal consent is only client-side and can be bypassed.'); $assets = $read('web/resources/views/venix/layout_assets.tpl'); -$assert(str_contains($assets, "value='2026102505'") +$assert(str_contains($assets, "value='2026102506'") && str_contains($admin, '?v={$venix_asset_version}') && str_contains($read('web/resources/views/venix/home/layout_header.tpl'), '?v={$venix_asset_version}'), 'Shared browser asset version was not coupled to the new assets.'); diff --git a/web/tests/PublicPageLocaleContractTest.php b/web/tests/PublicPageLocaleContractTest.php index 9b391856..89bb2af6 100644 --- a/web/tests/PublicPageLocaleContractTest.php +++ b/web/tests/PublicPageLocaleContractTest.php @@ -80,7 +80,7 @@ $assert(preg_match('/\.landing-site-header\s*\{[^}]*position:\s*fixed;/s', $css) && str_contains($accountCss, 'padding-top: 88px;') && str_contains($accountCss, 'padding-top: 76px;'), 'Persistent desktop or mobile public-navigation styling is incomplete.'); -$assert(str_contains($read('web/resources/views/venix/layout_assets.tpl'), "value='2026102505'"), +$assert(str_contains($read('web/resources/views/venix/layout_assets.tpl'), "value='2026102506'"), 'The shared browser asset version was not advanced with the navigation styles.'); echo "Public page locale and shared-layout contract tests passed.\n"; diff --git a/web/tests/SiteBrandContractTest.php b/web/tests/SiteBrandContractTest.php index d9cba479..3d3c9039 100644 --- a/web/tests/SiteBrandContractTest.php +++ b/web/tests/SiteBrandContractTest.php @@ -69,7 +69,7 @@ foreach ($templates as $template) { } $assets = $read('web/resources/views/venix/layout_assets.tpl'); -$assert(str_contains($assets, "value='2026102505'"), +$assert(str_contains($assets, "value='2026102506'"), 'The shared asset version was not updated for the canonical logo.'); echo "Shared site-brand contract passed across public, auth, user, cloud and admin layouts.\n";