From 896fbeaf55b83912971bd7c2378f560fbcc68fe8 Mon Sep 17 00:00:00 2001 From: codex Date: Mon, 26 Oct 2026 21:31:05 +0800 Subject: [PATCH] fix(cloud): preserve disk guest identity on recovery --- agent/hypervisor/admission.go | 1 + agent/hypervisor/backup_test.go | 1 + agent/hypervisor/feature_helpers.go | 213 +++++++++++++-- agent/hypervisor/feature_helpers_test.go | 258 ++++++++++++++++++ agent/hypervisor/libvirt_backup_recovery.go | 55 +++- agent/hypervisor/libvirt_operation_admin.go | 50 +++- .../libvirt_operation_admin_test.go | 24 ++ agent/hypervisor/types.go | 7 +- web/app/Services/Cloud/CloudBackupService.php | 12 +- .../CloudPortableBackupFinalizerTest.php | 11 +- 10 files changed, 573 insertions(+), 59 deletions(-) diff --git a/agent/hypervisor/admission.go b/agent/hypervisor/admission.go index 7931867d..1bef98ed 100644 --- a/agent/hypervisor/admission.go +++ b/agent/hypervisor/admission.go @@ -153,6 +153,7 @@ func RecoverBackupAdmission(uuid string, request RecoverBackupRequest) MutationA domainLockKey(uuid), vmManagedVolumesLockKey(uuid), volumeLockKey(managedStoragePoolName, snapshotRestoreVolumeName(uuid, request.OperationID)), + volumeLockKey(seedStoragePoolName, seedVolumeName(uuid)), catalogLockKey("backup", request.BackupKey), }} } diff --git a/agent/hypervisor/backup_test.go b/agent/hypervisor/backup_test.go index 958e7d75..3c6a956a 100644 --- a/agent/hypervisor/backup_test.go +++ b/agent/hypervisor/backup_test.go @@ -165,6 +165,7 @@ func TestBackupAdmissionsFenceSourceAndNewVMArtifacts(t *testing.T) { domainLockKey(uuid), vmManagedVolumesLockKey(uuid), volumeLockKey(managedStoragePoolName, snapshotRestoreVolumeName(uuid, recoveryRequest.OperationID)), + volumeLockKey(seedStoragePoolName, seedVolumeName(uuid)), catalogLockKey("backup", recoveryRequest.BackupKey), } { if !containsString(recovery.LockKeys, key) || !recovery.HeavyIO { diff --git a/agent/hypervisor/feature_helpers.go b/agent/hypervisor/feature_helpers.go index 86b38506..5c7400a9 100644 --- a/agent/hypervisor/feature_helpers.go +++ b/agent/hypervisor/feature_helpers.go @@ -28,11 +28,17 @@ import ( ) const ( - maxSeedImageBytes = 2 << 20 - maxTemplateBytes = int64(4) << 30 - maxUserISOBytes = int64(32) << 30 - maxRescueISOBytes = int64(8) << 30 - minImportBytes = int64(1) << 20 + maxSeedImageBytes = 2 << 20 + maxTemplateBytes = int64(4) << 30 + maxUserISOBytes = int64(32) << 30 + maxRescueISOBytes = int64(8) << 30 + minImportBytes = int64(1) << 20 + backupRecoveryOperationMetadata = "virt-cloud-backup-recovery-operation" + diskGuestIdentityMetadata = "virt-cloud-guest-identity" + diskGuestIdentityValue = "disk" + diskSSHAuthorizedKeysPath = "/root/.ssh/authorized_keys" + diskSSHDirectoryCommand = "install -d -m 0700 -o root -g root /root/.ssh" + diskSSHRemoveCommand = "rm -f -- /root/.ssh/authorized_keys" ) var ( @@ -125,28 +131,67 @@ func mutateSeedSettings(contents []byte, request SettingsRequest, transitionID s if err := json.Unmarshal(bytes.TrimSpace(files.UserData[len(header):]), &userData); err != nil { return nil, "", NewError(ErrConflict, "invalid_seed", "NoCloud user-data is invalid", err) } - users, ok := userData["users"].([]any) - if !ok || len(users) != 1 { - return nil, "", NewError(ErrConflict, "invalid_seed", "NoCloud root account definition is invalid", nil) + var metadata map[string]any + if err := json.Unmarshal(bytes.TrimSpace(files.MetaData), &metadata); err != nil { + return nil, "", NewError(ErrConflict, "invalid_seed", "NoCloud meta-data is invalid", err) } - root, ok := users[0].(map[string]any) - if !ok || root["name"] != "root" { - return nil, "", NewError(ErrConflict, "invalid_seed", "NoCloud root account definition is invalid", nil) + instanceID, ok := metadata["instance-id"].(string) + if !ok || strings.TrimSpace(instanceID) == "" { + return nil, "", NewError(ErrConflict, "invalid_seed", "NoCloud instance identity is invalid", nil) + } + diskAuthoritative := metadata[diskGuestIdentityMetadata] == diskGuestIdentityValue + var root map[string]any + if diskAuthoritative { + // Recovery itself removes every pre-recovery identity instruction. + // From this point on, however, settings explicitly submitted by the + // user are new intent and must survive later settings transitions until + // that same field is changed again. This also prevents a quick second + // operation from deleting a password/key before cloud-init consumes it. + if users, exists := userData["users"]; exists { + parsed, valid := users.([]any) + if !valid || len(parsed) != 1 { + return nil, "", NewError(ErrConflict, "invalid_seed", "NoCloud root account definition is invalid", nil) + } + root, valid = parsed[0].(map[string]any) + if !valid || root["name"] != "root" { + return nil, "", NewError(ErrConflict, "invalid_seed", "NoCloud root account definition is invalid", nil) + } + } + if request.RootPassword != nil { + if root == nil { + root = map[string]any{"name": "root", "shell": "/bin/bash"} + userData["users"] = []any{root} + } + } + } else { + users, valid := userData["users"].([]any) + if !valid || len(users) != 1 { + return nil, "", NewError(ErrConflict, "invalid_seed", "NoCloud root account definition is invalid", nil) + } + root, valid = users[0].(map[string]any) + if !valid || root["name"] != "root" { + return nil, "", NewError(ErrConflict, "invalid_seed", "NoCloud root account definition is invalid", nil) + } } if request.SSHPublicKey != nil { key := *request.SSHPublicKey if err := validateOptionalSSHKey(key); err != nil { return nil, "", err } - if key == "" { + if diskAuthoritative { + removeDiskAuthoritativeSSHKeyMutation(userData) + setDiskAuthoritativeSSHKeyMutation(userData, key) + } else if key == "" { delete(root, "ssh_authorized_keys") } else { root["ssh_authorized_keys"] = []string{key} } } passwordEnabled := false - if hash, ok := root["hashed_passwd"].(string); ok && hash != "" { - passwordEnabled = true + if root != nil { + if hash, present := root["hashed_passwd"].(string); present && hash != "" { + passwordEnabled = true + } } if request.RootPassword != nil { password := *request.RootPassword @@ -172,22 +217,23 @@ func mutateSeedSettings(contents []byte, request SettingsRequest, transitionID s passwordEnabled = true } } - keys, _ := root["ssh_authorized_keys"].([]any) - stringKeys, _ := root["ssh_authorized_keys"].([]string) - if !passwordEnabled && len(keys) == 0 && len(stringKeys) == 0 { - return nil, "", NewError(ErrInvalid, "guest_credentials_required", "at least one root login method must remain enabled", nil) - } - userData["ssh_pwauth"] = passwordEnabled - updateSSHDWriteFile(userData, passwordEnabled) - - var metadata map[string]any - if err := json.Unmarshal(bytes.TrimSpace(files.MetaData), &metadata); err != nil { - return nil, "", NewError(ErrConflict, "invalid_seed", "NoCloud meta-data is invalid", err) + if diskAuthoritative { + if request.RootPassword != nil && request.SSHPublicKey != nil && + *request.RootPassword == "" && *request.SSHPublicKey == "" { + return nil, "", NewError(ErrInvalid, "guest_credentials_required", "at least one root login method must remain enabled", nil) + } + } else { + keys, _ := root["ssh_authorized_keys"].([]any) + stringKeys, _ := root["ssh_authorized_keys"].([]string) + if root != nil && !passwordEnabled && len(keys) == 0 && len(stringKeys) == 0 { + return nil, "", NewError(ErrInvalid, "guest_credentials_required", "at least one root login method must remain enabled", nil) + } } - instanceID, ok := metadata["instance-id"].(string) - if !ok || strings.TrimSpace(instanceID) == "" { - return nil, "", NewError(ErrConflict, "invalid_seed", "NoCloud instance identity is invalid", nil) + if !diskAuthoritative || request.RootPassword != nil { + userData["ssh_pwauth"] = passwordEnabled + updateSSHDWriteFile(userData, passwordEnabled) } + hostname := "" if request.Hostname != nil { hostname = *request.Hostname @@ -195,6 +241,9 @@ func mutateSeedSettings(contents []byte, request SettingsRequest, transitionID s return nil, "", NewError(ErrInvalid, "invalid_hostname", "hostname must be a lower-case DNS name", nil) } metadata["local-hostname"] = hostname + if diskAuthoritative { + userData["preserve_hostname"] = false + } } generation, err := newRandomID() if err != nil { @@ -215,6 +264,114 @@ func mutateSeedSettings(contents []byte, request SettingsRequest, transitionID s return updated, hostname, err } +// setDiskAuthoritativeSSHKeyMutation updates only authorized_keys. In +// particular it does not use cloud-init's users module: lock_passwd defaults +// to true there, which would silently lock a password restored from disk when +// the user only asked to change an SSH key. +func setDiskAuthoritativeSSHKeyMutation(userData map[string]any, key string) { + if key == "" { + appendSeedCommand(userData, "runcmd", diskSSHRemoveCommand) + return + } + appendSeedCommand(userData, "bootcmd", diskSSHDirectoryCommand) + files, _ := userData["write_files"].([]any) + files = append(files, map[string]any{ + "path": diskSSHAuthorizedKeysPath, "content": key + "\n", + "owner": "root:root", "permissions": "0600", + }) + userData["write_files"] = files +} + +func removeDiskAuthoritativeSSHKeyMutation(userData map[string]any) { + files, _ := userData["write_files"].([]any) + kept := make([]any, 0, len(files)) + for _, raw := range files { + entry, ok := raw.(map[string]any) + if ok && entry["path"] == diskSSHAuthorizedKeysPath { + continue + } + kept = append(kept, raw) + } + if len(kept) == 0 { + delete(userData, "write_files") + } else { + userData["write_files"] = kept + } + removeSeedCommand(userData, "bootcmd", diskSSHDirectoryCommand) + removeSeedCommand(userData, "runcmd", diskSSHRemoveCommand) +} + +func appendSeedCommand(userData map[string]any, field, script string) { + commands, _ := userData[field].([]any) + commands = append(commands, []string{"/bin/sh", "-ec", script}) + userData[field] = commands +} + +func removeSeedCommand(userData map[string]any, field, script string) { + commands, _ := userData[field].([]any) + kept := make([]any, 0, len(commands)) + for _, raw := range commands { + parts, ok := raw.([]any) + if ok && len(parts) == 3 && parts[0] == "/bin/sh" && + parts[1] == "-ec" && parts[2] == script { + continue + } + kept = append(kept, raw) + } + if len(kept) == 0 { + delete(userData, field) + } else { + userData[field] = kept + } +} + +func sanitizeSeedForDiskAuthoritativeRecovery(contents []byte, + transitionID string) ([]byte, error) { + if !operationIdentifierPattern.MatchString(transitionID) { + return nil, NewError(ErrInvalid, "invalid_operation_id", "backup recovery transition id is invalid", nil) + } + files, err := readSeedISO(contents) + if err != nil { + return nil, err + } + const header = "#cloud-config\n" + if !bytes.HasPrefix(files.UserData, []byte(header)) { + return nil, NewError(ErrConflict, "invalid_seed", "NoCloud user-data is not managed by Virt Cloud", nil) + } + var original map[string]any + if err := json.Unmarshal(bytes.TrimSpace(files.UserData[len(header):]), &original); err != nil { + return nil, NewError(ErrConflict, "invalid_seed", "NoCloud user-data is invalid", err) + } + userData := map[string]any{ + "preserve_hostname": true, + } + for _, key := range []string{"updates", "bootcmd"} { + if value, exists := original[key]; exists { + userData[key] = value + } + } + var metadata map[string]any + if err := json.Unmarshal(bytes.TrimSpace(files.MetaData), &metadata); err != nil { + return nil, NewError(ErrConflict, "invalid_seed", "NoCloud meta-data is invalid", err) + } + instanceID, ok := metadata["instance-id"].(string) + if !ok || strings.TrimSpace(instanceID) == "" { + return nil, NewError(ErrConflict, "invalid_seed", "NoCloud instance identity is invalid", nil) + } + delete(metadata, "local-hostname") + metadata[diskGuestIdentityMetadata] = diskGuestIdentityValue + metadata[backupRecoveryOperationMetadata] = transitionID + files.UserData, err = marshalCloudConfig(userData) + if err != nil { + return nil, err + } + files.MetaData, err = marshalJSONLine(metadata) + if err != nil { + return nil, err + } + return buildSeedISO(files) +} + func updateSSHDWriteFile(userData map[string]any, passwordEnabled bool) { content := "PermitRootLogin prohibit-password\nPasswordAuthentication no\nKbdInteractiveAuthentication no\n" if passwordEnabled { diff --git a/agent/hypervisor/feature_helpers_test.go b/agent/hypervisor/feature_helpers_test.go index e3a2c183..f9edb115 100644 --- a/agent/hypervisor/feature_helpers_test.go +++ b/agent/hypervisor/feature_helpers_test.go @@ -2,6 +2,7 @@ package hypervisor import ( "bytes" + "encoding/base64" "encoding/binary" "encoding/json" "net/http" @@ -13,6 +14,18 @@ import ( "git.luoli.live/root/virt/agent/cloudinit" ) +func validSSHKeyForTest() string { + keyBlob := make([]byte, 4+len("ssh-ed25519")+4+32) + binary.BigEndian.PutUint32(keyBlob[:4], uint32(len("ssh-ed25519"))) + copy(keyBlob[4:], "ssh-ed25519") + offset := 4 + len("ssh-ed25519") + binary.BigEndian.PutUint32(keyBlob[offset:offset+4], 32) + for index := offset + 4; index < len(keyBlob); index++ { + keyBlob[index] = byte(index) + } + return "ssh-ed25519 " + base64.StdEncoding.EncodeToString(keyBlob) + " user@test" +} + func managedSeedForTest(t *testing.T) []byte { t.Helper() image, err := cloudinit.BuildISO(cloudinit.Request{ @@ -75,6 +88,251 @@ func TestMutateSeedSettingsPreservesStartupAndHashesNewPassword(t *testing.T) { } } +func TestSanitizeSeedForDiskAuthoritativeRecoveryKeepsOnlyNetworkIntent(t *testing.T) { + original := managedSeedForTest(t) + originalFiles, err := readSeedISO(original) + if err != nil { + t.Fatal(err) + } + sanitized, err := sanitizeSeedForDiskAuthoritativeRecovery( + original, "operation-recover-backup", + ) + if err != nil { + t.Fatal(err) + } + files, err := readSeedISO(sanitized) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(files.NetworkConfig, originalFiles.NetworkConfig) { + t.Fatal("backup recovery changed the current network-config") + } + var userData map[string]any + if err := json.Unmarshal(bytes.TrimPrefix( + files.UserData, []byte("#cloud-config\n")), &userData); err != nil { + t.Fatal(err) + } + for _, forbidden := range []string{ + "users", "chpasswd", "ssh_pwauth", "disable_root", + "write_files", "runcmd", + } { + if _, exists := userData[forbidden]; exists { + t.Fatalf("disk-authoritative recovery seed contains %q: %#v", forbidden, userData) + } + } + if userData["preserve_hostname"] != true { + t.Fatalf("preserve_hostname = %#v", userData["preserve_hostname"]) + } + if bytes.Contains(files.UserData, []byte("echo once")) || + bytes.Contains(files.UserData, []byte("hashed_passwd")) { + t.Fatalf("recovery seed retained guest identity or startup payload: %s", files.UserData) + } + var originalMetadata, metadata map[string]any + if err := json.Unmarshal(bytes.TrimSpace(originalFiles.MetaData), &originalMetadata); err != nil { + t.Fatal(err) + } + if err := json.Unmarshal(bytes.TrimSpace(files.MetaData), &metadata); err != nil { + t.Fatal(err) + } + if metadata["instance-id"] != originalMetadata["instance-id"] { + t.Fatalf("instance-id changed: old=%#v new=%#v", + originalMetadata["instance-id"], metadata["instance-id"]) + } + if _, exists := metadata["local-hostname"]; exists { + t.Fatalf("recovery seed retained local-hostname: %#v", metadata) + } + if metadata["virt-cloud-backup-recovery-operation"] != + "operation-recover-backup" { + t.Fatalf("backup recovery marker = %#v", + metadata["virt-cloud-backup-recovery-operation"]) + } +} + +func TestDiskAuthoritativeRecoverySeedAllowsExplicitFutureSettings(t *testing.T) { + sanitized, err := sanitizeSeedForDiskAuthoritativeRecovery( + managedSeedForTest(t), "operation-recover-backup", + ) + if err != nil { + t.Fatal(err) + } + + hostname := "after-recovery.example.com" + hostnameOnly, resultHostname, err := mutateSeedSettings( + sanitized, SettingsRequest{Hostname: &hostname}, "operation-hostname", + ) + if err != nil { + t.Fatal(err) + } + if resultHostname != hostname { + t.Fatalf("hostname = %q", resultHostname) + } + hostnameFiles, err := readSeedISO(hostnameOnly) + if err != nil { + t.Fatal(err) + } + var hostnameData map[string]any + if err := json.Unmarshal(bytes.TrimPrefix( + hostnameFiles.UserData, []byte("#cloud-config\n")), &hostnameData); err != nil { + t.Fatal(err) + } + for _, forbidden := range []string{"users", "chpasswd", "ssh_pwauth"} { + if _, exists := hostnameData[forbidden]; exists { + t.Fatalf("hostname-only update injected %q: %#v", forbidden, hostnameData) + } + } + + password := "ExplicitReset!456" + passwordReset, _, err := mutateSeedSettings( + sanitized, SettingsRequest{RootPassword: &password}, "operation-password", + ) + if err != nil { + t.Fatal(err) + } + if bytes.Contains(passwordReset, []byte(password)) { + t.Fatal("explicit password reset stored the plaintext password") + } + passwordFiles, err := readSeedISO(passwordReset) + if err != nil { + t.Fatal(err) + } + var passwordData map[string]any + if err := json.Unmarshal(bytes.TrimPrefix( + passwordFiles.UserData, []byte("#cloud-config\n")), &passwordData); err != nil { + t.Fatal(err) + } + users, ok := passwordData["users"].([]any) + if !ok || len(users) != 1 || passwordData["chpasswd"] == nil { + t.Fatalf("explicit password reset did not create managed credentials: %#v", passwordData) + } + + passwordAfterHostname, _, err := mutateSeedSettings( + hostnameOnly, SettingsRequest{RootPassword: &password}, + "operation-password-after-hostname", + ) + if err != nil { + t.Fatal(err) + } + passwordAfterHostnameFiles, err := readSeedISO(passwordAfterHostname) + if err != nil { + t.Fatal(err) + } + var passwordAfterHostnameData, passwordAfterHostnameMetadata map[string]any + if err := json.Unmarshal(bytes.TrimPrefix( + passwordAfterHostnameFiles.UserData, []byte("#cloud-config\n")), + &passwordAfterHostnameData); err != nil { + t.Fatal(err) + } + if err := json.Unmarshal(bytes.TrimSpace( + passwordAfterHostnameFiles.MetaData), &passwordAfterHostnameMetadata); err != nil { + t.Fatal(err) + } + if passwordAfterHostnameData["preserve_hostname"] != false { + t.Fatalf("password-only update did not retain the explicit hostname: %#v", + passwordAfterHostnameData) + } + if passwordAfterHostnameMetadata["local-hostname"] != hostname { + t.Fatalf("password-only update lost the explicit hostname: %#v", + passwordAfterHostnameMetadata) + } + + hostnameAfterPassword, _, err := mutateSeedSettings( + passwordReset, SettingsRequest{Hostname: &hostname}, + "operation-hostname-after-password", + ) + if err != nil { + t.Fatal(err) + } + hostnameAfterPasswordFiles, err := readSeedISO(hostnameAfterPassword) + if err != nil { + t.Fatal(err) + } + var hostnameAfterPasswordData map[string]any + if err := json.Unmarshal(bytes.TrimPrefix( + hostnameAfterPasswordFiles.UserData, []byte("#cloud-config\n")), + &hostnameAfterPasswordData); err != nil { + t.Fatal(err) + } + users, ok = hostnameAfterPasswordData["users"].([]any) + if !ok || len(users) != 1 || hostnameAfterPasswordData["chpasswd"] == nil || + hostnameAfterPasswordData["ssh_pwauth"] != true { + t.Fatalf("hostname-only update lost the explicit password intent: %#v", + hostnameAfterPasswordData) + } +} + +func TestDiskAuthoritativeRecoverySSHKeyDoesNotChangePasswordState(t *testing.T) { + sanitized, err := sanitizeSeedForDiskAuthoritativeRecovery( + managedSeedForTest(t), "operation-recover-backup", + ) + if err != nil { + t.Fatal(err) + } + key := validSSHKeyForTest() + updated, _, err := mutateSeedSettings( + sanitized, SettingsRequest{SSHPublicKey: &key}, "operation-ssh-key", + ) + if err != nil { + t.Fatal(err) + } + files, err := readSeedISO(updated) + if err != nil { + t.Fatal(err) + } + var userData map[string]any + if err := json.Unmarshal(bytes.TrimPrefix( + files.UserData, []byte("#cloud-config\n")), &userData); err != nil { + t.Fatal(err) + } + for _, forbidden := range []string{ + "users", "chpasswd", "ssh_pwauth", "disable_root", + } { + if _, exists := userData[forbidden]; exists { + t.Fatalf("SSH-key-only update changed password state through %q: %#v", + forbidden, userData) + } + } + if bytes.Contains(files.UserData, []byte("lock_passwd")) || + bytes.Contains(files.UserData, []byte("hashed_passwd")) { + t.Fatalf("SSH-key-only update included password state: %s", files.UserData) + } + writeFiles, ok := userData["write_files"].([]any) + if !ok || len(writeFiles) != 1 { + t.Fatalf("SSH-key-only update write_files = %#v", userData["write_files"]) + } + entry, ok := writeFiles[0].(map[string]any) + if !ok || entry["path"] != diskSSHAuthorizedKeysPath || + entry["content"] != key+"\n" || entry["permissions"] != "0600" { + t.Fatalf("SSH-key-only update file = %#v", writeFiles[0]) + } + if !bytes.Equal(files.NetworkConfig, + mustReadSeedFiles(t, sanitized).NetworkConfig) { + t.Fatal("SSH-key-only update changed network-config") + } + + hostname := "after-key.example.com" + hostnameOnly, _, err := mutateSeedSettings( + updated, SettingsRequest{Hostname: &hostname}, "operation-hostname-after-key", + ) + if err != nil { + t.Fatal(err) + } + hostnameFiles := mustReadSeedFiles(t, hostnameOnly) + if !bytes.Contains(hostnameFiles.UserData, []byte(diskSSHAuthorizedKeysPath)) || + !bytes.Contains(hostnameFiles.UserData, []byte(key)) { + t.Fatalf("later hostname update lost the explicit SSH key intent: %s", + hostnameFiles.UserData) + } +} + +func mustReadSeedFiles(t *testing.T, contents []byte) seedFiles { + t.Helper() + files, err := readSeedISO(contents) + if err != nil { + t.Fatal(err) + } + return files +} + func TestMutateSeedNetworkChangesOnlyAddressIntent(t *testing.T) { updated, mac, err := mutateSeedNetwork(managedSeedForTest(t), "9.9.9.10/24", "9.9.9.1", []string{"1.1.1.1", "8.8.8.8"}, "operation-network") if err != nil { diff --git a/agent/hypervisor/libvirt_backup_recovery.go b/agent/hypervisor/libvirt_backup_recovery.go index 3029a66b..5a0ee65c 100644 --- a/agent/hypervisor/libvirt_backup_recovery.go +++ b/agent/hypervisor/libvirt_backup_recovery.go @@ -12,10 +12,11 @@ import ( "libvirt.org/go/libvirtxml" ) -// RecoverBackup replaces only the managed root disk of an existing domain. -// The UUID, domain definition, NoCloud seed, public IP, firewall and service -// identity remain untouched. The disk transition follows the same -// fail-closed two-root protocol as snapshot restore. +// RecoverBackup replaces the managed root disk of an existing domain and +// reduces its NoCloud seed to current network intent. Guest credentials and +// hostname remain authoritative on the restored disk. The disk/seed +// transition follows the same fail-closed two-root protocol as snapshot +// restore. func (l *Libvirt) RecoverBackup(ctx context.Context, uuid string, request RecoverBackupRequest) (_ RecoverBackupResult, resultErr error) { mutationStarted := false @@ -38,6 +39,27 @@ func (l *Libvirt) RecoverBackup(ctx context.Context, uuid string, } defer pool.Free() defer oldRoot.Free() + seed, err := l.lookupOptionalManagedVolume( + l.storagePoolName(StorageRoleSeed), seedVolumeName(uuid), + ) + if err != nil { + return RecoverBackupResult{}, err + } + if seed == nil { + return RecoverBackupResult{}, NewError(ErrConflict, "seed_not_found", + "managed NoCloud seed volume was not found", nil) + } + defer seed.Free() + originalSeed, err := l.downloadSmallVolume(ctx, seed, maxSeedImageBytes) + if err != nil { + return RecoverBackupResult{}, err + } + recoverySeed, err := sanitizeSeedForDiskAuthoritativeRecovery( + originalSeed, request.OperationID, + ) + if err != nil { + return RecoverBackupResult{}, err + } if metadata.PendingCleanupVolume != "" { return RecoverBackupResult{}, NewError(ErrConflict, "backup_disk_transition_active", @@ -104,6 +126,10 @@ func (l *Libvirt) RecoverBackup(ctx context.Context, uuid string, stored.DiskTransitionKind = "backup_recover" stored.DiskTransitionID = request.OperationID stored.DiskTransitionRunning = wasRunning + // The recovered system disk owns the guest hostname. Keeping the + // pre-recovery value here would make later inspection and operations + // incorrectly present panel metadata as the guest's current hostname. + stored.Hostname = "" return nil }) if err != nil { @@ -116,23 +142,26 @@ func (l *Libvirt) RecoverBackup(ctx context.Context, uuid string, } return RecoverBackupResult{}, operation.MarkNotApplied(err) } - defined, err := l.conn.DomainDefineXML(updatedXML) - if err != nil { - _, observed, inspectErr := requireManagedDomain(domain) - if inspectErr != nil || observed.RootVolumeName != metadata.RootVolumeName { - return RecoverBackupResult{}, operation.MarkUncertain(errors.Join( - mutationLibvirtError("switch backup recovery system disk", err), inspectErr)) + if err := l.replaceSeedAndDefinition( + ctx, seed, originalSeed, recoverySeed, originalXML, updatedXML, false, + ); err != nil { + if operation.IsUncertain(err) { + return RecoverBackupResult{}, err } restartErr := restartOriginalDomain(domain, wasRunning) cleanupErr := replacement.Delete(libvirt.STORAGE_VOL_DELETE_NORMAL) if restartErr != nil || cleanupErr != nil { return RecoverBackupResult{}, operation.MarkUncertain(errors.Join( - mutationLibvirtError("switch backup recovery system disk", err), + err, restartErr, optionalMutationLibvirtError("roll back backup recovery volume", cleanupErr))) } - return RecoverBackupResult{}, operation.MarkNotApplied( - mutationLibvirtError("switch backup recovery system disk", err)) + return RecoverBackupResult{}, operation.MarkNotApplied(err) + } + defined, err := l.domainByUUID(ctx, uuid) + if err != nil { + return RecoverBackupResult{}, operation.MarkUncertain( + mutationLibvirtError("resolve recovered domain", err)) } defer defined.Free() if err := defined.SetAutostart(autostart); err != nil { diff --git a/agent/hypervisor/libvirt_operation_admin.go b/agent/hypervisor/libvirt_operation_admin.go index 87ab3973..97cf0f5a 100644 --- a/agent/hypervisor/libvirt_operation_admin.go +++ b/agent/hypervisor/libvirt_operation_admin.go @@ -146,7 +146,7 @@ func (l *Libvirt) inspectSuccessfulBackupCleanupPending(ctx context.Context, } defer domain.Free() - _, cleanup := l.inspectSnapshotOperation(record, domain) + _, cleanup := l.inspectSnapshotOperation(ctx, record, domain) return appliedBackupInspectionWithCleanup(record, inspection, cleanup) } @@ -245,7 +245,7 @@ func (l *Libvirt) inspectExistingDomainOperation(ctx context.Context, record ope strings.HasPrefix(record.Action, "delete-snapshot:") || strings.HasPrefix(record.Action, "restore-snapshot:") || strings.HasPrefix(record.Action, "recover-backup:") { - resolution, steps := l.inspectSnapshotOperation(record, domain) + resolution, steps := l.inspectSnapshotOperation(ctx, record, domain) return resolution, steps, nil } if record.Action == "enter-rescue" || record.Action == "force-start-rescue" || @@ -598,7 +598,7 @@ func (l *Libvirt) inspectReinstallOperation(record operation.Record, return operation.Resolution{}, operationCleanupInspection{} } -func (l *Libvirt) inspectSnapshotOperation(record operation.Record, +func (l *Libvirt) inspectSnapshotOperation(ctx context.Context, record operation.Record, domain *libvirt.Domain) (operation.Resolution, operationCleanupInspection) { parts := strings.Split(record.Action, ":") if len(parts) < 2 || !catalogIDPattern.MatchString(parts[1]) { @@ -733,8 +733,24 @@ func (l *Libvirt) inspectSnapshotOperation(record operation.Record, if replacement != nil { replacement.Free() } - if diskTransitionCommitted(metadata, replacementName, - "backup_recover", record.OperationID) && replacementExists { + seedContents, seedErr := l.managedSeedContents(ctx, record.ResourceID) + if seedErr != nil { + return operation.Resolution{}, operationCleanupInspection{} + } + seedOperation, seedErr := seedMetadataMarker( + seedContents, backupRecoveryOperationMetadata, + ) + if seedErr != nil { + return operation.Resolution{}, operationCleanupInspection{} + } + seedCommitted := seedOperation == record.OperationID + diskCommitted := diskTransitionCommitted(metadata, replacementName, + "backup_recover", record.OperationID) + evidence := classifyBackupRecoveryEvidence( + replacementExists, diskCommitted, + metadata.DiskTransitionID == record.OperationID, seedCommitted, + ) + if evidence == backupRecoveryEvidenceApplied { state, _, stateErr := domain.GetState() if stateErr != nil || !diskTransitionPowerStateMatches( metadata, state, "backup_recover", record.OperationID) { @@ -751,7 +767,7 @@ func (l *Libvirt) inspectSnapshotOperation(record operation.Record, return operation.Resolution{Resolved: true, Succeeded: true, Result: result}, operationCleanupInspection{} } - if !replacementExists && metadata.DiskTransitionID != record.OperationID { + if evidence == backupRecoveryEvidenceNotApplied { return operation.Resolution{Resolved: true, Succeeded: false, ErrorCode: "backup_not_recovered", ErrorMessage: "host evidence confirms the root volume was not switched"}, @@ -761,6 +777,28 @@ func (l *Libvirt) inspectSnapshotOperation(record operation.Record, return operation.Resolution{}, operationCleanupInspection{} } +type backupRecoveryEvidence uint8 + +const ( + backupRecoveryEvidenceUnknown backupRecoveryEvidence = iota + backupRecoveryEvidenceApplied + backupRecoveryEvidenceNotApplied +) + +// classifyBackupRecoveryEvidence is deliberately fail-closed. Recovery is +// applied only when both the root-disk transition and the seed transition are +// visible. Any partial evidence remains unresolved and fenced. +func classifyBackupRecoveryEvidence(replacementExists, diskCommitted, + diskTransitionClaimed, seedCommitted bool) backupRecoveryEvidence { + if replacementExists && diskCommitted && seedCommitted { + return backupRecoveryEvidenceApplied + } + if !replacementExists && !diskTransitionClaimed && !seedCommitted { + return backupRecoveryEvidenceNotApplied + } + return backupRecoveryEvidenceUnknown +} + func backupResidueCleanupAllowed(record operation.Record, stored *domainSnapshotMetadataXML) bool { parts := strings.Split(record.Action, ":") if len(parts) != 3 || parts[0] != "create-backup" || diff --git a/agent/hypervisor/libvirt_operation_admin_test.go b/agent/hypervisor/libvirt_operation_admin_test.go index efc8691c..23f87980 100644 --- a/agent/hypervisor/libvirt_operation_admin_test.go +++ b/agent/hypervisor/libvirt_operation_admin_test.go @@ -99,6 +99,30 @@ func TestCleanupJobEvidenceRequiresIdleDomainAndBlockJobs(t *testing.T) { } } +func TestBackupRecoveryEvidenceRequiresRootAndSeedConvergence(t *testing.T) { + tests := []struct { + name string + replacement, disk, diskClaim, seed bool + want backupRecoveryEvidence + }{ + {name: "root and seed committed", replacement: true, disk: true, diskClaim: true, seed: true, want: backupRecoveryEvidenceApplied}, + {name: "nothing dispatched", want: backupRecoveryEvidenceNotApplied}, + {name: "seed only", seed: true, want: backupRecoveryEvidenceUnknown}, + {name: "root only", replacement: true, disk: true, diskClaim: true, want: backupRecoveryEvidenceUnknown}, + {name: "replacement residue only", replacement: true, want: backupRecoveryEvidenceUnknown}, + {name: "transition claim without volume", diskClaim: true, want: backupRecoveryEvidenceUnknown}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + if got := classifyBackupRecoveryEvidence( + test.replacement, test.disk, test.diskClaim, test.seed, + ); got != test.want { + t.Fatalf("evidence = %v, want %v", got, test.want) + } + }) + } +} + func TestCleanupPlanIdentifierBindsPrivateExactEvidence(t *testing.T) { record := operation.Record{ OperationID: "operation-exact-cleanup-plan", ResourceID: "123e4567-e89b-42d3-a456-426614174000", diff --git a/agent/hypervisor/types.go b/agent/hypervisor/types.go index 7d4548eb..4cbc88ef 100644 --- a/agent/hypervisor/types.go +++ b/agent/hypervisor/types.go @@ -298,9 +298,10 @@ type RestoreBackupResult struct { } // RecoverBackupRequest replaces the managed root disk of an existing VM with -// one immutable portable-backup object. Network identity, NoCloud seed, -// billing identity and the domain UUID are deliberately outside this request -// and remain attached to the existing domain. +// one immutable portable-backup object. Network and domain identity remain +// attached to the existing VM, while the Agent reduces its NoCloud seed to +// network-only intent so the restored disk keeps its own credentials and +// hostname. type RecoverBackupRequest struct { OperationID string `json:"operation_id,omitempty"` BackupKey string `json:"backup_key"` diff --git a/web/app/Services/Cloud/CloudBackupService.php b/web/app/Services/Cloud/CloudBackupService.php index 0cc5d349..328c61c4 100644 --- a/web/app/Services/Cloud/CloudBackupService.php +++ b/web/app/Services/Cloud/CloudBackupService.php @@ -1503,7 +1503,7 @@ final class CloudBackupService foreach ([ 'template_id', 'template_revision', 'template_generation_token', 'os_source', 'os_name', - 'login_name', 'debian_12_network_fix', 'guest_hostname', + 'login_name', 'debian_12_network_fix', ] as $field) { if (array_key_exists($field, $backupSpec)) { $instanceSpec[$field] = $backupSpec[$field]; @@ -1512,6 +1512,11 @@ final class CloudBackupService } } unset($instanceSpec['install_iso']); + // Passwords and hostnames may have been changed inside the guest + // without passing through the panel. The restored system disk is + // authoritative, so the panel must not claim either value is + // known or cause the Agent's NoCloud seed to inject an old value. + unset($instanceSpec['guest_hostname'], $instanceSpec['hostname']); $instanceSpec['managed'] = true; $instanceSpec['restored_from_backup'] = (string) $backup->backup_key; @@ -1520,10 +1525,7 @@ final class CloudBackupService JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE ); - // The restored disk contains the backup-time guest credentials. - // Retaining a newer panel credential would mislead the owner and - // can leave them unable to log in after a successful rollback. - $instance->sealed_credential = $backup->sealed_credential; + $instance->sealed_credential = null; $instance->updated_at = $now; $instance->save(); $recovery->state = 'succeeded'; diff --git a/web/tests/CloudPortableBackupFinalizerTest.php b/web/tests/CloudPortableBackupFinalizerTest.php index ed75b167..c6c0a214 100644 --- a/web/tests/CloudPortableBackupFinalizerTest.php +++ b/web/tests/CloudPortableBackupFinalizerTest.php @@ -460,6 +460,7 @@ $capsule->table('cloud_instances')->insert([ 'network_status' => 'active', 'spec_json' => json_encode(array_replace($sourceSpec, [ 'guest_hostname' => 'current-host', + 'public_ip_cidr' => '198.51.100.99/24', ]), JSON_THROW_ON_ERROR), 'sealed_credential' => 'sealed-current-credential', 'updated_at' => $now, @@ -518,11 +519,13 @@ $assert((string) $recovery->state === 'succeeded' && (string) $recoveredInstance->uuid === '11111111-1111-4111-8111-111111111111' && (string) $recoveredInstance->status === 'active' - && (string) $recoveredInstance->sealed_credential - === 'sealed-source-credential' - && ($recoveredSpec['guest_hostname'] ?? null) === 'backup-host' + && $recoveredInstance->sealed_credential === null + && !array_key_exists('guest_hostname', $recoveredSpec) + && !array_key_exists('hostname', $recoveredSpec) + && ($recoveredSpec['public_ip_cidr'] ?? null) + === '198.51.100.99/24' && ($recoveredSpec['restored_from_backup'] ?? null) === str_repeat('a', 32), - 'In-place backup recovery did not preserve the business instance while restoring guest identity.'); + 'In-place backup recovery did not preserve disk-authoritative guest identity and the current network.'); echo "Portable backup create/restore finalizer behavior tests passed.\n";