fix(cloud): preserve disk guest identity on recovery

go
codex 3 weeks ago
parent dcdb33afff
commit 896fbeaf55

@ -153,6 +153,7 @@ func RecoverBackupAdmission(uuid string, request RecoverBackupRequest) MutationA
domainLockKey(uuid), vmManagedVolumesLockKey(uuid),
volumeLockKey(managedStoragePoolName,
snapshotRestoreVolumeName(uuid, request.OperationID)),
volumeLockKey(seedStoragePoolName, seedVolumeName(uuid)),
catalogLockKey("backup", request.BackupKey),
}}
}

@ -165,6 +165,7 @@ func TestBackupAdmissionsFenceSourceAndNewVMArtifacts(t *testing.T) {
domainLockKey(uuid), vmManagedVolumesLockKey(uuid),
volumeLockKey(managedStoragePoolName,
snapshotRestoreVolumeName(uuid, recoveryRequest.OperationID)),
volumeLockKey(seedStoragePoolName, seedVolumeName(uuid)),
catalogLockKey("backup", recoveryRequest.BackupKey),
} {
if !containsString(recovery.LockKeys, key) || !recovery.HeavyIO {

@ -28,11 +28,17 @@ import (
)
const (
maxSeedImageBytes = 2 << 20
maxTemplateBytes = int64(4) << 30
maxUserISOBytes = int64(32) << 30
maxRescueISOBytes = int64(8) << 30
minImportBytes = int64(1) << 20
maxSeedImageBytes = 2 << 20
maxTemplateBytes = int64(4) << 30
maxUserISOBytes = int64(32) << 30
maxRescueISOBytes = int64(8) << 30
minImportBytes = int64(1) << 20
backupRecoveryOperationMetadata = "virt-cloud-backup-recovery-operation"
diskGuestIdentityMetadata = "virt-cloud-guest-identity"
diskGuestIdentityValue = "disk"
diskSSHAuthorizedKeysPath = "/root/.ssh/authorized_keys"
diskSSHDirectoryCommand = "install -d -m 0700 -o root -g root /root/.ssh"
diskSSHRemoveCommand = "rm -f -- /root/.ssh/authorized_keys"
)
var (
@ -125,28 +131,67 @@ func mutateSeedSettings(contents []byte, request SettingsRequest, transitionID s
if err := json.Unmarshal(bytes.TrimSpace(files.UserData[len(header):]), &userData); err != nil {
return nil, "", NewError(ErrConflict, "invalid_seed", "NoCloud user-data is invalid", err)
}
users, ok := userData["users"].([]any)
if !ok || len(users) != 1 {
return nil, "", NewError(ErrConflict, "invalid_seed", "NoCloud root account definition is invalid", nil)
var metadata map[string]any
if err := json.Unmarshal(bytes.TrimSpace(files.MetaData), &metadata); err != nil {
return nil, "", NewError(ErrConflict, "invalid_seed", "NoCloud meta-data is invalid", err)
}
root, ok := users[0].(map[string]any)
if !ok || root["name"] != "root" {
return nil, "", NewError(ErrConflict, "invalid_seed", "NoCloud root account definition is invalid", nil)
instanceID, ok := metadata["instance-id"].(string)
if !ok || strings.TrimSpace(instanceID) == "" {
return nil, "", NewError(ErrConflict, "invalid_seed", "NoCloud instance identity is invalid", nil)
}
diskAuthoritative := metadata[diskGuestIdentityMetadata] == diskGuestIdentityValue
var root map[string]any
if diskAuthoritative {
// Recovery itself removes every pre-recovery identity instruction.
// From this point on, however, settings explicitly submitted by the
// user are new intent and must survive later settings transitions until
// that same field is changed again. This also prevents a quick second
// operation from deleting a password/key before cloud-init consumes it.
if users, exists := userData["users"]; exists {
parsed, valid := users.([]any)
if !valid || len(parsed) != 1 {
return nil, "", NewError(ErrConflict, "invalid_seed", "NoCloud root account definition is invalid", nil)
}
root, valid = parsed[0].(map[string]any)
if !valid || root["name"] != "root" {
return nil, "", NewError(ErrConflict, "invalid_seed", "NoCloud root account definition is invalid", nil)
}
}
if request.RootPassword != nil {
if root == nil {
root = map[string]any{"name": "root", "shell": "/bin/bash"}
userData["users"] = []any{root}
}
}
} else {
users, valid := userData["users"].([]any)
if !valid || len(users) != 1 {
return nil, "", NewError(ErrConflict, "invalid_seed", "NoCloud root account definition is invalid", nil)
}
root, valid = users[0].(map[string]any)
if !valid || root["name"] != "root" {
return nil, "", NewError(ErrConflict, "invalid_seed", "NoCloud root account definition is invalid", nil)
}
}
if request.SSHPublicKey != nil {
key := *request.SSHPublicKey
if err := validateOptionalSSHKey(key); err != nil {
return nil, "", err
}
if key == "" {
if diskAuthoritative {
removeDiskAuthoritativeSSHKeyMutation(userData)
setDiskAuthoritativeSSHKeyMutation(userData, key)
} else if key == "" {
delete(root, "ssh_authorized_keys")
} else {
root["ssh_authorized_keys"] = []string{key}
}
}
passwordEnabled := false
if hash, ok := root["hashed_passwd"].(string); ok && hash != "" {
passwordEnabled = true
if root != nil {
if hash, present := root["hashed_passwd"].(string); present && hash != "" {
passwordEnabled = true
}
}
if request.RootPassword != nil {
password := *request.RootPassword
@ -172,22 +217,23 @@ func mutateSeedSettings(contents []byte, request SettingsRequest, transitionID s
passwordEnabled = true
}
}
keys, _ := root["ssh_authorized_keys"].([]any)
stringKeys, _ := root["ssh_authorized_keys"].([]string)
if !passwordEnabled && len(keys) == 0 && len(stringKeys) == 0 {
return nil, "", NewError(ErrInvalid, "guest_credentials_required", "at least one root login method must remain enabled", nil)
}
userData["ssh_pwauth"] = passwordEnabled
updateSSHDWriteFile(userData, passwordEnabled)
var metadata map[string]any
if err := json.Unmarshal(bytes.TrimSpace(files.MetaData), &metadata); err != nil {
return nil, "", NewError(ErrConflict, "invalid_seed", "NoCloud meta-data is invalid", err)
if diskAuthoritative {
if request.RootPassword != nil && request.SSHPublicKey != nil &&
*request.RootPassword == "" && *request.SSHPublicKey == "" {
return nil, "", NewError(ErrInvalid, "guest_credentials_required", "at least one root login method must remain enabled", nil)
}
} else {
keys, _ := root["ssh_authorized_keys"].([]any)
stringKeys, _ := root["ssh_authorized_keys"].([]string)
if root != nil && !passwordEnabled && len(keys) == 0 && len(stringKeys) == 0 {
return nil, "", NewError(ErrInvalid, "guest_credentials_required", "at least one root login method must remain enabled", nil)
}
}
instanceID, ok := metadata["instance-id"].(string)
if !ok || strings.TrimSpace(instanceID) == "" {
return nil, "", NewError(ErrConflict, "invalid_seed", "NoCloud instance identity is invalid", nil)
if !diskAuthoritative || request.RootPassword != nil {
userData["ssh_pwauth"] = passwordEnabled
updateSSHDWriteFile(userData, passwordEnabled)
}
hostname := ""
if request.Hostname != nil {
hostname = *request.Hostname
@ -195,6 +241,9 @@ func mutateSeedSettings(contents []byte, request SettingsRequest, transitionID s
return nil, "", NewError(ErrInvalid, "invalid_hostname", "hostname must be a lower-case DNS name", nil)
}
metadata["local-hostname"] = hostname
if diskAuthoritative {
userData["preserve_hostname"] = false
}
}
generation, err := newRandomID()
if err != nil {
@ -215,6 +264,114 @@ func mutateSeedSettings(contents []byte, request SettingsRequest, transitionID s
return updated, hostname, err
}
// setDiskAuthoritativeSSHKeyMutation updates only authorized_keys. In
// particular it does not use cloud-init's users module: lock_passwd defaults
// to true there, which would silently lock a password restored from disk when
// the user only asked to change an SSH key.
func setDiskAuthoritativeSSHKeyMutation(userData map[string]any, key string) {
if key == "" {
appendSeedCommand(userData, "runcmd", diskSSHRemoveCommand)
return
}
appendSeedCommand(userData, "bootcmd", diskSSHDirectoryCommand)
files, _ := userData["write_files"].([]any)
files = append(files, map[string]any{
"path": diskSSHAuthorizedKeysPath, "content": key + "\n",
"owner": "root:root", "permissions": "0600",
})
userData["write_files"] = files
}
func removeDiskAuthoritativeSSHKeyMutation(userData map[string]any) {
files, _ := userData["write_files"].([]any)
kept := make([]any, 0, len(files))
for _, raw := range files {
entry, ok := raw.(map[string]any)
if ok && entry["path"] == diskSSHAuthorizedKeysPath {
continue
}
kept = append(kept, raw)
}
if len(kept) == 0 {
delete(userData, "write_files")
} else {
userData["write_files"] = kept
}
removeSeedCommand(userData, "bootcmd", diskSSHDirectoryCommand)
removeSeedCommand(userData, "runcmd", diskSSHRemoveCommand)
}
func appendSeedCommand(userData map[string]any, field, script string) {
commands, _ := userData[field].([]any)
commands = append(commands, []string{"/bin/sh", "-ec", script})
userData[field] = commands
}
func removeSeedCommand(userData map[string]any, field, script string) {
commands, _ := userData[field].([]any)
kept := make([]any, 0, len(commands))
for _, raw := range commands {
parts, ok := raw.([]any)
if ok && len(parts) == 3 && parts[0] == "/bin/sh" &&
parts[1] == "-ec" && parts[2] == script {
continue
}
kept = append(kept, raw)
}
if len(kept) == 0 {
delete(userData, field)
} else {
userData[field] = kept
}
}
func sanitizeSeedForDiskAuthoritativeRecovery(contents []byte,
transitionID string) ([]byte, error) {
if !operationIdentifierPattern.MatchString(transitionID) {
return nil, NewError(ErrInvalid, "invalid_operation_id", "backup recovery transition id is invalid", nil)
}
files, err := readSeedISO(contents)
if err != nil {
return nil, err
}
const header = "#cloud-config\n"
if !bytes.HasPrefix(files.UserData, []byte(header)) {
return nil, NewError(ErrConflict, "invalid_seed", "NoCloud user-data is not managed by Virt Cloud", nil)
}
var original map[string]any
if err := json.Unmarshal(bytes.TrimSpace(files.UserData[len(header):]), &original); err != nil {
return nil, NewError(ErrConflict, "invalid_seed", "NoCloud user-data is invalid", err)
}
userData := map[string]any{
"preserve_hostname": true,
}
for _, key := range []string{"updates", "bootcmd"} {
if value, exists := original[key]; exists {
userData[key] = value
}
}
var metadata map[string]any
if err := json.Unmarshal(bytes.TrimSpace(files.MetaData), &metadata); err != nil {
return nil, NewError(ErrConflict, "invalid_seed", "NoCloud meta-data is invalid", err)
}
instanceID, ok := metadata["instance-id"].(string)
if !ok || strings.TrimSpace(instanceID) == "" {
return nil, NewError(ErrConflict, "invalid_seed", "NoCloud instance identity is invalid", nil)
}
delete(metadata, "local-hostname")
metadata[diskGuestIdentityMetadata] = diskGuestIdentityValue
metadata[backupRecoveryOperationMetadata] = transitionID
files.UserData, err = marshalCloudConfig(userData)
if err != nil {
return nil, err
}
files.MetaData, err = marshalJSONLine(metadata)
if err != nil {
return nil, err
}
return buildSeedISO(files)
}
func updateSSHDWriteFile(userData map[string]any, passwordEnabled bool) {
content := "PermitRootLogin prohibit-password\nPasswordAuthentication no\nKbdInteractiveAuthentication no\n"
if passwordEnabled {

@ -2,6 +2,7 @@ package hypervisor
import (
"bytes"
"encoding/base64"
"encoding/binary"
"encoding/json"
"net/http"
@ -13,6 +14,18 @@ import (
"git.luoli.live/root/virt/agent/cloudinit"
)
func validSSHKeyForTest() string {
keyBlob := make([]byte, 4+len("ssh-ed25519")+4+32)
binary.BigEndian.PutUint32(keyBlob[:4], uint32(len("ssh-ed25519")))
copy(keyBlob[4:], "ssh-ed25519")
offset := 4 + len("ssh-ed25519")
binary.BigEndian.PutUint32(keyBlob[offset:offset+4], 32)
for index := offset + 4; index < len(keyBlob); index++ {
keyBlob[index] = byte(index)
}
return "ssh-ed25519 " + base64.StdEncoding.EncodeToString(keyBlob) + " user@test"
}
func managedSeedForTest(t *testing.T) []byte {
t.Helper()
image, err := cloudinit.BuildISO(cloudinit.Request{
@ -75,6 +88,251 @@ func TestMutateSeedSettingsPreservesStartupAndHashesNewPassword(t *testing.T) {
}
}
func TestSanitizeSeedForDiskAuthoritativeRecoveryKeepsOnlyNetworkIntent(t *testing.T) {
original := managedSeedForTest(t)
originalFiles, err := readSeedISO(original)
if err != nil {
t.Fatal(err)
}
sanitized, err := sanitizeSeedForDiskAuthoritativeRecovery(
original, "operation-recover-backup",
)
if err != nil {
t.Fatal(err)
}
files, err := readSeedISO(sanitized)
if err != nil {
t.Fatal(err)
}
if !bytes.Equal(files.NetworkConfig, originalFiles.NetworkConfig) {
t.Fatal("backup recovery changed the current network-config")
}
var userData map[string]any
if err := json.Unmarshal(bytes.TrimPrefix(
files.UserData, []byte("#cloud-config\n")), &userData); err != nil {
t.Fatal(err)
}
for _, forbidden := range []string{
"users", "chpasswd", "ssh_pwauth", "disable_root",
"write_files", "runcmd",
} {
if _, exists := userData[forbidden]; exists {
t.Fatalf("disk-authoritative recovery seed contains %q: %#v", forbidden, userData)
}
}
if userData["preserve_hostname"] != true {
t.Fatalf("preserve_hostname = %#v", userData["preserve_hostname"])
}
if bytes.Contains(files.UserData, []byte("echo once")) ||
bytes.Contains(files.UserData, []byte("hashed_passwd")) {
t.Fatalf("recovery seed retained guest identity or startup payload: %s", files.UserData)
}
var originalMetadata, metadata map[string]any
if err := json.Unmarshal(bytes.TrimSpace(originalFiles.MetaData), &originalMetadata); err != nil {
t.Fatal(err)
}
if err := json.Unmarshal(bytes.TrimSpace(files.MetaData), &metadata); err != nil {
t.Fatal(err)
}
if metadata["instance-id"] != originalMetadata["instance-id"] {
t.Fatalf("instance-id changed: old=%#v new=%#v",
originalMetadata["instance-id"], metadata["instance-id"])
}
if _, exists := metadata["local-hostname"]; exists {
t.Fatalf("recovery seed retained local-hostname: %#v", metadata)
}
if metadata["virt-cloud-backup-recovery-operation"] !=
"operation-recover-backup" {
t.Fatalf("backup recovery marker = %#v",
metadata["virt-cloud-backup-recovery-operation"])
}
}
func TestDiskAuthoritativeRecoverySeedAllowsExplicitFutureSettings(t *testing.T) {
sanitized, err := sanitizeSeedForDiskAuthoritativeRecovery(
managedSeedForTest(t), "operation-recover-backup",
)
if err != nil {
t.Fatal(err)
}
hostname := "after-recovery.example.com"
hostnameOnly, resultHostname, err := mutateSeedSettings(
sanitized, SettingsRequest{Hostname: &hostname}, "operation-hostname",
)
if err != nil {
t.Fatal(err)
}
if resultHostname != hostname {
t.Fatalf("hostname = %q", resultHostname)
}
hostnameFiles, err := readSeedISO(hostnameOnly)
if err != nil {
t.Fatal(err)
}
var hostnameData map[string]any
if err := json.Unmarshal(bytes.TrimPrefix(
hostnameFiles.UserData, []byte("#cloud-config\n")), &hostnameData); err != nil {
t.Fatal(err)
}
for _, forbidden := range []string{"users", "chpasswd", "ssh_pwauth"} {
if _, exists := hostnameData[forbidden]; exists {
t.Fatalf("hostname-only update injected %q: %#v", forbidden, hostnameData)
}
}
password := "ExplicitReset!456"
passwordReset, _, err := mutateSeedSettings(
sanitized, SettingsRequest{RootPassword: &password}, "operation-password",
)
if err != nil {
t.Fatal(err)
}
if bytes.Contains(passwordReset, []byte(password)) {
t.Fatal("explicit password reset stored the plaintext password")
}
passwordFiles, err := readSeedISO(passwordReset)
if err != nil {
t.Fatal(err)
}
var passwordData map[string]any
if err := json.Unmarshal(bytes.TrimPrefix(
passwordFiles.UserData, []byte("#cloud-config\n")), &passwordData); err != nil {
t.Fatal(err)
}
users, ok := passwordData["users"].([]any)
if !ok || len(users) != 1 || passwordData["chpasswd"] == nil {
t.Fatalf("explicit password reset did not create managed credentials: %#v", passwordData)
}
passwordAfterHostname, _, err := mutateSeedSettings(
hostnameOnly, SettingsRequest{RootPassword: &password},
"operation-password-after-hostname",
)
if err != nil {
t.Fatal(err)
}
passwordAfterHostnameFiles, err := readSeedISO(passwordAfterHostname)
if err != nil {
t.Fatal(err)
}
var passwordAfterHostnameData, passwordAfterHostnameMetadata map[string]any
if err := json.Unmarshal(bytes.TrimPrefix(
passwordAfterHostnameFiles.UserData, []byte("#cloud-config\n")),
&passwordAfterHostnameData); err != nil {
t.Fatal(err)
}
if err := json.Unmarshal(bytes.TrimSpace(
passwordAfterHostnameFiles.MetaData), &passwordAfterHostnameMetadata); err != nil {
t.Fatal(err)
}
if passwordAfterHostnameData["preserve_hostname"] != false {
t.Fatalf("password-only update did not retain the explicit hostname: %#v",
passwordAfterHostnameData)
}
if passwordAfterHostnameMetadata["local-hostname"] != hostname {
t.Fatalf("password-only update lost the explicit hostname: %#v",
passwordAfterHostnameMetadata)
}
hostnameAfterPassword, _, err := mutateSeedSettings(
passwordReset, SettingsRequest{Hostname: &hostname},
"operation-hostname-after-password",
)
if err != nil {
t.Fatal(err)
}
hostnameAfterPasswordFiles, err := readSeedISO(hostnameAfterPassword)
if err != nil {
t.Fatal(err)
}
var hostnameAfterPasswordData map[string]any
if err := json.Unmarshal(bytes.TrimPrefix(
hostnameAfterPasswordFiles.UserData, []byte("#cloud-config\n")),
&hostnameAfterPasswordData); err != nil {
t.Fatal(err)
}
users, ok = hostnameAfterPasswordData["users"].([]any)
if !ok || len(users) != 1 || hostnameAfterPasswordData["chpasswd"] == nil ||
hostnameAfterPasswordData["ssh_pwauth"] != true {
t.Fatalf("hostname-only update lost the explicit password intent: %#v",
hostnameAfterPasswordData)
}
}
func TestDiskAuthoritativeRecoverySSHKeyDoesNotChangePasswordState(t *testing.T) {
sanitized, err := sanitizeSeedForDiskAuthoritativeRecovery(
managedSeedForTest(t), "operation-recover-backup",
)
if err != nil {
t.Fatal(err)
}
key := validSSHKeyForTest()
updated, _, err := mutateSeedSettings(
sanitized, SettingsRequest{SSHPublicKey: &key}, "operation-ssh-key",
)
if err != nil {
t.Fatal(err)
}
files, err := readSeedISO(updated)
if err != nil {
t.Fatal(err)
}
var userData map[string]any
if err := json.Unmarshal(bytes.TrimPrefix(
files.UserData, []byte("#cloud-config\n")), &userData); err != nil {
t.Fatal(err)
}
for _, forbidden := range []string{
"users", "chpasswd", "ssh_pwauth", "disable_root",
} {
if _, exists := userData[forbidden]; exists {
t.Fatalf("SSH-key-only update changed password state through %q: %#v",
forbidden, userData)
}
}
if bytes.Contains(files.UserData, []byte("lock_passwd")) ||
bytes.Contains(files.UserData, []byte("hashed_passwd")) {
t.Fatalf("SSH-key-only update included password state: %s", files.UserData)
}
writeFiles, ok := userData["write_files"].([]any)
if !ok || len(writeFiles) != 1 {
t.Fatalf("SSH-key-only update write_files = %#v", userData["write_files"])
}
entry, ok := writeFiles[0].(map[string]any)
if !ok || entry["path"] != diskSSHAuthorizedKeysPath ||
entry["content"] != key+"\n" || entry["permissions"] != "0600" {
t.Fatalf("SSH-key-only update file = %#v", writeFiles[0])
}
if !bytes.Equal(files.NetworkConfig,
mustReadSeedFiles(t, sanitized).NetworkConfig) {
t.Fatal("SSH-key-only update changed network-config")
}
hostname := "after-key.example.com"
hostnameOnly, _, err := mutateSeedSettings(
updated, SettingsRequest{Hostname: &hostname}, "operation-hostname-after-key",
)
if err != nil {
t.Fatal(err)
}
hostnameFiles := mustReadSeedFiles(t, hostnameOnly)
if !bytes.Contains(hostnameFiles.UserData, []byte(diskSSHAuthorizedKeysPath)) ||
!bytes.Contains(hostnameFiles.UserData, []byte(key)) {
t.Fatalf("later hostname update lost the explicit SSH key intent: %s",
hostnameFiles.UserData)
}
}
func mustReadSeedFiles(t *testing.T, contents []byte) seedFiles {
t.Helper()
files, err := readSeedISO(contents)
if err != nil {
t.Fatal(err)
}
return files
}
func TestMutateSeedNetworkChangesOnlyAddressIntent(t *testing.T) {
updated, mac, err := mutateSeedNetwork(managedSeedForTest(t), "9.9.9.10/24", "9.9.9.1", []string{"1.1.1.1", "8.8.8.8"}, "operation-network")
if err != nil {

@ -12,10 +12,11 @@ import (
"libvirt.org/go/libvirtxml"
)
// RecoverBackup replaces only the managed root disk of an existing domain.
// The UUID, domain definition, NoCloud seed, public IP, firewall and service
// identity remain untouched. The disk transition follows the same
// fail-closed two-root protocol as snapshot restore.
// RecoverBackup replaces the managed root disk of an existing domain and
// reduces its NoCloud seed to current network intent. Guest credentials and
// hostname remain authoritative on the restored disk. The disk/seed
// transition follows the same fail-closed two-root protocol as snapshot
// restore.
func (l *Libvirt) RecoverBackup(ctx context.Context, uuid string,
request RecoverBackupRequest) (_ RecoverBackupResult, resultErr error) {
mutationStarted := false
@ -38,6 +39,27 @@ func (l *Libvirt) RecoverBackup(ctx context.Context, uuid string,
}
defer pool.Free()
defer oldRoot.Free()
seed, err := l.lookupOptionalManagedVolume(
l.storagePoolName(StorageRoleSeed), seedVolumeName(uuid),
)
if err != nil {
return RecoverBackupResult{}, err
}
if seed == nil {
return RecoverBackupResult{}, NewError(ErrConflict, "seed_not_found",
"managed NoCloud seed volume was not found", nil)
}
defer seed.Free()
originalSeed, err := l.downloadSmallVolume(ctx, seed, maxSeedImageBytes)
if err != nil {
return RecoverBackupResult{}, err
}
recoverySeed, err := sanitizeSeedForDiskAuthoritativeRecovery(
originalSeed, request.OperationID,
)
if err != nil {
return RecoverBackupResult{}, err
}
if metadata.PendingCleanupVolume != "" {
return RecoverBackupResult{}, NewError(ErrConflict,
"backup_disk_transition_active",
@ -104,6 +126,10 @@ func (l *Libvirt) RecoverBackup(ctx context.Context, uuid string,
stored.DiskTransitionKind = "backup_recover"
stored.DiskTransitionID = request.OperationID
stored.DiskTransitionRunning = wasRunning
// The recovered system disk owns the guest hostname. Keeping the
// pre-recovery value here would make later inspection and operations
// incorrectly present panel metadata as the guest's current hostname.
stored.Hostname = ""
return nil
})
if err != nil {
@ -116,23 +142,26 @@ func (l *Libvirt) RecoverBackup(ctx context.Context, uuid string,
}
return RecoverBackupResult{}, operation.MarkNotApplied(err)
}
defined, err := l.conn.DomainDefineXML(updatedXML)
if err != nil {
_, observed, inspectErr := requireManagedDomain(domain)
if inspectErr != nil || observed.RootVolumeName != metadata.RootVolumeName {
return RecoverBackupResult{}, operation.MarkUncertain(errors.Join(
mutationLibvirtError("switch backup recovery system disk", err), inspectErr))
if err := l.replaceSeedAndDefinition(
ctx, seed, originalSeed, recoverySeed, originalXML, updatedXML, false,
); err != nil {
if operation.IsUncertain(err) {
return RecoverBackupResult{}, err
}
restartErr := restartOriginalDomain(domain, wasRunning)
cleanupErr := replacement.Delete(libvirt.STORAGE_VOL_DELETE_NORMAL)
if restartErr != nil || cleanupErr != nil {
return RecoverBackupResult{}, operation.MarkUncertain(errors.Join(
mutationLibvirtError("switch backup recovery system disk", err),
err,
restartErr,
optionalMutationLibvirtError("roll back backup recovery volume", cleanupErr)))
}
return RecoverBackupResult{}, operation.MarkNotApplied(
mutationLibvirtError("switch backup recovery system disk", err))
return RecoverBackupResult{}, operation.MarkNotApplied(err)
}
defined, err := l.domainByUUID(ctx, uuid)
if err != nil {
return RecoverBackupResult{}, operation.MarkUncertain(
mutationLibvirtError("resolve recovered domain", err))
}
defer defined.Free()
if err := defined.SetAutostart(autostart); err != nil {

@ -146,7 +146,7 @@ func (l *Libvirt) inspectSuccessfulBackupCleanupPending(ctx context.Context,
}
defer domain.Free()
_, cleanup := l.inspectSnapshotOperation(record, domain)
_, cleanup := l.inspectSnapshotOperation(ctx, record, domain)
return appliedBackupInspectionWithCleanup(record, inspection, cleanup)
}
@ -245,7 +245,7 @@ func (l *Libvirt) inspectExistingDomainOperation(ctx context.Context, record ope
strings.HasPrefix(record.Action, "delete-snapshot:") ||
strings.HasPrefix(record.Action, "restore-snapshot:") ||
strings.HasPrefix(record.Action, "recover-backup:") {
resolution, steps := l.inspectSnapshotOperation(record, domain)
resolution, steps := l.inspectSnapshotOperation(ctx, record, domain)
return resolution, steps, nil
}
if record.Action == "enter-rescue" || record.Action == "force-start-rescue" ||
@ -598,7 +598,7 @@ func (l *Libvirt) inspectReinstallOperation(record operation.Record,
return operation.Resolution{}, operationCleanupInspection{}
}
func (l *Libvirt) inspectSnapshotOperation(record operation.Record,
func (l *Libvirt) inspectSnapshotOperation(ctx context.Context, record operation.Record,
domain *libvirt.Domain) (operation.Resolution, operationCleanupInspection) {
parts := strings.Split(record.Action, ":")
if len(parts) < 2 || !catalogIDPattern.MatchString(parts[1]) {
@ -733,8 +733,24 @@ func (l *Libvirt) inspectSnapshotOperation(record operation.Record,
if replacement != nil {
replacement.Free()
}
if diskTransitionCommitted(metadata, replacementName,
"backup_recover", record.OperationID) && replacementExists {
seedContents, seedErr := l.managedSeedContents(ctx, record.ResourceID)
if seedErr != nil {
return operation.Resolution{}, operationCleanupInspection{}
}
seedOperation, seedErr := seedMetadataMarker(
seedContents, backupRecoveryOperationMetadata,
)
if seedErr != nil {
return operation.Resolution{}, operationCleanupInspection{}
}
seedCommitted := seedOperation == record.OperationID
diskCommitted := diskTransitionCommitted(metadata, replacementName,
"backup_recover", record.OperationID)
evidence := classifyBackupRecoveryEvidence(
replacementExists, diskCommitted,
metadata.DiskTransitionID == record.OperationID, seedCommitted,
)
if evidence == backupRecoveryEvidenceApplied {
state, _, stateErr := domain.GetState()
if stateErr != nil || !diskTransitionPowerStateMatches(
metadata, state, "backup_recover", record.OperationID) {
@ -751,7 +767,7 @@ func (l *Libvirt) inspectSnapshotOperation(record operation.Record,
return operation.Resolution{Resolved: true, Succeeded: true,
Result: result}, operationCleanupInspection{}
}
if !replacementExists && metadata.DiskTransitionID != record.OperationID {
if evidence == backupRecoveryEvidenceNotApplied {
return operation.Resolution{Resolved: true, Succeeded: false,
ErrorCode: "backup_not_recovered",
ErrorMessage: "host evidence confirms the root volume was not switched"},
@ -761,6 +777,28 @@ func (l *Libvirt) inspectSnapshotOperation(record operation.Record,
return operation.Resolution{}, operationCleanupInspection{}
}
type backupRecoveryEvidence uint8
const (
backupRecoveryEvidenceUnknown backupRecoveryEvidence = iota
backupRecoveryEvidenceApplied
backupRecoveryEvidenceNotApplied
)
// classifyBackupRecoveryEvidence is deliberately fail-closed. Recovery is
// applied only when both the root-disk transition and the seed transition are
// visible. Any partial evidence remains unresolved and fenced.
func classifyBackupRecoveryEvidence(replacementExists, diskCommitted,
diskTransitionClaimed, seedCommitted bool) backupRecoveryEvidence {
if replacementExists && diskCommitted && seedCommitted {
return backupRecoveryEvidenceApplied
}
if !replacementExists && !diskTransitionClaimed && !seedCommitted {
return backupRecoveryEvidenceNotApplied
}
return backupRecoveryEvidenceUnknown
}
func backupResidueCleanupAllowed(record operation.Record, stored *domainSnapshotMetadataXML) bool {
parts := strings.Split(record.Action, ":")
if len(parts) != 3 || parts[0] != "create-backup" ||

@ -99,6 +99,30 @@ func TestCleanupJobEvidenceRequiresIdleDomainAndBlockJobs(t *testing.T) {
}
}
func TestBackupRecoveryEvidenceRequiresRootAndSeedConvergence(t *testing.T) {
tests := []struct {
name string
replacement, disk, diskClaim, seed bool
want backupRecoveryEvidence
}{
{name: "root and seed committed", replacement: true, disk: true, diskClaim: true, seed: true, want: backupRecoveryEvidenceApplied},
{name: "nothing dispatched", want: backupRecoveryEvidenceNotApplied},
{name: "seed only", seed: true, want: backupRecoveryEvidenceUnknown},
{name: "root only", replacement: true, disk: true, diskClaim: true, want: backupRecoveryEvidenceUnknown},
{name: "replacement residue only", replacement: true, want: backupRecoveryEvidenceUnknown},
{name: "transition claim without volume", diskClaim: true, want: backupRecoveryEvidenceUnknown},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
if got := classifyBackupRecoveryEvidence(
test.replacement, test.disk, test.diskClaim, test.seed,
); got != test.want {
t.Fatalf("evidence = %v, want %v", got, test.want)
}
})
}
}
func TestCleanupPlanIdentifierBindsPrivateExactEvidence(t *testing.T) {
record := operation.Record{
OperationID: "operation-exact-cleanup-plan", ResourceID: "123e4567-e89b-42d3-a456-426614174000",

@ -298,9 +298,10 @@ type RestoreBackupResult struct {
}
// RecoverBackupRequest replaces the managed root disk of an existing VM with
// one immutable portable-backup object. Network identity, NoCloud seed,
// billing identity and the domain UUID are deliberately outside this request
// and remain attached to the existing domain.
// one immutable portable-backup object. Network and domain identity remain
// attached to the existing VM, while the Agent reduces its NoCloud seed to
// network-only intent so the restored disk keeps its own credentials and
// hostname.
type RecoverBackupRequest struct {
OperationID string `json:"operation_id,omitempty"`
BackupKey string `json:"backup_key"`

@ -1503,7 +1503,7 @@ final class CloudBackupService
foreach ([
'template_id', 'template_revision',
'template_generation_token', 'os_source', 'os_name',
'login_name', 'debian_12_network_fix', 'guest_hostname',
'login_name', 'debian_12_network_fix',
] as $field) {
if (array_key_exists($field, $backupSpec)) {
$instanceSpec[$field] = $backupSpec[$field];
@ -1512,6 +1512,11 @@ final class CloudBackupService
}
}
unset($instanceSpec['install_iso']);
// Passwords and hostnames may have been changed inside the guest
// without passing through the panel. The restored system disk is
// authoritative, so the panel must not claim either value is
// known or cause the Agent's NoCloud seed to inject an old value.
unset($instanceSpec['guest_hostname'], $instanceSpec['hostname']);
$instanceSpec['managed'] = true;
$instanceSpec['restored_from_backup'] =
(string) $backup->backup_key;
@ -1520,10 +1525,7 @@ final class CloudBackupService
JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES
| JSON_UNESCAPED_UNICODE
);
// The restored disk contains the backup-time guest credentials.
// Retaining a newer panel credential would mislead the owner and
// can leave them unable to log in after a successful rollback.
$instance->sealed_credential = $backup->sealed_credential;
$instance->sealed_credential = null;
$instance->updated_at = $now;
$instance->save();
$recovery->state = 'succeeded';

@ -460,6 +460,7 @@ $capsule->table('cloud_instances')->insert([
'network_status' => 'active',
'spec_json' => json_encode(array_replace($sourceSpec, [
'guest_hostname' => 'current-host',
'public_ip_cidr' => '198.51.100.99/24',
]), JSON_THROW_ON_ERROR),
'sealed_credential' => 'sealed-current-credential',
'updated_at' => $now,
@ -518,11 +519,13 @@ $assert((string) $recovery->state === 'succeeded'
&& (string) $recoveredInstance->uuid
=== '11111111-1111-4111-8111-111111111111'
&& (string) $recoveredInstance->status === 'active'
&& (string) $recoveredInstance->sealed_credential
=== 'sealed-source-credential'
&& ($recoveredSpec['guest_hostname'] ?? null) === 'backup-host'
&& $recoveredInstance->sealed_credential === null
&& !array_key_exists('guest_hostname', $recoveredSpec)
&& !array_key_exists('hostname', $recoveredSpec)
&& ($recoveredSpec['public_ip_cidr'] ?? null)
=== '198.51.100.99/24'
&& ($recoveredSpec['restored_from_backup'] ?? null)
=== str_repeat('a', 32),
'In-place backup recovery did not preserve the business instance while restoring guest identity.');
'In-place backup recovery did not preserve disk-authoritative guest identity and the current network.');
echo "Portable backup create/restore finalizer behavior tests passed.\n";

Loading…
Cancel
Save