You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
virt/agent/hypervisor/libvirt_reinstall.go

273 lines
11 KiB
Go

//go:build libvirt
package hypervisor
import (
"context"
"errors"
"git.luoli.live/root/virt/agent/cloudinit"
"git.luoli.live/root/virt/agent/operation"
"libvirt.org/go/libvirt"
"libvirt.org/go/libvirtxml"
)
func (l *Libvirt) Reinstall(ctx context.Context, uuid string, request ReinstallRequest) (_ VMMetadata, resultErr error) {
mutationStarted := false
defer func() {
if !mutationStarted {
resultErr = operation.MarkNotApplied(resultErr)
}
}()
if err := ValidateTemplateSnapshot(request.TemplateID, request.TemplateRevision, request.TemplateGenerationToken); err != nil {
return VMMetadata{}, err
}
if !operationIdentifierPattern.MatchString(request.OperationID) {
return VMMetadata{}, NewError(ErrInvalid, "invalid_operation_id", "operation_id is required for safe reinstall", nil)
}
if request.PreserveData {
return VMMetadata{}, NewError(ErrInvalid, "preserve_data_not_allowed", "a clean reinstall cannot preserve the system disk", nil)
}
if err := validateGuestCredentials(request.SSHPublicKey, request.RootPassword); err != nil {
return VMMetadata{}, err
}
template, err := l.loadImageTemplate(ctx, request.TemplateID)
if err != nil {
return VMMetadata{}, err
}
if template.GenerationToken != request.TemplateGenerationToken {
return VMMetadata{}, NewError(ErrConflict, "template_generation_changed", "template generation token no longer matches", nil)
}
domain, err := l.domainByUUID(ctx, uuid)
if err != nil {
return VMMetadata{}, err
}
defer domain.Free()
originalXML, _, metadata, managedPool, oldRoot, err := l.managedRootContext(domain)
if err != nil {
return VMMetadata{}, err
}
defer managedPool.Free()
defer oldRoot.Free()
autostart, err := domain.GetAutostart()
if err != nil {
return VMMetadata{}, translateLibvirtError("read reinstall domain autostart", err)
}
if len(metadata.Snapshots) != 0 {
return VMMetadata{}, NewError(ErrConflict, "snapshots_present", "reinstall is blocked while managed snapshots exist", nil)
}
seed, err := l.lookupOptionalManagedVolume(l.storagePoolName(StorageRoleSeed), seedVolumeName(uuid))
if err != nil {
return VMMetadata{}, err
}
if seed == nil {
return VMMetadata{}, NewError(ErrConflict, "seed_not_found", "managed NoCloud seed volume was not found", nil)
}
defer seed.Free()
originalSeed, err := l.downloadSmallVolume(ctx, seed, maxSeedImageBytes)
if err != nil {
return VMMetadata{}, err
}
rollbackSeed := func() error {
rollbackCtx, cancelRollback := storageRollbackContext(ctx)
defer cancelRollback()
return l.uploadVolume(rollbackCtx, seed, originalSeed)
}
mac, publicCIDR, gateway, dns, err := seedNetwork(originalSeed)
if err != nil {
return VMMetadata{}, NewError(ErrConflict, "invalid_seed", "existing NoCloud network cannot be preserved", err)
}
privateNetwork, err := preservedVPCCloudInitNetwork(originalSeed, metadata.VPCPort)
if err != nil {
return VMMetadata{}, err
}
replacementSeed, err := cloudinit.BuildISO(cloudinit.Request{
UUID: uuid, Name: metadataName(originalXML), Hostname: metadata.Hostname,
GuestSSHPublicKey: request.SSHPublicKey, RootPassword: request.RootPassword,
Debian12NetworkFix: isDebian12Template(template),
Network: cloudinit.Network{
MAC: mac, PublicCIDR: publicCIDR, Gateway: gateway,
DNSServers: dns, InterfaceKey: "public0", Private: privateNetwork,
},
})
if err != nil {
return VMMetadata{}, NewError(ErrInvalid, "invalid_cloud_init", "reinstall credentials or network are invalid", err)
}
templatePool, templateVolume, err := l.templateVolume(request.TemplateID)
if err != nil {
return VMMetadata{}, err
}
defer templatePool.Free()
defer templateVolume.Free()
oldInfo, err := oldRoot.GetInfo()
if err != nil {
return VMMetadata{}, translateLibvirtError("read current root capacity", err)
}
templateInfo, err := templateVolume.GetInfo()
if err != nil {
return VMMetadata{}, translateLibvirtError("read reinstall template capacity", err)
}
if oldInfo.Capacity < templateInfo.Capacity {
return VMMetadata{}, NewError(ErrConflict, "disk_smaller_than_template", "existing root capacity is smaller than the selected template", nil)
}
replacementName := reinstallVolumeName(uuid, request.OperationID)
if existing, lookupErr := l.lookupOptionalManagedVolume(l.storagePoolName(StorageRoleManaged), replacementName); lookupErr != nil {
return VMMetadata{}, lookupErr
} else if existing != nil {
existing.Free()
return VMMetadata{}, operation.MarkUncertain(NewError(ErrConflict, "reinstall_volume_exists", "deterministic reinstall volume already exists", nil))
}
volumeXML, err := storageVolumeXML(replacementName, oldInfo.Capacity, "qcow2")
if err != nil {
return VMMetadata{}, err
}
mutationStarted = true
replacement, err := managedPool.StorageVolCreateXMLFrom(volumeXML, templateVolume, 0)
if err != nil {
observed, inspectErr := l.lookupOptionalManagedVolume(l.storagePoolName(StorageRoleManaged), replacementName)
artifactExists := observed != nil
if observed != nil {
observed.Free()
}
return VMMetadata{}, deterministicVolumeCreateFailure(
"clone reinstall system disk", err, inspectErr, artifactExists,
)
}
if err := establishNewRootVolumeCapacity(replacement, oldInfo.Capacity); err != nil {
return VMMetadata{}, l.rollbackUnpublishedVolume(
managedPool, replacement, replacementName, err,
"roll back incomplete reinstall volume",
)
}
defer replacement.Free()
replacementPath, err := replacement.GetPath()
if err != nil {
return VMMetadata{}, operation.MarkUncertain(translateLibvirtError("resolve reinstall volume path", err))
}
wasRunning, err := stopForDefinitionChange(ctx, domain)
if err != nil {
cleanupErr := replacement.Delete(libvirt.STORAGE_VOL_DELETE_NORMAL)
if cleanupErr != nil {
return VMMetadata{}, operation.MarkUncertain(errors.Join(err, mutationLibvirtError("roll back reinstall volume", cleanupErr)))
}
return VMMetadata{}, err
}
if err := l.uploadVolume(ctx, seed, replacementSeed); err != nil {
if operation.IsUncertain(err) {
return VMMetadata{}, err
}
seedRollback := rollbackSeed()
restartErr := restartOriginalDomain(domain, wasRunning)
cleanupErr := replacement.Delete(libvirt.STORAGE_VOL_DELETE_NORMAL)
if seedRollback != nil || restartErr != nil || cleanupErr != nil {
return VMMetadata{}, operation.MarkUncertain(errors.Join(err, seedRollback, restartErr, mutationLibvirtError("roll back reinstall volume", cleanupErr)))
}
return VMMetadata{}, err
}
updatedXML, err := mutateDomainDocument(originalXML, func(definition *libvirtxml.Domain, stored *domainMetadataXML) error {
root, rootErr := managedRootDisk(definition)
if rootErr != nil {
return rootErr
}
root.Source = &libvirtxml.DomainDiskSource{File: &libvirtxml.DomainDiskSourceFile{File: replacementPath}}
stored.PendingCleanupVolume = stored.RootVolumeName
stored.RootVolumeName = replacementName
stored.DiskTransitionKind = "reinstall"
stored.DiskTransitionID = request.OperationID
stored.DiskTransitionRunning = wasRunning
stored.TemplateID = request.TemplateID
stored.RescueTemplateID = ""
stored.RescueTemplateRevision = 0
stored.RescueTemplateGeneration = ""
stored.RescueWasRunning = false
return nil
})
if err != nil {
seedRollback := rollbackSeed()
restartErr := restartOriginalDomain(domain, wasRunning)
cleanupErr := replacement.Delete(libvirt.STORAGE_VOL_DELETE_NORMAL)
if seedRollback != nil || restartErr != nil || cleanupErr != nil {
return VMMetadata{}, operation.MarkUncertain(errors.Join(
err, seedRollback, restartErr,
optionalMutationLibvirtError("roll back reinstall volume", cleanupErr),
))
}
return VMMetadata{}, err
}
defined, err := l.conn.DomainDefineXML(updatedXML)
if err != nil {
_, observed, inspectErr := requireManagedDomain(domain)
if inspectErr != nil || observed.RootVolumeName != metadata.RootVolumeName {
return VMMetadata{}, operation.MarkUncertain(errors.Join(mutationLibvirtError("switch reinstalled system disk", err), inspectErr))
}
seedRollback := rollbackSeed()
restartErr := restartOriginalDomain(domain, wasRunning)
cleanupErr := replacement.Delete(libvirt.STORAGE_VOL_DELETE_NORMAL)
if seedRollback != nil || restartErr != nil || cleanupErr != nil {
return VMMetadata{}, operation.MarkUncertain(errors.Join(
mutationLibvirtError("switch reinstalled system disk", err), seedRollback, restartErr,
optionalMutationLibvirtError("roll back reinstall volume", cleanupErr),
))
}
return VMMetadata{}, mutationLibvirtError("switch reinstalled system disk", err)
}
defer defined.Free()
if err := defined.SetAutostart(autostart); err != nil {
return VMMetadata{}, operation.MarkUncertain(mutationLibvirtError("preserve reinstalled domain autostart", err))
}
if wasRunning {
if err := defined.Create(); err != nil {
return VMMetadata{}, operation.MarkUncertain(mutationLibvirtError("start reinstalled domain", err))
}
}
observedXML, observed, err := requireManagedDomain(defined)
if err != nil || observed.RootVolumeName != replacementName || observed.DiskTransitionKind != "reinstall" || observed.DiskTransitionID != request.OperationID {
return VMMetadata{}, operation.MarkUncertain(errors.Join(NewError(ErrConflict, "reinstall_observation_failed", "reinstalled domain did not expose the committed disk transition", nil), err))
}
oldPath, err := oldRoot.GetPath()
if err != nil {
return VMMetadata{}, operation.MarkUncertain(translateLibvirtError("resolve previous reinstall root", err))
}
used, err := l.anyDomainUsesVolume(
oldPath, l.storagePoolName(StorageRoleManaged), metadata.RootVolumeName,
)
if err != nil || used {
return VMMetadata{}, operation.MarkUncertain(errors.Join(err, NewError(ErrConflict, "reinstall_cleanup_pending", "reinstall committed but the old root is still referenced", nil)))
}
if err := oldRoot.Delete(libvirt.STORAGE_VOL_DELETE_NORMAL); err != nil {
return VMMetadata{}, operation.MarkUncertain(NewError(ErrUnavailable, "reinstall_cleanup_pending", "reinstall committed but old root cleanup failed", err))
}
cleanXML, err := mutateManagedMetadata(observedXML, func(stored *domainMetadataXML) error {
stored.PendingCleanupVolume = ""
// Retain the completed transition marker. If the Agent exits after
// redefining the domain but before persisting journal success, the
// same operation can still prove that this root switch committed.
return nil
})
if err != nil {
return VMMetadata{}, operation.MarkUncertain(err)
}
cleaned, err := l.conn.DomainDefineXML(cleanXML)
if err != nil {
return VMMetadata{}, operation.MarkUncertain(mutationLibvirtError("clear reinstall transition metadata", err))
}
cleaned.Free()
result, err := metadataFromXML(cleanXML)
if err != nil {
return VMMetadata{}, operation.MarkUncertain(err)
}
result.Status = VMStateShutoff
if wasRunning {
result.Status = VMStateRunning
}
result.ReinstallCommitted = true
return result, nil
}
func metadataName(document string) string {
if metadata, err := metadataFromXML(document); err == nil && metadata.Name != "" {
return metadata.Name
}
return "vm"
}