package api
import (
"bytes"
"context"
"crypto/sha256"
"encoding/json"
"errors"
"fmt"
"io"
"log"
"net/http"
"net/url"
"regexp"
"strconv"
"strings"
"sync"
"time"
"unicode/utf8"
"git.luoli.live/root/virt/agent/auth"
consolebroker "git.luoli.live/root/virt/agent/console"
"git.luoli.live/root/virt/agent/hypervisor"
"git.luoli.live/root/virt/agent/operation"
"git.luoli.live/root/virt/agent/vpcrouting"
"git.luoli.live/root/virt/agent/worker"
)
const Version = "0.1.0"
var (
apiUUIDPattern = regexp . MustCompile ( ` ^[0-9a-fA-F] { 8}-[0-9a-fA-F] { 4}-[1-5][0-9a-fA-F] { 3}-[89abAB][0-9a-fA-F] { 3}-[0-9a-fA-F] { 12}$ ` )
resourceIDPattern = regexp . MustCompile ( ` ^[A-Za-z0-9_.-] { 1,128}$ ` )
operationIDPattern = regexp . MustCompile ( ` ^[A-Za-z0-9_-] { 8,80}$ ` )
stableCodePattern = regexp . MustCompile ( ` ^[a-z][a-z0-9]*(_[a-z0-9]+)*$ ` )
sha256Pattern = regexp . MustCompile ( ` ^[0-9a-f] { 64}$ ` )
catalogIDPattern = regexp . MustCompile ( ` ^[0-9a-f] { 32}$ ` )
sourceTokenPattern = regexp . MustCompile ( ` ^[A-Za-z0-9._~-] { 32,512}$ ` )
)
const (
catalogImportTimeout = 2 * time . Hour
backupOperationTimeout = 24 * time . Hour
storageConfigurationWorkerTimeout = 120 * time . Second
defaultAdmissionTimeout = 2 * time . Second
maxAsyncImports = 2
)
type Server struct {
Authenticator * auth . Authenticator
Journal * operation . Journal
Hypervisor hypervisor . Hypervisor
Console * consolebroker . Broker
MaxBodyBytes int64
OperationTimeout time . Duration
AdmissionTimeout time . Duration
MutationExecutor worker . Executor
asyncMu sync . Mutex
asyncRunning map [ string ] struct { }
asyncSlots chan struct { }
asyncWG sync . WaitGroup
lifecycleMu sync . Mutex
closing bool
requestWG sync . WaitGroup
backgroundCtx context . Context
backgroundCancel context . CancelFunc
networkObserverMu sync . RWMutex
networkObserver NetworkObservationReader
}
type trustedWorkerRequestKey struct { }
type trustedSupersededOperationKey struct { }
// SetMutationExecutor enables the clean-break process-isolated mutation path.
// It must be called during startup before request admission begins.
func ( s * Server ) SetMutationExecutor ( executor worker . Executor ) error {
if executor == nil {
return errors . New ( "mutation executor is required" )
}
s . MutationExecutor = executor
return nil
}
// ServeTrustedWorkerRequest dispatches a request whose bounded envelope was
// already authenticated and validated by the parent Agent. Hidden workers do
// not have access to the panel secret through HTTP headers and cannot recurse
// because they never install a MutationExecutor.
func ( s * Server ) ServeTrustedWorkerRequest ( w http . ResponseWriter , r * http . Request , body [ ] byte , actor auth . Actor ,
superseded ... operation . Record ) {
setResponseHeaders ( w )
if ! s . beginRequest ( ) {
writePreDispatchError ( w , http . StatusServiceUnavailable , "agent_shutting_down" , "Agent is shutting down" )
return
}
defer s . requestWG . Done ( )
if int64 ( len ( body ) ) > s . MaxBodyBytes || actor . OperationID == "" {
writePreDispatchError ( w , http . StatusBadRequest , "invalid_worker_request" , "worker request is invalid" )
return
}
trusted := context . WithValue ( r . Context ( ) , trustedWorkerRequestKey { } , struct { } { } )
if len ( superseded ) > 0 {
prior := superseded [ 0 ]
trusted = context . WithValue ( trusted , trustedSupersededOperationKey { } , prior )
}
s . route ( w , r . WithContext ( trusted ) , body , actor )
}
func trustedSupersededOperation ( ctx context . Context , operationID string ) ( operation . Record , bool ) {
prior , ok := ctx . Value ( trustedSupersededOperationKey { } ) . ( operation . Record )
return prior , ok && prior . OperationID == operationID
}
func isVPCRepairTarget ( target string ) bool {
return strings . HasSuffix ( target , "/repair" ) &&
( strings . Contains ( target , "/v1/vpc/networks/" ) ||
strings . Contains ( target , "/vpc-port/" ) )
}
type errorBody struct {
Code string ` json:"code" `
Message string ` json:"message" `
Retryable bool ` json:"retryable" `
DispatchState string ` json:"dispatch_state,omitempty" `
Operation * operation . Record ` json:"operation,omitempty" `
}
type errorEnvelope struct {
Error errorBody ` json:"error" `
}
type operationResponse struct {
OperationID string ` json:"operation_id" `
Action string ` json:"action" `
ResourceID string ` json:"resource_id" `
RequestFingerprint string ` json:"request_fingerprint" `
Status operation . State ` json:"status" `
Replayed bool ` json:"replayed" `
Result json . RawMessage ` json:"result,omitempty" `
Error * operation . PublicError ` json:"error,omitempty" `
CauseCode string ` json:"cause_code,omitempty" `
EffectMayHaveApplied bool ` json:"effect_may_have_applied" `
SafeToResubmit bool ` json:"safe_to_resubmit" `
Retryable bool ` json:"retryable" `
}
type fenceResolutionResponse struct {
operation . FenceResolution
Replayed bool ` json:"replayed" `
}
func NewServer (
authenticator * auth . Authenticator ,
journal * operation . Journal ,
hv hypervisor . Hypervisor ,
console * consolebroker . Broker ,
maxBodyBytes int64 ,
operationTimeout time . Duration ,
) ( * Server , error ) {
if authenticator == nil || journal == nil || hv == nil || console == nil {
return nil , errors . New ( "authenticator, journal, hypervisor, and console broker are required" )
}
if maxBodyBytes < 1024 {
return nil , errors . New ( "max body size must be at least 1024 bytes" )
}
if operationTimeout <= 0 {
return nil , errors . New ( "operation timeout must be positive" )
}
backgroundCtx , backgroundCancel := context . WithCancel ( context . Background ( ) )
return & Server {
Authenticator : authenticator , Journal : journal , Hypervisor : hv , Console : console ,
MaxBodyBytes : maxBodyBytes , OperationTimeout : operationTimeout ,
AdmissionTimeout : min ( operationTimeout , defaultAdmissionTimeout ) ,
asyncRunning : make ( map [ string ] struct { } ) , asyncSlots : make ( chan struct { } , maxAsyncImports ) ,
backgroundCtx : backgroundCtx , backgroundCancel : backgroundCancel ,
} , nil
}
func ( s * Server ) ServeHTTP ( w http . ResponseWriter , r * http . Request ) {
setResponseHeaders ( w )
if ! s . beginRequest ( ) {
writePreDispatchError ( w , http . StatusServiceUnavailable , "agent_shutting_down" , "Agent is shutting down" )
return
}
defer s . requestWG . Done ( )
if ! strings . HasPrefix ( r . URL . Path , "/v1/" ) || strings . HasSuffix ( r . URL . Path , "/" ) {
writePreDispatchError ( w , http . StatusNotFound , "not_found" , "endpoint not found" )
return
}
body , err := readBoundedBody ( w , r , s . MaxBodyBytes )
if err != nil {
var tooLarge * http . MaxBytesError
if errors . As ( err , & tooLarge ) {
writePreDispatchError ( w , http . StatusRequestEntityTooLarge , "body_too_large" , "request body exceeds configured limit" )
} else {
writePreDispatchError ( w , http . StatusBadRequest , "invalid_body" , "could not read request body" )
}
return
}
actor , err := s . Authenticator . Authenticate ( r , body )
if err != nil {
if errors . Is ( err , auth . ErrNonceStore ) {
writePreDispatchError ( w , http . StatusServiceUnavailable , "nonce_store_unavailable" , "authentication state is unavailable" )
return
}
if errors . Is ( err , auth . ErrReplay ) {
// The matching nonce may belong to an earlier request that already
// crossed the durable operation boundary. A replay rejection is not
// proof that the operation was never dispatched.
writeError ( w , http . StatusUnauthorized , "replayed_request" , "request nonce has already been used" , nil )
return
}
writePreDispatchError ( w , http . StatusUnauthorized , "unauthorized" , "request authentication failed" )
return
}
s . route ( w , r , body , actor )
}
func ( s * Server ) route ( w http . ResponseWriter , r * http . Request , body [ ] byte , actor auth . Actor ) {
segments := strings . Split ( strings . TrimPrefix ( r . URL . Path , "/" ) , "/" )
if len ( segments ) < 2 || segments [ 0 ] != "v1" {
writePreDispatchError ( w , http . StatusNotFound , "not_found" , "endpoint not found" )
return
}
if actor . SystemRecovery && ! systemRecoveryRequestAllowed ( actor , r . Method , segments ) {
writePreDispatchError ( w , http . StatusForbidden , "forbidden" , "system recovery actor is not authorized for this endpoint" )
return
}
switch segments [ 1 ] {
case "health" :
if len ( segments ) != 2 || r . Method != http . MethodGet {
methodOrNotFound ( w , r , len ( segments ) == 2 , http . MethodGet )
return
}
if err := s . Hypervisor . Ping ( r . Context ( ) ) ; err != nil {
writeMappedError ( w , err , nil )
return
}
writeJSON ( w , http . StatusOK , map [ string ] any { "status" : "ok" , "service" : "virt-host-agent" , "version" : Version } )
case "capabilities" :
if len ( segments ) != 2 || r . Method != http . MethodGet {
methodOrNotFound ( w , r , len ( segments ) == 2 , http . MethodGet )
return
}
result , err := s . Hypervisor . Capabilities ( r . Context ( ) )
if err != nil {
writeMappedError ( w , err , nil )
return
}
writeJSON ( w , http . StatusOK , s . capabilitiesResponse ( result ) )
case "network-observations" :
s . routeNetworkObservations ( w , r , segments , actor )
case "storage" :
s . routeStorage ( w , r , segments , body , actor )
case "vpc" :
s . routeVPC ( w , r , segments , body , actor )
case "vms" :
s . routeVMs ( w , r , segments , body , actor )
case "templates" :
s . routeTemplates ( w , r , segments , body , actor )
case "rescue-templates" :
s . routeRescueTemplates ( w , r , segments , body , actor )
case "user-isos" :
s . routeUserISOs ( w , r , segments , body , actor )
case "operations" :
s . routeOperations ( w , r , segments , body , actor )
default :
writePreDispatchError ( w , http . StatusNotFound , "not_found" , "endpoint not found" )
}
}
func ( s * Server ) routeVPC ( w http . ResponseWriter , r * http . Request , segments [ ] string , body [ ] byte , actor auth . Actor ) {
repairPath := len ( segments ) == 6 && segments [ 5 ] == "repair"
if ( len ( segments ) != 5 && ! repairPath ) || segments [ 2 ] != "networks" || ! catalogIDPattern . MatchString ( segments [ 3 ] ) ||
( segments [ 4 ] != "apply" && segments [ 4 ] != "remove" ) {
writePreDispatchError ( w , http . StatusNotFound , "not_found" , "endpoint not found" )
return
}
if ! requireSuperAdminManage ( w , actor ) {
return
}
if r . Method != http . MethodPost {
methodOrNotFound ( w , r , true , http . MethodPost )
return
}
vpcKey := segments [ 3 ]
switch segments [ 4 ] {
case "apply" :
request , ok := decodeRequired [ vpcrouting . ApplyRequest ] ( w , body )
if ! ok {
return
}
if request . Network . VPCKey != vpcKey {
writePreDispatchError ( w , http . StatusUnprocessableEntity , "vpc_key_mismatch" , "body vpc_key must match the request target" )
return
}
operationID , err := resolveOperationID ( actor . OperationID , request . OperationID )
if err != nil {
writePreDispatchError ( w , http . StatusUnprocessableEntity , "invalid_operation_id" , err . Error ( ) )
return
}
request . OperationID = operationID
normalized , err := vpcrouting . ValidateApplyRequest ( request )
if err != nil {
writePreDispatchMappedError ( w , hypervisor . NewError ( hypervisor . ErrInvalid , publicErrorCode ( err , "invalid_vpc_projection" ) , publicErrorMessage ( err , "VPC projection is invalid" ) , err ) )
return
}
request = normalized
if request . VPCRepair != repairPath ||
( request . VPCRepair != ( request . SupersedesOperationID != "" ) ) {
writePreDispatchError ( w , http . StatusUnprocessableEntity , "invalid_vpc_repair" , "VPC repair requires the repair endpoint and one predecessor operation" )
return
}
action := "apply-vpc-network:" + vpcKey
expected := & operation . ExpectedState {
Kind : operation . ExpectedVPCNetwork , VPCKey : vpcKey ,
VPCRevision : request . Network . DesiredRevision ,
VPCDigest : request . DesiredDigest , VPCPresent : true ,
}
admission := hypervisor . VPCNetworkAdmission ( vpcKey )
supersedes , ok := s . authorizeVPCRepair (
w , r , operationID , request . SupersedesOperationID ,
"vpc:" + vpcKey , action , expected , admission , repairPath ,
)
if ! ok {
return
}
s . runMutationExpected ( w , r , body , actor , operationID , "vpc:" + vpcKey , action , expected , true , http . StatusOK ,
admission ,
func ( ctx context . Context ) ( any , error ) { return s . Hypervisor . ApplyVPCNetwork ( ctx , request ) } , supersedes ... )
case "remove" :
request , ok := decodeRequired [ vpcrouting . RemoveRequest ] ( w , body )
if ! ok {
return
}
if request . VPCKey != vpcKey {
writePreDispatchError ( w , http . StatusUnprocessableEntity , "vpc_key_mismatch" , "body vpc_key must match the request target" )
return
}
operationID , err := resolveOperationID ( actor . OperationID , request . OperationID )
if err != nil {
writePreDispatchError ( w , http . StatusUnprocessableEntity , "invalid_operation_id" , err . Error ( ) )
return
}
request . OperationID = operationID
if err := vpcrouting . ValidateRemoveRequest ( request ) ; err != nil {
writePreDispatchMappedError ( w , hypervisor . NewError ( hypervisor . ErrInvalid , publicErrorCode ( err , "invalid_vpc_remove" ) , publicErrorMessage ( err , "VPC removal request is invalid" ) , err ) )
return
}
if request . VPCRepair != repairPath ||
( request . VPCRepair != ( request . SupersedesOperationID != "" ) ) {
writePreDispatchError ( w , http . StatusUnprocessableEntity , "invalid_vpc_repair" , "VPC repair requires the repair endpoint and one predecessor operation" )
return
}
action := "remove-vpc-network:" + vpcKey
expected := & operation . ExpectedState {
Kind : operation . ExpectedVPCNetwork , VPCKey : vpcKey ,
VPCRevision : request . ExpectedRevision ,
VPCDigest : request . ExpectedDigest , VPCPresent : false ,
}
admission := hypervisor . VPCNetworkAdmission ( vpcKey )
supersedes , ok := s . authorizeVPCRepair (
w , r , operationID , request . SupersedesOperationID ,
"vpc:" + vpcKey , action , expected , admission , repairPath ,
)
if ! ok {
return
}
s . runMutationExpected ( w , r , body , actor , operationID , "vpc:" + vpcKey , action , expected , true , http . StatusOK ,
admission ,
func ( ctx context . Context ) ( any , error ) { return s . Hypervisor . RemoveVPCNetwork ( ctx , request ) } , supersedes ... )
}
}
func ( s * Server ) authorizeVPCRepair ( w http . ResponseWriter , r * http . Request ,
operationID , predecessorID , resourceID , action string ,
expected * operation . ExpectedState , admission hypervisor . MutationAdmission ,
repair bool ) ( [ ] string , bool ) {
if ! repair {
return nil , true
}
if ! operationIDPattern . MatchString ( predecessorID ) ||
operationID == predecessorID {
writePreDispatchError ( w , http . StatusUnprocessableEntity , "invalid_vpc_repair_predecessor" , "VPC repair requires a different valid predecessor operation" )
return nil , false
}
trustedPrior , trusted := trustedSupersededOperation (
r . Context ( ) , predecessorID ,
)
existing , existingErr := s . Journal . Get ( operationID )
if existingErr == nil {
if existing . Action != action || existing . ResourceID != resourceID ||
existing . SupersedesOperationID != predecessorID {
writePreDispatchError ( w , http . StatusConflict , "operation_id_conflict" , "operation_id is already bound to a different request" )
return nil , false
}
} else if ! errors . Is ( existingErr , operation . ErrNotFound ) {
writeMappedError ( w , existingErr , nil )
return nil , false
} else {
prior := trustedPrior
var err error
if ! trusted {
prior , err = s . Journal . Get ( predecessorID )
}
spec := operation . Spec {
OperationID : operationID , ResourceID : resourceID ,
Action : action , ExpectedState : expected ,
LockKeys : append ( [ ] string ( nil ) , admission . LockKeys ... ) ,
SupersedesOperationID : predecessorID ,
}
if err != nil || ! operation . ValidVPCRepairSupersession ( spec , prior ) {
writePreDispatchError ( w , http . StatusConflict , "vpc_repair_not_authorized" , "VPC repair no longer matches one exact fenced predecessor" )
return nil , false
}
if ! trusted {
_ , incident , incidentErr := s . Journal . CompletionDeliveryIncident (
predecessorID ,
)
if incidentErr != nil {
writeMappedError ( w , incidentErr , nil )
return nil , false
}
if incident {
writePreDispatchError ( w , http . StatusConflict , "vpc_repair_not_authorized" , "VPC repair is blocked by a completion-delivery incident" )
return nil , false
}
}
}
if trusted {
return nil , true
}
return [ ] string { predecessorID } , true
}
func ( s * Server ) routeStorage ( w http . ResponseWriter , r * http . Request , segments [ ] string , body [ ] byte , actor auth . Actor ) {
if len ( segments ) != 3 || segments [ 2 ] != "configuration" {
writePreDispatchError ( w , http . StatusNotFound , "not_found" , "endpoint not found" )
return
}
if ! requireSuperAdminManage ( w , actor ) {
return
}
if r . Method != http . MethodPost {
methodOrNotFound ( w , r , true , http . MethodPost )
return
}
request , ok := decodeRequired [ hypervisor . StorageConfigurationRequest ] ( w , body )
if ! ok {
return
}
if err := hypervisor . ValidateStorageConfigurationRequest ( request ) ; err != nil {
writePreDispatchMappedError ( w , err )
return
}
digests := hypervisor . StorageConfigurationPathDigests ( request )
expected := & operation . ExpectedState {
Kind : operation . ExpectedStorageConfiguration ,
ManagedPathDigest : digests [ hypervisor . StorageRoleManaged ] ,
TemplatePathDigest : digests [ hypervisor . StorageRoleTemplate ] ,
SeedPathDigest : digests [ hypervisor . StorageRoleSeed ] ,
UserISOPathDigest : digests [ hypervisor . StorageRoleUserISO ] ,
RescuePathDigest : digests [ hypervisor . StorageRoleRescue ] ,
}
s . runMutationExpected ( w , r , body , actor , request . OperationID , "host-storage" , "configure-storage" , expected , true , http . StatusOK ,
hypervisor . StorageConfigurationAdmission ( ) ,
func ( ctx context . Context ) ( any , error ) { return s . Hypervisor . ConfigureStorage ( ctx , request ) } )
}
func ( s * Server ) routeVMs ( w http . ResponseWriter , r * http . Request , segments [ ] string , body [ ] byte , actor auth . Actor ) {
if len ( segments ) == 2 {
switch r . Method {
case http . MethodGet :
result , err := s . Hypervisor . ListVMs ( r . Context ( ) )
writeNamedList ( w , "vms" , result , err )
case http . MethodPost :
if ! requireManage ( w , actor ) {
return
}
request , ok := decodeRequired [ hypervisor . CreateVMRequest ] ( w , body )
if ! ok {
return
}
normalized , err := hypervisor . NormalizeStartupScript ( request . StartupScript )
if err != nil {
writePreDispatchMappedError ( w , err )
return
}
request . StartupScript = normalized
resolvedOperationID , err := resolveOperationID ( actor . OperationID , request . OperationID )
if err != nil {
writePreDispatchError ( w , http . StatusUnprocessableEntity , "invalid_operation_id" , err . Error ( ) )
return
}
request . OperationID = resolvedOperationID
if request . UUID == "" {
request . UUID = operationUUID ( resolvedOperationID )
}
request . DomainName , err = hypervisor . LibvirtDomainName ( actor . UserID , request . UUID )
if err != nil {
writePreDispatchMappedError ( w , err )
return
}
if err := hypervisor . ValidateCreateVM ( request ) ; err != nil {
writePreDispatchMappedError ( w , err )
return
}
s . runMutation ( w , r , body , actor , request . OperationID , request . UUID , "create-vm" , true , http . StatusCreated ,
hypervisor . CreateVMAdmission ( request ) ,
func ( ctx context . Context ) ( any , error ) { return s . Hypervisor . CreateVM ( ctx , request ) } )
default :
methodOrNotFound ( w , r , true , http . MethodGet , http . MethodPost )
}
return
}
if len ( segments ) < 3 || ! apiUUIDPattern . MatchString ( segments [ 2 ] ) {
writePreDispatchError ( w , http . StatusUnprocessableEntity , "invalid_uuid" , "VM UUID must be canonical" )
return
}
uuid := strings . ToLower ( segments [ 2 ] )
if len ( segments ) == 3 {
switch r . Method {
case http . MethodGet :
result , err := s . Hypervisor . GetVMMetadata ( r . Context ( ) , uuid )
writeResult ( w , result , err )
case http . MethodDelete :
if ! requireManage ( w , actor ) {
return
}
request , ok := decodeOptional [ hypervisor . DeleteVMRequest ] ( w , body )
if ! ok {
return
}
if ! request . RemoveManagedDisks {
writePreDispatchError ( w , http . StatusUnprocessableEntity , "managed_disk_cleanup_required" , "remove_managed_disks must be true" )
return
}
s . runMutation ( w , r , body , actor , request . OperationID , uuid , "delete-vm" , true , http . StatusOK ,
hypervisor . DeleteVMAdmission ( uuid ) ,
func ( ctx context . Context ) ( any , error ) {
return s . Hypervisor . DeleteVM ( ctx , uuid , request )
} )
default :
methodOrNotFound ( w , r , true , http . MethodGet , http . MethodDelete )
}
return
}
switch segments [ 3 ] {
case "backup-readiness" :
if len ( segments ) == 4 && r . Method == http . MethodGet {
if ! requireManage ( w , actor ) {
return
}
result , err := s . Hypervisor . GetBackupReadiness ( r . Context ( ) , uuid )
writeResult ( w , result , err )
return
}
case "backups" :
if len ( segments ) == 4 && r . Method == http . MethodPost {
if ! requireManage ( w , actor ) {
return
}
request , ok := decodeRequired [ hypervisor . BackupRequest ] ( w , body )
if ! ok {
return
}
resolvedOperationID , err := resolveOperationID ( actor . OperationID , request . OperationID )
if err != nil {
writePreDispatchError ( w , http . StatusUnprocessableEntity , "invalid_operation_id" , err . Error ( ) )
return
}
request . OperationID = resolvedOperationID
if err := hypervisor . ValidateBackupRequest ( request ) ; err != nil {
writePreDispatchMappedError ( w , err )
return
}
action := "create-backup:" + request . BackupKey + ":" + request . ConsistencyMode
s . runAsyncMutationWithDeadline (
w , r , body , actor , request . OperationID , uuid , action , false ,
hypervisor . CreateBackupAdmission ( uuid , request ) , backupOperationTimeout ,
func ( ctx context . Context ) ( any , error ) { return s . Hypervisor . CreateBackup ( ctx , uuid , request ) } ,
)
return
}
case "restore-backup" :
if len ( segments ) == 4 && r . Method == http . MethodPost {
if ! requireManage ( w , actor ) {
return
}
request , ok := decodeRequired [ hypervisor . RestoreBackupRequest ] ( w , body )
if ! ok {
return
}
if request . UUID != "" && ! strings . EqualFold ( request . UUID , uuid ) {
writePreDispatchError ( w , http . StatusUnprocessableEntity , "uuid_mismatch" , "body UUID must match the restore target" )
return
}
request . UUID = uuid
resolvedOperationID , err := resolveOperationID ( actor . OperationID , request . OperationID )
if err != nil {
writePreDispatchError ( w , http . StatusUnprocessableEntity , "invalid_operation_id" , err . Error ( ) )
return
}
request . OperationID = resolvedOperationID
derivedDomainName , err := hypervisor . LibvirtDomainName ( actor . UserID , uuid )
if err != nil {
writePreDispatchMappedError ( w , err )
return
}
if request . RequestedDomainName != "" && request . RequestedDomainName != derivedDomainName {
writePreDispatchError ( w , http . StatusUnprocessableEntity , "domain_name_mismatch" , "domain_name must match the Agent-derived ownership name" )
return
}
request . DomainName = derivedDomainName
if err := hypervisor . ValidateRestoreBackupRequest ( request ) ; err != nil {
writePreDispatchMappedError ( w , err )
return
}
expected , err := s . createLikeNetworkExpectedState ( r . Context ( ) , request . CreateVMRequest , nil )
if err != nil {
writePreDispatchMappedError ( w , err )
return
}
action := "restore-backup:" + request . BackupKey
s . runAsyncMutationWithDeadlineExpected (
w , r , body , actor , request . OperationID , uuid , action , expected , true ,
hypervisor . RestoreBackupAdmission ( request ) , backupOperationTimeout ,
func ( ctx context . Context ) ( any , error ) { return s . Hypervisor . RestoreBackup ( ctx , request ) } ,
)
return
}
case "recover-backup" :
if len ( segments ) == 4 && r . Method == http . MethodPost {
if ! requireManage ( w , actor ) {
return
}
request , ok := decodeRequired [ hypervisor . RecoverBackupRequest ] ( w , body )
if ! ok {
return
}
resolvedOperationID , err := resolveOperationID ( actor . OperationID , request . OperationID )
if err != nil {
writePreDispatchError ( w , http . StatusUnprocessableEntity , "invalid_operation_id" , err . Error ( ) )
return
}
request . OperationID = resolvedOperationID
if err := hypervisor . ValidateRecoverBackupRequest ( request ) ; err != nil {
writePreDispatchMappedError ( w , err )
return
}
action := "recover-backup:" + request . BackupKey + ":" + request . SourceSHA256
s . runAsyncMutationWithDeadline (
w , r , body , actor , request . OperationID , uuid , action , true ,
hypervisor . RecoverBackupAdmission ( uuid , request ) , backupOperationTimeout ,
func ( ctx context . Context ) ( any , error ) {
return s . Hypervisor . RecoverBackup ( ctx , uuid , request )
} ,
)
return
}
case "peer-migration" :
if len ( segments ) != 5 || ! requireManage ( w , actor ) {
if len ( segments ) != 5 {
writePreDispatchError ( w , http . StatusNotFound , "not_found" , "endpoint not found" )
}
return
}
switch segments [ 4 ] {
case "prepare" :
if r . Method != http . MethodPost {
methodOrNotFound ( w , r , true , http . MethodPost )
return
}
request , ok := decodeRequired [ hypervisor . PreparePeerMigrationRequest ] ( w , body )
if ! ok {
return
}
operationID , err := resolveOperationID ( actor . OperationID , request . OperationID )
if err != nil {
writePreDispatchError ( w , http . StatusUnprocessableEntity , "invalid_operation_id" , err . Error ( ) )
return
}
request . OperationID = operationID
if err := hypervisor . ValidatePreparePeerMigrationRequest ( request ) ; err != nil {
writePreDispatchMappedError ( w , err )
return
}
s . runAsyncMutationWithDeadline ( w , r , body , actor , operationID , uuid ,
"prepare-peer-migration:" + request . MigrationID , true ,
hypervisor . PreparePeerMigrationAdmission ( uuid , request ) , backupOperationTimeout ,
func ( ctx context . Context ) ( any , error ) {
return s . Hypervisor . PreparePeerMigration ( ctx , uuid , request )
} )
case "receive" :
if r . Method != http . MethodPost {
methodOrNotFound ( w , r , true , http . MethodPost )
return
}
request , ok := decodeRequired [ hypervisor . ReceivePeerMigrationRequest ] ( w , body )
if ! ok {
return
}
if request . UUID != "" && ! strings . EqualFold ( request . UUID , uuid ) {
writePreDispatchError ( w , http . StatusUnprocessableEntity , "uuid_mismatch" , "body UUID must match the migration target" )
return
}
request . UUID = uuid
operationID , err := resolveOperationID ( actor . OperationID , request . OperationID )
if err != nil {
writePreDispatchError ( w , http . StatusUnprocessableEntity , "invalid_operation_id" , err . Error ( ) )
return
}
request . OperationID = operationID
request . DomainName , err = hypervisor . LibvirtDomainName ( actor . UserID , uuid )
if err != nil {
writePreDispatchMappedError ( w , err )
return
}
if err := hypervisor . ValidateReceivePeerMigrationRequest ( request ) ; err != nil {
writePreDispatchMappedError ( w , err )
return
}
expected , err := s . createLikeNetworkExpectedState ( r . Context ( ) , request . CreateVMRequest , request . VPCPort )
if err != nil {
writePreDispatchMappedError ( w , err )
return
}
s . runAsyncMutationWithDeadlineExpected ( w , r , body , actor , operationID , uuid ,
"receive-peer-migration:" + request . MigrationID , expected , true ,
hypervisor . ReceivePeerMigrationAdmission ( request ) , backupOperationTimeout ,
func ( ctx context . Context ) ( any , error ) {
return s . Hypervisor . ReceivePeerMigration ( ctx , request )
} )
case "rollback" :
if r . Method != http . MethodPost {
methodOrNotFound ( w , r , true , http . MethodPost )
return
}
request , ok := decodeRequired [ hypervisor . RollbackPeerMigrationRequest ] ( w , body )
if ! ok {
return
}
operationID , err := resolveOperationID ( actor . OperationID , request . OperationID )
if err != nil {
writePreDispatchError ( w , http . StatusUnprocessableEntity , "invalid_operation_id" , err . Error ( ) )
return
}
request . OperationID = operationID
if err := hypervisor . ValidateRollbackPeerMigrationRequest ( request ) ; err != nil {
writePreDispatchMappedError ( w , err )
return
}
expected := & operation . ExpectedState {
Kind : operation . ExpectedPeerMigrationState , Enabled : request . SourceAutostartEnabled ,
StartIfSuspended : request . SourceWasRunning ,
}
s . runMutationExpected ( w , r , body , actor , operationID , uuid ,
"rollback-peer-migration:" + request . MigrationID ,
expected , true , http . StatusOK ,
hypervisor . DomainMutationAdmission ( uuid ) ,
func ( ctx context . Context ) ( any , error ) {
return s . Hypervisor . RollbackPeerMigration ( ctx , uuid , request )
} )
case "activate" :
if r . Method != http . MethodPost {
methodOrNotFound ( w , r , true , http . MethodPost )
return
}
request , ok := decodeRequired [ hypervisor . ActivatePeerMigrationRequest ] ( w , body )
if ! ok {
return
}
operationID , err := resolveOperationID ( actor . OperationID , request . OperationID )
if err != nil {
writePreDispatchError ( w , http . StatusUnprocessableEntity , "invalid_operation_id" , err . Error ( ) )
return
}
request . OperationID = operationID
if err := hypervisor . ValidateActivatePeerMigrationRequest ( request ) ; err != nil {
writePreDispatchMappedError ( w , err )
return
}
expected := & operation . ExpectedState {
Kind : operation . ExpectedPeerMigrationState ,
Enabled : request . TargetAutostartEnabled ,
StartIfSuspended : request . TargetWasRunning ,
}
s . runMutationExpected ( w , r , body , actor , operationID , uuid ,
"activate-peer-migration:" + request . MigrationID ,
expected , true , http . StatusOK ,
hypervisor . DomainMutationAdmission ( uuid ) ,
func ( ctx context . Context ) ( any , error ) {
return s . Hypervisor . ActivatePeerMigration ( ctx , uuid , request )
} )
case "cleanup" :
if r . Method != http . MethodPost {
methodOrNotFound ( w , r , true , http . MethodPost )
return
}
request , ok := decodeRequired [ hypervisor . CleanupPeerMigrationRequest ] ( w , body )
if ! ok {
return
}
operationID , err := resolveOperationID ( actor . OperationID , request . OperationID )
if err != nil {
writePreDispatchError ( w , http . StatusUnprocessableEntity , "invalid_operation_id" , err . Error ( ) )
return
}
request . OperationID = operationID
if err := hypervisor . ValidateCleanupPeerMigrationRequest ( request ) ; err != nil {
writePreDispatchMappedError ( w , err )
return
}
s . runMutation ( w , r , body , actor , operationID , uuid ,
"delete-vm" , true , http . StatusOK ,
hypervisor . DeleteVMAdmission ( uuid ) ,
func ( ctx context . Context ) ( any , error ) {
return s . Hypervisor . CleanupPeerMigration ( ctx , uuid , request )
} )
case "cleanup-target" :
if r . Method != http . MethodPost {
methodOrNotFound ( w , r , true , http . MethodPost )
return
}
request , ok := decodeRequired [ hypervisor . CleanupPeerMigrationTargetRequest ] ( w , body )
if ! ok {
return
}
operationID , err := resolveOperationID ( actor . OperationID , request . OperationID )
if err != nil {
writePreDispatchError ( w , http . StatusUnprocessableEntity , "invalid_operation_id" , err . Error ( ) )
return
}
request . OperationID = operationID
if err := hypervisor . ValidateCleanupPeerMigrationTargetRequest ( request ) ; err != nil {
writePreDispatchMappedError ( w , err )
return
}
action := "cleanup-peer-migration-target:" + request . MigrationID
existingReplay := false
trustedPrior , trustedPriorFound := trustedSupersededOperation (
r . Context ( ) , request . PriorOperationID ,
)
existing , existingErr := s . Journal . Get ( operationID )
if existingErr == nil {
existingReplay = existing . Action == action && existing . ResourceID == uuid &&
existing . SupersedesOperationID == request . PriorOperationID
} else if ! errors . Is ( existingErr , operation . ErrNotFound ) {
writeMappedError ( w , existingErr , nil )
return
}
if ! existingReplay {
prior := trustedPrior
var priorErr error
if ! trustedPriorFound {
prior , priorErr = s . Journal . Get ( request . PriorOperationID )
}
if priorErr != nil || prior . ResourceID != uuid ||
prior . Action != "receive-peer-migration:" + request . MigrationID ||
prior . State != operation . StateFailed || ! prior . EffectMayHaveApplied ||
prior . SafeToResubmit || prior . ExecutionFenced {
writePreDispatchError ( w , http . StatusConflict , "peer_migration_target_cleanup_not_authorized" , "target cleanup requires the exact safely reaped failed receive operation" )
return
}
if ! trustedPriorFound {
_ , incidentFound , incidentErr := s . Journal . CompletionDeliveryIncident ( prior . OperationID )
if incidentErr != nil {
writeMappedError ( w , incidentErr , nil )
return
}
if incidentFound {
writePreDispatchError ( w , http . StatusConflict , "peer_migration_target_cleanup_not_authorized" , "target cleanup is blocked by a completion-delivery incident" )
return
}
}
ctx , cancel := context . WithTimeout ( r . Context ( ) , s . OperationTimeout )
inspection , inspectErr := s . Hypervisor . InspectOperation ( ctx , prior )
cancel ( )
if inspectErr != nil {
writeMappedError ( w , inspectErr , nil )
return
}
if ! peerMigrationTargetCleanupInspectionMatches ( prior , request . CleanupPlanID , inspection ) {
writePreDispatchError ( w , http . StatusConflict , "peer_migration_cleanup_plan_changed" , "target cleanup plan is stale or no longer safe" )
return
}
request . PriorOperation = prior
}
supersedes := [ ] string { request . PriorOperationID }
if trustedPriorFound {
// The isolated worker has its own Journal. The parent Journal
// already admitted this exact supersession and retains the audit
// link; the child receives the immutable predecessor separately.
supersedes = nil
}
s . runMutationExpected ( w , r , body , actor , operationID , uuid , action , nil ,
true , http . StatusOK ,
hypervisor . CleanupPeerMigrationTargetAdmission ( uuid , request . MigrationID ) ,
func ( ctx context . Context ) ( any , error ) {
return s . Hypervisor . CleanupPeerMigrationTarget ( ctx , uuid , request )
} , supersedes ... )
default :
writePreDispatchError ( w , http . StatusNotFound , "not_found" , "endpoint not found" )
}
return
case "force-delete" :
if len ( segments ) == 4 && r . Method == http . MethodPost {
if ! requireSuperAdminPrivilegedOperation ( w , actor ) {
return
}
request , ok := decodeRequired [ hypervisor . ForceDeleteVMRequest ] ( w , body )
if ! ok {
return
}
if ! request . RemoveManagedDisks {
writePreDispatchError ( w , http . StatusUnprocessableEntity , "managed_disk_cleanup_required" , "remove_managed_disks must be true" )
return
}
if ! request . AckDataLoss {
writePreDispatchError ( w , http . StatusUnprocessableEntity , "data_loss_ack_required" , "ack_data_loss must be true" )
return
}
operationID , err := resolveOperationID ( actor . OperationID , request . OperationID )
if err != nil {
writePreDispatchError ( w , http . StatusUnprocessableEntity , "invalid_operation_id" , err . Error ( ) )
return
}
if ! operationIDPattern . MatchString ( request . PriorOperationID ) {
writePreDispatchError ( w , http . StatusUnprocessableEntity , "invalid_prior_operation_id" , "prior_operation_id must contain 8-80 URL-safe characters" )
return
}
if operationID == request . PriorOperationID {
writePreDispatchError ( w , http . StatusUnprocessableEntity , "force_operation_id_reused" , "force delete requires a new operation_id" )
return
}
existingReplay := false
trustedPrior , trustedPriorFound := trustedSupersededOperation (
r . Context ( ) , request . PriorOperationID ,
)
existing , existingErr := s . Journal . Get ( operationID )
if existingErr == nil {
existingReplay = existing . Action == "force-delete-vm" && existing . ResourceID == uuid &&
existing . SupersedesOperationID == request . PriorOperationID
} else if ! errors . Is ( existingErr , operation . ErrNotFound ) {
writeMappedError ( w , existingErr , nil )
return
}
if ! existingReplay {
prior := trustedPrior
var priorErr error
if ! trustedPriorFound {
prior , priorErr = s . Journal . Get ( request . PriorOperationID )
}
priorActionAllowed := priorErr == nil && ( prior . Action == "delete-vm" || prior . Action == "service-suspend" )
if ! priorActionAllowed || prior . ResourceID != uuid ||
prior . State != operation . StateFailed || prior . Error == nil || prior . Error . Code != "force_required" {
writePreDispatchError ( w , http . StatusConflict , "force_delete_not_authorized" , "force delete requires the matching graceful stop to require force" )
return
}
}
supersedes := [ ] string { request . PriorOperationID }
if trustedPriorFound {
// The parent Journal already authorized and records the force
// successor. The isolated worker Journal receives the immutable
// predecessor separately and must not try to load it locally.
supersedes = nil
}
s . runMutationExpected ( w , r , body , actor , request . OperationID , uuid , "force-delete-vm" , nil ,
true , http . StatusOK , hypervisor . DeleteVMAdmission ( uuid ) ,
func ( ctx context . Context ) ( any , error ) {
return s . Hypervisor . ForceDeleteVM ( ctx , uuid , request )
} , supersedes ... )
return
}
case "state" :
if len ( segments ) == 4 && r . Method == http . MethodGet {
result , err := s . Hypervisor . GetVMState ( r . Context ( ) , uuid )
writeResult ( w , result , err )
return
}
case "metadata" :
if len ( segments ) == 4 && r . Method == http . MethodGet {
result , err := s . Hypervisor . GetVMMetadata ( r . Context ( ) , uuid )
writeResult ( w , result , err )
return
}
case "actions" :
if len ( segments ) == 4 && r . Method == http . MethodPost {
if ! requireManage ( w , actor ) {
return
}
request , ok := decodeRequired [ hypervisor . PowerRequest ] ( w , body )
if ! ok {
return
}
if ! validPowerAction ( request . Action ) {
writePreDispatchError ( w , http . StatusUnprocessableEntity , "invalid_power_action" , "action must be start, shutdown, reboot, reset, or destroy" )
return
}
s . runMutation ( w , r , body , actor , request . OperationID , uuid , "power-" + string ( request . Action ) , true , http . StatusOK ,
hypervisor . DomainMutationAdmission ( uuid ) ,
func ( ctx context . Context ) ( any , error ) { return s . Hypervisor . Power ( ctx , uuid , request ) } )
return
}
case "settings" :
if len ( segments ) == 4 && ( r . Method == http . MethodPatch || r . Method == http . MethodPost ) {
if ! requireManage ( w , actor ) {
return
}
request , ok := decodeRequired [ hypervisor . SettingsRequest ] ( w , body )
if ! ok {
return
}
if request . Hostname == nil && request . RootPassword == nil && request . SSHPublicKey == nil {
writePreDispatchError ( w , http . StatusUnprocessableEntity , "empty_settings" , "at least one setting is required" )
return
}
if ! request . ConfirmRestart {
writePreDispatchError ( w , http . StatusUnprocessableEntity , "restart_confirmation_required" , "confirm_restart must be true" )
return
}
expected := & operation . ExpectedState {
Kind : operation . ExpectedUpdateSettings , HostnameChanged : request . Hostname != nil ,
RootPasswordChanged : request . RootPassword != nil , SSHKeyChanged : request . SSHPublicKey != nil ,
}
if request . Hostname != nil {
expected . Hostname = * request . Hostname
}
s . runMutationExpected ( w , r , body , actor , request . OperationID , uuid , "update-settings" , expected , false , http . StatusOK ,
hypervisor . SeedMutationAdmission ( uuid ) ,
func ( ctx context . Context ) ( any , error ) { return s . Hypervisor . UpdateSettings ( ctx , uuid , request ) } )
return
}
case "resources" :
if len ( segments ) == 4 {
if r . Method != http . MethodPost {
methodOrNotFound ( w , r , true , http . MethodPost )
return
}
if ! requireManage ( w , actor ) {
return
}
request , ok := decodeRequired [ hypervisor . ResizeResourcesRequest ] ( w , body )
if ! ok {
return
}
if err := hypervisor . ValidateResizeResources ( request ) ; err != nil {
writePreDispatchMappedError ( w , err )
return
}
expected := & operation . ExpectedState {
Kind : operation . ExpectedResizeResources ,
FromVCPUs : request . FromVCPUs ,
TargetVCPUs : request . TargetVCPUs ,
FromMemoryMiB : request . FromMemoryMiB ,
TargetMemoryMiB : request . TargetMemoryMiB ,
FromDiskGiB : request . FromDiskGiB ,
TargetDiskGiB : request . TargetDiskGiB ,
FromBandwidthMbps : request . FromBandwidthMbps ,
TargetBandwidthMbps : request . TargetBandwidthMbps ,
}
s . runMutationExpected ( w , r , body , actor , request . OperationID , uuid , "resize-vm" , expected , true , http . StatusOK ,
hypervisor . ResizeResourcesAdmission ( uuid , request ) ,
func ( ctx context . Context ) ( any , error ) { return s . Hypervisor . ResizeResources ( ctx , uuid , request ) } )
return
}
case "network" :
if len ( segments ) == 4 && r . Method == http . MethodPost {
if ! requireManage ( w , actor ) {
return
}
request , ok := decodeRequired [ hypervisor . ChangeNetworkRequest ] ( w , body )
if ! ok {
return
}
if ! request . ConfirmRestart {
writePreDispatchError ( w , http . StatusUnprocessableEntity , "restart_confirmation_required" , "confirm_restart must be true" )
return
}
if err := hypervisor . ValidatePublicNetworkIntent ( request . PublicNetwork ) ; err != nil {
writePreDispatchMappedError ( w , err )
return
}
var source hypervisor . LocalNetworkSource
var err error
if s . MutationExecutor != nil {
source , err = hypervisor . ParseNetworkSelector ( request . PublicNetwork . NetworkSelector )
} else {
source , err = s . Hypervisor . ResolveNetworkSelector ( r . Context ( ) , request . PublicNetwork . NetworkSelector )
}
if err != nil {
writePreDispatchMappedError ( w , err )
return
}
expected := & operation . ExpectedState {
Kind : operation . ExpectedChangeNetwork , PublicNetwork : request . PublicNetwork ,
NetworkSourceType : source . Type , NetworkSourceName : source . Source ,
}
s . runMutationExpected ( w , r , body , actor , request . OperationID , uuid , "change-network" , expected , true , http . StatusAccepted ,
hypervisor . ChangeNetworkAdmission ( uuid , request . PublicNetwork ) ,
func ( ctx context . Context ) ( any , error ) { return s . Hypervisor . ChangeNetwork ( ctx , uuid , request ) } )
return
}
if len ( segments ) == 5 && r . Method == http . MethodPost && ( segments [ 4 ] == "suspend" || segments [ 4 ] == "resume" ) {
if ! requireManage ( w , actor ) {
return
}
request , ok := decodeOptional [ hypervisor . NetworkStateRequest ] ( w , body )
if ! ok {
return
}
enabled := segments [ 4 ] == "resume"
action := "network-" + segments [ 4 ]
expected := & operation . ExpectedState { Kind : operation . ExpectedNetworkState , Enabled : enabled }
s . runMutationExpected ( w , r , body , actor , request . OperationID , uuid , action , expected , false , http . StatusOK ,
hypervisor . DomainMutationAdmission ( uuid ) ,
func ( ctx context . Context ) ( any , error ) {
err := s . Hypervisor . SetNetworkState ( ctx , uuid , enabled , request )
return map [ string ] bool { "network_enabled" : enabled } , err
} )
return
}
case "vpc-port" :
repairPath := len ( segments ) == 5 && segments [ 4 ] == "repair"
if ( len ( segments ) == 4 || repairPath ) &&
( r . Method == http . MethodPost || r . Method == http . MethodDelete ) {
if ! requireManage ( w , actor ) {
return
}
request , ok := decodeRequired [ hypervisor . VPCPortRequest ] ( w , body )
if ! ok {
return
}
operationID , err := resolveOperationID ( actor . OperationID , request . OperationID )
if err != nil {
writePreDispatchError ( w , http . StatusUnprocessableEntity , "invalid_operation_id" , err . Error ( ) )
return
}
request . OperationID = operationID
request , err = hypervisor . NormalizeVPCPortRequest ( request )
if err != nil {
writePreDispatchMappedError ( w , err )
return
}
if request . VPCRepair != repairPath ||
( request . VPCRepair != ( request . SupersedesOperationID != "" ) ) {
writePreDispatchError ( w , http . StatusUnprocessableEntity , "invalid_vpc_repair" , "VPC repair requires the repair endpoint and one predecessor operation" )
return
}
bridge , _ := vpcrouting . BridgeName ( request . VPCKey )
present := r . Method == http . MethodPost
action := "apply-vpc-port"
if ! present {
action = "remove-vpc-port"
}
expected := & operation . ExpectedState {
Kind : operation . ExpectedVPCPort , VPCPresent : present ,
VPCPortKey : request . PortKey , VPCKey : request . VPCKey ,
VPCPrivateIP : request . PrivateIP , VPCCIDR : request . CIDR ,
VPCGateway : request . Gateway , VPCMAC : request . MAC ,
VPCBridge : bridge , VPCRevision : request . HostRevision ,
VPCDigest : request . HostDigest ,
}
admission := hypervisor . VPCPortAdmission ( uuid , request )
supersedes , ok := s . authorizeVPCRepair (
w , r , operationID , request . SupersedesOperationID ,
uuid , action , expected , admission , repairPath ,
)
if ! ok {
return
}
s . runMutationExpected ( w , r , body , actor , operationID , uuid ,
action , expected , true , http . StatusOK ,
admission ,
func ( ctx context . Context ) ( any , error ) {
if present {
return s . Hypervisor . ApplyVPCPort ( ctx , uuid , request )
}
return s . Hypervisor . RemoveVPCPort ( ctx , uuid , request )
} , supersedes ... )
return
}
case "service" :
if len ( segments ) == 5 && r . Method == http . MethodPost && ( segments [ 4 ] == "suspend" || segments [ 4 ] == "resume" ) {
if ! requireManage ( w , actor ) {
return
}
request , ok := decodeOptional [ hypervisor . ServiceStateRequest ] ( w , body )
if ! ok {
return
}
enabled := segments [ 4 ] == "resume"
expected := & operation . ExpectedState { Kind : operation . ExpectedServiceState , Enabled : enabled , StartIfSuspended : request . StartIfSuspended }
s . runMutationExpected ( w , r , body , actor , request . OperationID , uuid , "service-" + segments [ 4 ] , expected , true , http . StatusOK ,
hypervisor . DomainMutationAdmission ( uuid ) ,
func ( ctx context . Context ) ( any , error ) {
return s . Hypervisor . SetServiceState ( ctx , uuid , enabled , request )
} )
return
}
case "reinstall" :
if len ( segments ) == 4 && r . Method == http . MethodPost {
if ! requireManage ( w , actor ) {
return
}
request , ok := decodeRequired [ hypervisor . ReinstallRequest ] ( w , body )
if ! ok {
return
}
if err := hypervisor . ValidateTemplateSnapshot ( request . TemplateID , request . TemplateRevision , request . TemplateGenerationToken ) ; err != nil {
writePreDispatchMappedError ( w , err )
return
}
resolvedOperationID , err := resolveOperationID ( actor . OperationID , request . OperationID )
if err != nil {
writePreDispatchError ( w , http . StatusUnprocessableEntity , "invalid_operation_id" , err . Error ( ) )
return
}
request . OperationID = resolvedOperationID
s . runMutation ( w , r , body , actor , request . OperationID , uuid , "reinstall-vm" , true , http . StatusAccepted ,
hypervisor . ReinstallVMAdmission ( uuid , request ) ,
func ( ctx context . Context ) ( any , error ) { return s . Hypervisor . Reinstall ( ctx , uuid , request ) } )
return
}
case "metrics" :
if len ( segments ) == 4 && r . Method == http . MethodGet {
result , err := s . Hypervisor . GetMetrics ( r . Context ( ) , uuid )
writeResult ( w , result , err )
return
}
case "console-ticket" :
if len ( segments ) == 4 && r . Method == http . MethodPost {
if ! requireManage ( w , actor ) {
return
}
target , err := s . Hypervisor . GetConsole ( r . Context ( ) , uuid )
if err != nil {
writeMappedError ( w , err , nil )
return
}
if target . Protocol != "vnc" {
writeError ( w , http . StatusNotImplemented , "unsupported_console" , "only VNC console targets are supported" , nil )
return
}
ticket , err := s . Console . Issue ( target . Address )
if err != nil {
writeError ( w , http . StatusServiceUnavailable , "console_unavailable" , "console target is unavailable" , nil )
return
}
writeJSON ( w , http . StatusCreated , ticket )
return
}
case "snapshots" :
s . routeSnapshots ( w , r , segments , body , actor , uuid )
return
case "firewall" :
if len ( segments ) == 4 {
switch r . Method {
case http . MethodGet :
result , err := s . Hypervisor . GetFirewall ( r . Context ( ) , uuid )
writeResult ( w , result , err )
case http . MethodPut :
if ! requireManage ( w , actor ) {
return
}
request , ok := decodeRequired [ hypervisor . FirewallPolicy ] ( w , body )
if ! ok {
return
}
operationID := request . OperationID
request , err := hypervisor . NormalizeFirewallPolicy ( request )
if err != nil {
writePreDispatchMappedError ( w , err )
return
}
expected := & operation . ExpectedState { Kind : operation . ExpectedFirewall , FirewallRevision : request . Revision , FirewallDigest : request . PolicyDigest }
s . runMutationExpected ( w , r , body , actor , operationID , uuid , "apply-firewall" , expected , false , http . StatusOK ,
hypervisor . FirewallMutationAdmission ( uuid ) ,
func ( ctx context . Context ) ( any , error ) { return s . Hypervisor . ApplyFirewall ( ctx , uuid , request ) } )
default :
methodOrNotFound ( w , r , true , http . MethodGet , http . MethodPut )
}
return
}
case "rescue" :
s . routeRescue ( w , r , segments , body , actor , uuid )
return
}
writePreDispatchError ( w , http . StatusNotFound , "not_found" , "endpoint not found" )
}
func peerMigrationTargetCleanupInspectionMatches ( prior operation . Record , planID string ,
inspection hypervisor . OperationInspection ) bool {
if inspection . OperationID != prior . OperationID || inspection . Action != prior . Action ||
inspection . ResourceID != prior . ResourceID || inspection . RequestFingerprint != prior . Fingerprint ||
inspection . OperationStatus != prior . State || inspection . CheckedAt <= 0 ||
inspection . SummaryCode != hypervisor . InspectionSummaryCleanupRequired ||
inspection . EffectResolution != hypervisor . InspectionEffectPartial ||
inspection . CompletionDeliveryState != "" || inspection . CompletionDeliveryErrorCode != "" ||
inspection . AbortJob . Available || ! inspection . Cleanup . Available ||
! strings . EqualFold ( inspection . Cleanup . PlanID , planID ) || len ( inspection . Cleanup . Steps ) == 0 {
return false
}
for _ , step := range inspection . Cleanup . Steps {
switch step {
case hypervisor . CleanupStepDeleteManagedVolumes ,
hypervisor . CleanupStepDeleteSeedVolume ,
hypervisor . CleanupStepUndefineNetworkFilter ,
hypervisor . CleanupStepDeleteNVRAM ,
hypervisor . CleanupStepRemoveTrafficState :
default :
return false
}
}
return true
}
func backupWorkspaceCleanupInspectionMatches ( prior operation . Record , planID string ,
inspection hypervisor . OperationInspection ) bool {
if inspection . OperationID != prior . OperationID || inspection . Action != prior . Action ||
inspection . ResourceID != prior . ResourceID || inspection . RequestFingerprint != prior . Fingerprint ||
inspection . OperationStatus != prior . State || inspection . CheckedAt <= 0 ||
inspection . Outcome != hypervisor . InspectionOutcomeSucceeded ||
inspection . EffectResolution != hypervisor . InspectionEffectApplied ||
inspection . CompletionDeliveryState != "" || inspection . CompletionDeliveryErrorCode != "" ||
inspection . AbortJob . Available {
return false
}
if ! inspection . Cleanup . Available {
return planID == "" &&
inspection . SummaryCode == hypervisor . InspectionSummarySucceeded
}
if inspection . SummaryCode != hypervisor . InspectionSummaryCleanupRequired ||
planID == "" || ! strings . EqualFold ( inspection . Cleanup . PlanID , planID ) ||
len ( inspection . Cleanup . Steps ) == 0 {
return false
}
for _ , step := range inspection . Cleanup . Steps {
switch step {
case hypervisor . CleanupStepDeleteSnapshotVolume ,
hypervisor . CleanupStepRemoveSnapshotMetadata :
default :
return false
}
}
return true
}
func ( s * Server ) routeSnapshots ( w http . ResponseWriter , r * http . Request , segments [ ] string , body [ ] byte , actor auth . Actor , uuid string ) {
if len ( segments ) == 4 {
switch r . Method {
case http . MethodGet :
result , err := s . Hypervisor . ListSnapshots ( r . Context ( ) , uuid )
writeNamedList ( w , "snapshots" , result , err )
case http . MethodPost :
if ! requireManage ( w , actor ) {
return
}
request , ok := decodeRequired [ hypervisor . SnapshotRequest ] ( w , body )
if ! ok {
return
}
if request . SnapshotKey == "" {
request . SnapshotKey = request . SnapshotID
}
if ! catalogIDPattern . MatchString ( request . SnapshotKey ) {
writePreDispatchError ( w , http . StatusUnprocessableEntity , "invalid_snapshot" , "snapshot key is invalid" )
return
}
resolvedOperationID , err := resolveOperationID ( actor . OperationID , request . OperationID )
if err != nil {
writePreDispatchError ( w , http . StatusUnprocessableEntity , "invalid_operation_id" , err . Error ( ) )
return
}
request . OperationID = resolvedOperationID
action := "create-snapshot:" + request . SnapshotKey + ":" + request . ConsistencyMode
s . runMutation ( w , r , body , actor , request . OperationID , uuid , action , true , http . StatusCreated ,
hypervisor . CreateSnapshotAdmission ( uuid , request . SnapshotKey ) ,
func ( ctx context . Context ) ( any , error ) { return s . Hypervisor . CreateSnapshot ( ctx , uuid , request ) } )
default :
methodOrNotFound ( w , r , true , http . MethodGet , http . MethodPost )
}
return
}
if len ( segments ) < 5 || ! resourceIDPattern . MatchString ( segments [ 4 ] ) {
writePreDispatchError ( w , http . StatusUnprocessableEntity , "invalid_snapshot_id" , "snapshot ID is invalid" )
return
}
snapshotID := segments [ 4 ]
if len ( segments ) == 5 {
switch r . Method {
case http . MethodGet :
snapshots , err := s . Hypervisor . ListSnapshots ( r . Context ( ) , uuid )
if err != nil {
writeMappedError ( w , err , nil )
return
}
for _ , snapshot := range snapshots {
if snapshot . ID == snapshotID {
writeJSON ( w , http . StatusOK , snapshot )
return
}
}
writeError ( w , http . StatusNotFound , "snapshot_not_found" , "snapshot was not found" , nil )
case http . MethodDelete :
if ! requireManage ( w , actor ) {
return
}
request , ok := decodeOptional [ hypervisor . SnapshotMutationRequest ] ( w , body )
if ! ok {
return
}
resolvedOperationID , err := resolveOperationID ( actor . OperationID , request . OperationID )
if err != nil {
writePreDispatchError ( w , http . StatusUnprocessableEntity , "invalid_operation_id" , err . Error ( ) )
return
}
request . OperationID = resolvedOperationID
s . runMutation ( w , r , body , actor , request . OperationID , uuid , "delete-snapshot:" + snapshotID , true , http . StatusOK ,
hypervisor . DeleteSnapshotAdmission ( uuid , snapshotID ) ,
func ( ctx context . Context ) ( any , error ) {
return s . Hypervisor . DeleteSnapshot ( ctx , uuid , snapshotID , request )
} )
default :
methodOrNotFound ( w , r , true , http . MethodGet , http . MethodDelete )
}
return
}
if len ( segments ) == 6 && segments [ 5 ] == "restore" && r . Method == http . MethodPost {
if ! requireManage ( w , actor ) {
return
}
request , ok := decodeOptional [ hypervisor . SnapshotMutationRequest ] ( w , body )
if ! ok {
return
}
resolvedOperationID , err := resolveOperationID ( actor . OperationID , request . OperationID )
if err != nil {
writePreDispatchError ( w , http . StatusUnprocessableEntity , "invalid_operation_id" , err . Error ( ) )
return
}
request . OperationID = resolvedOperationID
s . runMutation ( w , r , body , actor , request . OperationID , uuid , "restore-snapshot:" + snapshotID , true , http . StatusAccepted ,
hypervisor . RestoreSnapshotAdmission ( uuid , snapshotID , request . OperationID ) ,
func ( ctx context . Context ) ( any , error ) {
return s . Hypervisor . RestoreSnapshot ( ctx , uuid , snapshotID , request )
} )
return
}
writePreDispatchError ( w , http . StatusNotFound , "not_found" , "endpoint not found" )
}
func ( s * Server ) routeRescue ( w http . ResponseWriter , r * http . Request , segments [ ] string , body [ ] byte , actor auth . Actor , uuid string ) {
if len ( segments ) == 4 && r . Method == http . MethodGet {
result , err := s . Hypervisor . GetRescue ( r . Context ( ) , uuid )
writeResult ( w , result , err )
return
}
if len ( segments ) == 5 && r . Method == http . MethodPost && ( segments [ 4 ] == "enter" || segments [ 4 ] == "exit" || segments [ 4 ] == "force-start" ) {
if ! requireManage ( w , actor ) {
return
}
if segments [ 4 ] == "enter" {
request , ok := decodeRequired [ hypervisor . RescueRequest ] ( w , body )
if ! ok {
return
}
if err := hypervisor . ValidateRescueTemplateSnapshot (
request . TemplateID ,
request . Revision ,
request . GenerationToken ,
) ; err != nil {
writePreDispatchMappedError ( w , err )
return
}
s . runMutation ( w , r , body , actor , request . OperationID , uuid , "enter-rescue" , true , http . StatusAccepted ,
hypervisor . EnterRescueAdmission ( uuid , request . TemplateID ) ,
func ( ctx context . Context ) ( any , error ) { return s . Hypervisor . EnterRescue ( ctx , uuid , request ) } )
} else if segments [ 4 ] == "force-start" {
request , ok := decodeRequired [ hypervisor . ForceRescueStartRequest ] ( w , body )
if ! ok {
return
}
if ! request . AckDataLoss {
writePreDispatchError ( w , http . StatusUnprocessableEntity , "data_loss_ack_required" , "ack_data_loss must be true" )
return
}
s . runMutation ( w , r , body , actor , request . OperationID , uuid , "force-start-rescue" , true , http . StatusAccepted ,
hypervisor . DomainMutationAdmission ( uuid ) ,
func ( ctx context . Context ) ( any , error ) { return s . Hypervisor . ForceStartRescue ( ctx , uuid , request ) } )
} else {
request , ok := decodeRequired [ hypervisor . ExitRescueRequest ] ( w , body )
if ! ok {
return
}
if ! request . AckDataLoss {
writePreDispatchError ( w , http . StatusUnprocessableEntity , "data_loss_ack_required" , "ack_data_loss must be true" )
return
}
if request . NetworkSuspended == nil {
writePreDispatchError ( w , http . StatusUnprocessableEntity , "network_state_required" , "network_suspended must be provided" )
return
}
action := "exit-rescue:connected"
if * request . NetworkSuspended {
action = "exit-rescue:suspended"
}
s . runMutation ( w , r , body , actor , request . OperationID , uuid , action , true , http . StatusAccepted ,
hypervisor . DomainMutationAdmission ( uuid ) ,
func ( ctx context . Context ) ( any , error ) { return s . Hypervisor . ExitRescue ( ctx , uuid , request ) } )
}
return
}
writePreDispatchError ( w , http . StatusNotFound , "not_found" , "endpoint not found" )
}
func ( s * Server ) routeTemplates ( w http . ResponseWriter , r * http . Request , segments [ ] string , body [ ] byte , actor auth . Actor ) {
if ! actor . IsAdmin && ! actor . IsSuperAdmin {
writePreDispatchError ( w , http . StatusForbidden , "forbidden" , "administrator privileges are required" )
return
}
if len ( segments ) == 2 {
switch r . Method {
case http . MethodGet :
result , err := s . Hypervisor . ListTemplates ( r . Context ( ) )
writeNamedList ( w , "templates" , result , err )
case http . MethodPost :
if ! requireManage ( w , actor ) {
return
}
request , ok := decodeRequired [ hypervisor . ImportTemplateRequest ] ( w , body )
if ! ok {
return
}
if err := hypervisor . ValidateTemplateSnapshot ( request . ID , request . Revision , request . GenerationToken ) ; err != nil {
writePreDispatchMappedError ( w , err )
return
}
if err := validateImportSource ( request . SourceURL , request . SourceToken , request . SHA256 ) ; err != nil {
writePreDispatchMappedError ( w , err )
return
}
s . runAsyncMutation ( w , r , body , actor , request . OperationID , "template:" + request . ID , "import-template" , false ,
hypervisor . ImportTemplateAdmission ( request ) ,
func ( ctx context . Context ) ( any , error ) { return s . Hypervisor . ImportTemplate ( ctx , request ) } )
default :
methodOrNotFound ( w , r , true , http . MethodGet , http . MethodPost )
}
return
}
if len ( segments ) == 3 && resourceIDPattern . MatchString ( segments [ 2 ] ) {
templateID := segments [ 2 ]
switch r . Method {
case http . MethodPatch :
if ! requireManage ( w , actor ) {
return
}
request , ok := decodeRequired [ hypervisor . UpdateTemplateRequest ] ( w , body )
if ! ok {
return
}
if strings . TrimSpace ( request . Name ) == "" || len ( request . Name ) > 120 {
writePreDispatchError ( w , http . StatusUnprocessableEntity , "invalid_template_name" , "template name must contain 1-120 characters" )
return
}
s . runMutation ( w , r , body , actor , request . OperationID , "template:" + templateID , "update-template" , false , http . StatusOK ,
hypervisor . UpdateTemplateAdmission ( templateID , request ) ,
func ( ctx context . Context ) ( any , error ) { return s . Hypervisor . UpdateTemplate ( ctx , templateID , request ) } )
case http . MethodDelete :
if ! requireManage ( w , actor ) {
return
}
request , ok := decodeOptional [ hypervisor . MutationRequest ] ( w , body )
if ! ok {
return
}
s . runMutation ( w , r , body , actor , request . OperationID , "template:" + templateID , "delete-template" , true , http . StatusOK ,
hypervisor . DeleteTemplateAdmission ( templateID ) ,
func ( ctx context . Context ) ( any , error ) {
err := s . Hypervisor . DeleteTemplate ( ctx , templateID )
return catalogDeleteResult ( "template" , templateID , 0 , err == nil ) , err
} )
default :
methodOrNotFound ( w , r , true , http . MethodPatch , http . MethodDelete )
}
return
}
writePreDispatchError ( w , http . StatusNotFound , "not_found" , "endpoint not found" )
}
func ( s * Server ) routeRescueTemplates ( w http . ResponseWriter , r * http . Request , segments [ ] string , body [ ] byte , actor auth . Actor ) {
if ! actor . IsAdmin && ! actor . IsSuperAdmin {
writePreDispatchError ( w , http . StatusForbidden , "forbidden" , "administrator privileges are required" )
return
}
if len ( segments ) == 2 {
switch r . Method {
case http . MethodGet :
result , err := s . Hypervisor . ListRescueTemplates ( r . Context ( ) )
writeNamedList ( w , "templates" , result , err )
case http . MethodPost :
if ! requireManage ( w , actor ) {
return
}
request , ok := decodeRequired [ hypervisor . ImportRescueTemplateRequest ] ( w , body )
if ! ok {
return
}
if err := hypervisor . ValidateRescueTemplateSnapshot ( request . ID , request . Revision , request . GenerationToken ) ; err != nil {
writePreDispatchMappedError ( w , err )
return
}
if err := validateImportSource ( request . SourceURL , request . SourceToken , request . SHA256 ) ; err != nil {
writePreDispatchMappedError ( w , err )
return
}
s . runAsyncMutation ( w , r , body , actor , request . OperationID , "rescue-template:" + request . ID , "import-rescue-template" , false ,
hypervisor . ImportRescueTemplateAdmission ( request ) ,
func ( ctx context . Context ) ( any , error ) { return s . Hypervisor . ImportRescueTemplate ( ctx , request ) } )
default :
methodOrNotFound ( w , r , true , http . MethodGet , http . MethodPost )
}
return
}
if len ( segments ) == 3 && catalogIDPattern . MatchString ( segments [ 2 ] ) && r . Method == http . MethodDelete {
if ! requireManage ( w , actor ) {
return
}
request , ok := decodeOptional [ hypervisor . MutationRequest ] ( w , body )
if ! ok {
return
}
id := segments [ 2 ]
s . runMutation ( w , r , body , actor , request . OperationID , "rescue-template:" + id , "delete-rescue-template" , true , http . StatusOK ,
hypervisor . DeleteRescueTemplateAdmission ( id ) ,
func ( ctx context . Context ) ( any , error ) {
err := s . Hypervisor . DeleteRescueTemplate ( ctx , id )
return catalogDeleteResult ( "rescue_template" , id , 0 , err == nil ) , err
} )
return
}
writePreDispatchError ( w , http . StatusNotFound , "not_found" , "endpoint not found" )
}
func ( s * Server ) routeUserISOs ( w http . ResponseWriter , r * http . Request , segments [ ] string , body [ ] byte , actor auth . Actor ) {
ownerID , err := ownerFromRequest ( r , actor )
if err != nil {
writePreDispatchError ( w , http . StatusForbidden , "forbidden" , err . Error ( ) )
return
}
if len ( segments ) == 2 {
switch r . Method {
case http . MethodGet :
result , err := s . Hypervisor . ListUserISOs ( r . Context ( ) , ownerID )
writeNamedList ( w , "user_isos" , result , err )
case http . MethodPost :
if ! requireManage ( w , actor ) {
return
}
request , ok := decodeRequired [ hypervisor . ImportUserISORequest ] ( w , body )
if ! ok {
return
}
if request . OwnerID == 0 {
request . OwnerID = ownerID
}
if request . OwnerID != ownerID && ! actor . IsAdmin && ! actor . IsSuperAdmin {
writePreDispatchError ( w , http . StatusForbidden , "forbidden" , "cannot import ISO for another owner" )
return
}
operationID , err := resolveOperationID ( actor . OperationID , request . OperationID )
if err != nil {
writePreDispatchError ( w , http . StatusUnprocessableEntity , "invalid_operation_id" , err . Error ( ) )
return
}
if request . ID == "" {
request . ID = operationCatalogID ( operationID )
}
if ! catalogIDPattern . MatchString ( request . ID ) {
writePreDispatchError ( w , http . StatusUnprocessableEntity , "invalid_user_iso_id" , "id must contain 32 lowercase hexadecimal characters" )
return
}
if err := validateImportSource ( request . SourceURL , request . SourceToken , request . SHA256 ) ; err != nil {
writePreDispatchMappedError ( w , err )
return
}
s . runAsyncMutation ( w , r , body , actor , request . OperationID , fmt . Sprintf ( "user-iso:%d:%s" , request . OwnerID , request . ID ) , "import-user-iso" , false ,
hypervisor . ImportUserISOAdmission ( request ) ,
func ( ctx context . Context ) ( any , error ) { return s . Hypervisor . ImportUserISO ( ctx , request ) } )
default :
methodOrNotFound ( w , r , true , http . MethodGet , http . MethodPost )
}
return
}
if len ( segments ) == 3 && catalogIDPattern . MatchString ( segments [ 2 ] ) && r . Method == http . MethodDelete {
if ! requireManage ( w , actor ) {
return
}
request , ok := decodeOptional [ hypervisor . MutationRequest ] ( w , body )
if ! ok {
return
}
isoID := segments [ 2 ]
s . runMutation ( w , r , body , actor , request . OperationID , fmt . Sprintf ( "user-iso:%d:%s" , ownerID , isoID ) , "delete-user-iso" , true , http . StatusOK ,
hypervisor . DeleteUserISOAdmission ( ownerID , isoID ) ,
func ( ctx context . Context ) ( any , error ) {
err := s . Hypervisor . DeleteUserISO ( ctx , isoID , ownerID )
return catalogDeleteResult ( "user_iso" , isoID , ownerID , err == nil ) , err
} )
return
}
writePreDispatchError ( w , http . StatusNotFound , "not_found" , "endpoint not found" )
}
func catalogDeleteResult ( resourceType , id string , ownerID int64 , deleted bool ) hypervisor . CatalogDeleteResult {
return hypervisor . CatalogDeleteResult {
ResourceType : resourceType , ID : id , OwnerID : ownerID , Deleted : deleted ,
ManagedImageAbsent : deleted , ManagedMetadataAbsent : deleted , ManagedArtifactsAbsent : deleted ,
}
}
func ( s * Server ) routeOperations ( w http . ResponseWriter , r * http . Request , segments [ ] string , body [ ] byte , actor auth . Actor ) {
if len ( segments ) < 3 || ! resourceIDPattern . MatchString ( segments [ 2 ] ) {
writeError ( w , http . StatusNotFound , "operation_not_found" , "operation was not found" , nil )
return
}
operationID := segments [ 2 ]
if len ( segments ) == 3 && r . Method == http . MethodGet {
record , err := s . Journal . Get ( operationID )
if err != nil {
writeMappedError ( w , err , nil )
return
}
if record . Action == "configure-storage" && ! requireSuperAdminManage ( w , actor ) {
return
}
writeJSON ( w , http . StatusOK , record )
return
}
if len ( segments ) == 4 && segments [ 3 ] == "inspect" && r . Method == http . MethodGet {
if ! requireSuperAdminInspection ( w , actor , operationID ) {
return
}
stored , err := s . Journal . GetForInspection ( operationID )
if err != nil {
writeMappedError ( w , err , nil )
return
}
if ! requireSystemRecoveryRecord ( w , actor , stored ) {
return
}
incident , incidentFound , err := s . Journal . CompletionDeliveryIncident ( operationID )
if err != nil {
writeMappedError ( w , err , nil )
return
}
var inspection hypervisor . OperationInspection
if incidentFound {
// Delivery corruption is already sufficient to make the panel view
// inconclusive. Do not make reporting that durable Agent incident
// depend on a live host read which may itself time out or fail.
inspection = hypervisor . OperationInspection {
OperationStatus : stored . State ,
CheckedAt : time . Now ( ) . UTC ( ) . Unix ( ) ,
}
if stored . Error != nil {
inspection . ErrorCode = stored . Error . Code
}
applyCompletionDeliveryIncident ( & inspection , incident )
} else {
ctx , cancel := context . WithTimeout ( r . Context ( ) , s . OperationTimeout )
defer cancel ( )
inspection , err = s . Hypervisor . InspectOperation ( ctx , stored )
if err != nil {
writeMappedError ( w , err , nil )
return
}
}
applyExecutionFencedCreateInspection ( & inspection , stored )
// Identity always comes from the immutable Journal record, never from a
// hypervisor adapter or its observation of current host state.
inspection . OperationID = stored . OperationID
inspection . Action = stored . Action
inspection . ResourceID = stored . ResourceID
inspection . RequestFingerprint = stored . Fingerprint
inspection . FenceState , err = s . Journal . FenceState ( operationID )
if err != nil {
writeMappedError ( w , err , nil )
return
}
inspection . FenceResolutionRequired =
inspection . FenceState == operation . FenceStateActive
writeJSON ( w , http . StatusOK , inspection )
return
}
if len ( segments ) == 4 && segments [ 3 ] == "cleanup-backup-workspace" &&
r . Method == http . MethodPost {
if ! requireSuperAdminPrivilegedOperation ( w , actor ) {
return
}
if actor . OperationID == "" || actor . OperationID != operationID {
writePreDispatchError ( w , http . StatusBadRequest , "operation_id_mismatch" , "actor operation_id must match the cleanup operation" )
return
}
request , ok := decodeRequired [ hypervisor . CleanupBackupWorkspaceRequest ] ( w , body )
if ! ok {
return
}
if err := hypervisor . ValidateCleanupBackupWorkspaceRequest ( request ) ; err != nil {
writePreDispatchMappedError ( w , err )
return
}
action := ""
existingReplay := false
trustedPrior , trustedPriorFound := trustedSupersededOperation (
r . Context ( ) , request . PriorOperationID ,
)
existing , existingErr := s . Journal . Get ( operationID )
if existingErr == nil {
action = existing . Action
existingReplay = strings . HasPrefix ( action , "cleanup-backup-workspace:" ) &&
existing . SupersedesOperationID == request . PriorOperationID
} else if ! errors . Is ( existingErr , operation . ErrNotFound ) {
writeMappedError ( w , existingErr , nil )
return
}
prior := trustedPrior
if ! existingReplay {
var priorErr error
if ! trustedPriorFound {
prior , priorErr = s . Journal . Get ( request . PriorOperationID )
}
parts := strings . Split ( prior . Action , ":" )
if priorErr != nil || len ( parts ) != 3 || parts [ 0 ] != "create-backup" ||
! catalogIDPattern . MatchString ( parts [ 1 ] ) ||
prior . State != operation . StateSucceeded ||
prior . ResourceID == "" || prior . ExecutionFenced {
writePreDispatchError ( w , http . StatusConflict , "backup_workspace_cleanup_not_authorized" , "cleanup requires the exact successful backup operation" )
return
}
action = "cleanup-backup-workspace:" + parts [ 1 ]
if ! trustedPriorFound {
_ , incidentFound , incidentErr := s . Journal . CompletionDeliveryIncident ( prior . OperationID )
if incidentErr != nil {
writeMappedError ( w , incidentErr , nil )
return
}
if incidentFound {
writePreDispatchError ( w , http . StatusConflict , "backup_workspace_cleanup_not_authorized" , "cleanup is blocked by a completion-delivery incident" )
return
}
}
ctx , cancel := context . WithTimeout ( r . Context ( ) , s . OperationTimeout )
inspection , inspectErr := s . Hypervisor . InspectOperation ( ctx , prior )
cancel ( )
if inspectErr != nil {
writeMappedError ( w , inspectErr , nil )
return
}
if ! backupWorkspaceCleanupInspectionMatches (
prior , request . CleanupPlanID , inspection ,
) {
writePreDispatchError ( w , http . StatusConflict , "backup_workspace_cleanup_plan_changed" , "cleanup plan is stale or no longer safe" )
return
}
request . PriorOperation = prior
}
parts := strings . Split ( action , ":" )
if len ( parts ) != 2 || ! catalogIDPattern . MatchString ( parts [ 1 ] ) {
writePreDispatchError ( w , http . StatusConflict , "backup_workspace_cleanup_not_authorized" , "cleanup action identity is invalid" )
return
}
resourceID := prior . ResourceID
if existingReplay {
resourceID = existing . ResourceID
}
supersedes := [ ] string { request . PriorOperationID }
if trustedPriorFound {
supersedes = nil
}
s . runMutationExpected ( w , r , body , actor , operationID , resourceID ,
action , nil , true , http . StatusOK ,
hypervisor . CleanupBackupWorkspaceAdmission ( resourceID , parts [ 1 ] ) ,
func ( ctx context . Context ) ( any , error ) {
return s . Hypervisor . CleanupBackupWorkspace ( ctx , resourceID , request )
} , supersedes ... )
return
}
if len ( segments ) == 4 && segments [ 3 ] == "fence-resolution" && r . Method == http . MethodGet {
if ! requireSuperAdminInspection ( w , actor , operationID ) {
return
}
stored , err := s . Journal . Get ( operationID )
if err != nil {
writeMappedError ( w , err , nil )
return
}
if ! requireSystemRecoveryRecord ( w , actor , stored ) {
return
}
resolution , err := s . Journal . GetFenceResolution ( operationID )
if errors . Is ( err , operation . ErrNotFound ) {
writeError ( w , http . StatusNotFound , "fence_resolution_not_found" , "operation fence resolution was not found" , nil )
return
}
if err != nil {
writeMappedError ( w , err , nil )
return
}
writeJSON ( w , http . StatusOK , fenceResolutionResponse { FenceResolution : resolution , Replayed : false } )
return
}
if len ( segments ) == 4 && segments [ 3 ] == "resolve-fence" && r . Method == http . MethodPost {
if ! requireSuperAdminFenceResolution ( w , actor , operationID ) {
return
}
if actor . OperationID == "" || actor . OperationID != operationID {
writeError ( w , http . StatusBadRequest , "operation_id_mismatch" , "actor operation_id must match the fenced operation" , nil )
return
}
stored , err := s . Journal . Get ( operationID )
if err != nil {
writeMappedError ( w , err , nil )
return
}
if ! requireSystemRecoveryRecord ( w , actor , stored ) {
return
}
request , ok := decodeRequired [ operation . FenceResolutionRequest ] ( w , body )
if ! ok {
return
}
if err := operation . ValidateFenceResolutionRequest ( request ) ; err != nil {
writeError ( w , http . StatusBadRequest , "invalid_fence_resolution" , err . Error ( ) , nil )
return
}
if actor . SystemRecovery && actor . UserID == 0 && operation . IsCreateLikeAction ( stored . Action ) {
if request . Resolution != operation . FenceResolutionNotApplied {
writeError ( w , http . StatusConflict , "operation_fence_not_resolvable" , "automatic create recovery can only confirm that creation was not applied" , nil )
return
}
if stored . ExecutionFenced {
writeError ( w , http . StatusConflict , "create_absence_not_confirmed" , "automatic create recovery cannot conclude while the original worker may still be running" , nil )
return
}
fenceState , err := s . Journal . FenceState ( operationID )
if err != nil {
writeMappedError ( w , err , nil )
return
}
if fenceState == operation . FenceStateActive {
_ , incidentFound , err := s . Journal . CompletionDeliveryIncident ( operationID )
if err != nil {
writeMappedError ( w , err , nil )
return
}
if incidentFound {
writeError ( w , http . StatusConflict , "create_absence_not_confirmed" , "automatic create recovery requires fresh complete host absence evidence" , nil )
return
}
ctx , cancel := context . WithTimeout ( r . Context ( ) , s . OperationTimeout )
inspection , inspectErr := s . Hypervisor . InspectOperation ( ctx , stored )
cancel ( )
if inspectErr != nil {
writeMappedError ( w , inspectErr , nil )
return
}
if ! systemCreateAbsenceConfirmed ( stored , request , inspection ) {
writeError ( w , http . StatusConflict , "create_absence_not_confirmed" , "automatic create recovery requires fresh complete host absence evidence" , nil )
return
}
}
}
resolution , replayed , err := s . Journal . ResolveResourceFence ( operationID , actor . UserID , request )
switch {
case errors . Is ( err , operation . ErrNotFound ) :
writeError ( w , http . StatusNotFound , "operation_not_found" , "operation was not found" , nil )
return
case errors . Is ( err , operation . ErrFenceNotResolvable ) :
writeError ( w , http . StatusConflict , "operation_fence_not_resolvable" , "operation does not hold an active effect fence" , nil )
return
case errors . Is ( err , operation . ErrFenceIdentityConflict ) :
writeError ( w , http . StatusConflict , "operation_identity_conflict" , "fence resolution identity does not match the immutable operation" , nil )
return
case errors . Is ( err , operation . ErrFenceResolutionConflict ) :
writeError ( w , http . StatusConflict , "operation_fence_resolution_conflict" , "operation fence already has a different audit resolution" , nil )
return
case errors . Is ( err , operation . ErrFenceInspectionStale ) :
writeError ( w , http . StatusConflict , "fence_inspection_stale" , "operation fence resolution requires a fresh read-only inspection" , nil )
return
case err != nil :
writeMappedError ( w , err , nil )
return
}
writeJSON ( w , http . StatusOK , fenceResolutionResponse { FenceResolution : resolution , Replayed : replayed } )
return
}
writePreDispatchError ( w , http . StatusNotFound , "not_found" , "endpoint not found" )
}
func ( s * Server ) createLikeNetworkExpectedState (
ctx context . Context ,
request hypervisor . CreateVMRequest ,
port * hypervisor . VPCPortMetadata ,
) ( * operation . ExpectedState , error ) {
var source hypervisor . LocalNetworkSource
var err error
if s . MutationExecutor != nil {
source , err = hypervisor . ParseNetworkSelector ( request . PublicNetwork . NetworkSelector )
} else {
source , err = s . Hypervisor . ResolveNetworkSelector ( ctx , request . PublicNetwork . NetworkSelector )
}
if err != nil {
return nil , err
}
expected := & operation . ExpectedState {
Kind : operation . ExpectedCreateLikeNetwork ,
NetworkSourceType : source . Type , NetworkSourceName : source . Source ,
PublicNetwork : request . PublicNetwork ,
VPCPresent : port != nil ,
}
if port != nil {
expected . VPCKey = port . VPCKey
expected . VPCRevision = port . HostRevision
expected . VPCDigest = port . HostDigest
expected . VPCPortKey = port . PortKey
expected . VPCCIDR = port . CIDR
expected . VPCGateway = port . Gateway
expected . VPCPrivateIP = port . PrivateIP
expected . VPCMAC = port . MAC
expected . VPCBridge = port . Bridge
}
return expected , nil
}
func ( s * Server ) runMutation (
w http . ResponseWriter ,
r * http . Request ,
body [ ] byte ,
actor auth . Actor ,
bodyOperationID , resourceID , action string ,
destructive bool ,
successStatus int ,
admission hypervisor . MutationAdmission ,
call func ( context . Context ) ( any , error ) ,
) {
s . runMutationExpected ( w , r , body , actor , bodyOperationID , resourceID , action , nil , destructive , successStatus , admission , call )
}
func ( s * Server ) runMutationExpected (
w http . ResponseWriter ,
r * http . Request ,
body [ ] byte ,
actor auth . Actor ,
bodyOperationID , resourceID , action string ,
expectedState * operation . ExpectedState ,
destructive bool ,
successStatus int ,
admission hypervisor . MutationAdmission ,
call func ( context . Context ) ( any , error ) ,
supersedesOperationID ... string ,
) {
operationID , err := resolveOperationID ( actor . OperationID , bodyOperationID )
if err != nil {
writePreDispatchError ( w , http . StatusUnprocessableEntity , "invalid_operation_id" , err . Error ( ) )
return
}
if expectedState != nil {
copyExpected := * expectedState
copyExpected . TransitionID = operationID
copyExpected . PublicNetwork . DNSServers = append ( [ ] string ( nil ) , expectedState . PublicNetwork . DNSServers ... )
expectedState = & copyExpected
}
supersedes := ""
if len ( supersedesOperationID ) != 0 {
supersedes = supersedesOperationID [ 0 ]
}
spec := operation . Spec {
OperationID : operationID , ResourceID : resourceID , Action : action ,
Fingerprint : operation . Fingerprint ( r . Method , auth . RequestTarget ( r ) , body ) ,
Destructive : destructive , ExpectedState : expectedState ,
LockKeys : append ( [ ] string ( nil ) , admission . LockKeys ... ) , HeavyIO : admission . HeavyIO ,
SupersedesOperationID : supersedes ,
}
if s . MutationExecutor != nil {
s . runIsolatedMutation ( w , r , body , actor , spec , s . isolatedMutationDeadline ( action ) )
return
}
if _ , trusted := r . Context ( ) . Value ( trustedWorkerRequestKey { } ) . ( struct { } ) ; ! trusted {
writePreDispatchError ( w , http . StatusServiceUnavailable , "mutation_worker_unavailable" , "isolated mutation worker is unavailable" )
return
}
ctx , cancel := context . WithTimeout ( r . Context ( ) , s . OperationTimeout )
defer cancel ( )
record , replayed , callErr := s . Journal . Execute ( ctx , spec , func ( ctx context . Context ) ( json . RawMessage , error ) {
result , err := call ( ctx )
if err != nil {
return nil , err
}
encoded , err := json . Marshal ( result )
if err != nil {
return nil , fmt . Errorf ( "encode operation result: %w" , err )
}
return encoded , nil
} )
logOperationOutcome ( record )
if errors . Is ( callErr , operation . ErrFingerprint ) {
writeError ( w , http . StatusConflict , "operation_id_conflict" , "operation_id is already bound to a different request" , nil )
return
}
if errors . Is ( callErr , operation . ErrOperationStarting ) {
writeOperationStarting ( w , record )
return
}
if operation . WasNotDispatched ( record ) {
writeOperationFailure ( w , record , replayed )
return
}
if callErr != nil {
if record . State == operation . StateFailed {
writeOperationFailure ( w , record , replayed )
return
}
writeMappedError ( w , callErr , & record )
return
}
if record . State == operation . StateFailed {
writeOperationFailure ( w , record , replayed )
return
}
if record . State == operation . StateDispatching || record . State == operation . StateReserved {
writeJSON ( w , http . StatusAccepted , operationWire ( record , replayed ) )
return
}
if replayed {
successStatus = http . StatusOK
}
writeJSON ( w , successStatus , operationWire ( record , replayed ) )
}
func ( s * Server ) runAsyncMutation (
w http . ResponseWriter ,
r * http . Request ,
body [ ] byte ,
actor auth . Actor ,
bodyOperationID , resourceID , action string ,
destructive bool ,
admission hypervisor . MutationAdmission ,
call func ( context . Context ) ( any , error ) ,
) {
s . runAsyncMutationWithDeadline (
w , r , body , actor , bodyOperationID , resourceID , action ,
destructive , admission , catalogImportTimeout , call ,
)
}
func ( s * Server ) runAsyncMutationWithDeadline (
w http . ResponseWriter ,
r * http . Request ,
body [ ] byte ,
actor auth . Actor ,
bodyOperationID , resourceID , action string ,
destructive bool ,
admission hypervisor . MutationAdmission ,
deadline time . Duration ,
call func ( context . Context ) ( any , error ) ,
) {
s . runAsyncMutationWithDeadlineExpected (
w , r , body , actor , bodyOperationID , resourceID , action , nil ,
destructive , admission , deadline , call ,
)
}
func ( s * Server ) runAsyncMutationWithDeadlineExpected (
w http . ResponseWriter ,
r * http . Request ,
body [ ] byte ,
actor auth . Actor ,
bodyOperationID , resourceID , action string ,
expectedState * operation . ExpectedState ,
destructive bool ,
admission hypervisor . MutationAdmission ,
deadline time . Duration ,
call func ( context . Context ) ( any , error ) ,
) {
operationID , err := resolveOperationID ( actor . OperationID , bodyOperationID )
if err != nil {
writePreDispatchError ( w , http . StatusUnprocessableEntity , "invalid_operation_id" , err . Error ( ) )
return
}
if expectedState != nil {
copyExpected := * expectedState
copyExpected . TransitionID = operationID
copyExpected . PublicNetwork . DNSServers = append ( [ ] string ( nil ) , expectedState . PublicNetwork . DNSServers ... )
expectedState = & copyExpected
}
spec := operation . Spec {
OperationID : operationID , ResourceID : resourceID , Action : action ,
Fingerprint : operation . Fingerprint ( r . Method , auth . RequestTarget ( r ) , body ) ,
Destructive : destructive , ExpectedState : expectedState ,
LockKeys : append ( [ ] string ( nil ) , admission . LockKeys ... ) , HeavyIO : admission . HeavyIO ,
}
if s . MutationExecutor != nil {
s . runIsolatedMutation ( w , r , body , actor , spec , deadline )
return
}
if _ , trusted := r . Context ( ) . Value ( trustedWorkerRequestKey { } ) . ( struct { } ) ; ! trusted {
writePreDispatchError ( w , http . StatusServiceUnavailable , "mutation_worker_unavailable" , "isolated mutation worker is unavailable" )
return
}
record , replayed , err := s . Journal . Reserve ( spec )
if errors . Is ( err , operation . ErrFingerprint ) {
writeError ( w , http . StatusConflict , "operation_id_conflict" , "operation_id is already bound to a different request" , nil )
return
}
if err != nil {
writeMappedError ( w , err , nil )
return
}
canStart := record . State == operation . StateReserved
if canStart && ! s . claimAsync ( operationID ) {
writeOperationStarting ( w , record )
return
}
if canStart {
select {
case s . asyncSlots <- struct { } { } :
default :
failed , failErr := s . Journal . FailNotDispatched ( operationID )
s . releaseAsync ( operationID )
if failErr != nil {
writeMappedError ( w , failErr , & record )
return
}
writeOperationFailure ( w , failed , false )
return
}
dispatchCtx , dispatchCancel := context . WithTimeout ( r . Context ( ) , s . AdmissionTimeout )
started , _ , done , startErr := s . Journal . ExecuteBackground (
dispatchCtx , s . backgroundCtx , deadline , spec ,
func ( ctx context . Context ) ( json . RawMessage , error ) {
result , callErr := call ( ctx )
if callErr != nil {
return nil , callErr
}
return json . Marshal ( result )
} ,
)
dispatchCancel ( )
if startErr != nil || done == nil {
<- s . asyncSlots
s . releaseAsync ( operationID )
if startErr != nil {
if errors . Is ( startErr , operation . ErrOperationStarting ) {
writeOperationStarting ( w , started )
return
}
if started . State == operation . StateFailed && started . Error != nil && started . Error . Code == "operation_not_dispatched" {
writeOperationFailure ( w , started , false )
return
}
writeMappedError ( w , startErr , & started )
return
}
record = started
} else {
record = started
s . asyncWG . Add ( 1 )
go func ( ) {
defer s . asyncWG . Done ( )
defer func ( ) { <- s . asyncSlots } ( )
defer s . releaseAsync ( operationID )
<- done
if final , getErr := s . Journal . Get ( operationID ) ; getErr == nil {
logOperationOutcome ( final )
}
} ( )
}
}
writeJSON ( w , http . StatusAccepted , operationWire ( record , replayed ) )
}
func ( s * Server ) runIsolatedMutation (
w http . ResponseWriter ,
r * http . Request ,
body [ ] byte ,
actor auth . Actor ,
spec operation . Spec ,
deadline time . Duration ,
) {
if deadline <= 0 {
deadline = s . OperationTimeout
}
actor . OperationID = spec . OperationID
request := worker . Request {
Version : worker . ProtocolVersion , Method : r . Method ,
RequestTarget : auth . RequestTarget ( r ) , Body : string ( body ) , Actor : actor ,
OperationID : spec . OperationID , Fingerprint : spec . Fingerprint ,
}
if spec . SupersedesOperationID != "" {
prior , err := s . Journal . Get ( spec . SupersedesOperationID )
if err != nil {
writeMappedError ( w , err , nil )
return
}
request . SupersededOperation = & prior
if isVPCRepairTarget ( request . RequestTarget ) {
request . SupersededExpectedState = prior . ExpectedState
}
}
dispatchCtx , dispatchCancel := context . WithTimeout ( r . Context ( ) , s . AdmissionTimeout )
record , replayed , done , err := s . Journal . ExecuteBackground (
dispatchCtx , s . backgroundCtx , deadline , spec ,
func ( callCtx context . Context ) ( json . RawMessage , error ) {
return s . MutationExecutor . Execute ( callCtx , request )
} ,
)
dispatchCancel ( )
if done != nil {
s . asyncWG . Add ( 1 )
go func ( operationID string ) {
defer s . asyncWG . Done ( )
<- done
if final , getErr := s . Journal . Get ( operationID ) ; getErr == nil {
logOperationOutcome ( final )
}
} ( spec . OperationID )
}
if errors . Is ( err , operation . ErrFingerprint ) {
writeError ( w , http . StatusConflict , "operation_id_conflict" , "operation_id is already bound to a different request" , nil )
return
}
if errors . Is ( err , operation . ErrOperationStarting ) {
writeOperationStarting ( w , record )
return
}
if operation . WasNotDispatched ( record ) {
writeJSON ( w , http . StatusOK , operationWire ( record , replayed ) )
return
}
if err != nil {
writeMappedError ( w , err , & record )
return
}
status := http . StatusAccepted
if record . State == operation . StateSucceeded || record . State == operation . StateFailed {
status = http . StatusOK
}
writeJSON ( w , status , operationWire ( record , replayed ) )
}
func ( s * Server ) isolatedMutationDeadline ( action string ) time . Duration {
deadline := s . OperationTimeout
if action == "configure-storage" && deadline > storageConfigurationWorkerTimeout {
return storageConfigurationWorkerTimeout
}
return deadline
}
func ( s * Server ) beginRequest ( ) bool {
s . lifecycleMu . Lock ( )
defer s . lifecycleMu . Unlock ( )
if s . closing {
return false
}
s . requestWG . Add ( 1 )
return true
}
// Shutdown stops background imports and waits for every admitted API handler
// and import watcher. The journal's own workers are drained separately before
// libvirt or BoltDB are closed.
func ( s * Server ) Shutdown ( ctx context . Context ) error {
s . lifecycleMu . Lock ( )
if ! s . closing {
s . closing = true
s . backgroundCancel ( )
}
s . lifecycleMu . Unlock ( )
if err := waitGroup ( ctx , & s . requestWG ) ; err != nil {
return err
}
return waitGroup ( ctx , & s . asyncWG )
}
func waitGroup ( ctx context . Context , group * sync . WaitGroup ) error {
done := make ( chan struct { } )
go func ( ) {
group . Wait ( )
close ( done )
} ( )
select {
case <- done :
return nil
case <- ctx . Done ( ) :
return ctx . Err ( )
}
}
func ( s * Server ) claimAsync ( operationID string ) bool {
s . asyncMu . Lock ( )
defer s . asyncMu . Unlock ( )
if s . asyncRunning == nil {
s . asyncRunning = make ( map [ string ] struct { } )
}
if _ , exists := s . asyncRunning [ operationID ] ; exists {
return false
}
s . asyncRunning [ operationID ] = struct { } { }
return true
}
func applyCompletionDeliveryIncident ( inspection * hypervisor . OperationInspection ,
incident operation . CompletionDeliveryIncident ) {
if inspection == nil {
return
}
inspection . Outcome = hypervisor . InspectionOutcomeUncertain
inspection . SummaryCode = hypervisor . InspectionSummaryIncomplete
inspection . EffectResolution = hypervisor . InspectionEffectInconclusive
inspection . EffectProjection = nil
inspection . AbortJob = hypervisor . OperationAbortInspection { }
inspection . Cleanup = hypervisor . OperationCleanupPlan { }
inspection . CompletionDeliveryState = operation . CompletionDeliveryStateQuarantined
inspection . CompletionDeliveryErrorCode = incident . ErrorCode
}
func applyExecutionFencedCreateInspection ( inspection * hypervisor . OperationInspection , record operation . Record ) {
if inspection == nil || ! operation . IsCreateLikeAction ( record . Action ) || ! record . ExecutionFenced {
return
}
inspection . Outcome = hypervisor . InspectionOutcomeUncertain
inspection . SummaryCode = hypervisor . InspectionSummaryIncomplete
inspection . ErrorCode = ""
if record . Error != nil {
inspection . ErrorCode = record . Error . Code
}
inspection . EffectResolution = hypervisor . InspectionEffectInconclusive
inspection . EffectProjection = nil
inspection . AbortJob = hypervisor . OperationAbortInspection { }
inspection . Cleanup = hypervisor . OperationCleanupPlan { }
}
// systemCreateAbsenceConfirmed accepts only a fresh, exact read-only host
// observation that both the VM and all panel-managed creation artifacts are
// absent. It does not infer absence from the original operation error and does
// not authorize any create replay or host mutation.
func systemCreateAbsenceConfirmed ( record operation . Record , request operation . FenceResolutionRequest ,
inspection hypervisor . OperationInspection ) bool {
if ! operation . IsCreateLikeAction ( record . Action ) || record . State != operation . StateFailed ||
inspection . OperationID != record . OperationID || inspection . Action != record . Action ||
inspection . ResourceID != record . ResourceID || inspection . RequestFingerprint != record . Fingerprint ||
inspection . OperationStatus != record . State || inspection . CheckedAt < request . InspectionCheckedAt ||
inspection . Outcome != hypervisor . InspectionOutcomeFailed ||
inspection . SummaryCode != hypervisor . InspectionSummaryFailed ||
inspection . ErrorCode != "creation_absent" ||
inspection . EffectResolution != hypervisor . InspectionEffectNotApplied ||
inspection . CompletionDeliveryState != "" || inspection . CompletionDeliveryErrorCode != "" ||
inspection . AbortJob . Available || inspection . Cleanup . Available ||
len ( inspection . EffectProjection ) == 0 || len ( inspection . EffectProjection ) > 1024 {
return false
}
var projection map [ string ] json . RawMessage
if err := json . Unmarshal ( inspection . EffectProjection , & projection ) ; err != nil || len ( projection ) != 2 {
return false
}
var creationAbsent , managedArtifactsAbsent bool
if err := json . Unmarshal ( projection [ "creation_absent" ] , & creationAbsent ) ; err != nil || ! creationAbsent {
return false
}
if err := json . Unmarshal ( projection [ "managed_artifacts_absent" ] , & managedArtifactsAbsent ) ; err != nil || ! managedArtifactsAbsent {
return false
}
return true
}
func ( s * Server ) releaseAsync ( operationID string ) {
s . asyncMu . Lock ( )
defer s . asyncMu . Unlock ( )
delete ( s . asyncRunning , operationID )
}
func operationWire ( record operation . Record , replayed bool ) operationResponse {
return operationResponse {
OperationID : record . OperationID , Action : record . Action , ResourceID : record . ResourceID ,
RequestFingerprint : record . Fingerprint , Status : record . State , Replayed : replayed ,
Result : record . Result , Error : record . Error , CauseCode : record . CauseCode ,
EffectMayHaveApplied : record . EffectMayHaveApplied ,
SafeToResubmit : record . SafeToResubmit ,
Retryable : false ,
}
}
// Log only bounded identifiers and stable codes. Free-form backend errors,
// request bodies, guest credentials, and host paths never enter this line.
func logOperationOutcome ( record operation . Record ) {
if record . State != operation . StateFailed {
return
}
errorCode := ""
if record . Error != nil && len ( record . Error . Code ) <= 64 &&
stableCodePattern . MatchString ( record . Error . Code ) {
errorCode = record . Error . Code
}
log . Printf ( "Agent operation outcome operation_id=%s action=%s status=%s error_code=%s cause_code=%s" ,
record . OperationID , record . Action , record . State , errorCode , record . CauseCode )
}
func resolveOperationID ( actorID , bodyID string ) ( string , error ) {
if actorID == "" && bodyID == "" {
return "" , errors . New ( "operation_id is required in X-Virt-Actor or request body" )
}
if actorID != "" && bodyID != "" && actorID != bodyID {
return "" , errors . New ( "actor and body operation_id values differ" )
}
resolved := bodyID
if actorID != "" {
resolved = actorID
}
if ! operationIDPattern . MatchString ( resolved ) {
return "" , errors . New ( "operation_id must contain 8-80 URL-safe characters" )
}
return resolved , nil
}
func operationUUID ( operationID string ) string {
digest := sha256 . Sum256 ( [ ] byte ( "virt-cloud-operation:" + operationID ) )
digest [ 6 ] = digest [ 6 ] & 0x0f | 0x50
digest [ 8 ] = digest [ 8 ] & 0x3f | 0x80
return fmt . Sprintf ( "%08x-%04x-%04x-%04x-%012x" ,
digest [ 0 : 4 ] , digest [ 4 : 6 ] , digest [ 6 : 8 ] , digest [ 8 : 10 ] , digest [ 10 : 16 ] )
}
func operationCatalogID ( operationID string ) string {
digest := sha256 . Sum256 ( [ ] byte ( "virt-cloud-catalog:" + operationID ) )
return fmt . Sprintf ( "%x" , digest [ : 16 ] )
}
func validPowerAction ( action hypervisor . PowerAction ) bool {
switch action {
case hypervisor . PowerStart , hypervisor . PowerShutdown , hypervisor . PowerReboot , hypervisor . PowerReset , hypervisor . PowerDestroy :
return true
default :
return false
}
}
func validateImportSource ( raw , token , digest string ) error {
parsed , err := url . Parse ( raw )
if err != nil || parsed . Scheme != "https" || parsed . Host == "" || parsed . User != nil || parsed . Fragment != "" {
return hypervisor . NewError ( hypervisor . ErrInvalid , "invalid_source_url" , "source_url must be an HTTPS URL without credentials or fragment" , err )
}
if digest != "" && ! sha256Pattern . MatchString ( digest ) {
return hypervisor . NewError ( hypervisor . ErrInvalid , "invalid_sha256" , "sha256 must be 64 lowercase hexadecimal characters" , nil )
}
if token != "" && ! sourceTokenPattern . MatchString ( token ) {
return hypervisor . NewError ( hypervisor . ErrInvalid , "invalid_source_token" , "source_token must contain 32-512 URL-safe characters" , nil )
}
return nil
}
func ownerFromRequest ( r * http . Request , actor auth . Actor ) ( int64 , error ) {
value := r . URL . Query ( ) . Get ( "owner_id" )
if value == "" {
return actor . UserID , nil
}
ownerID , err := strconv . ParseInt ( value , 10 , 64 )
if err != nil || ownerID <= 0 {
return 0 , errors . New ( "owner_id is invalid" )
}
if ownerID != actor . UserID && ! actor . IsAdmin && ! actor . IsSuperAdmin {
return 0 , errors . New ( "cannot access another owner's ISO" )
}
return ownerID , nil
}
func requireManage ( w http . ResponseWriter , actor auth . Actor ) bool {
if ! actor . CanManage {
writePreDispatchError ( w , http . StatusForbidden , "forbidden" , "actor is not authorized to manage this resource" )
return false
}
return true
}
func requireSuperAdminManage ( w http . ResponseWriter , actor auth . Actor ) bool {
if ! actor . IsSuperAdmin || ! actor . CanManage {
writePreDispatchError ( w , http . StatusForbidden , "forbidden" , "actor is not authorized to configure host storage" )
return false
}
return true
}
func requireSuperAdminPrivilegedOperation ( w http . ResponseWriter , actor auth . Actor ) bool {
if ! actor . IsSuperAdmin || ! actor . CanManage {
writePreDispatchError ( w , http . StatusForbidden , "forbidden" , "actor is not authorized to perform this privileged operation" )
return false
}
return true
}
func requireSuperAdminInspection ( w http . ResponseWriter , actor auth . Actor , operationID string ) bool {
if actor . SystemRecovery && actor . UserID == 0 && actor . OperationID == operationID {
return true
}
if actor . UserID < 1 || ! actor . IsSuperAdmin || ! actor . CanManage {
writePreDispatchError ( w , http . StatusForbidden , "forbidden" , "actor is not authorized to inspect operation state" )
return false
}
return true
}
func requireSuperAdminFenceResolution ( w http . ResponseWriter , actor auth . Actor , operationID string ) bool {
if actor . SystemRecovery && actor . UserID == 0 && actor . OperationID == operationID {
return true
}
if actor . UserID < 1 || ! actor . IsSuperAdmin || ! actor . CanManage {
writePreDispatchError ( w , http . StatusForbidden , "forbidden" , "actor is not authorized to resolve operation effect fences" )
return false
}
return true
}
// Route shape and operation identity are necessary but not sufficient for
// system recovery: authorization is completed only after reading the
// immutable Journal action and applying the explicit Agent action allowlist.
func requireSystemRecoveryRecord ( w http . ResponseWriter , actor auth . Actor , record operation . Record ) bool {
if actor . SystemRecovery && ! operation . SystemRecoveryActionAllowed ( record . Action ) {
writePreDispatchError ( w , http . StatusForbidden , "forbidden" , "system recovery is not enabled for this operation kind" )
return false
}
return true
}
// systemRecoveryRequestAllowed is a deny-by-default capability boundary. The
// signed system actor is bound to one operation and cannot reach health,
// inventory, raw Journal records, catalog reads, or any mutation route.
func systemRecoveryRequestAllowed ( actor auth . Actor , method string , segments [ ] string ) bool {
if ! actor . SystemRecovery || actor . UserID != 0 || actor . OperationID == "" ||
len ( segments ) != 4 || segments [ 0 ] != "v1" || segments [ 1 ] != "operations" ||
segments [ 2 ] != actor . OperationID {
return false
}
switch segments [ 3 ] {
case "inspect" , "fence-resolution" :
return method == http . MethodGet
case "resolve-fence" :
return method == http . MethodPost
default :
return false
}
}
func decodeRequired [ T any ] ( w http . ResponseWriter , body [ ] byte ) ( T , bool ) {
var value T
if len ( body ) == 0 {
writePreDispatchError ( w , http . StatusBadRequest , "invalid_json" , "JSON request body is required" )
return value , false
}
if err := decodeStrict ( body , & value ) ; err != nil {
writePreDispatchError ( w , http . StatusBadRequest , "invalid_json" , err . Error ( ) )
return value , false
}
return value , true
}
func decodeOptional [ T any ] ( w http . ResponseWriter , body [ ] byte ) ( T , bool ) {
var value T
if len ( body ) == 0 {
return value , true
}
if err := decodeStrict ( body , & value ) ; err != nil {
writePreDispatchError ( w , http . StatusBadRequest , "invalid_json" , err . Error ( ) )
return value , false
}
return value , true
}
func decodeStrict ( body [ ] byte , target any ) error {
if ! utf8 . Valid ( body ) {
return errors . New ( "invalid JSON: body must be valid UTF-8" )
}
decoder := json . NewDecoder ( bytes . NewReader ( body ) )
decoder . DisallowUnknownFields ( )
if err := decoder . Decode ( target ) ; err != nil {
return fmt . Errorf ( "invalid JSON: %w" , err )
}
var extra any
if err := decoder . Decode ( & extra ) ; err != io . EOF {
return errors . New ( "invalid JSON: exactly one object is required" )
}
return nil
}
func readBoundedBody ( w http . ResponseWriter , r * http . Request , limit int64 ) ( [ ] byte , error ) {
if r . Body == nil {
return [ ] byte { } , nil
}
r . Body = http . MaxBytesReader ( w , r . Body , limit )
defer r . Body . Close ( )
return io . ReadAll ( r . Body )
}
func writeResult ( w http . ResponseWriter , value any , err error ) {
if err != nil {
writeMappedError ( w , err , nil )
return
}
writeJSON ( w , http . StatusOK , value )
}
func writeNamedList [ T any ] ( w http . ResponseWriter , name string , value [ ] T , err error ) {
if err != nil {
writeMappedError ( w , err , nil )
return
}
if value == nil {
value = make ( [ ] T , 0 )
}
writeJSON ( w , http . StatusOK , map [ string ] any { name : value } )
}
func writeMappedError ( w http . ResponseWriter , err error , record * operation . Record ) {
status , code , message := classifyError ( err )
writeError ( w , status , code , message , record )
}
// writePreDispatchMappedError is only for validation or dependency checks
// completed before Journal.Execute/Reserve. The explicit marker lets the
// panel distinguish a proven no-journal outcome from a lost response after a
// mutation may already have crossed its durable dispatch boundary.
func writePreDispatchMappedError ( w http . ResponseWriter , err error ) {
status , code , message := classifyError ( err )
writePreDispatchError ( w , status , code , message )
}
func publicErrorCode ( err error , fallback string ) string {
var public interface { PublicCode ( ) string }
if errors . As ( err , & public ) && stableCodePattern . MatchString ( public . PublicCode ( ) ) {
return public . PublicCode ( )
}
return fallback
}
func publicErrorMessage ( err error , fallback string ) string {
var public interface { PublicMessage ( ) string }
if errors . As ( err , & public ) && public . PublicMessage ( ) != "" {
return public . PublicMessage ( )
}
return fallback
}
// writeOperationStarting reports the active admission owner without exposing a
// stale failed/not-dispatched record from before that owner acquired admission.
// The projected reserved state is diagnostic only; it never authorizes a
// caller to dispatch the mutation again. The journal remains authoritative.
func writeOperationStarting ( w http . ResponseWriter , record operation . Record ) {
record . State = operation . StateReserved
record . Result = nil
record . Error = nil
record . CauseCode = ""
record . Tombstone = false
writeError ( w , http . StatusServiceUnavailable , "operation_starting" , "operation dispatch is already starting" , & record )
}
func classifyError ( err error ) ( int , string , string ) {
if errors . Is ( err , operation . ErrNotFound ) {
return http . StatusNotFound , "operation_not_found" , "operation was not found"
}
if errors . Is ( err , operation . ErrBusy ) {
return http . StatusServiceUnavailable , "agent_busy" , "Agent mutation capacity is exhausted"
}
if errors . Is ( err , operation . ErrOperationStarting ) {
return http . StatusServiceUnavailable , "operation_starting" , "operation dispatch is already starting"
}
if errors . Is ( err , operation . ErrResourceBlocked ) {
return http . StatusConflict , "resource_operation_blocked" , "resource is fenced by a prior failed operation whose effect may have applied"
}
if errors . Is ( err , operation . ErrInvalidSupersession ) {
return http . StatusConflict , "force_delete_not_authorized" , "force delete requires the matching graceful operation"
}
var backend * hypervisor . Error
if errors . As ( err , & backend ) {
status := statusForKind ( backend . Kind )
code := backend . Code
if code == "" {
code = "backend_error"
}
message := backend . Message
if message == "" {
message = http . StatusText ( status )
}
return status , code , message
}
switch {
case errors . Is ( err , hypervisor . ErrNotFound ) :
return http . StatusNotFound , "not_found" , "resource was not found"
case errors . Is ( err , hypervisor . ErrConflict ) :
return http . StatusConflict , "conflict" , "resource conflict"
case errors . Is ( err , hypervisor . ErrUnsupported ) :
return http . StatusNotImplemented , "unsupported" , "operation is not supported"
case errors . Is ( err , hypervisor . ErrInvalid ) :
return http . StatusUnprocessableEntity , "invalid_request" , "request is invalid"
case errors . Is ( err , context . DeadlineExceeded ) , errors . Is ( err , context . Canceled ) :
return http . StatusServiceUnavailable , "backend_timeout" , "backend operation did not complete"
default :
return http . StatusServiceUnavailable , "backend_error" , "backend operation failed"
}
}
func statusForKind ( kind error ) int {
switch kind {
case hypervisor . ErrNotFound :
return http . StatusNotFound
case hypervisor . ErrConflict :
return http . StatusConflict
case hypervisor . ErrUnsupported :
return http . StatusNotImplemented
case hypervisor . ErrInvalid :
return http . StatusUnprocessableEntity
default :
return http . StatusServiceUnavailable
}
}
func writeOperationFailure ( w http . ResponseWriter , record operation . Record , replayed bool ) {
status := http . StatusServiceUnavailable
if record . Error != nil {
switch record . Error . Code {
case "unsupported" :
status = http . StatusNotImplemented
case "not_found" :
status = http . StatusNotFound
case "conflict" :
status = http . StatusConflict
case "force_required" , "force_delete_not_authorized" :
status = http . StatusConflict
default :
if strings . HasSuffix ( record . Error . Code , "_not_found" ) {
status = http . StatusNotFound
} else if strings . HasPrefix ( record . Error . Code , "invalid_" ) {
status = http . StatusUnprocessableEntity
}
}
}
writeJSON ( w , status , operationWire ( record , replayed ) )
}
func methodOrNotFound ( w http . ResponseWriter , r * http . Request , exists bool , methods ... string ) {
if ! exists {
writePreDispatchError ( w , http . StatusNotFound , "not_found" , "endpoint not found" )
return
}
w . Header ( ) . Set ( "Allow" , strings . Join ( methods , ", " ) )
writePreDispatchError ( w , http . StatusMethodNotAllowed , "method_not_allowed" , "method is not allowed" )
}
func writePreDispatchError ( w http . ResponseWriter , status int , code , message string ) {
writeJSON ( w , status , errorEnvelope { Error : errorBody {
Code : code , Message : message , Retryable : retryableError ( status , code ) ,
DispatchState : "not_dispatched" ,
} } )
}
func writeError ( w http . ResponseWriter , status int , code , message string , record * operation . Record ) {
writeJSON ( w , status , errorEnvelope { Error : errorBody {
Code : code , Message : message , Retryable : retryableError ( status , code ) , Operation : record ,
} } )
}
func retryableError ( status int , code string ) bool {
return false
}
func writeJSON ( w http . ResponseWriter , status int , value any ) {
w . Header ( ) . Set ( "Content-Type" , "application/json; charset=utf-8" )
w . WriteHeader ( status )
_ = json . NewEncoder ( w ) . Encode ( value )
}
func setResponseHeaders ( w http . ResponseWriter ) {
w . Header ( ) . Set ( "Cache-Control" , "no-store" )
w . Header ( ) . Set ( "X-Content-Type-Options" , "nosniff" )
}