From 371110d9fc412ff546891da99bc58179aa02979b Mon Sep 17 00:00:00 2001 From: codex Date: Wed, 21 Oct 2026 15:57:37 +0800 Subject: [PATCH] refactor(cloud): unify recovery commits and operation contracts Share applied/not_applied preparation and atomic publication while preserving direction-specific evidence checks. Define recovery capabilities once, embed them in the Agent, and generate the PHP inventory with drift and full-kind behavior contracts. Permit only bounded public template generation IDs in inspection projections. --- agent/operation/journal.go | 99 +- agent/operation/recovery_contract.go | 288 ++++ agent/operation/recovery_contract.json | 368 ++++ agent/operation/recovery_contract_test.go | 179 ++ docs/TASK_EXECUTION.md | 4 + docs/VALIDATION.md | 14 + tools/generate-recovery-contract.php | 146 ++ .../CloudAdminOperationRecoveryService.php | 27 +- ...CloudAutomaticOperationRecoveryService.php | 2 +- .../Cloud/CloudOperationRecoveryContract.php | 221 +++ .../Cloud/CloudOperationRecoveryIdentity.php | 22 +- .../Cloud/CloudOperationRecoveryPolicy.php | 77 +- .../Cloud/CloudOperationRecoveryService.php | 249 ++- .../CloudCatalogLockOrderContractTest.php | 79 +- .../CloudOperationRecoveryContractTest.php | 98 ++ .../CloudOperationRecoveryMatrixTest.php | 1529 +++++++++++++++++ .../CloudSingleDispatchArchitectureTest.php | 4 +- 17 files changed, 3149 insertions(+), 257 deletions(-) create mode 100644 agent/operation/recovery_contract.go create mode 100644 agent/operation/recovery_contract.json create mode 100644 agent/operation/recovery_contract_test.go create mode 100644 tools/generate-recovery-contract.php create mode 100644 web/app/Services/Cloud/CloudOperationRecoveryContract.php create mode 100644 web/tests/CloudOperationRecoveryContractTest.php create mode 100644 web/tests/CloudOperationRecoveryMatrixTest.php diff --git a/agent/operation/journal.go b/agent/operation/journal.go index fc82b7bb..da17886d 100644 --- a/agent/operation/journal.go +++ b/agent/operation/journal.go @@ -42,36 +42,32 @@ const ( ) var ( - ErrNotFound = errors.New("operation not found") - ErrFingerprint = errors.New("operation_id payload conflict") - ErrOperationStarting = errors.New("operation dispatch is already starting") - ErrBusy = errors.New("operation execution capacity is exhausted") - ErrResourceBlocked = errors.New("resource has an unresolved operation") - ErrInvalidSupersession = errors.New("operation cannot supersede the requested predecessor") - ErrInspectionInconclusive = errors.New("read-only inspection did not confirm a terminal outcome") - ErrExecutionNotReaped = errors.New("isolated mutation process did not exit after forced termination") - ErrFenceNotResolvable = errors.New("operation does not hold an effect fence") - ErrFenceIdentityConflict = errors.New("fence resolution identity does not match operation") - ErrFenceResolutionConflict = errors.New("operation effect fence already has a different resolution") - ErrFenceInspectionStale = errors.New("operation effect fence resolution requires a fresh inspection") - ErrTerminalEvidencePending = errors.New("worker journal has no terminal operation evidence") - ErrTerminalEvidenceInvalid = errors.New("worker journal terminal evidence is invalid") - operationIDPattern = regexp.MustCompile(`^[A-Za-z0-9_-]{8,80}$`) - operationActionPattern = regexp.MustCompile(`^[a-z][a-z0-9]*(?:[-:][a-z0-9]+)*$`) - operationResourceIDPattern = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9_.:-]*$`) - hexDigestPattern = regexp.MustCompile(`^[0-9a-f]{64}$`) - noncePattern = regexp.MustCompile(`^[A-Za-z0-9_-]{16,128}$`) - networkSourcePattern = regexp.MustCompile(`^[A-Za-z0-9_.:-]{1,128}$`) - causeCodePattern = regexp.MustCompile(`^[a-z][a-z0-9]*(_[a-z0-9]+)*$`) - canonicalUUIDPattern = regexp.MustCompile(`^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$`) - resourcePartPattern = regexp.MustCompile(`^[A-Za-z0-9_.:-]{1,192}$`) - resourceKindPattern = regexp.MustCompile(`^[a-z][a-z0-9_-]{0,63}$`) - fenceResolutionIDPattern = regexp.MustCompile(`^[a-f0-9]{32}$`) - completionSensitiveKeyPattern = regexp.MustCompile(`(?i)password|secret|(?:public|private).?key|ssh.?key|passphrase|credential|(?:(?:access|refresh|auth|api|bearer|session|source).?token)|^token$`) - snapshotRecoveryActionPattern = regexp.MustCompile( - `^(?:create-snapshot:[a-f0-9]{32}:(?:filesystem|crash)|(?:delete|restore)-snapshot:[a-f0-9]{32})$`, - ) - + ErrNotFound = errors.New("operation not found") + ErrFingerprint = errors.New("operation_id payload conflict") + ErrOperationStarting = errors.New("operation dispatch is already starting") + ErrBusy = errors.New("operation execution capacity is exhausted") + ErrResourceBlocked = errors.New("resource has an unresolved operation") + ErrInvalidSupersession = errors.New("operation cannot supersede the requested predecessor") + ErrInspectionInconclusive = errors.New("read-only inspection did not confirm a terminal outcome") + ErrExecutionNotReaped = errors.New("isolated mutation process did not exit after forced termination") + ErrFenceNotResolvable = errors.New("operation does not hold an effect fence") + ErrFenceIdentityConflict = errors.New("fence resolution identity does not match operation") + ErrFenceResolutionConflict = errors.New("operation effect fence already has a different resolution") + ErrFenceInspectionStale = errors.New("operation effect fence resolution requires a fresh inspection") + ErrTerminalEvidencePending = errors.New("worker journal has no terminal operation evidence") + ErrTerminalEvidenceInvalid = errors.New("worker journal terminal evidence is invalid") + operationIDPattern = regexp.MustCompile(`^[A-Za-z0-9_-]{8,80}$`) + operationActionPattern = regexp.MustCompile(`^[a-z][a-z0-9]*(?:[-:][a-z0-9]+)*$`) + operationResourceIDPattern = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9_.:-]*$`) + hexDigestPattern = regexp.MustCompile(`^[0-9a-f]{64}$`) + noncePattern = regexp.MustCompile(`^[A-Za-z0-9_-]{16,128}$`) + networkSourcePattern = regexp.MustCompile(`^[A-Za-z0-9_.:-]{1,128}$`) + causeCodePattern = regexp.MustCompile(`^[a-z][a-z0-9]*(_[a-z0-9]+)*$`) + canonicalUUIDPattern = regexp.MustCompile(`^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$`) + resourcePartPattern = regexp.MustCompile(`^[A-Za-z0-9_.:-]{1,192}$`) + resourceKindPattern = regexp.MustCompile(`^[a-z][a-z0-9_-]{0,63}$`) + fenceResolutionIDPattern = regexp.MustCompile(`^[a-f0-9]{32}$`) + completionSensitiveKeyPattern = regexp.MustCompile(`(?i)password|secret|(?:public|private).?key|ssh.?key|passphrase|credential|(?:(?:access|refresh|auth|api|bearer|session|source).?token)|^token$`) metadataBucket = []byte("metadata") operationsBucket = []byte("operations") inflightBucket = []byte("inflight") @@ -176,29 +172,6 @@ type Spec struct { SupersedesOperationID string } -// SystemRecoveryActionAllowed is the deny-by-default capability boundary for -// the panel's read-only inspector and Journal-only fence resolver. Adding a -// new mutation route does not implicitly grant system recovery access: every -// leaf action must be reviewed and added here explicitly. -func SystemRecoveryActionAllowed(action string) bool { - switch action { - case "create-vm", - "power-start", "power-shutdown", "power-reboot", "power-reset", "power-destroy", - "reinstall-vm", - "enter-rescue", "force-start-rescue", "exit-rescue:connected", "exit-rescue:suspended", - "change-network", "network-suspend", "network-resume", - "update-settings", "apply-firewall", - "delete-vm", "force-delete-vm", - "service-suspend", - "import-template", "update-template", "delete-template", - "import-rescue-template", "delete-rescue-template", - "import-user-iso", "delete-user-iso": - return true - default: - return snapshotRecoveryActionPattern.MatchString(action) - } -} - // ValidOperationIdentity validates the immutable non-secret identity copied // from a Journal record into inspections and completion deliveries. func ValidOperationIdentity(action, resourceID, requestFingerprint string) bool { @@ -873,20 +846,16 @@ func (j *Journal) ResolveResourceFence(operationID string, resolvedBy int64, return ErrFenceIdentityConflict } // The zero audit actor is a deliberately narrow internal capability. - // Only explicitly reviewed automatic recovery actions may use it; - // human administrators retain their positive audited identity. - if resolvedBy == 0 && !SystemRecoveryActionAllowed(record.Action) { + // Only an action/resolution pair explicitly granted by the embedded + // contract may use it; human administrators retain their positive + // audited identity. + if resolvedBy == 0 && !SystemRecoveryResolutionAllowed(record.Action, request.Resolution) { return ErrFenceNotResolvable } - // Automatic create recovery is deliberately one-way: fresh host - // absence evidence may release a failed create fence as not applied, - // but the zero-ID system actor can never declare that a create applied. - // Nor may it release a fence while an unreaped worker could still create - // the resource after the inspection. Materializing an applied create - // requires the original immutable completion result and remains PHP's - // responsibility. - if resolvedBy == 0 && record.Action == "create-vm" && - (record.ExecutionFenced || request.Resolution != FenceResolutionNotApplied) { + // An unreaped create worker could still create the resource after the + // inspection. Its fence therefore remains active even though the + // contract permits system not_applied for a safely reaped create. + if resolvedBy == 0 && record.Action == "create-vm" && record.ExecutionFenced { return ErrFenceNotResolvable } if encoded := resolutions.Get([]byte(operationID)); encoded != nil { diff --git a/agent/operation/recovery_contract.go b/agent/operation/recovery_contract.go new file mode 100644 index 00000000..872eba7b --- /dev/null +++ b/agent/operation/recovery_contract.go @@ -0,0 +1,288 @@ +package operation + +import ( + "bytes" + _ "embed" + "encoding/json" + "errors" + "fmt" + "io" + "regexp" +) + +const recoveryContractSchemaVersion = 1 + +var recoveryKindPattern = regexp.MustCompile(`^[a-z][a-z0-9_]{0,63}$`) + +//go:embed recovery_contract.json +var embeddedRecoveryContractJSON []byte + +var embeddedRecoveryContract = mustParseRecoveryContract(embeddedRecoveryContractJSON) + +type recoveryContractDocument struct { + SchemaVersion int `json:"schema_version"` + Operations []recoveryContractOperation `json:"operations"` +} + +type recoveryContractOperation struct { + Kind string `json:"kind"` + RecoveryMode string `json:"recovery_mode"` + IdentityScope string `json:"identity_scope"` + Actions []string `json:"actions"` + ActionPatterns []string `json:"action_patterns"` + SystemResolutions []string `json:"system_resolutions"` +} + +type recoveryResolutionSet map[string]struct{} + +type recoveryActionPattern struct { + source string + pattern *regexp.Regexp + resolutions recoveryResolutionSet +} + +type recoveryContract struct { + actions map[string]recoveryResolutionSet + patterns []recoveryActionPattern +} + +// SystemRecoveryActionAllowed is the deny-by-default capability boundary for +// the panel's read-only inspector. A manifest entry without a permitted system +// resolution (for example host storage coordination) does not grant access. +func SystemRecoveryActionAllowed(action string) bool { + return embeddedRecoveryContract.actionAllowed(action) +} + +// SystemRecoveryResolutionAllowed binds an automatic fence decision to the +// exact immutable Agent action. This prevents a manifest entry that only +// permits not_applied from inheriting the generic applied capability. +func SystemRecoveryResolutionAllowed(action, resolution string) bool { + return embeddedRecoveryContract.resolutionAllowed(action, resolution) +} + +func (contract recoveryContract) actionAllowed(action string) bool { + if len(action) > 128 || !operationActionPattern.MatchString(action) { + return false + } + if resolutions, ok := contract.actions[action]; ok && len(resolutions) > 0 { + return true + } + for _, candidate := range contract.patterns { + if len(candidate.resolutions) > 0 && candidate.pattern.MatchString(action) { + return true + } + } + return false +} + +func (contract recoveryContract) resolutionAllowed(action, resolution string) bool { + if len(action) > 128 || !operationActionPattern.MatchString(action) { + return false + } + if resolutions, ok := contract.actions[action]; ok { + _, allowed := resolutions[resolution] + return allowed + } + for _, candidate := range contract.patterns { + if candidate.pattern.MatchString(action) { + _, allowed := candidate.resolutions[resolution] + return allowed + } + } + return false +} + +func mustParseRecoveryContract(document []byte) recoveryContract { + contract, err := parseRecoveryContract(document) + if err != nil { + panic(fmt.Sprintf("invalid embedded recovery contract: %v", err)) + } + return contract +} + +func parseRecoveryContract(document []byte) (recoveryContract, error) { + decoder := json.NewDecoder(bytes.NewReader(document)) + decoder.DisallowUnknownFields() + + var decoded recoveryContractDocument + if err := decoder.Decode(&decoded); err != nil { + return recoveryContract{}, fmt.Errorf("decode recovery contract: %w", err) + } + var trailing json.RawMessage + if err := decoder.Decode(&trailing); !errors.Is(err, io.EOF) { + if err == nil { + return recoveryContract{}, errors.New("recovery contract contains a trailing JSON value") + } + return recoveryContract{}, fmt.Errorf("decode recovery contract trailer: %w", err) + } + if decoded.SchemaVersion != recoveryContractSchemaVersion { + return recoveryContract{}, fmt.Errorf("unsupported recovery contract schema version %d", decoded.SchemaVersion) + } + if len(decoded.Operations) == 0 { + return recoveryContract{}, errors.New("recovery contract operations must not be empty") + } + + contract := recoveryContract{actions: make(map[string]recoveryResolutionSet)} + seenKinds := make(map[string]struct{}, len(decoded.Operations)) + seenPatterns := make(map[string]recoveryResolutionSet) + var patternedResolutions recoveryResolutionSet + for index, operation := range decoded.Operations { + if err := validateRecoveryContractOperation(operation); err != nil { + return recoveryContract{}, fmt.Errorf("recovery contract operation %d: %w", index, err) + } + if _, exists := seenKinds[operation.Kind]; exists { + return recoveryContract{}, fmt.Errorf("recovery contract kind %q is duplicated", operation.Kind) + } + seenKinds[operation.Kind] = struct{}{} + + resolutions := recoveryResolutionSetFrom(operation.SystemResolutions) + for _, action := range operation.Actions { + if existing, exists := contract.actions[action]; exists { + if !sameRecoveryResolutionSet(existing, resolutions) { + return recoveryContract{}, fmt.Errorf("action %q has conflicting system resolutions", action) + } + continue + } + contract.actions[action] = resolutions + } + for _, source := range operation.ActionPatterns { + if existing, exists := seenPatterns[source]; exists { + if !sameRecoveryResolutionSet(existing, resolutions) { + return recoveryContract{}, fmt.Errorf("action pattern %q has conflicting system resolutions", source) + } + continue + } + // This loader intentionally does not analyze regex intersections. + // Requiring every pattern to carry one identical grant prevents + // overlaps from unioning different system-recovery capabilities. + if patternedResolutions != nil && !sameRecoveryResolutionSet(patternedResolutions, resolutions) { + return recoveryContract{}, errors.New("action patterns have conflicting system resolutions") + } + compiled, err := regexp.Compile(source) + if err != nil { + return recoveryContract{}, fmt.Errorf("compile action pattern %q: %w", source, err) + } + patternedResolutions = resolutions + seenPatterns[source] = resolutions + contract.patterns = append(contract.patterns, recoveryActionPattern{ + source: source, pattern: compiled, resolutions: resolutions, + }) + } + } + + // No exact action may silently acquire a second, different capability from + // a pattern. The reviewed patterns are currently disjoint, but this check + // keeps the invariant explicit if the manifest grammar is extended later. + for action, exactResolutions := range contract.actions { + for _, candidate := range contract.patterns { + if candidate.pattern.MatchString(action) && + !sameRecoveryResolutionSet(exactResolutions, candidate.resolutions) { + return recoveryContract{}, fmt.Errorf("action %q matches pattern %q with conflicting system resolutions", action, candidate.source) + } + } + } + + return contract, nil +} + +func validateRecoveryContractOperation(operation recoveryContractOperation) error { + if !recoveryKindPattern.MatchString(operation.Kind) { + return fmt.Errorf("invalid kind %q", operation.Kind) + } + if operation.Actions == nil || operation.ActionPatterns == nil || operation.SystemResolutions == nil { + return errors.New("actions, action_patterns, and system_resolutions must all be present") + } + + allowedScopes := map[string]bool{} + expectedResolutions := []string(nil) + requiresActions := false + switch operation.RecoveryMode { + case "generic": + allowedScopes["instance"] = true + allowedScopes["catalog"] = true + expectedResolutions = []string{FenceResolutionApplied, FenceResolutionNotApplied} + requiresActions = true + case "create": + allowedScopes["create"] = true + expectedResolutions = []string{FenceResolutionNotApplied} + requiresActions = true + case "storage": + allowedScopes["storage"] = true + expectedResolutions = []string{} + requiresActions = true + case "aggregate", "payment": + allowedScopes["none"] = true + expectedResolutions = []string{} + default: + return fmt.Errorf("unsupported recovery_mode %q", operation.RecoveryMode) + } + if !allowedScopes[operation.IdentityScope] { + return fmt.Errorf("identity_scope %q is invalid for recovery_mode %q", operation.IdentityScope, operation.RecoveryMode) + } + if requiresActions == (len(operation.Actions)+len(operation.ActionPatterns) == 0) { + if requiresActions { + return fmt.Errorf("recovery_mode %q requires an action or action pattern", operation.RecoveryMode) + } + return fmt.Errorf("recovery_mode %q must not define actions or action patterns", operation.RecoveryMode) + } + if !sameStringSlice(operation.SystemResolutions, expectedResolutions) { + return fmt.Errorf("system_resolutions do not match recovery_mode %q", operation.RecoveryMode) + } + + seenActions := make(map[string]struct{}, len(operation.Actions)) + for _, action := range operation.Actions { + if len(action) > 128 || !operationActionPattern.MatchString(action) { + return fmt.Errorf("invalid action %q", action) + } + if _, exists := seenActions[action]; exists { + return fmt.Errorf("action %q is duplicated", action) + } + seenActions[action] = struct{}{} + } + seenPatterns := make(map[string]struct{}, len(operation.ActionPatterns)) + for _, source := range operation.ActionPatterns { + if len(source) < 2 || source[0] != '^' || source[len(source)-1] != '$' { + return fmt.Errorf("action pattern %q must be anchored", source) + } + if _, err := regexp.Compile(source); err != nil { + return fmt.Errorf("invalid action pattern %q: %w", source, err) + } + if _, exists := seenPatterns[source]; exists { + return fmt.Errorf("action pattern %q is duplicated", source) + } + seenPatterns[source] = struct{}{} + } + return nil +} + +func recoveryResolutionSetFrom(resolutions []string) recoveryResolutionSet { + result := make(recoveryResolutionSet, len(resolutions)) + for _, resolution := range resolutions { + result[resolution] = struct{}{} + } + return result +} + +func sameRecoveryResolutionSet(left, right recoveryResolutionSet) bool { + if len(left) != len(right) { + return false + } + for resolution := range left { + if _, exists := right[resolution]; !exists { + return false + } + } + return true +} + +func sameStringSlice(left, right []string) bool { + if len(left) != len(right) { + return false + } + for index := range left { + if left[index] != right[index] { + return false + } + } + return true +} diff --git a/agent/operation/recovery_contract.json b/agent/operation/recovery_contract.json new file mode 100644 index 00000000..8a85d260 --- /dev/null +++ b/agent/operation/recovery_contract.json @@ -0,0 +1,368 @@ +{ + "schema_version": 1, + "operations": [ + { + "kind": "action", + "recovery_mode": "generic", + "identity_scope": "instance", + "actions": [ + "power-start", + "power-shutdown", + "power-reboot", + "power-reset", + "power-destroy" + ], + "action_patterns": [], + "system_resolutions": [ + "applied", + "not_applied" + ] + }, + { + "kind": "reinstall", + "recovery_mode": "generic", + "identity_scope": "instance", + "actions": [ + "reinstall-vm" + ], + "action_patterns": [], + "system_resolutions": [ + "applied", + "not_applied" + ] + }, + { + "kind": "rescue", + "recovery_mode": "generic", + "identity_scope": "instance", + "actions": [ + "enter-rescue", + "force-start-rescue", + "exit-rescue:connected", + "exit-rescue:suspended" + ], + "action_patterns": [], + "system_resolutions": [ + "applied", + "not_applied" + ] + }, + { + "kind": "delete", + "recovery_mode": "generic", + "identity_scope": "instance", + "actions": [ + "delete-vm" + ], + "action_patterns": [], + "system_resolutions": [ + "applied", + "not_applied" + ] + }, + { + "kind": "delete_force", + "recovery_mode": "generic", + "identity_scope": "instance", + "actions": [ + "force-delete-vm" + ], + "action_patterns": [], + "system_resolutions": [ + "applied", + "not_applied" + ] + }, + { + "kind": "expiry_delete", + "recovery_mode": "generic", + "identity_scope": "instance", + "actions": [ + "delete-vm" + ], + "action_patterns": [], + "system_resolutions": [ + "applied", + "not_applied" + ] + }, + { + "kind": "network_change", + "recovery_mode": "generic", + "identity_scope": "instance", + "actions": [ + "change-network" + ], + "action_patterns": [], + "system_resolutions": [ + "applied", + "not_applied" + ] + }, + { + "kind": "instance_settings", + "recovery_mode": "generic", + "identity_scope": "instance", + "actions": [ + "update-settings" + ], + "action_patterns": [], + "system_resolutions": [ + "applied", + "not_applied" + ] + }, + { + "kind": "snapshot_create", + "recovery_mode": "generic", + "identity_scope": "instance", + "actions": [], + "action_patterns": [ + "^create-snapshot:[a-f0-9]{32}:(?:filesystem|crash)$" + ], + "system_resolutions": [ + "applied", + "not_applied" + ] + }, + { + "kind": "snapshot_restore", + "recovery_mode": "generic", + "identity_scope": "instance", + "actions": [], + "action_patterns": [ + "^restore-snapshot:[a-f0-9]{32}$" + ], + "system_resolutions": [ + "applied", + "not_applied" + ] + }, + { + "kind": "snapshot_delete", + "recovery_mode": "generic", + "identity_scope": "instance", + "actions": [], + "action_patterns": [ + "^delete-snapshot:[a-f0-9]{32}$" + ], + "system_resolutions": [ + "applied", + "not_applied" + ] + }, + { + "kind": "firewall_apply", + "recovery_mode": "generic", + "identity_scope": "instance", + "actions": [ + "apply-firewall" + ], + "action_patterns": [], + "system_resolutions": [ + "applied", + "not_applied" + ] + }, + { + "kind": "service_suspend", + "recovery_mode": "generic", + "identity_scope": "instance", + "actions": [ + "service-suspend" + ], + "action_patterns": [], + "system_resolutions": [ + "applied", + "not_applied" + ] + }, + { + "kind": "network_suspend", + "recovery_mode": "generic", + "identity_scope": "instance", + "actions": [ + "network-suspend" + ], + "action_patterns": [], + "system_resolutions": [ + "applied", + "not_applied" + ] + }, + { + "kind": "network_resume", + "recovery_mode": "generic", + "identity_scope": "instance", + "actions": [ + "network-resume" + ], + "action_patterns": [], + "system_resolutions": [ + "applied", + "not_applied" + ] + }, + { + "kind": "template_import_replica", + "recovery_mode": "generic", + "identity_scope": "catalog", + "actions": [ + "import-template" + ], + "action_patterns": [], + "system_resolutions": [ + "applied", + "not_applied" + ] + }, + { + "kind": "template_metadata_replica", + "recovery_mode": "generic", + "identity_scope": "catalog", + "actions": [ + "update-template" + ], + "action_patterns": [], + "system_resolutions": [ + "applied", + "not_applied" + ] + }, + { + "kind": "template_delete_replica", + "recovery_mode": "generic", + "identity_scope": "catalog", + "actions": [ + "delete-template" + ], + "action_patterns": [], + "system_resolutions": [ + "applied", + "not_applied" + ] + }, + { + "kind": "rescue_template_import", + "recovery_mode": "generic", + "identity_scope": "catalog", + "actions": [ + "import-rescue-template" + ], + "action_patterns": [], + "system_resolutions": [ + "applied", + "not_applied" + ] + }, + { + "kind": "rescue_template_delete", + "recovery_mode": "generic", + "identity_scope": "catalog", + "actions": [ + "delete-rescue-template" + ], + "action_patterns": [], + "system_resolutions": [ + "applied", + "not_applied" + ] + }, + { + "kind": "user_iso_import_replica", + "recovery_mode": "generic", + "identity_scope": "catalog", + "actions": [ + "import-user-iso" + ], + "action_patterns": [], + "system_resolutions": [ + "applied", + "not_applied" + ] + }, + { + "kind": "user_iso_delete_replica", + "recovery_mode": "generic", + "identity_scope": "catalog", + "actions": [ + "delete-user-iso" + ], + "action_patterns": [], + "system_resolutions": [ + "applied", + "not_applied" + ] + }, + { + "kind": "create", + "recovery_mode": "create", + "identity_scope": "create", + "actions": [ + "create-vm" + ], + "action_patterns": [], + "system_resolutions": [ + "not_applied" + ] + }, + { + "kind": "host_storage_configure", + "recovery_mode": "storage", + "identity_scope": "storage", + "actions": [ + "configure-storage" + ], + "action_patterns": [], + "system_resolutions": [] + }, + { + "kind": "payment_create", + "recovery_mode": "payment", + "identity_scope": "none", + "actions": [], + "action_patterns": [], + "system_resolutions": [] + }, + { + "kind": "template_import", + "recovery_mode": "aggregate", + "identity_scope": "none", + "actions": [], + "action_patterns": [], + "system_resolutions": [] + }, + { + "kind": "template_metadata_sync", + "recovery_mode": "aggregate", + "identity_scope": "none", + "actions": [], + "action_patterns": [], + "system_resolutions": [] + }, + { + "kind": "template_delete", + "recovery_mode": "aggregate", + "identity_scope": "none", + "actions": [], + "action_patterns": [], + "system_resolutions": [] + }, + { + "kind": "user_iso_import", + "recovery_mode": "aggregate", + "identity_scope": "none", + "actions": [], + "action_patterns": [], + "system_resolutions": [] + }, + { + "kind": "user_iso_delete", + "recovery_mode": "aggregate", + "identity_scope": "none", + "actions": [], + "action_patterns": [], + "system_resolutions": [] + } + ] +} diff --git a/agent/operation/recovery_contract_test.go b/agent/operation/recovery_contract_test.go new file mode 100644 index 00000000..6a805f5f --- /dev/null +++ b/agent/operation/recovery_contract_test.go @@ -0,0 +1,179 @@ +package operation + +import ( + "strings" + "testing" +) + +func TestEmbeddedRecoveryContractControlsSystemResolutions(t *testing.T) { + tests := []struct { + name string + action string + resolution string + want bool + }{ + {name: "generic applied", action: "power-start", resolution: FenceResolutionApplied, want: true}, + {name: "generic not applied", action: "power-start", resolution: FenceResolutionNotApplied, want: true}, + {name: "parameterized snapshot", action: "create-snapshot:0123456789abcdef0123456789abcdef:filesystem", resolution: FenceResolutionApplied, want: true}, + {name: "create not applied", action: "create-vm", resolution: FenceResolutionNotApplied, want: true}, + {name: "create applied", action: "create-vm", resolution: FenceResolutionApplied, want: false}, + {name: "storage excluded", action: "configure-storage", resolution: FenceResolutionNotApplied, want: false}, + {name: "unknown action", action: "future-operation", resolution: FenceResolutionApplied, want: false}, + {name: "unknown resolution", action: "power-start", resolution: "partial", want: false}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + if got := SystemRecoveryResolutionAllowed(test.action, test.resolution); got != test.want { + t.Fatalf("SystemRecoveryResolutionAllowed(%q, %q) = %v, want %v", test.action, test.resolution, got, test.want) + } + }) + } +} + +func TestParseRecoveryContractRejectsMalformedSchemas(t *testing.T) { + validOperation := `{ + "kind":"action", + "recovery_mode":"generic", + "identity_scope":"instance", + "actions":["power-start"], + "action_patterns":[], + "system_resolutions":["applied","not_applied"] + }` + valid := `{"schema_version":1,"operations":[` + validOperation + `]}` + if _, err := parseRecoveryContract([]byte(valid)); err != nil { + t.Fatalf("valid recovery contract rejected: %v", err) + } + + tests := map[string]string{ + "unknown envelope field": `{"schema_version":1,"operations":[` + validOperation + `],"future":true}`, + "unknown operation field": strings.Replace(valid, `"system_resolutions":["applied","not_applied"]`, `"system_resolutions":["applied","not_applied"],"future":true`, 1), + "unsupported schema": strings.Replace(valid, `"schema_version":1`, `"schema_version":2`, 1), + "duplicate kind": `{"schema_version":1,"operations":[` + validOperation + `,` + validOperation + `]}`, + "missing actions": strings.Replace(valid, `"actions":["power-start"],`, ``, 1), + "mode resolution mismatch": strings.Replace(valid, `"system_resolutions":["applied","not_applied"]`, `"system_resolutions":["not_applied"]`, 1), + "scope mode mismatch": strings.Replace(valid, `"identity_scope":"instance"`, `"identity_scope":"none"`, 1), + "unanchored pattern": strings.Replace(valid, + `"actions":["power-start"], + "action_patterns":[]`, + `"actions":[], + "action_patterns":["power-.*"]`, 1), + "invalid pattern": strings.Replace(valid, + `"actions":["power-start"], + "action_patterns":[]`, + `"actions":[], + "action_patterns":["^power-($"]`, 1), + "trailing document": valid + `{}`, + } + for name, document := range tests { + t.Run(name, func(t *testing.T) { + if _, err := parseRecoveryContract([]byte(document)); err == nil { + t.Fatal("malformed recovery contract was accepted") + } + }) + } +} + +func TestParseRecoveryContractAcceptsAnchoredManifestPattern(t *testing.T) { + document := `{ + "schema_version":1, + "operations":[{ + "kind":"custom_action", + "recovery_mode":"generic", + "identity_scope":"instance", + "actions":[], + "action_patterns":["^custom-action:[a-f0-9]{8}$"], + "system_resolutions":["applied","not_applied"] + }] + }` + contract, err := parseRecoveryContract([]byte(document)) + if err != nil { + t.Fatalf("valid manifest pattern rejected: %v", err) + } + if !contract.resolutionAllowed("custom-action:0123abcd", FenceResolutionApplied) { + t.Fatal("manifest pattern did not grant its declared resolution") + } + if contract.resolutionAllowed("custom-action:0123abcg", FenceResolutionApplied) { + t.Fatal("manifest pattern matched an invalid action") + } +} + +func TestParseRecoveryContractRejectsCrossModeActionGrant(t *testing.T) { + document := `{ + "schema_version":1, + "operations":[ + { + "kind":"action", + "recovery_mode":"generic", + "identity_scope":"instance", + "actions":["power-start"], + "action_patterns":[], + "system_resolutions":["applied","not_applied"] + }, + { + "kind":"host_storage_configure", + "recovery_mode":"storage", + "identity_scope":"storage", + "actions":["power-start"], + "action_patterns":[], + "system_resolutions":[] + } + ] + }` + if _, err := parseRecoveryContract([]byte(document)); err == nil { + t.Fatal("one Agent action acquired conflicting system-recovery grants") + } +} + +func TestParseRecoveryContractRejectsConflictingPatternGrants(t *testing.T) { + document := `{ + "schema_version":1, + "operations":[ + { + "kind":"generic_pattern", + "recovery_mode":"generic", + "identity_scope":"instance", + "actions":[], + "action_patterns":["^generic-[a-z]+$"], + "system_resolutions":["applied","not_applied"] + }, + { + "kind":"storage_pattern", + "recovery_mode":"storage", + "identity_scope":"storage", + "actions":[], + "action_patterns":["^storage-[a-z]+$"], + "system_resolutions":[] + } + ] + }` + if _, err := parseRecoveryContract([]byte(document)); err == nil { + t.Fatal("patterns with conflicting system-recovery grants were accepted") + } +} + +func TestParseRecoveryContractRejectsPatternOverrideOfExactAction(t *testing.T) { + document := `{ + "schema_version":1, + "operations":[ + { + "kind":"generic_pattern", + "recovery_mode":"generic", + "identity_scope":"instance", + "actions":[], + "action_patterns":["^create-vm$"], + "system_resolutions":["applied","not_applied"] + }, + { + "kind":"create", + "recovery_mode":"create", + "identity_scope":"create", + "actions":["create-vm"], + "action_patterns":[], + "system_resolutions":["not_applied"] + } + ] + }` + if _, err := parseRecoveryContract([]byte(document)); err == nil { + t.Fatal("pattern overrode an exact action's narrower system-recovery grant") + } +} diff --git a/docs/TASK_EXECUTION.md b/docs/TASK_EXECUTION.md index 973a56ae..98e55029 100644 --- a/docs/TASK_EXECUTION.md +++ b/docs/TASK_EXECUTION.md @@ -61,6 +61,10 @@ PHP 的 `receipt_deadline_at` 到期时直接把任务终态化为失败。之 ## 资源与资金安全 +普通 `applied/not_applied` 恢复保留各自的证据检查,但共用准备与提交实现,避免资源锁、恢复 escrow、父任务汇总和回执 ACK 在两个方向上漂移。公开入口不变;共同提交事务仍按父任务、目录、子任务、回执的既定顺序加锁,账本及效果结论提交后才确认回执。开通、支付和固定存储的专用收尾不进入这个通用入口。 + +跨端恢复能力由 `agent/operation/recovery_contract.json` 定义:Agent 嵌入读取,PHP 使用随源码发布的生成常量。清单区分普通叶子、开通、存储、支付及汇总任务,记录身份范围、动作及 system actor 可提交的结论;未知类型默认拒绝。请求身份重建、业务账本终结器和新鲜证据校验仍是必需条件,登记类型本身不等于可以解锁。维护步骤和跨端契约测试见 `docs/VALIDATION.md`。 + 任务终态不等于副作用已被证明。当失败回执表明操作效果可能已经发生时: - 任务本身立即结束,不再轮询、不再自动重试;内部恢复协调器只能低频读取允许种类的同一 Agent operation 证据; diff --git a/docs/VALIDATION.md b/docs/VALIDATION.md index c8752a11..0b28bb6f 100644 --- a/docs/VALIDATION.md +++ b/docs/VALIDATION.md @@ -8,6 +8,7 @@ ```bash git diff --check +php tools/generate-recovery-contract.php --check cd agent go mod verify @@ -34,6 +35,19 @@ jq empty deploy/*.json.example 测试产物只能写入临时目录;不能覆盖 `release/` 中的用户文件。macOS 没有生产 libvirt 头文件时,必须在与目标宿主机 ABI 一致的 Linux 环境补跑两个 libvirt tag 组合和 race 测试。 +### 恢复类型契约 + +`agent/operation/recovery_contract.json` 是恢复种类、身份范围、Agent action 和 system actor 可提交结论的唯一清单。Go 将它嵌入二进制;PHP 使用 `tools/generate-recovery-contract.php` 生成的 `CloudOperationRecoveryContract.php`,生产面板不读取 Agent 源码目录,也不在启动时生成文件。 + +新增或调整操作时: + +1. 修改清单,明确普通叶子、开通、存储、支付或汇总模式;不能为了通过测试把特殊任务加入普通恢复。 +2. 实现该类型的请求身份、业务终结器和种类专用证据校验;清单只定义能力边界,不替代这些安全检查。 +3. 执行 `php tools/generate-recovery-contract.php` 并一起提交生成物。`--check` 和 PHP 契约测试必须在清单/生成物不一致时失败。 +4. 补齐 `CloudOperationRecoveryMatrixTest.php` 的真实请求/账本 fixture,以及 Agent 契约测试。新增种类没有覆盖、动作映射遗漏、权限方向冲突、快照参数越界,都必须让测试失败。 + +开通的 system fence 只允许 `not_applied`,仍需专用清理与资金证据;支付、固定存储和汇总任务不得获得普通 system recovery 权限。两种普通效果结论共享一个 PHP 提交事务:保留各自证据校验,先处理账本/父任务,再在同一事务确认晚到回执。现有幂等、冲突证据、prepared crash-resume 和失败不 ACK 的回归不可删除。 + ## 2. 单次派发契约 自动测试至少证明: diff --git a/tools/generate-recovery-contract.php b/tools/generate-recovery-contract.php new file mode 100644 index 00000000..7a84b86e --- /dev/null +++ b/tools/generate-recovery-contract.php @@ -0,0 +1,146 @@ + 2 || (isset($argv[1]) && !$check)) { + fwrite(STDERR, "Usage: php tools/generate-recovery-contract.php [--check]\n"); + exit(2); +} +$contract = json_decode(file_get_contents($source), true, 512, + JSON_THROW_ON_ERROR); +$hasKeys = static function (array $value, array $keys): bool { + $actual = array_keys($value); + sort($actual); + sort($keys); + return $actual === $keys; +}; +if (!$hasKeys($contract, ['schema_version', 'operations']) + || ($contract['schema_version'] ?? null) !== 1 + || !is_array($contract['operations'] ?? null) + || !array_is_list($contract['operations']) + || $contract['operations'] === []) { + throw new RuntimeException('Invalid recovery contract envelope.'); +} +$operations = []; +foreach ($contract['operations'] as $entry) { + if (!is_array($entry) || !$hasKeys($entry, [ + 'kind', 'recovery_mode', 'identity_scope', 'actions', + 'action_patterns', 'system_resolutions', + ])) { + throw new RuntimeException('Invalid recovery operation fields.'); + } + $kind = $entry['kind'] ?? null; + $mode = $entry['recovery_mode'] ?? null; + $scope = $entry['identity_scope'] ?? null; + if (!is_string($kind) || preg_match('/^[a-z][a-z0-9_]{0,63}$/D', $kind) !== 1 + || isset($operations[$kind]) + || !in_array($mode, ['generic', 'create', 'storage', 'aggregate', 'payment'], true) + || !in_array($scope, ['instance', 'catalog', 'create', 'storage', 'none'], true)) { + throw new RuntimeException('Invalid or duplicate recovery operation.'); + } + foreach (['actions', 'action_patterns', 'system_resolutions'] as $field) { + if (!is_array($entry[$field] ?? null) || !array_is_list($entry[$field]) + || count(array_unique($entry[$field])) !== count($entry[$field])) { + throw new RuntimeException('Invalid recovery list: ' . $kind . '.' . $field); + } + foreach ($entry[$field] as $value) { + if (!is_string($value) || $value === '') { + throw new RuntimeException('Invalid recovery entry: ' . $kind . '.' . $field); + } + } + } + $resolutions = match ($mode) { + 'generic' => ['applied', 'not_applied'], + 'create' => ['not_applied'], + default => [], + }; + $validScope = match ($mode) { + 'generic' => in_array($scope, ['instance', 'catalog'], true), + 'create', 'storage' => $scope === $mode, + default => $scope === 'none', + }; + if (!$validScope || $entry['system_resolutions'] !== $resolutions + || (($scope === 'none') !== ($entry['actions'] === [] && $entry['action_patterns'] === []))) { + throw new RuntimeException('Unsafe recovery mode boundary: ' . $kind); + } + foreach ($entry['actions'] as $action) { + if (strlen($action) > 128 + || preg_match('/^[a-z][a-z0-9]*(?:[-:][a-z0-9]+)*$/D', $action) !== 1) { + throw new RuntimeException('Invalid recovery action: ' . $kind); + } + } + foreach ($entry['action_patterns'] as $pattern) { + if (!str_starts_with($pattern, '^') || !str_ends_with($pattern, '$') + || str_contains($pattern, '~') || @preg_match('~' . $pattern . '~D', '') === false) { + throw new RuntimeException('Invalid anchored recovery pattern: ' . $kind); + } + } + unset($entry['kind']); + $operations[$kind] = $entry; +} +// The Agent authorizes by action, not PHP kind. Aliases such as delete and +// expiry_delete are legal only when they grant the same resolution capability. +$actionPermissions = []; +$patterns = []; +foreach ($operations as $entry) { + foreach ($entry['actions'] as $action) { + if (isset($actionPermissions[$action]) + && $actionPermissions[$action] !== $entry['system_resolutions']) { + throw new RuntimeException('Conflicting recovery permissions: ' . $action); + } + $actionPermissions[$action] = $entry['system_resolutions']; + } + foreach ($entry['action_patterns'] as $pattern) { + // This generator deliberately does not analyze regex intersections. + // Keep patterned capabilities uniform instead of unioning permissions. + foreach ($patterns as $permissions) { + if ($permissions !== $entry['system_resolutions']) { + throw new RuntimeException('Conflicting patterned recovery permissions.'); + } + } + $patterns[$pattern] = $entry['system_resolutions']; + } +} +foreach ($actionPermissions as $action => $permissions) { + foreach ($patterns as $pattern => $patternPermissions) { + if (preg_match('~' . $pattern . '~D', $action) === 1 + && $permissions !== $patternPermissions) { + throw new RuntimeException('Recovery pattern overrides permissions: ' . $action); + } + } +} +$lines = []; +foreach ($operations as $kind => $entry) { + $lines[] = " '$kind' => ["; + foreach ($entry as $key => $value) { + $encoded = is_array($value) + ? '[' . implode(', ', array_map(static fn (string $item): string => var_export($item, true), $value)) . ']' + : var_export($value, true); + $lines[] = " '$key' => $encoded,"; + } + $lines[] = ' ],'; +} +$output = "kind, + $resolution + ) || (string) $job->execution_mode !== 'agent' || (string) $job->phase !== 'terminal' || (string) $job->outcome !== 'failed' @@ -339,6 +343,10 @@ final class CloudAdminOperationRecoveryService ); } if (!self::supportsAutomaticRecovery($job) + || !CloudOperationRecoveryPolicy::allowsSystemResolution( + (string) $job->kind, + $resolution + ) || (string) $job->execution_mode !== 'agent' || (string) $job->phase !== 'terminal' || (string) $job->outcome !== 'failed' @@ -592,7 +600,7 @@ final class CloudAdminOperationRecoveryService /** Create is inspectable, but never enters generic effect finalization. */ private static function automaticRecoveryKinds(): array { - return [...CloudOperationRecoveryPolicy::kinds(), 'create']; + return CloudOperationRecoveryPolicy::automaticKinds(); } private static function supportsAutomaticRecovery(CloudJob $job): bool @@ -858,7 +866,10 @@ final class CloudAdminOperationRecoveryService } $nodes = 0; if (strlen($encoded) > self::MAX_PROJECTION_BYTES - || !self::projectionWithinBounds($value, 1, $nodes)) { + || !self::projectionWithinBounds($value, 1, $nodes, + in_array($kind, [ + 'template_import_replica', 'template_metadata_replica', + ], true))) { self::invalidInspection(); } $schema = self::projectionSchema($kind, $resolution); @@ -1058,7 +1069,8 @@ final class CloudAdminOperationRecoveryService private static function projectionWithinBounds( mixed $value, int $depth, - int &$nodes + int &$nodes, + bool $allowTemplateGeneration = false ): bool { $nodes++; if ($nodes > self::MAX_PROJECTION_NODES @@ -1069,7 +1081,14 @@ final class CloudAdminOperationRecoveryService return true; } foreach ($value as $key => $item) { - if (is_string($key) && self::sensitiveProjectionKey($key)) { + // This is a public template revision identifier, not a bearer + // credential. Only its exact top-level, bounded form is exempt; + // recursive values and every other token remain forbidden. + $templateGeneration = $allowTemplateGeneration && $depth === 1 + && $key === 'generation_token' && is_string($item) + && preg_match('/^[a-f0-9]{32}$/D', $item) === 1; + if (is_string($key) && self::sensitiveProjectionKey($key) + && !$templateGeneration) { return false; } if (!self::projectionWithinBounds( diff --git a/web/app/Services/Cloud/CloudAutomaticOperationRecoveryService.php b/web/app/Services/Cloud/CloudAutomaticOperationRecoveryService.php index 60c03d39..cfae4f59 100644 --- a/web/app/Services/Cloud/CloudAutomaticOperationRecoveryService.php +++ b/web/app/Services/Cloud/CloudAutomaticOperationRecoveryService.php @@ -547,7 +547,7 @@ final class CloudAutomaticOperationRecoveryService // Create uses a separate financial // finalizer, never the generic replay or // effect-projection path below. - [...CloudOperationRecoveryPolicy::kinds(), 'create'] + CloudOperationRecoveryPolicy::automaticKinds() )->where( static function ($ready) use ( $now, diff --git a/web/app/Services/Cloud/CloudOperationRecoveryContract.php b/web/app/Services/Cloud/CloudOperationRecoveryContract.php new file mode 100644 index 00000000..df1754c4 --- /dev/null +++ b/web/app/Services/Cloud/CloudOperationRecoveryContract.php @@ -0,0 +1,221 @@ + [ + 'recovery_mode' => 'generic', + 'identity_scope' => 'instance', + 'actions' => ['power-start', 'power-shutdown', 'power-reboot', 'power-reset', 'power-destroy'], + 'action_patterns' => [], + 'system_resolutions' => ['applied', 'not_applied'], + ], + 'reinstall' => [ + 'recovery_mode' => 'generic', + 'identity_scope' => 'instance', + 'actions' => ['reinstall-vm'], + 'action_patterns' => [], + 'system_resolutions' => ['applied', 'not_applied'], + ], + 'rescue' => [ + 'recovery_mode' => 'generic', + 'identity_scope' => 'instance', + 'actions' => ['enter-rescue', 'force-start-rescue', 'exit-rescue:connected', 'exit-rescue:suspended'], + 'action_patterns' => [], + 'system_resolutions' => ['applied', 'not_applied'], + ], + 'delete' => [ + 'recovery_mode' => 'generic', + 'identity_scope' => 'instance', + 'actions' => ['delete-vm'], + 'action_patterns' => [], + 'system_resolutions' => ['applied', 'not_applied'], + ], + 'delete_force' => [ + 'recovery_mode' => 'generic', + 'identity_scope' => 'instance', + 'actions' => ['force-delete-vm'], + 'action_patterns' => [], + 'system_resolutions' => ['applied', 'not_applied'], + ], + 'expiry_delete' => [ + 'recovery_mode' => 'generic', + 'identity_scope' => 'instance', + 'actions' => ['delete-vm'], + 'action_patterns' => [], + 'system_resolutions' => ['applied', 'not_applied'], + ], + 'network_change' => [ + 'recovery_mode' => 'generic', + 'identity_scope' => 'instance', + 'actions' => ['change-network'], + 'action_patterns' => [], + 'system_resolutions' => ['applied', 'not_applied'], + ], + 'instance_settings' => [ + 'recovery_mode' => 'generic', + 'identity_scope' => 'instance', + 'actions' => ['update-settings'], + 'action_patterns' => [], + 'system_resolutions' => ['applied', 'not_applied'], + ], + 'snapshot_create' => [ + 'recovery_mode' => 'generic', + 'identity_scope' => 'instance', + 'actions' => [], + 'action_patterns' => ['^create-snapshot:[a-f0-9]{32}:(?:filesystem|crash)$'], + 'system_resolutions' => ['applied', 'not_applied'], + ], + 'snapshot_restore' => [ + 'recovery_mode' => 'generic', + 'identity_scope' => 'instance', + 'actions' => [], + 'action_patterns' => ['^restore-snapshot:[a-f0-9]{32}$'], + 'system_resolutions' => ['applied', 'not_applied'], + ], + 'snapshot_delete' => [ + 'recovery_mode' => 'generic', + 'identity_scope' => 'instance', + 'actions' => [], + 'action_patterns' => ['^delete-snapshot:[a-f0-9]{32}$'], + 'system_resolutions' => ['applied', 'not_applied'], + ], + 'firewall_apply' => [ + 'recovery_mode' => 'generic', + 'identity_scope' => 'instance', + 'actions' => ['apply-firewall'], + 'action_patterns' => [], + 'system_resolutions' => ['applied', 'not_applied'], + ], + 'service_suspend' => [ + 'recovery_mode' => 'generic', + 'identity_scope' => 'instance', + 'actions' => ['service-suspend'], + 'action_patterns' => [], + 'system_resolutions' => ['applied', 'not_applied'], + ], + 'network_suspend' => [ + 'recovery_mode' => 'generic', + 'identity_scope' => 'instance', + 'actions' => ['network-suspend'], + 'action_patterns' => [], + 'system_resolutions' => ['applied', 'not_applied'], + ], + 'network_resume' => [ + 'recovery_mode' => 'generic', + 'identity_scope' => 'instance', + 'actions' => ['network-resume'], + 'action_patterns' => [], + 'system_resolutions' => ['applied', 'not_applied'], + ], + 'template_import_replica' => [ + 'recovery_mode' => 'generic', + 'identity_scope' => 'catalog', + 'actions' => ['import-template'], + 'action_patterns' => [], + 'system_resolutions' => ['applied', 'not_applied'], + ], + 'template_metadata_replica' => [ + 'recovery_mode' => 'generic', + 'identity_scope' => 'catalog', + 'actions' => ['update-template'], + 'action_patterns' => [], + 'system_resolutions' => ['applied', 'not_applied'], + ], + 'template_delete_replica' => [ + 'recovery_mode' => 'generic', + 'identity_scope' => 'catalog', + 'actions' => ['delete-template'], + 'action_patterns' => [], + 'system_resolutions' => ['applied', 'not_applied'], + ], + 'rescue_template_import' => [ + 'recovery_mode' => 'generic', + 'identity_scope' => 'catalog', + 'actions' => ['import-rescue-template'], + 'action_patterns' => [], + 'system_resolutions' => ['applied', 'not_applied'], + ], + 'rescue_template_delete' => [ + 'recovery_mode' => 'generic', + 'identity_scope' => 'catalog', + 'actions' => ['delete-rescue-template'], + 'action_patterns' => [], + 'system_resolutions' => ['applied', 'not_applied'], + ], + 'user_iso_import_replica' => [ + 'recovery_mode' => 'generic', + 'identity_scope' => 'catalog', + 'actions' => ['import-user-iso'], + 'action_patterns' => [], + 'system_resolutions' => ['applied', 'not_applied'], + ], + 'user_iso_delete_replica' => [ + 'recovery_mode' => 'generic', + 'identity_scope' => 'catalog', + 'actions' => ['delete-user-iso'], + 'action_patterns' => [], + 'system_resolutions' => ['applied', 'not_applied'], + ], + 'create' => [ + 'recovery_mode' => 'create', + 'identity_scope' => 'create', + 'actions' => ['create-vm'], + 'action_patterns' => [], + 'system_resolutions' => ['not_applied'], + ], + 'host_storage_configure' => [ + 'recovery_mode' => 'storage', + 'identity_scope' => 'storage', + 'actions' => ['configure-storage'], + 'action_patterns' => [], + 'system_resolutions' => [], + ], + 'payment_create' => [ + 'recovery_mode' => 'payment', + 'identity_scope' => 'none', + 'actions' => [], + 'action_patterns' => [], + 'system_resolutions' => [], + ], + 'template_import' => [ + 'recovery_mode' => 'aggregate', + 'identity_scope' => 'none', + 'actions' => [], + 'action_patterns' => [], + 'system_resolutions' => [], + ], + 'template_metadata_sync' => [ + 'recovery_mode' => 'aggregate', + 'identity_scope' => 'none', + 'actions' => [], + 'action_patterns' => [], + 'system_resolutions' => [], + ], + 'template_delete' => [ + 'recovery_mode' => 'aggregate', + 'identity_scope' => 'none', + 'actions' => [], + 'action_patterns' => [], + 'system_resolutions' => [], + ], + 'user_iso_import' => [ + 'recovery_mode' => 'aggregate', + 'identity_scope' => 'none', + 'actions' => [], + 'action_patterns' => [], + 'system_resolutions' => [], + ], + 'user_iso_delete' => [ + 'recovery_mode' => 'aggregate', + 'identity_scope' => 'none', + 'actions' => [], + 'action_patterns' => [], + 'system_resolutions' => [], + ], + ]; +} diff --git a/web/app/Services/Cloud/CloudOperationRecoveryIdentity.php b/web/app/Services/Cloud/CloudOperationRecoveryIdentity.php index 955573b4..4d225474 100644 --- a/web/app/Services/Cloud/CloudOperationRecoveryIdentity.php +++ b/web/app/Services/Cloud/CloudOperationRecoveryIdentity.php @@ -34,8 +34,7 @@ final class CloudOperationRecoveryIdentity */ public static function supports(string $kind): bool { - return in_array($kind, ['create', 'host_storage_configure'], true) - || CloudOperationRecoveryPolicy::supports($kind); + return CloudOperationRecoveryPolicy::identityScope($kind) !== null; } /** @@ -138,7 +137,8 @@ final class CloudOperationRecoveryIdentity self::assertBodyOperation($job, $body); $kind = (string) $job->kind; - if ($kind === 'create') { + $scope = CloudOperationRecoveryPolicy::identityScope($kind); + if ($scope === 'create') { [$action, $resourceId] = self::createIdentity( $job, $payload, @@ -147,7 +147,7 @@ final class CloudOperationRecoveryIdentity $body, $forAppliedEffect ); - } elseif ($kind === 'host_storage_configure') { + } elseif ($scope === 'storage') { [$action, $resourceId] = self::storageIdentity( $job, $payload, @@ -155,13 +155,7 @@ final class CloudOperationRecoveryIdentity $target, $body ); - } elseif (in_array($kind, [ - 'action', 'reinstall', 'rescue', 'delete', 'delete_force', - 'expiry_delete', 'network_change', 'instance_settings', - 'snapshot_create', 'snapshot_restore', 'snapshot_delete', - 'firewall_apply', 'service_suspend', 'network_suspend', - 'network_resume', - ], true)) { + } elseif ($scope === 'instance') { [$action, $resourceId] = self::instanceIdentity( $job, $payload, @@ -179,6 +173,12 @@ final class CloudOperationRecoveryIdentity $body ); } + if (!CloudOperationRecoveryPolicy::allowsAction($kind, $action)) { + throw new CloudRecoveryOperationIdentityConflict( + '任务种类与 Agent 操作契约不匹配', + 409 + ); + } $wire = self::wireBody($kind, $body); $encoded = json_encode( $wire === [] ? (object) [] : $wire, diff --git a/web/app/Services/Cloud/CloudOperationRecoveryPolicy.php b/web/app/Services/Cloud/CloudOperationRecoveryPolicy.php index bf25a115..cd271e87 100644 --- a/web/app/Services/Cloud/CloudOperationRecoveryPolicy.php +++ b/web/app/Services/Cloud/CloudOperationRecoveryPolicy.php @@ -11,39 +11,62 @@ namespace App\Services\Cloud; */ final class CloudOperationRecoveryPolicy { - private const AGENT_LEAF_KINDS = [ - 'action', - 'reinstall', - 'rescue', - 'delete', - 'delete_force', - 'expiry_delete', - 'network_change', - 'instance_settings', - 'snapshot_create', - 'snapshot_restore', - 'snapshot_delete', - 'firewall_apply', - 'service_suspend', - 'network_suspend', - 'network_resume', - 'template_import_replica', - 'template_metadata_replica', - 'template_delete_replica', - 'rescue_template_import', - 'rescue_template_delete', - 'user_iso_import_replica', - 'user_iso_delete_replica', - ]; - /** @return list */ public static function kinds(): array { - return self::AGENT_LEAF_KINDS; + return self::kindsForModes(['generic']); } public static function supports(string $kind): bool { - return in_array($kind, self::AGENT_LEAF_KINDS, true); + return (CloudOperationRecoveryContract::OPERATIONS[$kind]['recovery_mode'] ?? null) + === 'generic'; + } + + /** Includes dedicated create inspection, never generic create finalization. */ + public static function automaticKinds(): array + { + return self::kindsForModes(['generic', 'create']); + } + + public static function identityScope(string $kind): ?string + { + $scope = CloudOperationRecoveryContract::OPERATIONS[$kind]['identity_scope'] ?? null; + return $scope === 'none' ? null : $scope; + } + + public static function allowsSystemResolution(string $kind, string $resolution): bool + { + return in_array($resolution, + CloudOperationRecoveryContract::OPERATIONS[$kind]['system_resolutions'] ?? [], + true); + } + + /** Check a rebuilt wire action without replacing its kind-specific evidence checks. */ + public static function allowsAction(string $kind, string $action): bool + { + $entry = CloudOperationRecoveryContract::OPERATIONS[$kind] ?? null; + if ($entry === null || strlen($action) > 128 + || preg_match('/^[a-z][a-z0-9]*(?:[-:][a-z0-9]+)*$/D', $action) !== 1) { + return false; + } + if (in_array($action, $entry['actions'], true)) { + return true; + } + foreach ($entry['action_patterns'] as $pattern) { + if (preg_match('~' . $pattern . '~D', $action) === 1) { + return true; + } + } + return false; + } + + /** @return list */ + private static function kindsForModes(array $modes): array + { + return array_keys(array_filter( + CloudOperationRecoveryContract::OPERATIONS, + static fn (array $entry): bool => in_array($entry['recovery_mode'], $modes, true) + )); } } diff --git a/web/app/Services/Cloud/CloudOperationRecoveryService.php b/web/app/Services/Cloud/CloudOperationRecoveryService.php index de4b4d73..5db981fd 100644 --- a/web/app/Services/Cloud/CloudOperationRecoveryService.php +++ b/web/app/Services/Cloud/CloudOperationRecoveryService.php @@ -30,26 +30,12 @@ final class CloudOperationRecoveryService array $operation, string $evidenceSource ): CloudJob { - $job = CloudJob::find($jobId); - if ($job === null) { - throw new CloudError('任务不存在', 404); - } - self::assertRecoverableAppliedEvidence( - $job, + return self::prepareResolution( + $jobId, $operation, - $evidenceSource - ); - $marker = self::transitionPreparedMarker( - (string) $job->effect_resolution_reason, - self::preparedMarker($operation, $evidenceSource), $evidenceSource, 'applied' ); - return (new CloudWorkerService())->prepareRecoveredEffect( - $jobId, - $operation, - $marker - ); } /** @@ -63,108 +49,13 @@ final class CloudOperationRecoveryService ?int $resolvedBy = null, ?CloudAgentCompletionEvent $completionEvent = null ): CloudJob { - self::assertResolvedBy($resolvedBy); - $aggregateParentId = (int) (CloudJob::where('id', $jobId) - ->value('parent_job_id') ?? 0); - return (new CloudJob())->getConnection()->transaction( - function () use ( - $jobId, - $operation, - $evidenceSource, - $resolvedBy, - $completionEvent, - $aggregateParentId - ): CloudJob { - if ($aggregateParentId > 0) { - $aggregateParent = CloudJob::where( - 'id', - $aggregateParentId - ) - ->lockForUpdate()->first(); - if ($aggregateParent === null) { - throw new CloudRecoveryAggregateStateConflict( - '任务的汇总父任务不存在', - 409 - ); - } - CloudTemplateCatalogService::lockAggregateCatalogLocked( - $aggregateParent - ); - CloudUserIsoService::lockAggregateCatalogLocked( - $aggregateParent - ); - } - $job = CloudJob::where('id', $jobId) - ->lockForUpdate()->first(); - if ($job === null) { - throw new CloudError('任务不存在', 404); - } - if ((int) ($job->parent_job_id ?? 0) - !== $aggregateParentId) { - throw new CloudRecoveryAggregateStateConflict( - '任务的汇总父级关系在提交期间发生变化', - 409 - ); - } - self::assertLockedAgentRecoveryEnvelope($job); - self::assertEvidenceSource($evidenceSource); - self::assertAppliedOperationIdentity($job, $operation); - $lockedEvent = self::lockLateReceiptEvent( - $job, - $completionEvent, - $evidenceSource - ); - self::assertRecoveryCommitMayPublish( - $job, - $evidenceSource, - $resolvedBy, - $completionEvent - ); - $finalReason = $evidenceSource; - if ((string) $job->effect_resolution === 'applied' - && hash_equals( - $finalReason, - (string) $job->effect_resolution_reason - )) { - self::markLateReceiptApplied($job, $lockedEvent); - return $job; - } - self::assertRecoverableAppliedEvidence( - $job, - $operation, - $evidenceSource - ); - $marker = self::preparedMarker( - $operation, - $evidenceSource - ); - if (!self::preparedMarkerMatches( - (string) $job->effect_resolution_reason, - $marker - )) { - throw new CloudError('任务效果尚未完成账本预提交', 409); - } - $job->effect_resolution = 'applied'; - $job->effect_resolved_at = time(); - $job->effect_resolved_by = $resolvedBy; - $job->effect_resolution_reason = $finalReason; - $job->sealed_payload = self::finalizedRecoveryPayload($job); - $job->lease_token = null; - $job->lease_until = 0; - $job->available_at = 0; - $job->finalization_incident = 0; - $job->finalization_error_code = null; - $job->read_at = null; - $job->updated_at = time(); - $job->save(); - CloudJobService::reconcileAggregateParentEffectLocked( - $job, - $resolvedBy - ); - self::markLateReceiptApplied($job, $lockedEvent); - return $job; - }, - 3 + return self::commitResolution( + $jobId, + $operation, + $evidenceSource, + 'applied', + $resolvedBy, + $completionEvent ); } @@ -176,26 +67,63 @@ final class CloudOperationRecoveryService int $jobId, array $operation, string $evidenceSource + ): CloudJob { + // A proved no-effect operation must still drive the original failed + // finalizer. Network reservations, snapshots, firewall revisions and + // catalog replicas all have PHP-owned pending/uncertain ledger state + // which only that finalizer can roll back safely. + return self::prepareResolution( + $jobId, + $operation, + $evidenceSource, + 'not_applied' + ); + } + + /** + * Publish a prepared no-effect conclusion after the matching Agent fence + * has been durably released, then erase the encrypted recovery escrow. + */ + public static function commitNotApplied( + int $jobId, + array $operation, + string $evidenceSource, + ?int $resolvedBy = null, + ?CloudAgentCompletionEvent $completionEvent = null + ): CloudJob { + return self::commitResolution( + $jobId, + $operation, + $evidenceSource, + 'not_applied', + $resolvedBy, + $completionEvent + ); + } + + /** Prepare one validated resolution without publishing its effect. */ + private static function prepareResolution( + int $jobId, + array $operation, + string $evidenceSource, + string $resolution ): CloudJob { $job = CloudJob::find($jobId); if ($job === null) { throw new CloudError('任务不存在', 404); } - self::assertRecoverableNotAppliedEvidence( + self::assertRecoverableEvidence( $job, $operation, - $evidenceSource + $evidenceSource, + $resolution ); $marker = self::transitionPreparedMarker( (string) $job->effect_resolution_reason, self::preparedMarker($operation, $evidenceSource), $evidenceSource, - 'not_applied' + $resolution ); - // A proved no-effect operation must still drive the original failed - // finalizer. Network reservations, snapshots, firewall revisions and - // catalog replicas all have PHP-owned pending/uncertain ledger state - // which only that finalizer can roll back safely. return (new CloudWorkerService())->prepareRecoveredEffect( $jobId, $operation, @@ -203,17 +131,16 @@ final class CloudOperationRecoveryService ); } - /** - * Publish a prepared no-effect conclusion after the matching Agent fence - * has been durably released, then erase the encrypted recovery escrow. - */ - public static function commitNotApplied( + /** Publish one prepared resolution through the shared locked commit seam. */ + private static function commitResolution( int $jobId, array $operation, string $evidenceSource, - ?int $resolvedBy = null, - ?CloudAgentCompletionEvent $completionEvent = null + string $resolution, + ?int $resolvedBy, + ?CloudAgentCompletionEvent $completionEvent ): CloudJob { + self::assertResolution($resolution); self::assertResolvedBy($resolvedBy); $aggregateParentId = (int) (CloudJob::where('id', $jobId) ->value('parent_job_id') ?? 0); @@ -222,6 +149,7 @@ final class CloudOperationRecoveryService $jobId, $operation, $evidenceSource, + $resolution, $resolvedBy, $completionEvent, $aggregateParentId @@ -259,7 +187,11 @@ final class CloudOperationRecoveryService } self::assertLockedAgentRecoveryEnvelope($job); self::assertEvidenceSource($evidenceSource); - self::assertNotAppliedOperationIdentity($job, $operation); + self::assertOperationIdentity( + $job, + $operation, + $resolution + ); $lockedEvent = self::lockLateReceiptEvent( $job, $completionEvent, @@ -272,7 +204,7 @@ final class CloudOperationRecoveryService $completionEvent ); $finalReason = $evidenceSource; - if ((string) $job->effect_resolution === 'not_applied' + if ((string) $job->effect_resolution === $resolution && hash_equals( $finalReason, (string) $job->effect_resolution_reason @@ -280,10 +212,11 @@ final class CloudOperationRecoveryService self::markLateReceiptApplied($job, $lockedEvent); return $job; } - self::assertRecoverableNotAppliedEvidence( + self::assertRecoverableEvidence( $job, $operation, - $evidenceSource + $evidenceSource, + $resolution ); $marker = self::preparedMarker( $operation, @@ -295,7 +228,7 @@ final class CloudOperationRecoveryService )) { throw new CloudError('任务效果尚未完成账本预提交', 409); } - $job->effect_resolution = 'not_applied'; + $job->effect_resolution = $resolution; $job->effect_resolved_at = time(); $job->effect_resolved_by = $resolvedBy; $job->effect_resolution_reason = $finalReason; @@ -974,6 +907,48 @@ final class CloudOperationRecoveryService && !(bool) $event->safe_to_resubmit)); } + private static function assertResolution(string $resolution): void + { + if (!in_array($resolution, ['applied', 'not_applied'], true)) { + throw new \LogicException('Unsupported recovery resolution.'); + } + } + + private static function assertRecoverableEvidence( + CloudJob $job, + array $operation, + string $evidenceSource, + string $resolution + ): void { + self::assertResolution($resolution); + if ($resolution === 'applied') { + self::assertRecoverableAppliedEvidence( + $job, + $operation, + $evidenceSource + ); + return; + } + self::assertRecoverableNotAppliedEvidence( + $job, + $operation, + $evidenceSource + ); + } + + private static function assertOperationIdentity( + CloudJob $job, + array $operation, + string $resolution + ): void { + self::assertResolution($resolution); + if ($resolution === 'applied') { + self::assertAppliedOperationIdentity($job, $operation); + return; + } + self::assertNotAppliedOperationIdentity($job, $operation); + } + private static function assertRecoverableAppliedEvidence( CloudJob $job, array $operation, diff --git a/web/tests/CloudCatalogLockOrderContractTest.php b/web/tests/CloudCatalogLockOrderContractTest.php index 135c0a58..bbc90348 100644 --- a/web/tests/CloudCatalogLockOrderContractTest.php +++ b/web/tests/CloudCatalogLockOrderContractTest.php @@ -70,22 +70,81 @@ $assert($ordered($finish, [ 'CloudTemplateCatalogService::finishJobLocked(', ]), 'Worker completion does not lock parent, catalog, child, replica in order.'); -foreach (['commitApplied', 'commitNotApplied'] as $name) { - $commit = $method( +foreach ([ + 'prepareApplied' => [ + 'end' => 'public static function commitApplied(', + 'resolution' => 'applied', + ], + 'prepareNotApplied' => [ + 'end' => 'public static function commitNotApplied(', + 'resolution' => 'not_applied', + ], +] as $name => $contract) { + $wrapper = $method( + $recovery, + 'public static function ' . $name . '(', + $contract['end'] + ); + $assert($ordered($wrapper, [ + 'return self::prepareResolution(', + "'" . $contract['resolution'] . "'", + ]), $name . ' does not preserve its public resolution adapter.'); +} + +foreach ([ + 'commitApplied' => 'applied', + 'commitNotApplied' => 'not_applied', +] as $name => $resolution) { + $wrapper = $method( $recovery, 'public static function ' . $name . '(', $name === 'commitApplied' ? 'public static function prepareNotApplied(' - : 'private static function assertResolvedBy(' + : 'private static function prepareResolution(' ); - $assert($ordered($commit, [ - '$aggregateParent = CloudJob::where(', - '$aggregateParentId', - 'CloudTemplateCatalogService::lockAggregateCatalogLocked(', - 'CloudUserIsoService::lockAggregateCatalogLocked(', - "CloudJob::where('id', \$jobId)", - ]), $name . ' takes a child lock before its catalog lock.'); + $assert($ordered($wrapper, [ + 'return self::commitResolution(', + "'" . $resolution . "'", + '$resolvedBy', + '$completionEvent', + ]), $name . ' does not preserve its public resolution adapter.'); } +$commit = $method( + $recovery, + 'private static function commitResolution(', + 'public static function settleLateReceipt(' +); +$assert($ordered($commit, [ + '$aggregateParent = CloudJob::where(', + '$aggregateParentId', + 'CloudTemplateCatalogService::lockAggregateCatalogLocked(', + 'CloudUserIsoService::lockAggregateCatalogLocked(', + "CloudJob::where('id', \$jobId)", +]), 'Shared recovery commit does not lock parent, catalog, then child.'); +$assert($ordered($commit, [ + 'self::assertLockedAgentRecoveryEnvelope(', + 'self::assertEvidenceSource(', + 'self::assertOperationIdentity(', + 'self::lockLateReceiptEvent(', + 'self::assertRecoveryCommitMayPublish(', + 'self::assertRecoverableEvidence(', + 'self::preparedMarker(', + 'self::preparedMarkerMatches(', + '$job->effect_resolution = $resolution;', + '$job->sealed_payload = self::finalizedRecoveryPayload($job);', + '$job->save();', + 'CloudJobService::reconcileAggregateParentEffectLocked(', +]), 'Shared recovery commit changed validation or ledger publish order.'); +$commitSave = strpos($commit, '$job->save();'); +$commitReconcile = strpos( + $commit, + 'CloudJobService::reconcileAggregateParentEffectLocked(' +); +$commitAck = strrpos($commit, 'self::markLateReceiptApplied('); +$assert($commitSave !== false && $commitReconcile !== false + && $commitAck !== false && $commitSave < $commitReconcile + && $commitReconcile < $commitAck, + 'Shared recovery commit ACKs a receipt before ledger convergence.'); $automaticCommit = $method( $automatic, diff --git a/web/tests/CloudOperationRecoveryContractTest.php b/web/tests/CloudOperationRecoveryContractTest.php new file mode 100644 index 00000000..246372e6 --- /dev/null +++ b/web/tests/CloudOperationRecoveryContractTest.php @@ -0,0 +1,98 @@ + $entry) { + $mode = $entry['recovery_mode']; + if ($mode === 'generic') { + $generic[] = $kind; + $assert($entry['system_resolutions'] === ['applied', 'not_applied'], + 'A generic leaf lost a recovery direction: ' . $kind); + } + if (in_array($mode, ['generic', 'create'], true)) { + $automatic[] = $kind; + } + $assert(CloudOperationRecoveryPolicy::supports($kind) === ($mode === 'generic'), + 'Generic policy mode differs from inventory: ' . $kind); + $assert(CloudOperationRecoveryIdentity::supports($kind) + === ($entry['identity_scope'] !== 'none'), + 'Identity support differs from inventory: ' . $kind); + foreach (['applied', 'not_applied', 'unknown', ''] as $resolution) { + $assert(CloudOperationRecoveryPolicy::allowsSystemResolution($kind, $resolution) + === in_array($resolution, $entry['system_resolutions'], true), + 'System resolution differs from inventory: ' . $kind); + } + foreach ($entry['actions'] as $action) { + $assert(CloudOperationRecoveryPolicy::allowsAction($kind, $action), + 'Declared wire action is not recognized: ' . $kind . '/' . $action); + $assert(!CloudOperationRecoveryPolicy::allowsAction($kind, $action . ':future') + && !CloudOperationRecoveryPolicy::allowsAction($kind, $action . "\n"), + 'Action matching is not exact: ' . $kind); + } +} +$assert(CloudOperationRecoveryPolicy::kinds() === $generic, + 'Generic selector does not follow the inventory.'); +$assert(CloudOperationRecoveryPolicy::automaticKinds() === $automatic, + 'Automatic selector does not follow the inventory.'); +$assert($expected['create']['recovery_mode'] === 'create' + && $expected['create']['system_resolutions'] === ['not_applied'] + && !CloudOperationRecoveryPolicy::supports('create'), + 'Create must retain dedicated finance recovery and one-way fence resolution.'); +foreach (['host_storage_configure', 'payment_create', 'template_import', + 'template_metadata_sync', 'template_delete', 'user_iso_import', 'user_iso_delete'] as $kind) { + $assert($expected[$kind]['system_resolutions'] === [] + && !in_array($kind, $automatic, true), + 'A separately finalized operation gained system recovery: ' . $kind); +} +foreach (['', 'future_kind'] as $kind) { + $assert(!CloudOperationRecoveryPolicy::supports($kind) + && !CloudOperationRecoveryIdentity::supports($kind) + && !CloudOperationRecoveryPolicy::allowsAction($kind, 'create-vm'), + 'Unknown kinds must be denied.'); +} +$key = str_repeat('ab', 16); +foreach ([ + ['snapshot_create', 'create-snapshot:' . $key . ':filesystem'], + ['snapshot_create', 'create-snapshot:' . $key . ':crash'], + ['snapshot_restore', 'restore-snapshot:' . $key], + ['snapshot_delete', 'delete-snapshot:' . $key], +] as [$kind, $action]) { + $assert(CloudOperationRecoveryPolicy::allowsAction($kind, $action), + 'Valid parameterized snapshot action was rejected.'); + foreach ([$action . "\n", $action . ':extra', strtoupper($action), + str_replace($key, substr($key, 1), $action)] as $invalid) { + $assert(!CloudOperationRecoveryPolicy::allowsAction($kind, $invalid), + 'Malformed parameterized snapshot action was accepted.'); + } + $assert(!CloudOperationRecoveryPolicy::allowsAction('instance_settings', $action), + 'An action can escape its owning kind.'); +} + +echo "PHP/Agent generated recovery inventory contract passed.\n"; diff --git a/web/tests/CloudOperationRecoveryMatrixTest.php b/web/tests/CloudOperationRecoveryMatrixTest.php new file mode 100644 index 00000000..7eee0404 --- /dev/null +++ b/web/tests/CloudOperationRecoveryMatrixTest.php @@ -0,0 +1,1529 @@ +httpStatus === 409, + 'Unexpected recovery state-conflict status.'); + return; + } + throw new RuntimeException( + 'A dedicated recovery kind entered generic worker recovery.' + ); + }; + + $manifestPath = dirname(__DIR__, 2) + . '/agent/operation/recovery_contract.json'; + $manifest = json_decode( + file_get_contents($manifestPath), + true, + 64, + JSON_THROW_ON_ERROR + ); + $assert(($manifest['schema_version'] ?? null) === 1, + 'Unsupported recovery contract schema version.'); + $manifestOperations = []; + foreach ($manifest['operations'] ?? [] as $entry) { + $kind = $entry['kind'] ?? null; + $assert(is_string($kind) && $kind !== '' + && !array_key_exists($kind, $manifestOperations), + 'Recovery contract contains a missing or duplicate kind.'); + unset($entry['kind']); + $manifestOperations[$kind] = $entry; + } + $assert(CloudOperationRecoveryContract::OPERATIONS + === $manifestOperations, + 'Generated PHP recovery contract is stale.'); + + $capsule = new Capsule(); + $capsule->addConnection([ + 'driver' => 'sqlite', + 'database' => ':memory:', + ], 'default'); + $capsule->setAsGlobal(); + $capsule->bootEloquent(); + $schema = $capsule->schema('default'); + + $schema->create('cloud_jobs', static function (Blueprint $table): void { + $table->increments('id'); + $table->string('operation_id')->unique(); + $table->string('request_idempotency_key')->unique(); + $table->string('request_fingerprint'); + $table->integer('parent_job_id')->nullable(); + $table->integer('order_id')->nullable(); + $table->integer('invoice_id')->nullable(); + $table->integer('instance_id')->nullable(); + $table->integer('snapshot_id')->nullable(); + $table->integer('user_id'); + $table->integer('host_id')->nullable(); + $table->string('kind'); + $table->string('execution_mode'); + $table->string('phase'); + $table->string('outcome')->nullable(); + $table->string('status'); + $table->text('sealed_payload'); + $table->integer('attempts')->default(1); + $table->integer('dispatch_attempts')->default(1); + $table->integer('dispatch_deadline_at')->nullable(); + $table->integer('dispatch_started_at')->nullable(); + $table->integer('agent_accepted_at')->nullable(); + $table->integer('receipt_deadline_at')->nullable(); + $table->integer('dispatched_at')->nullable(); + $table->integer('terminal_at')->nullable(); + $table->boolean('effect_may_have_applied')->nullable(); + $table->boolean('safe_to_resubmit')->nullable(); + $table->string('effect_resolution')->default('not_applicable'); + $table->integer('effect_resolved_at')->nullable(); + $table->integer('effect_resolved_by')->nullable(); + $table->string('effect_resolution_reason')->default(''); + $table->string('completion_event_id')->nullable(); + $table->integer('read_at')->nullable(); + $table->string('lease_token')->nullable(); + $table->integer('lease_until')->default(0); + $table->integer('available_at')->default(0); + $table->string('error_code')->nullable(); + $table->boolean('finalization_incident')->default(0); + $table->string('finalization_error_code')->nullable(); + $table->text('result_json')->nullable(); + $table->integer('created_at'); + $table->integer('updated_at'); + }); + $schema->create('cloud_instances', static function (Blueprint $table): void { + $table->increments('id'); + $table->string('uuid'); + $table->integer('host_id'); + $table->integer('user_id'); + $table->string('name'); + $table->string('status'); + $table->string('service_status')->default('exempt'); + $table->string('network_status')->default('active'); + $table->text('spec_json')->default('{}'); + $table->integer('created_at'); + $table->integer('updated_at'); + }); + $schema->create('cloud_instance_snapshots', + static function (Blueprint $table): void { + $table->increments('id'); + $table->string('snapshot_key')->unique(); + $table->integer('instance_id'); + $table->integer('user_id'); + $table->integer('host_id'); + $table->string('name'); + $table->string('state'); + $table->string('consistency'); + $table->integer('size_bytes')->nullable(); + $table->string('last_operation_id')->nullable(); + $table->integer('ready_at')->nullable(); + $table->integer('last_restored_at')->nullable(); + $table->integer('deleted_at')->nullable(); + $table->integer('created_at'); + $table->integer('updated_at'); + }); + $schema->create('cloud_instance_firewall_versions', + static function (Blueprint $table): void { + $table->increments('id'); + $table->integer('instance_id'); + $table->integer('revision'); + $table->boolean('enabled')->default(1); + $table->string('policy_digest'); + $table->string('request_hash'); + $table->string('idempotency_key'); + $table->string('state'); + $table->integer('job_id')->nullable(); + $table->string('operation_id')->nullable(); + $table->string('error_code')->nullable(); + $table->integer('applied_at')->nullable(); + $table->integer('created_at'); + $table->integer('updated_at'); + }); + $schema->create('cloud_templates', + static function (Blueprint $table): void { + $table->string('id')->primary(); + $table->string('name'); + $table->integer('revision'); + $table->string('generation_token'); + $table->string('state'); + $table->integer('active_job_id')->nullable(); + $table->integer('created_at'); + $table->integer('updated_at'); + }); + $schema->create('cloud_template_replicas', + static function (Blueprint $table): void { + $table->integer('host_id'); + $table->string('template_id'); + $table->integer('revision'); + $table->string('generation_token'); + $table->string('state'); + $table->integer('image_bytes')->default(0); + $table->string('sha256')->default(''); + $table->string('operation_id'); + $table->integer('job_id')->nullable(); + $table->string('error_code')->default(''); + $table->integer('created_at'); + $table->integer('updated_at'); + $table->primary(['host_id', 'template_id']); + }); + $schema->create('cloud_rescue_template_imports', + static function (Blueprint $table): void { + $table->string('operation_id')->primary(); + $table->integer('host_id'); + $table->string('template_id'); + $table->integer('job_id'); + $table->string('import_type'); + $table->string('name'); + $table->integer('size_bytes')->default(0); + $table->string('sha256')->default(''); + $table->string('state'); + $table->string('error_code')->default(''); + $table->integer('created_at'); + $table->integer('updated_at'); + }); + $schema->create('cloud_rescue_template_deletions', + static function (Blueprint $table): void { + $table->integer('host_id'); + $table->string('template_id'); + $table->string('operation_id'); + $table->integer('job_id'); + $table->string('state'); + $table->string('error_code')->default(''); + $table->integer('created_at'); + $table->integer('updated_at'); + $table->primary(['host_id', 'template_id']); + }); + $schema->create('cloud_user_isos', + static function (Blueprint $table): void { + $table->string('id')->primary(); + $table->integer('user_id'); + $table->string('name'); + $table->string('state'); + $table->integer('active_job_id')->nullable(); + $table->integer('created_at'); + $table->integer('updated_at'); + }); + $schema->create('cloud_user_iso_replicas', + static function (Blueprint $table): void { + $table->integer('host_id'); + $table->string('user_iso_id'); + $table->integer('user_id'); + $table->string('state'); + $table->integer('size_bytes')->default(0); + $table->string('sha256')->default(''); + $table->string('operation_id'); + $table->integer('job_id')->nullable(); + $table->string('error_code')->default(''); + $table->integer('created_at'); + $table->integer('updated_at'); + $table->primary(['host_id', 'user_iso_id']); + }); + $schema->create('cloud_host_storage_configurations', + static function (Blueprint $table): void { + $table->integer('host_id')->primary(); + $table->integer('revision'); + foreach (['managed', 'template', 'seed', 'user_iso', 'rescue'] + as $role) { + $table->text($role . '_directory'); + } + $table->string('state'); + $table->string('operation_id')->nullable(); + $table->integer('job_id')->nullable(); + $table->integer('created_at'); + $table->integer('updated_at'); + }); + + $now = time(); + $hostId = 7; + $ownerId = 41; + $instanceId = 1; + $instanceUuid = '123e4567-e89b-42d3-a456-426614174000'; + Capsule::table('cloud_instances')->insert([ + 'id' => $instanceId, + 'uuid' => $instanceUuid, + 'host_id' => $hostId, + 'user_id' => $ownerId, + 'name' => 'matrix-vm', + 'status' => 'active', + 'service_status' => 'exempt', + 'network_status' => 'active', + 'spec_json' => '{}', + 'created_at' => $now - 3600, + 'updated_at' => $now - 3600, + ]); + + $nextJobId = 1; + /** + * @return array{id:int,operation_id:string,job:CloudJob,payload:array} + */ + $insertJob = static function ( + string $kind, + callable $payloadFactory, + ?int $boundInstanceId = null, + ?int $snapshotId = null, + ?int $parentJobId = null, + ?int $orderId = null, + string $executionMode = 'agent', + int $userId = 41 + ) use (&$nextJobId, $hostId, $now): array { + $id = $nextJobId++; + $operationId = hash('md5', 'recovery-matrix-job:' . $id); + $payload = $payloadFactory($id, $operationId); + Capsule::table('cloud_jobs')->insert([ + 'id' => $id, + 'operation_id' => $operationId, + 'request_idempotency_key' => 'recovery-matrix-' . $operationId, + 'request_fingerprint' => hash('sha256', $operationId), + 'parent_job_id' => $parentJobId, + 'order_id' => $orderId, + 'invoice_id' => null, + 'instance_id' => $boundInstanceId, + 'snapshot_id' => $snapshotId, + 'user_id' => $userId, + 'host_id' => $hostId, + 'kind' => $kind, + 'execution_mode' => $executionMode, + 'phase' => 'terminal', + 'outcome' => 'failed', + 'status' => 'failed', + 'sealed_payload' => CloudSecrets::encrypt( + $payload, + 'job-payload' + ), + 'attempts' => 1, + 'dispatch_attempts' => 1, + 'dispatch_deadline_at' => $now - 120, + 'dispatch_started_at' => $now - 110, + 'agent_accepted_at' => $now - 100, + 'receipt_deadline_at' => $now - 30, + 'dispatched_at' => $now - 110, + 'terminal_at' => $now - 20, + 'effect_may_have_applied' => 1, + 'safe_to_resubmit' => 0, + 'effect_resolution' => 'unknown', + 'effect_resolution_reason' => '', + 'lease_until' => 0, + 'available_at' => 0, + 'finalization_incident' => 0, + 'created_at' => $now - 180, + 'updated_at' => $now - 20, + ]); + return [ + 'id' => $id, + 'operation_id' => $operationId, + 'job' => CloudJob::findOrFail($id), + 'payload' => $payload, + ]; + }; + + $fixtures = []; + $addFixture = static function ( + string $kind, + string $variant, + array $record, + string $expectedAction, + string $expectedResource, + array $projection + ) use (&$fixtures): void { + $fixtures[$kind][] = $record + [ + 'variant' => $variant, + 'expected_action' => $expectedAction, + 'expected_resource' => $expectedResource, + 'projection' => $projection, + ]; + }; + + $instanceFixture = static function ( + string $kind, + string $variant, + string $method, + string $suffix, + array $body, + string $expectedAction, + array $projection, + array $metadata = [] + ) use ( + $insertJob, + $addFixture, + $instanceId, + $instanceUuid, + $ownerId + ): array { + $record = $insertJob( + $kind, + static function (int $id, string $operationId) use ( + $method, + $suffix, + $body, + $metadata, + $instanceUuid, + $ownerId + ): array { + $resolvedBody = $body; + foreach ($resolvedBody as $field => $value) { + if ($value === '__OPERATION_ID__') { + $resolvedBody[$field] = $operationId; + } + } + return $metadata + [ + 'method' => $method, + 'request_target' => '/v1/vms/' + . rawurlencode($instanceUuid) . $suffix, + 'body' => $resolvedBody, + 'owner_id' => $ownerId, + 'can_manage' => false, + ]; + }, + $instanceId + ); + $addFixture( + $kind, + $variant, + $record, + $expectedAction, + $instanceUuid, + $projection + ); + return $record; + }; + + $powerProjection = ['uuid' => $instanceUuid, 'state' => 'running']; + foreach ([ + 'start' => 'power-start', + 'shutdown' => 'power-shutdown', + 'reboot' => 'power-reboot', + 'reset' => 'power-reset', + 'poweroff' => 'power-destroy', + ] as $panelAction => $wireAction) { + $instanceFixture( + 'action', + $panelAction, + 'POST', + '/actions', + ['action' => $panelAction], + $wireAction, + $powerProjection + ); + } + + $instanceFixture( + 'reinstall', + 'default', + 'POST', + '/reinstall', + [ + 'template_id' => 'debian-13', + 'template_revision' => 4, + 'template_generation_token' => str_repeat('a', 32), + 'preserve_data' => false, + 'root_password' => 'MatrixPassword!123', + ], + 'reinstall-vm', + ['uuid' => $instanceUuid, 'reinstall_committed' => true] + ); + + $instanceFixture( + 'rescue', + 'enter', + 'POST', + '/rescue/enter', + ['template_id' => 'rescue-debian-13'], + 'enter-rescue', + ['active' => true, 'staged' => true, 'network_suspended' => false] + ); + $instanceFixture( + 'rescue', + 'force-start', + 'POST', + '/rescue/force-start', + ['ack_data_loss' => true], + 'force-start-rescue', + ['active' => true, 'staged' => true, 'network_suspended' => false] + ); + foreach ([ + ['network_suspended' => false, 'state' => 'connected'], + ['network_suspended' => true, 'state' => 'suspended'], + ] as $rescueExit) { + $networkSuspended = $rescueExit['network_suspended']; + $state = $rescueExit['state']; + $instanceFixture( + 'rescue', + 'exit-' . $state, + 'POST', + '/rescue/exit', + ['network_suspended' => $networkSuspended], + 'exit-rescue:' . $state, + [ + 'active' => false, + 'staged' => false, + 'network_suspended' => $networkSuspended, + ] + ); + } + + $deleteProjection = [ + 'deleted' => true, + 'managed_artifacts_absent' => true, + 'forced' => false, + ]; + $instanceFixture( + 'delete', + 'user-delete', + 'DELETE', + '', + ['remove_managed_disks' => true], + 'delete-vm', + $deleteProjection + ); + $instanceFixture( + 'expiry_delete', + 'expiry-delete', + 'DELETE', + '', + ['remove_managed_disks' => true], + 'delete-vm', + $deleteProjection + ); + $priorOperationId = str_repeat('b', 32); + $instanceFixture( + 'delete_force', + 'force-delete', + 'POST', + '/force-delete', + [ + 'operation_id' => '__OPERATION_ID__', + 'prior_operation_id' => $priorOperationId, + 'ack_data_loss' => true, + 'remove_managed_disks' => true, + ], + 'force-delete-vm', + array_replace($deleteProjection, ['forced' => true]), + ['prior_operation_id' => $priorOperationId] + ); + + $instanceFixture( + 'network_change', + 'public-ip-change', + 'POST', + '/network', + [ + 'network' => 'bridge:br0', + 'public_pool_key' => str_repeat('c', 32), + 'public_pool_cidr' => '203.0.113.0/24', + 'public_ip_cidr' => '203.0.113.20/24', + 'gateway' => '203.0.113.1', + 'dns_servers' => ['1.1.1.1'], + 'confirm_restart' => true, + ], + 'change-network', + ['uuid' => $instanceUuid, 'network_changed' => true] + ); + $instanceFixture( + 'instance_settings', + 'hostname', + 'PATCH', + '/settings', + [ + 'hostname' => 'matrix.example', + 'confirm_restart' => true, + ], + 'update-settings', + ['uuid' => $instanceUuid, 'settings_changed' => true], + ['settings_fields' => ['hostname'], 'credential' => null] + ); + + $snapshotFixture = static function ( + string $kind, + string $variant, + string $method, + string $suffix, + array $body, + string $expectedAction, + array $projection, + string $consistency = 'crash' + ) use ( + $insertJob, + $addFixture, + $instanceId, + $instanceUuid, + $ownerId, + $hostId, + $now + ): void { + static $snapshotId = 100; + $id = $snapshotId++; + $key = hash('md5', 'recovery-matrix-snapshot:' . $id); + $record = $insertJob( + $kind, + static function (int $jobId, string $operationId) use ( + $kind, + $method, + $suffix, + $body, + $key, + $instanceUuid, + $ownerId + ): array { + return [ + 'method' => $method, + 'request_target' => '/v1/vms/' + . rawurlencode($instanceUuid) . '/snapshots' + . ($kind === 'snapshot_create' ? '' + : '/' . rawurlencode($key) . $suffix), + 'body' => $body + ($body === [] ? [] : [ + 'snapshot_key' => $key, + ]), + 'owner_id' => $ownerId, + 'can_manage' => false, + 'snapshot_key' => $key, + 'snapshot_previous_state' => 'ready', + ]; + }, + $instanceId, + $id + ); + Capsule::table('cloud_instance_snapshots')->insert([ + 'id' => $id, + 'snapshot_key' => $key, + 'instance_id' => $instanceId, + 'user_id' => $ownerId, + 'host_id' => $hostId, + 'name' => 'matrix-snapshot-' . $id, + 'state' => 'uncertain', + 'consistency' => $consistency, + 'size_bytes' => null, + 'last_operation_id' => $record['operation_id'], + 'ready_at' => $now - 300, + 'last_restored_at' => null, + 'deleted_at' => null, + 'created_at' => $now - 600, + 'updated_at' => $now - 20, + ]); + $resolvedAction = str_replace('__SNAPSHOT_KEY__', $key, + $expectedAction); + $resolvedProjection = array_map( + static fn (mixed $value): mixed => + $value === '__SNAPSHOT_KEY__' ? $key : $value, + $projection + ); + $addFixture( + $kind, + $variant, + $record, + $resolvedAction, + $instanceUuid, + $resolvedProjection + ); + }; + foreach (['filesystem', 'crash'] as $consistency) { + $snapshotFixture( + 'snapshot_create', + $consistency, + 'POST', + '', + [ + 'name' => 'matrix-' . $consistency, + 'description' => '', + 'quiesce' => $consistency === 'filesystem', + 'consistency_mode' => $consistency, + ], + 'create-snapshot:__SNAPSHOT_KEY__:' . $consistency, + [ + 'id' => 'matrix-id', + 'snapshot_key' => '__SNAPSHOT_KEY__', + 'created_at' => '2026-10-21T00:00:00Z', + 'state' => 'ready', + 'consistency' => $consistency, + ], + $consistency + ); + } + $snapshotFixture( + 'snapshot_restore', + 'restore', + 'POST', + '/restore', + [], + 'restore-snapshot:__SNAPSHOT_KEY__', + ['snapshot_key' => '__SNAPSHOT_KEY__', 'restored' => true] + ); + $snapshotFixture( + 'snapshot_delete', + 'delete', + 'DELETE', + '', + [], + 'delete-snapshot:__SNAPSHOT_KEY__', + ['snapshot_key' => '__SNAPSHOT_KEY__', 'deleted' => true] + ); + + $firewallVersionId = 1; + $firewallRevision = 9; + $firewallDigest = str_repeat('d', 64); + $firewallRecord = $instanceFixture( + 'firewall_apply', + 'apply', + 'PUT', + '/firewall', + [ + 'revision' => $firewallRevision, + 'default_in' => 'drop', + 'default_out' => 'accept', + 'policy_digest' => $firewallDigest, + 'rules' => [], + ], + 'apply-firewall', + [ + 'revision' => $firewallRevision, + 'default_in' => 'drop', + 'default_out' => 'accept', + 'rules' => [], + ], + [ + 'firewall_version_id' => $firewallVersionId, + 'firewall_revision' => $firewallRevision, + 'agent_firewall_policy_digest' => $firewallDigest, + ] + ); + Capsule::table('cloud_instance_firewall_versions')->insert([ + 'id' => $firewallVersionId, + 'instance_id' => $instanceId, + 'revision' => $firewallRevision, + 'enabled' => 1, + 'policy_digest' => $firewallDigest, + 'request_hash' => str_repeat('e', 64), + 'idempotency_key' => 'matrix-firewall-version', + 'state' => 'uncertain', + 'job_id' => $firewallRecord['id'], + 'operation_id' => $firewallRecord['operation_id'], + 'error_code' => '', + 'created_at' => $now - 120, + 'updated_at' => $now - 20, + ]); + + $instanceFixture( + 'service_suspend', + 'purge', + 'POST', + '/service/suspend', + ['reason' => 'purge'], + 'service-suspend', + ['uuid' => $instanceUuid, 'state' => 'shutoff'] + ); + $instanceFixture( + 'network_suspend', + 'traffic', + 'POST', + '/network/suspend', + ['reason' => 'traffic'], + 'network-suspend', + ['network_enabled' => false] + ); + $instanceFixture( + 'network_resume', + 'resume', + 'POST', + '/network/resume', + [], + 'network-resume', + ['network_enabled' => true] + ); + + $insertAggregateParent = static function ( + string $kind, + int $userId = 41 + ) use ($insertJob): array { + return $insertJob( + $kind, + static fn (int $id, string $operationId): array => [ + 'aggregate_kind' => $kind, + ], + null, + null, + null, + null, + 'aggregate', + $userId + ); + }; + + $templateFixture = static function ( + string $kind, + string $parentKind, + string $method, + string $expectedAction, + array $projection + ) use ( + $insertAggregateParent, + $insertJob, + $addFixture, + $hostId, + $ownerId, + $now + ): void { + $parent = $insertAggregateParent($parentKind); + $templateId = 'matrix-' . str_replace('_', '-', $kind); + $revision = 5; + $generationToken = hash('md5', 'generation:' . $kind); + $record = $insertJob( + $kind, + static function (int $jobId, string $operationId) use ( + $kind, + $method, + $templateId, + $revision, + $generationToken, + $ownerId + ): array { + $target = $kind === 'template_import_replica' + ? '/v1/templates' + : '/v1/templates/' . rawurlencode($templateId); + $body = ['operation_id' => $operationId]; + if ($kind === 'template_import_replica') { + $body += [ + 'id' => $templateId, + 'name' => 'Matrix Template', + 'source_url' => 'https://example.invalid/template.qcow2', + 'source_token' => 'matrix-token', + 'os_family' => 'linux', + 'version' => '13', + 'architecture' => 'x86_64', + 'sha256' => str_repeat('1', 64), + 'revision' => $revision, + 'generation_token' => $generationToken, + ]; + } elseif ($kind === 'template_metadata_replica') { + $body += [ + 'name' => 'Matrix Template Updated', + 'revision' => $revision, + 'generation_token' => $generationToken, + ]; + } + return [ + 'method' => $method, + 'request_target' => $target, + 'body' => $body, + 'owner_id' => $ownerId, + 'can_manage' => true, + 'template_id' => $templateId, + ]; + }, + null, + null, + $parent['id'] + ); + Capsule::table('cloud_templates')->insert([ + 'id' => $templateId, + 'name' => 'Matrix Template', + 'revision' => $revision, + 'generation_token' => $generationToken, + 'state' => 'active', + 'active_job_id' => $parent['id'], + 'created_at' => $now - 300, + 'updated_at' => $now - 20, + ]); + Capsule::table('cloud_template_replicas')->insert([ + 'host_id' => $hostId, + 'template_id' => $templateId, + 'revision' => $revision, + 'generation_token' => $generationToken, + 'state' => 'uncertain', + 'image_bytes' => 0, + 'sha256' => '', + 'operation_id' => $record['operation_id'], + 'job_id' => $record['id'], + 'error_code' => '', + 'created_at' => $now - 120, + 'updated_at' => $now - 20, + ]); + $addFixture( + $kind, + 'catalog', + $record, + $expectedAction, + 'template:' . $templateId, + $projection + ); + }; + $templateFixture( + 'template_import_replica', + 'template_import', + 'POST', + 'import-template', + [ + 'id' => 'matrix-template-import-replica', + 'generation_token' => hash('md5', + 'generation:template_import_replica'), + 'created_at' => '2026-10-21', + ] + ); + $templateFixture( + 'template_metadata_replica', + 'template_metadata_sync', + 'PATCH', + 'update-template', + [ + 'id' => 'matrix-template-metadata-replica', + 'name' => 'Matrix Template Updated', + 'revision' => 5, + 'generation_token' => hash('md5', + 'generation:template_metadata_replica'), + ] + ); + $templateFixture( + 'template_delete_replica', + 'template_delete', + 'DELETE', + 'delete-template', + [ + 'resource_type' => 'template', + 'id' => 'matrix-template-delete-replica', + 'deleted' => true, + 'managed_image_absent' => true, + 'managed_metadata_absent' => true, + 'managed_artifacts_absent' => true, + ] + ); + + $rescueImportId = hash('md5', 'matrix-rescue-import'); + $rescueImport = $insertJob( + 'rescue_template_import', + static fn (int $jobId, string $operationId): array => [ + 'method' => 'POST', + 'request_target' => '/v1/rescue-templates', + 'body' => [ + 'operation_id' => $operationId, + 'id' => $rescueImportId, + 'name' => 'Matrix Rescue', + 'source_url' => 'https://example.invalid/rescue.iso', + 'source_token' => 'matrix-token', + 'sha256' => str_repeat('2', 64), + ], + 'owner_id' => $ownerId, + 'can_manage' => true, + 'rescue_template_id' => $rescueImportId, + ] + ); + Capsule::table('cloud_rescue_template_imports')->insert([ + 'operation_id' => $rescueImport['operation_id'], + 'host_id' => $hostId, + 'template_id' => $rescueImportId, + 'job_id' => $rescueImport['id'], + 'import_type' => 'url', + 'name' => 'Matrix Rescue', + 'size_bytes' => 0, + 'sha256' => '', + 'state' => 'uncertain', + 'error_code' => '', + 'created_at' => $now - 120, + 'updated_at' => $now - 20, + ]); + $addFixture( + 'rescue_template_import', + 'catalog', + $rescueImport, + 'import-rescue-template', + 'rescue-template:' . $rescueImportId, + ['id' => $rescueImportId, 'created_at' => '2026-10-21'] + ); + + $rescueDeleteId = hash('md5', 'matrix-rescue-delete'); + $rescueDelete = $insertJob( + 'rescue_template_delete', + static fn (int $jobId, string $operationId): array => [ + 'method' => 'DELETE', + 'request_target' => '/v1/rescue-templates/' + . rawurlencode($rescueDeleteId), + 'body' => ['operation_id' => $operationId], + 'owner_id' => $ownerId, + 'can_manage' => true, + 'rescue_template_id' => $rescueDeleteId, + ] + ); + Capsule::table('cloud_rescue_template_deletions')->insert([ + 'host_id' => $hostId, + 'template_id' => $rescueDeleteId, + 'operation_id' => $rescueDelete['operation_id'], + 'job_id' => $rescueDelete['id'], + 'state' => 'uncertain', + 'error_code' => '', + 'created_at' => $now - 120, + 'updated_at' => $now - 20, + ]); + $addFixture( + 'rescue_template_delete', + 'catalog', + $rescueDelete, + 'delete-rescue-template', + 'rescue-template:' . $rescueDeleteId, + [ + 'resource_type' => 'rescue_template', + 'id' => $rescueDeleteId, + 'deleted' => true, + 'managed_image_absent' => true, + 'managed_metadata_absent' => true, + 'managed_artifacts_absent' => true, + ] + ); + + $isoFixture = static function ( + string $kind, + string $parentKind, + string $method, + string $expectedAction, + array $projection + ) use ( + $insertAggregateParent, + $insertJob, + $addFixture, + $hostId, + $ownerId, + $now + ): void { + $parent = $insertAggregateParent($parentKind, $ownerId); + $isoId = hash('md5', 'matrix-iso:' . $kind); + $record = $insertJob( + $kind, + static function (int $jobId, string $operationId) use ( + $kind, + $method, + $isoId, + $ownerId + ): array { + $target = $kind === 'user_iso_import_replica' + ? '/v1/user-isos' + : '/v1/user-isos/' . rawurlencode($isoId) + . '?owner_id=' . $ownerId; + $body = ['operation_id' => $operationId]; + if ($kind === 'user_iso_import_replica') { + $body += [ + 'id' => $isoId, + 'owner_id' => $ownerId, + 'name' => 'Matrix ISO', + 'source_url' => 'https://example.invalid/user.iso', + 'source_token' => 'matrix-token', + 'sha256' => str_repeat('3', 64), + ]; + } + return [ + 'method' => $method, + 'request_target' => $target, + 'body' => $body, + 'owner_id' => $ownerId, + 'can_manage' => false, + 'user_iso_id' => $isoId, + ]; + }, + null, + null, + $parent['id'], + null, + 'agent', + $ownerId + ); + Capsule::table('cloud_user_isos')->insert([ + 'id' => $isoId, + 'user_id' => $ownerId, + 'name' => 'Matrix ISO', + 'state' => 'active', + 'active_job_id' => $parent['id'], + 'created_at' => $now - 300, + 'updated_at' => $now - 20, + ]); + Capsule::table('cloud_user_iso_replicas')->insert([ + 'host_id' => $hostId, + 'user_iso_id' => $isoId, + 'user_id' => $ownerId, + 'state' => 'uncertain', + 'size_bytes' => 0, + 'sha256' => '', + 'operation_id' => $record['operation_id'], + 'job_id' => $record['id'], + 'error_code' => '', + 'created_at' => $now - 120, + 'updated_at' => $now - 20, + ]); + $addFixture( + $kind, + 'catalog', + $record, + $expectedAction, + 'user-iso:' . $ownerId . ':' . $isoId, + $projection + ); + }; + $isoFixture( + 'user_iso_import_replica', + 'user_iso_import', + 'POST', + 'import-user-iso', + ['id' => hash('md5', 'matrix-iso:user_iso_import_replica'), + 'created_at' => '2026-10-21'] + ); + $isoDeleteId = hash('md5', 'matrix-iso:user_iso_delete_replica'); + $isoFixture( + 'user_iso_delete_replica', + 'user_iso_delete', + 'DELETE', + 'delete-user-iso', + [ + 'resource_type' => 'user_iso', + 'id' => $isoDeleteId, + 'deleted' => true, + 'managed_image_absent' => true, + 'managed_metadata_absent' => true, + 'managed_artifacts_absent' => true, + ] + ); + + // Dedicated create/storage identities are covered without admitting them + // to the generic worker/finalizer matrix. + $dedicatedFixtures = []; + $createRecord = $insertJob( + 'create', + static fn (int $jobId, string $operationId): array => [ + 'method' => 'POST', + 'request_target' => '/v1/vms', + 'body' => [ + 'operation_id' => $operationId, + 'source' => 'cloud', + 'name' => 'matrix-create', + 'vcpu' => 2, + 'memory_mib' => 2048, + 'disk_gib' => 40, + 'bandwidth_mbps' => 100, + 'network' => 'bridge:br0', + 'template_id' => 'debian-13', + 'template_revision' => 1, + 'template_generation_token' => str_repeat('4', 32), + 'root_password' => 'MatrixPassword!123', + 'public_pool_key' => str_repeat('5', 32), + 'public_pool_cidr' => '203.0.113.0/24', + 'public_ip_cidr' => '203.0.113.30/24', + 'gateway' => '203.0.113.1', + 'dns_servers' => ['1.1.1.1'], + 'platform_network_policy_snapshot' => [ + 'schema_version' => 1, + ], + ], + 'credential' => [ + 'password' => 'MatrixPassword!123', + 'login_name' => 'root', + 'auth_mode' => 'password', + ], + 'template_metadata' => [ + 'template_id' => 'debian-13', + 'os_source' => 'cloud_template', + 'os_name' => 'Debian 13', + 'login_name' => 'root', + ], + ], + null, + null, + null, + 9001 + ); + $dedicatedFixtures['create'] = $createRecord + [ + 'expected_action' => 'create-vm', + ]; + + $storageBody = [ + 'managed_directory' => '/srv/virt/managed', + 'template_directory' => '/srv/virt/templates', + 'seed_directory' => '/srv/virt/seeds', + 'user_iso_directory' => '/srv/virt/user-isos', + 'rescue_directory' => '/srv/virt/rescue', + ]; + $storageRecord = $insertJob( + 'host_storage_configure', + static fn (int $jobId, string $operationId): array => [ + 'method' => 'POST', + 'request_target' => '/v1/storage/configuration', + 'body' => $storageBody + ['operation_id' => $operationId], + 'owner_id' => $ownerId, + 'can_manage' => true, + 'storage_revision' => 3, + ] + ); + Capsule::table('cloud_host_storage_configurations')->insert([ + 'host_id' => $hostId, + 'revision' => 3, + 'managed_directory' => $storageBody['managed_directory'], + 'template_directory' => $storageBody['template_directory'], + 'seed_directory' => $storageBody['seed_directory'], + 'user_iso_directory' => $storageBody['user_iso_directory'], + 'rescue_directory' => $storageBody['rescue_directory'], + 'state' => 'uncertain', + 'operation_id' => $storageRecord['operation_id'], + 'job_id' => $storageRecord['id'], + 'created_at' => $now - 120, + 'updated_at' => $now - 20, + ]); + $dedicatedFixtures['host_storage_configure'] = $storageRecord + [ + 'expected_action' => 'configure-storage', + 'expected_resource' => 'host-storage', + ]; + + $classificationOnlyKinds = [ + 'payment_create' => 'payment', + 'template_import' => 'aggregate', + 'template_metadata_sync' => 'aggregate', + 'template_delete' => 'aggregate', + 'user_iso_import' => 'aggregate', + 'user_iso_delete' => 'aggregate', + ]; + $genericKinds = array_keys(array_filter( + $manifestOperations, + static fn (array $operation): bool => + $operation['recovery_mode'] === 'generic' + )); + $fixtureKinds = array_keys($fixtures); + sort($genericKinds, SORT_STRING); + sort($fixtureKinds, SORT_STRING); + $assert($fixtureKinds === $genericKinds, + 'Every generic recovery kind must have an executable fixture.'); + + $coveredKinds = array_merge( + $fixtureKinds, + array_keys($dedicatedFixtures), + array_keys($classificationOnlyKinds) + ); + $contractKinds = array_keys($manifestOperations); + sort($coveredKinds, SORT_STRING); + sort($contractKinds, SORT_STRING); + $assert($coveredKinds === $contractKinds, + 'A recovery contract kind was added without a matrix fixture.'); + + $expectedAutomaticKinds = array_keys(array_filter( + $manifestOperations, + static fn (array $operation): bool => in_array( + $operation['recovery_mode'], + ['generic', 'create'], + true + ) + )); + $assert(CloudOperationRecoveryPolicy::kinds() + === array_keys(array_filter( + $manifestOperations, + static fn (array $operation): bool => + $operation['recovery_mode'] === 'generic' + )) + && CloudOperationRecoveryPolicy::automaticKinds() + === $expectedAutomaticKinds, + 'Policy kind lists diverged from the recovery modes.'); + + $projectionMethod = new ReflectionMethod( + CloudAdminOperationRecoveryService::class, + 'effectProjection' + ); + $expectInvalidProjection = static function ( + array $projection, + string $kind, + string $message + ) use ($projectionMethod, $assert): void { + try { + $projectionMethod->invoke(null, $projection, $kind, 'applied'); + } catch (CloudAgentError $error) { + $assert($error->agentCode === 'agent_invalid_inspection' + && $error->agentHttpStatus === 200, + 'Unexpected invalid-projection error for ' . $kind . '.'); + return; + } + throw new RuntimeException($message); + }; + + foreach ($fixtures as $kind => $kindFixtures) { + $entry = $manifestOperations[$kind]; + $assert(CloudOperationRecoveryPolicy::supports($kind) + && CloudOperationRecoveryPolicy::identityScope($kind) + === $entry['identity_scope'] + && CloudOperationRecoveryIdentity::supports($kind), + 'Generic policy/identity support diverged for ' . $kind . '.'); + + $actualActions = []; + foreach ($kindFixtures as $fixture) { + $job = $fixture['job']; + $payload = $fixture['payload']; + $fromPayload = CloudOperationRecoveryIdentity::expectedFromPayload( + $job, + $payload + ); + $fromEncryptedJob = CloudOperationRecoveryIdentity::expected($job); + $label = $kind . ':' . $fixture['variant']; + $assert($fromPayload === $fromEncryptedJob, + 'Encrypted identity reconstruction diverged for ' . $label); + $assert($fromPayload['action'] + === $fixture['expected_action'] + && $fromPayload['resource_id'] + === $fixture['expected_resource'], + 'Identity action/resource mismatch for ' . $label); + $assert(CloudOperationRecoveryPolicy::allowsAction( + $kind, + $fromPayload['action'] + ), + 'Contract rejected the rebuilt action for ' . $label); + + $wireBody = $payload['body']; + if ($kind === 'action' + && ($wireBody['action'] ?? null) === 'poweroff') { + $wireBody['action'] = 'destroy'; + } + $encodedBody = json_encode( + $wireBody === [] ? (object) [] : $wireBody, + JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES + | JSON_UNESCAPED_UNICODE + ); + $expectedFingerprint = hash( + 'sha256', + strtoupper($payload['method']) . "\n" + . $payload['request_target'] . "\n" . $encodedBody + ); + $assert(hash_equals( + $expectedFingerprint, + $fromPayload['request_fingerprint'] + ), + 'Wire fingerprint mismatch for ' . $label); + + $context = $kind === 'instance_settings' + ? CloudWorkerService::settingsRecoveryContext($job) + : CloudWorkerService::recoveryContext($job); + $assert($context['payload'] === $payload, + 'Worker lost the encrypted recovery payload for ' . $label); + if ($entry['identity_scope'] === 'instance') { + $assert(($context['instance']->id ?? null) === $instanceId, + 'Worker lost the instance binding for ' . $label); + } else { + $assert($context['instance'] === null, + 'Catalog recovery unexpectedly acquired an instance for ' + . $label); + } + + $projection = $projectionMethod->invoke( + null, + $fixture['projection'], + $kind, + 'applied' + ); + $assert($projection === $fixture['projection'], + 'Evidence projection branch is missing for ' . $label); + $actualActions[] = $fromPayload['action']; + } + + foreach ($entry['actions'] as $action) { + $assert(in_array($action, $actualActions, true), + 'Literal action lacks a real identity fixture: ' + . $kind . ':' . $action); + } + foreach ($entry['action_patterns'] as $pattern) { + $matched = array_filter( + $actualActions, + static fn (string $action): bool => + preg_match('~' . $pattern . '~D', $action) === 1 + ); + $assert($matched !== [], + 'Pattern action lacks a real identity fixture: ' + . $kind . ':' . $pattern); + } + foreach ($actualActions as $action) { + $literal = in_array($action, $entry['actions'], true); + $pattern = false; + foreach ($entry['action_patterns'] as $candidate) { + $pattern = $pattern + || preg_match('~' . $candidate . '~D', $action) === 1; + } + $assert($literal || $pattern, + 'Fixture produced an uncontracted action: ' + . $kind . ':' . $action); + } + } + + $snapshotCreateActions = array_column( + $fixtures['snapshot_create'], + 'expected_action' + ); + $snapshotKeyPattern = '[a-f0-9]{32}'; + $assert(count($snapshotCreateActions) === 2 + && count(array_filter( + $snapshotCreateActions, + static fn (string $action): bool => preg_match( + '~^create-snapshot:' . $snapshotKeyPattern + . ':filesystem$~D', + $action + ) === 1 + )) === 1 + && count(array_filter( + $snapshotCreateActions, + static fn (string $action): bool => preg_match( + '~^create-snapshot:' . $snapshotKeyPattern . ':crash$~D', + $action + ) === 1 + )) === 1, + 'Snapshot create must cover both filesystem and crash consistency.'); + + $validTemplateMetadataProjection = + $fixtures['template_metadata_replica'][0]['projection']; + $invalidGeneration = $validTemplateMetadataProjection; + $invalidGeneration['generation_token'] = 'NOT-32-LOWER-HEX'; + $expectInvalidProjection( + $invalidGeneration, + 'template_metadata_replica', + 'An invalid template generation token crossed the projection boundary.' + ); + $expectInvalidProjection( + [ + 'id' => 'matrix-rescue', + 'created_at' => '2026-10-21', + 'generation_token' => str_repeat('a', 32), + ], + 'rescue_template_import', + 'A generation token escaped its two template projection kinds.' + ); + $nestedToken = $validTemplateMetadataProjection; + $nestedToken['generation_token'] = [ + 'generation_token' => str_repeat('a', 32), + ]; + $expectInvalidProjection( + $nestedToken, + 'template_metadata_replica', + 'A nested generation token bypassed sensitive-key filtering.' + ); + $accessToken = $validTemplateMetadataProjection; + $accessToken['access_token'] = str_repeat('a', 32); + $expectInvalidProjection( + $accessToken, + 'template_metadata_replica', + 'An access token bypassed sensitive-key filtering.' + ); + + foreach ($dedicatedFixtures as $kind => $fixture) { + $identity = CloudOperationRecoveryIdentity::expectedFromPayload( + $fixture['job'], + $fixture['payload'] + ); + $assert($identity === CloudOperationRecoveryIdentity::expected( + $fixture['job'] + ) + && $identity['action'] === $fixture['expected_action'] + && CloudOperationRecoveryPolicy::allowsAction( + $kind, + $identity['action'] + ) + && !CloudOperationRecoveryPolicy::supports($kind), + 'Dedicated identity contract diverged for ' . $kind . '.'); + if ($kind === 'create') { + $assert(CloudOperationRecoveryPolicy::identityScope($kind) + === 'create' + && preg_match( + '/^[a-f0-9]{8}-[a-f0-9]{4}-5[a-f0-9]{3}-[89ab][a-f0-9]{3}-[a-f0-9]{12}$/D', + $identity['resource_id'] + ) === 1, + 'Create identity lost its deterministic UUID scope.'); + $projection = $projectionMethod->invoke(null, [ + 'creation_absent' => true, + 'managed_artifacts_absent' => true, + ], 'create', 'not_applied'); + $assert(($projection['creation_absent'] ?? null) === true + && ($projection['managed_artifacts_absent'] ?? null) + === true, + 'Create absence projection is no longer complete.'); + } else { + $assert(CloudOperationRecoveryPolicy::identityScope($kind) + === 'storage' + && $identity['resource_id'] + === $fixture['expected_resource'], + 'Storage identity lost its dedicated scope.'); + $assert($projectionMethod->invoke( + null, + ['configured' => true], + 'host_storage_configure', + 'applied' + ) === ['configured' => true], + 'Storage projection branch is missing.'); + } + $expectStateConflict(static fn () => + CloudWorkerService::recoveryContext($fixture['job'])); + } + + foreach ($classificationOnlyKinds as $kind => $mode) { + $entry = $manifestOperations[$kind]; + $assert($entry['recovery_mode'] === $mode + && $entry['identity_scope'] === 'none' + && $entry['actions'] === [] + && $entry['action_patterns'] === [] + && !CloudOperationRecoveryPolicy::supports($kind) + && CloudOperationRecoveryPolicy::identityScope($kind) === null + && !CloudOperationRecoveryIdentity::supports($kind), + 'Aggregate/payment classification diverged for ' . $kind . '.'); + } + + foreach ($manifestOperations as $kind => $entry) { + foreach (['applied', 'not_applied', 'partial'] as $resolution) { + $assert(CloudOperationRecoveryPolicy::allowsSystemResolution( + $kind, + $resolution + ) === in_array( + $resolution, + $entry['system_resolutions'], + true + ), + 'System resolution policy diverged for ' + . $kind . ':' . $resolution); + } + $assert(!CloudOperationRecoveryPolicy::allowsAction( + $kind, + 'unlisted-operation' + ), + 'Recovery contract accepted an unlisted action for ' . $kind); + } + + echo "Cloud operation recovery matrix tests passed\n"; +} diff --git a/web/tests/CloudSingleDispatchArchitectureTest.php b/web/tests/CloudSingleDispatchArchitectureTest.php index 651337e6..128ef011 100644 --- a/web/tests/CloudSingleDispatchArchitectureTest.php +++ b/web/tests/CloudSingleDispatchArchitectureTest.php @@ -144,9 +144,9 @@ $assert(str_contains($completion, "(string) \$job->phase === 'terminal'") && str_contains($operationRecovery, "(string) \$job->outcome === 'failed'") && str_contains($operationRecovery, - "\$job->effect_resolution = 'applied'") + "if (!in_array(\$resolution, ['applied', 'not_applied'], true))") && str_contains($operationRecovery, - "\$job->effect_resolution = 'not_applied'") + '$job->effect_resolution = $resolution;') && !str_contains($operationRecovery, "\$job->outcome = 'succeeded'") && str_contains($completion, "['succeeded', 'failed']")