refactor(cloud): unify recovery commits and operation contracts

Share applied/not_applied preparation and atomic publication while preserving direction-specific evidence checks. Define recovery capabilities once, embed them in the Agent, and generate the PHP inventory with drift and full-kind behavior contracts. Permit only bounded public template generation IDs in inspection projections.
go
codex 4 weeks ago
parent 813f9f58a6
commit 371110d9fc

@ -42,36 +42,32 @@ const (
)
var (
ErrNotFound = errors.New("operation not found")
ErrFingerprint = errors.New("operation_id payload conflict")
ErrOperationStarting = errors.New("operation dispatch is already starting")
ErrBusy = errors.New("operation execution capacity is exhausted")
ErrResourceBlocked = errors.New("resource has an unresolved operation")
ErrInvalidSupersession = errors.New("operation cannot supersede the requested predecessor")
ErrInspectionInconclusive = errors.New("read-only inspection did not confirm a terminal outcome")
ErrExecutionNotReaped = errors.New("isolated mutation process did not exit after forced termination")
ErrFenceNotResolvable = errors.New("operation does not hold an effect fence")
ErrFenceIdentityConflict = errors.New("fence resolution identity does not match operation")
ErrFenceResolutionConflict = errors.New("operation effect fence already has a different resolution")
ErrFenceInspectionStale = errors.New("operation effect fence resolution requires a fresh inspection")
ErrTerminalEvidencePending = errors.New("worker journal has no terminal operation evidence")
ErrTerminalEvidenceInvalid = errors.New("worker journal terminal evidence is invalid")
operationIDPattern = regexp.MustCompile(`^[A-Za-z0-9_-]{8,80}$`)
operationActionPattern = regexp.MustCompile(`^[a-z][a-z0-9]*(?:[-:][a-z0-9]+)*$`)
operationResourceIDPattern = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9_.:-]*$`)
hexDigestPattern = regexp.MustCompile(`^[0-9a-f]{64}$`)
noncePattern = regexp.MustCompile(`^[A-Za-z0-9_-]{16,128}$`)
networkSourcePattern = regexp.MustCompile(`^[A-Za-z0-9_.:-]{1,128}$`)
causeCodePattern = regexp.MustCompile(`^[a-z][a-z0-9]*(_[a-z0-9]+)*$`)
canonicalUUIDPattern = regexp.MustCompile(`^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$`)
resourcePartPattern = regexp.MustCompile(`^[A-Za-z0-9_.:-]{1,192}$`)
resourceKindPattern = regexp.MustCompile(`^[a-z][a-z0-9_-]{0,63}$`)
fenceResolutionIDPattern = regexp.MustCompile(`^[a-f0-9]{32}$`)
completionSensitiveKeyPattern = regexp.MustCompile(`(?i)password|secret|(?:public|private).?key|ssh.?key|passphrase|credential|(?:(?:access|refresh|auth|api|bearer|session|source).?token)|^token$`)
snapshotRecoveryActionPattern = regexp.MustCompile(
`^(?:create-snapshot:[a-f0-9]{32}:(?:filesystem|crash)|(?:delete|restore)-snapshot:[a-f0-9]{32})$`,
)
ErrNotFound = errors.New("operation not found")
ErrFingerprint = errors.New("operation_id payload conflict")
ErrOperationStarting = errors.New("operation dispatch is already starting")
ErrBusy = errors.New("operation execution capacity is exhausted")
ErrResourceBlocked = errors.New("resource has an unresolved operation")
ErrInvalidSupersession = errors.New("operation cannot supersede the requested predecessor")
ErrInspectionInconclusive = errors.New("read-only inspection did not confirm a terminal outcome")
ErrExecutionNotReaped = errors.New("isolated mutation process did not exit after forced termination")
ErrFenceNotResolvable = errors.New("operation does not hold an effect fence")
ErrFenceIdentityConflict = errors.New("fence resolution identity does not match operation")
ErrFenceResolutionConflict = errors.New("operation effect fence already has a different resolution")
ErrFenceInspectionStale = errors.New("operation effect fence resolution requires a fresh inspection")
ErrTerminalEvidencePending = errors.New("worker journal has no terminal operation evidence")
ErrTerminalEvidenceInvalid = errors.New("worker journal terminal evidence is invalid")
operationIDPattern = regexp.MustCompile(`^[A-Za-z0-9_-]{8,80}$`)
operationActionPattern = regexp.MustCompile(`^[a-z][a-z0-9]*(?:[-:][a-z0-9]+)*$`)
operationResourceIDPattern = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9_.:-]*$`)
hexDigestPattern = regexp.MustCompile(`^[0-9a-f]{64}$`)
noncePattern = regexp.MustCompile(`^[A-Za-z0-9_-]{16,128}$`)
networkSourcePattern = regexp.MustCompile(`^[A-Za-z0-9_.:-]{1,128}$`)
causeCodePattern = regexp.MustCompile(`^[a-z][a-z0-9]*(_[a-z0-9]+)*$`)
canonicalUUIDPattern = regexp.MustCompile(`^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$`)
resourcePartPattern = regexp.MustCompile(`^[A-Za-z0-9_.:-]{1,192}$`)
resourceKindPattern = regexp.MustCompile(`^[a-z][a-z0-9_-]{0,63}$`)
fenceResolutionIDPattern = regexp.MustCompile(`^[a-f0-9]{32}$`)
completionSensitiveKeyPattern = regexp.MustCompile(`(?i)password|secret|(?:public|private).?key|ssh.?key|passphrase|credential|(?:(?:access|refresh|auth|api|bearer|session|source).?token)|^token$`)
metadataBucket = []byte("metadata")
operationsBucket = []byte("operations")
inflightBucket = []byte("inflight")
@ -176,29 +172,6 @@ type Spec struct {
SupersedesOperationID string
}
// SystemRecoveryActionAllowed is the deny-by-default capability boundary for
// the panel's read-only inspector and Journal-only fence resolver. Adding a
// new mutation route does not implicitly grant system recovery access: every
// leaf action must be reviewed and added here explicitly.
func SystemRecoveryActionAllowed(action string) bool {
switch action {
case "create-vm",
"power-start", "power-shutdown", "power-reboot", "power-reset", "power-destroy",
"reinstall-vm",
"enter-rescue", "force-start-rescue", "exit-rescue:connected", "exit-rescue:suspended",
"change-network", "network-suspend", "network-resume",
"update-settings", "apply-firewall",
"delete-vm", "force-delete-vm",
"service-suspend",
"import-template", "update-template", "delete-template",
"import-rescue-template", "delete-rescue-template",
"import-user-iso", "delete-user-iso":
return true
default:
return snapshotRecoveryActionPattern.MatchString(action)
}
}
// ValidOperationIdentity validates the immutable non-secret identity copied
// from a Journal record into inspections and completion deliveries.
func ValidOperationIdentity(action, resourceID, requestFingerprint string) bool {
@ -873,20 +846,16 @@ func (j *Journal) ResolveResourceFence(operationID string, resolvedBy int64,
return ErrFenceIdentityConflict
}
// The zero audit actor is a deliberately narrow internal capability.
// Only explicitly reviewed automatic recovery actions may use it;
// human administrators retain their positive audited identity.
if resolvedBy == 0 && !SystemRecoveryActionAllowed(record.Action) {
// Only an action/resolution pair explicitly granted by the embedded
// contract may use it; human administrators retain their positive
// audited identity.
if resolvedBy == 0 && !SystemRecoveryResolutionAllowed(record.Action, request.Resolution) {
return ErrFenceNotResolvable
}
// Automatic create recovery is deliberately one-way: fresh host
// absence evidence may release a failed create fence as not applied,
// but the zero-ID system actor can never declare that a create applied.
// Nor may it release a fence while an unreaped worker could still create
// the resource after the inspection. Materializing an applied create
// requires the original immutable completion result and remains PHP's
// responsibility.
if resolvedBy == 0 && record.Action == "create-vm" &&
(record.ExecutionFenced || request.Resolution != FenceResolutionNotApplied) {
// An unreaped create worker could still create the resource after the
// inspection. Its fence therefore remains active even though the
// contract permits system not_applied for a safely reaped create.
if resolvedBy == 0 && record.Action == "create-vm" && record.ExecutionFenced {
return ErrFenceNotResolvable
}
if encoded := resolutions.Get([]byte(operationID)); encoded != nil {

@ -0,0 +1,288 @@
package operation
import (
"bytes"
_ "embed"
"encoding/json"
"errors"
"fmt"
"io"
"regexp"
)
const recoveryContractSchemaVersion = 1
var recoveryKindPattern = regexp.MustCompile(`^[a-z][a-z0-9_]{0,63}$`)
//go:embed recovery_contract.json
var embeddedRecoveryContractJSON []byte
var embeddedRecoveryContract = mustParseRecoveryContract(embeddedRecoveryContractJSON)
type recoveryContractDocument struct {
SchemaVersion int `json:"schema_version"`
Operations []recoveryContractOperation `json:"operations"`
}
type recoveryContractOperation struct {
Kind string `json:"kind"`
RecoveryMode string `json:"recovery_mode"`
IdentityScope string `json:"identity_scope"`
Actions []string `json:"actions"`
ActionPatterns []string `json:"action_patterns"`
SystemResolutions []string `json:"system_resolutions"`
}
type recoveryResolutionSet map[string]struct{}
type recoveryActionPattern struct {
source string
pattern *regexp.Regexp
resolutions recoveryResolutionSet
}
type recoveryContract struct {
actions map[string]recoveryResolutionSet
patterns []recoveryActionPattern
}
// SystemRecoveryActionAllowed is the deny-by-default capability boundary for
// the panel's read-only inspector. A manifest entry without a permitted system
// resolution (for example host storage coordination) does not grant access.
func SystemRecoveryActionAllowed(action string) bool {
return embeddedRecoveryContract.actionAllowed(action)
}
// SystemRecoveryResolutionAllowed binds an automatic fence decision to the
// exact immutable Agent action. This prevents a manifest entry that only
// permits not_applied from inheriting the generic applied capability.
func SystemRecoveryResolutionAllowed(action, resolution string) bool {
return embeddedRecoveryContract.resolutionAllowed(action, resolution)
}
func (contract recoveryContract) actionAllowed(action string) bool {
if len(action) > 128 || !operationActionPattern.MatchString(action) {
return false
}
if resolutions, ok := contract.actions[action]; ok && len(resolutions) > 0 {
return true
}
for _, candidate := range contract.patterns {
if len(candidate.resolutions) > 0 && candidate.pattern.MatchString(action) {
return true
}
}
return false
}
func (contract recoveryContract) resolutionAllowed(action, resolution string) bool {
if len(action) > 128 || !operationActionPattern.MatchString(action) {
return false
}
if resolutions, ok := contract.actions[action]; ok {
_, allowed := resolutions[resolution]
return allowed
}
for _, candidate := range contract.patterns {
if candidate.pattern.MatchString(action) {
_, allowed := candidate.resolutions[resolution]
return allowed
}
}
return false
}
func mustParseRecoveryContract(document []byte) recoveryContract {
contract, err := parseRecoveryContract(document)
if err != nil {
panic(fmt.Sprintf("invalid embedded recovery contract: %v", err))
}
return contract
}
func parseRecoveryContract(document []byte) (recoveryContract, error) {
decoder := json.NewDecoder(bytes.NewReader(document))
decoder.DisallowUnknownFields()
var decoded recoveryContractDocument
if err := decoder.Decode(&decoded); err != nil {
return recoveryContract{}, fmt.Errorf("decode recovery contract: %w", err)
}
var trailing json.RawMessage
if err := decoder.Decode(&trailing); !errors.Is(err, io.EOF) {
if err == nil {
return recoveryContract{}, errors.New("recovery contract contains a trailing JSON value")
}
return recoveryContract{}, fmt.Errorf("decode recovery contract trailer: %w", err)
}
if decoded.SchemaVersion != recoveryContractSchemaVersion {
return recoveryContract{}, fmt.Errorf("unsupported recovery contract schema version %d", decoded.SchemaVersion)
}
if len(decoded.Operations) == 0 {
return recoveryContract{}, errors.New("recovery contract operations must not be empty")
}
contract := recoveryContract{actions: make(map[string]recoveryResolutionSet)}
seenKinds := make(map[string]struct{}, len(decoded.Operations))
seenPatterns := make(map[string]recoveryResolutionSet)
var patternedResolutions recoveryResolutionSet
for index, operation := range decoded.Operations {
if err := validateRecoveryContractOperation(operation); err != nil {
return recoveryContract{}, fmt.Errorf("recovery contract operation %d: %w", index, err)
}
if _, exists := seenKinds[operation.Kind]; exists {
return recoveryContract{}, fmt.Errorf("recovery contract kind %q is duplicated", operation.Kind)
}
seenKinds[operation.Kind] = struct{}{}
resolutions := recoveryResolutionSetFrom(operation.SystemResolutions)
for _, action := range operation.Actions {
if existing, exists := contract.actions[action]; exists {
if !sameRecoveryResolutionSet(existing, resolutions) {
return recoveryContract{}, fmt.Errorf("action %q has conflicting system resolutions", action)
}
continue
}
contract.actions[action] = resolutions
}
for _, source := range operation.ActionPatterns {
if existing, exists := seenPatterns[source]; exists {
if !sameRecoveryResolutionSet(existing, resolutions) {
return recoveryContract{}, fmt.Errorf("action pattern %q has conflicting system resolutions", source)
}
continue
}
// This loader intentionally does not analyze regex intersections.
// Requiring every pattern to carry one identical grant prevents
// overlaps from unioning different system-recovery capabilities.
if patternedResolutions != nil && !sameRecoveryResolutionSet(patternedResolutions, resolutions) {
return recoveryContract{}, errors.New("action patterns have conflicting system resolutions")
}
compiled, err := regexp.Compile(source)
if err != nil {
return recoveryContract{}, fmt.Errorf("compile action pattern %q: %w", source, err)
}
patternedResolutions = resolutions
seenPatterns[source] = resolutions
contract.patterns = append(contract.patterns, recoveryActionPattern{
source: source, pattern: compiled, resolutions: resolutions,
})
}
}
// No exact action may silently acquire a second, different capability from
// a pattern. The reviewed patterns are currently disjoint, but this check
// keeps the invariant explicit if the manifest grammar is extended later.
for action, exactResolutions := range contract.actions {
for _, candidate := range contract.patterns {
if candidate.pattern.MatchString(action) &&
!sameRecoveryResolutionSet(exactResolutions, candidate.resolutions) {
return recoveryContract{}, fmt.Errorf("action %q matches pattern %q with conflicting system resolutions", action, candidate.source)
}
}
}
return contract, nil
}
func validateRecoveryContractOperation(operation recoveryContractOperation) error {
if !recoveryKindPattern.MatchString(operation.Kind) {
return fmt.Errorf("invalid kind %q", operation.Kind)
}
if operation.Actions == nil || operation.ActionPatterns == nil || operation.SystemResolutions == nil {
return errors.New("actions, action_patterns, and system_resolutions must all be present")
}
allowedScopes := map[string]bool{}
expectedResolutions := []string(nil)
requiresActions := false
switch operation.RecoveryMode {
case "generic":
allowedScopes["instance"] = true
allowedScopes["catalog"] = true
expectedResolutions = []string{FenceResolutionApplied, FenceResolutionNotApplied}
requiresActions = true
case "create":
allowedScopes["create"] = true
expectedResolutions = []string{FenceResolutionNotApplied}
requiresActions = true
case "storage":
allowedScopes["storage"] = true
expectedResolutions = []string{}
requiresActions = true
case "aggregate", "payment":
allowedScopes["none"] = true
expectedResolutions = []string{}
default:
return fmt.Errorf("unsupported recovery_mode %q", operation.RecoveryMode)
}
if !allowedScopes[operation.IdentityScope] {
return fmt.Errorf("identity_scope %q is invalid for recovery_mode %q", operation.IdentityScope, operation.RecoveryMode)
}
if requiresActions == (len(operation.Actions)+len(operation.ActionPatterns) == 0) {
if requiresActions {
return fmt.Errorf("recovery_mode %q requires an action or action pattern", operation.RecoveryMode)
}
return fmt.Errorf("recovery_mode %q must not define actions or action patterns", operation.RecoveryMode)
}
if !sameStringSlice(operation.SystemResolutions, expectedResolutions) {
return fmt.Errorf("system_resolutions do not match recovery_mode %q", operation.RecoveryMode)
}
seenActions := make(map[string]struct{}, len(operation.Actions))
for _, action := range operation.Actions {
if len(action) > 128 || !operationActionPattern.MatchString(action) {
return fmt.Errorf("invalid action %q", action)
}
if _, exists := seenActions[action]; exists {
return fmt.Errorf("action %q is duplicated", action)
}
seenActions[action] = struct{}{}
}
seenPatterns := make(map[string]struct{}, len(operation.ActionPatterns))
for _, source := range operation.ActionPatterns {
if len(source) < 2 || source[0] != '^' || source[len(source)-1] != '$' {
return fmt.Errorf("action pattern %q must be anchored", source)
}
if _, err := regexp.Compile(source); err != nil {
return fmt.Errorf("invalid action pattern %q: %w", source, err)
}
if _, exists := seenPatterns[source]; exists {
return fmt.Errorf("action pattern %q is duplicated", source)
}
seenPatterns[source] = struct{}{}
}
return nil
}
func recoveryResolutionSetFrom(resolutions []string) recoveryResolutionSet {
result := make(recoveryResolutionSet, len(resolutions))
for _, resolution := range resolutions {
result[resolution] = struct{}{}
}
return result
}
func sameRecoveryResolutionSet(left, right recoveryResolutionSet) bool {
if len(left) != len(right) {
return false
}
for resolution := range left {
if _, exists := right[resolution]; !exists {
return false
}
}
return true
}
func sameStringSlice(left, right []string) bool {
if len(left) != len(right) {
return false
}
for index := range left {
if left[index] != right[index] {
return false
}
}
return true
}

@ -0,0 +1,368 @@
{
"schema_version": 1,
"operations": [
{
"kind": "action",
"recovery_mode": "generic",
"identity_scope": "instance",
"actions": [
"power-start",
"power-shutdown",
"power-reboot",
"power-reset",
"power-destroy"
],
"action_patterns": [],
"system_resolutions": [
"applied",
"not_applied"
]
},
{
"kind": "reinstall",
"recovery_mode": "generic",
"identity_scope": "instance",
"actions": [
"reinstall-vm"
],
"action_patterns": [],
"system_resolutions": [
"applied",
"not_applied"
]
},
{
"kind": "rescue",
"recovery_mode": "generic",
"identity_scope": "instance",
"actions": [
"enter-rescue",
"force-start-rescue",
"exit-rescue:connected",
"exit-rescue:suspended"
],
"action_patterns": [],
"system_resolutions": [
"applied",
"not_applied"
]
},
{
"kind": "delete",
"recovery_mode": "generic",
"identity_scope": "instance",
"actions": [
"delete-vm"
],
"action_patterns": [],
"system_resolutions": [
"applied",
"not_applied"
]
},
{
"kind": "delete_force",
"recovery_mode": "generic",
"identity_scope": "instance",
"actions": [
"force-delete-vm"
],
"action_patterns": [],
"system_resolutions": [
"applied",
"not_applied"
]
},
{
"kind": "expiry_delete",
"recovery_mode": "generic",
"identity_scope": "instance",
"actions": [
"delete-vm"
],
"action_patterns": [],
"system_resolutions": [
"applied",
"not_applied"
]
},
{
"kind": "network_change",
"recovery_mode": "generic",
"identity_scope": "instance",
"actions": [
"change-network"
],
"action_patterns": [],
"system_resolutions": [
"applied",
"not_applied"
]
},
{
"kind": "instance_settings",
"recovery_mode": "generic",
"identity_scope": "instance",
"actions": [
"update-settings"
],
"action_patterns": [],
"system_resolutions": [
"applied",
"not_applied"
]
},
{
"kind": "snapshot_create",
"recovery_mode": "generic",
"identity_scope": "instance",
"actions": [],
"action_patterns": [
"^create-snapshot:[a-f0-9]{32}:(?:filesystem|crash)$"
],
"system_resolutions": [
"applied",
"not_applied"
]
},
{
"kind": "snapshot_restore",
"recovery_mode": "generic",
"identity_scope": "instance",
"actions": [],
"action_patterns": [
"^restore-snapshot:[a-f0-9]{32}$"
],
"system_resolutions": [
"applied",
"not_applied"
]
},
{
"kind": "snapshot_delete",
"recovery_mode": "generic",
"identity_scope": "instance",
"actions": [],
"action_patterns": [
"^delete-snapshot:[a-f0-9]{32}$"
],
"system_resolutions": [
"applied",
"not_applied"
]
},
{
"kind": "firewall_apply",
"recovery_mode": "generic",
"identity_scope": "instance",
"actions": [
"apply-firewall"
],
"action_patterns": [],
"system_resolutions": [
"applied",
"not_applied"
]
},
{
"kind": "service_suspend",
"recovery_mode": "generic",
"identity_scope": "instance",
"actions": [
"service-suspend"
],
"action_patterns": [],
"system_resolutions": [
"applied",
"not_applied"
]
},
{
"kind": "network_suspend",
"recovery_mode": "generic",
"identity_scope": "instance",
"actions": [
"network-suspend"
],
"action_patterns": [],
"system_resolutions": [
"applied",
"not_applied"
]
},
{
"kind": "network_resume",
"recovery_mode": "generic",
"identity_scope": "instance",
"actions": [
"network-resume"
],
"action_patterns": [],
"system_resolutions": [
"applied",
"not_applied"
]
},
{
"kind": "template_import_replica",
"recovery_mode": "generic",
"identity_scope": "catalog",
"actions": [
"import-template"
],
"action_patterns": [],
"system_resolutions": [
"applied",
"not_applied"
]
},
{
"kind": "template_metadata_replica",
"recovery_mode": "generic",
"identity_scope": "catalog",
"actions": [
"update-template"
],
"action_patterns": [],
"system_resolutions": [
"applied",
"not_applied"
]
},
{
"kind": "template_delete_replica",
"recovery_mode": "generic",
"identity_scope": "catalog",
"actions": [
"delete-template"
],
"action_patterns": [],
"system_resolutions": [
"applied",
"not_applied"
]
},
{
"kind": "rescue_template_import",
"recovery_mode": "generic",
"identity_scope": "catalog",
"actions": [
"import-rescue-template"
],
"action_patterns": [],
"system_resolutions": [
"applied",
"not_applied"
]
},
{
"kind": "rescue_template_delete",
"recovery_mode": "generic",
"identity_scope": "catalog",
"actions": [
"delete-rescue-template"
],
"action_patterns": [],
"system_resolutions": [
"applied",
"not_applied"
]
},
{
"kind": "user_iso_import_replica",
"recovery_mode": "generic",
"identity_scope": "catalog",
"actions": [
"import-user-iso"
],
"action_patterns": [],
"system_resolutions": [
"applied",
"not_applied"
]
},
{
"kind": "user_iso_delete_replica",
"recovery_mode": "generic",
"identity_scope": "catalog",
"actions": [
"delete-user-iso"
],
"action_patterns": [],
"system_resolutions": [
"applied",
"not_applied"
]
},
{
"kind": "create",
"recovery_mode": "create",
"identity_scope": "create",
"actions": [
"create-vm"
],
"action_patterns": [],
"system_resolutions": [
"not_applied"
]
},
{
"kind": "host_storage_configure",
"recovery_mode": "storage",
"identity_scope": "storage",
"actions": [
"configure-storage"
],
"action_patterns": [],
"system_resolutions": []
},
{
"kind": "payment_create",
"recovery_mode": "payment",
"identity_scope": "none",
"actions": [],
"action_patterns": [],
"system_resolutions": []
},
{
"kind": "template_import",
"recovery_mode": "aggregate",
"identity_scope": "none",
"actions": [],
"action_patterns": [],
"system_resolutions": []
},
{
"kind": "template_metadata_sync",
"recovery_mode": "aggregate",
"identity_scope": "none",
"actions": [],
"action_patterns": [],
"system_resolutions": []
},
{
"kind": "template_delete",
"recovery_mode": "aggregate",
"identity_scope": "none",
"actions": [],
"action_patterns": [],
"system_resolutions": []
},
{
"kind": "user_iso_import",
"recovery_mode": "aggregate",
"identity_scope": "none",
"actions": [],
"action_patterns": [],
"system_resolutions": []
},
{
"kind": "user_iso_delete",
"recovery_mode": "aggregate",
"identity_scope": "none",
"actions": [],
"action_patterns": [],
"system_resolutions": []
}
]
}

@ -0,0 +1,179 @@
package operation
import (
"strings"
"testing"
)
func TestEmbeddedRecoveryContractControlsSystemResolutions(t *testing.T) {
tests := []struct {
name string
action string
resolution string
want bool
}{
{name: "generic applied", action: "power-start", resolution: FenceResolutionApplied, want: true},
{name: "generic not applied", action: "power-start", resolution: FenceResolutionNotApplied, want: true},
{name: "parameterized snapshot", action: "create-snapshot:0123456789abcdef0123456789abcdef:filesystem", resolution: FenceResolutionApplied, want: true},
{name: "create not applied", action: "create-vm", resolution: FenceResolutionNotApplied, want: true},
{name: "create applied", action: "create-vm", resolution: FenceResolutionApplied, want: false},
{name: "storage excluded", action: "configure-storage", resolution: FenceResolutionNotApplied, want: false},
{name: "unknown action", action: "future-operation", resolution: FenceResolutionApplied, want: false},
{name: "unknown resolution", action: "power-start", resolution: "partial", want: false},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
if got := SystemRecoveryResolutionAllowed(test.action, test.resolution); got != test.want {
t.Fatalf("SystemRecoveryResolutionAllowed(%q, %q) = %v, want %v", test.action, test.resolution, got, test.want)
}
})
}
}
func TestParseRecoveryContractRejectsMalformedSchemas(t *testing.T) {
validOperation := `{
"kind":"action",
"recovery_mode":"generic",
"identity_scope":"instance",
"actions":["power-start"],
"action_patterns":[],
"system_resolutions":["applied","not_applied"]
}`
valid := `{"schema_version":1,"operations":[` + validOperation + `]}`
if _, err := parseRecoveryContract([]byte(valid)); err != nil {
t.Fatalf("valid recovery contract rejected: %v", err)
}
tests := map[string]string{
"unknown envelope field": `{"schema_version":1,"operations":[` + validOperation + `],"future":true}`,
"unknown operation field": strings.Replace(valid, `"system_resolutions":["applied","not_applied"]`, `"system_resolutions":["applied","not_applied"],"future":true`, 1),
"unsupported schema": strings.Replace(valid, `"schema_version":1`, `"schema_version":2`, 1),
"duplicate kind": `{"schema_version":1,"operations":[` + validOperation + `,` + validOperation + `]}`,
"missing actions": strings.Replace(valid, `"actions":["power-start"],`, ``, 1),
"mode resolution mismatch": strings.Replace(valid, `"system_resolutions":["applied","not_applied"]`, `"system_resolutions":["not_applied"]`, 1),
"scope mode mismatch": strings.Replace(valid, `"identity_scope":"instance"`, `"identity_scope":"none"`, 1),
"unanchored pattern": strings.Replace(valid,
`"actions":["power-start"],
"action_patterns":[]`,
`"actions":[],
"action_patterns":["power-.*"]`, 1),
"invalid pattern": strings.Replace(valid,
`"actions":["power-start"],
"action_patterns":[]`,
`"actions":[],
"action_patterns":["^power-($"]`, 1),
"trailing document": valid + `{}`,
}
for name, document := range tests {
t.Run(name, func(t *testing.T) {
if _, err := parseRecoveryContract([]byte(document)); err == nil {
t.Fatal("malformed recovery contract was accepted")
}
})
}
}
func TestParseRecoveryContractAcceptsAnchoredManifestPattern(t *testing.T) {
document := `{
"schema_version":1,
"operations":[{
"kind":"custom_action",
"recovery_mode":"generic",
"identity_scope":"instance",
"actions":[],
"action_patterns":["^custom-action:[a-f0-9]{8}$"],
"system_resolutions":["applied","not_applied"]
}]
}`
contract, err := parseRecoveryContract([]byte(document))
if err != nil {
t.Fatalf("valid manifest pattern rejected: %v", err)
}
if !contract.resolutionAllowed("custom-action:0123abcd", FenceResolutionApplied) {
t.Fatal("manifest pattern did not grant its declared resolution")
}
if contract.resolutionAllowed("custom-action:0123abcg", FenceResolutionApplied) {
t.Fatal("manifest pattern matched an invalid action")
}
}
func TestParseRecoveryContractRejectsCrossModeActionGrant(t *testing.T) {
document := `{
"schema_version":1,
"operations":[
{
"kind":"action",
"recovery_mode":"generic",
"identity_scope":"instance",
"actions":["power-start"],
"action_patterns":[],
"system_resolutions":["applied","not_applied"]
},
{
"kind":"host_storage_configure",
"recovery_mode":"storage",
"identity_scope":"storage",
"actions":["power-start"],
"action_patterns":[],
"system_resolutions":[]
}
]
}`
if _, err := parseRecoveryContract([]byte(document)); err == nil {
t.Fatal("one Agent action acquired conflicting system-recovery grants")
}
}
func TestParseRecoveryContractRejectsConflictingPatternGrants(t *testing.T) {
document := `{
"schema_version":1,
"operations":[
{
"kind":"generic_pattern",
"recovery_mode":"generic",
"identity_scope":"instance",
"actions":[],
"action_patterns":["^generic-[a-z]+$"],
"system_resolutions":["applied","not_applied"]
},
{
"kind":"storage_pattern",
"recovery_mode":"storage",
"identity_scope":"storage",
"actions":[],
"action_patterns":["^storage-[a-z]+$"],
"system_resolutions":[]
}
]
}`
if _, err := parseRecoveryContract([]byte(document)); err == nil {
t.Fatal("patterns with conflicting system-recovery grants were accepted")
}
}
func TestParseRecoveryContractRejectsPatternOverrideOfExactAction(t *testing.T) {
document := `{
"schema_version":1,
"operations":[
{
"kind":"generic_pattern",
"recovery_mode":"generic",
"identity_scope":"instance",
"actions":[],
"action_patterns":["^create-vm$"],
"system_resolutions":["applied","not_applied"]
},
{
"kind":"create",
"recovery_mode":"create",
"identity_scope":"create",
"actions":["create-vm"],
"action_patterns":[],
"system_resolutions":["not_applied"]
}
]
}`
if _, err := parseRecoveryContract([]byte(document)); err == nil {
t.Fatal("pattern overrode an exact action's narrower system-recovery grant")
}
}

@ -61,6 +61,10 @@ PHP 的 `receipt_deadline_at` 到期时直接把任务终态化为失败。之
## 资源与资金安全
普通 `applied/not_applied` 恢复保留各自的证据检查,但共用准备与提交实现,避免资源锁、恢复 escrow、父任务汇总和回执 ACK 在两个方向上漂移。公开入口不变;共同提交事务仍按父任务、目录、子任务、回执的既定顺序加锁,账本及效果结论提交后才确认回执。开通、支付和固定存储的专用收尾不进入这个通用入口。
跨端恢复能力由 `agent/operation/recovery_contract.json` 定义:Agent 嵌入读取,PHP 使用随源码发布的生成常量。清单区分普通叶子、开通、存储、支付及汇总任务,记录身份范围、动作及 system actor 可提交的结论;未知类型默认拒绝。请求身份重建、业务账本终结器和新鲜证据校验仍是必需条件,登记类型本身不等于可以解锁。维护步骤和跨端契约测试见 `docs/VALIDATION.md`。
任务终态不等于副作用已被证明。当失败回执表明操作效果可能已经发生时:
- 任务本身立即结束,不再轮询、不再自动重试;内部恢复协调器只能低频读取允许种类的同一 Agent operation 证据;

@ -8,6 +8,7 @@
```bash
git diff --check
php tools/generate-recovery-contract.php --check
cd agent
go mod verify
@ -34,6 +35,19 @@ jq empty deploy/*.json.example
测试产物只能写入临时目录;不能覆盖 `release/` 中的用户文件。macOS 没有生产 libvirt 头文件时,必须在与目标宿主机 ABI 一致的 Linux 环境补跑两个 libvirt tag 组合和 race 测试。
### 恢复类型契约
`agent/operation/recovery_contract.json` 是恢复种类、身份范围、Agent action 和 system actor 可提交结论的唯一清单。Go 将它嵌入二进制;PHP 使用 `tools/generate-recovery-contract.php` 生成的 `CloudOperationRecoveryContract.php`,生产面板不读取 Agent 源码目录,也不在启动时生成文件。
新增或调整操作时:
1. 修改清单,明确普通叶子、开通、存储、支付或汇总模式;不能为了通过测试把特殊任务加入普通恢复。
2. 实现该类型的请求身份、业务终结器和种类专用证据校验;清单只定义能力边界,不替代这些安全检查。
3. 执行 `php tools/generate-recovery-contract.php` 并一起提交生成物。`--check` 和 PHP 契约测试必须在清单/生成物不一致时失败。
4. 补齐 `CloudOperationRecoveryMatrixTest.php` 的真实请求/账本 fixture,以及 Agent 契约测试。新增种类没有覆盖、动作映射遗漏、权限方向冲突、快照参数越界,都必须让测试失败。
开通的 system fence 只允许 `not_applied`,仍需专用清理与资金证据;支付、固定存储和汇总任务不得获得普通 system recovery 权限。两种普通效果结论共享一个 PHP 提交事务:保留各自证据校验,先处理账本/父任务,再在同一事务确认晚到回执。现有幂等、冲突证据、prepared crash-resume 和失败不 ACK 的回归不可删除。
## 2. 单次派发契约
自动测试至少证明:

@ -0,0 +1,146 @@
<?php
declare(strict_types=1);
// Development-only generator. Production PHP uses the generated constant and
// never needs the Agent source tree, JSON parsing, or a writable deployment.
$root = dirname(__DIR__);
$source = $root . '/agent/operation/recovery_contract.json';
$target = $root . '/web/app/Services/Cloud/CloudOperationRecoveryContract.php';
$check = ($argv[1] ?? '') === '--check';
if (count($argv) > 2 || (isset($argv[1]) && !$check)) {
fwrite(STDERR, "Usage: php tools/generate-recovery-contract.php [--check]\n");
exit(2);
}
$contract = json_decode(file_get_contents($source), true, 512,
JSON_THROW_ON_ERROR);
$hasKeys = static function (array $value, array $keys): bool {
$actual = array_keys($value);
sort($actual);
sort($keys);
return $actual === $keys;
};
if (!$hasKeys($contract, ['schema_version', 'operations'])
|| ($contract['schema_version'] ?? null) !== 1
|| !is_array($contract['operations'] ?? null)
|| !array_is_list($contract['operations'])
|| $contract['operations'] === []) {
throw new RuntimeException('Invalid recovery contract envelope.');
}
$operations = [];
foreach ($contract['operations'] as $entry) {
if (!is_array($entry) || !$hasKeys($entry, [
'kind', 'recovery_mode', 'identity_scope', 'actions',
'action_patterns', 'system_resolutions',
])) {
throw new RuntimeException('Invalid recovery operation fields.');
}
$kind = $entry['kind'] ?? null;
$mode = $entry['recovery_mode'] ?? null;
$scope = $entry['identity_scope'] ?? null;
if (!is_string($kind) || preg_match('/^[a-z][a-z0-9_]{0,63}$/D', $kind) !== 1
|| isset($operations[$kind])
|| !in_array($mode, ['generic', 'create', 'storage', 'aggregate', 'payment'], true)
|| !in_array($scope, ['instance', 'catalog', 'create', 'storage', 'none'], true)) {
throw new RuntimeException('Invalid or duplicate recovery operation.');
}
foreach (['actions', 'action_patterns', 'system_resolutions'] as $field) {
if (!is_array($entry[$field] ?? null) || !array_is_list($entry[$field])
|| count(array_unique($entry[$field])) !== count($entry[$field])) {
throw new RuntimeException('Invalid recovery list: ' . $kind . '.' . $field);
}
foreach ($entry[$field] as $value) {
if (!is_string($value) || $value === '') {
throw new RuntimeException('Invalid recovery entry: ' . $kind . '.' . $field);
}
}
}
$resolutions = match ($mode) {
'generic' => ['applied', 'not_applied'],
'create' => ['not_applied'],
default => [],
};
$validScope = match ($mode) {
'generic' => in_array($scope, ['instance', 'catalog'], true),
'create', 'storage' => $scope === $mode,
default => $scope === 'none',
};
if (!$validScope || $entry['system_resolutions'] !== $resolutions
|| (($scope === 'none') !== ($entry['actions'] === [] && $entry['action_patterns'] === []))) {
throw new RuntimeException('Unsafe recovery mode boundary: ' . $kind);
}
foreach ($entry['actions'] as $action) {
if (strlen($action) > 128
|| preg_match('/^[a-z][a-z0-9]*(?:[-:][a-z0-9]+)*$/D', $action) !== 1) {
throw new RuntimeException('Invalid recovery action: ' . $kind);
}
}
foreach ($entry['action_patterns'] as $pattern) {
if (!str_starts_with($pattern, '^') || !str_ends_with($pattern, '$')
|| str_contains($pattern, '~') || @preg_match('~' . $pattern . '~D', '') === false) {
throw new RuntimeException('Invalid anchored recovery pattern: ' . $kind);
}
}
unset($entry['kind']);
$operations[$kind] = $entry;
}
// The Agent authorizes by action, not PHP kind. Aliases such as delete and
// expiry_delete are legal only when they grant the same resolution capability.
$actionPermissions = [];
$patterns = [];
foreach ($operations as $entry) {
foreach ($entry['actions'] as $action) {
if (isset($actionPermissions[$action])
&& $actionPermissions[$action] !== $entry['system_resolutions']) {
throw new RuntimeException('Conflicting recovery permissions: ' . $action);
}
$actionPermissions[$action] = $entry['system_resolutions'];
}
foreach ($entry['action_patterns'] as $pattern) {
// This generator deliberately does not analyze regex intersections.
// Keep patterned capabilities uniform instead of unioning permissions.
foreach ($patterns as $permissions) {
if ($permissions !== $entry['system_resolutions']) {
throw new RuntimeException('Conflicting patterned recovery permissions.');
}
}
$patterns[$pattern] = $entry['system_resolutions'];
}
}
foreach ($actionPermissions as $action => $permissions) {
foreach ($patterns as $pattern => $patternPermissions) {
if (preg_match('~' . $pattern . '~D', $action) === 1
&& $permissions !== $patternPermissions) {
throw new RuntimeException('Recovery pattern overrides permissions: ' . $action);
}
}
}
$lines = [];
foreach ($operations as $kind => $entry) {
$lines[] = " '$kind' => [";
foreach ($entry as $key => $value) {
$encoded = is_array($value)
? '[' . implode(', ', array_map(static fn (string $item): string => var_export($item, true), $value)) . ']'
: var_export($value, true);
$lines[] = " '$key' => $encoded,";
}
$lines[] = ' ],';
}
$output = "<?php\n\nnamespace App\\Services\\Cloud;\n\n"
. "// Generated from agent/operation/recovery_contract.json. Do not edit.\n"
. "// Regenerate: php tools/generate-recovery-contract.php\n"
. "final class CloudOperationRecoveryContract\n{\n"
. " public const OPERATIONS = [\n" . implode("\n", $lines)
. "\n ];\n}\n";
if ($check) {
if (!is_file($target) || file_get_contents($target) !== $output) {
fwrite(STDERR, "Recovery contract is stale; run php tools/generate-recovery-contract.php\n");
exit(1);
}
echo "Generated recovery contract is current.\n";
exit(0);
}
if (file_put_contents($target, $output) !== strlen($output)) {
throw new RuntimeException('Could not write PHP recovery contract.');
}
echo "Generated PHP recovery contract.\n";

@ -264,6 +264,10 @@ final class CloudAdminOperationRecoveryService
$inspection['request_fingerprint'] ?? null,
]);
if (!self::supportsAutomaticRecovery($job)
|| !CloudOperationRecoveryPolicy::allowsSystemResolution(
(string) $job->kind,
$resolution
)
|| (string) $job->execution_mode !== 'agent'
|| (string) $job->phase !== 'terminal'
|| (string) $job->outcome !== 'failed'
@ -339,6 +343,10 @@ final class CloudAdminOperationRecoveryService
);
}
if (!self::supportsAutomaticRecovery($job)
|| !CloudOperationRecoveryPolicy::allowsSystemResolution(
(string) $job->kind,
$resolution
)
|| (string) $job->execution_mode !== 'agent'
|| (string) $job->phase !== 'terminal'
|| (string) $job->outcome !== 'failed'
@ -592,7 +600,7 @@ final class CloudAdminOperationRecoveryService
/** Create is inspectable, but never enters generic effect finalization. */
private static function automaticRecoveryKinds(): array
{
return [...CloudOperationRecoveryPolicy::kinds(), 'create'];
return CloudOperationRecoveryPolicy::automaticKinds();
}
private static function supportsAutomaticRecovery(CloudJob $job): bool
@ -858,7 +866,10 @@ final class CloudAdminOperationRecoveryService
}
$nodes = 0;
if (strlen($encoded) > self::MAX_PROJECTION_BYTES
|| !self::projectionWithinBounds($value, 1, $nodes)) {
|| !self::projectionWithinBounds($value, 1, $nodes,
in_array($kind, [
'template_import_replica', 'template_metadata_replica',
], true))) {
self::invalidInspection();
}
$schema = self::projectionSchema($kind, $resolution);
@ -1058,7 +1069,8 @@ final class CloudAdminOperationRecoveryService
private static function projectionWithinBounds(
mixed $value,
int $depth,
int &$nodes
int &$nodes,
bool $allowTemplateGeneration = false
): bool {
$nodes++;
if ($nodes > self::MAX_PROJECTION_NODES
@ -1069,7 +1081,14 @@ final class CloudAdminOperationRecoveryService
return true;
}
foreach ($value as $key => $item) {
if (is_string($key) && self::sensitiveProjectionKey($key)) {
// This is a public template revision identifier, not a bearer
// credential. Only its exact top-level, bounded form is exempt;
// recursive values and every other token remain forbidden.
$templateGeneration = $allowTemplateGeneration && $depth === 1
&& $key === 'generation_token' && is_string($item)
&& preg_match('/^[a-f0-9]{32}$/D', $item) === 1;
if (is_string($key) && self::sensitiveProjectionKey($key)
&& !$templateGeneration) {
return false;
}
if (!self::projectionWithinBounds(

@ -547,7 +547,7 @@ final class CloudAutomaticOperationRecoveryService
// Create uses a separate financial
// finalizer, never the generic replay or
// effect-projection path below.
[...CloudOperationRecoveryPolicy::kinds(), 'create']
CloudOperationRecoveryPolicy::automaticKinds()
)->where(
static function ($ready) use (
$now,

@ -0,0 +1,221 @@
<?php
namespace App\Services\Cloud;
// Generated from agent/operation/recovery_contract.json. Do not edit.
// Regenerate: php tools/generate-recovery-contract.php
final class CloudOperationRecoveryContract
{
public const OPERATIONS = [
'action' => [
'recovery_mode' => 'generic',
'identity_scope' => 'instance',
'actions' => ['power-start', 'power-shutdown', 'power-reboot', 'power-reset', 'power-destroy'],
'action_patterns' => [],
'system_resolutions' => ['applied', 'not_applied'],
],
'reinstall' => [
'recovery_mode' => 'generic',
'identity_scope' => 'instance',
'actions' => ['reinstall-vm'],
'action_patterns' => [],
'system_resolutions' => ['applied', 'not_applied'],
],
'rescue' => [
'recovery_mode' => 'generic',
'identity_scope' => 'instance',
'actions' => ['enter-rescue', 'force-start-rescue', 'exit-rescue:connected', 'exit-rescue:suspended'],
'action_patterns' => [],
'system_resolutions' => ['applied', 'not_applied'],
],
'delete' => [
'recovery_mode' => 'generic',
'identity_scope' => 'instance',
'actions' => ['delete-vm'],
'action_patterns' => [],
'system_resolutions' => ['applied', 'not_applied'],
],
'delete_force' => [
'recovery_mode' => 'generic',
'identity_scope' => 'instance',
'actions' => ['force-delete-vm'],
'action_patterns' => [],
'system_resolutions' => ['applied', 'not_applied'],
],
'expiry_delete' => [
'recovery_mode' => 'generic',
'identity_scope' => 'instance',
'actions' => ['delete-vm'],
'action_patterns' => [],
'system_resolutions' => ['applied', 'not_applied'],
],
'network_change' => [
'recovery_mode' => 'generic',
'identity_scope' => 'instance',
'actions' => ['change-network'],
'action_patterns' => [],
'system_resolutions' => ['applied', 'not_applied'],
],
'instance_settings' => [
'recovery_mode' => 'generic',
'identity_scope' => 'instance',
'actions' => ['update-settings'],
'action_patterns' => [],
'system_resolutions' => ['applied', 'not_applied'],
],
'snapshot_create' => [
'recovery_mode' => 'generic',
'identity_scope' => 'instance',
'actions' => [],
'action_patterns' => ['^create-snapshot:[a-f0-9]{32}:(?:filesystem|crash)$'],
'system_resolutions' => ['applied', 'not_applied'],
],
'snapshot_restore' => [
'recovery_mode' => 'generic',
'identity_scope' => 'instance',
'actions' => [],
'action_patterns' => ['^restore-snapshot:[a-f0-9]{32}$'],
'system_resolutions' => ['applied', 'not_applied'],
],
'snapshot_delete' => [
'recovery_mode' => 'generic',
'identity_scope' => 'instance',
'actions' => [],
'action_patterns' => ['^delete-snapshot:[a-f0-9]{32}$'],
'system_resolutions' => ['applied', 'not_applied'],
],
'firewall_apply' => [
'recovery_mode' => 'generic',
'identity_scope' => 'instance',
'actions' => ['apply-firewall'],
'action_patterns' => [],
'system_resolutions' => ['applied', 'not_applied'],
],
'service_suspend' => [
'recovery_mode' => 'generic',
'identity_scope' => 'instance',
'actions' => ['service-suspend'],
'action_patterns' => [],
'system_resolutions' => ['applied', 'not_applied'],
],
'network_suspend' => [
'recovery_mode' => 'generic',
'identity_scope' => 'instance',
'actions' => ['network-suspend'],
'action_patterns' => [],
'system_resolutions' => ['applied', 'not_applied'],
],
'network_resume' => [
'recovery_mode' => 'generic',
'identity_scope' => 'instance',
'actions' => ['network-resume'],
'action_patterns' => [],
'system_resolutions' => ['applied', 'not_applied'],
],
'template_import_replica' => [
'recovery_mode' => 'generic',
'identity_scope' => 'catalog',
'actions' => ['import-template'],
'action_patterns' => [],
'system_resolutions' => ['applied', 'not_applied'],
],
'template_metadata_replica' => [
'recovery_mode' => 'generic',
'identity_scope' => 'catalog',
'actions' => ['update-template'],
'action_patterns' => [],
'system_resolutions' => ['applied', 'not_applied'],
],
'template_delete_replica' => [
'recovery_mode' => 'generic',
'identity_scope' => 'catalog',
'actions' => ['delete-template'],
'action_patterns' => [],
'system_resolutions' => ['applied', 'not_applied'],
],
'rescue_template_import' => [
'recovery_mode' => 'generic',
'identity_scope' => 'catalog',
'actions' => ['import-rescue-template'],
'action_patterns' => [],
'system_resolutions' => ['applied', 'not_applied'],
],
'rescue_template_delete' => [
'recovery_mode' => 'generic',
'identity_scope' => 'catalog',
'actions' => ['delete-rescue-template'],
'action_patterns' => [],
'system_resolutions' => ['applied', 'not_applied'],
],
'user_iso_import_replica' => [
'recovery_mode' => 'generic',
'identity_scope' => 'catalog',
'actions' => ['import-user-iso'],
'action_patterns' => [],
'system_resolutions' => ['applied', 'not_applied'],
],
'user_iso_delete_replica' => [
'recovery_mode' => 'generic',
'identity_scope' => 'catalog',
'actions' => ['delete-user-iso'],
'action_patterns' => [],
'system_resolutions' => ['applied', 'not_applied'],
],
'create' => [
'recovery_mode' => 'create',
'identity_scope' => 'create',
'actions' => ['create-vm'],
'action_patterns' => [],
'system_resolutions' => ['not_applied'],
],
'host_storage_configure' => [
'recovery_mode' => 'storage',
'identity_scope' => 'storage',
'actions' => ['configure-storage'],
'action_patterns' => [],
'system_resolutions' => [],
],
'payment_create' => [
'recovery_mode' => 'payment',
'identity_scope' => 'none',
'actions' => [],
'action_patterns' => [],
'system_resolutions' => [],
],
'template_import' => [
'recovery_mode' => 'aggregate',
'identity_scope' => 'none',
'actions' => [],
'action_patterns' => [],
'system_resolutions' => [],
],
'template_metadata_sync' => [
'recovery_mode' => 'aggregate',
'identity_scope' => 'none',
'actions' => [],
'action_patterns' => [],
'system_resolutions' => [],
],
'template_delete' => [
'recovery_mode' => 'aggregate',
'identity_scope' => 'none',
'actions' => [],
'action_patterns' => [],
'system_resolutions' => [],
],
'user_iso_import' => [
'recovery_mode' => 'aggregate',
'identity_scope' => 'none',
'actions' => [],
'action_patterns' => [],
'system_resolutions' => [],
],
'user_iso_delete' => [
'recovery_mode' => 'aggregate',
'identity_scope' => 'none',
'actions' => [],
'action_patterns' => [],
'system_resolutions' => [],
],
];
}

@ -34,8 +34,7 @@ final class CloudOperationRecoveryIdentity
*/
public static function supports(string $kind): bool
{
return in_array($kind, ['create', 'host_storage_configure'], true)
|| CloudOperationRecoveryPolicy::supports($kind);
return CloudOperationRecoveryPolicy::identityScope($kind) !== null;
}
/**
@ -138,7 +137,8 @@ final class CloudOperationRecoveryIdentity
self::assertBodyOperation($job, $body);
$kind = (string) $job->kind;
if ($kind === 'create') {
$scope = CloudOperationRecoveryPolicy::identityScope($kind);
if ($scope === 'create') {
[$action, $resourceId] = self::createIdentity(
$job,
$payload,
@ -147,7 +147,7 @@ final class CloudOperationRecoveryIdentity
$body,
$forAppliedEffect
);
} elseif ($kind === 'host_storage_configure') {
} elseif ($scope === 'storage') {
[$action, $resourceId] = self::storageIdentity(
$job,
$payload,
@ -155,13 +155,7 @@ final class CloudOperationRecoveryIdentity
$target,
$body
);
} elseif (in_array($kind, [
'action', 'reinstall', 'rescue', 'delete', 'delete_force',
'expiry_delete', 'network_change', 'instance_settings',
'snapshot_create', 'snapshot_restore', 'snapshot_delete',
'firewall_apply', 'service_suspend', 'network_suspend',
'network_resume',
], true)) {
} elseif ($scope === 'instance') {
[$action, $resourceId] = self::instanceIdentity(
$job,
$payload,
@ -179,6 +173,12 @@ final class CloudOperationRecoveryIdentity
$body
);
}
if (!CloudOperationRecoveryPolicy::allowsAction($kind, $action)) {
throw new CloudRecoveryOperationIdentityConflict(
'任务种类与 Agent 操作契约不匹配',
409
);
}
$wire = self::wireBody($kind, $body);
$encoded = json_encode(
$wire === [] ? (object) [] : $wire,

@ -11,39 +11,62 @@ namespace App\Services\Cloud;
*/
final class CloudOperationRecoveryPolicy
{
private const AGENT_LEAF_KINDS = [
'action',
'reinstall',
'rescue',
'delete',
'delete_force',
'expiry_delete',
'network_change',
'instance_settings',
'snapshot_create',
'snapshot_restore',
'snapshot_delete',
'firewall_apply',
'service_suspend',
'network_suspend',
'network_resume',
'template_import_replica',
'template_metadata_replica',
'template_delete_replica',
'rescue_template_import',
'rescue_template_delete',
'user_iso_import_replica',
'user_iso_delete_replica',
];
/** @return list<string> */
public static function kinds(): array
{
return self::AGENT_LEAF_KINDS;
return self::kindsForModes(['generic']);
}
public static function supports(string $kind): bool
{
return in_array($kind, self::AGENT_LEAF_KINDS, true);
return (CloudOperationRecoveryContract::OPERATIONS[$kind]['recovery_mode'] ?? null)
=== 'generic';
}
/** Includes dedicated create inspection, never generic create finalization. */
public static function automaticKinds(): array
{
return self::kindsForModes(['generic', 'create']);
}
public static function identityScope(string $kind): ?string
{
$scope = CloudOperationRecoveryContract::OPERATIONS[$kind]['identity_scope'] ?? null;
return $scope === 'none' ? null : $scope;
}
public static function allowsSystemResolution(string $kind, string $resolution): bool
{
return in_array($resolution,
CloudOperationRecoveryContract::OPERATIONS[$kind]['system_resolutions'] ?? [],
true);
}
/** Check a rebuilt wire action without replacing its kind-specific evidence checks. */
public static function allowsAction(string $kind, string $action): bool
{
$entry = CloudOperationRecoveryContract::OPERATIONS[$kind] ?? null;
if ($entry === null || strlen($action) > 128
|| preg_match('/^[a-z][a-z0-9]*(?:[-:][a-z0-9]+)*$/D', $action) !== 1) {
return false;
}
if (in_array($action, $entry['actions'], true)) {
return true;
}
foreach ($entry['action_patterns'] as $pattern) {
if (preg_match('~' . $pattern . '~D', $action) === 1) {
return true;
}
}
return false;
}
/** @return list<string> */
private static function kindsForModes(array $modes): array
{
return array_keys(array_filter(
CloudOperationRecoveryContract::OPERATIONS,
static fn (array $entry): bool => in_array($entry['recovery_mode'], $modes, true)
));
}
}

@ -30,26 +30,12 @@ final class CloudOperationRecoveryService
array $operation,
string $evidenceSource
): CloudJob {
$job = CloudJob::find($jobId);
if ($job === null) {
throw new CloudError('任务不存在', 404);
}
self::assertRecoverableAppliedEvidence(
$job,
return self::prepareResolution(
$jobId,
$operation,
$evidenceSource
);
$marker = self::transitionPreparedMarker(
(string) $job->effect_resolution_reason,
self::preparedMarker($operation, $evidenceSource),
$evidenceSource,
'applied'
);
return (new CloudWorkerService())->prepareRecoveredEffect(
$jobId,
$operation,
$marker
);
}
/**
@ -63,108 +49,13 @@ final class CloudOperationRecoveryService
?int $resolvedBy = null,
?CloudAgentCompletionEvent $completionEvent = null
): CloudJob {
self::assertResolvedBy($resolvedBy);
$aggregateParentId = (int) (CloudJob::where('id', $jobId)
->value('parent_job_id') ?? 0);
return (new CloudJob())->getConnection()->transaction(
function () use (
$jobId,
$operation,
$evidenceSource,
$resolvedBy,
$completionEvent,
$aggregateParentId
): CloudJob {
if ($aggregateParentId > 0) {
$aggregateParent = CloudJob::where(
'id',
$aggregateParentId
)
->lockForUpdate()->first();
if ($aggregateParent === null) {
throw new CloudRecoveryAggregateStateConflict(
'任务的汇总父任务不存在',
409
);
}
CloudTemplateCatalogService::lockAggregateCatalogLocked(
$aggregateParent
);
CloudUserIsoService::lockAggregateCatalogLocked(
$aggregateParent
);
}
$job = CloudJob::where('id', $jobId)
->lockForUpdate()->first();
if ($job === null) {
throw new CloudError('任务不存在', 404);
}
if ((int) ($job->parent_job_id ?? 0)
!== $aggregateParentId) {
throw new CloudRecoveryAggregateStateConflict(
'任务的汇总父级关系在提交期间发生变化',
409
);
}
self::assertLockedAgentRecoveryEnvelope($job);
self::assertEvidenceSource($evidenceSource);
self::assertAppliedOperationIdentity($job, $operation);
$lockedEvent = self::lockLateReceiptEvent(
$job,
$completionEvent,
$evidenceSource
);
self::assertRecoveryCommitMayPublish(
$job,
$evidenceSource,
$resolvedBy,
$completionEvent
);
$finalReason = $evidenceSource;
if ((string) $job->effect_resolution === 'applied'
&& hash_equals(
$finalReason,
(string) $job->effect_resolution_reason
)) {
self::markLateReceiptApplied($job, $lockedEvent);
return $job;
}
self::assertRecoverableAppliedEvidence(
$job,
$operation,
$evidenceSource
);
$marker = self::preparedMarker(
$operation,
$evidenceSource
);
if (!self::preparedMarkerMatches(
(string) $job->effect_resolution_reason,
$marker
)) {
throw new CloudError('任务效果尚未完成账本预提交', 409);
}
$job->effect_resolution = 'applied';
$job->effect_resolved_at = time();
$job->effect_resolved_by = $resolvedBy;
$job->effect_resolution_reason = $finalReason;
$job->sealed_payload = self::finalizedRecoveryPayload($job);
$job->lease_token = null;
$job->lease_until = 0;
$job->available_at = 0;
$job->finalization_incident = 0;
$job->finalization_error_code = null;
$job->read_at = null;
$job->updated_at = time();
$job->save();
CloudJobService::reconcileAggregateParentEffectLocked(
$job,
$resolvedBy
);
self::markLateReceiptApplied($job, $lockedEvent);
return $job;
},
3
return self::commitResolution(
$jobId,
$operation,
$evidenceSource,
'applied',
$resolvedBy,
$completionEvent
);
}
@ -176,26 +67,63 @@ final class CloudOperationRecoveryService
int $jobId,
array $operation,
string $evidenceSource
): CloudJob {
// A proved no-effect operation must still drive the original failed
// finalizer. Network reservations, snapshots, firewall revisions and
// catalog replicas all have PHP-owned pending/uncertain ledger state
// which only that finalizer can roll back safely.
return self::prepareResolution(
$jobId,
$operation,
$evidenceSource,
'not_applied'
);
}
/**
* Publish a prepared no-effect conclusion after the matching Agent fence
* has been durably released, then erase the encrypted recovery escrow.
*/
public static function commitNotApplied(
int $jobId,
array $operation,
string $evidenceSource,
?int $resolvedBy = null,
?CloudAgentCompletionEvent $completionEvent = null
): CloudJob {
return self::commitResolution(
$jobId,
$operation,
$evidenceSource,
'not_applied',
$resolvedBy,
$completionEvent
);
}
/** Prepare one validated resolution without publishing its effect. */
private static function prepareResolution(
int $jobId,
array $operation,
string $evidenceSource,
string $resolution
): CloudJob {
$job = CloudJob::find($jobId);
if ($job === null) {
throw new CloudError('任务不存在', 404);
}
self::assertRecoverableNotAppliedEvidence(
self::assertRecoverableEvidence(
$job,
$operation,
$evidenceSource
$evidenceSource,
$resolution
);
$marker = self::transitionPreparedMarker(
(string) $job->effect_resolution_reason,
self::preparedMarker($operation, $evidenceSource),
$evidenceSource,
'not_applied'
$resolution
);
// A proved no-effect operation must still drive the original failed
// finalizer. Network reservations, snapshots, firewall revisions and
// catalog replicas all have PHP-owned pending/uncertain ledger state
// which only that finalizer can roll back safely.
return (new CloudWorkerService())->prepareRecoveredEffect(
$jobId,
$operation,
@ -203,17 +131,16 @@ final class CloudOperationRecoveryService
);
}
/**
* Publish a prepared no-effect conclusion after the matching Agent fence
* has been durably released, then erase the encrypted recovery escrow.
*/
public static function commitNotApplied(
/** Publish one prepared resolution through the shared locked commit seam. */
private static function commitResolution(
int $jobId,
array $operation,
string $evidenceSource,
?int $resolvedBy = null,
?CloudAgentCompletionEvent $completionEvent = null
string $resolution,
?int $resolvedBy,
?CloudAgentCompletionEvent $completionEvent
): CloudJob {
self::assertResolution($resolution);
self::assertResolvedBy($resolvedBy);
$aggregateParentId = (int) (CloudJob::where('id', $jobId)
->value('parent_job_id') ?? 0);
@ -222,6 +149,7 @@ final class CloudOperationRecoveryService
$jobId,
$operation,
$evidenceSource,
$resolution,
$resolvedBy,
$completionEvent,
$aggregateParentId
@ -259,7 +187,11 @@ final class CloudOperationRecoveryService
}
self::assertLockedAgentRecoveryEnvelope($job);
self::assertEvidenceSource($evidenceSource);
self::assertNotAppliedOperationIdentity($job, $operation);
self::assertOperationIdentity(
$job,
$operation,
$resolution
);
$lockedEvent = self::lockLateReceiptEvent(
$job,
$completionEvent,
@ -272,7 +204,7 @@ final class CloudOperationRecoveryService
$completionEvent
);
$finalReason = $evidenceSource;
if ((string) $job->effect_resolution === 'not_applied'
if ((string) $job->effect_resolution === $resolution
&& hash_equals(
$finalReason,
(string) $job->effect_resolution_reason
@ -280,10 +212,11 @@ final class CloudOperationRecoveryService
self::markLateReceiptApplied($job, $lockedEvent);
return $job;
}
self::assertRecoverableNotAppliedEvidence(
self::assertRecoverableEvidence(
$job,
$operation,
$evidenceSource
$evidenceSource,
$resolution
);
$marker = self::preparedMarker(
$operation,
@ -295,7 +228,7 @@ final class CloudOperationRecoveryService
)) {
throw new CloudError('任务效果尚未完成账本预提交', 409);
}
$job->effect_resolution = 'not_applied';
$job->effect_resolution = $resolution;
$job->effect_resolved_at = time();
$job->effect_resolved_by = $resolvedBy;
$job->effect_resolution_reason = $finalReason;
@ -974,6 +907,48 @@ final class CloudOperationRecoveryService
&& !(bool) $event->safe_to_resubmit));
}
private static function assertResolution(string $resolution): void
{
if (!in_array($resolution, ['applied', 'not_applied'], true)) {
throw new \LogicException('Unsupported recovery resolution.');
}
}
private static function assertRecoverableEvidence(
CloudJob $job,
array $operation,
string $evidenceSource,
string $resolution
): void {
self::assertResolution($resolution);
if ($resolution === 'applied') {
self::assertRecoverableAppliedEvidence(
$job,
$operation,
$evidenceSource
);
return;
}
self::assertRecoverableNotAppliedEvidence(
$job,
$operation,
$evidenceSource
);
}
private static function assertOperationIdentity(
CloudJob $job,
array $operation,
string $resolution
): void {
self::assertResolution($resolution);
if ($resolution === 'applied') {
self::assertAppliedOperationIdentity($job, $operation);
return;
}
self::assertNotAppliedOperationIdentity($job, $operation);
}
private static function assertRecoverableAppliedEvidence(
CloudJob $job,
array $operation,

@ -70,22 +70,81 @@ $assert($ordered($finish, [
'CloudTemplateCatalogService::finishJobLocked(',
]), 'Worker completion does not lock parent, catalog, child, replica in order.');
foreach (['commitApplied', 'commitNotApplied'] as $name) {
$commit = $method(
foreach ([
'prepareApplied' => [
'end' => 'public static function commitApplied(',
'resolution' => 'applied',
],
'prepareNotApplied' => [
'end' => 'public static function commitNotApplied(',
'resolution' => 'not_applied',
],
] as $name => $contract) {
$wrapper = $method(
$recovery,
'public static function ' . $name . '(',
$contract['end']
);
$assert($ordered($wrapper, [
'return self::prepareResolution(',
"'" . $contract['resolution'] . "'",
]), $name . ' does not preserve its public resolution adapter.');
}
foreach ([
'commitApplied' => 'applied',
'commitNotApplied' => 'not_applied',
] as $name => $resolution) {
$wrapper = $method(
$recovery,
'public static function ' . $name . '(',
$name === 'commitApplied'
? 'public static function prepareNotApplied('
: 'private static function assertResolvedBy('
: 'private static function prepareResolution('
);
$assert($ordered($commit, [
'$aggregateParent = CloudJob::where(',
'$aggregateParentId',
'CloudTemplateCatalogService::lockAggregateCatalogLocked(',
'CloudUserIsoService::lockAggregateCatalogLocked(',
"CloudJob::where('id', \$jobId)",
]), $name . ' takes a child lock before its catalog lock.');
$assert($ordered($wrapper, [
'return self::commitResolution(',
"'" . $resolution . "'",
'$resolvedBy',
'$completionEvent',
]), $name . ' does not preserve its public resolution adapter.');
}
$commit = $method(
$recovery,
'private static function commitResolution(',
'public static function settleLateReceipt('
);
$assert($ordered($commit, [
'$aggregateParent = CloudJob::where(',
'$aggregateParentId',
'CloudTemplateCatalogService::lockAggregateCatalogLocked(',
'CloudUserIsoService::lockAggregateCatalogLocked(',
"CloudJob::where('id', \$jobId)",
]), 'Shared recovery commit does not lock parent, catalog, then child.');
$assert($ordered($commit, [
'self::assertLockedAgentRecoveryEnvelope(',
'self::assertEvidenceSource(',
'self::assertOperationIdentity(',
'self::lockLateReceiptEvent(',
'self::assertRecoveryCommitMayPublish(',
'self::assertRecoverableEvidence(',
'self::preparedMarker(',
'self::preparedMarkerMatches(',
'$job->effect_resolution = $resolution;',
'$job->sealed_payload = self::finalizedRecoveryPayload($job);',
'$job->save();',
'CloudJobService::reconcileAggregateParentEffectLocked(',
]), 'Shared recovery commit changed validation or ledger publish order.');
$commitSave = strpos($commit, '$job->save();');
$commitReconcile = strpos(
$commit,
'CloudJobService::reconcileAggregateParentEffectLocked('
);
$commitAck = strrpos($commit, 'self::markLateReceiptApplied(');
$assert($commitSave !== false && $commitReconcile !== false
&& $commitAck !== false && $commitSave < $commitReconcile
&& $commitReconcile < $commitAck,
'Shared recovery commit ACKs a receipt before ledger convergence.');
$automaticCommit = $method(
$automatic,

@ -0,0 +1,98 @@
<?php
declare(strict_types=1);
require dirname(__DIR__) . '/vendor/autoload.php';
use App\Services\Cloud\CloudOperationRecoveryContract;
use App\Services\Cloud\CloudOperationRecoveryIdentity;
use App\Services\Cloud\CloudOperationRecoveryPolicy;
$assert = static function (bool $condition, string $message): void {
if (!$condition) {
throw new RuntimeException($message);
}
};
$contract = json_decode(file_get_contents(
dirname(__DIR__, 2) . '/agent/operation/recovery_contract.json'
), true, 512, JSON_THROW_ON_ERROR);
$assert($contract['schema_version'] === 1, 'Unsupported recovery contract version.');
$expected = [];
foreach ($contract['operations'] as $entry) {
$kind = $entry['kind'];
$assert(!isset($expected[$kind]), 'Duplicate recovery kind: ' . $kind);
unset($entry['kind']);
$expected[$kind] = $entry;
}
$assert(CloudOperationRecoveryContract::OPERATIONS === $expected,
'PHP/Agent recovery inventory drift: run php tools/generate-recovery-contract.php');
$generic = [];
$automatic = [];
foreach ($expected as $kind => $entry) {
$mode = $entry['recovery_mode'];
if ($mode === 'generic') {
$generic[] = $kind;
$assert($entry['system_resolutions'] === ['applied', 'not_applied'],
'A generic leaf lost a recovery direction: ' . $kind);
}
if (in_array($mode, ['generic', 'create'], true)) {
$automatic[] = $kind;
}
$assert(CloudOperationRecoveryPolicy::supports($kind) === ($mode === 'generic'),
'Generic policy mode differs from inventory: ' . $kind);
$assert(CloudOperationRecoveryIdentity::supports($kind)
=== ($entry['identity_scope'] !== 'none'),
'Identity support differs from inventory: ' . $kind);
foreach (['applied', 'not_applied', 'unknown', ''] as $resolution) {
$assert(CloudOperationRecoveryPolicy::allowsSystemResolution($kind, $resolution)
=== in_array($resolution, $entry['system_resolutions'], true),
'System resolution differs from inventory: ' . $kind);
}
foreach ($entry['actions'] as $action) {
$assert(CloudOperationRecoveryPolicy::allowsAction($kind, $action),
'Declared wire action is not recognized: ' . $kind . '/' . $action);
$assert(!CloudOperationRecoveryPolicy::allowsAction($kind, $action . ':future')
&& !CloudOperationRecoveryPolicy::allowsAction($kind, $action . "\n"),
'Action matching is not exact: ' . $kind);
}
}
$assert(CloudOperationRecoveryPolicy::kinds() === $generic,
'Generic selector does not follow the inventory.');
$assert(CloudOperationRecoveryPolicy::automaticKinds() === $automatic,
'Automatic selector does not follow the inventory.');
$assert($expected['create']['recovery_mode'] === 'create'
&& $expected['create']['system_resolutions'] === ['not_applied']
&& !CloudOperationRecoveryPolicy::supports('create'),
'Create must retain dedicated finance recovery and one-way fence resolution.');
foreach (['host_storage_configure', 'payment_create', 'template_import',
'template_metadata_sync', 'template_delete', 'user_iso_import', 'user_iso_delete'] as $kind) {
$assert($expected[$kind]['system_resolutions'] === []
&& !in_array($kind, $automatic, true),
'A separately finalized operation gained system recovery: ' . $kind);
}
foreach (['', 'future_kind'] as $kind) {
$assert(!CloudOperationRecoveryPolicy::supports($kind)
&& !CloudOperationRecoveryIdentity::supports($kind)
&& !CloudOperationRecoveryPolicy::allowsAction($kind, 'create-vm'),
'Unknown kinds must be denied.');
}
$key = str_repeat('ab', 16);
foreach ([
['snapshot_create', 'create-snapshot:' . $key . ':filesystem'],
['snapshot_create', 'create-snapshot:' . $key . ':crash'],
['snapshot_restore', 'restore-snapshot:' . $key],
['snapshot_delete', 'delete-snapshot:' . $key],
] as [$kind, $action]) {
$assert(CloudOperationRecoveryPolicy::allowsAction($kind, $action),
'Valid parameterized snapshot action was rejected.');
foreach ([$action . "\n", $action . ':extra', strtoupper($action),
str_replace($key, substr($key, 1), $action)] as $invalid) {
$assert(!CloudOperationRecoveryPolicy::allowsAction($kind, $invalid),
'Malformed parameterized snapshot action was accepted.');
}
$assert(!CloudOperationRecoveryPolicy::allowsAction('instance_settings', $action),
'An action can escape its owning kind.');
}
echo "PHP/Agent generated recovery inventory contract passed.\n";

File diff suppressed because it is too large Load Diff

@ -144,9 +144,9 @@ $assert(str_contains($completion, "(string) \$job->phase === 'terminal'")
&& str_contains($operationRecovery,
"(string) \$job->outcome === 'failed'")
&& str_contains($operationRecovery,
"\$job->effect_resolution = 'applied'")
"if (!in_array(\$resolution, ['applied', 'not_applied'], true))")
&& str_contains($operationRecovery,
"\$job->effect_resolution = 'not_applied'")
'$job->effect_resolution = $resolution;')
&& !str_contains($operationRecovery,
"\$job->outcome = 'succeeded'")
&& str_contains($completion, "['succeeded', 'failed']")

Loading…
Cancel
Save