diff --git a/agent/api/server.go b/agent/api/server.go
index 6f3c6155..d536d050 100644
--- a/agent/api/server.go
+++ b/agent/api/server.go
@@ -1083,6 +1083,7 @@ func (s *Server) routeOperations(w http.ResponseWriter, r *http.Request, segment
return
}
}
+ applyExecutionFencedCreateInspection(&inspection, stored)
// Identity always comes from the immutable Journal record, never from a
// hypervisor adapter or its observation of current host state.
inspection.OperationID = stored.OperationID
@@ -1147,6 +1148,43 @@ func (s *Server) routeOperations(w http.ResponseWriter, r *http.Request, segment
writeError(w, http.StatusBadRequest, "invalid_fence_resolution", err.Error(), nil)
return
}
+ if actor.SystemRecovery && actor.UserID == 0 && stored.Action == "create-vm" {
+ if request.Resolution != operation.FenceResolutionNotApplied {
+ writeError(w, http.StatusConflict, "operation_fence_not_resolvable", "automatic create recovery can only confirm that creation was not applied", nil)
+ return
+ }
+ if stored.ExecutionFenced {
+ writeError(w, http.StatusConflict, "create_absence_not_confirmed", "automatic create recovery cannot conclude while the original worker may still be running", nil)
+ return
+ }
+ fenceState, err := s.Journal.FenceState(operationID)
+ if err != nil {
+ writeMappedError(w, err, nil)
+ return
+ }
+ if fenceState == operation.FenceStateActive {
+ _, incidentFound, err := s.Journal.CompletionDeliveryIncident(operationID)
+ if err != nil {
+ writeMappedError(w, err, nil)
+ return
+ }
+ if incidentFound {
+ writeError(w, http.StatusConflict, "create_absence_not_confirmed", "automatic create recovery requires fresh complete host absence evidence", nil)
+ return
+ }
+ ctx, cancel := context.WithTimeout(r.Context(), s.OperationTimeout)
+ inspection, inspectErr := s.Hypervisor.InspectOperation(ctx, stored)
+ cancel()
+ if inspectErr != nil {
+ writeMappedError(w, inspectErr, nil)
+ return
+ }
+ if !systemCreateAbsenceConfirmed(stored, request, inspection) {
+ writeError(w, http.StatusConflict, "create_absence_not_confirmed", "automatic create recovery requires fresh complete host absence evidence", nil)
+ return
+ }
+ }
+ }
resolution, replayed, err := s.Journal.ResolveResourceFence(operationID, actor.UserID, request)
switch {
case errors.Is(err, operation.ErrNotFound):
@@ -1514,6 +1552,55 @@ func applyCompletionDeliveryIncident(inspection *hypervisor.OperationInspection,
inspection.CompletionDeliveryErrorCode = incident.ErrorCode
}
+func applyExecutionFencedCreateInspection(inspection *hypervisor.OperationInspection, record operation.Record) {
+ if inspection == nil || record.Action != "create-vm" || !record.ExecutionFenced {
+ return
+ }
+ inspection.Outcome = hypervisor.InspectionOutcomeUncertain
+ inspection.SummaryCode = hypervisor.InspectionSummaryIncomplete
+ inspection.ErrorCode = ""
+ if record.Error != nil {
+ inspection.ErrorCode = record.Error.Code
+ }
+ inspection.EffectResolution = hypervisor.InspectionEffectInconclusive
+ inspection.EffectProjection = nil
+ inspection.AbortJob = hypervisor.OperationAbortInspection{}
+ inspection.Cleanup = hypervisor.OperationCleanupPlan{}
+}
+
+// systemCreateAbsenceConfirmed accepts only a fresh, exact read-only host
+// observation that both the VM and all panel-managed creation artifacts are
+// absent. It does not infer absence from the original operation error and does
+// not authorize any create replay or host mutation.
+func systemCreateAbsenceConfirmed(record operation.Record, request operation.FenceResolutionRequest,
+ inspection hypervisor.OperationInspection) bool {
+ if record.Action != "create-vm" || record.State != operation.StateFailed ||
+ inspection.OperationID != record.OperationID || inspection.Action != record.Action ||
+ inspection.ResourceID != record.ResourceID || inspection.RequestFingerprint != record.Fingerprint ||
+ inspection.OperationStatus != record.State || inspection.CheckedAt < request.InspectionCheckedAt ||
+ inspection.Outcome != hypervisor.InspectionOutcomeFailed ||
+ inspection.SummaryCode != hypervisor.InspectionSummaryFailed ||
+ inspection.ErrorCode != "creation_absent" ||
+ inspection.EffectResolution != hypervisor.InspectionEffectNotApplied ||
+ inspection.CompletionDeliveryState != "" || inspection.CompletionDeliveryErrorCode != "" ||
+ inspection.AbortJob.Available || inspection.Cleanup.Available ||
+ len(inspection.EffectProjection) == 0 || len(inspection.EffectProjection) > 1024 {
+ return false
+ }
+ var projection map[string]json.RawMessage
+ if err := json.Unmarshal(inspection.EffectProjection, &projection); err != nil || len(projection) != 2 {
+ return false
+ }
+ var creationAbsent, managedArtifactsAbsent bool
+ if err := json.Unmarshal(projection["creation_absent"], &creationAbsent); err != nil || !creationAbsent {
+ return false
+ }
+ if err := json.Unmarshal(projection["managed_artifacts_absent"], &managedArtifactsAbsent); err != nil || !managedArtifactsAbsent {
+ return false
+ }
+ return true
+}
+
func (s *Server) releaseAsync(operationID string) {
s.asyncMu.Lock()
defer s.asyncMu.Unlock()
diff --git a/agent/api/server_test.go b/agent/api/server_test.go
index 1e35ac7d..e260b90f 100644
--- a/agent/api/server_test.go
+++ b/agent/api/server_test.go
@@ -23,9 +23,11 @@ import (
type testHypervisor struct {
hypervisor.Unsupported
- mu sync.Mutex
- inspections int
- configureCalls int
+ mu sync.Mutex
+ inspections int
+ configureCalls int
+ inspectionResult *hypervisor.OperationInspection
+ inspectionErr error
}
func (h *testHypervisor) Ping(context.Context) error { return nil }
@@ -43,8 +45,16 @@ func (h *testHypervisor) ConfigureStorage(context.Context, hypervisor.StorageCon
func (h *testHypervisor) InspectOperation(_ context.Context, record operation.Record) (hypervisor.OperationInspection, error) {
h.mu.Lock()
+ defer h.mu.Unlock()
h.inspections++
- h.mu.Unlock()
+ if h.inspectionErr != nil {
+ return hypervisor.OperationInspection{}, h.inspectionErr
+ }
+ if h.inspectionResult != nil {
+ inspection := *h.inspectionResult
+ inspection.EffectProjection = append(json.RawMessage(nil), inspection.EffectProjection...)
+ return inspection, nil
+ }
return hypervisor.OperationInspection{
OperationID: record.OperationID,
OperationStatus: record.State,
@@ -548,6 +558,7 @@ func TestSystemRecoveryActorCanOnlyInspectAndResolveItsExactOperation(t *testing
{"raw operation", http.MethodGet, "/v1/operations/" + operationID, nil},
{"other operation", http.MethodGet, "/v1/operations/operation-other-recovery/inspect", nil},
{"host mutation", http.MethodPost, "/v1/storage/configuration", []byte(`{"operation_id":"operation-system-recovery"}`)},
+ {"create mutation", http.MethodPost, "/v1/vms", []byte(`{"operation_id":"operation-system-recovery"}`)},
} {
t.Run(test.name, func(t *testing.T) {
response := api.request(t, test.method, test.target, test.body, actor)
@@ -571,6 +582,260 @@ func TestSystemRecoveryActorCanOnlyInspectAndResolveItsExactOperation(t *testing
}
}
+func TestSystemCreateRecoveryRequiresFreshCompleteAbsenceEvidence(t *testing.T) {
+ api := newTestAPI(t)
+ operationID := "operation-system-create-recovery"
+ resourceID := "123e4567-e89b-42d3-a456-426614174000"
+ requestBody := []byte(`{"operation_id":"operation-system-create-recovery"}`)
+ spec := operation.Spec{
+ OperationID: operationID,
+ ResourceID: resourceID,
+ Action: "create-vm",
+ Fingerprint: operation.Fingerprint(http.MethodPost, "/v1/vms", requestBody),
+ Destructive: true,
+ }
+ if _, _, err := api.journal.Execute(context.Background(), spec, func(context.Context) (json.RawMessage, error) {
+ return nil, &worker.Error{
+ Code: "result_unconfirmed", Message: "create result was not confirmed",
+ EffectMayHaveApplied: true,
+ }
+ }); err == nil {
+ t.Fatal("seed create failure unexpectedly succeeded")
+ }
+ actor := auth.Actor{UserID: 0, SystemRecovery: true, OperationID: operationID}
+ request := operation.FenceResolutionRequest{
+ ResolutionID: "40404040404040404040404040404040",
+ Resolution: operation.FenceResolutionApplied,
+ InspectionCheckedAt: api.now.Unix(),
+ ReasonSHA256: "cdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcd",
+ Action: spec.Action,
+ ResourceID: spec.ResourceID,
+ RequestFingerprint: spec.Fingerprint,
+ }
+ encodeRequest := func() []byte {
+ t.Helper()
+ body, err := json.Marshal(request)
+ if err != nil {
+ t.Fatal(err)
+ }
+ return body
+ }
+ assertActive := func() {
+ t.Helper()
+ state, err := api.journal.FenceState(operationID)
+ if err != nil || state != operation.FenceStateActive {
+ t.Fatalf("create fence state=%q err=%v", state, err)
+ }
+ }
+
+ applied := api.request(
+ t,
+ http.MethodPost,
+ "/v1/operations/"+operationID+"/resolve-fence",
+ encodeRequest(),
+ actor,
+ )
+ if applied.Code != http.StatusConflict {
+ t.Fatalf("system create applied resolution = %d %s", applied.Code, applied.Body.String())
+ }
+ assertActive()
+
+ request.Resolution = operation.FenceResolutionNotApplied
+ inconclusive := api.request(
+ t,
+ http.MethodPost,
+ "/v1/operations/"+operationID+"/resolve-fence",
+ encodeRequest(),
+ actor,
+ )
+ if inconclusive.Code != http.StatusConflict ||
+ !bytes.Contains(inconclusive.Body.Bytes(), []byte(`"code":"create_absence_not_confirmed"`)) {
+ t.Fatalf("inconclusive create resolution = %d %s", inconclusive.Code, inconclusive.Body.String())
+ }
+ assertActive()
+
+ exactInspection := func(checkedAt int64) hypervisor.OperationInspection {
+ return hypervisor.OperationInspection{
+ OperationID: operationID,
+ Action: spec.Action,
+ ResourceID: resourceID,
+ RequestFingerprint: spec.Fingerprint,
+ OperationStatus: operation.StateFailed,
+ CheckedAt: checkedAt,
+ Outcome: hypervisor.InspectionOutcomeFailed,
+ SummaryCode: hypervisor.InspectionSummaryFailed,
+ ErrorCode: "creation_absent",
+ EffectResolution: hypervisor.InspectionEffectNotApplied,
+ EffectProjection: json.RawMessage(
+ `{"creation_absent":true,"managed_artifacts_absent":true}`,
+ ),
+ }
+ }
+ stale := exactInspection(api.now.Add(-time.Second).Unix())
+ api.hv.mu.Lock()
+ api.hv.inspectionResult = &stale
+ api.hv.mu.Unlock()
+ staleResponse := api.request(
+ t,
+ http.MethodPost,
+ "/v1/operations/"+operationID+"/resolve-fence",
+ encodeRequest(),
+ actor,
+ )
+ if staleResponse.Code != http.StatusConflict ||
+ !bytes.Contains(staleResponse.Body.Bytes(), []byte(`"code":"create_absence_not_confirmed"`)) {
+ t.Fatalf("stale create resolution = %d %s", staleResponse.Code, staleResponse.Body.String())
+ }
+ assertActive()
+
+ incomplete := exactInspection(api.now.Unix())
+ incomplete.EffectProjection = json.RawMessage(`{"creation_absent":true,"managed_artifacts_absent":false}`)
+ api.hv.mu.Lock()
+ api.hv.inspectionResult = &incomplete
+ api.hv.mu.Unlock()
+ incompleteResponse := api.request(
+ t,
+ http.MethodPost,
+ "/v1/operations/"+operationID+"/resolve-fence",
+ encodeRequest(),
+ actor,
+ )
+ if incompleteResponse.Code != http.StatusConflict ||
+ !bytes.Contains(incompleteResponse.Body.Bytes(), []byte(`"code":"create_absence_not_confirmed"`)) {
+ t.Fatalf("incomplete create absence = %d %s", incompleteResponse.Code, incompleteResponse.Body.String())
+ }
+ assertActive()
+
+ exact := exactInspection(api.now.Unix())
+ api.hv.mu.Lock()
+ api.hv.inspectionResult = &exact
+ api.hv.mu.Unlock()
+ inspection := api.request(
+ t,
+ http.MethodGet,
+ "/v1/operations/"+operationID+"/inspect",
+ nil,
+ actor,
+ )
+ if inspection.Code != http.StatusOK ||
+ !bytes.Contains(inspection.Body.Bytes(), []byte(`"error_code":"creation_absent"`)) ||
+ !bytes.Contains(inspection.Body.Bytes(), []byte(`"effect_resolution":"not_applied"`)) {
+ t.Fatalf("system create inspection = %d %s", inspection.Code, inspection.Body.String())
+ }
+ resolved := api.request(
+ t,
+ http.MethodPost,
+ "/v1/operations/"+operationID+"/resolve-fence",
+ encodeRequest(),
+ actor,
+ )
+ if resolved.Code != http.StatusOK ||
+ !bytes.Contains(resolved.Body.Bytes(), []byte(`"resolution":"not_applied"`)) ||
+ !bytes.Contains(resolved.Body.Bytes(), []byte(`"resolved_by":0`)) {
+ t.Fatalf("system create not-applied resolution = %d %s", resolved.Code, resolved.Body.String())
+ }
+ replayed := api.request(
+ t,
+ http.MethodPost,
+ "/v1/operations/"+operationID+"/resolve-fence",
+ encodeRequest(),
+ actor,
+ )
+ if replayed.Code != http.StatusOK ||
+ !bytes.Contains(replayed.Body.Bytes(), []byte(`"resolution":"not_applied"`)) ||
+ !bytes.Contains(replayed.Body.Bytes(), []byte(`"replayed":true`)) {
+ t.Fatalf("system create resolution replay = %d %s", replayed.Code, replayed.Body.String())
+ }
+ api.hv.mu.Lock()
+ inspections := api.hv.inspections
+ configureCalls := api.hv.configureCalls
+ api.hv.mu.Unlock()
+ if inspections != 5 || configureCalls != 0 {
+ t.Fatalf("system create recovery inspections=%d mutations=%d", inspections, configureCalls)
+ }
+}
+
+func TestSystemCreateRecoveryRejectsExecutionFencedAbsence(t *testing.T) {
+ api := newTestAPI(t)
+ operationID := "operation-system-create-unreaped"
+ resourceID := "123e4567-e89b-42d3-a456-426614174002"
+ spec := operation.Spec{
+ OperationID: operationID,
+ ResourceID: resourceID,
+ Action: "create-vm",
+ Fingerprint: operation.Fingerprint(http.MethodPost, "/v1/vms", []byte(`{"operation_id":"operation-system-create-unreaped"}`)),
+ Destructive: true,
+ }
+ if _, _, err := api.journal.Execute(context.Background(), spec, func(context.Context) (json.RawMessage, error) {
+ return nil, &worker.Error{
+ Code: "worker_reap_unconfirmed", Message: "worker may still be running",
+ EffectMayHaveApplied: true, ExecutionFenced: true,
+ }
+ }); err == nil {
+ t.Fatal("seed execution-fenced create unexpectedly succeeded")
+ }
+ exactAbsence := hypervisor.OperationInspection{
+ OperationID: operationID,
+ Action: spec.Action,
+ ResourceID: resourceID,
+ RequestFingerprint: spec.Fingerprint,
+ OperationStatus: operation.StateFailed,
+ CheckedAt: api.now.Unix(),
+ Outcome: hypervisor.InspectionOutcomeFailed,
+ SummaryCode: hypervisor.InspectionSummaryFailed,
+ ErrorCode: "creation_absent",
+ EffectResolution: hypervisor.InspectionEffectNotApplied,
+ EffectProjection: json.RawMessage(`{"creation_absent":true,"managed_artifacts_absent":true}`),
+ }
+ api.hv.mu.Lock()
+ api.hv.inspectionResult = &exactAbsence
+ api.hv.mu.Unlock()
+ actor := auth.Actor{UserID: 0, SystemRecovery: true, OperationID: operationID}
+
+ inspected := api.request(t, http.MethodGet, "/v1/operations/"+operationID+"/inspect", nil, actor)
+ if inspected.Code != http.StatusOK {
+ t.Fatalf("execution-fenced create inspection = %d %s", inspected.Code, inspected.Body.String())
+ }
+ var inspection hypervisor.OperationInspection
+ if err := json.Unmarshal(inspected.Body.Bytes(), &inspection); err != nil {
+ t.Fatal(err)
+ }
+ if inspection.Outcome != hypervisor.InspectionOutcomeUncertain ||
+ inspection.SummaryCode != hypervisor.InspectionSummaryIncomplete ||
+ inspection.EffectResolution != hypervisor.InspectionEffectInconclusive ||
+ len(inspection.EffectProjection) != 0 {
+ t.Fatalf("execution-fenced create absence was treated as final: %#v", inspection)
+ }
+
+ request := operation.FenceResolutionRequest{
+ ResolutionID: "50505050505050505050505050505050",
+ Resolution: operation.FenceResolutionNotApplied,
+ InspectionCheckedAt: api.now.Unix(),
+ ReasonSHA256: "dededededededededededededededededededededededededededededededede",
+ Action: spec.Action,
+ ResourceID: spec.ResourceID,
+ RequestFingerprint: spec.Fingerprint,
+ }
+ body, err := json.Marshal(request)
+ if err != nil {
+ t.Fatal(err)
+ }
+ resolved := api.request(t, http.MethodPost, "/v1/operations/"+operationID+"/resolve-fence", body, actor)
+ if resolved.Code != http.StatusConflict ||
+ !bytes.Contains(resolved.Body.Bytes(), []byte(`"code":"create_absence_not_confirmed"`)) {
+ t.Fatalf("execution-fenced create resolution = %d %s", resolved.Code, resolved.Body.String())
+ }
+ if state, err := api.journal.FenceState(operationID); err != nil || state != operation.FenceStateActive {
+ t.Fatalf("execution-fenced create was released state=%q err=%v", state, err)
+ }
+ api.hv.mu.Lock()
+ inspections := api.hv.inspections
+ api.hv.mu.Unlock()
+ if inspections != 1 {
+ t.Fatalf("execution-fenced create resolve repeated host inspection: %d", inspections)
+ }
+}
+
func TestMutationUsesIsolatedExecutorExactlyOnce(t *testing.T) {
api := newTestAPI(t)
executor := &fakeMutationExecutor{result: json.RawMessage(`{"configured":true}`)}
diff --git a/agent/hypervisor/libvirt_operation_admin_test.go b/agent/hypervisor/libvirt_operation_admin_test.go
index a468e659..2b2c2e1a 100644
--- a/agent/hypervisor/libvirt_operation_admin_test.go
+++ b/agent/hypervisor/libvirt_operation_admin_test.go
@@ -648,6 +648,27 @@ func TestAbsentCreateTrafficResidueRequiresExplicitCleanup(t *testing.T) {
}
}
+func TestFailedCreateAlwaysRequiresLiveInspection(t *testing.T) {
+ for name, record := range map[string]operation.Record{
+ "ordinary failure without effect flag": {
+ OperationID: "operation-create-ordinary-failure", ResourceID: "123e4567-e89b-42d3-a456-426614174000",
+ Action: "create-vm", State: operation.StateFailed,
+ Error: &operation.PublicError{Code: "operation_failed", Message: "create failed"},
+ },
+ "unknown effect failure": {
+ OperationID: "operation-create-unknown-effect", ResourceID: "123e4567-e89b-42d3-a456-426614174001",
+ Action: "create-vm", State: operation.StateFailed, EffectMayHaveApplied: true,
+ Error: &operation.PublicError{Code: "result_unconfirmed", Message: "create result was not confirmed"},
+ },
+ } {
+ t.Run(name, func(t *testing.T) {
+ if storedOperationInspectionAvailable(record) {
+ t.Fatal("failed create reused its original Journal error instead of requiring live host inspection")
+ }
+ })
+ }
+}
+
func TestBackupPushMatchesOnlyExactOwnedTarget(t *testing.T) {
target := "/var/lib/virt-cloud/managed/snapshot.qcow2"
matching := ``
diff --git a/agent/operation/journal.go b/agent/operation/journal.go
index fe987610..fc82b7bb 100644
--- a/agent/operation/journal.go
+++ b/agent/operation/journal.go
@@ -182,7 +182,8 @@ type Spec struct {
// leaf action must be reviewed and added here explicitly.
func SystemRecoveryActionAllowed(action string) bool {
switch action {
- case "power-start", "power-shutdown", "power-reboot", "power-reset", "power-destroy",
+ case "create-vm",
+ "power-start", "power-shutdown", "power-reboot", "power-reset", "power-destroy",
"reinstall-vm",
"enter-rescue", "force-start-rescue", "exit-rescue:connected", "exit-rescue:suspended",
"change-network", "network-suspend", "network-resume",
@@ -877,6 +878,17 @@ func (j *Journal) ResolveResourceFence(operationID string, resolvedBy int64,
if resolvedBy == 0 && !SystemRecoveryActionAllowed(record.Action) {
return ErrFenceNotResolvable
}
+ // Automatic create recovery is deliberately one-way: fresh host
+ // absence evidence may release a failed create fence as not applied,
+ // but the zero-ID system actor can never declare that a create applied.
+ // Nor may it release a fence while an unreaped worker could still create
+ // the resource after the inspection. Materializing an applied create
+ // requires the original immutable completion result and remains PHP's
+ // responsibility.
+ if resolvedBy == 0 && record.Action == "create-vm" &&
+ (record.ExecutionFenced || request.Resolution != FenceResolutionNotApplied) {
+ return ErrFenceNotResolvable
+ }
if encoded := resolutions.Get([]byte(operationID)); encoded != nil {
if err := json.Unmarshal(encoded, &resolution); err != nil || resolution.OperationID != operationID {
return fmt.Errorf("decode fence resolution %q", operationID)
diff --git a/agent/operation/journal_test.go b/agent/operation/journal_test.go
index 79d15c74..53462489 100644
--- a/agent/operation/journal_test.go
+++ b/agent/operation/journal_test.go
@@ -1445,8 +1445,97 @@ func TestSystemFenceResolutionUsesZeroAuditActorButRejectsNegativeActors(t *test
}
}
+func TestSystemCreateFenceResolutionOnlyAcceptsNotApplied(t *testing.T) {
+ j := openTestJournal(t)
+ now := time.Date(2026, 10, 21, 2, 0, 0, 0, time.UTC)
+ j.SetClockForTesting(func() time.Time { return now })
+ newCreate := func(operationID, resourceID string) Spec {
+ return Spec{
+ OperationID: operationID,
+ ResourceID: resourceID,
+ Action: "create-vm",
+ Fingerprint: Fingerprint("POST", "/v1/vms", []byte(`{"operation_id":"`+operationID+`"}`)),
+ Destructive: true,
+ }
+ }
+ seedUnknown := func(spec Spec) {
+ t.Helper()
+ if _, _, err := j.Execute(context.Background(), spec, func(context.Context) (json.RawMessage, error) {
+ return nil, testFailure{code: "result_unconfirmed", message: "create result is unknown", effect: true}
+ }); err == nil {
+ t.Fatal("seed create failure unexpectedly succeeded")
+ }
+ }
+ requestFor := func(spec Spec, resolutionID, resolution string) FenceResolutionRequest {
+ return FenceResolutionRequest{
+ ResolutionID: resolutionID,
+ Resolution: resolution,
+ InspectionCheckedAt: now.Unix(),
+ ReasonSHA256: "abababababababababababababababababababababababababababababababab",
+ Action: spec.Action,
+ ResourceID: spec.ResourceID,
+ RequestFingerprint: spec.Fingerprint,
+ }
+ }
+
+ systemSpec := newCreate("operation-system-create", "vm-system-create")
+ seedUnknown(systemSpec)
+ applied := requestFor(
+ systemSpec,
+ "10101010101010101010101010101010",
+ FenceResolutionApplied,
+ )
+ if _, _, err := j.ResolveResourceFence(systemSpec.OperationID, 0, applied); !errors.Is(err, ErrFenceNotResolvable) {
+ t.Fatalf("system create applied resolution err=%v", err)
+ }
+ if state, err := j.FenceState(systemSpec.OperationID); err != nil || state != FenceStateActive {
+ t.Fatalf("rejected system create resolution released fence state=%q err=%v", state, err)
+ }
+ notApplied := applied
+ notApplied.ResolutionID = "20202020202020202020202020202020"
+ notApplied.Resolution = FenceResolutionNotApplied
+ resolved, replayed, err := j.ResolveResourceFence(systemSpec.OperationID, 0, notApplied)
+ if err != nil || replayed || resolved.Resolution != FenceResolutionNotApplied || resolved.ResolvedBy != 0 {
+ t.Fatalf("system create not-applied resolution=%#v replayed=%v err=%v", resolved, replayed, err)
+ }
+
+ manualSpec := newCreate("operation-manual-create", "vm-manual-create")
+ seedUnknown(manualSpec)
+ manualApplied := requestFor(
+ manualSpec,
+ "30303030303030303030303030303030",
+ FenceResolutionApplied,
+ )
+ resolved, replayed, err = j.ResolveResourceFence(manualSpec.OperationID, 42, manualApplied)
+ if err != nil || replayed || resolved.Resolution != FenceResolutionApplied || resolved.ResolvedBy != 42 {
+ t.Fatalf("manual create applied resolution=%#v replayed=%v err=%v", resolved, replayed, err)
+ }
+
+ fencedSpec := newCreate("operation-system-create-unreaped", "vm-system-create-unreaped")
+ if _, _, err := j.Execute(context.Background(), fencedSpec, func(context.Context) (json.RawMessage, error) {
+ return nil, testFailure{
+ code: "worker_reap_unconfirmed", message: "worker may still be running",
+ effect: true, fence: true,
+ }
+ }); err == nil {
+ t.Fatal("seed execution-fenced create unexpectedly succeeded")
+ }
+ fencedRequest := requestFor(
+ fencedSpec,
+ "40404040404040404040404040404040",
+ FenceResolutionNotApplied,
+ )
+ if _, _, err := j.ResolveResourceFence(fencedSpec.OperationID, 0, fencedRequest); !errors.Is(err, ErrFenceNotResolvable) {
+ t.Fatalf("system execution-fenced create resolution err=%v", err)
+ }
+ if state, err := j.FenceState(fencedSpec.OperationID); err != nil || state != FenceStateActive {
+ t.Fatalf("execution-fenced create was released state=%q err=%v", state, err)
+ }
+}
+
func TestSystemRecoveryActionAllowlistIsExplicit(t *testing.T) {
allowed := []string{
+ "create-vm",
"power-start", "power-shutdown", "power-reboot", "power-reset", "power-destroy",
"reinstall-vm",
"enter-rescue", "force-start-rescue", "exit-rescue:connected", "exit-rescue:suspended",
@@ -1469,7 +1558,7 @@ func TestSystemRecoveryActionAllowlistIsExplicit(t *testing.T) {
}
denied := []string{
- "", "create-vm", "configure-storage", "service-resume", "future-unknown", "power-future",
+ "", "configure-storage", "service-resume", "future-unknown", "power-future",
"exit-rescue:future",
"create-snapshot:0123456789abcdef0123456789abcdef:future",
"create-snapshot:0123456789ABCDEF0123456789ABCDEF:filesystem",
diff --git a/docs/DEPLOYMENT.md b/docs/DEPLOYMENT.md
index a9ce5ec3..a9099d88 100644
--- a/docs/DEPLOYMENT.md
+++ b/docs/DEPLOYMENT.md
@@ -162,6 +162,8 @@ systemctl enable --now virt-cloud-worker.service \
普通 worker 负责购买、创建、重装、邮件和最终删除;至少两个策略 worker 处理到期断网、网络恢复与删除前策略;lifecycle 生成续费账单并推进账期、流量、宽限和删除状态。`virt-cloud-recovery-worker.service` 是必需的独立 operation 恢复进程,覆盖策略表中的实例电源、重装/救援、公网 IP、实例设置、快照、防火墙、删除、生命周期网络、模板与 ISO Agent 叶子任务。它只做 operation-bound inspection/栅栏审计、持久晚到回执的 PHP 收敛或聚合父任务的本地汇总,不投递邮件,也绝不重放原 mutation。离线、身份冲突或账本异常按固定低频间隔持续核对,没有次数或总时长上限。它与普通队列隔离,避免某个离线 Agent 的核对占住正常 worker。
+恢复进程也覆盖已派发开通任务的失败收尾:完整缺失证据回执可直接完成本地退款和资源释放;回执不足时,只读检查原 operation,确认实例及托管资源均不存在后才收尾。升级此能力需同时更新面板代码与宿主机 Agent,并重启已有 worker/recovery worker 使其加载新代码;无需数据库迁移或新增 service。旧 Agent 拒绝开通自动检查时会保留订单占用并低频重试,不会绕过授权退款。已有失败且效果未知的开通订单会被后台领取,不必重发开通请求。
+
在面板机器构建并安装 Agent Gateway:
```bash
diff --git a/docs/TASK_EXECUTION.md b/docs/TASK_EXECUTION.md
index 8b21eb13..973a56ae 100644
--- a/docs/TASK_EXECUTION.md
+++ b/docs/TASK_EXECUTION.md
@@ -51,7 +51,9 @@ Go context 不能可靠中止已经进入 CGo/libvirt 的调用,因此每个
完成回执是不可变事件,不是新的变更请求。Agent 持续投递同一 `event_id`,直到 PHP 返回 ACK;重启、旧 outbox 中保存的尝试次数或历史过期时间都不能淘汰未确认回执。PHP 以 `event_id`、`operation_id` 和正文摘要幂等接收。ACK 只删除投递 outbox,不删除主 Journal 记录。
-PHP 的 `receipt_deadline_at` 到期时直接把任务终态化为失败。之后收到相同回执不能更新原任务终态;除开通任务的精确成功回执专用路径外,不支持恢复的类型只返回晚到 ACK。对策略表允许的任务,精确匹配 `agent_id + operation_id + event_id + 正文摘要` 的成功回执会先以原加密请求收敛对应 PHP 业务账本,再提交 `effect_resolution=applied`;明确无副作用的失败回执则收敛为 `not_applied`。开通任务不进入通用 inspection、fence 或 mutation replay;只有已持久化且身份完全匹配的成功回执,才能在重新验证未退款订单、已支付开通发票、库存、IP、placement、服务周期和凭据 escrow 后原子补完本地账本。该回执路径只使用 PHP 已持久化事件,不联系 Agent;任务效果结论与事件 ACK 在同一事务提交,结论提交后立即清除恢复密文。如果进程在事件提交后或预提交后崩溃,专用 recovery worker 会从 terminal job 出发,以已有 `(host_id,operation_id)` 唯一索引精确读取 `received` 事件,重算正文摘要后继续 PHP 提交。它只有在既有 prepared 证据同方向、同完整 operation hash 且栅栏语义一致时才可继续收敛;方向、哈希、ID、栅栏语义、密文或账本状态冲突时,必须记录具体 `recovery_*` 异常并保留 blocker/escrow。异常事件可以被隔离,但任务仍按固定低频退避留在恢复队列,修复数据、身份、网络或 Agent 后继续核对;不存在 6 小时、12 次或其他永久退出上限。它不扫描事件全表,也不依赖 Agent 再次投递。相同回执重传只返回幂等 ACK;不同正文复用事件 ID 或 operation ID 必须拒绝并记录安全日志。
+PHP 的 `receipt_deadline_at` 到期时直接把任务终态化为失败。之后收到相同回执不能更新原任务终态;不支持恢复的类型只返回晚到 ACK。对策略表允许的任务,精确匹配 `agent_id + operation_id + event_id + 正文摘要` 的成功回执会先以原加密请求收敛对应 PHP 业务账本,再提交 `effect_resolution=applied`;明确无副作用的失败回执则收敛为 `not_applied`。回执路径只使用 PHP 已持久化事件,不联系 Agent;任务效果结论与事件 ACK 在同一事务提交,结论提交后立即清除恢复密文。如果进程在事件提交后或预提交后崩溃,专用 recovery worker 会从 terminal job 出发,以已有 `(host_id,operation_id)` 唯一索引精确读取 `received` 事件,重算正文摘要后继续 PHP 提交。它只有在既有 prepared 证据同方向、同完整 operation hash 且栅栏语义一致时才可继续收敛;方向、哈希、ID、栅栏语义、密文或账本状态冲突时,必须记录具体 `recovery_*` 异常并保留 blocker/escrow。异常事件可以被隔离,但任务仍按固定低频退避留在恢复队列,修复数据、身份、网络或 Agent 后继续核对;不存在 6 小时、12 次或其他永久退出上限。它不扫描事件全表,也不依赖 Agent 再次投递。相同回执重传只返回幂等 ACK;不同正文复用事件 ID 或 operation ID 必须拒绝并记录安全日志。
+
+开通任务使用独立的资金恢复终结器,不加入通用效果物化策略,也不重放创建请求。精确成功回执在重新验证未退款订单、已支付开通发票、库存、IP、placement、服务周期和凭据 escrow 后原子补完实例账本。晚到失败回执必须同时证明 `creation_absent=true`、`managed_artifacts_absent=true` 且没有未知副作用,才能原子退款、释放库存/IP/预留并 ACK;仅有失败状态或 `safe_to_resubmit` 不足以退款。没有完整失败回执时,恢复 worker 仍会领取该订单,使用绑定原 operation 的 system-recovery 身份只读检查;只有新鲜、身份一致且明确的 `creation_absent` 证据才进入专用预提交。预提交持久化证据但不提前退款,确认必要的 Agent 栅栏审计后再原子提交资金和资源账本。Agent 已提交栅栏但 PHP 提交失败时,后续轮次可依据同一预提交读取既有栅栏结论并继续,不会重新开通或重复退款。系统身份对开通只允许 `not_applied` 栅栏结论,不能凭只读状态推断成功并物化实例;残留资源、运行中操作、身份冲突、无效证据和矛盾成功回执继续保留保护。此能力需更新面板和 Agent,并保持 recovery worker 运行;不需要数据库迁移或新增 systemd 服务。
任何回执或 inspection 改变 PHP 账本前,都必须重新验证加密请求、任务/宿主机/资源身份,以及该种类终结器会消费的请求与结果字段;可解密但不完整的 escrow 不得以空账本假收敛。prepared 证据绑定规范化后的完整 operation;重试次数只属于调度状态,不参与 immutable operation hash,也不能破坏 prepare 后的崩溃续跑。如果恢复必须写 Agent 栅栏审计结论,该有界网络写入与随后的 PHP effect commit 必须在同一 job-row 锁下串行,写前再次验证租约、incident 和优先回执,防止陈旧 worker 在冲突证据落盘后解除 Agent fence。
@@ -63,7 +65,7 @@ PHP 的 `receipt_deadline_at` 到期时直接把任务终态化为失败。之
- 任务本身立即结束,不再轮询、不再自动重试;内部恢复协调器只能低频读取允许种类的同一 Agent operation 证据;
- 同一 VM、卷、网络过滤器、地址或账单的冲突变更立即返回明确错误,不进入等待队列;
-- 不自动退款、不释放 reservation、不再次删除磁盘;
+- 在可信清理证据确认前不自动退款、不释放 reservation、不再次删除磁盘;
- 管理员可以执行独立的只读检查,但检查不会翻转原任务终态,也不会续作原 mutation;
- 策略表中的 Agent 叶子任务可以自动消费同一 operation 的不可变证据:晚到终态回执是纯 PHP 收敛;没有回执时,由独立的 `cloudWorker --recovery-only` 进程每轮最多领取一项。常驻进程在精确 `received` 事件、普通叶子任务、聚合任务三个通道间轮转,空通道同轮跳过;持续有任务时,每三个完成的 tick 给每个通道一次机会,避免回执或叶子队列持续非空导致聚合任务饥饿。普通/策略 worker 从不联系恢复目标,因此离线宿主的核对不会阻塞正常派发。恢复使用绑定 operation ID 的 system-recovery actor 调用 Agent 只读 `inspect`,且只接受新鲜、种类专用、资源身份一致的 `applied/not_applied` 证据。它先预提交 PHP 账本并继续保留 blocker;只有活动失败栅栏才写入一次 operation-bound Journal 审计结论,再提交 `effect_resolution`。原 mutation 永不调用。`partial/inconclusive`、Agent 离线、ID 冲突、无效回执、密文损坏或账本/实例状态矛盾都会保留 blocker,向页面公开具体异常,并按固定低频退避持续重检;没有次数和总时长上限。修复根因后,同一 operation 可继续自动收敛并清除 incident;相互矛盾的持久证据不得被覆盖或猜测解锁;
- 超级管理员只能对已有种类专用原子终结器的任务提交审计结论。开通任务仅支持由新鲜宿主机缺失证据确认 `not_applied`,并在同一事务退款、释放库存/IP/预留后再解栅栏;支付任务只根据签名回调或外部支付平台审计证据收敛本地账本,不重新下单;
diff --git a/web/app/Services/Cloud/CloudAdminOperationRecoveryService.php b/web/app/Services/Cloud/CloudAdminOperationRecoveryService.php
index fc31fab0..180e5e77 100644
--- a/web/app/Services/Cloud/CloudAdminOperationRecoveryService.php
+++ b/web/app/Services/Cloud/CloudAdminOperationRecoveryService.php
@@ -184,9 +184,9 @@ final class CloudAdminOperationRecoveryService
[$job, $client] = self::jobContext(
$id,
30.0,
- CloudOperationRecoveryPolicy::kinds()
+ self::automaticRecoveryKinds()
);
- if (!CloudOperationRecoveryPolicy::supports((string) $job->kind)
+ if (!self::supportsAutomaticRecovery($job)
|| (string) $job->execution_mode !== 'agent'
|| (string) $job->phase !== 'terminal'
|| (string) $job->outcome !== 'failed'
@@ -238,8 +238,20 @@ final class CloudAdminOperationRecoveryService
[$job, $client] = self::jobContext(
$id,
30.0,
- CloudOperationRecoveryPolicy::kinds()
+ self::automaticRecoveryKinds()
);
+ if ((string) $job->kind === 'create') {
+ CloudCreateFailureRecoveryService::assertPreparedForFence(
+ $job,
+ $resolution,
+ $reason
+ );
+ CloudCreateFailureFinalizer::assertResolutionSupported(
+ $job,
+ $resolution,
+ $inspection
+ );
+ }
$checkedAt = $inspection['checked_at'] ?? null;
$identity = CloudOperationRecoveryIdentity::expected(
$job,
@@ -251,7 +263,7 @@ final class CloudAdminOperationRecoveryService
'request_fingerprint' =>
$inspection['request_fingerprint'] ?? null,
]);
- if (!CloudOperationRecoveryPolicy::supports((string) $job->kind)
+ if (!self::supportsAutomaticRecovery($job)
|| (string) $job->execution_mode !== 'agent'
|| (string) $job->phase !== 'terminal'
|| (string) $job->outcome !== 'failed'
@@ -260,9 +272,10 @@ final class CloudAdminOperationRecoveryService
|| !(bool) $job->effect_may_have_applied
|| (bool) $job->safe_to_resubmit
|| (string) $job->effect_resolution !== 'unknown'
- || !self::automaticPreparedReason(
- (string) $job->effect_resolution_reason
- )
+ || ((string) $job->kind !== 'create'
+ && !self::automaticPreparedReason(
+ (string) $job->effect_resolution_reason
+ ))
|| ($inspection['operation_id'] ?? null)
!== (string) $job->operation_id
|| ($inspection['effect_resolution'] ?? null) !== $resolution
@@ -316,9 +329,16 @@ final class CloudAdminOperationRecoveryService
[$job, $client] = self::jobContext(
$id,
30.0,
- CloudOperationRecoveryPolicy::kinds()
+ self::automaticRecoveryKinds()
);
- if (!CloudOperationRecoveryPolicy::supports((string) $job->kind)
+ if ((string) $job->kind === 'create') {
+ CloudCreateFailureRecoveryService::assertPreparedForFence(
+ $job,
+ $resolution,
+ $reason
+ );
+ }
+ if (!self::supportsAutomaticRecovery($job)
|| (string) $job->execution_mode !== 'agent'
|| (string) $job->phase !== 'terminal'
|| (string) $job->outcome !== 'failed'
@@ -327,9 +347,10 @@ final class CloudAdminOperationRecoveryService
|| !(bool) $job->effect_may_have_applied
|| (bool) $job->safe_to_resubmit
|| (string) $job->effect_resolution !== 'unknown'
- || !self::automaticPreparedReason(
- (string) $job->effect_resolution_reason
- )) {
+ || ((string) $job->kind !== 'create'
+ && !self::automaticPreparedReason(
+ (string) $job->effect_resolution_reason
+ ))) {
throw new CloudError('任务尚未完成可验证的账本预提交', 409);
}
// A prepared escrow persists the identity validated before the PHP
@@ -568,6 +589,18 @@ final class CloudAdminOperationRecoveryService
) === 1;
}
+ /** Create is inspectable, but never enters generic effect finalization. */
+ private static function automaticRecoveryKinds(): array
+ {
+ return [...CloudOperationRecoveryPolicy::kinds(), 'create'];
+ }
+
+ private static function supportsAutomaticRecovery(CloudJob $job): bool
+ {
+ return CloudOperationRecoveryPolicy::supports((string) $job->kind)
+ || ((string) $job->kind === 'create' && $job->order_id !== null);
+ }
+
private static function assertCompletionDeliveryUsable(
array $inspection
): void {
diff --git a/web/app/Services/Cloud/CloudAutomaticOperationRecoveryService.php b/web/app/Services/Cloud/CloudAutomaticOperationRecoveryService.php
index 8298383a..60c03d39 100644
--- a/web/app/Services/Cloud/CloudAutomaticOperationRecoveryService.php
+++ b/web/app/Services/Cloud/CloudAutomaticOperationRecoveryService.php
@@ -544,7 +544,10 @@ final class CloudAutomaticOperationRecoveryService
): void {
$policy->whereIn(
'kind',
- CloudOperationRecoveryPolicy::kinds()
+ // Create uses a separate financial
+ // finalizer, never the generic replay or
+ // effect-projection path below.
+ [...CloudOperationRecoveryPolicy::kinds(), 'create']
)->where(
static function ($ready) use (
$now,
@@ -580,21 +583,6 @@ final class CloudAutomaticOperationRecoveryService
}
);
}
- )->orWhere(
- static function ($create): void {
- // Create keeps its specialised financial
- // finalizer. It is recoverable here only from
- // a durable exact completion receipt; it must
- // never enter generic inspection or replay.
- $create->where('kind', 'create')
- ->whereExists(
- static function ($events): void {
- self::receivedCreateSuccessEventExists(
- $events
- );
- }
- );
- }
);
})
->when(
@@ -703,13 +691,6 @@ final class CloudAutomaticOperationRecoveryService
'operation_id',
(string) $job->operation_id
)->where('disposition', 'received')
- ->when(
- (string) $job->kind === 'create',
- static fn ($events) => $events->where(
- 'outcome',
- 'succeeded'
- )
- )
->lockForUpdate()
->first();
if ($event !== null) {
@@ -726,6 +707,16 @@ final class CloudAutomaticOperationRecoveryService
(string) $job->effect_resolution_reason,
$now
);
+ // The dedicated create coordinator owns its encrypted
+ // prepare/commit binding. Never wrap or replace that marker
+ // when resuming a fence-write/financial-commit crash window.
+ if ((string) $job->kind === 'create'
+ && preg_match(
+ '/^prepared:create_not_applied:v1:[a-f0-9]{64}$/D',
+ (string) $job->effect_resolution_reason
+ ) === 1) {
+ $attempt = (string) $job->effect_resolution_reason;
+ }
if ($attempt === false) {
$job->finalization_incident = 1;
$job->finalization_error_code =
@@ -792,12 +783,6 @@ final class CloudAutomaticOperationRecoveryService
->where('recovery_events.disposition', 'received');
}
- private static function receivedCreateSuccessEventExists($events): void
- {
- self::receivedEventExists($events);
- $events->where('recovery_events.outcome', 'succeeded');
- }
-
/**
* @return array{delay:int,error_code:string}|null Retry state, or null
* after a committed resolution.
@@ -813,6 +798,10 @@ final class CloudAutomaticOperationRecoveryService
)) {
throw new CloudError('自动核对租约已失效', 409);
}
+ if ((string) $job->kind === 'create') {
+ CloudCreateFailureRecoveryService::recover($job);
+ return null;
+ }
$context = self::recoveryContext($job);
// A late immutable completion receipt is a PHP-local two-phase
diff --git a/web/app/Services/Cloud/CloudCreateFailureRecoveryService.php b/web/app/Services/Cloud/CloudCreateFailureRecoveryService.php
new file mode 100644
index 00000000..408c9af4
--- /dev/null
+++ b/web/app/Services/Cloud/CloudCreateFailureRecoveryService.php
@@ -0,0 +1,712 @@
+getConnection()->transaction(
+ static function () use ($source, $sourceEvent): string {
+ [$user, $order, $job] = self::lockLedger($source);
+ $event = CloudAgentCompletionEvent::where(
+ 'event_id',
+ (string) $sourceEvent->event_id
+ )->lockForUpdate()->first();
+ if ($event === null) {
+ return 'received';
+ }
+ if ((string) $event->disposition !== 'received') {
+ return (string) $event->disposition;
+ }
+ if ((int) $event->host_id !== (int) $job->host_id
+ || !hash_equals(
+ (string) $job->operation_id,
+ (string) $event->operation_id
+ )) {
+ throw new CloudRecoveryOperationIdentityConflict(
+ '开通失败回执与原任务不匹配',
+ 409
+ );
+ }
+ CloudOperationCompletionService::assertEventIdentityForJob(
+ $job,
+ $event
+ );
+ $operation = CloudOperationCompletionService::operation($event);
+ if (self::settledNotApplied($job, $order, $event)) {
+ // A fresh inspection may have completed the atomic PHP
+ // finalizer before the Agent could deliver its immutable
+ // failure. The receipt no longer decides the ledger, but
+ // the exact identity and committed audit allow it to be
+ // ACKed without reopening or rewriting that resolution.
+ $event->disposition = 'applied';
+ $event->applied_at = time();
+ $event->save();
+ return 'applied';
+ }
+ self::assertRecoverableJob($job);
+ if (!self::receiptProvesAbsence($event, $operation)) {
+ $event->disposition = 'late';
+ $event->applied_at = null;
+ $event->save();
+ return 'late';
+ }
+
+ self::assertNoSuccessfulReceiptLocked($job);
+ self::finalizeLocked(
+ $job,
+ $user,
+ $order,
+ 'late_completion_receipt:' . (string) $event->event_id,
+ $event
+ );
+ return 'applied';
+ },
+ 3
+ );
+ }
+
+ /** Recover one lease-owned create without replaying its Agent mutation. */
+ public static function recover(CloudJob $claimed): void
+ {
+ $job = self::claimedJob($claimed);
+ $prepared = self::preparedEvidence($job);
+ if ($prepared !== null) {
+ if (!$prepared['fence_resolution_required']) {
+ self::finishPrepared($claimed, $prepared, false);
+ return;
+ }
+ try {
+ self::finishPrepared($claimed, $prepared, null);
+ return;
+ } catch (CloudAgentError $error) {
+ if ($error->agentHttpStatus !== 404) {
+ throw $error;
+ }
+ // No fence verdict was committed before the prior PHP crash.
+ // A new write requires new, bounded read-only evidence.
+ }
+ }
+
+ $inspection = CloudAdminOperationRecoveryService::inspectForRecovery(
+ (string) $job->id
+ );
+ $prepared = self::prepareInspection($claimed, $inspection);
+ self::finishPrepared(
+ $claimed,
+ $prepared,
+ $prepared['fence_resolution_required'] ? $inspection : false
+ );
+ }
+
+ /**
+ * Authenticate the durable PHP intent before an automatic Agent fence read
+ * or write. This is deliberately create-specific and does not broaden the
+ * generic recovery allow-list.
+ */
+ public static function assertPreparedForFence(
+ CloudJob $job,
+ string $resolution,
+ string $reason
+ ): void {
+ if ($resolution !== 'not_applied') {
+ throw new CloudError('开通任务只支持确认未应用', 409);
+ }
+ self::assertRecoverableJob($job);
+ $prepared = self::preparedEvidence($job);
+ if ($prepared === null
+ || !hash_equals($prepared['reason'], $reason)) {
+ throw new CloudRecoveryEvidenceConflict(
+ '开通任务的恢复预提交证据不匹配',
+ 409
+ );
+ }
+ }
+
+ /**
+ * @return array{marker:string,reason:string,identity:array,inspection:array,fence_resolution_required:bool}
+ */
+ private static function prepareInspection(
+ CloudJob $claimed,
+ array $inspection
+ ): array {
+ $job = self::claimedJob($claimed);
+ $identity = CloudOperationRecoveryIdentity::expected($job, false);
+ CloudOperationRecoveryIdentity::assertMatches($identity, [
+ 'action' => $inspection['action'] ?? null,
+ 'resource_id' => $inspection['resource_id'] ?? null,
+ 'request_fingerprint' =>
+ $inspection['request_fingerprint'] ?? null,
+ ]);
+ self::assertInspectionProvesAbsence($job, $inspection);
+ $evidence = self::inspectionEvidence($inspection);
+ $marker = self::inspectionMarker($evidence);
+ $reason = self::fenceReason($job, $marker);
+
+ return (new CloudJob())->getConnection()->transaction(
+ static function () use (
+ $claimed,
+ $identity,
+ $evidence,
+ $marker,
+ $reason
+ ): array {
+ [$user, $order, $locked] = self::lockLedger($claimed);
+ self::assertClaimedJob($locked, $claimed);
+ self::assertNoSuccessfulReceiptLocked($locked);
+
+ $payload = self::payload($locked);
+ $lockedIdentity = CloudOperationRecoveryIdentity::
+ expectedFromPayload($locked, $payload, false);
+ CloudOperationRecoveryIdentity::assertMatches(
+ $lockedIdentity,
+ $identity
+ );
+ $payload[self::ESCROW_KEY] = [
+ 'version' => self::ESCROW_VERSION,
+ 'marker' => $marker,
+ 'reason' => $reason,
+ 'identity' => CloudOperationRecoveryIdentity::persisted(
+ $lockedIdentity
+ ),
+ 'inspection' => $evidence,
+ ];
+ $encoded = json_encode(
+ $payload,
+ JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES
+ | JSON_UNESCAPED_UNICODE
+ );
+ if (strlen($encoded) > 196608) {
+ throw new CloudRecoveryPayloadUnavailable(
+ '开通任务恢复预提交数据过大',
+ 409
+ );
+ }
+ $locked->effect_resolution_reason = $marker;
+ $locked->sealed_payload = CloudSecrets::encrypt(
+ $payload,
+ 'job-payload'
+ );
+ $locked->updated_at = time();
+ $locked->save();
+
+ return [
+ 'marker' => $marker,
+ 'reason' => $reason,
+ 'identity' => $lockedIdentity,
+ 'inspection' => $evidence,
+ 'fence_resolution_required' => (bool)
+ $evidence['fence_resolution_required'],
+ ];
+ },
+ 3
+ );
+ }
+
+ /**
+ * Resolve or verify the Agent fence while holding the canonical financial
+ * locks, then commit every PHP ledger transition in that same transaction.
+ *
+ * @param array{marker:string,reason:string,identity:array,inspection:array,fence_resolution_required:bool} $prepared
+ * @param array|false|null $inspection Fresh evidence, false when no fence
+ * is required, null when resuming an already committed fence resolution.
+ */
+ private static function finishPrepared(
+ CloudJob $claimed,
+ array $prepared,
+ array|false|null $inspection
+ ): void {
+ (new CloudJob())->getConnection()->transaction(
+ static function () use ($claimed, $prepared, $inspection): void {
+ [$user, $order, $job] = self::lockLedger($claimed);
+ self::assertClaimedJob($job, $claimed);
+ $lockedPrepared = self::preparedEvidence($job);
+ if ($lockedPrepared === null
+ || !hash_equals(
+ $prepared['marker'],
+ $lockedPrepared['marker']
+ )
+ || !hash_equals(
+ $prepared['reason'],
+ $lockedPrepared['reason']
+ )) {
+ throw new CloudRecoveryEvidenceConflict(
+ '开通任务的恢复预提交已变化',
+ 409
+ );
+ }
+ self::assertNoSuccessfulReceiptLocked($job);
+
+ if ($inspection === null) {
+ CloudAdminOperationRecoveryService::
+ preparedFenceResolutionForRecovery(
+ (string) $job->id,
+ 'not_applied',
+ $lockedPrepared['reason']
+ );
+ } elseif ($inspection !== false) {
+ CloudAdminOperationRecoveryService::resolveFenceForRecovery(
+ (string) $job->id,
+ 'not_applied',
+ $lockedPrepared['reason'],
+ $inspection
+ );
+ } elseif ($lockedPrepared['fence_resolution_required']) {
+ throw new CloudRecoveryEvidenceConflict(
+ '开通任务仍需解除宿主机资源栅栏',
+ 409
+ );
+ }
+
+ self::assertNoSuccessfulReceiptLocked($job);
+ self::finalizeLocked(
+ $job,
+ $user,
+ $order,
+ 'automatic_create_not_applied:'
+ . substr($lockedPrepared['marker'], -64)
+ );
+ },
+ 3
+ );
+ }
+
+ /**
+ * @return array{marker:string,reason:string,identity:array,inspection:array,fence_resolution_required:bool}|null
+ */
+ private static function preparedEvidence(CloudJob $job): ?array
+ {
+ $marker = (string) ($job->effect_resolution_reason ?? '');
+ if (preg_match(
+ '/^prepared:create_not_applied:v1:[a-f0-9]{64}$/D',
+ $marker
+ ) !== 1) {
+ return null;
+ }
+ $payload = self::payload($job);
+ $value = $payload[self::ESCROW_KEY] ?? null;
+ $identity = is_array($value) ? ($value['identity'] ?? null) : null;
+ $inspection = is_array($value) ? ($value['inspection'] ?? null) : null;
+ if (!is_array($value) || array_is_list($value)
+ || array_diff(array_keys($value), [
+ 'version', 'marker', 'reason', 'identity', 'inspection',
+ ]) !== []
+ || ($value['version'] ?? null) !== self::ESCROW_VERSION
+ || !is_string($value['marker'] ?? null)
+ || !hash_equals($marker, $value['marker'])
+ || !is_string($value['reason'] ?? null)
+ || mb_strlen($value['reason']) < 20
+ || mb_strlen($value['reason']) > 500
+ || !is_array($identity) || array_is_list($identity)
+ || !is_array($inspection) || array_is_list($inspection)) {
+ throw new CloudRecoveryPayloadUnavailable(
+ '开通任务恢复预提交数据无效',
+ 409
+ );
+ }
+ $expected = CloudOperationRecoveryIdentity::expectedFromPayload(
+ $job,
+ $payload,
+ false
+ );
+ CloudOperationRecoveryIdentity::assertMatches($expected, $identity);
+ self::assertInspectionProvesAbsence($job, $inspection, false);
+ CloudOperationRecoveryIdentity::assertMatches($expected, [
+ 'action' => $inspection['action'] ?? null,
+ 'resource_id' => $inspection['resource_id'] ?? null,
+ 'request_fingerprint' =>
+ $inspection['request_fingerprint'] ?? null,
+ ]);
+ $evidence = self::inspectionEvidence($inspection);
+ $expectedMarker = self::inspectionMarker($evidence);
+ if (!hash_equals($expectedMarker, $marker)
+ || !hash_equals(self::fenceReason($job, $marker), $value['reason'])) {
+ throw new CloudRecoveryEvidenceConflict(
+ '开通任务恢复预提交摘要不匹配',
+ 409
+ );
+ }
+ return [
+ 'marker' => $marker,
+ 'reason' => $value['reason'],
+ 'identity' => $expected,
+ 'inspection' => $evidence,
+ 'fence_resolution_required' =>
+ (bool) $evidence['fence_resolution_required'],
+ ];
+ }
+
+ private static function assertInspectionProvesAbsence(
+ CloudJob $job,
+ array $inspection,
+ bool $requireFresh = true
+ ): void {
+ $checkedAt = $inspection['checked_at'] ?? null;
+ $projection = $inspection['effect_projection'] ?? null;
+ if (($inspection['operation_id'] ?? null)
+ !== (string) $job->operation_id
+ || ($inspection['journal_status'] ?? null) !== 'failed'
+ || ($inspection['outcome'] ?? null) !== 'failed'
+ || ($inspection['summary_code'] ?? null) !== 'operation_failed'
+ || ($inspection['effect_resolution'] ?? null) !== 'not_applied'
+ || !is_int($checkedAt)
+ || ($requireFresh && ($checkedAt < time()
+ - self::INSPECTION_MAX_AGE_SECONDS
+ || $checkedAt > time() + 5))
+ || !is_array($projection) || array_is_list($projection)
+ || ($projection['creation_absent'] ?? null) !== true
+ || ($projection['managed_artifacts_absent'] ?? null) !== true
+ || !in_array(
+ $inspection['fence_state'] ?? null,
+ ['not_required', 'active'],
+ true
+ )
+ || !is_bool($inspection['fence_resolution_required'] ?? null)
+ || (bool) $inspection['fence_resolution_required']
+ !== (($inspection['fence_state'] ?? null) === 'active')
+ || array_key_exists('completion_delivery_state', $inspection)) {
+ throw new CloudRecoveryEvidenceConflict(
+ '宿主机检查没有同时确认虚拟机及面板管理资源均不存在',
+ 409
+ );
+ }
+ CloudCreateFailureFinalizer::assertResolutionSupported(
+ $job,
+ 'not_applied',
+ $inspection
+ );
+ }
+
+ private static function receiptProvesAbsence(
+ CloudAgentCompletionEvent $event,
+ array $operation
+ ): bool {
+ $result = $operation['result'] ?? null;
+ return (string) $event->outcome === 'failed'
+ && !(bool) $event->effect_may_have_applied
+ && is_array($result) && !array_is_list($result)
+ && ($result['creation_absent'] ?? null) === true
+ && ($result['managed_artifacts_absent'] ?? null) === true;
+ }
+
+ /** @return array{0:User,1:CloudOrder,2:CloudJob} */
+ private static function lockLedger(CloudJob $source): array
+ {
+ $user = User::where('id', (int) $source->user_id)
+ ->lockForUpdate()->first();
+ $order = CloudOrder::where('id', (int) $source->order_id)
+ ->lockForUpdate()->first();
+ $job = CloudJob::where('id', (int) $source->id)
+ ->lockForUpdate()->first();
+ if ($user === null || $order === null || $job === null
+ || (int) $job->user_id !== (int) $user->id
+ || (int) $job->order_id !== (int) $order->id
+ || (int) $order->user_id !== (int) $user->id) {
+ throw new CloudRecoveryInstanceStateConflict(
+ '开通任务与资金账本关联不一致',
+ 409
+ );
+ }
+ return [$user, $order, $job];
+ }
+
+ private static function assertRecoverableJob(CloudJob $job): void
+ {
+ self::assertDispatchedEnvelope($job);
+ if ((string) $job->kind !== 'create'
+ || $job->order_id === null
+ || (string) $job->execution_mode !== 'agent'
+ || (string) $job->phase !== 'terminal'
+ || (string) $job->outcome !== 'failed'
+ || (string) $job->status !== 'failed'
+ || !(bool) $job->effect_may_have_applied
+ || (bool) $job->safe_to_resubmit
+ || (string) $job->effect_resolution !== 'unknown') {
+ throw new CloudRecoveryInstanceStateConflict(
+ '开通任务不处于可自动核对状态',
+ 409
+ );
+ }
+ }
+
+ private static function settledNotApplied(
+ CloudJob $job,
+ CloudOrder $order,
+ CloudAgentCompletionEvent $event
+ ): bool {
+ self::assertDispatchedEnvelope($job);
+ $reason = $job->effect_resolution_reason;
+ $completionEventId = $job->completion_event_id;
+ return (string) $event->outcome === 'failed'
+ && (string) $job->kind === 'create'
+ && $job->order_id !== null
+ && (string) $job->execution_mode === 'agent'
+ && (string) $job->phase === 'terminal'
+ && (string) $job->outcome === 'failed'
+ && (string) $job->status === 'failed'
+ && (int) $job->dispatch_attempts === 1
+ && !(bool) $job->safe_to_resubmit
+ && (string) $job->effect_resolution === 'not_applied'
+ && (int) ($job->effect_resolved_at ?? 0) > 0
+ && is_string($reason)
+ && mb_strlen($reason) >= 20
+ && mb_strlen($reason) <= 500
+ && !(bool) $job->finalization_incident
+ && $job->finalization_error_code === null
+ && $job->instance_id === null
+ && $order->instance_id === null
+ && (string) $order->status === 'failed'
+ && $order->refunded_at !== null
+ && ($completionEventId === null
+ || (is_string($completionEventId)
+ && hash_equals(
+ $completionEventId,
+ (string) $event->event_id
+ )));
+ }
+
+ /** Validate create's immutable generation without broadening policy. */
+ private static function assertDispatchedEnvelope(CloudJob $job): void
+ {
+ CloudDispatchLedger::assertDispatched($job);
+ if ((string) $job->attempts !== '1'
+ || (int) ($job->host_id ?? 0) < 1
+ || (int) ($job->dispatch_deadline_at ?? 0) < 1
+ || (int) ($job->dispatch_started_at ?? 0) < 1
+ || (($job->agent_accepted_at ?? null) !== null
+ && (int) $job->agent_accepted_at < 1)
+ || (int) ($job->dispatched_at ?? 0) < 1
+ || (int) ($job->receipt_deadline_at ?? 0) < 1
+ || (int) ($job->terminal_at ?? 0) < 1
+ || preg_match(
+ '/^[a-f0-9]{32}$/D',
+ (string) ($job->operation_id ?? '')
+ ) !== 1
+ || preg_match(
+ '/^[a-f0-9]{64}$/D',
+ (string) ($job->request_fingerprint ?? '')
+ ) !== 1
+ || preg_match(
+ '/^[A-Za-z0-9_.-]{16,128}$/D',
+ (string) ($job->request_idempotency_key ?? '')
+ ) !== 1) {
+ throw new CloudRecoveryInstanceStateConflict(
+ '开通任务单次派发信封不完整',
+ 409
+ );
+ }
+ }
+
+ private static function assertClaimedJob(
+ CloudJob $job,
+ CloudJob $claimed
+ ): void {
+ self::assertRecoverableJob($job);
+ if (!is_string($claimed->lease_token)
+ || !is_string($job->lease_token)
+ || !hash_equals(
+ (string) $claimed->lease_token,
+ (string) $job->lease_token
+ )
+ || (int) $job->lease_until <= time()) {
+ throw new CloudRecoveryInstanceStateConflict(
+ '开通任务自动核对租约已失效',
+ 409
+ );
+ }
+ }
+
+ private static function claimedJob(CloudJob $claimed): CloudJob
+ {
+ $job = CloudJob::find((int) $claimed->id);
+ if ($job === null) {
+ throw new CloudError('任务不存在', 404);
+ }
+ self::assertClaimedJob($job, $claimed);
+ return $job;
+ }
+
+ private static function assertNoSuccessfulReceiptLocked(CloudJob $job): void
+ {
+ $event = CloudAgentCompletionEvent::where(
+ 'host_id',
+ (int) $job->host_id
+ )->where(
+ 'operation_id',
+ (string) $job->operation_id
+ )->where('outcome', 'succeeded')->lockForUpdate()->first();
+ if ($event === null) {
+ return;
+ }
+ $exact = CloudOperationCompletionService::isExactStoredEvent($event)
+ || CloudOperationCompletionService::isExactAppliedEvent($event);
+ if (!$exact) {
+ throw new CloudRecoveryOperationIdentityConflict(
+ '开通成功回执状态异常,不能执行退款',
+ 409
+ );
+ }
+ $payload = self::payload($job);
+ $identity = CloudOperationRecoveryIdentity::expectedFromPayload(
+ $job,
+ $payload,
+ true
+ );
+ if (CloudOperationRecoveryIdentity::same(
+ $identity,
+ CloudOperationCompletionService::identity($event)
+ )) {
+ throw new CloudRecoveryEvidenceConflict(
+ '已收到匹配的实例创建成功回执,不能按未应用退款',
+ 409
+ );
+ }
+ throw new CloudRecoveryOperationIdentityConflict(
+ '开通成功回执身份冲突,不能执行退款',
+ 409
+ );
+ }
+
+ private static function finalizeLocked(
+ CloudJob $job,
+ User $user,
+ CloudOrder $order,
+ string $resolutionReason,
+ ?CloudAgentCompletionEvent $event = null
+ ): void {
+ $payload = self::payload($job);
+ $identity = CloudOperationRecoveryIdentity::expectedFromPayload(
+ $job,
+ $payload,
+ false
+ );
+ CloudCreateFailureFinalizer::refundAndReleaseLocked(
+ $job,
+ $user,
+ $order,
+ 'create_effect_not_applied'
+ );
+
+ $safePayload = CloudInput::safeResult($payload);
+ unset($safePayload[self::ESCROW_KEY]);
+ $safePayload[CloudOperationRecoveryIdentity::payloadKey()] =
+ CloudOperationRecoveryIdentity::persisted($identity);
+ $now = time();
+ $job->effect_resolution = 'not_applied';
+ $job->effect_resolved_at = $now;
+ $job->effect_resolved_by = null;
+ $job->effect_resolution_reason = $resolutionReason;
+ if ($event !== null) {
+ $job->completion_event_id = (string) $event->event_id;
+ }
+ $job->sealed_payload = CloudSecrets::encrypt(
+ $safePayload,
+ 'job-payload'
+ );
+ $job->finalization_incident = 0;
+ $job->finalization_error_code = null;
+ $job->read_at = null;
+ $job->lease_token = null;
+ $job->lease_until = 0;
+ $job->available_at = 0;
+ $job->updated_at = $now;
+ $job->save();
+ if ($event !== null) {
+ $event->disposition = 'applied';
+ $event->applied_at = $now;
+ $event->save();
+ }
+ }
+
+ private static function payload(CloudJob $job): array
+ {
+ try {
+ $payload = CloudSecrets::decrypt(
+ (string) $job->sealed_payload,
+ 'job-payload'
+ );
+ } catch (\Throwable $error) {
+ throw new CloudRecoveryPayloadUnavailable(
+ '开通任务恢复数据无法解密',
+ 409
+ );
+ }
+ if (!is_array($payload) || array_is_list($payload)) {
+ throw new CloudRecoveryPayloadUnavailable(
+ '开通任务恢复数据结构不完整',
+ 409
+ );
+ }
+ return $payload;
+ }
+
+ private static function fenceReason(CloudJob $job, string $marker): string
+ {
+ return '自动恢复已确认原开通操作未创建虚拟机或面板管理资源;operation_id='
+ . (string) $job->operation_id . ';evidence='
+ . substr($marker, -64);
+ }
+
+ /** Hash only stable evidence so a lost fence response can be resumed. */
+ private static function inspectionMarker(array $evidence): string
+ {
+ unset($evidence['checked_at']);
+ return self::MARKER_PREFIX . hash(
+ 'sha256',
+ json_encode(
+ $evidence,
+ JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES
+ | JSON_UNESCAPED_UNICODE
+ )
+ );
+ }
+
+ /** Return the bounded, canonical evidence persisted in encrypted escrow. */
+ private static function inspectionEvidence(array $inspection): array
+ {
+ return [
+ 'operation_id' => $inspection['operation_id'],
+ 'action' => $inspection['action'],
+ 'resource_id' => $inspection['resource_id'],
+ 'request_fingerprint' => $inspection['request_fingerprint'],
+ 'journal_status' => $inspection['journal_status'],
+ 'checked_at' => $inspection['checked_at'],
+ 'outcome' => $inspection['outcome'],
+ 'summary_code' => $inspection['summary_code'],
+ 'error_code' => $inspection['error_code'] ?? '',
+ 'effect_resolution' => $inspection['effect_resolution'],
+ 'fence_state' => $inspection['fence_state'],
+ 'fence_resolution_required' =>
+ $inspection['fence_resolution_required'],
+ 'effect_projection' => [
+ 'creation_absent' => true,
+ 'managed_artifacts_absent' => true,
+ ],
+ ];
+ }
+}
diff --git a/web/app/Services/Cloud/CloudOperationRecoveryService.php b/web/app/Services/Cloud/CloudOperationRecoveryService.php
index ba705b76..de4b4d73 100644
--- a/web/app/Services/Cloud/CloudOperationRecoveryService.php
+++ b/web/app/Services/Cloud/CloudOperationRecoveryService.php
@@ -365,6 +365,12 @@ final class CloudOperationRecoveryService
);
}
if ((string) $current->kind === 'create') {
+ if ((string) $event->outcome === 'failed') {
+ return CloudCreateFailureRecoveryService::settleReceipt(
+ $current,
+ $event
+ );
+ }
if ((string) $event->outcome !== 'succeeded'
|| (bool) $event->effect_may_have_applied
|| (bool) $event->safe_to_resubmit
@@ -548,10 +554,11 @@ final class CloudOperationRecoveryService
// eligible for the immutable receipt, terminate the event
// instead of leaving an unclaimable "received" row.
if (self::retryableCreateCompletion($job, $event)) {
- // Create is intentionally outside the generic
- // inspection policy. Its exact success receipt is the
- // only safe recovery evidence, so a temporary local
- // ledger fault must not consume that evidence.
+ // Create uses dedicated financial finalizers. Keep
+ // either exact outcome retryable when PHP could not
+ // finish inspecting or settling its ledger. An
+ // incomplete failure is ACKed late by the dedicated
+ // finalizer only after those local checks succeed.
return 'received';
}
if (self::lateReceiptResolution($job, $event) === null) {
@@ -961,9 +968,10 @@ final class CloudOperationRecoveryService
&& (bool) $job->effect_may_have_applied
&& !(bool) $job->safe_to_resubmit
&& (string) $job->effect_resolution === 'unknown'
- && (string) $event->outcome === 'succeeded'
- && !(bool) $event->effect_may_have_applied
- && !(bool) $event->safe_to_resubmit;
+ && ((string) $event->outcome === 'failed'
+ || ((string) $event->outcome === 'succeeded'
+ && !(bool) $event->effect_may_have_applied
+ && !(bool) $event->safe_to_resubmit));
}
private static function assertRecoverableAppliedEvidence(
diff --git a/web/tests/CloudAdminOperationRecoveryServiceTest.php b/web/tests/CloudAdminOperationRecoveryServiceTest.php
index 9b9c68d5..e71094ea 100644
--- a/web/tests/CloudAdminOperationRecoveryServiceTest.php
+++ b/web/tests/CloudAdminOperationRecoveryServiceTest.php
@@ -363,11 +363,14 @@ namespace {
]);
$before = count($client->requests);
+ Capsule::table('cloud_jobs')->where('id', 41)->update([
+ 'kind' => 'host_storage_configure',
+ ]);
$reject(static fn () => Recovery::inspectForRecovery('41'), 409);
$reject(static fn () => Recovery::resolveFenceForRecovery(
'41',
'not_applied',
- 'automatic recovery must reject non-settings tasks',
+ 'automatic recovery must reject unregistered task kinds',
[
'operation_id' => 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa',
'checked_at' => $now,
@@ -379,13 +382,16 @@ namespace {
$reject(static fn () => Recovery::preparedFenceResolutionForRecovery(
'41',
'not_applied',
- 'automatic recovery must reject non-settings tasks'
+ 'automatic recovery must reject unregistered task kinds'
), 409);
$assert(count($client->requests) === $before,
- 'A non-settings task reached the system recovery Agent client.');
+ 'An unregistered task reached the system recovery Agent client.');
+ Capsule::table('cloud_jobs')->where('id', 41)->update([
+ 'kind' => 'create',
+ ]);
$operationA = 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa';
- $client->responses[] = [
+ $client->responses[] = $createInspectionResponse = [
'operation_id' => $operationA,
'operation_status' => 'failed',
'checked_at' => $now,
@@ -430,6 +436,36 @@ namespace {
->value('effect_resolution') === 'unknown',
'Read-only inspection changed the task or its safety fence.');
+ $client->responses[] = $createInspectionResponse;
+ $createInspection = Recovery::inspectForRecovery('41');
+ $createRequest = $client->requests[array_key_last($client->requests)];
+ $assert(($createInspection['error_code'] ?? null) === 'creation_absent'
+ && ($createInspection['effect_projection']['creation_absent'] ?? null) === true
+ && ($createInspection['effect_projection']['managed_artifacts_absent'] ?? null) === true
+ && $createRequest['method'] === 'GET'
+ && ($createRequest['system_recovery'] ?? false) === true
+ && $createRequest['actor']['operation_id'] === $operationA
+ && Capsule::table('cloud_jobs')->where('id', 41)
+ ->value('effect_resolution') === 'unknown',
+ 'Create inspection did not remain operation-bound and read-only.');
+ $before = count($client->requests);
+ $reject(static fn () => Recovery::resolveFenceForRecovery(
+ '41', 'not_applied',
+ 'automatic create recovery requires durable financial intent',
+ $createInspection
+ ), 409);
+ $reject(static fn () => Recovery::resolveFenceForRecovery(
+ '41', 'applied',
+ 'automatic create recovery cannot infer a successful instance',
+ $createInspection
+ ), 409);
+ $reject(static fn () => Recovery::preparedFenceResolutionForRecovery(
+ '41', 'not_applied',
+ 'automatic create recovery requires durable financial intent'
+ ), 409);
+ $assert(count($client->requests) === $before,
+ 'An unprepared create reached the system Agent fence routes.');
+
$client->responses[] = ['operation_id' => $operationA,
'operation_status' => 'uncertain', 'checked_at' => $now,
'outcome' => 'uncertain', 'summary_code' => 'evidence_incomplete'];
diff --git a/web/tests/CloudFinancialWorkflowTest.php b/web/tests/CloudFinancialWorkflowTest.php
index b50c512f..018e9498 100644
--- a/web/tests/CloudFinancialWorkflowTest.php
+++ b/web/tests/CloudFinancialWorkflowTest.php
@@ -25,6 +25,61 @@ namespace App\Services\Cloud {
public static function encrypt(array $payload, string $purpose): string { $id = bin2hex(random_bytes(16)); self::$memory[$id] = $payload; return 'test-only:' . $id; }
public static function decrypt(string $cipher, string $purpose): array { return self::$memory[substr($cipher, 10)]; }
}
+
+ /** Isolated read-only Agent transport for create recovery checks. */
+ final class FinancialCreateRecoveryAgentClient
+ {
+ public array $requests = [];
+ public array $responses = [];
+
+ public function systemRecoveryRequest(
+ string $method,
+ string $target,
+ string $operationId,
+ ?array $payload = null
+ ): array {
+ $this->requests[] = compact(
+ 'method', 'target', 'operationId', 'payload'
+ );
+ if ($this->responses === []) {
+ throw new \RuntimeException(
+ 'Unexpected create recovery Agent request.'
+ );
+ }
+ $response = array_shift($this->responses);
+ if ($response instanceof \Throwable) {
+ throw $response;
+ }
+ if (is_callable($response)) {
+ $response = $response(end($this->requests));
+ }
+ if (!is_array($response)) {
+ throw new \RuntimeException(
+ 'Invalid create recovery Agent response.'
+ );
+ }
+ return $response;
+ }
+ }
+
+ final class CloudAgentSettingsService
+ {
+ public static ?FinancialCreateRecoveryAgentClient $client = null;
+ public static int $hostId = 0;
+
+ public static function clientForConfiguration(
+ int $hostId,
+ float $timeoutSeconds = 120.0
+ ): FinancialCreateRecoveryAgentClient {
+ if ($hostId !== self::$hostId || $timeoutSeconds !== 30.0
+ || self::$client === null) {
+ throw new \RuntimeException(
+ 'Unexpected create recovery client request.'
+ );
+ }
+ return self::$client;
+ }
+ }
}
namespace {
require dirname(__DIR__) . '/vendor/autoload.php';
@@ -69,6 +124,10 @@ namespace {
use App\Services\Cloud\CloudPlatformTestNetworkPolicy;
use App\Services\Cloud\CloudDeploymentPageService;
use App\Services\Cloud\CloudTemplateCatalogService;
+ use App\Services\Cloud\CloudAgentSettingsService;
+ use App\Services\Cloud\CloudAgentError;
+ use App\Services\Cloud\CloudCreateFailureRecoveryService;
+ use App\Services\Cloud\FinancialCreateRecoveryAgentClient;
use GuzzleHttp\Client;
use GuzzleHttp\Handler\MockHandler;
use GuzzleHttp\HandlerStack;
@@ -2915,5 +2974,892 @@ namespace {
'Unknown, applied, or sticky lifecycle evidence created a new mutation generation.');
}
+ // A create which crossed the Agent boundary may time out locally before
+ // the authenticated terminal failure reaches PHP. Exact Agent proof that
+ // both the guest and every managed artifact are absent must finish the
+ // original paid order locally; it must never dispatch a second create.
+ foreach (['203.0.113.20', '203.0.113.21', '203.0.113.22'] as $address) {
+ if (!CloudPublicIp::where('address', $address)->exists()) {
+ $ip = new CloudPublicIp();
+ $ip->host_id = $host->id;
+ $ip->public_ip_pool_id = $sourcePublicPool->id;
+ $ip->address = $address;
+ $ip->prefix_length = 28;
+ $ip->state = 'available';
+ $ip->created_at = $ip->updated_at = time();
+ $ip->save();
+ }
+ }
+ $lateFailureOrder = CloudOrderService::purchase(array_replace($input, [
+ 'name' => 'vm-late-failure',
+ 'idempotency_key' => 'test-request-key-late-failure',
+ ]), $actor);
+ $lateFailureInvoice = CloudInvoice::where(
+ 'order_id',
+ $lateFailureOrder->id
+ )->firstOrFail();
+ CloudInvoiceService::payBalance(
+ $lateFailureInvoice,
+ $actor,
+ 'provision-balance-late-failure'
+ );
+ $lateFailureJob = CloudJob::where(
+ 'order_id',
+ $lateFailureOrder->id
+ )->firstOrFail();
+ // Earlier lifecycle tests intentionally leave separately audited retries
+ // queued. Park those unrelated rows so this tracer observes exactly one
+ // create dispatch rather than depending on suite-wide queue order.
+ Capsule::table('cloud_jobs')
+ ->where('id', '<>', (int) $lateFailureJob->id)
+ ->where('phase', '<>', 'terminal')
+ ->update([
+ 'available_at' => time() + 3600,
+ 'dispatch_deadline_at' => time() + 3600,
+ 'receipt_deadline_at' => time() + 3600,
+ ]);
+ Capsule::table('cloud_jobs')->where(
+ 'id',
+ (int) $lateFailureJob->id
+ )->update(['available_at' => 0]);
+ $lateFailureJob->refresh();
+ $lateFailureIdentity = CloudOperationRecoveryIdentity::expected(
+ $lateFailureJob,
+ false
+ );
+ $mock->append(new Response(202, [], json_encode([
+ 'operation_id' => $lateFailureJob->operation_id,
+ 'status' => 'reserved',
+ ...$lateFailureIdentity,
+ 'replayed' => false,
+ 'retryable' => false,
+ ], JSON_THROW_ON_ERROR)));
+ $beforeLateFailureDispatch = count($history);
+ $assert($worker->tick(),
+ 'Late-failure create did not cross the Agent dispatch boundary.');
+ $lateFailureJob->refresh();
+ $assert((string) $lateFailureJob->phase === 'awaiting_receipt'
+ && (int) $lateFailureJob->dispatch_attempts === 1
+ && count($history) === $beforeLateFailureDispatch + 1,
+ 'Late-failure create did not persist its single dispatch generation.');
+ Capsule::table('cloud_jobs')->where(
+ 'id',
+ (int) $lateFailureJob->id
+ )->update([
+ 'receipt_deadline_at' => time() - 1,
+ 'lease_token' => null,
+ 'lease_until' => 0,
+ ]);
+ $assert(CloudWorkerService::expireDueTaskForAdmin(
+ (int) $lateFailureJob->id
+ ) === 1, 'Late-failure create did not reach its local deadline.');
+ $lateFailureJob->refresh();
+ $lateFailureOrder->refresh();
+ $lateFailureInvoice->refresh();
+ $lateFailureReservation = CloudResourceReservation::where(
+ 'order_id',
+ $lateFailureOrder->id
+ )->firstOrFail();
+ $lateFailureIp = CloudPublicIp::findOrFail(
+ (int) $lateFailureReservation->public_ip_id
+ );
+ $walletWhileFenced = (string) User::findOrFail(1)->money;
+ $stockWhileFenced = (int) CloudOffer::findOrFail(1)->stock;
+ $assert((string) $lateFailureJob->phase === 'terminal'
+ && (string) $lateFailureJob->outcome === 'failed'
+ && (string) $lateFailureJob->effect_resolution === 'unknown'
+ && (string) $lateFailureOrder->status === 'failed'
+ && $lateFailureOrder->refunded_at === null
+ && (string) $lateFailureInvoice->status === 'paid'
+ && (string) $lateFailureReservation->state === 'uncertain'
+ && (string) $lateFailureIp->state === 'uncertain',
+ 'Timed-out dispatched create did not retain its financial fence.');
+
+ $lateFailureEventId = 'create-late-failure-event-0001';
+ $lateFailureReceipt = [
+ 'type' => 'completion',
+ 'id' => $lateFailureEventId,
+ 'agent_id' => (string) $lateFailureJob->host_id,
+ 'operation_id' => (string) $lateFailureJob->operation_id,
+ ...$lateFailureIdentity,
+ 'status' => 'failed',
+ 'effect_may_have_applied' => false,
+ 'safe_to_resubmit' => false,
+ 'error_code' => 'creation_absent',
+ 'result' => [
+ 'uuid' => $lateFailureIdentity['resource_id'],
+ 'creation_absent' => true,
+ 'managed_artifacts_absent' => true,
+ ],
+ ];
+ $lateFailureAccepted = CloudOperationCompletionService::accept(
+ $lateFailureReceipt
+ );
+ $lateFailureJob->refresh();
+ $lateFailureOrder->refresh();
+ $lateFailureInvoice->refresh();
+ $lateFailureReservation->refresh();
+ $lateFailureIp->refresh();
+ $lateFailureEvent = \App\Models\CloudAgentCompletionEvent::where(
+ 'event_id',
+ $lateFailureEventId
+ )->firstOrFail();
+ $assert($lateFailureAccepted === [
+ 'accepted' => true,
+ 'disposition' => 'applied',
+ ]
+ && (string) $lateFailureEvent->disposition === 'applied'
+ && $lateFailureEvent->applied_at !== null
+ && (string) $lateFailureJob->effect_resolution === 'not_applied'
+ && !(bool) $lateFailureJob->finalization_incident
+ && (string) $lateFailureOrder->status === 'failed'
+ && $lateFailureOrder->refunded_at !== null
+ && (string) $lateFailureInvoice->status === 'refunded'
+ && (string) $lateFailureReservation->state === 'released'
+ && (string) $lateFailureIp->state === 'available'
+ && (string) User::findOrFail(1)->money
+ === CloudMoney::decimal(
+ CloudMoney::cents($walletWhileFenced) + 999
+ )
+ && (int) CloudOffer::findOrFail(1)->stock
+ === $stockWhileFenced + 1
+ && count($history) === $beforeLateFailureDispatch + 1,
+ 'Exact late create failure did not atomically refund, release, ACK, '
+ . 'or replayed the original mutation.');
+ $lateFailureWalletAfter = (string) User::findOrFail(1)->money;
+ $lateFailureStockAfter = (int) CloudOffer::findOrFail(1)->stock;
+ $lateFailureRefundedAt = (int) $lateFailureOrder->refunded_at;
+ $lateFailureJobUpdatedAt = (int) $lateFailureJob->updated_at;
+ $lateFailureAppliedAt = (int) $lateFailureEvent->applied_at;
+ $lateFailureDuplicate = CloudOperationCompletionService::accept(
+ $lateFailureReceipt
+ );
+ $assert($lateFailureDuplicate === [
+ 'accepted' => true,
+ 'disposition' => 'applied',
+ ]
+ && (string) User::findOrFail(1)->money
+ === $lateFailureWalletAfter
+ && (int) CloudOffer::findOrFail(1)->stock
+ === $lateFailureStockAfter
+ && (int) CloudOrder::findOrFail(
+ (int) $lateFailureOrder->id
+ )->refunded_at === $lateFailureRefundedAt
+ && (int) CloudJob::findOrFail(
+ (int) $lateFailureJob->id
+ )->updated_at === $lateFailureJobUpdatedAt
+ && (int) \App\Models\CloudAgentCompletionEvent::findOrFail(
+ $lateFailureEventId
+ )->applied_at === $lateFailureAppliedAt
+ && count($history) === $beforeLateFailureDispatch + 1,
+ 'Duplicate late create failure refunded, released, ACKed, or dispatched '
+ . 'more than once.');
+
+ /**
+ * Create one paid order, cross the Agent boundary exactly once, and force
+ * the panel-side receipt deadline to the terminal unknown-effect state.
+ * Later recovery assertions can then focus on finance and evidence rather
+ * than reproducing dispatch bookkeeping by hand.
+ *
+ * @return array
+ */
+ $makeTimedOutCreate = static function (
+ string $suffix
+ ) use (
+ $input,
+ $actor,
+ $mock,
+ $worker,
+ &$history,
+ $assert
+ ): array {
+ $order = CloudOrderService::purchase(array_replace($input, [
+ 'name' => 'vm-create-recovery-' . $suffix,
+ 'idempotency_key' => 'test-create-recovery-' . $suffix,
+ ]), $actor);
+ $invoice = CloudInvoice::where('order_id', $order->id)
+ ->firstOrFail();
+ CloudInvoiceService::payBalance(
+ $invoice,
+ $actor,
+ 'provision-create-recovery-' . $suffix
+ );
+ $job = CloudJob::where('order_id', $order->id)->firstOrFail();
+ $future = time() + 3600;
+ Capsule::table('cloud_jobs')
+ ->where('id', '<>', (int) $job->id)
+ ->update([
+ 'available_at' => $future,
+ 'lease_until' => $future,
+ ]);
+ Capsule::table('cloud_jobs')->where('id', (int) $job->id)
+ ->update(['available_at' => 0, 'lease_until' => 0]);
+ $job->refresh();
+ $identity = CloudOperationRecoveryIdentity::expected($job, false);
+ $mock->append(new Response(202, [], json_encode([
+ 'operation_id' => $job->operation_id,
+ 'status' => 'reserved',
+ ...$identity,
+ 'replayed' => false,
+ 'retryable' => false,
+ ], JSON_THROW_ON_ERROR)));
+ $historyBefore = count($history);
+ $assert($worker->tick(),
+ 'Create recovery fixture did not dispatch: ' . $suffix);
+ $job->refresh();
+ $assert((string) $job->phase === 'awaiting_receipt'
+ && (int) $job->dispatch_attempts === 1
+ && count($history) === $historyBefore + 1,
+ 'Create recovery fixture did not persist one dispatch: '
+ . $suffix);
+ Capsule::table('cloud_jobs')->where('id', (int) $job->id)
+ ->update([
+ 'receipt_deadline_at' => time() - 1,
+ 'lease_token' => null,
+ 'lease_until' => 0,
+ ]);
+ $assert(CloudWorkerService::expireDueTaskForAdmin((int) $job->id)
+ === 1, 'Create recovery fixture did not expire: ' . $suffix);
+ $job->refresh();
+ $order->refresh();
+ $invoice->refresh();
+ $reservation = CloudResourceReservation::where(
+ 'order_id',
+ $order->id
+ )->firstOrFail();
+ $ip = CloudPublicIp::findOrFail((int) $reservation->public_ip_id);
+ $assert((string) $job->phase === 'terminal'
+ && (string) $job->outcome === 'failed'
+ && (string) $job->effect_resolution === 'unknown'
+ && (string) $order->status === 'failed'
+ && $order->refunded_at === null
+ && (string) $invoice->status === 'paid'
+ && (string) $reservation->state === 'uncertain'
+ && (string) $ip->state === 'uncertain',
+ 'Create recovery fixture did not retain its safety fence: '
+ . $suffix);
+ return compact(
+ 'order', 'invoice', 'job', 'identity', 'reservation', 'ip',
+ 'historyBefore'
+ ) + [
+ 'walletBefore' => (string) User::findOrFail(1)->money,
+ 'stockBefore' => (int) CloudOffer::findOrFail(1)->stock,
+ ];
+ };
+ $makeRecoveryReady = static function (CloudJob $job): void {
+ $future = time() + 3600;
+ Capsule::table('cloud_jobs')
+ ->where('id', '<>', (int) $job->id)
+ ->update([
+ 'available_at' => $future,
+ 'lease_until' => $future,
+ ]);
+ Capsule::table('cloud_jobs')->where('id', (int) $job->id)
+ ->update([
+ 'available_at' => 0,
+ 'lease_token' => null,
+ 'lease_until' => 0,
+ ]);
+ $job->refresh();
+ };
+ $inspectionFor = static function (
+ CloudJob $job,
+ array $identity,
+ bool $fenced = false
+ ): array {
+ return [
+ 'operation_id' => (string) $job->operation_id,
+ ...$identity,
+ 'operation_status' => 'failed',
+ 'checked_at' => time(),
+ 'outcome' => 'failed',
+ 'summary_code' => 'operation_failed',
+ 'error_code' => 'creation_absent',
+ 'effect_resolution' => 'not_applied',
+ 'fence_state' => $fenced ? 'active' : 'not_required',
+ 'fence_resolution_required' => $fenced,
+ 'effect_projection' => [
+ 'creation_absent' => true,
+ 'managed_artifacts_absent' => true,
+ ],
+ 'abort_job' => ['available' => false],
+ 'cleanup' => ['available' => false],
+ ];
+ };
+ $assertCreateSettled = static function (
+ array $case,
+ string $message
+ ) use ($assert, &$history): void {
+ $case['job']->refresh();
+ $case['order']->refresh();
+ $case['invoice']->refresh();
+ $case['reservation']->refresh();
+ $case['ip']->refresh();
+ $assert((string) $case['job']->effect_resolution === 'not_applied'
+ && !(bool) $case['job']->finalization_incident
+ && $case['job']->lease_token === null
+ && (string) $case['order']->status === 'failed'
+ && $case['order']->refunded_at !== null
+ && (string) $case['invoice']->status === 'refunded'
+ && (string) $case['reservation']->state === 'released'
+ && (string) $case['ip']->state === 'available'
+ && (string) User::findOrFail(1)->money
+ === CloudMoney::decimal(
+ CloudMoney::cents($case['walletBefore']) + 999
+ )
+ && (int) CloudOffer::findOrFail(1)->stock
+ === $case['stockBefore'] + 1
+ && count($history) === $case['historyBefore'] + 1,
+ $message);
+ };
+ $failedCreateReceipt = static function (
+ array $case,
+ string $eventId
+ ): array {
+ return [
+ 'type' => 'completion',
+ 'id' => $eventId,
+ 'agent_id' => (string) $case['job']->host_id,
+ 'operation_id' => (string) $case['job']->operation_id,
+ ...$case['identity'],
+ 'status' => 'failed',
+ 'effect_may_have_applied' => false,
+ 'safe_to_resubmit' => false,
+ 'error_code' => 'creation_absent',
+ 'result' => [
+ 'uuid' => $case['identity']['resource_id'],
+ 'creation_absent' => true,
+ 'managed_artifacts_absent' => true,
+ ],
+ ];
+ };
+
+ // Immutable host absence evidence must wait for a temporarily inconsistent
+ // local invoice. Keeping the event received allows the exact same receipt
+ // to finish atomically after the PHP ledger is repaired.
+ $invoiceGuardCase = $makeTimedOutCreate('receipt-invoice-guard');
+ $invoiceGuardCase['invoice']->purpose = 'renewal';
+ $invoiceGuardCase['invoice']->save();
+ $invoiceGuardReceipt = $failedCreateReceipt(
+ $invoiceGuardCase,
+ 'create-invoice-guard-failure-event-0001'
+ );
+ $invoiceGuardPending = CloudOperationCompletionService::accept(
+ $invoiceGuardReceipt
+ );
+ $invoiceGuardCase['job']->refresh();
+ $invoiceGuardCase['order']->refresh();
+ $invoiceGuardCase['reservation']->refresh();
+ $invoiceGuardCase['ip']->refresh();
+ $invoiceGuardEvent = \App\Models\CloudAgentCompletionEvent::findOrFail(
+ $invoiceGuardReceipt['id']
+ );
+ $assert($invoiceGuardPending === [
+ 'accepted' => true,
+ 'disposition' => 'received',
+ ]
+ && (string) $invoiceGuardEvent->disposition === 'received'
+ && $invoiceGuardEvent->applied_at === null
+ && (string) $invoiceGuardCase['job']->effect_resolution === 'unknown'
+ && $invoiceGuardCase['order']->refunded_at === null
+ && (string) $invoiceGuardCase['reservation']->state === 'uncertain'
+ && (string) $invoiceGuardCase['ip']->state === 'uncertain'
+ && (string) User::findOrFail(1)->money
+ === $invoiceGuardCase['walletBefore']
+ && (int) CloudOffer::findOrFail(1)->stock
+ === $invoiceGuardCase['stockBefore'],
+ 'Exact create failure bypassed an inconsistent local invoice: '
+ . json_encode([
+ 'accepted' => $invoiceGuardPending,
+ 'event_disposition' => $invoiceGuardEvent->disposition,
+ 'event_applied_at' => $invoiceGuardEvent->applied_at,
+ 'job_resolution' =>
+ $invoiceGuardCase['job']->effect_resolution,
+ 'refunded_at' => $invoiceGuardCase['order']->refunded_at,
+ 'reservation' => $invoiceGuardCase['reservation']->state,
+ 'ip' => $invoiceGuardCase['ip']->state,
+ 'wallet_before' => $invoiceGuardCase['walletBefore'],
+ 'wallet_after' => User::findOrFail(1)->money,
+ 'stock_before' => $invoiceGuardCase['stockBefore'],
+ 'stock_after' => CloudOffer::findOrFail(1)->stock,
+ ], JSON_THROW_ON_ERROR));
+ $invoiceGuardCase['invoice']->purpose = 'provision';
+ $invoiceGuardCase['invoice']->save();
+ $assert(CloudOperationCompletionService::accept($invoiceGuardReceipt)
+ === ['accepted' => true, 'disposition' => 'applied'],
+ 'Repaired local invoice did not resume the same create receipt.');
+ $assertCreateSettled(
+ $invoiceGuardCase,
+ 'Repaired local invoice did not atomically settle create finance.'
+ );
+
+ // The specialised create finalizer must validate the complete dispatched
+ // envelope itself. Generic recovery policy deliberately does not own
+ // create, so neither a damaged attempt count nor a missing receipt
+ // deadline may reach the refund path.
+ foreach ([
+ 'attempts' => 2,
+ 'receipt_deadline_at' => null,
+ ] as $damagedField => $damagedValue) {
+ $suffix = 'receipt-envelope-' . str_replace('_', '-', $damagedField);
+ $envelopeCase = $makeTimedOutCreate($suffix);
+ $eventId = 'create-envelope-' . str_replace('_', '-', $damagedField)
+ . '-event-0001';
+ $envelopeReceipt = $failedCreateReceipt($envelopeCase, $eventId);
+ $originalValue = $envelopeCase['job']->{$damagedField};
+ Capsule::table('cloud_jobs')->where(
+ 'id',
+ (int) $envelopeCase['job']->id
+ )->update([$damagedField => $damagedValue]);
+ $envelopePending = CloudOperationCompletionService::accept(
+ $envelopeReceipt
+ );
+ $envelopeCase['job']->refresh();
+ $envelopeCase['order']->refresh();
+ $envelopeCase['invoice']->refresh();
+ $envelopeCase['reservation']->refresh();
+ $envelopeCase['ip']->refresh();
+ $envelopeEvent = \App\Models\CloudAgentCompletionEvent::findOrFail(
+ $eventId
+ );
+ $assert($envelopePending === [
+ 'accepted' => true,
+ 'disposition' => 'received',
+ ]
+ && (string) $envelopeEvent->disposition === 'received'
+ && $envelopeEvent->applied_at === null
+ && (string) $envelopeCase['job']->effect_resolution === 'unknown'
+ && $envelopeCase['order']->refunded_at === null
+ && (string) $envelopeCase['invoice']->status === 'paid'
+ && (string) $envelopeCase['reservation']->state === 'uncertain'
+ && (string) $envelopeCase['ip']->state === 'uncertain'
+ && (string) User::findOrFail(1)->money
+ === $envelopeCase['walletBefore']
+ && (int) CloudOffer::findOrFail(1)->stock
+ === $envelopeCase['stockBefore'],
+ 'Damaged create dispatch envelope released finance: '
+ . $damagedField);
+ Capsule::table('cloud_jobs')->where(
+ 'id',
+ (int) $envelopeCase['job']->id
+ )->update([$damagedField => $originalValue]);
+ $assert(CloudOperationCompletionService::accept($envelopeReceipt)
+ === ['accepted' => true, 'disposition' => 'applied'],
+ 'Repaired create dispatch envelope did not resume receipt: '
+ . $damagedField);
+ $assertCreateSettled(
+ $envelopeCase,
+ 'Repaired dispatch envelope did not settle create finance: '
+ . $damagedField
+ );
+ }
+
+ // A fresh read-only Journal failure with complete absence projection can
+ // finish a timed-out create even when no completion event ever arrived.
+ $inspectionCase = $makeTimedOutCreate('inspection-no-receipt');
+ $makeRecoveryReady($inspectionCase['job']);
+ $inspectionClient = new FinancialCreateRecoveryAgentClient();
+ $inspectionClient->responses[] = $inspectionFor(
+ $inspectionCase['job'],
+ $inspectionCase['identity']
+ );
+ CloudAgentSettingsService::$hostId =
+ (int) $inspectionCase['job']->host_id;
+ CloudAgentSettingsService::$client = $inspectionClient;
+ $inspectionRecovery = new CloudAutomaticOperationRecoveryService();
+ $assert($inspectionRecovery->tick(),
+ 'No-receipt create was not claimed for read-only recovery.');
+ $assert(count($inspectionClient->requests) === 1
+ && $inspectionClient->requests[0]['method'] === 'GET'
+ && str_ends_with(
+ $inspectionClient->requests[0]['target'],
+ '/inspect'
+ )
+ && $inspectionClient->requests[0]['payload'] === null,
+ 'No-receipt create recovery used something other than one read-only '
+ . 'inspection.');
+ $assertCreateSettled(
+ $inspectionCase,
+ 'Read-only create absence evidence did not atomically settle finance.'
+ );
+
+ // Once inspection has settled the ledger, the first later failure receipt
+ // is an idempotent ACK of the late delivery. It must not re-enter the received queue,
+ // rewrite the job incident, or credit/release anything twice.
+ $inspectionCase['job']->refresh();
+ $inspectionJobUpdatedAt = (int) $inspectionCase['job']->updated_at;
+ $inspectionWalletAfter = (string) User::findOrFail(1)->money;
+ $inspectionStockAfter = (int) CloudOffer::findOrFail(1)->stock;
+ $inspectionLateReceipt = [
+ 'type' => 'completion',
+ 'id' => 'create-inspection-late-failure-event-0001',
+ 'agent_id' => (string) $inspectionCase['job']->host_id,
+ 'operation_id' => (string) $inspectionCase['job']->operation_id,
+ ...$inspectionCase['identity'],
+ 'status' => 'failed',
+ 'effect_may_have_applied' => false,
+ 'safe_to_resubmit' => false,
+ 'error_code' => 'creation_absent',
+ 'result' => [
+ 'uuid' => $inspectionCase['identity']['resource_id'],
+ 'creation_absent' => true,
+ 'managed_artifacts_absent' => true,
+ ],
+ ];
+ $inspectionLateAccepted = CloudOperationCompletionService::accept(
+ $inspectionLateReceipt
+ );
+ $inspectionLateDuplicate = CloudOperationCompletionService::accept(
+ $inspectionLateReceipt
+ );
+ $inspectionCase['job']->refresh();
+ $inspectionLateEvent = \App\Models\CloudAgentCompletionEvent::findOrFail(
+ $inspectionLateReceipt['id']
+ );
+ $assert($inspectionLateAccepted === [
+ 'accepted' => true,
+ 'disposition' => 'applied',
+ ]
+ && $inspectionLateDuplicate === $inspectionLateAccepted
+ && (string) $inspectionLateEvent->disposition === 'applied'
+ && $inspectionLateEvent->applied_at !== null
+ && (int) $inspectionCase['job']->updated_at
+ === $inspectionJobUpdatedAt
+ && !(bool) $inspectionCase['job']->finalization_incident
+ && (string) User::findOrFail(1)->money === $inspectionWalletAfter
+ && (int) CloudOffer::findOrFail(1)->stock === $inspectionStockAfter,
+ 'A failure receipt arriving after inspection settlement was requeued '
+ . 'or changed the financial ledger: ' . json_encode([
+ 'first' => $inspectionLateAccepted,
+ 'duplicate' => $inspectionLateDuplicate,
+ 'event_disposition' => $inspectionLateEvent->disposition,
+ 'event_applied_at' => $inspectionLateEvent->applied_at,
+ 'job_updated_at_before' => $inspectionJobUpdatedAt,
+ 'job_updated_at_after' => $inspectionCase['job']->updated_at,
+ 'job_incident' => $inspectionCase['job']->finalization_incident,
+ 'wallet_before' => $inspectionWalletAfter,
+ 'wallet_after' => User::findOrFail(1)->money,
+ 'stock_before' => $inspectionStockAfter,
+ 'stock_after' => CloudOffer::findOrFail(1)->stock,
+ ], JSON_THROW_ON_ERROR));
+
+ // An incomplete failed receipt is retained as a late audit event, then a
+ // fresh read-only inspection supplies the missing managed-artifact proof.
+ $incompleteCase = $makeTimedOutCreate('incomplete-receipt');
+ $incompleteReceipt = [
+ 'type' => 'completion',
+ 'id' => 'create-incomplete-failure-event-0001',
+ 'agent_id' => (string) $incompleteCase['job']->host_id,
+ 'operation_id' => (string) $incompleteCase['job']->operation_id,
+ ...$incompleteCase['identity'],
+ 'status' => 'failed',
+ 'effect_may_have_applied' => false,
+ 'safe_to_resubmit' => false,
+ 'error_code' => 'creation_absent',
+ 'result' => [
+ 'uuid' => $incompleteCase['identity']['resource_id'],
+ 'creation_absent' => true,
+ ],
+ ];
+ $assert(CloudOperationCompletionService::accept($incompleteReceipt)
+ === ['accepted' => true, 'disposition' => 'late'],
+ 'Incomplete create absence receipt was not ACKed as late.');
+ $makeRecoveryReady($incompleteCase['job']);
+ $incompleteClient = new FinancialCreateRecoveryAgentClient();
+ $incompleteClient->responses[] = $inspectionFor(
+ $incompleteCase['job'],
+ $incompleteCase['identity']
+ );
+ CloudAgentSettingsService::$client = $incompleteClient;
+ $assert((new CloudAutomaticOperationRecoveryService())->tick(),
+ 'Incomplete receipt create did not enter fresh inspection recovery.');
+ $assertCreateSettled(
+ $incompleteCase,
+ 'Fresh inspection did not finish an incomplete failed receipt create.'
+ );
+
+ // If PHP fails after durably preparing a no-fence conclusion, a later
+ // tick must finish locally even while the Agent is offline.
+ $noFenceRetry = $makeTimedOutCreate('nofence-db-retry');
+ $noFenceRetry['invoice']->purpose = 'renewal';
+ $noFenceRetry['invoice']->save();
+ $makeRecoveryReady($noFenceRetry['job']);
+ $noFenceClient = new FinancialCreateRecoveryAgentClient();
+ $noFenceClient->responses[] = $inspectionFor(
+ $noFenceRetry['job'],
+ $noFenceRetry['identity']
+ );
+ CloudAgentSettingsService::$client = $noFenceClient;
+ $assert((new CloudAutomaticOperationRecoveryService())->tick(),
+ 'No-fence DB-failure create was not claimed.');
+ $noFenceRetry['job']->refresh();
+ $noFenceRetry['order']->refresh();
+ $noFenceRetry['invoice']->refresh();
+ $assert(preg_match(
+ '/^prepared:create_not_applied:v1:[a-f0-9]{64}$/D',
+ (string) $noFenceRetry['job']->effect_resolution_reason
+ ) === 1
+ && (string) $noFenceRetry['job']->effect_resolution === 'unknown'
+ && $noFenceRetry['order']->refunded_at === null
+ && (string) $noFenceRetry['invoice']->status === 'paid'
+ && count($noFenceClient->requests) === 1,
+ 'No-fence financial failure did not retain its durable prepare.');
+ $noFenceRetry['invoice']->purpose = 'provision';
+ $noFenceRetry['invoice']->save();
+ $makeRecoveryReady($noFenceRetry['job']);
+ // Deliberately leave the response queue empty: any Agent call now fails.
+ $assert((new CloudAutomaticOperationRecoveryService())->tick(),
+ 'Prepared no-fence create was not retried locally.');
+ $assert(count($noFenceClient->requests) === 1,
+ 'Prepared no-fence retry contacted an offline Agent.');
+ $assertCreateSettled(
+ $noFenceRetry,
+ 'Prepared no-fence create did not settle after the DB fault cleared.'
+ );
+
+ // If the Agent fence write committed but the same PHP transaction later
+ // failed, recovery reads that immutable verdict once and never POSTs it a
+ // second time.
+ $fenceRetry = $makeTimedOutCreate('fence-db-retry');
+ $fenceRetry['invoice']->purpose = 'renewal';
+ $fenceRetry['invoice']->save();
+ $makeRecoveryReady($fenceRetry['job']);
+ $fenceClient = new FinancialCreateRecoveryAgentClient();
+ $fenceClient->responses[] = $inspectionFor(
+ $fenceRetry['job'],
+ $fenceRetry['identity'],
+ true
+ );
+ $fenceClient->responses[] = new CloudAgentError(
+ 'no committed fence',
+ 404,
+ 'not_found',
+ false,
+ 404
+ );
+ $committedFence = null;
+ $fenceClient->responses[] = static function (
+ array $request
+ ) use (&$committedFence, $fenceRetry): array {
+ $payload = $request['payload'];
+ $committedFence = [
+ 'operation_id' => (string) $fenceRetry['job']->operation_id,
+ 'resolution_id' => (string) $payload['resolution_id'],
+ 'resolution' => (string) $payload['resolution'],
+ 'inspection_checked_at' =>
+ (int) $payload['inspection_checked_at'],
+ 'reason_sha256' => (string) $payload['reason_sha256'],
+ 'resolved_at' => time(),
+ 'resolved_by' => 0,
+ 'replayed' => false,
+ ];
+ return $committedFence;
+ };
+ CloudAgentSettingsService::$client = $fenceClient;
+ $assert((new CloudAutomaticOperationRecoveryService())->tick(),
+ 'Active-fence DB-failure create was not claimed.');
+ $fenceRetry['job']->refresh();
+ $assert(is_array($committedFence)
+ && count($fenceClient->requests) === 3
+ && $fenceClient->requests[0]['method'] === 'GET'
+ && str_ends_with($fenceClient->requests[0]['target'], '/inspect')
+ && $fenceClient->requests[1]['method'] === 'GET'
+ && str_ends_with(
+ $fenceClient->requests[1]['target'],
+ '/fence-resolution'
+ )
+ && $fenceClient->requests[2]['method'] === 'POST'
+ && str_ends_with(
+ $fenceClient->requests[2]['target'],
+ '/resolve-fence'
+ )
+ && preg_match(
+ '/^prepared:create_not_applied:v1:[a-f0-9]{64}$/D',
+ (string) $fenceRetry['job']->effect_resolution_reason
+ ) === 1
+ && (string) $fenceRetry['job']->effect_resolution === 'unknown',
+ 'Fence commit plus PHP failure did not preserve resumable evidence.');
+ $fenceRetry['invoice']->purpose = 'provision';
+ $fenceRetry['invoice']->save();
+ $makeRecoveryReady($fenceRetry['job']);
+ $fenceClient->responses[] = $committedFence;
+ $assert((new CloudAutomaticOperationRecoveryService())->tick(),
+ 'Prepared active-fence create was not retried.');
+ $assert(count($fenceClient->requests) === 4
+ && $fenceClient->requests[3]['method'] === 'GET'
+ && str_ends_with(
+ $fenceClient->requests[3]['target'],
+ '/fence-resolution'
+ ),
+ 'Prepared active-fence retry wrote or re-inspected Agent state.');
+ $assertCreateSettled(
+ $fenceRetry,
+ 'Prepared active-fence create did not settle after DB recovery.'
+ );
+
+ // A fence write can fail before the Agent commits it. The next tick first
+ // proves that no prior verdict exists, then obtains a fresh inspection.
+ // checked_at is freshness metadata, not part of the durable conclusion:
+ // the same facts must produce the same marker and fence reason.
+ $stableCase = $makeTimedOutCreate('stable-inspection-marker');
+ $makeRecoveryReady($stableCase['job']);
+ $stableClient = new FinancialCreateRecoveryAgentClient();
+ $stableFirstInspection = $inspectionFor(
+ $stableCase['job'],
+ $stableCase['identity'],
+ true
+ );
+ $stableClient->responses = [
+ $stableFirstInspection,
+ new CloudAgentError('no committed fence', 404, 'not_found', false, 404),
+ new CloudAgentError('fence write unavailable', 503, 'unavailable', true, 503),
+ new CloudAgentError('no committed fence', 404, 'not_found', false, 404),
+ ];
+ CloudAgentSettingsService::$client = $stableClient;
+ $assert((new CloudAutomaticOperationRecoveryService())->tick(),
+ 'Stable-marker first recovery attempt was not claimed.');
+ $stableCase['job']->refresh();
+ $stableMarker = (string) $stableCase['job']->effect_resolution_reason;
+ $assert(preg_match(
+ '/^prepared:create_not_applied:v1:[a-f0-9]{64}$/D',
+ $stableMarker
+ ) === 1
+ && (string) $stableCase['job']->effect_resolution === 'unknown'
+ && count($stableClient->requests) === 4,
+ 'Failed fence write did not retain a stable prepared create marker.');
+ $makeRecoveryReady($stableCase['job']);
+ $stableSecondInspection = $stableFirstInspection;
+ $stableSecondInspection['checked_at'] =
+ (int) $stableFirstInspection['checked_at'] + 1;
+ $stableCommittedFence = null;
+ $stableClient->responses = [
+ new CloudAgentError('no committed fence', 404, 'not_found', false, 404),
+ $stableSecondInspection,
+ new CloudAgentError('no committed fence', 404, 'not_found', false, 404),
+ static function (array $request) use (
+ &$stableCommittedFence,
+ $stableCase
+ ): array {
+ $payload = $request['payload'];
+ $stableCommittedFence = [
+ 'operation_id' => (string) $stableCase['job']->operation_id,
+ 'resolution_id' => (string) $payload['resolution_id'],
+ 'resolution' => (string) $payload['resolution'],
+ 'inspection_checked_at' =>
+ (int) $payload['inspection_checked_at'],
+ 'reason_sha256' => (string) $payload['reason_sha256'],
+ 'resolved_at' => time(),
+ 'resolved_by' => 0,
+ 'replayed' => false,
+ ];
+ return $stableCommittedFence;
+ },
+ ];
+ $assert((new CloudAutomaticOperationRecoveryService())->tick(),
+ 'Stable-marker create did not resume from fresh inspection.');
+ $stableExpectedReason =
+ '自动恢复已确认原开通操作未创建虚拟机或面板管理资源;operation_id='
+ . (string) $stableCase['job']->operation_id . ';evidence='
+ . substr($stableMarker, -64);
+ $assert(is_array($stableCommittedFence)
+ && count($stableClient->requests) === 8
+ && $stableClient->requests[4]['method'] === 'GET'
+ && str_ends_with(
+ $stableClient->requests[4]['target'],
+ '/fence-resolution'
+ )
+ && $stableClient->requests[5]['method'] === 'GET'
+ && str_ends_with($stableClient->requests[5]['target'], '/inspect')
+ && $stableClient->requests[7]['method'] === 'POST'
+ && (string) $stableClient->requests[7]['payload']['reason_sha256']
+ === hash('sha256', $stableExpectedReason),
+ 'Fresh equivalent inspection changed the prepared fence reason or '
+ . 'replayed the original create.');
+ $assertCreateSettled(
+ $stableCase,
+ 'Stable-marker retry did not finish the create financial ledger.'
+ );
+
+ // Partial/residual host state and an identity mismatch are both
+ // contradictions, not refund evidence. They remain fenced and retryable.
+ $conflictCase = $makeTimedOutCreate('inspection-conflicts');
+ $makeRecoveryReady($conflictCase['job']);
+ $conflictClient = new FinancialCreateRecoveryAgentClient();
+ $partialInspection = $inspectionFor(
+ $conflictCase['job'],
+ $conflictCase['identity'],
+ true
+ );
+ $partialInspection['outcome'] = 'uncertain';
+ $partialInspection['summary_code'] = 'evidence_incomplete';
+ $partialInspection['effect_resolution'] = 'partial';
+ $partialInspection['error_code'] = 'cleanup_required';
+ $partialInspection['cleanup'] = [
+ 'available' => true,
+ 'plan_id' => str_repeat('a', 64),
+ 'steps' => ['delete_managed_volumes'],
+ ];
+ unset($partialInspection['effect_projection']);
+ $conflictClient->responses[] = $partialInspection;
+ CloudAgentSettingsService::$client = $conflictClient;
+ $assert((new CloudAutomaticOperationRecoveryService())->tick(),
+ 'Partial create evidence was not safely deferred.');
+ $conflictCase['job']->refresh();
+ $conflictCase['order']->refresh();
+ $conflictCase['invoice']->refresh();
+ $assert((string) $conflictCase['job']->effect_resolution === 'unknown'
+ && (bool) $conflictCase['job']->finalization_incident
+ && $conflictCase['order']->refunded_at === null
+ && (string) $conflictCase['invoice']->status === 'paid'
+ && (string) User::findOrFail(1)->money
+ === $conflictCase['walletBefore']
+ && (int) CloudOffer::findOrFail(1)->stock
+ === $conflictCase['stockBefore'],
+ 'Partial create evidence released a financial/resource fence.');
+ $makeRecoveryReady($conflictCase['job']);
+ $wrongIdentity = $inspectionFor(
+ $conflictCase['job'],
+ $conflictCase['identity']
+ );
+ $wrongIdentity['resource_id'] = 'wrong-create-resource';
+ $conflictClient->responses[] = $wrongIdentity;
+ $assert((new CloudAutomaticOperationRecoveryService())->tick(),
+ 'Create identity contradiction was not safely deferred.');
+ $conflictCase['job']->refresh();
+ $assert((string) $conflictCase['job']->effect_resolution === 'unknown'
+ && (bool) $conflictCase['job']->finalization_incident
+ && $conflictCase['order']->fresh()->refunded_at === null
+ && (string) User::findOrFail(1)->money
+ === $conflictCase['walletBefore']
+ && (int) CloudOffer::findOrFail(1)->stock
+ === $conflictCase['stockBefore'],
+ 'Create identity contradiction released a financial/resource fence.');
+
+ // A caller cannot reuse an automatic claim after its lease has expired.
+ $leaseToken = bin2hex(random_bytes(16));
+ Capsule::table('cloud_jobs')->where('id', (int) $conflictCase['job']->id)
+ ->update([
+ 'available_at' => 0,
+ 'lease_token' => $leaseToken,
+ 'lease_until' => time() + 180,
+ ]);
+ $staleClaim = CloudJob::findOrFail((int) $conflictCase['job']->id);
+ Capsule::table('cloud_jobs')->where('id', (int) $conflictCase['job']->id)
+ ->update(['lease_until' => time() - 1]);
+ $staleRequestCount = count($conflictClient->requests);
+ $staleRejected = false;
+ try {
+ CloudCreateFailureRecoveryService::recover($staleClaim);
+ } catch (CloudError $error) {
+ $staleRejected = true;
+ }
+ $assert($staleRejected
+ && count($conflictClient->requests) === $staleRequestCount
+ && (string) CloudJob::findOrFail(
+ (int) $conflictCase['job']->id
+ )->effect_resolution === 'unknown'
+ && $conflictCase['order']->fresh()->refunded_at === null,
+ 'Expired create recovery lease contacted Agent or released finance.');
+
echo "Cloud SQLite purchase, provisioning, refund, ambiguity, quota and tenant checks passed (crypto double only).\n";
}