From 813f9f58a6407da95dfb2a7ec437af8f33f2cfa7 Mon Sep 17 00:00:00 2001 From: codex Date: Wed, 21 Oct 2026 11:01:21 +0800 Subject: [PATCH] fix(cloud): recover failed provisioning after lost receipts Keep dispatched create failures eligible for dedicated recovery. Atomically settle exact absence receipts, or persist identity-bound absence evidence before resolving the Agent fence and refunding the original order. Preserve failed receipts across local ledger faults and resume fence commits without replaying create or refunding twice. Restrict system create fence resolution to confirmed not-applied outcomes and reject unreaped execution. Cover late receipts, database faults, invalid envelopes, crash recovery, duplicate delivery and unsafe evidence. --- agent/api/server.go | 87 ++ agent/api/server_test.go | 273 ++++- .../libvirt_operation_admin_test.go | 21 + agent/operation/journal.go | 14 +- agent/operation/journal_test.go | 91 +- docs/DEPLOYMENT.md | 2 + docs/TASK_EXECUTION.md | 6 +- .../CloudAdminOperationRecoveryService.php | 57 +- ...CloudAutomaticOperationRecoveryService.php | 47 +- .../CloudCreateFailureRecoveryService.php | 712 +++++++++++++ .../Cloud/CloudOperationRecoveryService.php | 22 +- ...CloudAdminOperationRecoveryServiceTest.php | 44 +- web/tests/CloudFinancialWorkflowTest.php | 946 ++++++++++++++++++ 13 files changed, 2262 insertions(+), 60 deletions(-) create mode 100644 web/app/Services/Cloud/CloudCreateFailureRecoveryService.php diff --git a/agent/api/server.go b/agent/api/server.go index 6f3c6155..d536d050 100644 --- a/agent/api/server.go +++ b/agent/api/server.go @@ -1083,6 +1083,7 @@ func (s *Server) routeOperations(w http.ResponseWriter, r *http.Request, segment return } } + applyExecutionFencedCreateInspection(&inspection, stored) // Identity always comes from the immutable Journal record, never from a // hypervisor adapter or its observation of current host state. inspection.OperationID = stored.OperationID @@ -1147,6 +1148,43 @@ func (s *Server) routeOperations(w http.ResponseWriter, r *http.Request, segment writeError(w, http.StatusBadRequest, "invalid_fence_resolution", err.Error(), nil) return } + if actor.SystemRecovery && actor.UserID == 0 && stored.Action == "create-vm" { + if request.Resolution != operation.FenceResolutionNotApplied { + writeError(w, http.StatusConflict, "operation_fence_not_resolvable", "automatic create recovery can only confirm that creation was not applied", nil) + return + } + if stored.ExecutionFenced { + writeError(w, http.StatusConflict, "create_absence_not_confirmed", "automatic create recovery cannot conclude while the original worker may still be running", nil) + return + } + fenceState, err := s.Journal.FenceState(operationID) + if err != nil { + writeMappedError(w, err, nil) + return + } + if fenceState == operation.FenceStateActive { + _, incidentFound, err := s.Journal.CompletionDeliveryIncident(operationID) + if err != nil { + writeMappedError(w, err, nil) + return + } + if incidentFound { + writeError(w, http.StatusConflict, "create_absence_not_confirmed", "automatic create recovery requires fresh complete host absence evidence", nil) + return + } + ctx, cancel := context.WithTimeout(r.Context(), s.OperationTimeout) + inspection, inspectErr := s.Hypervisor.InspectOperation(ctx, stored) + cancel() + if inspectErr != nil { + writeMappedError(w, inspectErr, nil) + return + } + if !systemCreateAbsenceConfirmed(stored, request, inspection) { + writeError(w, http.StatusConflict, "create_absence_not_confirmed", "automatic create recovery requires fresh complete host absence evidence", nil) + return + } + } + } resolution, replayed, err := s.Journal.ResolveResourceFence(operationID, actor.UserID, request) switch { case errors.Is(err, operation.ErrNotFound): @@ -1514,6 +1552,55 @@ func applyCompletionDeliveryIncident(inspection *hypervisor.OperationInspection, inspection.CompletionDeliveryErrorCode = incident.ErrorCode } +func applyExecutionFencedCreateInspection(inspection *hypervisor.OperationInspection, record operation.Record) { + if inspection == nil || record.Action != "create-vm" || !record.ExecutionFenced { + return + } + inspection.Outcome = hypervisor.InspectionOutcomeUncertain + inspection.SummaryCode = hypervisor.InspectionSummaryIncomplete + inspection.ErrorCode = "" + if record.Error != nil { + inspection.ErrorCode = record.Error.Code + } + inspection.EffectResolution = hypervisor.InspectionEffectInconclusive + inspection.EffectProjection = nil + inspection.AbortJob = hypervisor.OperationAbortInspection{} + inspection.Cleanup = hypervisor.OperationCleanupPlan{} +} + +// systemCreateAbsenceConfirmed accepts only a fresh, exact read-only host +// observation that both the VM and all panel-managed creation artifacts are +// absent. It does not infer absence from the original operation error and does +// not authorize any create replay or host mutation. +func systemCreateAbsenceConfirmed(record operation.Record, request operation.FenceResolutionRequest, + inspection hypervisor.OperationInspection) bool { + if record.Action != "create-vm" || record.State != operation.StateFailed || + inspection.OperationID != record.OperationID || inspection.Action != record.Action || + inspection.ResourceID != record.ResourceID || inspection.RequestFingerprint != record.Fingerprint || + inspection.OperationStatus != record.State || inspection.CheckedAt < request.InspectionCheckedAt || + inspection.Outcome != hypervisor.InspectionOutcomeFailed || + inspection.SummaryCode != hypervisor.InspectionSummaryFailed || + inspection.ErrorCode != "creation_absent" || + inspection.EffectResolution != hypervisor.InspectionEffectNotApplied || + inspection.CompletionDeliveryState != "" || inspection.CompletionDeliveryErrorCode != "" || + inspection.AbortJob.Available || inspection.Cleanup.Available || + len(inspection.EffectProjection) == 0 || len(inspection.EffectProjection) > 1024 { + return false + } + var projection map[string]json.RawMessage + if err := json.Unmarshal(inspection.EffectProjection, &projection); err != nil || len(projection) != 2 { + return false + } + var creationAbsent, managedArtifactsAbsent bool + if err := json.Unmarshal(projection["creation_absent"], &creationAbsent); err != nil || !creationAbsent { + return false + } + if err := json.Unmarshal(projection["managed_artifacts_absent"], &managedArtifactsAbsent); err != nil || !managedArtifactsAbsent { + return false + } + return true +} + func (s *Server) releaseAsync(operationID string) { s.asyncMu.Lock() defer s.asyncMu.Unlock() diff --git a/agent/api/server_test.go b/agent/api/server_test.go index 1e35ac7d..e260b90f 100644 --- a/agent/api/server_test.go +++ b/agent/api/server_test.go @@ -23,9 +23,11 @@ import ( type testHypervisor struct { hypervisor.Unsupported - mu sync.Mutex - inspections int - configureCalls int + mu sync.Mutex + inspections int + configureCalls int + inspectionResult *hypervisor.OperationInspection + inspectionErr error } func (h *testHypervisor) Ping(context.Context) error { return nil } @@ -43,8 +45,16 @@ func (h *testHypervisor) ConfigureStorage(context.Context, hypervisor.StorageCon func (h *testHypervisor) InspectOperation(_ context.Context, record operation.Record) (hypervisor.OperationInspection, error) { h.mu.Lock() + defer h.mu.Unlock() h.inspections++ - h.mu.Unlock() + if h.inspectionErr != nil { + return hypervisor.OperationInspection{}, h.inspectionErr + } + if h.inspectionResult != nil { + inspection := *h.inspectionResult + inspection.EffectProjection = append(json.RawMessage(nil), inspection.EffectProjection...) + return inspection, nil + } return hypervisor.OperationInspection{ OperationID: record.OperationID, OperationStatus: record.State, @@ -548,6 +558,7 @@ func TestSystemRecoveryActorCanOnlyInspectAndResolveItsExactOperation(t *testing {"raw operation", http.MethodGet, "/v1/operations/" + operationID, nil}, {"other operation", http.MethodGet, "/v1/operations/operation-other-recovery/inspect", nil}, {"host mutation", http.MethodPost, "/v1/storage/configuration", []byte(`{"operation_id":"operation-system-recovery"}`)}, + {"create mutation", http.MethodPost, "/v1/vms", []byte(`{"operation_id":"operation-system-recovery"}`)}, } { t.Run(test.name, func(t *testing.T) { response := api.request(t, test.method, test.target, test.body, actor) @@ -571,6 +582,260 @@ func TestSystemRecoveryActorCanOnlyInspectAndResolveItsExactOperation(t *testing } } +func TestSystemCreateRecoveryRequiresFreshCompleteAbsenceEvidence(t *testing.T) { + api := newTestAPI(t) + operationID := "operation-system-create-recovery" + resourceID := "123e4567-e89b-42d3-a456-426614174000" + requestBody := []byte(`{"operation_id":"operation-system-create-recovery"}`) + spec := operation.Spec{ + OperationID: operationID, + ResourceID: resourceID, + Action: "create-vm", + Fingerprint: operation.Fingerprint(http.MethodPost, "/v1/vms", requestBody), + Destructive: true, + } + if _, _, err := api.journal.Execute(context.Background(), spec, func(context.Context) (json.RawMessage, error) { + return nil, &worker.Error{ + Code: "result_unconfirmed", Message: "create result was not confirmed", + EffectMayHaveApplied: true, + } + }); err == nil { + t.Fatal("seed create failure unexpectedly succeeded") + } + actor := auth.Actor{UserID: 0, SystemRecovery: true, OperationID: operationID} + request := operation.FenceResolutionRequest{ + ResolutionID: "40404040404040404040404040404040", + Resolution: operation.FenceResolutionApplied, + InspectionCheckedAt: api.now.Unix(), + ReasonSHA256: "cdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcd", + Action: spec.Action, + ResourceID: spec.ResourceID, + RequestFingerprint: spec.Fingerprint, + } + encodeRequest := func() []byte { + t.Helper() + body, err := json.Marshal(request) + if err != nil { + t.Fatal(err) + } + return body + } + assertActive := func() { + t.Helper() + state, err := api.journal.FenceState(operationID) + if err != nil || state != operation.FenceStateActive { + t.Fatalf("create fence state=%q err=%v", state, err) + } + } + + applied := api.request( + t, + http.MethodPost, + "/v1/operations/"+operationID+"/resolve-fence", + encodeRequest(), + actor, + ) + if applied.Code != http.StatusConflict { + t.Fatalf("system create applied resolution = %d %s", applied.Code, applied.Body.String()) + } + assertActive() + + request.Resolution = operation.FenceResolutionNotApplied + inconclusive := api.request( + t, + http.MethodPost, + "/v1/operations/"+operationID+"/resolve-fence", + encodeRequest(), + actor, + ) + if inconclusive.Code != http.StatusConflict || + !bytes.Contains(inconclusive.Body.Bytes(), []byte(`"code":"create_absence_not_confirmed"`)) { + t.Fatalf("inconclusive create resolution = %d %s", inconclusive.Code, inconclusive.Body.String()) + } + assertActive() + + exactInspection := func(checkedAt int64) hypervisor.OperationInspection { + return hypervisor.OperationInspection{ + OperationID: operationID, + Action: spec.Action, + ResourceID: resourceID, + RequestFingerprint: spec.Fingerprint, + OperationStatus: operation.StateFailed, + CheckedAt: checkedAt, + Outcome: hypervisor.InspectionOutcomeFailed, + SummaryCode: hypervisor.InspectionSummaryFailed, + ErrorCode: "creation_absent", + EffectResolution: hypervisor.InspectionEffectNotApplied, + EffectProjection: json.RawMessage( + `{"creation_absent":true,"managed_artifacts_absent":true}`, + ), + } + } + stale := exactInspection(api.now.Add(-time.Second).Unix()) + api.hv.mu.Lock() + api.hv.inspectionResult = &stale + api.hv.mu.Unlock() + staleResponse := api.request( + t, + http.MethodPost, + "/v1/operations/"+operationID+"/resolve-fence", + encodeRequest(), + actor, + ) + if staleResponse.Code != http.StatusConflict || + !bytes.Contains(staleResponse.Body.Bytes(), []byte(`"code":"create_absence_not_confirmed"`)) { + t.Fatalf("stale create resolution = %d %s", staleResponse.Code, staleResponse.Body.String()) + } + assertActive() + + incomplete := exactInspection(api.now.Unix()) + incomplete.EffectProjection = json.RawMessage(`{"creation_absent":true,"managed_artifacts_absent":false}`) + api.hv.mu.Lock() + api.hv.inspectionResult = &incomplete + api.hv.mu.Unlock() + incompleteResponse := api.request( + t, + http.MethodPost, + "/v1/operations/"+operationID+"/resolve-fence", + encodeRequest(), + actor, + ) + if incompleteResponse.Code != http.StatusConflict || + !bytes.Contains(incompleteResponse.Body.Bytes(), []byte(`"code":"create_absence_not_confirmed"`)) { + t.Fatalf("incomplete create absence = %d %s", incompleteResponse.Code, incompleteResponse.Body.String()) + } + assertActive() + + exact := exactInspection(api.now.Unix()) + api.hv.mu.Lock() + api.hv.inspectionResult = &exact + api.hv.mu.Unlock() + inspection := api.request( + t, + http.MethodGet, + "/v1/operations/"+operationID+"/inspect", + nil, + actor, + ) + if inspection.Code != http.StatusOK || + !bytes.Contains(inspection.Body.Bytes(), []byte(`"error_code":"creation_absent"`)) || + !bytes.Contains(inspection.Body.Bytes(), []byte(`"effect_resolution":"not_applied"`)) { + t.Fatalf("system create inspection = %d %s", inspection.Code, inspection.Body.String()) + } + resolved := api.request( + t, + http.MethodPost, + "/v1/operations/"+operationID+"/resolve-fence", + encodeRequest(), + actor, + ) + if resolved.Code != http.StatusOK || + !bytes.Contains(resolved.Body.Bytes(), []byte(`"resolution":"not_applied"`)) || + !bytes.Contains(resolved.Body.Bytes(), []byte(`"resolved_by":0`)) { + t.Fatalf("system create not-applied resolution = %d %s", resolved.Code, resolved.Body.String()) + } + replayed := api.request( + t, + http.MethodPost, + "/v1/operations/"+operationID+"/resolve-fence", + encodeRequest(), + actor, + ) + if replayed.Code != http.StatusOK || + !bytes.Contains(replayed.Body.Bytes(), []byte(`"resolution":"not_applied"`)) || + !bytes.Contains(replayed.Body.Bytes(), []byte(`"replayed":true`)) { + t.Fatalf("system create resolution replay = %d %s", replayed.Code, replayed.Body.String()) + } + api.hv.mu.Lock() + inspections := api.hv.inspections + configureCalls := api.hv.configureCalls + api.hv.mu.Unlock() + if inspections != 5 || configureCalls != 0 { + t.Fatalf("system create recovery inspections=%d mutations=%d", inspections, configureCalls) + } +} + +func TestSystemCreateRecoveryRejectsExecutionFencedAbsence(t *testing.T) { + api := newTestAPI(t) + operationID := "operation-system-create-unreaped" + resourceID := "123e4567-e89b-42d3-a456-426614174002" + spec := operation.Spec{ + OperationID: operationID, + ResourceID: resourceID, + Action: "create-vm", + Fingerprint: operation.Fingerprint(http.MethodPost, "/v1/vms", []byte(`{"operation_id":"operation-system-create-unreaped"}`)), + Destructive: true, + } + if _, _, err := api.journal.Execute(context.Background(), spec, func(context.Context) (json.RawMessage, error) { + return nil, &worker.Error{ + Code: "worker_reap_unconfirmed", Message: "worker may still be running", + EffectMayHaveApplied: true, ExecutionFenced: true, + } + }); err == nil { + t.Fatal("seed execution-fenced create unexpectedly succeeded") + } + exactAbsence := hypervisor.OperationInspection{ + OperationID: operationID, + Action: spec.Action, + ResourceID: resourceID, + RequestFingerprint: spec.Fingerprint, + OperationStatus: operation.StateFailed, + CheckedAt: api.now.Unix(), + Outcome: hypervisor.InspectionOutcomeFailed, + SummaryCode: hypervisor.InspectionSummaryFailed, + ErrorCode: "creation_absent", + EffectResolution: hypervisor.InspectionEffectNotApplied, + EffectProjection: json.RawMessage(`{"creation_absent":true,"managed_artifacts_absent":true}`), + } + api.hv.mu.Lock() + api.hv.inspectionResult = &exactAbsence + api.hv.mu.Unlock() + actor := auth.Actor{UserID: 0, SystemRecovery: true, OperationID: operationID} + + inspected := api.request(t, http.MethodGet, "/v1/operations/"+operationID+"/inspect", nil, actor) + if inspected.Code != http.StatusOK { + t.Fatalf("execution-fenced create inspection = %d %s", inspected.Code, inspected.Body.String()) + } + var inspection hypervisor.OperationInspection + if err := json.Unmarshal(inspected.Body.Bytes(), &inspection); err != nil { + t.Fatal(err) + } + if inspection.Outcome != hypervisor.InspectionOutcomeUncertain || + inspection.SummaryCode != hypervisor.InspectionSummaryIncomplete || + inspection.EffectResolution != hypervisor.InspectionEffectInconclusive || + len(inspection.EffectProjection) != 0 { + t.Fatalf("execution-fenced create absence was treated as final: %#v", inspection) + } + + request := operation.FenceResolutionRequest{ + ResolutionID: "50505050505050505050505050505050", + Resolution: operation.FenceResolutionNotApplied, + InspectionCheckedAt: api.now.Unix(), + ReasonSHA256: "dededededededededededededededededededededededededededededededede", + Action: spec.Action, + ResourceID: spec.ResourceID, + RequestFingerprint: spec.Fingerprint, + } + body, err := json.Marshal(request) + if err != nil { + t.Fatal(err) + } + resolved := api.request(t, http.MethodPost, "/v1/operations/"+operationID+"/resolve-fence", body, actor) + if resolved.Code != http.StatusConflict || + !bytes.Contains(resolved.Body.Bytes(), []byte(`"code":"create_absence_not_confirmed"`)) { + t.Fatalf("execution-fenced create resolution = %d %s", resolved.Code, resolved.Body.String()) + } + if state, err := api.journal.FenceState(operationID); err != nil || state != operation.FenceStateActive { + t.Fatalf("execution-fenced create was released state=%q err=%v", state, err) + } + api.hv.mu.Lock() + inspections := api.hv.inspections + api.hv.mu.Unlock() + if inspections != 1 { + t.Fatalf("execution-fenced create resolve repeated host inspection: %d", inspections) + } +} + func TestMutationUsesIsolatedExecutorExactlyOnce(t *testing.T) { api := newTestAPI(t) executor := &fakeMutationExecutor{result: json.RawMessage(`{"configured":true}`)} diff --git a/agent/hypervisor/libvirt_operation_admin_test.go b/agent/hypervisor/libvirt_operation_admin_test.go index a468e659..2b2c2e1a 100644 --- a/agent/hypervisor/libvirt_operation_admin_test.go +++ b/agent/hypervisor/libvirt_operation_admin_test.go @@ -648,6 +648,27 @@ func TestAbsentCreateTrafficResidueRequiresExplicitCleanup(t *testing.T) { } } +func TestFailedCreateAlwaysRequiresLiveInspection(t *testing.T) { + for name, record := range map[string]operation.Record{ + "ordinary failure without effect flag": { + OperationID: "operation-create-ordinary-failure", ResourceID: "123e4567-e89b-42d3-a456-426614174000", + Action: "create-vm", State: operation.StateFailed, + Error: &operation.PublicError{Code: "operation_failed", Message: "create failed"}, + }, + "unknown effect failure": { + OperationID: "operation-create-unknown-effect", ResourceID: "123e4567-e89b-42d3-a456-426614174001", + Action: "create-vm", State: operation.StateFailed, EffectMayHaveApplied: true, + Error: &operation.PublicError{Code: "result_unconfirmed", Message: "create result was not confirmed"}, + }, + } { + t.Run(name, func(t *testing.T) { + if storedOperationInspectionAvailable(record) { + t.Fatal("failed create reused its original Journal error instead of requiring live host inspection") + } + }) + } +} + func TestBackupPushMatchesOnlyExactOwnedTarget(t *testing.T) { target := "/var/lib/virt-cloud/managed/snapshot.qcow2" matching := `` diff --git a/agent/operation/journal.go b/agent/operation/journal.go index fe987610..fc82b7bb 100644 --- a/agent/operation/journal.go +++ b/agent/operation/journal.go @@ -182,7 +182,8 @@ type Spec struct { // leaf action must be reviewed and added here explicitly. func SystemRecoveryActionAllowed(action string) bool { switch action { - case "power-start", "power-shutdown", "power-reboot", "power-reset", "power-destroy", + case "create-vm", + "power-start", "power-shutdown", "power-reboot", "power-reset", "power-destroy", "reinstall-vm", "enter-rescue", "force-start-rescue", "exit-rescue:connected", "exit-rescue:suspended", "change-network", "network-suspend", "network-resume", @@ -877,6 +878,17 @@ func (j *Journal) ResolveResourceFence(operationID string, resolvedBy int64, if resolvedBy == 0 && !SystemRecoveryActionAllowed(record.Action) { return ErrFenceNotResolvable } + // Automatic create recovery is deliberately one-way: fresh host + // absence evidence may release a failed create fence as not applied, + // but the zero-ID system actor can never declare that a create applied. + // Nor may it release a fence while an unreaped worker could still create + // the resource after the inspection. Materializing an applied create + // requires the original immutable completion result and remains PHP's + // responsibility. + if resolvedBy == 0 && record.Action == "create-vm" && + (record.ExecutionFenced || request.Resolution != FenceResolutionNotApplied) { + return ErrFenceNotResolvable + } if encoded := resolutions.Get([]byte(operationID)); encoded != nil { if err := json.Unmarshal(encoded, &resolution); err != nil || resolution.OperationID != operationID { return fmt.Errorf("decode fence resolution %q", operationID) diff --git a/agent/operation/journal_test.go b/agent/operation/journal_test.go index 79d15c74..53462489 100644 --- a/agent/operation/journal_test.go +++ b/agent/operation/journal_test.go @@ -1445,8 +1445,97 @@ func TestSystemFenceResolutionUsesZeroAuditActorButRejectsNegativeActors(t *test } } +func TestSystemCreateFenceResolutionOnlyAcceptsNotApplied(t *testing.T) { + j := openTestJournal(t) + now := time.Date(2026, 10, 21, 2, 0, 0, 0, time.UTC) + j.SetClockForTesting(func() time.Time { return now }) + newCreate := func(operationID, resourceID string) Spec { + return Spec{ + OperationID: operationID, + ResourceID: resourceID, + Action: "create-vm", + Fingerprint: Fingerprint("POST", "/v1/vms", []byte(`{"operation_id":"`+operationID+`"}`)), + Destructive: true, + } + } + seedUnknown := func(spec Spec) { + t.Helper() + if _, _, err := j.Execute(context.Background(), spec, func(context.Context) (json.RawMessage, error) { + return nil, testFailure{code: "result_unconfirmed", message: "create result is unknown", effect: true} + }); err == nil { + t.Fatal("seed create failure unexpectedly succeeded") + } + } + requestFor := func(spec Spec, resolutionID, resolution string) FenceResolutionRequest { + return FenceResolutionRequest{ + ResolutionID: resolutionID, + Resolution: resolution, + InspectionCheckedAt: now.Unix(), + ReasonSHA256: "abababababababababababababababababababababababababababababababab", + Action: spec.Action, + ResourceID: spec.ResourceID, + RequestFingerprint: spec.Fingerprint, + } + } + + systemSpec := newCreate("operation-system-create", "vm-system-create") + seedUnknown(systemSpec) + applied := requestFor( + systemSpec, + "10101010101010101010101010101010", + FenceResolutionApplied, + ) + if _, _, err := j.ResolveResourceFence(systemSpec.OperationID, 0, applied); !errors.Is(err, ErrFenceNotResolvable) { + t.Fatalf("system create applied resolution err=%v", err) + } + if state, err := j.FenceState(systemSpec.OperationID); err != nil || state != FenceStateActive { + t.Fatalf("rejected system create resolution released fence state=%q err=%v", state, err) + } + notApplied := applied + notApplied.ResolutionID = "20202020202020202020202020202020" + notApplied.Resolution = FenceResolutionNotApplied + resolved, replayed, err := j.ResolveResourceFence(systemSpec.OperationID, 0, notApplied) + if err != nil || replayed || resolved.Resolution != FenceResolutionNotApplied || resolved.ResolvedBy != 0 { + t.Fatalf("system create not-applied resolution=%#v replayed=%v err=%v", resolved, replayed, err) + } + + manualSpec := newCreate("operation-manual-create", "vm-manual-create") + seedUnknown(manualSpec) + manualApplied := requestFor( + manualSpec, + "30303030303030303030303030303030", + FenceResolutionApplied, + ) + resolved, replayed, err = j.ResolveResourceFence(manualSpec.OperationID, 42, manualApplied) + if err != nil || replayed || resolved.Resolution != FenceResolutionApplied || resolved.ResolvedBy != 42 { + t.Fatalf("manual create applied resolution=%#v replayed=%v err=%v", resolved, replayed, err) + } + + fencedSpec := newCreate("operation-system-create-unreaped", "vm-system-create-unreaped") + if _, _, err := j.Execute(context.Background(), fencedSpec, func(context.Context) (json.RawMessage, error) { + return nil, testFailure{ + code: "worker_reap_unconfirmed", message: "worker may still be running", + effect: true, fence: true, + } + }); err == nil { + t.Fatal("seed execution-fenced create unexpectedly succeeded") + } + fencedRequest := requestFor( + fencedSpec, + "40404040404040404040404040404040", + FenceResolutionNotApplied, + ) + if _, _, err := j.ResolveResourceFence(fencedSpec.OperationID, 0, fencedRequest); !errors.Is(err, ErrFenceNotResolvable) { + t.Fatalf("system execution-fenced create resolution err=%v", err) + } + if state, err := j.FenceState(fencedSpec.OperationID); err != nil || state != FenceStateActive { + t.Fatalf("execution-fenced create was released state=%q err=%v", state, err) + } +} + func TestSystemRecoveryActionAllowlistIsExplicit(t *testing.T) { allowed := []string{ + "create-vm", "power-start", "power-shutdown", "power-reboot", "power-reset", "power-destroy", "reinstall-vm", "enter-rescue", "force-start-rescue", "exit-rescue:connected", "exit-rescue:suspended", @@ -1469,7 +1558,7 @@ func TestSystemRecoveryActionAllowlistIsExplicit(t *testing.T) { } denied := []string{ - "", "create-vm", "configure-storage", "service-resume", "future-unknown", "power-future", + "", "configure-storage", "service-resume", "future-unknown", "power-future", "exit-rescue:future", "create-snapshot:0123456789abcdef0123456789abcdef:future", "create-snapshot:0123456789ABCDEF0123456789ABCDEF:filesystem", diff --git a/docs/DEPLOYMENT.md b/docs/DEPLOYMENT.md index a9ce5ec3..a9099d88 100644 --- a/docs/DEPLOYMENT.md +++ b/docs/DEPLOYMENT.md @@ -162,6 +162,8 @@ systemctl enable --now virt-cloud-worker.service \ 普通 worker 负责购买、创建、重装、邮件和最终删除;至少两个策略 worker 处理到期断网、网络恢复与删除前策略;lifecycle 生成续费账单并推进账期、流量、宽限和删除状态。`virt-cloud-recovery-worker.service` 是必需的独立 operation 恢复进程,覆盖策略表中的实例电源、重装/救援、公网 IP、实例设置、快照、防火墙、删除、生命周期网络、模板与 ISO Agent 叶子任务。它只做 operation-bound inspection/栅栏审计、持久晚到回执的 PHP 收敛或聚合父任务的本地汇总,不投递邮件,也绝不重放原 mutation。离线、身份冲突或账本异常按固定低频间隔持续核对,没有次数或总时长上限。它与普通队列隔离,避免某个离线 Agent 的核对占住正常 worker。 +恢复进程也覆盖已派发开通任务的失败收尾:完整缺失证据回执可直接完成本地退款和资源释放;回执不足时,只读检查原 operation,确认实例及托管资源均不存在后才收尾。升级此能力需同时更新面板代码与宿主机 Agent,并重启已有 worker/recovery worker 使其加载新代码;无需数据库迁移或新增 service。旧 Agent 拒绝开通自动检查时会保留订单占用并低频重试,不会绕过授权退款。已有失败且效果未知的开通订单会被后台领取,不必重发开通请求。 + 在面板机器构建并安装 Agent Gateway: ```bash diff --git a/docs/TASK_EXECUTION.md b/docs/TASK_EXECUTION.md index 8b21eb13..973a56ae 100644 --- a/docs/TASK_EXECUTION.md +++ b/docs/TASK_EXECUTION.md @@ -51,7 +51,9 @@ Go context 不能可靠中止已经进入 CGo/libvirt 的调用,因此每个 完成回执是不可变事件,不是新的变更请求。Agent 持续投递同一 `event_id`,直到 PHP 返回 ACK;重启、旧 outbox 中保存的尝试次数或历史过期时间都不能淘汰未确认回执。PHP 以 `event_id`、`operation_id` 和正文摘要幂等接收。ACK 只删除投递 outbox,不删除主 Journal 记录。 -PHP 的 `receipt_deadline_at` 到期时直接把任务终态化为失败。之后收到相同回执不能更新原任务终态;除开通任务的精确成功回执专用路径外,不支持恢复的类型只返回晚到 ACK。对策略表允许的任务,精确匹配 `agent_id + operation_id + event_id + 正文摘要` 的成功回执会先以原加密请求收敛对应 PHP 业务账本,再提交 `effect_resolution=applied`;明确无副作用的失败回执则收敛为 `not_applied`。开通任务不进入通用 inspection、fence 或 mutation replay;只有已持久化且身份完全匹配的成功回执,才能在重新验证未退款订单、已支付开通发票、库存、IP、placement、服务周期和凭据 escrow 后原子补完本地账本。该回执路径只使用 PHP 已持久化事件,不联系 Agent;任务效果结论与事件 ACK 在同一事务提交,结论提交后立即清除恢复密文。如果进程在事件提交后或预提交后崩溃,专用 recovery worker 会从 terminal job 出发,以已有 `(host_id,operation_id)` 唯一索引精确读取 `received` 事件,重算正文摘要后继续 PHP 提交。它只有在既有 prepared 证据同方向、同完整 operation hash 且栅栏语义一致时才可继续收敛;方向、哈希、ID、栅栏语义、密文或账本状态冲突时,必须记录具体 `recovery_*` 异常并保留 blocker/escrow。异常事件可以被隔离,但任务仍按固定低频退避留在恢复队列,修复数据、身份、网络或 Agent 后继续核对;不存在 6 小时、12 次或其他永久退出上限。它不扫描事件全表,也不依赖 Agent 再次投递。相同回执重传只返回幂等 ACK;不同正文复用事件 ID 或 operation ID 必须拒绝并记录安全日志。 +PHP 的 `receipt_deadline_at` 到期时直接把任务终态化为失败。之后收到相同回执不能更新原任务终态;不支持恢复的类型只返回晚到 ACK。对策略表允许的任务,精确匹配 `agent_id + operation_id + event_id + 正文摘要` 的成功回执会先以原加密请求收敛对应 PHP 业务账本,再提交 `effect_resolution=applied`;明确无副作用的失败回执则收敛为 `not_applied`。回执路径只使用 PHP 已持久化事件,不联系 Agent;任务效果结论与事件 ACK 在同一事务提交,结论提交后立即清除恢复密文。如果进程在事件提交后或预提交后崩溃,专用 recovery worker 会从 terminal job 出发,以已有 `(host_id,operation_id)` 唯一索引精确读取 `received` 事件,重算正文摘要后继续 PHP 提交。它只有在既有 prepared 证据同方向、同完整 operation hash 且栅栏语义一致时才可继续收敛;方向、哈希、ID、栅栏语义、密文或账本状态冲突时,必须记录具体 `recovery_*` 异常并保留 blocker/escrow。异常事件可以被隔离,但任务仍按固定低频退避留在恢复队列,修复数据、身份、网络或 Agent 后继续核对;不存在 6 小时、12 次或其他永久退出上限。它不扫描事件全表,也不依赖 Agent 再次投递。相同回执重传只返回幂等 ACK;不同正文复用事件 ID 或 operation ID 必须拒绝并记录安全日志。 + +开通任务使用独立的资金恢复终结器,不加入通用效果物化策略,也不重放创建请求。精确成功回执在重新验证未退款订单、已支付开通发票、库存、IP、placement、服务周期和凭据 escrow 后原子补完实例账本。晚到失败回执必须同时证明 `creation_absent=true`、`managed_artifacts_absent=true` 且没有未知副作用,才能原子退款、释放库存/IP/预留并 ACK;仅有失败状态或 `safe_to_resubmit` 不足以退款。没有完整失败回执时,恢复 worker 仍会领取该订单,使用绑定原 operation 的 system-recovery 身份只读检查;只有新鲜、身份一致且明确的 `creation_absent` 证据才进入专用预提交。预提交持久化证据但不提前退款,确认必要的 Agent 栅栏审计后再原子提交资金和资源账本。Agent 已提交栅栏但 PHP 提交失败时,后续轮次可依据同一预提交读取既有栅栏结论并继续,不会重新开通或重复退款。系统身份对开通只允许 `not_applied` 栅栏结论,不能凭只读状态推断成功并物化实例;残留资源、运行中操作、身份冲突、无效证据和矛盾成功回执继续保留保护。此能力需更新面板和 Agent,并保持 recovery worker 运行;不需要数据库迁移或新增 systemd 服务。 任何回执或 inspection 改变 PHP 账本前,都必须重新验证加密请求、任务/宿主机/资源身份,以及该种类终结器会消费的请求与结果字段;可解密但不完整的 escrow 不得以空账本假收敛。prepared 证据绑定规范化后的完整 operation;重试次数只属于调度状态,不参与 immutable operation hash,也不能破坏 prepare 后的崩溃续跑。如果恢复必须写 Agent 栅栏审计结论,该有界网络写入与随后的 PHP effect commit 必须在同一 job-row 锁下串行,写前再次验证租约、incident 和优先回执,防止陈旧 worker 在冲突证据落盘后解除 Agent fence。 @@ -63,7 +65,7 @@ PHP 的 `receipt_deadline_at` 到期时直接把任务终态化为失败。之 - 任务本身立即结束,不再轮询、不再自动重试;内部恢复协调器只能低频读取允许种类的同一 Agent operation 证据; - 同一 VM、卷、网络过滤器、地址或账单的冲突变更立即返回明确错误,不进入等待队列; -- 不自动退款、不释放 reservation、不再次删除磁盘; +- 在可信清理证据确认前不自动退款、不释放 reservation、不再次删除磁盘; - 管理员可以执行独立的只读检查,但检查不会翻转原任务终态,也不会续作原 mutation; - 策略表中的 Agent 叶子任务可以自动消费同一 operation 的不可变证据:晚到终态回执是纯 PHP 收敛;没有回执时,由独立的 `cloudWorker --recovery-only` 进程每轮最多领取一项。常驻进程在精确 `received` 事件、普通叶子任务、聚合任务三个通道间轮转,空通道同轮跳过;持续有任务时,每三个完成的 tick 给每个通道一次机会,避免回执或叶子队列持续非空导致聚合任务饥饿。普通/策略 worker 从不联系恢复目标,因此离线宿主的核对不会阻塞正常派发。恢复使用绑定 operation ID 的 system-recovery actor 调用 Agent 只读 `inspect`,且只接受新鲜、种类专用、资源身份一致的 `applied/not_applied` 证据。它先预提交 PHP 账本并继续保留 blocker;只有活动失败栅栏才写入一次 operation-bound Journal 审计结论,再提交 `effect_resolution`。原 mutation 永不调用。`partial/inconclusive`、Agent 离线、ID 冲突、无效回执、密文损坏或账本/实例状态矛盾都会保留 blocker,向页面公开具体异常,并按固定低频退避持续重检;没有次数和总时长上限。修复根因后,同一 operation 可继续自动收敛并清除 incident;相互矛盾的持久证据不得被覆盖或猜测解锁; - 超级管理员只能对已有种类专用原子终结器的任务提交审计结论。开通任务仅支持由新鲜宿主机缺失证据确认 `not_applied`,并在同一事务退款、释放库存/IP/预留后再解栅栏;支付任务只根据签名回调或外部支付平台审计证据收敛本地账本,不重新下单; diff --git a/web/app/Services/Cloud/CloudAdminOperationRecoveryService.php b/web/app/Services/Cloud/CloudAdminOperationRecoveryService.php index fc31fab0..180e5e77 100644 --- a/web/app/Services/Cloud/CloudAdminOperationRecoveryService.php +++ b/web/app/Services/Cloud/CloudAdminOperationRecoveryService.php @@ -184,9 +184,9 @@ final class CloudAdminOperationRecoveryService [$job, $client] = self::jobContext( $id, 30.0, - CloudOperationRecoveryPolicy::kinds() + self::automaticRecoveryKinds() ); - if (!CloudOperationRecoveryPolicy::supports((string) $job->kind) + if (!self::supportsAutomaticRecovery($job) || (string) $job->execution_mode !== 'agent' || (string) $job->phase !== 'terminal' || (string) $job->outcome !== 'failed' @@ -238,8 +238,20 @@ final class CloudAdminOperationRecoveryService [$job, $client] = self::jobContext( $id, 30.0, - CloudOperationRecoveryPolicy::kinds() + self::automaticRecoveryKinds() ); + if ((string) $job->kind === 'create') { + CloudCreateFailureRecoveryService::assertPreparedForFence( + $job, + $resolution, + $reason + ); + CloudCreateFailureFinalizer::assertResolutionSupported( + $job, + $resolution, + $inspection + ); + } $checkedAt = $inspection['checked_at'] ?? null; $identity = CloudOperationRecoveryIdentity::expected( $job, @@ -251,7 +263,7 @@ final class CloudAdminOperationRecoveryService 'request_fingerprint' => $inspection['request_fingerprint'] ?? null, ]); - if (!CloudOperationRecoveryPolicy::supports((string) $job->kind) + if (!self::supportsAutomaticRecovery($job) || (string) $job->execution_mode !== 'agent' || (string) $job->phase !== 'terminal' || (string) $job->outcome !== 'failed' @@ -260,9 +272,10 @@ final class CloudAdminOperationRecoveryService || !(bool) $job->effect_may_have_applied || (bool) $job->safe_to_resubmit || (string) $job->effect_resolution !== 'unknown' - || !self::automaticPreparedReason( - (string) $job->effect_resolution_reason - ) + || ((string) $job->kind !== 'create' + && !self::automaticPreparedReason( + (string) $job->effect_resolution_reason + )) || ($inspection['operation_id'] ?? null) !== (string) $job->operation_id || ($inspection['effect_resolution'] ?? null) !== $resolution @@ -316,9 +329,16 @@ final class CloudAdminOperationRecoveryService [$job, $client] = self::jobContext( $id, 30.0, - CloudOperationRecoveryPolicy::kinds() + self::automaticRecoveryKinds() ); - if (!CloudOperationRecoveryPolicy::supports((string) $job->kind) + if ((string) $job->kind === 'create') { + CloudCreateFailureRecoveryService::assertPreparedForFence( + $job, + $resolution, + $reason + ); + } + if (!self::supportsAutomaticRecovery($job) || (string) $job->execution_mode !== 'agent' || (string) $job->phase !== 'terminal' || (string) $job->outcome !== 'failed' @@ -327,9 +347,10 @@ final class CloudAdminOperationRecoveryService || !(bool) $job->effect_may_have_applied || (bool) $job->safe_to_resubmit || (string) $job->effect_resolution !== 'unknown' - || !self::automaticPreparedReason( - (string) $job->effect_resolution_reason - )) { + || ((string) $job->kind !== 'create' + && !self::automaticPreparedReason( + (string) $job->effect_resolution_reason + ))) { throw new CloudError('任务尚未完成可验证的账本预提交', 409); } // A prepared escrow persists the identity validated before the PHP @@ -568,6 +589,18 @@ final class CloudAdminOperationRecoveryService ) === 1; } + /** Create is inspectable, but never enters generic effect finalization. */ + private static function automaticRecoveryKinds(): array + { + return [...CloudOperationRecoveryPolicy::kinds(), 'create']; + } + + private static function supportsAutomaticRecovery(CloudJob $job): bool + { + return CloudOperationRecoveryPolicy::supports((string) $job->kind) + || ((string) $job->kind === 'create' && $job->order_id !== null); + } + private static function assertCompletionDeliveryUsable( array $inspection ): void { diff --git a/web/app/Services/Cloud/CloudAutomaticOperationRecoveryService.php b/web/app/Services/Cloud/CloudAutomaticOperationRecoveryService.php index 8298383a..60c03d39 100644 --- a/web/app/Services/Cloud/CloudAutomaticOperationRecoveryService.php +++ b/web/app/Services/Cloud/CloudAutomaticOperationRecoveryService.php @@ -544,7 +544,10 @@ final class CloudAutomaticOperationRecoveryService ): void { $policy->whereIn( 'kind', - CloudOperationRecoveryPolicy::kinds() + // Create uses a separate financial + // finalizer, never the generic replay or + // effect-projection path below. + [...CloudOperationRecoveryPolicy::kinds(), 'create'] )->where( static function ($ready) use ( $now, @@ -580,21 +583,6 @@ final class CloudAutomaticOperationRecoveryService } ); } - )->orWhere( - static function ($create): void { - // Create keeps its specialised financial - // finalizer. It is recoverable here only from - // a durable exact completion receipt; it must - // never enter generic inspection or replay. - $create->where('kind', 'create') - ->whereExists( - static function ($events): void { - self::receivedCreateSuccessEventExists( - $events - ); - } - ); - } ); }) ->when( @@ -703,13 +691,6 @@ final class CloudAutomaticOperationRecoveryService 'operation_id', (string) $job->operation_id )->where('disposition', 'received') - ->when( - (string) $job->kind === 'create', - static fn ($events) => $events->where( - 'outcome', - 'succeeded' - ) - ) ->lockForUpdate() ->first(); if ($event !== null) { @@ -726,6 +707,16 @@ final class CloudAutomaticOperationRecoveryService (string) $job->effect_resolution_reason, $now ); + // The dedicated create coordinator owns its encrypted + // prepare/commit binding. Never wrap or replace that marker + // when resuming a fence-write/financial-commit crash window. + if ((string) $job->kind === 'create' + && preg_match( + '/^prepared:create_not_applied:v1:[a-f0-9]{64}$/D', + (string) $job->effect_resolution_reason + ) === 1) { + $attempt = (string) $job->effect_resolution_reason; + } if ($attempt === false) { $job->finalization_incident = 1; $job->finalization_error_code = @@ -792,12 +783,6 @@ final class CloudAutomaticOperationRecoveryService ->where('recovery_events.disposition', 'received'); } - private static function receivedCreateSuccessEventExists($events): void - { - self::receivedEventExists($events); - $events->where('recovery_events.outcome', 'succeeded'); - } - /** * @return array{delay:int,error_code:string}|null Retry state, or null * after a committed resolution. @@ -813,6 +798,10 @@ final class CloudAutomaticOperationRecoveryService )) { throw new CloudError('自动核对租约已失效', 409); } + if ((string) $job->kind === 'create') { + CloudCreateFailureRecoveryService::recover($job); + return null; + } $context = self::recoveryContext($job); // A late immutable completion receipt is a PHP-local two-phase diff --git a/web/app/Services/Cloud/CloudCreateFailureRecoveryService.php b/web/app/Services/Cloud/CloudCreateFailureRecoveryService.php new file mode 100644 index 00000000..408c9af4 --- /dev/null +++ b/web/app/Services/Cloud/CloudCreateFailureRecoveryService.php @@ -0,0 +1,712 @@ +getConnection()->transaction( + static function () use ($source, $sourceEvent): string { + [$user, $order, $job] = self::lockLedger($source); + $event = CloudAgentCompletionEvent::where( + 'event_id', + (string) $sourceEvent->event_id + )->lockForUpdate()->first(); + if ($event === null) { + return 'received'; + } + if ((string) $event->disposition !== 'received') { + return (string) $event->disposition; + } + if ((int) $event->host_id !== (int) $job->host_id + || !hash_equals( + (string) $job->operation_id, + (string) $event->operation_id + )) { + throw new CloudRecoveryOperationIdentityConflict( + '开通失败回执与原任务不匹配', + 409 + ); + } + CloudOperationCompletionService::assertEventIdentityForJob( + $job, + $event + ); + $operation = CloudOperationCompletionService::operation($event); + if (self::settledNotApplied($job, $order, $event)) { + // A fresh inspection may have completed the atomic PHP + // finalizer before the Agent could deliver its immutable + // failure. The receipt no longer decides the ledger, but + // the exact identity and committed audit allow it to be + // ACKed without reopening or rewriting that resolution. + $event->disposition = 'applied'; + $event->applied_at = time(); + $event->save(); + return 'applied'; + } + self::assertRecoverableJob($job); + if (!self::receiptProvesAbsence($event, $operation)) { + $event->disposition = 'late'; + $event->applied_at = null; + $event->save(); + return 'late'; + } + + self::assertNoSuccessfulReceiptLocked($job); + self::finalizeLocked( + $job, + $user, + $order, + 'late_completion_receipt:' . (string) $event->event_id, + $event + ); + return 'applied'; + }, + 3 + ); + } + + /** Recover one lease-owned create without replaying its Agent mutation. */ + public static function recover(CloudJob $claimed): void + { + $job = self::claimedJob($claimed); + $prepared = self::preparedEvidence($job); + if ($prepared !== null) { + if (!$prepared['fence_resolution_required']) { + self::finishPrepared($claimed, $prepared, false); + return; + } + try { + self::finishPrepared($claimed, $prepared, null); + return; + } catch (CloudAgentError $error) { + if ($error->agentHttpStatus !== 404) { + throw $error; + } + // No fence verdict was committed before the prior PHP crash. + // A new write requires new, bounded read-only evidence. + } + } + + $inspection = CloudAdminOperationRecoveryService::inspectForRecovery( + (string) $job->id + ); + $prepared = self::prepareInspection($claimed, $inspection); + self::finishPrepared( + $claimed, + $prepared, + $prepared['fence_resolution_required'] ? $inspection : false + ); + } + + /** + * Authenticate the durable PHP intent before an automatic Agent fence read + * or write. This is deliberately create-specific and does not broaden the + * generic recovery allow-list. + */ + public static function assertPreparedForFence( + CloudJob $job, + string $resolution, + string $reason + ): void { + if ($resolution !== 'not_applied') { + throw new CloudError('开通任务只支持确认未应用', 409); + } + self::assertRecoverableJob($job); + $prepared = self::preparedEvidence($job); + if ($prepared === null + || !hash_equals($prepared['reason'], $reason)) { + throw new CloudRecoveryEvidenceConflict( + '开通任务的恢复预提交证据不匹配', + 409 + ); + } + } + + /** + * @return array{marker:string,reason:string,identity:array,inspection:array,fence_resolution_required:bool} + */ + private static function prepareInspection( + CloudJob $claimed, + array $inspection + ): array { + $job = self::claimedJob($claimed); + $identity = CloudOperationRecoveryIdentity::expected($job, false); + CloudOperationRecoveryIdentity::assertMatches($identity, [ + 'action' => $inspection['action'] ?? null, + 'resource_id' => $inspection['resource_id'] ?? null, + 'request_fingerprint' => + $inspection['request_fingerprint'] ?? null, + ]); + self::assertInspectionProvesAbsence($job, $inspection); + $evidence = self::inspectionEvidence($inspection); + $marker = self::inspectionMarker($evidence); + $reason = self::fenceReason($job, $marker); + + return (new CloudJob())->getConnection()->transaction( + static function () use ( + $claimed, + $identity, + $evidence, + $marker, + $reason + ): array { + [$user, $order, $locked] = self::lockLedger($claimed); + self::assertClaimedJob($locked, $claimed); + self::assertNoSuccessfulReceiptLocked($locked); + + $payload = self::payload($locked); + $lockedIdentity = CloudOperationRecoveryIdentity:: + expectedFromPayload($locked, $payload, false); + CloudOperationRecoveryIdentity::assertMatches( + $lockedIdentity, + $identity + ); + $payload[self::ESCROW_KEY] = [ + 'version' => self::ESCROW_VERSION, + 'marker' => $marker, + 'reason' => $reason, + 'identity' => CloudOperationRecoveryIdentity::persisted( + $lockedIdentity + ), + 'inspection' => $evidence, + ]; + $encoded = json_encode( + $payload, + JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES + | JSON_UNESCAPED_UNICODE + ); + if (strlen($encoded) > 196608) { + throw new CloudRecoveryPayloadUnavailable( + '开通任务恢复预提交数据过大', + 409 + ); + } + $locked->effect_resolution_reason = $marker; + $locked->sealed_payload = CloudSecrets::encrypt( + $payload, + 'job-payload' + ); + $locked->updated_at = time(); + $locked->save(); + + return [ + 'marker' => $marker, + 'reason' => $reason, + 'identity' => $lockedIdentity, + 'inspection' => $evidence, + 'fence_resolution_required' => (bool) + $evidence['fence_resolution_required'], + ]; + }, + 3 + ); + } + + /** + * Resolve or verify the Agent fence while holding the canonical financial + * locks, then commit every PHP ledger transition in that same transaction. + * + * @param array{marker:string,reason:string,identity:array,inspection:array,fence_resolution_required:bool} $prepared + * @param array|false|null $inspection Fresh evidence, false when no fence + * is required, null when resuming an already committed fence resolution. + */ + private static function finishPrepared( + CloudJob $claimed, + array $prepared, + array|false|null $inspection + ): void { + (new CloudJob())->getConnection()->transaction( + static function () use ($claimed, $prepared, $inspection): void { + [$user, $order, $job] = self::lockLedger($claimed); + self::assertClaimedJob($job, $claimed); + $lockedPrepared = self::preparedEvidence($job); + if ($lockedPrepared === null + || !hash_equals( + $prepared['marker'], + $lockedPrepared['marker'] + ) + || !hash_equals( + $prepared['reason'], + $lockedPrepared['reason'] + )) { + throw new CloudRecoveryEvidenceConflict( + '开通任务的恢复预提交已变化', + 409 + ); + } + self::assertNoSuccessfulReceiptLocked($job); + + if ($inspection === null) { + CloudAdminOperationRecoveryService:: + preparedFenceResolutionForRecovery( + (string) $job->id, + 'not_applied', + $lockedPrepared['reason'] + ); + } elseif ($inspection !== false) { + CloudAdminOperationRecoveryService::resolveFenceForRecovery( + (string) $job->id, + 'not_applied', + $lockedPrepared['reason'], + $inspection + ); + } elseif ($lockedPrepared['fence_resolution_required']) { + throw new CloudRecoveryEvidenceConflict( + '开通任务仍需解除宿主机资源栅栏', + 409 + ); + } + + self::assertNoSuccessfulReceiptLocked($job); + self::finalizeLocked( + $job, + $user, + $order, + 'automatic_create_not_applied:' + . substr($lockedPrepared['marker'], -64) + ); + }, + 3 + ); + } + + /** + * @return array{marker:string,reason:string,identity:array,inspection:array,fence_resolution_required:bool}|null + */ + private static function preparedEvidence(CloudJob $job): ?array + { + $marker = (string) ($job->effect_resolution_reason ?? ''); + if (preg_match( + '/^prepared:create_not_applied:v1:[a-f0-9]{64}$/D', + $marker + ) !== 1) { + return null; + } + $payload = self::payload($job); + $value = $payload[self::ESCROW_KEY] ?? null; + $identity = is_array($value) ? ($value['identity'] ?? null) : null; + $inspection = is_array($value) ? ($value['inspection'] ?? null) : null; + if (!is_array($value) || array_is_list($value) + || array_diff(array_keys($value), [ + 'version', 'marker', 'reason', 'identity', 'inspection', + ]) !== [] + || ($value['version'] ?? null) !== self::ESCROW_VERSION + || !is_string($value['marker'] ?? null) + || !hash_equals($marker, $value['marker']) + || !is_string($value['reason'] ?? null) + || mb_strlen($value['reason']) < 20 + || mb_strlen($value['reason']) > 500 + || !is_array($identity) || array_is_list($identity) + || !is_array($inspection) || array_is_list($inspection)) { + throw new CloudRecoveryPayloadUnavailable( + '开通任务恢复预提交数据无效', + 409 + ); + } + $expected = CloudOperationRecoveryIdentity::expectedFromPayload( + $job, + $payload, + false + ); + CloudOperationRecoveryIdentity::assertMatches($expected, $identity); + self::assertInspectionProvesAbsence($job, $inspection, false); + CloudOperationRecoveryIdentity::assertMatches($expected, [ + 'action' => $inspection['action'] ?? null, + 'resource_id' => $inspection['resource_id'] ?? null, + 'request_fingerprint' => + $inspection['request_fingerprint'] ?? null, + ]); + $evidence = self::inspectionEvidence($inspection); + $expectedMarker = self::inspectionMarker($evidence); + if (!hash_equals($expectedMarker, $marker) + || !hash_equals(self::fenceReason($job, $marker), $value['reason'])) { + throw new CloudRecoveryEvidenceConflict( + '开通任务恢复预提交摘要不匹配', + 409 + ); + } + return [ + 'marker' => $marker, + 'reason' => $value['reason'], + 'identity' => $expected, + 'inspection' => $evidence, + 'fence_resolution_required' => + (bool) $evidence['fence_resolution_required'], + ]; + } + + private static function assertInspectionProvesAbsence( + CloudJob $job, + array $inspection, + bool $requireFresh = true + ): void { + $checkedAt = $inspection['checked_at'] ?? null; + $projection = $inspection['effect_projection'] ?? null; + if (($inspection['operation_id'] ?? null) + !== (string) $job->operation_id + || ($inspection['journal_status'] ?? null) !== 'failed' + || ($inspection['outcome'] ?? null) !== 'failed' + || ($inspection['summary_code'] ?? null) !== 'operation_failed' + || ($inspection['effect_resolution'] ?? null) !== 'not_applied' + || !is_int($checkedAt) + || ($requireFresh && ($checkedAt < time() + - self::INSPECTION_MAX_AGE_SECONDS + || $checkedAt > time() + 5)) + || !is_array($projection) || array_is_list($projection) + || ($projection['creation_absent'] ?? null) !== true + || ($projection['managed_artifacts_absent'] ?? null) !== true + || !in_array( + $inspection['fence_state'] ?? null, + ['not_required', 'active'], + true + ) + || !is_bool($inspection['fence_resolution_required'] ?? null) + || (bool) $inspection['fence_resolution_required'] + !== (($inspection['fence_state'] ?? null) === 'active') + || array_key_exists('completion_delivery_state', $inspection)) { + throw new CloudRecoveryEvidenceConflict( + '宿主机检查没有同时确认虚拟机及面板管理资源均不存在', + 409 + ); + } + CloudCreateFailureFinalizer::assertResolutionSupported( + $job, + 'not_applied', + $inspection + ); + } + + private static function receiptProvesAbsence( + CloudAgentCompletionEvent $event, + array $operation + ): bool { + $result = $operation['result'] ?? null; + return (string) $event->outcome === 'failed' + && !(bool) $event->effect_may_have_applied + && is_array($result) && !array_is_list($result) + && ($result['creation_absent'] ?? null) === true + && ($result['managed_artifacts_absent'] ?? null) === true; + } + + /** @return array{0:User,1:CloudOrder,2:CloudJob} */ + private static function lockLedger(CloudJob $source): array + { + $user = User::where('id', (int) $source->user_id) + ->lockForUpdate()->first(); + $order = CloudOrder::where('id', (int) $source->order_id) + ->lockForUpdate()->first(); + $job = CloudJob::where('id', (int) $source->id) + ->lockForUpdate()->first(); + if ($user === null || $order === null || $job === null + || (int) $job->user_id !== (int) $user->id + || (int) $job->order_id !== (int) $order->id + || (int) $order->user_id !== (int) $user->id) { + throw new CloudRecoveryInstanceStateConflict( + '开通任务与资金账本关联不一致', + 409 + ); + } + return [$user, $order, $job]; + } + + private static function assertRecoverableJob(CloudJob $job): void + { + self::assertDispatchedEnvelope($job); + if ((string) $job->kind !== 'create' + || $job->order_id === null + || (string) $job->execution_mode !== 'agent' + || (string) $job->phase !== 'terminal' + || (string) $job->outcome !== 'failed' + || (string) $job->status !== 'failed' + || !(bool) $job->effect_may_have_applied + || (bool) $job->safe_to_resubmit + || (string) $job->effect_resolution !== 'unknown') { + throw new CloudRecoveryInstanceStateConflict( + '开通任务不处于可自动核对状态', + 409 + ); + } + } + + private static function settledNotApplied( + CloudJob $job, + CloudOrder $order, + CloudAgentCompletionEvent $event + ): bool { + self::assertDispatchedEnvelope($job); + $reason = $job->effect_resolution_reason; + $completionEventId = $job->completion_event_id; + return (string) $event->outcome === 'failed' + && (string) $job->kind === 'create' + && $job->order_id !== null + && (string) $job->execution_mode === 'agent' + && (string) $job->phase === 'terminal' + && (string) $job->outcome === 'failed' + && (string) $job->status === 'failed' + && (int) $job->dispatch_attempts === 1 + && !(bool) $job->safe_to_resubmit + && (string) $job->effect_resolution === 'not_applied' + && (int) ($job->effect_resolved_at ?? 0) > 0 + && is_string($reason) + && mb_strlen($reason) >= 20 + && mb_strlen($reason) <= 500 + && !(bool) $job->finalization_incident + && $job->finalization_error_code === null + && $job->instance_id === null + && $order->instance_id === null + && (string) $order->status === 'failed' + && $order->refunded_at !== null + && ($completionEventId === null + || (is_string($completionEventId) + && hash_equals( + $completionEventId, + (string) $event->event_id + ))); + } + + /** Validate create's immutable generation without broadening policy. */ + private static function assertDispatchedEnvelope(CloudJob $job): void + { + CloudDispatchLedger::assertDispatched($job); + if ((string) $job->attempts !== '1' + || (int) ($job->host_id ?? 0) < 1 + || (int) ($job->dispatch_deadline_at ?? 0) < 1 + || (int) ($job->dispatch_started_at ?? 0) < 1 + || (($job->agent_accepted_at ?? null) !== null + && (int) $job->agent_accepted_at < 1) + || (int) ($job->dispatched_at ?? 0) < 1 + || (int) ($job->receipt_deadline_at ?? 0) < 1 + || (int) ($job->terminal_at ?? 0) < 1 + || preg_match( + '/^[a-f0-9]{32}$/D', + (string) ($job->operation_id ?? '') + ) !== 1 + || preg_match( + '/^[a-f0-9]{64}$/D', + (string) ($job->request_fingerprint ?? '') + ) !== 1 + || preg_match( + '/^[A-Za-z0-9_.-]{16,128}$/D', + (string) ($job->request_idempotency_key ?? '') + ) !== 1) { + throw new CloudRecoveryInstanceStateConflict( + '开通任务单次派发信封不完整', + 409 + ); + } + } + + private static function assertClaimedJob( + CloudJob $job, + CloudJob $claimed + ): void { + self::assertRecoverableJob($job); + if (!is_string($claimed->lease_token) + || !is_string($job->lease_token) + || !hash_equals( + (string) $claimed->lease_token, + (string) $job->lease_token + ) + || (int) $job->lease_until <= time()) { + throw new CloudRecoveryInstanceStateConflict( + '开通任务自动核对租约已失效', + 409 + ); + } + } + + private static function claimedJob(CloudJob $claimed): CloudJob + { + $job = CloudJob::find((int) $claimed->id); + if ($job === null) { + throw new CloudError('任务不存在', 404); + } + self::assertClaimedJob($job, $claimed); + return $job; + } + + private static function assertNoSuccessfulReceiptLocked(CloudJob $job): void + { + $event = CloudAgentCompletionEvent::where( + 'host_id', + (int) $job->host_id + )->where( + 'operation_id', + (string) $job->operation_id + )->where('outcome', 'succeeded')->lockForUpdate()->first(); + if ($event === null) { + return; + } + $exact = CloudOperationCompletionService::isExactStoredEvent($event) + || CloudOperationCompletionService::isExactAppliedEvent($event); + if (!$exact) { + throw new CloudRecoveryOperationIdentityConflict( + '开通成功回执状态异常,不能执行退款', + 409 + ); + } + $payload = self::payload($job); + $identity = CloudOperationRecoveryIdentity::expectedFromPayload( + $job, + $payload, + true + ); + if (CloudOperationRecoveryIdentity::same( + $identity, + CloudOperationCompletionService::identity($event) + )) { + throw new CloudRecoveryEvidenceConflict( + '已收到匹配的实例创建成功回执,不能按未应用退款', + 409 + ); + } + throw new CloudRecoveryOperationIdentityConflict( + '开通成功回执身份冲突,不能执行退款', + 409 + ); + } + + private static function finalizeLocked( + CloudJob $job, + User $user, + CloudOrder $order, + string $resolutionReason, + ?CloudAgentCompletionEvent $event = null + ): void { + $payload = self::payload($job); + $identity = CloudOperationRecoveryIdentity::expectedFromPayload( + $job, + $payload, + false + ); + CloudCreateFailureFinalizer::refundAndReleaseLocked( + $job, + $user, + $order, + 'create_effect_not_applied' + ); + + $safePayload = CloudInput::safeResult($payload); + unset($safePayload[self::ESCROW_KEY]); + $safePayload[CloudOperationRecoveryIdentity::payloadKey()] = + CloudOperationRecoveryIdentity::persisted($identity); + $now = time(); + $job->effect_resolution = 'not_applied'; + $job->effect_resolved_at = $now; + $job->effect_resolved_by = null; + $job->effect_resolution_reason = $resolutionReason; + if ($event !== null) { + $job->completion_event_id = (string) $event->event_id; + } + $job->sealed_payload = CloudSecrets::encrypt( + $safePayload, + 'job-payload' + ); + $job->finalization_incident = 0; + $job->finalization_error_code = null; + $job->read_at = null; + $job->lease_token = null; + $job->lease_until = 0; + $job->available_at = 0; + $job->updated_at = $now; + $job->save(); + if ($event !== null) { + $event->disposition = 'applied'; + $event->applied_at = $now; + $event->save(); + } + } + + private static function payload(CloudJob $job): array + { + try { + $payload = CloudSecrets::decrypt( + (string) $job->sealed_payload, + 'job-payload' + ); + } catch (\Throwable $error) { + throw new CloudRecoveryPayloadUnavailable( + '开通任务恢复数据无法解密', + 409 + ); + } + if (!is_array($payload) || array_is_list($payload)) { + throw new CloudRecoveryPayloadUnavailable( + '开通任务恢复数据结构不完整', + 409 + ); + } + return $payload; + } + + private static function fenceReason(CloudJob $job, string $marker): string + { + return '自动恢复已确认原开通操作未创建虚拟机或面板管理资源;operation_id=' + . (string) $job->operation_id . ';evidence=' + . substr($marker, -64); + } + + /** Hash only stable evidence so a lost fence response can be resumed. */ + private static function inspectionMarker(array $evidence): string + { + unset($evidence['checked_at']); + return self::MARKER_PREFIX . hash( + 'sha256', + json_encode( + $evidence, + JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES + | JSON_UNESCAPED_UNICODE + ) + ); + } + + /** Return the bounded, canonical evidence persisted in encrypted escrow. */ + private static function inspectionEvidence(array $inspection): array + { + return [ + 'operation_id' => $inspection['operation_id'], + 'action' => $inspection['action'], + 'resource_id' => $inspection['resource_id'], + 'request_fingerprint' => $inspection['request_fingerprint'], + 'journal_status' => $inspection['journal_status'], + 'checked_at' => $inspection['checked_at'], + 'outcome' => $inspection['outcome'], + 'summary_code' => $inspection['summary_code'], + 'error_code' => $inspection['error_code'] ?? '', + 'effect_resolution' => $inspection['effect_resolution'], + 'fence_state' => $inspection['fence_state'], + 'fence_resolution_required' => + $inspection['fence_resolution_required'], + 'effect_projection' => [ + 'creation_absent' => true, + 'managed_artifacts_absent' => true, + ], + ]; + } +} diff --git a/web/app/Services/Cloud/CloudOperationRecoveryService.php b/web/app/Services/Cloud/CloudOperationRecoveryService.php index ba705b76..de4b4d73 100644 --- a/web/app/Services/Cloud/CloudOperationRecoveryService.php +++ b/web/app/Services/Cloud/CloudOperationRecoveryService.php @@ -365,6 +365,12 @@ final class CloudOperationRecoveryService ); } if ((string) $current->kind === 'create') { + if ((string) $event->outcome === 'failed') { + return CloudCreateFailureRecoveryService::settleReceipt( + $current, + $event + ); + } if ((string) $event->outcome !== 'succeeded' || (bool) $event->effect_may_have_applied || (bool) $event->safe_to_resubmit @@ -548,10 +554,11 @@ final class CloudOperationRecoveryService // eligible for the immutable receipt, terminate the event // instead of leaving an unclaimable "received" row. if (self::retryableCreateCompletion($job, $event)) { - // Create is intentionally outside the generic - // inspection policy. Its exact success receipt is the - // only safe recovery evidence, so a temporary local - // ledger fault must not consume that evidence. + // Create uses dedicated financial finalizers. Keep + // either exact outcome retryable when PHP could not + // finish inspecting or settling its ledger. An + // incomplete failure is ACKed late by the dedicated + // finalizer only after those local checks succeed. return 'received'; } if (self::lateReceiptResolution($job, $event) === null) { @@ -961,9 +968,10 @@ final class CloudOperationRecoveryService && (bool) $job->effect_may_have_applied && !(bool) $job->safe_to_resubmit && (string) $job->effect_resolution === 'unknown' - && (string) $event->outcome === 'succeeded' - && !(bool) $event->effect_may_have_applied - && !(bool) $event->safe_to_resubmit; + && ((string) $event->outcome === 'failed' + || ((string) $event->outcome === 'succeeded' + && !(bool) $event->effect_may_have_applied + && !(bool) $event->safe_to_resubmit)); } private static function assertRecoverableAppliedEvidence( diff --git a/web/tests/CloudAdminOperationRecoveryServiceTest.php b/web/tests/CloudAdminOperationRecoveryServiceTest.php index 9b9c68d5..e71094ea 100644 --- a/web/tests/CloudAdminOperationRecoveryServiceTest.php +++ b/web/tests/CloudAdminOperationRecoveryServiceTest.php @@ -363,11 +363,14 @@ namespace { ]); $before = count($client->requests); + Capsule::table('cloud_jobs')->where('id', 41)->update([ + 'kind' => 'host_storage_configure', + ]); $reject(static fn () => Recovery::inspectForRecovery('41'), 409); $reject(static fn () => Recovery::resolveFenceForRecovery( '41', 'not_applied', - 'automatic recovery must reject non-settings tasks', + 'automatic recovery must reject unregistered task kinds', [ 'operation_id' => 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa', 'checked_at' => $now, @@ -379,13 +382,16 @@ namespace { $reject(static fn () => Recovery::preparedFenceResolutionForRecovery( '41', 'not_applied', - 'automatic recovery must reject non-settings tasks' + 'automatic recovery must reject unregistered task kinds' ), 409); $assert(count($client->requests) === $before, - 'A non-settings task reached the system recovery Agent client.'); + 'An unregistered task reached the system recovery Agent client.'); + Capsule::table('cloud_jobs')->where('id', 41)->update([ + 'kind' => 'create', + ]); $operationA = 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa'; - $client->responses[] = [ + $client->responses[] = $createInspectionResponse = [ 'operation_id' => $operationA, 'operation_status' => 'failed', 'checked_at' => $now, @@ -430,6 +436,36 @@ namespace { ->value('effect_resolution') === 'unknown', 'Read-only inspection changed the task or its safety fence.'); + $client->responses[] = $createInspectionResponse; + $createInspection = Recovery::inspectForRecovery('41'); + $createRequest = $client->requests[array_key_last($client->requests)]; + $assert(($createInspection['error_code'] ?? null) === 'creation_absent' + && ($createInspection['effect_projection']['creation_absent'] ?? null) === true + && ($createInspection['effect_projection']['managed_artifacts_absent'] ?? null) === true + && $createRequest['method'] === 'GET' + && ($createRequest['system_recovery'] ?? false) === true + && $createRequest['actor']['operation_id'] === $operationA + && Capsule::table('cloud_jobs')->where('id', 41) + ->value('effect_resolution') === 'unknown', + 'Create inspection did not remain operation-bound and read-only.'); + $before = count($client->requests); + $reject(static fn () => Recovery::resolveFenceForRecovery( + '41', 'not_applied', + 'automatic create recovery requires durable financial intent', + $createInspection + ), 409); + $reject(static fn () => Recovery::resolveFenceForRecovery( + '41', 'applied', + 'automatic create recovery cannot infer a successful instance', + $createInspection + ), 409); + $reject(static fn () => Recovery::preparedFenceResolutionForRecovery( + '41', 'not_applied', + 'automatic create recovery requires durable financial intent' + ), 409); + $assert(count($client->requests) === $before, + 'An unprepared create reached the system Agent fence routes.'); + $client->responses[] = ['operation_id' => $operationA, 'operation_status' => 'uncertain', 'checked_at' => $now, 'outcome' => 'uncertain', 'summary_code' => 'evidence_incomplete']; diff --git a/web/tests/CloudFinancialWorkflowTest.php b/web/tests/CloudFinancialWorkflowTest.php index b50c512f..018e9498 100644 --- a/web/tests/CloudFinancialWorkflowTest.php +++ b/web/tests/CloudFinancialWorkflowTest.php @@ -25,6 +25,61 @@ namespace App\Services\Cloud { public static function encrypt(array $payload, string $purpose): string { $id = bin2hex(random_bytes(16)); self::$memory[$id] = $payload; return 'test-only:' . $id; } public static function decrypt(string $cipher, string $purpose): array { return self::$memory[substr($cipher, 10)]; } } + + /** Isolated read-only Agent transport for create recovery checks. */ + final class FinancialCreateRecoveryAgentClient + { + public array $requests = []; + public array $responses = []; + + public function systemRecoveryRequest( + string $method, + string $target, + string $operationId, + ?array $payload = null + ): array { + $this->requests[] = compact( + 'method', 'target', 'operationId', 'payload' + ); + if ($this->responses === []) { + throw new \RuntimeException( + 'Unexpected create recovery Agent request.' + ); + } + $response = array_shift($this->responses); + if ($response instanceof \Throwable) { + throw $response; + } + if (is_callable($response)) { + $response = $response(end($this->requests)); + } + if (!is_array($response)) { + throw new \RuntimeException( + 'Invalid create recovery Agent response.' + ); + } + return $response; + } + } + + final class CloudAgentSettingsService + { + public static ?FinancialCreateRecoveryAgentClient $client = null; + public static int $hostId = 0; + + public static function clientForConfiguration( + int $hostId, + float $timeoutSeconds = 120.0 + ): FinancialCreateRecoveryAgentClient { + if ($hostId !== self::$hostId || $timeoutSeconds !== 30.0 + || self::$client === null) { + throw new \RuntimeException( + 'Unexpected create recovery client request.' + ); + } + return self::$client; + } + } } namespace { require dirname(__DIR__) . '/vendor/autoload.php'; @@ -69,6 +124,10 @@ namespace { use App\Services\Cloud\CloudPlatformTestNetworkPolicy; use App\Services\Cloud\CloudDeploymentPageService; use App\Services\Cloud\CloudTemplateCatalogService; + use App\Services\Cloud\CloudAgentSettingsService; + use App\Services\Cloud\CloudAgentError; + use App\Services\Cloud\CloudCreateFailureRecoveryService; + use App\Services\Cloud\FinancialCreateRecoveryAgentClient; use GuzzleHttp\Client; use GuzzleHttp\Handler\MockHandler; use GuzzleHttp\HandlerStack; @@ -2915,5 +2974,892 @@ namespace { 'Unknown, applied, or sticky lifecycle evidence created a new mutation generation.'); } + // A create which crossed the Agent boundary may time out locally before + // the authenticated terminal failure reaches PHP. Exact Agent proof that + // both the guest and every managed artifact are absent must finish the + // original paid order locally; it must never dispatch a second create. + foreach (['203.0.113.20', '203.0.113.21', '203.0.113.22'] as $address) { + if (!CloudPublicIp::where('address', $address)->exists()) { + $ip = new CloudPublicIp(); + $ip->host_id = $host->id; + $ip->public_ip_pool_id = $sourcePublicPool->id; + $ip->address = $address; + $ip->prefix_length = 28; + $ip->state = 'available'; + $ip->created_at = $ip->updated_at = time(); + $ip->save(); + } + } + $lateFailureOrder = CloudOrderService::purchase(array_replace($input, [ + 'name' => 'vm-late-failure', + 'idempotency_key' => 'test-request-key-late-failure', + ]), $actor); + $lateFailureInvoice = CloudInvoice::where( + 'order_id', + $lateFailureOrder->id + )->firstOrFail(); + CloudInvoiceService::payBalance( + $lateFailureInvoice, + $actor, + 'provision-balance-late-failure' + ); + $lateFailureJob = CloudJob::where( + 'order_id', + $lateFailureOrder->id + )->firstOrFail(); + // Earlier lifecycle tests intentionally leave separately audited retries + // queued. Park those unrelated rows so this tracer observes exactly one + // create dispatch rather than depending on suite-wide queue order. + Capsule::table('cloud_jobs') + ->where('id', '<>', (int) $lateFailureJob->id) + ->where('phase', '<>', 'terminal') + ->update([ + 'available_at' => time() + 3600, + 'dispatch_deadline_at' => time() + 3600, + 'receipt_deadline_at' => time() + 3600, + ]); + Capsule::table('cloud_jobs')->where( + 'id', + (int) $lateFailureJob->id + )->update(['available_at' => 0]); + $lateFailureJob->refresh(); + $lateFailureIdentity = CloudOperationRecoveryIdentity::expected( + $lateFailureJob, + false + ); + $mock->append(new Response(202, [], json_encode([ + 'operation_id' => $lateFailureJob->operation_id, + 'status' => 'reserved', + ...$lateFailureIdentity, + 'replayed' => false, + 'retryable' => false, + ], JSON_THROW_ON_ERROR))); + $beforeLateFailureDispatch = count($history); + $assert($worker->tick(), + 'Late-failure create did not cross the Agent dispatch boundary.'); + $lateFailureJob->refresh(); + $assert((string) $lateFailureJob->phase === 'awaiting_receipt' + && (int) $lateFailureJob->dispatch_attempts === 1 + && count($history) === $beforeLateFailureDispatch + 1, + 'Late-failure create did not persist its single dispatch generation.'); + Capsule::table('cloud_jobs')->where( + 'id', + (int) $lateFailureJob->id + )->update([ + 'receipt_deadline_at' => time() - 1, + 'lease_token' => null, + 'lease_until' => 0, + ]); + $assert(CloudWorkerService::expireDueTaskForAdmin( + (int) $lateFailureJob->id + ) === 1, 'Late-failure create did not reach its local deadline.'); + $lateFailureJob->refresh(); + $lateFailureOrder->refresh(); + $lateFailureInvoice->refresh(); + $lateFailureReservation = CloudResourceReservation::where( + 'order_id', + $lateFailureOrder->id + )->firstOrFail(); + $lateFailureIp = CloudPublicIp::findOrFail( + (int) $lateFailureReservation->public_ip_id + ); + $walletWhileFenced = (string) User::findOrFail(1)->money; + $stockWhileFenced = (int) CloudOffer::findOrFail(1)->stock; + $assert((string) $lateFailureJob->phase === 'terminal' + && (string) $lateFailureJob->outcome === 'failed' + && (string) $lateFailureJob->effect_resolution === 'unknown' + && (string) $lateFailureOrder->status === 'failed' + && $lateFailureOrder->refunded_at === null + && (string) $lateFailureInvoice->status === 'paid' + && (string) $lateFailureReservation->state === 'uncertain' + && (string) $lateFailureIp->state === 'uncertain', + 'Timed-out dispatched create did not retain its financial fence.'); + + $lateFailureEventId = 'create-late-failure-event-0001'; + $lateFailureReceipt = [ + 'type' => 'completion', + 'id' => $lateFailureEventId, + 'agent_id' => (string) $lateFailureJob->host_id, + 'operation_id' => (string) $lateFailureJob->operation_id, + ...$lateFailureIdentity, + 'status' => 'failed', + 'effect_may_have_applied' => false, + 'safe_to_resubmit' => false, + 'error_code' => 'creation_absent', + 'result' => [ + 'uuid' => $lateFailureIdentity['resource_id'], + 'creation_absent' => true, + 'managed_artifacts_absent' => true, + ], + ]; + $lateFailureAccepted = CloudOperationCompletionService::accept( + $lateFailureReceipt + ); + $lateFailureJob->refresh(); + $lateFailureOrder->refresh(); + $lateFailureInvoice->refresh(); + $lateFailureReservation->refresh(); + $lateFailureIp->refresh(); + $lateFailureEvent = \App\Models\CloudAgentCompletionEvent::where( + 'event_id', + $lateFailureEventId + )->firstOrFail(); + $assert($lateFailureAccepted === [ + 'accepted' => true, + 'disposition' => 'applied', + ] + && (string) $lateFailureEvent->disposition === 'applied' + && $lateFailureEvent->applied_at !== null + && (string) $lateFailureJob->effect_resolution === 'not_applied' + && !(bool) $lateFailureJob->finalization_incident + && (string) $lateFailureOrder->status === 'failed' + && $lateFailureOrder->refunded_at !== null + && (string) $lateFailureInvoice->status === 'refunded' + && (string) $lateFailureReservation->state === 'released' + && (string) $lateFailureIp->state === 'available' + && (string) User::findOrFail(1)->money + === CloudMoney::decimal( + CloudMoney::cents($walletWhileFenced) + 999 + ) + && (int) CloudOffer::findOrFail(1)->stock + === $stockWhileFenced + 1 + && count($history) === $beforeLateFailureDispatch + 1, + 'Exact late create failure did not atomically refund, release, ACK, ' + . 'or replayed the original mutation.'); + $lateFailureWalletAfter = (string) User::findOrFail(1)->money; + $lateFailureStockAfter = (int) CloudOffer::findOrFail(1)->stock; + $lateFailureRefundedAt = (int) $lateFailureOrder->refunded_at; + $lateFailureJobUpdatedAt = (int) $lateFailureJob->updated_at; + $lateFailureAppliedAt = (int) $lateFailureEvent->applied_at; + $lateFailureDuplicate = CloudOperationCompletionService::accept( + $lateFailureReceipt + ); + $assert($lateFailureDuplicate === [ + 'accepted' => true, + 'disposition' => 'applied', + ] + && (string) User::findOrFail(1)->money + === $lateFailureWalletAfter + && (int) CloudOffer::findOrFail(1)->stock + === $lateFailureStockAfter + && (int) CloudOrder::findOrFail( + (int) $lateFailureOrder->id + )->refunded_at === $lateFailureRefundedAt + && (int) CloudJob::findOrFail( + (int) $lateFailureJob->id + )->updated_at === $lateFailureJobUpdatedAt + && (int) \App\Models\CloudAgentCompletionEvent::findOrFail( + $lateFailureEventId + )->applied_at === $lateFailureAppliedAt + && count($history) === $beforeLateFailureDispatch + 1, + 'Duplicate late create failure refunded, released, ACKed, or dispatched ' + . 'more than once.'); + + /** + * Create one paid order, cross the Agent boundary exactly once, and force + * the panel-side receipt deadline to the terminal unknown-effect state. + * Later recovery assertions can then focus on finance and evidence rather + * than reproducing dispatch bookkeeping by hand. + * + * @return array + */ + $makeTimedOutCreate = static function ( + string $suffix + ) use ( + $input, + $actor, + $mock, + $worker, + &$history, + $assert + ): array { + $order = CloudOrderService::purchase(array_replace($input, [ + 'name' => 'vm-create-recovery-' . $suffix, + 'idempotency_key' => 'test-create-recovery-' . $suffix, + ]), $actor); + $invoice = CloudInvoice::where('order_id', $order->id) + ->firstOrFail(); + CloudInvoiceService::payBalance( + $invoice, + $actor, + 'provision-create-recovery-' . $suffix + ); + $job = CloudJob::where('order_id', $order->id)->firstOrFail(); + $future = time() + 3600; + Capsule::table('cloud_jobs') + ->where('id', '<>', (int) $job->id) + ->update([ + 'available_at' => $future, + 'lease_until' => $future, + ]); + Capsule::table('cloud_jobs')->where('id', (int) $job->id) + ->update(['available_at' => 0, 'lease_until' => 0]); + $job->refresh(); + $identity = CloudOperationRecoveryIdentity::expected($job, false); + $mock->append(new Response(202, [], json_encode([ + 'operation_id' => $job->operation_id, + 'status' => 'reserved', + ...$identity, + 'replayed' => false, + 'retryable' => false, + ], JSON_THROW_ON_ERROR))); + $historyBefore = count($history); + $assert($worker->tick(), + 'Create recovery fixture did not dispatch: ' . $suffix); + $job->refresh(); + $assert((string) $job->phase === 'awaiting_receipt' + && (int) $job->dispatch_attempts === 1 + && count($history) === $historyBefore + 1, + 'Create recovery fixture did not persist one dispatch: ' + . $suffix); + Capsule::table('cloud_jobs')->where('id', (int) $job->id) + ->update([ + 'receipt_deadline_at' => time() - 1, + 'lease_token' => null, + 'lease_until' => 0, + ]); + $assert(CloudWorkerService::expireDueTaskForAdmin((int) $job->id) + === 1, 'Create recovery fixture did not expire: ' . $suffix); + $job->refresh(); + $order->refresh(); + $invoice->refresh(); + $reservation = CloudResourceReservation::where( + 'order_id', + $order->id + )->firstOrFail(); + $ip = CloudPublicIp::findOrFail((int) $reservation->public_ip_id); + $assert((string) $job->phase === 'terminal' + && (string) $job->outcome === 'failed' + && (string) $job->effect_resolution === 'unknown' + && (string) $order->status === 'failed' + && $order->refunded_at === null + && (string) $invoice->status === 'paid' + && (string) $reservation->state === 'uncertain' + && (string) $ip->state === 'uncertain', + 'Create recovery fixture did not retain its safety fence: ' + . $suffix); + return compact( + 'order', 'invoice', 'job', 'identity', 'reservation', 'ip', + 'historyBefore' + ) + [ + 'walletBefore' => (string) User::findOrFail(1)->money, + 'stockBefore' => (int) CloudOffer::findOrFail(1)->stock, + ]; + }; + $makeRecoveryReady = static function (CloudJob $job): void { + $future = time() + 3600; + Capsule::table('cloud_jobs') + ->where('id', '<>', (int) $job->id) + ->update([ + 'available_at' => $future, + 'lease_until' => $future, + ]); + Capsule::table('cloud_jobs')->where('id', (int) $job->id) + ->update([ + 'available_at' => 0, + 'lease_token' => null, + 'lease_until' => 0, + ]); + $job->refresh(); + }; + $inspectionFor = static function ( + CloudJob $job, + array $identity, + bool $fenced = false + ): array { + return [ + 'operation_id' => (string) $job->operation_id, + ...$identity, + 'operation_status' => 'failed', + 'checked_at' => time(), + 'outcome' => 'failed', + 'summary_code' => 'operation_failed', + 'error_code' => 'creation_absent', + 'effect_resolution' => 'not_applied', + 'fence_state' => $fenced ? 'active' : 'not_required', + 'fence_resolution_required' => $fenced, + 'effect_projection' => [ + 'creation_absent' => true, + 'managed_artifacts_absent' => true, + ], + 'abort_job' => ['available' => false], + 'cleanup' => ['available' => false], + ]; + }; + $assertCreateSettled = static function ( + array $case, + string $message + ) use ($assert, &$history): void { + $case['job']->refresh(); + $case['order']->refresh(); + $case['invoice']->refresh(); + $case['reservation']->refresh(); + $case['ip']->refresh(); + $assert((string) $case['job']->effect_resolution === 'not_applied' + && !(bool) $case['job']->finalization_incident + && $case['job']->lease_token === null + && (string) $case['order']->status === 'failed' + && $case['order']->refunded_at !== null + && (string) $case['invoice']->status === 'refunded' + && (string) $case['reservation']->state === 'released' + && (string) $case['ip']->state === 'available' + && (string) User::findOrFail(1)->money + === CloudMoney::decimal( + CloudMoney::cents($case['walletBefore']) + 999 + ) + && (int) CloudOffer::findOrFail(1)->stock + === $case['stockBefore'] + 1 + && count($history) === $case['historyBefore'] + 1, + $message); + }; + $failedCreateReceipt = static function ( + array $case, + string $eventId + ): array { + return [ + 'type' => 'completion', + 'id' => $eventId, + 'agent_id' => (string) $case['job']->host_id, + 'operation_id' => (string) $case['job']->operation_id, + ...$case['identity'], + 'status' => 'failed', + 'effect_may_have_applied' => false, + 'safe_to_resubmit' => false, + 'error_code' => 'creation_absent', + 'result' => [ + 'uuid' => $case['identity']['resource_id'], + 'creation_absent' => true, + 'managed_artifacts_absent' => true, + ], + ]; + }; + + // Immutable host absence evidence must wait for a temporarily inconsistent + // local invoice. Keeping the event received allows the exact same receipt + // to finish atomically after the PHP ledger is repaired. + $invoiceGuardCase = $makeTimedOutCreate('receipt-invoice-guard'); + $invoiceGuardCase['invoice']->purpose = 'renewal'; + $invoiceGuardCase['invoice']->save(); + $invoiceGuardReceipt = $failedCreateReceipt( + $invoiceGuardCase, + 'create-invoice-guard-failure-event-0001' + ); + $invoiceGuardPending = CloudOperationCompletionService::accept( + $invoiceGuardReceipt + ); + $invoiceGuardCase['job']->refresh(); + $invoiceGuardCase['order']->refresh(); + $invoiceGuardCase['reservation']->refresh(); + $invoiceGuardCase['ip']->refresh(); + $invoiceGuardEvent = \App\Models\CloudAgentCompletionEvent::findOrFail( + $invoiceGuardReceipt['id'] + ); + $assert($invoiceGuardPending === [ + 'accepted' => true, + 'disposition' => 'received', + ] + && (string) $invoiceGuardEvent->disposition === 'received' + && $invoiceGuardEvent->applied_at === null + && (string) $invoiceGuardCase['job']->effect_resolution === 'unknown' + && $invoiceGuardCase['order']->refunded_at === null + && (string) $invoiceGuardCase['reservation']->state === 'uncertain' + && (string) $invoiceGuardCase['ip']->state === 'uncertain' + && (string) User::findOrFail(1)->money + === $invoiceGuardCase['walletBefore'] + && (int) CloudOffer::findOrFail(1)->stock + === $invoiceGuardCase['stockBefore'], + 'Exact create failure bypassed an inconsistent local invoice: ' + . json_encode([ + 'accepted' => $invoiceGuardPending, + 'event_disposition' => $invoiceGuardEvent->disposition, + 'event_applied_at' => $invoiceGuardEvent->applied_at, + 'job_resolution' => + $invoiceGuardCase['job']->effect_resolution, + 'refunded_at' => $invoiceGuardCase['order']->refunded_at, + 'reservation' => $invoiceGuardCase['reservation']->state, + 'ip' => $invoiceGuardCase['ip']->state, + 'wallet_before' => $invoiceGuardCase['walletBefore'], + 'wallet_after' => User::findOrFail(1)->money, + 'stock_before' => $invoiceGuardCase['stockBefore'], + 'stock_after' => CloudOffer::findOrFail(1)->stock, + ], JSON_THROW_ON_ERROR)); + $invoiceGuardCase['invoice']->purpose = 'provision'; + $invoiceGuardCase['invoice']->save(); + $assert(CloudOperationCompletionService::accept($invoiceGuardReceipt) + === ['accepted' => true, 'disposition' => 'applied'], + 'Repaired local invoice did not resume the same create receipt.'); + $assertCreateSettled( + $invoiceGuardCase, + 'Repaired local invoice did not atomically settle create finance.' + ); + + // The specialised create finalizer must validate the complete dispatched + // envelope itself. Generic recovery policy deliberately does not own + // create, so neither a damaged attempt count nor a missing receipt + // deadline may reach the refund path. + foreach ([ + 'attempts' => 2, + 'receipt_deadline_at' => null, + ] as $damagedField => $damagedValue) { + $suffix = 'receipt-envelope-' . str_replace('_', '-', $damagedField); + $envelopeCase = $makeTimedOutCreate($suffix); + $eventId = 'create-envelope-' . str_replace('_', '-', $damagedField) + . '-event-0001'; + $envelopeReceipt = $failedCreateReceipt($envelopeCase, $eventId); + $originalValue = $envelopeCase['job']->{$damagedField}; + Capsule::table('cloud_jobs')->where( + 'id', + (int) $envelopeCase['job']->id + )->update([$damagedField => $damagedValue]); + $envelopePending = CloudOperationCompletionService::accept( + $envelopeReceipt + ); + $envelopeCase['job']->refresh(); + $envelopeCase['order']->refresh(); + $envelopeCase['invoice']->refresh(); + $envelopeCase['reservation']->refresh(); + $envelopeCase['ip']->refresh(); + $envelopeEvent = \App\Models\CloudAgentCompletionEvent::findOrFail( + $eventId + ); + $assert($envelopePending === [ + 'accepted' => true, + 'disposition' => 'received', + ] + && (string) $envelopeEvent->disposition === 'received' + && $envelopeEvent->applied_at === null + && (string) $envelopeCase['job']->effect_resolution === 'unknown' + && $envelopeCase['order']->refunded_at === null + && (string) $envelopeCase['invoice']->status === 'paid' + && (string) $envelopeCase['reservation']->state === 'uncertain' + && (string) $envelopeCase['ip']->state === 'uncertain' + && (string) User::findOrFail(1)->money + === $envelopeCase['walletBefore'] + && (int) CloudOffer::findOrFail(1)->stock + === $envelopeCase['stockBefore'], + 'Damaged create dispatch envelope released finance: ' + . $damagedField); + Capsule::table('cloud_jobs')->where( + 'id', + (int) $envelopeCase['job']->id + )->update([$damagedField => $originalValue]); + $assert(CloudOperationCompletionService::accept($envelopeReceipt) + === ['accepted' => true, 'disposition' => 'applied'], + 'Repaired create dispatch envelope did not resume receipt: ' + . $damagedField); + $assertCreateSettled( + $envelopeCase, + 'Repaired dispatch envelope did not settle create finance: ' + . $damagedField + ); + } + + // A fresh read-only Journal failure with complete absence projection can + // finish a timed-out create even when no completion event ever arrived. + $inspectionCase = $makeTimedOutCreate('inspection-no-receipt'); + $makeRecoveryReady($inspectionCase['job']); + $inspectionClient = new FinancialCreateRecoveryAgentClient(); + $inspectionClient->responses[] = $inspectionFor( + $inspectionCase['job'], + $inspectionCase['identity'] + ); + CloudAgentSettingsService::$hostId = + (int) $inspectionCase['job']->host_id; + CloudAgentSettingsService::$client = $inspectionClient; + $inspectionRecovery = new CloudAutomaticOperationRecoveryService(); + $assert($inspectionRecovery->tick(), + 'No-receipt create was not claimed for read-only recovery.'); + $assert(count($inspectionClient->requests) === 1 + && $inspectionClient->requests[0]['method'] === 'GET' + && str_ends_with( + $inspectionClient->requests[0]['target'], + '/inspect' + ) + && $inspectionClient->requests[0]['payload'] === null, + 'No-receipt create recovery used something other than one read-only ' + . 'inspection.'); + $assertCreateSettled( + $inspectionCase, + 'Read-only create absence evidence did not atomically settle finance.' + ); + + // Once inspection has settled the ledger, the first later failure receipt + // is an idempotent ACK of the late delivery. It must not re-enter the received queue, + // rewrite the job incident, or credit/release anything twice. + $inspectionCase['job']->refresh(); + $inspectionJobUpdatedAt = (int) $inspectionCase['job']->updated_at; + $inspectionWalletAfter = (string) User::findOrFail(1)->money; + $inspectionStockAfter = (int) CloudOffer::findOrFail(1)->stock; + $inspectionLateReceipt = [ + 'type' => 'completion', + 'id' => 'create-inspection-late-failure-event-0001', + 'agent_id' => (string) $inspectionCase['job']->host_id, + 'operation_id' => (string) $inspectionCase['job']->operation_id, + ...$inspectionCase['identity'], + 'status' => 'failed', + 'effect_may_have_applied' => false, + 'safe_to_resubmit' => false, + 'error_code' => 'creation_absent', + 'result' => [ + 'uuid' => $inspectionCase['identity']['resource_id'], + 'creation_absent' => true, + 'managed_artifacts_absent' => true, + ], + ]; + $inspectionLateAccepted = CloudOperationCompletionService::accept( + $inspectionLateReceipt + ); + $inspectionLateDuplicate = CloudOperationCompletionService::accept( + $inspectionLateReceipt + ); + $inspectionCase['job']->refresh(); + $inspectionLateEvent = \App\Models\CloudAgentCompletionEvent::findOrFail( + $inspectionLateReceipt['id'] + ); + $assert($inspectionLateAccepted === [ + 'accepted' => true, + 'disposition' => 'applied', + ] + && $inspectionLateDuplicate === $inspectionLateAccepted + && (string) $inspectionLateEvent->disposition === 'applied' + && $inspectionLateEvent->applied_at !== null + && (int) $inspectionCase['job']->updated_at + === $inspectionJobUpdatedAt + && !(bool) $inspectionCase['job']->finalization_incident + && (string) User::findOrFail(1)->money === $inspectionWalletAfter + && (int) CloudOffer::findOrFail(1)->stock === $inspectionStockAfter, + 'A failure receipt arriving after inspection settlement was requeued ' + . 'or changed the financial ledger: ' . json_encode([ + 'first' => $inspectionLateAccepted, + 'duplicate' => $inspectionLateDuplicate, + 'event_disposition' => $inspectionLateEvent->disposition, + 'event_applied_at' => $inspectionLateEvent->applied_at, + 'job_updated_at_before' => $inspectionJobUpdatedAt, + 'job_updated_at_after' => $inspectionCase['job']->updated_at, + 'job_incident' => $inspectionCase['job']->finalization_incident, + 'wallet_before' => $inspectionWalletAfter, + 'wallet_after' => User::findOrFail(1)->money, + 'stock_before' => $inspectionStockAfter, + 'stock_after' => CloudOffer::findOrFail(1)->stock, + ], JSON_THROW_ON_ERROR)); + + // An incomplete failed receipt is retained as a late audit event, then a + // fresh read-only inspection supplies the missing managed-artifact proof. + $incompleteCase = $makeTimedOutCreate('incomplete-receipt'); + $incompleteReceipt = [ + 'type' => 'completion', + 'id' => 'create-incomplete-failure-event-0001', + 'agent_id' => (string) $incompleteCase['job']->host_id, + 'operation_id' => (string) $incompleteCase['job']->operation_id, + ...$incompleteCase['identity'], + 'status' => 'failed', + 'effect_may_have_applied' => false, + 'safe_to_resubmit' => false, + 'error_code' => 'creation_absent', + 'result' => [ + 'uuid' => $incompleteCase['identity']['resource_id'], + 'creation_absent' => true, + ], + ]; + $assert(CloudOperationCompletionService::accept($incompleteReceipt) + === ['accepted' => true, 'disposition' => 'late'], + 'Incomplete create absence receipt was not ACKed as late.'); + $makeRecoveryReady($incompleteCase['job']); + $incompleteClient = new FinancialCreateRecoveryAgentClient(); + $incompleteClient->responses[] = $inspectionFor( + $incompleteCase['job'], + $incompleteCase['identity'] + ); + CloudAgentSettingsService::$client = $incompleteClient; + $assert((new CloudAutomaticOperationRecoveryService())->tick(), + 'Incomplete receipt create did not enter fresh inspection recovery.'); + $assertCreateSettled( + $incompleteCase, + 'Fresh inspection did not finish an incomplete failed receipt create.' + ); + + // If PHP fails after durably preparing a no-fence conclusion, a later + // tick must finish locally even while the Agent is offline. + $noFenceRetry = $makeTimedOutCreate('nofence-db-retry'); + $noFenceRetry['invoice']->purpose = 'renewal'; + $noFenceRetry['invoice']->save(); + $makeRecoveryReady($noFenceRetry['job']); + $noFenceClient = new FinancialCreateRecoveryAgentClient(); + $noFenceClient->responses[] = $inspectionFor( + $noFenceRetry['job'], + $noFenceRetry['identity'] + ); + CloudAgentSettingsService::$client = $noFenceClient; + $assert((new CloudAutomaticOperationRecoveryService())->tick(), + 'No-fence DB-failure create was not claimed.'); + $noFenceRetry['job']->refresh(); + $noFenceRetry['order']->refresh(); + $noFenceRetry['invoice']->refresh(); + $assert(preg_match( + '/^prepared:create_not_applied:v1:[a-f0-9]{64}$/D', + (string) $noFenceRetry['job']->effect_resolution_reason + ) === 1 + && (string) $noFenceRetry['job']->effect_resolution === 'unknown' + && $noFenceRetry['order']->refunded_at === null + && (string) $noFenceRetry['invoice']->status === 'paid' + && count($noFenceClient->requests) === 1, + 'No-fence financial failure did not retain its durable prepare.'); + $noFenceRetry['invoice']->purpose = 'provision'; + $noFenceRetry['invoice']->save(); + $makeRecoveryReady($noFenceRetry['job']); + // Deliberately leave the response queue empty: any Agent call now fails. + $assert((new CloudAutomaticOperationRecoveryService())->tick(), + 'Prepared no-fence create was not retried locally.'); + $assert(count($noFenceClient->requests) === 1, + 'Prepared no-fence retry contacted an offline Agent.'); + $assertCreateSettled( + $noFenceRetry, + 'Prepared no-fence create did not settle after the DB fault cleared.' + ); + + // If the Agent fence write committed but the same PHP transaction later + // failed, recovery reads that immutable verdict once and never POSTs it a + // second time. + $fenceRetry = $makeTimedOutCreate('fence-db-retry'); + $fenceRetry['invoice']->purpose = 'renewal'; + $fenceRetry['invoice']->save(); + $makeRecoveryReady($fenceRetry['job']); + $fenceClient = new FinancialCreateRecoveryAgentClient(); + $fenceClient->responses[] = $inspectionFor( + $fenceRetry['job'], + $fenceRetry['identity'], + true + ); + $fenceClient->responses[] = new CloudAgentError( + 'no committed fence', + 404, + 'not_found', + false, + 404 + ); + $committedFence = null; + $fenceClient->responses[] = static function ( + array $request + ) use (&$committedFence, $fenceRetry): array { + $payload = $request['payload']; + $committedFence = [ + 'operation_id' => (string) $fenceRetry['job']->operation_id, + 'resolution_id' => (string) $payload['resolution_id'], + 'resolution' => (string) $payload['resolution'], + 'inspection_checked_at' => + (int) $payload['inspection_checked_at'], + 'reason_sha256' => (string) $payload['reason_sha256'], + 'resolved_at' => time(), + 'resolved_by' => 0, + 'replayed' => false, + ]; + return $committedFence; + }; + CloudAgentSettingsService::$client = $fenceClient; + $assert((new CloudAutomaticOperationRecoveryService())->tick(), + 'Active-fence DB-failure create was not claimed.'); + $fenceRetry['job']->refresh(); + $assert(is_array($committedFence) + && count($fenceClient->requests) === 3 + && $fenceClient->requests[0]['method'] === 'GET' + && str_ends_with($fenceClient->requests[0]['target'], '/inspect') + && $fenceClient->requests[1]['method'] === 'GET' + && str_ends_with( + $fenceClient->requests[1]['target'], + '/fence-resolution' + ) + && $fenceClient->requests[2]['method'] === 'POST' + && str_ends_with( + $fenceClient->requests[2]['target'], + '/resolve-fence' + ) + && preg_match( + '/^prepared:create_not_applied:v1:[a-f0-9]{64}$/D', + (string) $fenceRetry['job']->effect_resolution_reason + ) === 1 + && (string) $fenceRetry['job']->effect_resolution === 'unknown', + 'Fence commit plus PHP failure did not preserve resumable evidence.'); + $fenceRetry['invoice']->purpose = 'provision'; + $fenceRetry['invoice']->save(); + $makeRecoveryReady($fenceRetry['job']); + $fenceClient->responses[] = $committedFence; + $assert((new CloudAutomaticOperationRecoveryService())->tick(), + 'Prepared active-fence create was not retried.'); + $assert(count($fenceClient->requests) === 4 + && $fenceClient->requests[3]['method'] === 'GET' + && str_ends_with( + $fenceClient->requests[3]['target'], + '/fence-resolution' + ), + 'Prepared active-fence retry wrote or re-inspected Agent state.'); + $assertCreateSettled( + $fenceRetry, + 'Prepared active-fence create did not settle after DB recovery.' + ); + + // A fence write can fail before the Agent commits it. The next tick first + // proves that no prior verdict exists, then obtains a fresh inspection. + // checked_at is freshness metadata, not part of the durable conclusion: + // the same facts must produce the same marker and fence reason. + $stableCase = $makeTimedOutCreate('stable-inspection-marker'); + $makeRecoveryReady($stableCase['job']); + $stableClient = new FinancialCreateRecoveryAgentClient(); + $stableFirstInspection = $inspectionFor( + $stableCase['job'], + $stableCase['identity'], + true + ); + $stableClient->responses = [ + $stableFirstInspection, + new CloudAgentError('no committed fence', 404, 'not_found', false, 404), + new CloudAgentError('fence write unavailable', 503, 'unavailable', true, 503), + new CloudAgentError('no committed fence', 404, 'not_found', false, 404), + ]; + CloudAgentSettingsService::$client = $stableClient; + $assert((new CloudAutomaticOperationRecoveryService())->tick(), + 'Stable-marker first recovery attempt was not claimed.'); + $stableCase['job']->refresh(); + $stableMarker = (string) $stableCase['job']->effect_resolution_reason; + $assert(preg_match( + '/^prepared:create_not_applied:v1:[a-f0-9]{64}$/D', + $stableMarker + ) === 1 + && (string) $stableCase['job']->effect_resolution === 'unknown' + && count($stableClient->requests) === 4, + 'Failed fence write did not retain a stable prepared create marker.'); + $makeRecoveryReady($stableCase['job']); + $stableSecondInspection = $stableFirstInspection; + $stableSecondInspection['checked_at'] = + (int) $stableFirstInspection['checked_at'] + 1; + $stableCommittedFence = null; + $stableClient->responses = [ + new CloudAgentError('no committed fence', 404, 'not_found', false, 404), + $stableSecondInspection, + new CloudAgentError('no committed fence', 404, 'not_found', false, 404), + static function (array $request) use ( + &$stableCommittedFence, + $stableCase + ): array { + $payload = $request['payload']; + $stableCommittedFence = [ + 'operation_id' => (string) $stableCase['job']->operation_id, + 'resolution_id' => (string) $payload['resolution_id'], + 'resolution' => (string) $payload['resolution'], + 'inspection_checked_at' => + (int) $payload['inspection_checked_at'], + 'reason_sha256' => (string) $payload['reason_sha256'], + 'resolved_at' => time(), + 'resolved_by' => 0, + 'replayed' => false, + ]; + return $stableCommittedFence; + }, + ]; + $assert((new CloudAutomaticOperationRecoveryService())->tick(), + 'Stable-marker create did not resume from fresh inspection.'); + $stableExpectedReason = + '自动恢复已确认原开通操作未创建虚拟机或面板管理资源;operation_id=' + . (string) $stableCase['job']->operation_id . ';evidence=' + . substr($stableMarker, -64); + $assert(is_array($stableCommittedFence) + && count($stableClient->requests) === 8 + && $stableClient->requests[4]['method'] === 'GET' + && str_ends_with( + $stableClient->requests[4]['target'], + '/fence-resolution' + ) + && $stableClient->requests[5]['method'] === 'GET' + && str_ends_with($stableClient->requests[5]['target'], '/inspect') + && $stableClient->requests[7]['method'] === 'POST' + && (string) $stableClient->requests[7]['payload']['reason_sha256'] + === hash('sha256', $stableExpectedReason), + 'Fresh equivalent inspection changed the prepared fence reason or ' + . 'replayed the original create.'); + $assertCreateSettled( + $stableCase, + 'Stable-marker retry did not finish the create financial ledger.' + ); + + // Partial/residual host state and an identity mismatch are both + // contradictions, not refund evidence. They remain fenced and retryable. + $conflictCase = $makeTimedOutCreate('inspection-conflicts'); + $makeRecoveryReady($conflictCase['job']); + $conflictClient = new FinancialCreateRecoveryAgentClient(); + $partialInspection = $inspectionFor( + $conflictCase['job'], + $conflictCase['identity'], + true + ); + $partialInspection['outcome'] = 'uncertain'; + $partialInspection['summary_code'] = 'evidence_incomplete'; + $partialInspection['effect_resolution'] = 'partial'; + $partialInspection['error_code'] = 'cleanup_required'; + $partialInspection['cleanup'] = [ + 'available' => true, + 'plan_id' => str_repeat('a', 64), + 'steps' => ['delete_managed_volumes'], + ]; + unset($partialInspection['effect_projection']); + $conflictClient->responses[] = $partialInspection; + CloudAgentSettingsService::$client = $conflictClient; + $assert((new CloudAutomaticOperationRecoveryService())->tick(), + 'Partial create evidence was not safely deferred.'); + $conflictCase['job']->refresh(); + $conflictCase['order']->refresh(); + $conflictCase['invoice']->refresh(); + $assert((string) $conflictCase['job']->effect_resolution === 'unknown' + && (bool) $conflictCase['job']->finalization_incident + && $conflictCase['order']->refunded_at === null + && (string) $conflictCase['invoice']->status === 'paid' + && (string) User::findOrFail(1)->money + === $conflictCase['walletBefore'] + && (int) CloudOffer::findOrFail(1)->stock + === $conflictCase['stockBefore'], + 'Partial create evidence released a financial/resource fence.'); + $makeRecoveryReady($conflictCase['job']); + $wrongIdentity = $inspectionFor( + $conflictCase['job'], + $conflictCase['identity'] + ); + $wrongIdentity['resource_id'] = 'wrong-create-resource'; + $conflictClient->responses[] = $wrongIdentity; + $assert((new CloudAutomaticOperationRecoveryService())->tick(), + 'Create identity contradiction was not safely deferred.'); + $conflictCase['job']->refresh(); + $assert((string) $conflictCase['job']->effect_resolution === 'unknown' + && (bool) $conflictCase['job']->finalization_incident + && $conflictCase['order']->fresh()->refunded_at === null + && (string) User::findOrFail(1)->money + === $conflictCase['walletBefore'] + && (int) CloudOffer::findOrFail(1)->stock + === $conflictCase['stockBefore'], + 'Create identity contradiction released a financial/resource fence.'); + + // A caller cannot reuse an automatic claim after its lease has expired. + $leaseToken = bin2hex(random_bytes(16)); + Capsule::table('cloud_jobs')->where('id', (int) $conflictCase['job']->id) + ->update([ + 'available_at' => 0, + 'lease_token' => $leaseToken, + 'lease_until' => time() + 180, + ]); + $staleClaim = CloudJob::findOrFail((int) $conflictCase['job']->id); + Capsule::table('cloud_jobs')->where('id', (int) $conflictCase['job']->id) + ->update(['lease_until' => time() - 1]); + $staleRequestCount = count($conflictClient->requests); + $staleRejected = false; + try { + CloudCreateFailureRecoveryService::recover($staleClaim); + } catch (CloudError $error) { + $staleRejected = true; + } + $assert($staleRejected + && count($conflictClient->requests) === $staleRequestCount + && (string) CloudJob::findOrFail( + (int) $conflictCase['job']->id + )->effect_resolution === 'unknown' + && $conflictCase['order']->fresh()->refunded_at === null, + 'Expired create recovery lease contacted Agent or released finance.'); + echo "Cloud SQLite purchase, provisioning, refund, ambiguity, quota and tenant checks passed (crypto double only).\n"; }