diff --git a/agent/api/server.go b/agent/api/server.go index 6f3c6155..d536d050 100644 --- a/agent/api/server.go +++ b/agent/api/server.go @@ -1083,6 +1083,7 @@ func (s *Server) routeOperations(w http.ResponseWriter, r *http.Request, segment return } } + applyExecutionFencedCreateInspection(&inspection, stored) // Identity always comes from the immutable Journal record, never from a // hypervisor adapter or its observation of current host state. inspection.OperationID = stored.OperationID @@ -1147,6 +1148,43 @@ func (s *Server) routeOperations(w http.ResponseWriter, r *http.Request, segment writeError(w, http.StatusBadRequest, "invalid_fence_resolution", err.Error(), nil) return } + if actor.SystemRecovery && actor.UserID == 0 && stored.Action == "create-vm" { + if request.Resolution != operation.FenceResolutionNotApplied { + writeError(w, http.StatusConflict, "operation_fence_not_resolvable", "automatic create recovery can only confirm that creation was not applied", nil) + return + } + if stored.ExecutionFenced { + writeError(w, http.StatusConflict, "create_absence_not_confirmed", "automatic create recovery cannot conclude while the original worker may still be running", nil) + return + } + fenceState, err := s.Journal.FenceState(operationID) + if err != nil { + writeMappedError(w, err, nil) + return + } + if fenceState == operation.FenceStateActive { + _, incidentFound, err := s.Journal.CompletionDeliveryIncident(operationID) + if err != nil { + writeMappedError(w, err, nil) + return + } + if incidentFound { + writeError(w, http.StatusConflict, "create_absence_not_confirmed", "automatic create recovery requires fresh complete host absence evidence", nil) + return + } + ctx, cancel := context.WithTimeout(r.Context(), s.OperationTimeout) + inspection, inspectErr := s.Hypervisor.InspectOperation(ctx, stored) + cancel() + if inspectErr != nil { + writeMappedError(w, inspectErr, nil) + return + } + if !systemCreateAbsenceConfirmed(stored, request, inspection) { + writeError(w, http.StatusConflict, "create_absence_not_confirmed", "automatic create recovery requires fresh complete host absence evidence", nil) + return + } + } + } resolution, replayed, err := s.Journal.ResolveResourceFence(operationID, actor.UserID, request) switch { case errors.Is(err, operation.ErrNotFound): @@ -1514,6 +1552,55 @@ func applyCompletionDeliveryIncident(inspection *hypervisor.OperationInspection, inspection.CompletionDeliveryErrorCode = incident.ErrorCode } +func applyExecutionFencedCreateInspection(inspection *hypervisor.OperationInspection, record operation.Record) { + if inspection == nil || record.Action != "create-vm" || !record.ExecutionFenced { + return + } + inspection.Outcome = hypervisor.InspectionOutcomeUncertain + inspection.SummaryCode = hypervisor.InspectionSummaryIncomplete + inspection.ErrorCode = "" + if record.Error != nil { + inspection.ErrorCode = record.Error.Code + } + inspection.EffectResolution = hypervisor.InspectionEffectInconclusive + inspection.EffectProjection = nil + inspection.AbortJob = hypervisor.OperationAbortInspection{} + inspection.Cleanup = hypervisor.OperationCleanupPlan{} +} + +// systemCreateAbsenceConfirmed accepts only a fresh, exact read-only host +// observation that both the VM and all panel-managed creation artifacts are +// absent. It does not infer absence from the original operation error and does +// not authorize any create replay or host mutation. +func systemCreateAbsenceConfirmed(record operation.Record, request operation.FenceResolutionRequest, + inspection hypervisor.OperationInspection) bool { + if record.Action != "create-vm" || record.State != operation.StateFailed || + inspection.OperationID != record.OperationID || inspection.Action != record.Action || + inspection.ResourceID != record.ResourceID || inspection.RequestFingerprint != record.Fingerprint || + inspection.OperationStatus != record.State || inspection.CheckedAt < request.InspectionCheckedAt || + inspection.Outcome != hypervisor.InspectionOutcomeFailed || + inspection.SummaryCode != hypervisor.InspectionSummaryFailed || + inspection.ErrorCode != "creation_absent" || + inspection.EffectResolution != hypervisor.InspectionEffectNotApplied || + inspection.CompletionDeliveryState != "" || inspection.CompletionDeliveryErrorCode != "" || + inspection.AbortJob.Available || inspection.Cleanup.Available || + len(inspection.EffectProjection) == 0 || len(inspection.EffectProjection) > 1024 { + return false + } + var projection map[string]json.RawMessage + if err := json.Unmarshal(inspection.EffectProjection, &projection); err != nil || len(projection) != 2 { + return false + } + var creationAbsent, managedArtifactsAbsent bool + if err := json.Unmarshal(projection["creation_absent"], &creationAbsent); err != nil || !creationAbsent { + return false + } + if err := json.Unmarshal(projection["managed_artifacts_absent"], &managedArtifactsAbsent); err != nil || !managedArtifactsAbsent { + return false + } + return true +} + func (s *Server) releaseAsync(operationID string) { s.asyncMu.Lock() defer s.asyncMu.Unlock() diff --git a/agent/api/server_test.go b/agent/api/server_test.go index 1e35ac7d..e260b90f 100644 --- a/agent/api/server_test.go +++ b/agent/api/server_test.go @@ -23,9 +23,11 @@ import ( type testHypervisor struct { hypervisor.Unsupported - mu sync.Mutex - inspections int - configureCalls int + mu sync.Mutex + inspections int + configureCalls int + inspectionResult *hypervisor.OperationInspection + inspectionErr error } func (h *testHypervisor) Ping(context.Context) error { return nil } @@ -43,8 +45,16 @@ func (h *testHypervisor) ConfigureStorage(context.Context, hypervisor.StorageCon func (h *testHypervisor) InspectOperation(_ context.Context, record operation.Record) (hypervisor.OperationInspection, error) { h.mu.Lock() + defer h.mu.Unlock() h.inspections++ - h.mu.Unlock() + if h.inspectionErr != nil { + return hypervisor.OperationInspection{}, h.inspectionErr + } + if h.inspectionResult != nil { + inspection := *h.inspectionResult + inspection.EffectProjection = append(json.RawMessage(nil), inspection.EffectProjection...) + return inspection, nil + } return hypervisor.OperationInspection{ OperationID: record.OperationID, OperationStatus: record.State, @@ -548,6 +558,7 @@ func TestSystemRecoveryActorCanOnlyInspectAndResolveItsExactOperation(t *testing {"raw operation", http.MethodGet, "/v1/operations/" + operationID, nil}, {"other operation", http.MethodGet, "/v1/operations/operation-other-recovery/inspect", nil}, {"host mutation", http.MethodPost, "/v1/storage/configuration", []byte(`{"operation_id":"operation-system-recovery"}`)}, + {"create mutation", http.MethodPost, "/v1/vms", []byte(`{"operation_id":"operation-system-recovery"}`)}, } { t.Run(test.name, func(t *testing.T) { response := api.request(t, test.method, test.target, test.body, actor) @@ -571,6 +582,260 @@ func TestSystemRecoveryActorCanOnlyInspectAndResolveItsExactOperation(t *testing } } +func TestSystemCreateRecoveryRequiresFreshCompleteAbsenceEvidence(t *testing.T) { + api := newTestAPI(t) + operationID := "operation-system-create-recovery" + resourceID := "123e4567-e89b-42d3-a456-426614174000" + requestBody := []byte(`{"operation_id":"operation-system-create-recovery"}`) + spec := operation.Spec{ + OperationID: operationID, + ResourceID: resourceID, + Action: "create-vm", + Fingerprint: operation.Fingerprint(http.MethodPost, "/v1/vms", requestBody), + Destructive: true, + } + if _, _, err := api.journal.Execute(context.Background(), spec, func(context.Context) (json.RawMessage, error) { + return nil, &worker.Error{ + Code: "result_unconfirmed", Message: "create result was not confirmed", + EffectMayHaveApplied: true, + } + }); err == nil { + t.Fatal("seed create failure unexpectedly succeeded") + } + actor := auth.Actor{UserID: 0, SystemRecovery: true, OperationID: operationID} + request := operation.FenceResolutionRequest{ + ResolutionID: "40404040404040404040404040404040", + Resolution: operation.FenceResolutionApplied, + InspectionCheckedAt: api.now.Unix(), + ReasonSHA256: "cdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcd", + Action: spec.Action, + ResourceID: spec.ResourceID, + RequestFingerprint: spec.Fingerprint, + } + encodeRequest := func() []byte { + t.Helper() + body, err := json.Marshal(request) + if err != nil { + t.Fatal(err) + } + return body + } + assertActive := func() { + t.Helper() + state, err := api.journal.FenceState(operationID) + if err != nil || state != operation.FenceStateActive { + t.Fatalf("create fence state=%q err=%v", state, err) + } + } + + applied := api.request( + t, + http.MethodPost, + "/v1/operations/"+operationID+"/resolve-fence", + encodeRequest(), + actor, + ) + if applied.Code != http.StatusConflict { + t.Fatalf("system create applied resolution = %d %s", applied.Code, applied.Body.String()) + } + assertActive() + + request.Resolution = operation.FenceResolutionNotApplied + inconclusive := api.request( + t, + http.MethodPost, + "/v1/operations/"+operationID+"/resolve-fence", + encodeRequest(), + actor, + ) + if inconclusive.Code != http.StatusConflict || + !bytes.Contains(inconclusive.Body.Bytes(), []byte(`"code":"create_absence_not_confirmed"`)) { + t.Fatalf("inconclusive create resolution = %d %s", inconclusive.Code, inconclusive.Body.String()) + } + assertActive() + + exactInspection := func(checkedAt int64) hypervisor.OperationInspection { + return hypervisor.OperationInspection{ + OperationID: operationID, + Action: spec.Action, + ResourceID: resourceID, + RequestFingerprint: spec.Fingerprint, + OperationStatus: operation.StateFailed, + CheckedAt: checkedAt, + Outcome: hypervisor.InspectionOutcomeFailed, + SummaryCode: hypervisor.InspectionSummaryFailed, + ErrorCode: "creation_absent", + EffectResolution: hypervisor.InspectionEffectNotApplied, + EffectProjection: json.RawMessage( + `{"creation_absent":true,"managed_artifacts_absent":true}`, + ), + } + } + stale := exactInspection(api.now.Add(-time.Second).Unix()) + api.hv.mu.Lock() + api.hv.inspectionResult = &stale + api.hv.mu.Unlock() + staleResponse := api.request( + t, + http.MethodPost, + "/v1/operations/"+operationID+"/resolve-fence", + encodeRequest(), + actor, + ) + if staleResponse.Code != http.StatusConflict || + !bytes.Contains(staleResponse.Body.Bytes(), []byte(`"code":"create_absence_not_confirmed"`)) { + t.Fatalf("stale create resolution = %d %s", staleResponse.Code, staleResponse.Body.String()) + } + assertActive() + + incomplete := exactInspection(api.now.Unix()) + incomplete.EffectProjection = json.RawMessage(`{"creation_absent":true,"managed_artifacts_absent":false}`) + api.hv.mu.Lock() + api.hv.inspectionResult = &incomplete + api.hv.mu.Unlock() + incompleteResponse := api.request( + t, + http.MethodPost, + "/v1/operations/"+operationID+"/resolve-fence", + encodeRequest(), + actor, + ) + if incompleteResponse.Code != http.StatusConflict || + !bytes.Contains(incompleteResponse.Body.Bytes(), []byte(`"code":"create_absence_not_confirmed"`)) { + t.Fatalf("incomplete create absence = %d %s", incompleteResponse.Code, incompleteResponse.Body.String()) + } + assertActive() + + exact := exactInspection(api.now.Unix()) + api.hv.mu.Lock() + api.hv.inspectionResult = &exact + api.hv.mu.Unlock() + inspection := api.request( + t, + http.MethodGet, + "/v1/operations/"+operationID+"/inspect", + nil, + actor, + ) + if inspection.Code != http.StatusOK || + !bytes.Contains(inspection.Body.Bytes(), []byte(`"error_code":"creation_absent"`)) || + !bytes.Contains(inspection.Body.Bytes(), []byte(`"effect_resolution":"not_applied"`)) { + t.Fatalf("system create inspection = %d %s", inspection.Code, inspection.Body.String()) + } + resolved := api.request( + t, + http.MethodPost, + "/v1/operations/"+operationID+"/resolve-fence", + encodeRequest(), + actor, + ) + if resolved.Code != http.StatusOK || + !bytes.Contains(resolved.Body.Bytes(), []byte(`"resolution":"not_applied"`)) || + !bytes.Contains(resolved.Body.Bytes(), []byte(`"resolved_by":0`)) { + t.Fatalf("system create not-applied resolution = %d %s", resolved.Code, resolved.Body.String()) + } + replayed := api.request( + t, + http.MethodPost, + "/v1/operations/"+operationID+"/resolve-fence", + encodeRequest(), + actor, + ) + if replayed.Code != http.StatusOK || + !bytes.Contains(replayed.Body.Bytes(), []byte(`"resolution":"not_applied"`)) || + !bytes.Contains(replayed.Body.Bytes(), []byte(`"replayed":true`)) { + t.Fatalf("system create resolution replay = %d %s", replayed.Code, replayed.Body.String()) + } + api.hv.mu.Lock() + inspections := api.hv.inspections + configureCalls := api.hv.configureCalls + api.hv.mu.Unlock() + if inspections != 5 || configureCalls != 0 { + t.Fatalf("system create recovery inspections=%d mutations=%d", inspections, configureCalls) + } +} + +func TestSystemCreateRecoveryRejectsExecutionFencedAbsence(t *testing.T) { + api := newTestAPI(t) + operationID := "operation-system-create-unreaped" + resourceID := "123e4567-e89b-42d3-a456-426614174002" + spec := operation.Spec{ + OperationID: operationID, + ResourceID: resourceID, + Action: "create-vm", + Fingerprint: operation.Fingerprint(http.MethodPost, "/v1/vms", []byte(`{"operation_id":"operation-system-create-unreaped"}`)), + Destructive: true, + } + if _, _, err := api.journal.Execute(context.Background(), spec, func(context.Context) (json.RawMessage, error) { + return nil, &worker.Error{ + Code: "worker_reap_unconfirmed", Message: "worker may still be running", + EffectMayHaveApplied: true, ExecutionFenced: true, + } + }); err == nil { + t.Fatal("seed execution-fenced create unexpectedly succeeded") + } + exactAbsence := hypervisor.OperationInspection{ + OperationID: operationID, + Action: spec.Action, + ResourceID: resourceID, + RequestFingerprint: spec.Fingerprint, + OperationStatus: operation.StateFailed, + CheckedAt: api.now.Unix(), + Outcome: hypervisor.InspectionOutcomeFailed, + SummaryCode: hypervisor.InspectionSummaryFailed, + ErrorCode: "creation_absent", + EffectResolution: hypervisor.InspectionEffectNotApplied, + EffectProjection: json.RawMessage(`{"creation_absent":true,"managed_artifacts_absent":true}`), + } + api.hv.mu.Lock() + api.hv.inspectionResult = &exactAbsence + api.hv.mu.Unlock() + actor := auth.Actor{UserID: 0, SystemRecovery: true, OperationID: operationID} + + inspected := api.request(t, http.MethodGet, "/v1/operations/"+operationID+"/inspect", nil, actor) + if inspected.Code != http.StatusOK { + t.Fatalf("execution-fenced create inspection = %d %s", inspected.Code, inspected.Body.String()) + } + var inspection hypervisor.OperationInspection + if err := json.Unmarshal(inspected.Body.Bytes(), &inspection); err != nil { + t.Fatal(err) + } + if inspection.Outcome != hypervisor.InspectionOutcomeUncertain || + inspection.SummaryCode != hypervisor.InspectionSummaryIncomplete || + inspection.EffectResolution != hypervisor.InspectionEffectInconclusive || + len(inspection.EffectProjection) != 0 { + t.Fatalf("execution-fenced create absence was treated as final: %#v", inspection) + } + + request := operation.FenceResolutionRequest{ + ResolutionID: "50505050505050505050505050505050", + Resolution: operation.FenceResolutionNotApplied, + InspectionCheckedAt: api.now.Unix(), + ReasonSHA256: "dededededededededededededededededededededededededededededededede", + Action: spec.Action, + ResourceID: spec.ResourceID, + RequestFingerprint: spec.Fingerprint, + } + body, err := json.Marshal(request) + if err != nil { + t.Fatal(err) + } + resolved := api.request(t, http.MethodPost, "/v1/operations/"+operationID+"/resolve-fence", body, actor) + if resolved.Code != http.StatusConflict || + !bytes.Contains(resolved.Body.Bytes(), []byte(`"code":"create_absence_not_confirmed"`)) { + t.Fatalf("execution-fenced create resolution = %d %s", resolved.Code, resolved.Body.String()) + } + if state, err := api.journal.FenceState(operationID); err != nil || state != operation.FenceStateActive { + t.Fatalf("execution-fenced create was released state=%q err=%v", state, err) + } + api.hv.mu.Lock() + inspections := api.hv.inspections + api.hv.mu.Unlock() + if inspections != 1 { + t.Fatalf("execution-fenced create resolve repeated host inspection: %d", inspections) + } +} + func TestMutationUsesIsolatedExecutorExactlyOnce(t *testing.T) { api := newTestAPI(t) executor := &fakeMutationExecutor{result: json.RawMessage(`{"configured":true}`)} diff --git a/agent/hypervisor/libvirt_operation_admin_test.go b/agent/hypervisor/libvirt_operation_admin_test.go index a468e659..2b2c2e1a 100644 --- a/agent/hypervisor/libvirt_operation_admin_test.go +++ b/agent/hypervisor/libvirt_operation_admin_test.go @@ -648,6 +648,27 @@ func TestAbsentCreateTrafficResidueRequiresExplicitCleanup(t *testing.T) { } } +func TestFailedCreateAlwaysRequiresLiveInspection(t *testing.T) { + for name, record := range map[string]operation.Record{ + "ordinary failure without effect flag": { + OperationID: "operation-create-ordinary-failure", ResourceID: "123e4567-e89b-42d3-a456-426614174000", + Action: "create-vm", State: operation.StateFailed, + Error: &operation.PublicError{Code: "operation_failed", Message: "create failed"}, + }, + "unknown effect failure": { + OperationID: "operation-create-unknown-effect", ResourceID: "123e4567-e89b-42d3-a456-426614174001", + Action: "create-vm", State: operation.StateFailed, EffectMayHaveApplied: true, + Error: &operation.PublicError{Code: "result_unconfirmed", Message: "create result was not confirmed"}, + }, + } { + t.Run(name, func(t *testing.T) { + if storedOperationInspectionAvailable(record) { + t.Fatal("failed create reused its original Journal error instead of requiring live host inspection") + } + }) + } +} + func TestBackupPushMatchesOnlyExactOwnedTarget(t *testing.T) { target := "/var/lib/virt-cloud/managed/snapshot.qcow2" matching := `` diff --git a/agent/operation/journal.go b/agent/operation/journal.go index fe987610..fc82b7bb 100644 --- a/agent/operation/journal.go +++ b/agent/operation/journal.go @@ -182,7 +182,8 @@ type Spec struct { // leaf action must be reviewed and added here explicitly. func SystemRecoveryActionAllowed(action string) bool { switch action { - case "power-start", "power-shutdown", "power-reboot", "power-reset", "power-destroy", + case "create-vm", + "power-start", "power-shutdown", "power-reboot", "power-reset", "power-destroy", "reinstall-vm", "enter-rescue", "force-start-rescue", "exit-rescue:connected", "exit-rescue:suspended", "change-network", "network-suspend", "network-resume", @@ -877,6 +878,17 @@ func (j *Journal) ResolveResourceFence(operationID string, resolvedBy int64, if resolvedBy == 0 && !SystemRecoveryActionAllowed(record.Action) { return ErrFenceNotResolvable } + // Automatic create recovery is deliberately one-way: fresh host + // absence evidence may release a failed create fence as not applied, + // but the zero-ID system actor can never declare that a create applied. + // Nor may it release a fence while an unreaped worker could still create + // the resource after the inspection. Materializing an applied create + // requires the original immutable completion result and remains PHP's + // responsibility. + if resolvedBy == 0 && record.Action == "create-vm" && + (record.ExecutionFenced || request.Resolution != FenceResolutionNotApplied) { + return ErrFenceNotResolvable + } if encoded := resolutions.Get([]byte(operationID)); encoded != nil { if err := json.Unmarshal(encoded, &resolution); err != nil || resolution.OperationID != operationID { return fmt.Errorf("decode fence resolution %q", operationID) diff --git a/agent/operation/journal_test.go b/agent/operation/journal_test.go index 79d15c74..53462489 100644 --- a/agent/operation/journal_test.go +++ b/agent/operation/journal_test.go @@ -1445,8 +1445,97 @@ func TestSystemFenceResolutionUsesZeroAuditActorButRejectsNegativeActors(t *test } } +func TestSystemCreateFenceResolutionOnlyAcceptsNotApplied(t *testing.T) { + j := openTestJournal(t) + now := time.Date(2026, 10, 21, 2, 0, 0, 0, time.UTC) + j.SetClockForTesting(func() time.Time { return now }) + newCreate := func(operationID, resourceID string) Spec { + return Spec{ + OperationID: operationID, + ResourceID: resourceID, + Action: "create-vm", + Fingerprint: Fingerprint("POST", "/v1/vms", []byte(`{"operation_id":"`+operationID+`"}`)), + Destructive: true, + } + } + seedUnknown := func(spec Spec) { + t.Helper() + if _, _, err := j.Execute(context.Background(), spec, func(context.Context) (json.RawMessage, error) { + return nil, testFailure{code: "result_unconfirmed", message: "create result is unknown", effect: true} + }); err == nil { + t.Fatal("seed create failure unexpectedly succeeded") + } + } + requestFor := func(spec Spec, resolutionID, resolution string) FenceResolutionRequest { + return FenceResolutionRequest{ + ResolutionID: resolutionID, + Resolution: resolution, + InspectionCheckedAt: now.Unix(), + ReasonSHA256: "abababababababababababababababababababababababababababababababab", + Action: spec.Action, + ResourceID: spec.ResourceID, + RequestFingerprint: spec.Fingerprint, + } + } + + systemSpec := newCreate("operation-system-create", "vm-system-create") + seedUnknown(systemSpec) + applied := requestFor( + systemSpec, + "10101010101010101010101010101010", + FenceResolutionApplied, + ) + if _, _, err := j.ResolveResourceFence(systemSpec.OperationID, 0, applied); !errors.Is(err, ErrFenceNotResolvable) { + t.Fatalf("system create applied resolution err=%v", err) + } + if state, err := j.FenceState(systemSpec.OperationID); err != nil || state != FenceStateActive { + t.Fatalf("rejected system create resolution released fence state=%q err=%v", state, err) + } + notApplied := applied + notApplied.ResolutionID = "20202020202020202020202020202020" + notApplied.Resolution = FenceResolutionNotApplied + resolved, replayed, err := j.ResolveResourceFence(systemSpec.OperationID, 0, notApplied) + if err != nil || replayed || resolved.Resolution != FenceResolutionNotApplied || resolved.ResolvedBy != 0 { + t.Fatalf("system create not-applied resolution=%#v replayed=%v err=%v", resolved, replayed, err) + } + + manualSpec := newCreate("operation-manual-create", "vm-manual-create") + seedUnknown(manualSpec) + manualApplied := requestFor( + manualSpec, + "30303030303030303030303030303030", + FenceResolutionApplied, + ) + resolved, replayed, err = j.ResolveResourceFence(manualSpec.OperationID, 42, manualApplied) + if err != nil || replayed || resolved.Resolution != FenceResolutionApplied || resolved.ResolvedBy != 42 { + t.Fatalf("manual create applied resolution=%#v replayed=%v err=%v", resolved, replayed, err) + } + + fencedSpec := newCreate("operation-system-create-unreaped", "vm-system-create-unreaped") + if _, _, err := j.Execute(context.Background(), fencedSpec, func(context.Context) (json.RawMessage, error) { + return nil, testFailure{ + code: "worker_reap_unconfirmed", message: "worker may still be running", + effect: true, fence: true, + } + }); err == nil { + t.Fatal("seed execution-fenced create unexpectedly succeeded") + } + fencedRequest := requestFor( + fencedSpec, + "40404040404040404040404040404040", + FenceResolutionNotApplied, + ) + if _, _, err := j.ResolveResourceFence(fencedSpec.OperationID, 0, fencedRequest); !errors.Is(err, ErrFenceNotResolvable) { + t.Fatalf("system execution-fenced create resolution err=%v", err) + } + if state, err := j.FenceState(fencedSpec.OperationID); err != nil || state != FenceStateActive { + t.Fatalf("execution-fenced create was released state=%q err=%v", state, err) + } +} + func TestSystemRecoveryActionAllowlistIsExplicit(t *testing.T) { allowed := []string{ + "create-vm", "power-start", "power-shutdown", "power-reboot", "power-reset", "power-destroy", "reinstall-vm", "enter-rescue", "force-start-rescue", "exit-rescue:connected", "exit-rescue:suspended", @@ -1469,7 +1558,7 @@ func TestSystemRecoveryActionAllowlistIsExplicit(t *testing.T) { } denied := []string{ - "", "create-vm", "configure-storage", "service-resume", "future-unknown", "power-future", + "", "configure-storage", "service-resume", "future-unknown", "power-future", "exit-rescue:future", "create-snapshot:0123456789abcdef0123456789abcdef:future", "create-snapshot:0123456789ABCDEF0123456789ABCDEF:filesystem", diff --git a/docs/DEPLOYMENT.md b/docs/DEPLOYMENT.md index a9ce5ec3..a9099d88 100644 --- a/docs/DEPLOYMENT.md +++ b/docs/DEPLOYMENT.md @@ -162,6 +162,8 @@ systemctl enable --now virt-cloud-worker.service \ 普通 worker 负责购买、创建、重装、邮件和最终删除;至少两个策略 worker 处理到期断网、网络恢复与删除前策略;lifecycle 生成续费账单并推进账期、流量、宽限和删除状态。`virt-cloud-recovery-worker.service` 是必需的独立 operation 恢复进程,覆盖策略表中的实例电源、重装/救援、公网 IP、实例设置、快照、防火墙、删除、生命周期网络、模板与 ISO Agent 叶子任务。它只做 operation-bound inspection/栅栏审计、持久晚到回执的 PHP 收敛或聚合父任务的本地汇总,不投递邮件,也绝不重放原 mutation。离线、身份冲突或账本异常按固定低频间隔持续核对,没有次数或总时长上限。它与普通队列隔离,避免某个离线 Agent 的核对占住正常 worker。 +恢复进程也覆盖已派发开通任务的失败收尾:完整缺失证据回执可直接完成本地退款和资源释放;回执不足时,只读检查原 operation,确认实例及托管资源均不存在后才收尾。升级此能力需同时更新面板代码与宿主机 Agent,并重启已有 worker/recovery worker 使其加载新代码;无需数据库迁移或新增 service。旧 Agent 拒绝开通自动检查时会保留订单占用并低频重试,不会绕过授权退款。已有失败且效果未知的开通订单会被后台领取,不必重发开通请求。 + 在面板机器构建并安装 Agent Gateway: ```bash diff --git a/docs/TASK_EXECUTION.md b/docs/TASK_EXECUTION.md index 8b21eb13..973a56ae 100644 --- a/docs/TASK_EXECUTION.md +++ b/docs/TASK_EXECUTION.md @@ -51,7 +51,9 @@ Go context 不能可靠中止已经进入 CGo/libvirt 的调用,因此每个 完成回执是不可变事件,不是新的变更请求。Agent 持续投递同一 `event_id`,直到 PHP 返回 ACK;重启、旧 outbox 中保存的尝试次数或历史过期时间都不能淘汰未确认回执。PHP 以 `event_id`、`operation_id` 和正文摘要幂等接收。ACK 只删除投递 outbox,不删除主 Journal 记录。 -PHP 的 `receipt_deadline_at` 到期时直接把任务终态化为失败。之后收到相同回执不能更新原任务终态;除开通任务的精确成功回执专用路径外,不支持恢复的类型只返回晚到 ACK。对策略表允许的任务,精确匹配 `agent_id + operation_id + event_id + 正文摘要` 的成功回执会先以原加密请求收敛对应 PHP 业务账本,再提交 `effect_resolution=applied`;明确无副作用的失败回执则收敛为 `not_applied`。开通任务不进入通用 inspection、fence 或 mutation replay;只有已持久化且身份完全匹配的成功回执,才能在重新验证未退款订单、已支付开通发票、库存、IP、placement、服务周期和凭据 escrow 后原子补完本地账本。该回执路径只使用 PHP 已持久化事件,不联系 Agent;任务效果结论与事件 ACK 在同一事务提交,结论提交后立即清除恢复密文。如果进程在事件提交后或预提交后崩溃,专用 recovery worker 会从 terminal job 出发,以已有 `(host_id,operation_id)` 唯一索引精确读取 `received` 事件,重算正文摘要后继续 PHP 提交。它只有在既有 prepared 证据同方向、同完整 operation hash 且栅栏语义一致时才可继续收敛;方向、哈希、ID、栅栏语义、密文或账本状态冲突时,必须记录具体 `recovery_*` 异常并保留 blocker/escrow。异常事件可以被隔离,但任务仍按固定低频退避留在恢复队列,修复数据、身份、网络或 Agent 后继续核对;不存在 6 小时、12 次或其他永久退出上限。它不扫描事件全表,也不依赖 Agent 再次投递。相同回执重传只返回幂等 ACK;不同正文复用事件 ID 或 operation ID 必须拒绝并记录安全日志。 +PHP 的 `receipt_deadline_at` 到期时直接把任务终态化为失败。之后收到相同回执不能更新原任务终态;不支持恢复的类型只返回晚到 ACK。对策略表允许的任务,精确匹配 `agent_id + operation_id + event_id + 正文摘要` 的成功回执会先以原加密请求收敛对应 PHP 业务账本,再提交 `effect_resolution=applied`;明确无副作用的失败回执则收敛为 `not_applied`。回执路径只使用 PHP 已持久化事件,不联系 Agent;任务效果结论与事件 ACK 在同一事务提交,结论提交后立即清除恢复密文。如果进程在事件提交后或预提交后崩溃,专用 recovery worker 会从 terminal job 出发,以已有 `(host_id,operation_id)` 唯一索引精确读取 `received` 事件,重算正文摘要后继续 PHP 提交。它只有在既有 prepared 证据同方向、同完整 operation hash 且栅栏语义一致时才可继续收敛;方向、哈希、ID、栅栏语义、密文或账本状态冲突时,必须记录具体 `recovery_*` 异常并保留 blocker/escrow。异常事件可以被隔离,但任务仍按固定低频退避留在恢复队列,修复数据、身份、网络或 Agent 后继续核对;不存在 6 小时、12 次或其他永久退出上限。它不扫描事件全表,也不依赖 Agent 再次投递。相同回执重传只返回幂等 ACK;不同正文复用事件 ID 或 operation ID 必须拒绝并记录安全日志。 + +开通任务使用独立的资金恢复终结器,不加入通用效果物化策略,也不重放创建请求。精确成功回执在重新验证未退款订单、已支付开通发票、库存、IP、placement、服务周期和凭据 escrow 后原子补完实例账本。晚到失败回执必须同时证明 `creation_absent=true`、`managed_artifacts_absent=true` 且没有未知副作用,才能原子退款、释放库存/IP/预留并 ACK;仅有失败状态或 `safe_to_resubmit` 不足以退款。没有完整失败回执时,恢复 worker 仍会领取该订单,使用绑定原 operation 的 system-recovery 身份只读检查;只有新鲜、身份一致且明确的 `creation_absent` 证据才进入专用预提交。预提交持久化证据但不提前退款,确认必要的 Agent 栅栏审计后再原子提交资金和资源账本。Agent 已提交栅栏但 PHP 提交失败时,后续轮次可依据同一预提交读取既有栅栏结论并继续,不会重新开通或重复退款。系统身份对开通只允许 `not_applied` 栅栏结论,不能凭只读状态推断成功并物化实例;残留资源、运行中操作、身份冲突、无效证据和矛盾成功回执继续保留保护。此能力需更新面板和 Agent,并保持 recovery worker 运行;不需要数据库迁移或新增 systemd 服务。 任何回执或 inspection 改变 PHP 账本前,都必须重新验证加密请求、任务/宿主机/资源身份,以及该种类终结器会消费的请求与结果字段;可解密但不完整的 escrow 不得以空账本假收敛。prepared 证据绑定规范化后的完整 operation;重试次数只属于调度状态,不参与 immutable operation hash,也不能破坏 prepare 后的崩溃续跑。如果恢复必须写 Agent 栅栏审计结论,该有界网络写入与随后的 PHP effect commit 必须在同一 job-row 锁下串行,写前再次验证租约、incident 和优先回执,防止陈旧 worker 在冲突证据落盘后解除 Agent fence。 @@ -63,7 +65,7 @@ PHP 的 `receipt_deadline_at` 到期时直接把任务终态化为失败。之 - 任务本身立即结束,不再轮询、不再自动重试;内部恢复协调器只能低频读取允许种类的同一 Agent operation 证据; - 同一 VM、卷、网络过滤器、地址或账单的冲突变更立即返回明确错误,不进入等待队列; -- 不自动退款、不释放 reservation、不再次删除磁盘; +- 在可信清理证据确认前不自动退款、不释放 reservation、不再次删除磁盘; - 管理员可以执行独立的只读检查,但检查不会翻转原任务终态,也不会续作原 mutation; - 策略表中的 Agent 叶子任务可以自动消费同一 operation 的不可变证据:晚到终态回执是纯 PHP 收敛;没有回执时,由独立的 `cloudWorker --recovery-only` 进程每轮最多领取一项。常驻进程在精确 `received` 事件、普通叶子任务、聚合任务三个通道间轮转,空通道同轮跳过;持续有任务时,每三个完成的 tick 给每个通道一次机会,避免回执或叶子队列持续非空导致聚合任务饥饿。普通/策略 worker 从不联系恢复目标,因此离线宿主的核对不会阻塞正常派发。恢复使用绑定 operation ID 的 system-recovery actor 调用 Agent 只读 `inspect`,且只接受新鲜、种类专用、资源身份一致的 `applied/not_applied` 证据。它先预提交 PHP 账本并继续保留 blocker;只有活动失败栅栏才写入一次 operation-bound Journal 审计结论,再提交 `effect_resolution`。原 mutation 永不调用。`partial/inconclusive`、Agent 离线、ID 冲突、无效回执、密文损坏或账本/实例状态矛盾都会保留 blocker,向页面公开具体异常,并按固定低频退避持续重检;没有次数和总时长上限。修复根因后,同一 operation 可继续自动收敛并清除 incident;相互矛盾的持久证据不得被覆盖或猜测解锁; - 超级管理员只能对已有种类专用原子终结器的任务提交审计结论。开通任务仅支持由新鲜宿主机缺失证据确认 `not_applied`,并在同一事务退款、释放库存/IP/预留后再解栅栏;支付任务只根据签名回调或外部支付平台审计证据收敛本地账本,不重新下单; diff --git a/web/app/Services/Cloud/CloudAdminOperationRecoveryService.php b/web/app/Services/Cloud/CloudAdminOperationRecoveryService.php index fc31fab0..180e5e77 100644 --- a/web/app/Services/Cloud/CloudAdminOperationRecoveryService.php +++ b/web/app/Services/Cloud/CloudAdminOperationRecoveryService.php @@ -184,9 +184,9 @@ final class CloudAdminOperationRecoveryService [$job, $client] = self::jobContext( $id, 30.0, - CloudOperationRecoveryPolicy::kinds() + self::automaticRecoveryKinds() ); - if (!CloudOperationRecoveryPolicy::supports((string) $job->kind) + if (!self::supportsAutomaticRecovery($job) || (string) $job->execution_mode !== 'agent' || (string) $job->phase !== 'terminal' || (string) $job->outcome !== 'failed' @@ -238,8 +238,20 @@ final class CloudAdminOperationRecoveryService [$job, $client] = self::jobContext( $id, 30.0, - CloudOperationRecoveryPolicy::kinds() + self::automaticRecoveryKinds() ); + if ((string) $job->kind === 'create') { + CloudCreateFailureRecoveryService::assertPreparedForFence( + $job, + $resolution, + $reason + ); + CloudCreateFailureFinalizer::assertResolutionSupported( + $job, + $resolution, + $inspection + ); + } $checkedAt = $inspection['checked_at'] ?? null; $identity = CloudOperationRecoveryIdentity::expected( $job, @@ -251,7 +263,7 @@ final class CloudAdminOperationRecoveryService 'request_fingerprint' => $inspection['request_fingerprint'] ?? null, ]); - if (!CloudOperationRecoveryPolicy::supports((string) $job->kind) + if (!self::supportsAutomaticRecovery($job) || (string) $job->execution_mode !== 'agent' || (string) $job->phase !== 'terminal' || (string) $job->outcome !== 'failed' @@ -260,9 +272,10 @@ final class CloudAdminOperationRecoveryService || !(bool) $job->effect_may_have_applied || (bool) $job->safe_to_resubmit || (string) $job->effect_resolution !== 'unknown' - || !self::automaticPreparedReason( - (string) $job->effect_resolution_reason - ) + || ((string) $job->kind !== 'create' + && !self::automaticPreparedReason( + (string) $job->effect_resolution_reason + )) || ($inspection['operation_id'] ?? null) !== (string) $job->operation_id || ($inspection['effect_resolution'] ?? null) !== $resolution @@ -316,9 +329,16 @@ final class CloudAdminOperationRecoveryService [$job, $client] = self::jobContext( $id, 30.0, - CloudOperationRecoveryPolicy::kinds() + self::automaticRecoveryKinds() ); - if (!CloudOperationRecoveryPolicy::supports((string) $job->kind) + if ((string) $job->kind === 'create') { + CloudCreateFailureRecoveryService::assertPreparedForFence( + $job, + $resolution, + $reason + ); + } + if (!self::supportsAutomaticRecovery($job) || (string) $job->execution_mode !== 'agent' || (string) $job->phase !== 'terminal' || (string) $job->outcome !== 'failed' @@ -327,9 +347,10 @@ final class CloudAdminOperationRecoveryService || !(bool) $job->effect_may_have_applied || (bool) $job->safe_to_resubmit || (string) $job->effect_resolution !== 'unknown' - || !self::automaticPreparedReason( - (string) $job->effect_resolution_reason - )) { + || ((string) $job->kind !== 'create' + && !self::automaticPreparedReason( + (string) $job->effect_resolution_reason + ))) { throw new CloudError('任务尚未完成可验证的账本预提交', 409); } // A prepared escrow persists the identity validated before the PHP @@ -568,6 +589,18 @@ final class CloudAdminOperationRecoveryService ) === 1; } + /** Create is inspectable, but never enters generic effect finalization. */ + private static function automaticRecoveryKinds(): array + { + return [...CloudOperationRecoveryPolicy::kinds(), 'create']; + } + + private static function supportsAutomaticRecovery(CloudJob $job): bool + { + return CloudOperationRecoveryPolicy::supports((string) $job->kind) + || ((string) $job->kind === 'create' && $job->order_id !== null); + } + private static function assertCompletionDeliveryUsable( array $inspection ): void { diff --git a/web/app/Services/Cloud/CloudAutomaticOperationRecoveryService.php b/web/app/Services/Cloud/CloudAutomaticOperationRecoveryService.php index 8298383a..60c03d39 100644 --- a/web/app/Services/Cloud/CloudAutomaticOperationRecoveryService.php +++ b/web/app/Services/Cloud/CloudAutomaticOperationRecoveryService.php @@ -544,7 +544,10 @@ final class CloudAutomaticOperationRecoveryService ): void { $policy->whereIn( 'kind', - CloudOperationRecoveryPolicy::kinds() + // Create uses a separate financial + // finalizer, never the generic replay or + // effect-projection path below. + [...CloudOperationRecoveryPolicy::kinds(), 'create'] )->where( static function ($ready) use ( $now, @@ -580,21 +583,6 @@ final class CloudAutomaticOperationRecoveryService } ); } - )->orWhere( - static function ($create): void { - // Create keeps its specialised financial - // finalizer. It is recoverable here only from - // a durable exact completion receipt; it must - // never enter generic inspection or replay. - $create->where('kind', 'create') - ->whereExists( - static function ($events): void { - self::receivedCreateSuccessEventExists( - $events - ); - } - ); - } ); }) ->when( @@ -703,13 +691,6 @@ final class CloudAutomaticOperationRecoveryService 'operation_id', (string) $job->operation_id )->where('disposition', 'received') - ->when( - (string) $job->kind === 'create', - static fn ($events) => $events->where( - 'outcome', - 'succeeded' - ) - ) ->lockForUpdate() ->first(); if ($event !== null) { @@ -726,6 +707,16 @@ final class CloudAutomaticOperationRecoveryService (string) $job->effect_resolution_reason, $now ); + // The dedicated create coordinator owns its encrypted + // prepare/commit binding. Never wrap or replace that marker + // when resuming a fence-write/financial-commit crash window. + if ((string) $job->kind === 'create' + && preg_match( + '/^prepared:create_not_applied:v1:[a-f0-9]{64}$/D', + (string) $job->effect_resolution_reason + ) === 1) { + $attempt = (string) $job->effect_resolution_reason; + } if ($attempt === false) { $job->finalization_incident = 1; $job->finalization_error_code = @@ -792,12 +783,6 @@ final class CloudAutomaticOperationRecoveryService ->where('recovery_events.disposition', 'received'); } - private static function receivedCreateSuccessEventExists($events): void - { - self::receivedEventExists($events); - $events->where('recovery_events.outcome', 'succeeded'); - } - /** * @return array{delay:int,error_code:string}|null Retry state, or null * after a committed resolution. @@ -813,6 +798,10 @@ final class CloudAutomaticOperationRecoveryService )) { throw new CloudError('自动核对租约已失效', 409); } + if ((string) $job->kind === 'create') { + CloudCreateFailureRecoveryService::recover($job); + return null; + } $context = self::recoveryContext($job); // A late immutable completion receipt is a PHP-local two-phase diff --git a/web/app/Services/Cloud/CloudCreateFailureRecoveryService.php b/web/app/Services/Cloud/CloudCreateFailureRecoveryService.php new file mode 100644 index 00000000..408c9af4 --- /dev/null +++ b/web/app/Services/Cloud/CloudCreateFailureRecoveryService.php @@ -0,0 +1,712 @@ +getConnection()->transaction( + static function () use ($source, $sourceEvent): string { + [$user, $order, $job] = self::lockLedger($source); + $event = CloudAgentCompletionEvent::where( + 'event_id', + (string) $sourceEvent->event_id + )->lockForUpdate()->first(); + if ($event === null) { + return 'received'; + } + if ((string) $event->disposition !== 'received') { + return (string) $event->disposition; + } + if ((int) $event->host_id !== (int) $job->host_id + || !hash_equals( + (string) $job->operation_id, + (string) $event->operation_id + )) { + throw new CloudRecoveryOperationIdentityConflict( + '开通失败回执与原任务不匹配', + 409 + ); + } + CloudOperationCompletionService::assertEventIdentityForJob( + $job, + $event + ); + $operation = CloudOperationCompletionService::operation($event); + if (self::settledNotApplied($job, $order, $event)) { + // A fresh inspection may have completed the atomic PHP + // finalizer before the Agent could deliver its immutable + // failure. The receipt no longer decides the ledger, but + // the exact identity and committed audit allow it to be + // ACKed without reopening or rewriting that resolution. + $event->disposition = 'applied'; + $event->applied_at = time(); + $event->save(); + return 'applied'; + } + self::assertRecoverableJob($job); + if (!self::receiptProvesAbsence($event, $operation)) { + $event->disposition = 'late'; + $event->applied_at = null; + $event->save(); + return 'late'; + } + + self::assertNoSuccessfulReceiptLocked($job); + self::finalizeLocked( + $job, + $user, + $order, + 'late_completion_receipt:' . (string) $event->event_id, + $event + ); + return 'applied'; + }, + 3 + ); + } + + /** Recover one lease-owned create without replaying its Agent mutation. */ + public static function recover(CloudJob $claimed): void + { + $job = self::claimedJob($claimed); + $prepared = self::preparedEvidence($job); + if ($prepared !== null) { + if (!$prepared['fence_resolution_required']) { + self::finishPrepared($claimed, $prepared, false); + return; + } + try { + self::finishPrepared($claimed, $prepared, null); + return; + } catch (CloudAgentError $error) { + if ($error->agentHttpStatus !== 404) { + throw $error; + } + // No fence verdict was committed before the prior PHP crash. + // A new write requires new, bounded read-only evidence. + } + } + + $inspection = CloudAdminOperationRecoveryService::inspectForRecovery( + (string) $job->id + ); + $prepared = self::prepareInspection($claimed, $inspection); + self::finishPrepared( + $claimed, + $prepared, + $prepared['fence_resolution_required'] ? $inspection : false + ); + } + + /** + * Authenticate the durable PHP intent before an automatic Agent fence read + * or write. This is deliberately create-specific and does not broaden the + * generic recovery allow-list. + */ + public static function assertPreparedForFence( + CloudJob $job, + string $resolution, + string $reason + ): void { + if ($resolution !== 'not_applied') { + throw new CloudError('开通任务只支持确认未应用', 409); + } + self::assertRecoverableJob($job); + $prepared = self::preparedEvidence($job); + if ($prepared === null + || !hash_equals($prepared['reason'], $reason)) { + throw new CloudRecoveryEvidenceConflict( + '开通任务的恢复预提交证据不匹配', + 409 + ); + } + } + + /** + * @return array{marker:string,reason:string,identity:array,inspection:array,fence_resolution_required:bool} + */ + private static function prepareInspection( + CloudJob $claimed, + array $inspection + ): array { + $job = self::claimedJob($claimed); + $identity = CloudOperationRecoveryIdentity::expected($job, false); + CloudOperationRecoveryIdentity::assertMatches($identity, [ + 'action' => $inspection['action'] ?? null, + 'resource_id' => $inspection['resource_id'] ?? null, + 'request_fingerprint' => + $inspection['request_fingerprint'] ?? null, + ]); + self::assertInspectionProvesAbsence($job, $inspection); + $evidence = self::inspectionEvidence($inspection); + $marker = self::inspectionMarker($evidence); + $reason = self::fenceReason($job, $marker); + + return (new CloudJob())->getConnection()->transaction( + static function () use ( + $claimed, + $identity, + $evidence, + $marker, + $reason + ): array { + [$user, $order, $locked] = self::lockLedger($claimed); + self::assertClaimedJob($locked, $claimed); + self::assertNoSuccessfulReceiptLocked($locked); + + $payload = self::payload($locked); + $lockedIdentity = CloudOperationRecoveryIdentity:: + expectedFromPayload($locked, $payload, false); + CloudOperationRecoveryIdentity::assertMatches( + $lockedIdentity, + $identity + ); + $payload[self::ESCROW_KEY] = [ + 'version' => self::ESCROW_VERSION, + 'marker' => $marker, + 'reason' => $reason, + 'identity' => CloudOperationRecoveryIdentity::persisted( + $lockedIdentity + ), + 'inspection' => $evidence, + ]; + $encoded = json_encode( + $payload, + JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES + | JSON_UNESCAPED_UNICODE + ); + if (strlen($encoded) > 196608) { + throw new CloudRecoveryPayloadUnavailable( + '开通任务恢复预提交数据过大', + 409 + ); + } + $locked->effect_resolution_reason = $marker; + $locked->sealed_payload = CloudSecrets::encrypt( + $payload, + 'job-payload' + ); + $locked->updated_at = time(); + $locked->save(); + + return [ + 'marker' => $marker, + 'reason' => $reason, + 'identity' => $lockedIdentity, + 'inspection' => $evidence, + 'fence_resolution_required' => (bool) + $evidence['fence_resolution_required'], + ]; + }, + 3 + ); + } + + /** + * Resolve or verify the Agent fence while holding the canonical financial + * locks, then commit every PHP ledger transition in that same transaction. + * + * @param array{marker:string,reason:string,identity:array,inspection:array,fence_resolution_required:bool} $prepared + * @param array|false|null $inspection Fresh evidence, false when no fence + * is required, null when resuming an already committed fence resolution. + */ + private static function finishPrepared( + CloudJob $claimed, + array $prepared, + array|false|null $inspection + ): void { + (new CloudJob())->getConnection()->transaction( + static function () use ($claimed, $prepared, $inspection): void { + [$user, $order, $job] = self::lockLedger($claimed); + self::assertClaimedJob($job, $claimed); + $lockedPrepared = self::preparedEvidence($job); + if ($lockedPrepared === null + || !hash_equals( + $prepared['marker'], + $lockedPrepared['marker'] + ) + || !hash_equals( + $prepared['reason'], + $lockedPrepared['reason'] + )) { + throw new CloudRecoveryEvidenceConflict( + '开通任务的恢复预提交已变化', + 409 + ); + } + self::assertNoSuccessfulReceiptLocked($job); + + if ($inspection === null) { + CloudAdminOperationRecoveryService:: + preparedFenceResolutionForRecovery( + (string) $job->id, + 'not_applied', + $lockedPrepared['reason'] + ); + } elseif ($inspection !== false) { + CloudAdminOperationRecoveryService::resolveFenceForRecovery( + (string) $job->id, + 'not_applied', + $lockedPrepared['reason'], + $inspection + ); + } elseif ($lockedPrepared['fence_resolution_required']) { + throw new CloudRecoveryEvidenceConflict( + '开通任务仍需解除宿主机资源栅栏', + 409 + ); + } + + self::assertNoSuccessfulReceiptLocked($job); + self::finalizeLocked( + $job, + $user, + $order, + 'automatic_create_not_applied:' + . substr($lockedPrepared['marker'], -64) + ); + }, + 3 + ); + } + + /** + * @return array{marker:string,reason:string,identity:array,inspection:array,fence_resolution_required:bool}|null + */ + private static function preparedEvidence(CloudJob $job): ?array + { + $marker = (string) ($job->effect_resolution_reason ?? ''); + if (preg_match( + '/^prepared:create_not_applied:v1:[a-f0-9]{64}$/D', + $marker + ) !== 1) { + return null; + } + $payload = self::payload($job); + $value = $payload[self::ESCROW_KEY] ?? null; + $identity = is_array($value) ? ($value['identity'] ?? null) : null; + $inspection = is_array($value) ? ($value['inspection'] ?? null) : null; + if (!is_array($value) || array_is_list($value) + || array_diff(array_keys($value), [ + 'version', 'marker', 'reason', 'identity', 'inspection', + ]) !== [] + || ($value['version'] ?? null) !== self::ESCROW_VERSION + || !is_string($value['marker'] ?? null) + || !hash_equals($marker, $value['marker']) + || !is_string($value['reason'] ?? null) + || mb_strlen($value['reason']) < 20 + || mb_strlen($value['reason']) > 500 + || !is_array($identity) || array_is_list($identity) + || !is_array($inspection) || array_is_list($inspection)) { + throw new CloudRecoveryPayloadUnavailable( + '开通任务恢复预提交数据无效', + 409 + ); + } + $expected = CloudOperationRecoveryIdentity::expectedFromPayload( + $job, + $payload, + false + ); + CloudOperationRecoveryIdentity::assertMatches($expected, $identity); + self::assertInspectionProvesAbsence($job, $inspection, false); + CloudOperationRecoveryIdentity::assertMatches($expected, [ + 'action' => $inspection['action'] ?? null, + 'resource_id' => $inspection['resource_id'] ?? null, + 'request_fingerprint' => + $inspection['request_fingerprint'] ?? null, + ]); + $evidence = self::inspectionEvidence($inspection); + $expectedMarker = self::inspectionMarker($evidence); + if (!hash_equals($expectedMarker, $marker) + || !hash_equals(self::fenceReason($job, $marker), $value['reason'])) { + throw new CloudRecoveryEvidenceConflict( + '开通任务恢复预提交摘要不匹配', + 409 + ); + } + return [ + 'marker' => $marker, + 'reason' => $value['reason'], + 'identity' => $expected, + 'inspection' => $evidence, + 'fence_resolution_required' => + (bool) $evidence['fence_resolution_required'], + ]; + } + + private static function assertInspectionProvesAbsence( + CloudJob $job, + array $inspection, + bool $requireFresh = true + ): void { + $checkedAt = $inspection['checked_at'] ?? null; + $projection = $inspection['effect_projection'] ?? null; + if (($inspection['operation_id'] ?? null) + !== (string) $job->operation_id + || ($inspection['journal_status'] ?? null) !== 'failed' + || ($inspection['outcome'] ?? null) !== 'failed' + || ($inspection['summary_code'] ?? null) !== 'operation_failed' + || ($inspection['effect_resolution'] ?? null) !== 'not_applied' + || !is_int($checkedAt) + || ($requireFresh && ($checkedAt < time() + - self::INSPECTION_MAX_AGE_SECONDS + || $checkedAt > time() + 5)) + || !is_array($projection) || array_is_list($projection) + || ($projection['creation_absent'] ?? null) !== true + || ($projection['managed_artifacts_absent'] ?? null) !== true + || !in_array( + $inspection['fence_state'] ?? null, + ['not_required', 'active'], + true + ) + || !is_bool($inspection['fence_resolution_required'] ?? null) + || (bool) $inspection['fence_resolution_required'] + !== (($inspection['fence_state'] ?? null) === 'active') + || array_key_exists('completion_delivery_state', $inspection)) { + throw new CloudRecoveryEvidenceConflict( + '宿主机检查没有同时确认虚拟机及面板管理资源均不存在', + 409 + ); + } + CloudCreateFailureFinalizer::assertResolutionSupported( + $job, + 'not_applied', + $inspection + ); + } + + private static function receiptProvesAbsence( + CloudAgentCompletionEvent $event, + array $operation + ): bool { + $result = $operation['result'] ?? null; + return (string) $event->outcome === 'failed' + && !(bool) $event->effect_may_have_applied + && is_array($result) && !array_is_list($result) + && ($result['creation_absent'] ?? null) === true + && ($result['managed_artifacts_absent'] ?? null) === true; + } + + /** @return array{0:User,1:CloudOrder,2:CloudJob} */ + private static function lockLedger(CloudJob $source): array + { + $user = User::where('id', (int) $source->user_id) + ->lockForUpdate()->first(); + $order = CloudOrder::where('id', (int) $source->order_id) + ->lockForUpdate()->first(); + $job = CloudJob::where('id', (int) $source->id) + ->lockForUpdate()->first(); + if ($user === null || $order === null || $job === null + || (int) $job->user_id !== (int) $user->id + || (int) $job->order_id !== (int) $order->id + || (int) $order->user_id !== (int) $user->id) { + throw new CloudRecoveryInstanceStateConflict( + '开通任务与资金账本关联不一致', + 409 + ); + } + return [$user, $order, $job]; + } + + private static function assertRecoverableJob(CloudJob $job): void + { + self::assertDispatchedEnvelope($job); + if ((string) $job->kind !== 'create' + || $job->order_id === null + || (string) $job->execution_mode !== 'agent' + || (string) $job->phase !== 'terminal' + || (string) $job->outcome !== 'failed' + || (string) $job->status !== 'failed' + || !(bool) $job->effect_may_have_applied + || (bool) $job->safe_to_resubmit + || (string) $job->effect_resolution !== 'unknown') { + throw new CloudRecoveryInstanceStateConflict( + '开通任务不处于可自动核对状态', + 409 + ); + } + } + + private static function settledNotApplied( + CloudJob $job, + CloudOrder $order, + CloudAgentCompletionEvent $event + ): bool { + self::assertDispatchedEnvelope($job); + $reason = $job->effect_resolution_reason; + $completionEventId = $job->completion_event_id; + return (string) $event->outcome === 'failed' + && (string) $job->kind === 'create' + && $job->order_id !== null + && (string) $job->execution_mode === 'agent' + && (string) $job->phase === 'terminal' + && (string) $job->outcome === 'failed' + && (string) $job->status === 'failed' + && (int) $job->dispatch_attempts === 1 + && !(bool) $job->safe_to_resubmit + && (string) $job->effect_resolution === 'not_applied' + && (int) ($job->effect_resolved_at ?? 0) > 0 + && is_string($reason) + && mb_strlen($reason) >= 20 + && mb_strlen($reason) <= 500 + && !(bool) $job->finalization_incident + && $job->finalization_error_code === null + && $job->instance_id === null + && $order->instance_id === null + && (string) $order->status === 'failed' + && $order->refunded_at !== null + && ($completionEventId === null + || (is_string($completionEventId) + && hash_equals( + $completionEventId, + (string) $event->event_id + ))); + } + + /** Validate create's immutable generation without broadening policy. */ + private static function assertDispatchedEnvelope(CloudJob $job): void + { + CloudDispatchLedger::assertDispatched($job); + if ((string) $job->attempts !== '1' + || (int) ($job->host_id ?? 0) < 1 + || (int) ($job->dispatch_deadline_at ?? 0) < 1 + || (int) ($job->dispatch_started_at ?? 0) < 1 + || (($job->agent_accepted_at ?? null) !== null + && (int) $job->agent_accepted_at < 1) + || (int) ($job->dispatched_at ?? 0) < 1 + || (int) ($job->receipt_deadline_at ?? 0) < 1 + || (int) ($job->terminal_at ?? 0) < 1 + || preg_match( + '/^[a-f0-9]{32}$/D', + (string) ($job->operation_id ?? '') + ) !== 1 + || preg_match( + '/^[a-f0-9]{64}$/D', + (string) ($job->request_fingerprint ?? '') + ) !== 1 + || preg_match( + '/^[A-Za-z0-9_.-]{16,128}$/D', + (string) ($job->request_idempotency_key ?? '') + ) !== 1) { + throw new CloudRecoveryInstanceStateConflict( + '开通任务单次派发信封不完整', + 409 + ); + } + } + + private static function assertClaimedJob( + CloudJob $job, + CloudJob $claimed + ): void { + self::assertRecoverableJob($job); + if (!is_string($claimed->lease_token) + || !is_string($job->lease_token) + || !hash_equals( + (string) $claimed->lease_token, + (string) $job->lease_token + ) + || (int) $job->lease_until <= time()) { + throw new CloudRecoveryInstanceStateConflict( + '开通任务自动核对租约已失效', + 409 + ); + } + } + + private static function claimedJob(CloudJob $claimed): CloudJob + { + $job = CloudJob::find((int) $claimed->id); + if ($job === null) { + throw new CloudError('任务不存在', 404); + } + self::assertClaimedJob($job, $claimed); + return $job; + } + + private static function assertNoSuccessfulReceiptLocked(CloudJob $job): void + { + $event = CloudAgentCompletionEvent::where( + 'host_id', + (int) $job->host_id + )->where( + 'operation_id', + (string) $job->operation_id + )->where('outcome', 'succeeded')->lockForUpdate()->first(); + if ($event === null) { + return; + } + $exact = CloudOperationCompletionService::isExactStoredEvent($event) + || CloudOperationCompletionService::isExactAppliedEvent($event); + if (!$exact) { + throw new CloudRecoveryOperationIdentityConflict( + '开通成功回执状态异常,不能执行退款', + 409 + ); + } + $payload = self::payload($job); + $identity = CloudOperationRecoveryIdentity::expectedFromPayload( + $job, + $payload, + true + ); + if (CloudOperationRecoveryIdentity::same( + $identity, + CloudOperationCompletionService::identity($event) + )) { + throw new CloudRecoveryEvidenceConflict( + '已收到匹配的实例创建成功回执,不能按未应用退款', + 409 + ); + } + throw new CloudRecoveryOperationIdentityConflict( + '开通成功回执身份冲突,不能执行退款', + 409 + ); + } + + private static function finalizeLocked( + CloudJob $job, + User $user, + CloudOrder $order, + string $resolutionReason, + ?CloudAgentCompletionEvent $event = null + ): void { + $payload = self::payload($job); + $identity = CloudOperationRecoveryIdentity::expectedFromPayload( + $job, + $payload, + false + ); + CloudCreateFailureFinalizer::refundAndReleaseLocked( + $job, + $user, + $order, + 'create_effect_not_applied' + ); + + $safePayload = CloudInput::safeResult($payload); + unset($safePayload[self::ESCROW_KEY]); + $safePayload[CloudOperationRecoveryIdentity::payloadKey()] = + CloudOperationRecoveryIdentity::persisted($identity); + $now = time(); + $job->effect_resolution = 'not_applied'; + $job->effect_resolved_at = $now; + $job->effect_resolved_by = null; + $job->effect_resolution_reason = $resolutionReason; + if ($event !== null) { + $job->completion_event_id = (string) $event->event_id; + } + $job->sealed_payload = CloudSecrets::encrypt( + $safePayload, + 'job-payload' + ); + $job->finalization_incident = 0; + $job->finalization_error_code = null; + $job->read_at = null; + $job->lease_token = null; + $job->lease_until = 0; + $job->available_at = 0; + $job->updated_at = $now; + $job->save(); + if ($event !== null) { + $event->disposition = 'applied'; + $event->applied_at = $now; + $event->save(); + } + } + + private static function payload(CloudJob $job): array + { + try { + $payload = CloudSecrets::decrypt( + (string) $job->sealed_payload, + 'job-payload' + ); + } catch (\Throwable $error) { + throw new CloudRecoveryPayloadUnavailable( + '开通任务恢复数据无法解密', + 409 + ); + } + if (!is_array($payload) || array_is_list($payload)) { + throw new CloudRecoveryPayloadUnavailable( + '开通任务恢复数据结构不完整', + 409 + ); + } + return $payload; + } + + private static function fenceReason(CloudJob $job, string $marker): string + { + return '自动恢复已确认原开通操作未创建虚拟机或面板管理资源;operation_id=' + . (string) $job->operation_id . ';evidence=' + . substr($marker, -64); + } + + /** Hash only stable evidence so a lost fence response can be resumed. */ + private static function inspectionMarker(array $evidence): string + { + unset($evidence['checked_at']); + return self::MARKER_PREFIX . hash( + 'sha256', + json_encode( + $evidence, + JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES + | JSON_UNESCAPED_UNICODE + ) + ); + } + + /** Return the bounded, canonical evidence persisted in encrypted escrow. */ + private static function inspectionEvidence(array $inspection): array + { + return [ + 'operation_id' => $inspection['operation_id'], + 'action' => $inspection['action'], + 'resource_id' => $inspection['resource_id'], + 'request_fingerprint' => $inspection['request_fingerprint'], + 'journal_status' => $inspection['journal_status'], + 'checked_at' => $inspection['checked_at'], + 'outcome' => $inspection['outcome'], + 'summary_code' => $inspection['summary_code'], + 'error_code' => $inspection['error_code'] ?? '', + 'effect_resolution' => $inspection['effect_resolution'], + 'fence_state' => $inspection['fence_state'], + 'fence_resolution_required' => + $inspection['fence_resolution_required'], + 'effect_projection' => [ + 'creation_absent' => true, + 'managed_artifacts_absent' => true, + ], + ]; + } +} diff --git a/web/app/Services/Cloud/CloudOperationRecoveryService.php b/web/app/Services/Cloud/CloudOperationRecoveryService.php index ba705b76..de4b4d73 100644 --- a/web/app/Services/Cloud/CloudOperationRecoveryService.php +++ b/web/app/Services/Cloud/CloudOperationRecoveryService.php @@ -365,6 +365,12 @@ final class CloudOperationRecoveryService ); } if ((string) $current->kind === 'create') { + if ((string) $event->outcome === 'failed') { + return CloudCreateFailureRecoveryService::settleReceipt( + $current, + $event + ); + } if ((string) $event->outcome !== 'succeeded' || (bool) $event->effect_may_have_applied || (bool) $event->safe_to_resubmit @@ -548,10 +554,11 @@ final class CloudOperationRecoveryService // eligible for the immutable receipt, terminate the event // instead of leaving an unclaimable "received" row. if (self::retryableCreateCompletion($job, $event)) { - // Create is intentionally outside the generic - // inspection policy. Its exact success receipt is the - // only safe recovery evidence, so a temporary local - // ledger fault must not consume that evidence. + // Create uses dedicated financial finalizers. Keep + // either exact outcome retryable when PHP could not + // finish inspecting or settling its ledger. An + // incomplete failure is ACKed late by the dedicated + // finalizer only after those local checks succeed. return 'received'; } if (self::lateReceiptResolution($job, $event) === null) { @@ -961,9 +968,10 @@ final class CloudOperationRecoveryService && (bool) $job->effect_may_have_applied && !(bool) $job->safe_to_resubmit && (string) $job->effect_resolution === 'unknown' - && (string) $event->outcome === 'succeeded' - && !(bool) $event->effect_may_have_applied - && !(bool) $event->safe_to_resubmit; + && ((string) $event->outcome === 'failed' + || ((string) $event->outcome === 'succeeded' + && !(bool) $event->effect_may_have_applied + && !(bool) $event->safe_to_resubmit)); } private static function assertRecoverableAppliedEvidence( diff --git a/web/tests/CloudAdminOperationRecoveryServiceTest.php b/web/tests/CloudAdminOperationRecoveryServiceTest.php index 9b9c68d5..e71094ea 100644 --- a/web/tests/CloudAdminOperationRecoveryServiceTest.php +++ b/web/tests/CloudAdminOperationRecoveryServiceTest.php @@ -363,11 +363,14 @@ namespace { ]); $before = count($client->requests); + Capsule::table('cloud_jobs')->where('id', 41)->update([ + 'kind' => 'host_storage_configure', + ]); $reject(static fn () => Recovery::inspectForRecovery('41'), 409); $reject(static fn () => Recovery::resolveFenceForRecovery( '41', 'not_applied', - 'automatic recovery must reject non-settings tasks', + 'automatic recovery must reject unregistered task kinds', [ 'operation_id' => 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa', 'checked_at' => $now, @@ -379,13 +382,16 @@ namespace { $reject(static fn () => Recovery::preparedFenceResolutionForRecovery( '41', 'not_applied', - 'automatic recovery must reject non-settings tasks' + 'automatic recovery must reject unregistered task kinds' ), 409); $assert(count($client->requests) === $before, - 'A non-settings task reached the system recovery Agent client.'); + 'An unregistered task reached the system recovery Agent client.'); + Capsule::table('cloud_jobs')->where('id', 41)->update([ + 'kind' => 'create', + ]); $operationA = 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa'; - $client->responses[] = [ + $client->responses[] = $createInspectionResponse = [ 'operation_id' => $operationA, 'operation_status' => 'failed', 'checked_at' => $now, @@ -430,6 +436,36 @@ namespace { ->value('effect_resolution') === 'unknown', 'Read-only inspection changed the task or its safety fence.'); + $client->responses[] = $createInspectionResponse; + $createInspection = Recovery::inspectForRecovery('41'); + $createRequest = $client->requests[array_key_last($client->requests)]; + $assert(($createInspection['error_code'] ?? null) === 'creation_absent' + && ($createInspection['effect_projection']['creation_absent'] ?? null) === true + && ($createInspection['effect_projection']['managed_artifacts_absent'] ?? null) === true + && $createRequest['method'] === 'GET' + && ($createRequest['system_recovery'] ?? false) === true + && $createRequest['actor']['operation_id'] === $operationA + && Capsule::table('cloud_jobs')->where('id', 41) + ->value('effect_resolution') === 'unknown', + 'Create inspection did not remain operation-bound and read-only.'); + $before = count($client->requests); + $reject(static fn () => Recovery::resolveFenceForRecovery( + '41', 'not_applied', + 'automatic create recovery requires durable financial intent', + $createInspection + ), 409); + $reject(static fn () => Recovery::resolveFenceForRecovery( + '41', 'applied', + 'automatic create recovery cannot infer a successful instance', + $createInspection + ), 409); + $reject(static fn () => Recovery::preparedFenceResolutionForRecovery( + '41', 'not_applied', + 'automatic create recovery requires durable financial intent' + ), 409); + $assert(count($client->requests) === $before, + 'An unprepared create reached the system Agent fence routes.'); + $client->responses[] = ['operation_id' => $operationA, 'operation_status' => 'uncertain', 'checked_at' => $now, 'outcome' => 'uncertain', 'summary_code' => 'evidence_incomplete']; diff --git a/web/tests/CloudFinancialWorkflowTest.php b/web/tests/CloudFinancialWorkflowTest.php index b50c512f..018e9498 100644 --- a/web/tests/CloudFinancialWorkflowTest.php +++ b/web/tests/CloudFinancialWorkflowTest.php @@ -25,6 +25,61 @@ namespace App\Services\Cloud { public static function encrypt(array $payload, string $purpose): string { $id = bin2hex(random_bytes(16)); self::$memory[$id] = $payload; return 'test-only:' . $id; } public static function decrypt(string $cipher, string $purpose): array { return self::$memory[substr($cipher, 10)]; } } + + /** Isolated read-only Agent transport for create recovery checks. */ + final class FinancialCreateRecoveryAgentClient + { + public array $requests = []; + public array $responses = []; + + public function systemRecoveryRequest( + string $method, + string $target, + string $operationId, + ?array $payload = null + ): array { + $this->requests[] = compact( + 'method', 'target', 'operationId', 'payload' + ); + if ($this->responses === []) { + throw new \RuntimeException( + 'Unexpected create recovery Agent request.' + ); + } + $response = array_shift($this->responses); + if ($response instanceof \Throwable) { + throw $response; + } + if (is_callable($response)) { + $response = $response(end($this->requests)); + } + if (!is_array($response)) { + throw new \RuntimeException( + 'Invalid create recovery Agent response.' + ); + } + return $response; + } + } + + final class CloudAgentSettingsService + { + public static ?FinancialCreateRecoveryAgentClient $client = null; + public static int $hostId = 0; + + public static function clientForConfiguration( + int $hostId, + float $timeoutSeconds = 120.0 + ): FinancialCreateRecoveryAgentClient { + if ($hostId !== self::$hostId || $timeoutSeconds !== 30.0 + || self::$client === null) { + throw new \RuntimeException( + 'Unexpected create recovery client request.' + ); + } + return self::$client; + } + } } namespace { require dirname(__DIR__) . '/vendor/autoload.php'; @@ -69,6 +124,10 @@ namespace { use App\Services\Cloud\CloudPlatformTestNetworkPolicy; use App\Services\Cloud\CloudDeploymentPageService; use App\Services\Cloud\CloudTemplateCatalogService; + use App\Services\Cloud\CloudAgentSettingsService; + use App\Services\Cloud\CloudAgentError; + use App\Services\Cloud\CloudCreateFailureRecoveryService; + use App\Services\Cloud\FinancialCreateRecoveryAgentClient; use GuzzleHttp\Client; use GuzzleHttp\Handler\MockHandler; use GuzzleHttp\HandlerStack; @@ -2915,5 +2974,892 @@ namespace { 'Unknown, applied, or sticky lifecycle evidence created a new mutation generation.'); } + // A create which crossed the Agent boundary may time out locally before + // the authenticated terminal failure reaches PHP. Exact Agent proof that + // both the guest and every managed artifact are absent must finish the + // original paid order locally; it must never dispatch a second create. + foreach (['203.0.113.20', '203.0.113.21', '203.0.113.22'] as $address) { + if (!CloudPublicIp::where('address', $address)->exists()) { + $ip = new CloudPublicIp(); + $ip->host_id = $host->id; + $ip->public_ip_pool_id = $sourcePublicPool->id; + $ip->address = $address; + $ip->prefix_length = 28; + $ip->state = 'available'; + $ip->created_at = $ip->updated_at = time(); + $ip->save(); + } + } + $lateFailureOrder = CloudOrderService::purchase(array_replace($input, [ + 'name' => 'vm-late-failure', + 'idempotency_key' => 'test-request-key-late-failure', + ]), $actor); + $lateFailureInvoice = CloudInvoice::where( + 'order_id', + $lateFailureOrder->id + )->firstOrFail(); + CloudInvoiceService::payBalance( + $lateFailureInvoice, + $actor, + 'provision-balance-late-failure' + ); + $lateFailureJob = CloudJob::where( + 'order_id', + $lateFailureOrder->id + )->firstOrFail(); + // Earlier lifecycle tests intentionally leave separately audited retries + // queued. Park those unrelated rows so this tracer observes exactly one + // create dispatch rather than depending on suite-wide queue order. + Capsule::table('cloud_jobs') + ->where('id', '<>', (int) $lateFailureJob->id) + ->where('phase', '<>', 'terminal') + ->update([ + 'available_at' => time() + 3600, + 'dispatch_deadline_at' => time() + 3600, + 'receipt_deadline_at' => time() + 3600, + ]); + Capsule::table('cloud_jobs')->where( + 'id', + (int) $lateFailureJob->id + )->update(['available_at' => 0]); + $lateFailureJob->refresh(); + $lateFailureIdentity = CloudOperationRecoveryIdentity::expected( + $lateFailureJob, + false + ); + $mock->append(new Response(202, [], json_encode([ + 'operation_id' => $lateFailureJob->operation_id, + 'status' => 'reserved', + ...$lateFailureIdentity, + 'replayed' => false, + 'retryable' => false, + ], JSON_THROW_ON_ERROR))); + $beforeLateFailureDispatch = count($history); + $assert($worker->tick(), + 'Late-failure create did not cross the Agent dispatch boundary.'); + $lateFailureJob->refresh(); + $assert((string) $lateFailureJob->phase === 'awaiting_receipt' + && (int) $lateFailureJob->dispatch_attempts === 1 + && count($history) === $beforeLateFailureDispatch + 1, + 'Late-failure create did not persist its single dispatch generation.'); + Capsule::table('cloud_jobs')->where( + 'id', + (int) $lateFailureJob->id + )->update([ + 'receipt_deadline_at' => time() - 1, + 'lease_token' => null, + 'lease_until' => 0, + ]); + $assert(CloudWorkerService::expireDueTaskForAdmin( + (int) $lateFailureJob->id + ) === 1, 'Late-failure create did not reach its local deadline.'); + $lateFailureJob->refresh(); + $lateFailureOrder->refresh(); + $lateFailureInvoice->refresh(); + $lateFailureReservation = CloudResourceReservation::where( + 'order_id', + $lateFailureOrder->id + )->firstOrFail(); + $lateFailureIp = CloudPublicIp::findOrFail( + (int) $lateFailureReservation->public_ip_id + ); + $walletWhileFenced = (string) User::findOrFail(1)->money; + $stockWhileFenced = (int) CloudOffer::findOrFail(1)->stock; + $assert((string) $lateFailureJob->phase === 'terminal' + && (string) $lateFailureJob->outcome === 'failed' + && (string) $lateFailureJob->effect_resolution === 'unknown' + && (string) $lateFailureOrder->status === 'failed' + && $lateFailureOrder->refunded_at === null + && (string) $lateFailureInvoice->status === 'paid' + && (string) $lateFailureReservation->state === 'uncertain' + && (string) $lateFailureIp->state === 'uncertain', + 'Timed-out dispatched create did not retain its financial fence.'); + + $lateFailureEventId = 'create-late-failure-event-0001'; + $lateFailureReceipt = [ + 'type' => 'completion', + 'id' => $lateFailureEventId, + 'agent_id' => (string) $lateFailureJob->host_id, + 'operation_id' => (string) $lateFailureJob->operation_id, + ...$lateFailureIdentity, + 'status' => 'failed', + 'effect_may_have_applied' => false, + 'safe_to_resubmit' => false, + 'error_code' => 'creation_absent', + 'result' => [ + 'uuid' => $lateFailureIdentity['resource_id'], + 'creation_absent' => true, + 'managed_artifacts_absent' => true, + ], + ]; + $lateFailureAccepted = CloudOperationCompletionService::accept( + $lateFailureReceipt + ); + $lateFailureJob->refresh(); + $lateFailureOrder->refresh(); + $lateFailureInvoice->refresh(); + $lateFailureReservation->refresh(); + $lateFailureIp->refresh(); + $lateFailureEvent = \App\Models\CloudAgentCompletionEvent::where( + 'event_id', + $lateFailureEventId + )->firstOrFail(); + $assert($lateFailureAccepted === [ + 'accepted' => true, + 'disposition' => 'applied', + ] + && (string) $lateFailureEvent->disposition === 'applied' + && $lateFailureEvent->applied_at !== null + && (string) $lateFailureJob->effect_resolution === 'not_applied' + && !(bool) $lateFailureJob->finalization_incident + && (string) $lateFailureOrder->status === 'failed' + && $lateFailureOrder->refunded_at !== null + && (string) $lateFailureInvoice->status === 'refunded' + && (string) $lateFailureReservation->state === 'released' + && (string) $lateFailureIp->state === 'available' + && (string) User::findOrFail(1)->money + === CloudMoney::decimal( + CloudMoney::cents($walletWhileFenced) + 999 + ) + && (int) CloudOffer::findOrFail(1)->stock + === $stockWhileFenced + 1 + && count($history) === $beforeLateFailureDispatch + 1, + 'Exact late create failure did not atomically refund, release, ACK, ' + . 'or replayed the original mutation.'); + $lateFailureWalletAfter = (string) User::findOrFail(1)->money; + $lateFailureStockAfter = (int) CloudOffer::findOrFail(1)->stock; + $lateFailureRefundedAt = (int) $lateFailureOrder->refunded_at; + $lateFailureJobUpdatedAt = (int) $lateFailureJob->updated_at; + $lateFailureAppliedAt = (int) $lateFailureEvent->applied_at; + $lateFailureDuplicate = CloudOperationCompletionService::accept( + $lateFailureReceipt + ); + $assert($lateFailureDuplicate === [ + 'accepted' => true, + 'disposition' => 'applied', + ] + && (string) User::findOrFail(1)->money + === $lateFailureWalletAfter + && (int) CloudOffer::findOrFail(1)->stock + === $lateFailureStockAfter + && (int) CloudOrder::findOrFail( + (int) $lateFailureOrder->id + )->refunded_at === $lateFailureRefundedAt + && (int) CloudJob::findOrFail( + (int) $lateFailureJob->id + )->updated_at === $lateFailureJobUpdatedAt + && (int) \App\Models\CloudAgentCompletionEvent::findOrFail( + $lateFailureEventId + )->applied_at === $lateFailureAppliedAt + && count($history) === $beforeLateFailureDispatch + 1, + 'Duplicate late create failure refunded, released, ACKed, or dispatched ' + . 'more than once.'); + + /** + * Create one paid order, cross the Agent boundary exactly once, and force + * the panel-side receipt deadline to the terminal unknown-effect state. + * Later recovery assertions can then focus on finance and evidence rather + * than reproducing dispatch bookkeeping by hand. + * + * @return array + */ + $makeTimedOutCreate = static function ( + string $suffix + ) use ( + $input, + $actor, + $mock, + $worker, + &$history, + $assert + ): array { + $order = CloudOrderService::purchase(array_replace($input, [ + 'name' => 'vm-create-recovery-' . $suffix, + 'idempotency_key' => 'test-create-recovery-' . $suffix, + ]), $actor); + $invoice = CloudInvoice::where('order_id', $order->id) + ->firstOrFail(); + CloudInvoiceService::payBalance( + $invoice, + $actor, + 'provision-create-recovery-' . $suffix + ); + $job = CloudJob::where('order_id', $order->id)->firstOrFail(); + $future = time() + 3600; + Capsule::table('cloud_jobs') + ->where('id', '<>', (int) $job->id) + ->update([ + 'available_at' => $future, + 'lease_until' => $future, + ]); + Capsule::table('cloud_jobs')->where('id', (int) $job->id) + ->update(['available_at' => 0, 'lease_until' => 0]); + $job->refresh(); + $identity = CloudOperationRecoveryIdentity::expected($job, false); + $mock->append(new Response(202, [], json_encode([ + 'operation_id' => $job->operation_id, + 'status' => 'reserved', + ...$identity, + 'replayed' => false, + 'retryable' => false, + ], JSON_THROW_ON_ERROR))); + $historyBefore = count($history); + $assert($worker->tick(), + 'Create recovery fixture did not dispatch: ' . $suffix); + $job->refresh(); + $assert((string) $job->phase === 'awaiting_receipt' + && (int) $job->dispatch_attempts === 1 + && count($history) === $historyBefore + 1, + 'Create recovery fixture did not persist one dispatch: ' + . $suffix); + Capsule::table('cloud_jobs')->where('id', (int) $job->id) + ->update([ + 'receipt_deadline_at' => time() - 1, + 'lease_token' => null, + 'lease_until' => 0, + ]); + $assert(CloudWorkerService::expireDueTaskForAdmin((int) $job->id) + === 1, 'Create recovery fixture did not expire: ' . $suffix); + $job->refresh(); + $order->refresh(); + $invoice->refresh(); + $reservation = CloudResourceReservation::where( + 'order_id', + $order->id + )->firstOrFail(); + $ip = CloudPublicIp::findOrFail((int) $reservation->public_ip_id); + $assert((string) $job->phase === 'terminal' + && (string) $job->outcome === 'failed' + && (string) $job->effect_resolution === 'unknown' + && (string) $order->status === 'failed' + && $order->refunded_at === null + && (string) $invoice->status === 'paid' + && (string) $reservation->state === 'uncertain' + && (string) $ip->state === 'uncertain', + 'Create recovery fixture did not retain its safety fence: ' + . $suffix); + return compact( + 'order', 'invoice', 'job', 'identity', 'reservation', 'ip', + 'historyBefore' + ) + [ + 'walletBefore' => (string) User::findOrFail(1)->money, + 'stockBefore' => (int) CloudOffer::findOrFail(1)->stock, + ]; + }; + $makeRecoveryReady = static function (CloudJob $job): void { + $future = time() + 3600; + Capsule::table('cloud_jobs') + ->where('id', '<>', (int) $job->id) + ->update([ + 'available_at' => $future, + 'lease_until' => $future, + ]); + Capsule::table('cloud_jobs')->where('id', (int) $job->id) + ->update([ + 'available_at' => 0, + 'lease_token' => null, + 'lease_until' => 0, + ]); + $job->refresh(); + }; + $inspectionFor = static function ( + CloudJob $job, + array $identity, + bool $fenced = false + ): array { + return [ + 'operation_id' => (string) $job->operation_id, + ...$identity, + 'operation_status' => 'failed', + 'checked_at' => time(), + 'outcome' => 'failed', + 'summary_code' => 'operation_failed', + 'error_code' => 'creation_absent', + 'effect_resolution' => 'not_applied', + 'fence_state' => $fenced ? 'active' : 'not_required', + 'fence_resolution_required' => $fenced, + 'effect_projection' => [ + 'creation_absent' => true, + 'managed_artifacts_absent' => true, + ], + 'abort_job' => ['available' => false], + 'cleanup' => ['available' => false], + ]; + }; + $assertCreateSettled = static function ( + array $case, + string $message + ) use ($assert, &$history): void { + $case['job']->refresh(); + $case['order']->refresh(); + $case['invoice']->refresh(); + $case['reservation']->refresh(); + $case['ip']->refresh(); + $assert((string) $case['job']->effect_resolution === 'not_applied' + && !(bool) $case['job']->finalization_incident + && $case['job']->lease_token === null + && (string) $case['order']->status === 'failed' + && $case['order']->refunded_at !== null + && (string) $case['invoice']->status === 'refunded' + && (string) $case['reservation']->state === 'released' + && (string) $case['ip']->state === 'available' + && (string) User::findOrFail(1)->money + === CloudMoney::decimal( + CloudMoney::cents($case['walletBefore']) + 999 + ) + && (int) CloudOffer::findOrFail(1)->stock + === $case['stockBefore'] + 1 + && count($history) === $case['historyBefore'] + 1, + $message); + }; + $failedCreateReceipt = static function ( + array $case, + string $eventId + ): array { + return [ + 'type' => 'completion', + 'id' => $eventId, + 'agent_id' => (string) $case['job']->host_id, + 'operation_id' => (string) $case['job']->operation_id, + ...$case['identity'], + 'status' => 'failed', + 'effect_may_have_applied' => false, + 'safe_to_resubmit' => false, + 'error_code' => 'creation_absent', + 'result' => [ + 'uuid' => $case['identity']['resource_id'], + 'creation_absent' => true, + 'managed_artifacts_absent' => true, + ], + ]; + }; + + // Immutable host absence evidence must wait for a temporarily inconsistent + // local invoice. Keeping the event received allows the exact same receipt + // to finish atomically after the PHP ledger is repaired. + $invoiceGuardCase = $makeTimedOutCreate('receipt-invoice-guard'); + $invoiceGuardCase['invoice']->purpose = 'renewal'; + $invoiceGuardCase['invoice']->save(); + $invoiceGuardReceipt = $failedCreateReceipt( + $invoiceGuardCase, + 'create-invoice-guard-failure-event-0001' + ); + $invoiceGuardPending = CloudOperationCompletionService::accept( + $invoiceGuardReceipt + ); + $invoiceGuardCase['job']->refresh(); + $invoiceGuardCase['order']->refresh(); + $invoiceGuardCase['reservation']->refresh(); + $invoiceGuardCase['ip']->refresh(); + $invoiceGuardEvent = \App\Models\CloudAgentCompletionEvent::findOrFail( + $invoiceGuardReceipt['id'] + ); + $assert($invoiceGuardPending === [ + 'accepted' => true, + 'disposition' => 'received', + ] + && (string) $invoiceGuardEvent->disposition === 'received' + && $invoiceGuardEvent->applied_at === null + && (string) $invoiceGuardCase['job']->effect_resolution === 'unknown' + && $invoiceGuardCase['order']->refunded_at === null + && (string) $invoiceGuardCase['reservation']->state === 'uncertain' + && (string) $invoiceGuardCase['ip']->state === 'uncertain' + && (string) User::findOrFail(1)->money + === $invoiceGuardCase['walletBefore'] + && (int) CloudOffer::findOrFail(1)->stock + === $invoiceGuardCase['stockBefore'], + 'Exact create failure bypassed an inconsistent local invoice: ' + . json_encode([ + 'accepted' => $invoiceGuardPending, + 'event_disposition' => $invoiceGuardEvent->disposition, + 'event_applied_at' => $invoiceGuardEvent->applied_at, + 'job_resolution' => + $invoiceGuardCase['job']->effect_resolution, + 'refunded_at' => $invoiceGuardCase['order']->refunded_at, + 'reservation' => $invoiceGuardCase['reservation']->state, + 'ip' => $invoiceGuardCase['ip']->state, + 'wallet_before' => $invoiceGuardCase['walletBefore'], + 'wallet_after' => User::findOrFail(1)->money, + 'stock_before' => $invoiceGuardCase['stockBefore'], + 'stock_after' => CloudOffer::findOrFail(1)->stock, + ], JSON_THROW_ON_ERROR)); + $invoiceGuardCase['invoice']->purpose = 'provision'; + $invoiceGuardCase['invoice']->save(); + $assert(CloudOperationCompletionService::accept($invoiceGuardReceipt) + === ['accepted' => true, 'disposition' => 'applied'], + 'Repaired local invoice did not resume the same create receipt.'); + $assertCreateSettled( + $invoiceGuardCase, + 'Repaired local invoice did not atomically settle create finance.' + ); + + // The specialised create finalizer must validate the complete dispatched + // envelope itself. Generic recovery policy deliberately does not own + // create, so neither a damaged attempt count nor a missing receipt + // deadline may reach the refund path. + foreach ([ + 'attempts' => 2, + 'receipt_deadline_at' => null, + ] as $damagedField => $damagedValue) { + $suffix = 'receipt-envelope-' . str_replace('_', '-', $damagedField); + $envelopeCase = $makeTimedOutCreate($suffix); + $eventId = 'create-envelope-' . str_replace('_', '-', $damagedField) + . '-event-0001'; + $envelopeReceipt = $failedCreateReceipt($envelopeCase, $eventId); + $originalValue = $envelopeCase['job']->{$damagedField}; + Capsule::table('cloud_jobs')->where( + 'id', + (int) $envelopeCase['job']->id + )->update([$damagedField => $damagedValue]); + $envelopePending = CloudOperationCompletionService::accept( + $envelopeReceipt + ); + $envelopeCase['job']->refresh(); + $envelopeCase['order']->refresh(); + $envelopeCase['invoice']->refresh(); + $envelopeCase['reservation']->refresh(); + $envelopeCase['ip']->refresh(); + $envelopeEvent = \App\Models\CloudAgentCompletionEvent::findOrFail( + $eventId + ); + $assert($envelopePending === [ + 'accepted' => true, + 'disposition' => 'received', + ] + && (string) $envelopeEvent->disposition === 'received' + && $envelopeEvent->applied_at === null + && (string) $envelopeCase['job']->effect_resolution === 'unknown' + && $envelopeCase['order']->refunded_at === null + && (string) $envelopeCase['invoice']->status === 'paid' + && (string) $envelopeCase['reservation']->state === 'uncertain' + && (string) $envelopeCase['ip']->state === 'uncertain' + && (string) User::findOrFail(1)->money + === $envelopeCase['walletBefore'] + && (int) CloudOffer::findOrFail(1)->stock + === $envelopeCase['stockBefore'], + 'Damaged create dispatch envelope released finance: ' + . $damagedField); + Capsule::table('cloud_jobs')->where( + 'id', + (int) $envelopeCase['job']->id + )->update([$damagedField => $originalValue]); + $assert(CloudOperationCompletionService::accept($envelopeReceipt) + === ['accepted' => true, 'disposition' => 'applied'], + 'Repaired create dispatch envelope did not resume receipt: ' + . $damagedField); + $assertCreateSettled( + $envelopeCase, + 'Repaired dispatch envelope did not settle create finance: ' + . $damagedField + ); + } + + // A fresh read-only Journal failure with complete absence projection can + // finish a timed-out create even when no completion event ever arrived. + $inspectionCase = $makeTimedOutCreate('inspection-no-receipt'); + $makeRecoveryReady($inspectionCase['job']); + $inspectionClient = new FinancialCreateRecoveryAgentClient(); + $inspectionClient->responses[] = $inspectionFor( + $inspectionCase['job'], + $inspectionCase['identity'] + ); + CloudAgentSettingsService::$hostId = + (int) $inspectionCase['job']->host_id; + CloudAgentSettingsService::$client = $inspectionClient; + $inspectionRecovery = new CloudAutomaticOperationRecoveryService(); + $assert($inspectionRecovery->tick(), + 'No-receipt create was not claimed for read-only recovery.'); + $assert(count($inspectionClient->requests) === 1 + && $inspectionClient->requests[0]['method'] === 'GET' + && str_ends_with( + $inspectionClient->requests[0]['target'], + '/inspect' + ) + && $inspectionClient->requests[0]['payload'] === null, + 'No-receipt create recovery used something other than one read-only ' + . 'inspection.'); + $assertCreateSettled( + $inspectionCase, + 'Read-only create absence evidence did not atomically settle finance.' + ); + + // Once inspection has settled the ledger, the first later failure receipt + // is an idempotent ACK of the late delivery. It must not re-enter the received queue, + // rewrite the job incident, or credit/release anything twice. + $inspectionCase['job']->refresh(); + $inspectionJobUpdatedAt = (int) $inspectionCase['job']->updated_at; + $inspectionWalletAfter = (string) User::findOrFail(1)->money; + $inspectionStockAfter = (int) CloudOffer::findOrFail(1)->stock; + $inspectionLateReceipt = [ + 'type' => 'completion', + 'id' => 'create-inspection-late-failure-event-0001', + 'agent_id' => (string) $inspectionCase['job']->host_id, + 'operation_id' => (string) $inspectionCase['job']->operation_id, + ...$inspectionCase['identity'], + 'status' => 'failed', + 'effect_may_have_applied' => false, + 'safe_to_resubmit' => false, + 'error_code' => 'creation_absent', + 'result' => [ + 'uuid' => $inspectionCase['identity']['resource_id'], + 'creation_absent' => true, + 'managed_artifacts_absent' => true, + ], + ]; + $inspectionLateAccepted = CloudOperationCompletionService::accept( + $inspectionLateReceipt + ); + $inspectionLateDuplicate = CloudOperationCompletionService::accept( + $inspectionLateReceipt + ); + $inspectionCase['job']->refresh(); + $inspectionLateEvent = \App\Models\CloudAgentCompletionEvent::findOrFail( + $inspectionLateReceipt['id'] + ); + $assert($inspectionLateAccepted === [ + 'accepted' => true, + 'disposition' => 'applied', + ] + && $inspectionLateDuplicate === $inspectionLateAccepted + && (string) $inspectionLateEvent->disposition === 'applied' + && $inspectionLateEvent->applied_at !== null + && (int) $inspectionCase['job']->updated_at + === $inspectionJobUpdatedAt + && !(bool) $inspectionCase['job']->finalization_incident + && (string) User::findOrFail(1)->money === $inspectionWalletAfter + && (int) CloudOffer::findOrFail(1)->stock === $inspectionStockAfter, + 'A failure receipt arriving after inspection settlement was requeued ' + . 'or changed the financial ledger: ' . json_encode([ + 'first' => $inspectionLateAccepted, + 'duplicate' => $inspectionLateDuplicate, + 'event_disposition' => $inspectionLateEvent->disposition, + 'event_applied_at' => $inspectionLateEvent->applied_at, + 'job_updated_at_before' => $inspectionJobUpdatedAt, + 'job_updated_at_after' => $inspectionCase['job']->updated_at, + 'job_incident' => $inspectionCase['job']->finalization_incident, + 'wallet_before' => $inspectionWalletAfter, + 'wallet_after' => User::findOrFail(1)->money, + 'stock_before' => $inspectionStockAfter, + 'stock_after' => CloudOffer::findOrFail(1)->stock, + ], JSON_THROW_ON_ERROR)); + + // An incomplete failed receipt is retained as a late audit event, then a + // fresh read-only inspection supplies the missing managed-artifact proof. + $incompleteCase = $makeTimedOutCreate('incomplete-receipt'); + $incompleteReceipt = [ + 'type' => 'completion', + 'id' => 'create-incomplete-failure-event-0001', + 'agent_id' => (string) $incompleteCase['job']->host_id, + 'operation_id' => (string) $incompleteCase['job']->operation_id, + ...$incompleteCase['identity'], + 'status' => 'failed', + 'effect_may_have_applied' => false, + 'safe_to_resubmit' => false, + 'error_code' => 'creation_absent', + 'result' => [ + 'uuid' => $incompleteCase['identity']['resource_id'], + 'creation_absent' => true, + ], + ]; + $assert(CloudOperationCompletionService::accept($incompleteReceipt) + === ['accepted' => true, 'disposition' => 'late'], + 'Incomplete create absence receipt was not ACKed as late.'); + $makeRecoveryReady($incompleteCase['job']); + $incompleteClient = new FinancialCreateRecoveryAgentClient(); + $incompleteClient->responses[] = $inspectionFor( + $incompleteCase['job'], + $incompleteCase['identity'] + ); + CloudAgentSettingsService::$client = $incompleteClient; + $assert((new CloudAutomaticOperationRecoveryService())->tick(), + 'Incomplete receipt create did not enter fresh inspection recovery.'); + $assertCreateSettled( + $incompleteCase, + 'Fresh inspection did not finish an incomplete failed receipt create.' + ); + + // If PHP fails after durably preparing a no-fence conclusion, a later + // tick must finish locally even while the Agent is offline. + $noFenceRetry = $makeTimedOutCreate('nofence-db-retry'); + $noFenceRetry['invoice']->purpose = 'renewal'; + $noFenceRetry['invoice']->save(); + $makeRecoveryReady($noFenceRetry['job']); + $noFenceClient = new FinancialCreateRecoveryAgentClient(); + $noFenceClient->responses[] = $inspectionFor( + $noFenceRetry['job'], + $noFenceRetry['identity'] + ); + CloudAgentSettingsService::$client = $noFenceClient; + $assert((new CloudAutomaticOperationRecoveryService())->tick(), + 'No-fence DB-failure create was not claimed.'); + $noFenceRetry['job']->refresh(); + $noFenceRetry['order']->refresh(); + $noFenceRetry['invoice']->refresh(); + $assert(preg_match( + '/^prepared:create_not_applied:v1:[a-f0-9]{64}$/D', + (string) $noFenceRetry['job']->effect_resolution_reason + ) === 1 + && (string) $noFenceRetry['job']->effect_resolution === 'unknown' + && $noFenceRetry['order']->refunded_at === null + && (string) $noFenceRetry['invoice']->status === 'paid' + && count($noFenceClient->requests) === 1, + 'No-fence financial failure did not retain its durable prepare.'); + $noFenceRetry['invoice']->purpose = 'provision'; + $noFenceRetry['invoice']->save(); + $makeRecoveryReady($noFenceRetry['job']); + // Deliberately leave the response queue empty: any Agent call now fails. + $assert((new CloudAutomaticOperationRecoveryService())->tick(), + 'Prepared no-fence create was not retried locally.'); + $assert(count($noFenceClient->requests) === 1, + 'Prepared no-fence retry contacted an offline Agent.'); + $assertCreateSettled( + $noFenceRetry, + 'Prepared no-fence create did not settle after the DB fault cleared.' + ); + + // If the Agent fence write committed but the same PHP transaction later + // failed, recovery reads that immutable verdict once and never POSTs it a + // second time. + $fenceRetry = $makeTimedOutCreate('fence-db-retry'); + $fenceRetry['invoice']->purpose = 'renewal'; + $fenceRetry['invoice']->save(); + $makeRecoveryReady($fenceRetry['job']); + $fenceClient = new FinancialCreateRecoveryAgentClient(); + $fenceClient->responses[] = $inspectionFor( + $fenceRetry['job'], + $fenceRetry['identity'], + true + ); + $fenceClient->responses[] = new CloudAgentError( + 'no committed fence', + 404, + 'not_found', + false, + 404 + ); + $committedFence = null; + $fenceClient->responses[] = static function ( + array $request + ) use (&$committedFence, $fenceRetry): array { + $payload = $request['payload']; + $committedFence = [ + 'operation_id' => (string) $fenceRetry['job']->operation_id, + 'resolution_id' => (string) $payload['resolution_id'], + 'resolution' => (string) $payload['resolution'], + 'inspection_checked_at' => + (int) $payload['inspection_checked_at'], + 'reason_sha256' => (string) $payload['reason_sha256'], + 'resolved_at' => time(), + 'resolved_by' => 0, + 'replayed' => false, + ]; + return $committedFence; + }; + CloudAgentSettingsService::$client = $fenceClient; + $assert((new CloudAutomaticOperationRecoveryService())->tick(), + 'Active-fence DB-failure create was not claimed.'); + $fenceRetry['job']->refresh(); + $assert(is_array($committedFence) + && count($fenceClient->requests) === 3 + && $fenceClient->requests[0]['method'] === 'GET' + && str_ends_with($fenceClient->requests[0]['target'], '/inspect') + && $fenceClient->requests[1]['method'] === 'GET' + && str_ends_with( + $fenceClient->requests[1]['target'], + '/fence-resolution' + ) + && $fenceClient->requests[2]['method'] === 'POST' + && str_ends_with( + $fenceClient->requests[2]['target'], + '/resolve-fence' + ) + && preg_match( + '/^prepared:create_not_applied:v1:[a-f0-9]{64}$/D', + (string) $fenceRetry['job']->effect_resolution_reason + ) === 1 + && (string) $fenceRetry['job']->effect_resolution === 'unknown', + 'Fence commit plus PHP failure did not preserve resumable evidence.'); + $fenceRetry['invoice']->purpose = 'provision'; + $fenceRetry['invoice']->save(); + $makeRecoveryReady($fenceRetry['job']); + $fenceClient->responses[] = $committedFence; + $assert((new CloudAutomaticOperationRecoveryService())->tick(), + 'Prepared active-fence create was not retried.'); + $assert(count($fenceClient->requests) === 4 + && $fenceClient->requests[3]['method'] === 'GET' + && str_ends_with( + $fenceClient->requests[3]['target'], + '/fence-resolution' + ), + 'Prepared active-fence retry wrote or re-inspected Agent state.'); + $assertCreateSettled( + $fenceRetry, + 'Prepared active-fence create did not settle after DB recovery.' + ); + + // A fence write can fail before the Agent commits it. The next tick first + // proves that no prior verdict exists, then obtains a fresh inspection. + // checked_at is freshness metadata, not part of the durable conclusion: + // the same facts must produce the same marker and fence reason. + $stableCase = $makeTimedOutCreate('stable-inspection-marker'); + $makeRecoveryReady($stableCase['job']); + $stableClient = new FinancialCreateRecoveryAgentClient(); + $stableFirstInspection = $inspectionFor( + $stableCase['job'], + $stableCase['identity'], + true + ); + $stableClient->responses = [ + $stableFirstInspection, + new CloudAgentError('no committed fence', 404, 'not_found', false, 404), + new CloudAgentError('fence write unavailable', 503, 'unavailable', true, 503), + new CloudAgentError('no committed fence', 404, 'not_found', false, 404), + ]; + CloudAgentSettingsService::$client = $stableClient; + $assert((new CloudAutomaticOperationRecoveryService())->tick(), + 'Stable-marker first recovery attempt was not claimed.'); + $stableCase['job']->refresh(); + $stableMarker = (string) $stableCase['job']->effect_resolution_reason; + $assert(preg_match( + '/^prepared:create_not_applied:v1:[a-f0-9]{64}$/D', + $stableMarker + ) === 1 + && (string) $stableCase['job']->effect_resolution === 'unknown' + && count($stableClient->requests) === 4, + 'Failed fence write did not retain a stable prepared create marker.'); + $makeRecoveryReady($stableCase['job']); + $stableSecondInspection = $stableFirstInspection; + $stableSecondInspection['checked_at'] = + (int) $stableFirstInspection['checked_at'] + 1; + $stableCommittedFence = null; + $stableClient->responses = [ + new CloudAgentError('no committed fence', 404, 'not_found', false, 404), + $stableSecondInspection, + new CloudAgentError('no committed fence', 404, 'not_found', false, 404), + static function (array $request) use ( + &$stableCommittedFence, + $stableCase + ): array { + $payload = $request['payload']; + $stableCommittedFence = [ + 'operation_id' => (string) $stableCase['job']->operation_id, + 'resolution_id' => (string) $payload['resolution_id'], + 'resolution' => (string) $payload['resolution'], + 'inspection_checked_at' => + (int) $payload['inspection_checked_at'], + 'reason_sha256' => (string) $payload['reason_sha256'], + 'resolved_at' => time(), + 'resolved_by' => 0, + 'replayed' => false, + ]; + return $stableCommittedFence; + }, + ]; + $assert((new CloudAutomaticOperationRecoveryService())->tick(), + 'Stable-marker create did not resume from fresh inspection.'); + $stableExpectedReason = + '自动恢复已确认原开通操作未创建虚拟机或面板管理资源;operation_id=' + . (string) $stableCase['job']->operation_id . ';evidence=' + . substr($stableMarker, -64); + $assert(is_array($stableCommittedFence) + && count($stableClient->requests) === 8 + && $stableClient->requests[4]['method'] === 'GET' + && str_ends_with( + $stableClient->requests[4]['target'], + '/fence-resolution' + ) + && $stableClient->requests[5]['method'] === 'GET' + && str_ends_with($stableClient->requests[5]['target'], '/inspect') + && $stableClient->requests[7]['method'] === 'POST' + && (string) $stableClient->requests[7]['payload']['reason_sha256'] + === hash('sha256', $stableExpectedReason), + 'Fresh equivalent inspection changed the prepared fence reason or ' + . 'replayed the original create.'); + $assertCreateSettled( + $stableCase, + 'Stable-marker retry did not finish the create financial ledger.' + ); + + // Partial/residual host state and an identity mismatch are both + // contradictions, not refund evidence. They remain fenced and retryable. + $conflictCase = $makeTimedOutCreate('inspection-conflicts'); + $makeRecoveryReady($conflictCase['job']); + $conflictClient = new FinancialCreateRecoveryAgentClient(); + $partialInspection = $inspectionFor( + $conflictCase['job'], + $conflictCase['identity'], + true + ); + $partialInspection['outcome'] = 'uncertain'; + $partialInspection['summary_code'] = 'evidence_incomplete'; + $partialInspection['effect_resolution'] = 'partial'; + $partialInspection['error_code'] = 'cleanup_required'; + $partialInspection['cleanup'] = [ + 'available' => true, + 'plan_id' => str_repeat('a', 64), + 'steps' => ['delete_managed_volumes'], + ]; + unset($partialInspection['effect_projection']); + $conflictClient->responses[] = $partialInspection; + CloudAgentSettingsService::$client = $conflictClient; + $assert((new CloudAutomaticOperationRecoveryService())->tick(), + 'Partial create evidence was not safely deferred.'); + $conflictCase['job']->refresh(); + $conflictCase['order']->refresh(); + $conflictCase['invoice']->refresh(); + $assert((string) $conflictCase['job']->effect_resolution === 'unknown' + && (bool) $conflictCase['job']->finalization_incident + && $conflictCase['order']->refunded_at === null + && (string) $conflictCase['invoice']->status === 'paid' + && (string) User::findOrFail(1)->money + === $conflictCase['walletBefore'] + && (int) CloudOffer::findOrFail(1)->stock + === $conflictCase['stockBefore'], + 'Partial create evidence released a financial/resource fence.'); + $makeRecoveryReady($conflictCase['job']); + $wrongIdentity = $inspectionFor( + $conflictCase['job'], + $conflictCase['identity'] + ); + $wrongIdentity['resource_id'] = 'wrong-create-resource'; + $conflictClient->responses[] = $wrongIdentity; + $assert((new CloudAutomaticOperationRecoveryService())->tick(), + 'Create identity contradiction was not safely deferred.'); + $conflictCase['job']->refresh(); + $assert((string) $conflictCase['job']->effect_resolution === 'unknown' + && (bool) $conflictCase['job']->finalization_incident + && $conflictCase['order']->fresh()->refunded_at === null + && (string) User::findOrFail(1)->money + === $conflictCase['walletBefore'] + && (int) CloudOffer::findOrFail(1)->stock + === $conflictCase['stockBefore'], + 'Create identity contradiction released a financial/resource fence.'); + + // A caller cannot reuse an automatic claim after its lease has expired. + $leaseToken = bin2hex(random_bytes(16)); + Capsule::table('cloud_jobs')->where('id', (int) $conflictCase['job']->id) + ->update([ + 'available_at' => 0, + 'lease_token' => $leaseToken, + 'lease_until' => time() + 180, + ]); + $staleClaim = CloudJob::findOrFail((int) $conflictCase['job']->id); + Capsule::table('cloud_jobs')->where('id', (int) $conflictCase['job']->id) + ->update(['lease_until' => time() - 1]); + $staleRequestCount = count($conflictClient->requests); + $staleRejected = false; + try { + CloudCreateFailureRecoveryService::recover($staleClaim); + } catch (CloudError $error) { + $staleRejected = true; + } + $assert($staleRejected + && count($conflictClient->requests) === $staleRequestCount + && (string) CloudJob::findOrFail( + (int) $conflictCase['job']->id + )->effect_resolution === 'unknown' + && $conflictCase['order']->fresh()->refunded_at === null, + 'Expired create recovery lease contacted Agent or released finance.'); + echo "Cloud SQLite purchase, provisioning, refund, ambiguity, quota and tenant checks passed (crypto double only).\n"; }