diff --git a/web/app/Services/Cloud/CloudAutomaticOperationRecoveryService.php b/web/app/Services/Cloud/CloudAutomaticOperationRecoveryService.php index 8bb8627d..845bf42f 100644 --- a/web/app/Services/Cloud/CloudAutomaticOperationRecoveryService.php +++ b/web/app/Services/Cloud/CloudAutomaticOperationRecoveryService.php @@ -295,15 +295,41 @@ final class CloudAutomaticOperationRecoveryService ->first(); } if ($job === null && !$receivedOnly) { - // A valid zero counter proves that no Agent mutation was - // sent. Retry only PHP's kind-specific not-applied ledger - // finalizer; never inspect or contact the host. + // A zero counter or the worker's exact persisted + // dispatch_state=not_dispatched proof establishes that no + // Agent mutation was sent. Retry only PHP's kind-specific + // not-applied ledger finalizer; never inspect or contact + // the host. $job = CloudJob::query() ->where('execution_mode', 'agent') ->where('phase', 'terminal') ->where('outcome', 'failed') ->where('status', 'failed') - ->where('dispatch_attempts', 0) + ->where(static function ($proof): void { + $proof->where('dispatch_attempts', 0) + ->orWhere(static function ($transport): void { + $transport->where('dispatch_attempts', 1) + ->where('attempts', 1) + ->where('dispatch_started_at', '>', 0) + ->where('dispatched_at', '>', 0) + ->where('receipt_deadline_at', '>', 0) + ->whereNull('agent_accepted_at') + ->whereNull('completion_event_id') + ->where('effect_may_have_applied', 0) + ->where('safe_to_resubmit', 0) + ->where( + 'effect_resolution', + 'not_applied' + )->where( + 'effect_resolution_reason', + CloudDispatchLedger:: + TRANSPORT_NOT_DISPATCHED_REASON + )->where( + 'error_code', + 'local_finalization_failed' + ); + }); + }) ->where('finalization_incident', 1) ->where(static function ($incident): void { $incident->whereNull('finalization_error_code') @@ -317,10 +343,10 @@ final class CloudAutomaticOperationRecoveryService 'kind', CloudOperationRecoveryPolicy::kinds() )->orWhere(static function ($create): void { - // Generation zero is definitive no-effect - // evidence for create too. Its financial and - // placement ledgers use the specialised atomic - // local finalizer, never the Agent mutation. + // Proven local no-effect is authoritative for + // create too. Its financial and placement + // ledgers use the specialised atomic local + // finalizer, never the Agent mutation. $create->where('kind', 'create') ->whereNotNull('order_id'); }); diff --git a/web/app/Services/Cloud/CloudCreateFailureFinalizer.php b/web/app/Services/Cloud/CloudCreateFailureFinalizer.php index 4e228bbb..bbff412d 100644 --- a/web/app/Services/Cloud/CloudCreateFailureFinalizer.php +++ b/web/app/Services/Cloud/CloudCreateFailureFinalizer.php @@ -315,7 +315,8 @@ final class CloudCreateFailureFinalizer || (string) $job->phase !== 'terminal' || (string) $job->outcome !== 'failed' || (string) $job->status !== 'failed' - || !CloudDispatchLedger::isNeverDispatched($job) + || !CloudDispatchLedger:: + isLocallyFinalizableNotDispatched($job) || !(bool) $job->finalization_incident || !in_array( (string) $job->effect_resolution, @@ -339,7 +340,7 @@ final class CloudCreateFailureFinalizer ); } CloudOperationCompletionService:: - assertNeverDispatchedRecoveryState($job); + assertLocalNotDispatchedRecoveryState($job); try { $payload = CloudSecrets::decrypt( diff --git a/web/app/Services/Cloud/CloudDispatchLedger.php b/web/app/Services/Cloud/CloudDispatchLedger.php index 274f11ff..662a98a6 100644 --- a/web/app/Services/Cloud/CloudDispatchLedger.php +++ b/web/app/Services/Cloud/CloudDispatchLedger.php @@ -5,14 +5,18 @@ namespace App\Services\Cloud; /** * Fail-closed interpretation of the immutable single-dispatch counter. * - * Zero is the only proof that no Agent mutation crossed the dispatch boundary. - * One is the only valid dispatched generation. Any other value is corruption: + * Zero is the only counter-level proof that no Agent request was attempted. + * One is the only valid attempted generation. A generation-one request may + * still carry a separate, exact Agent transport proof that its durable + * operation journal was never entered. Any other counter value is corruption: * callers must retain the resource fence and must not consume recovery proof * until an operator repairs the ledger. */ final class CloudDispatchLedger { public const CONFLICT_CODE = 'recovery_dispatch_ledger_conflict'; + public const TRANSPORT_NOT_DISPATCHED_REASON = + 'worker_transport_not_dispatched'; private const REPAIR_INCIDENT_CODES = [ self::CONFLICT_CODE, 'recovery_dispatch_repair_completion_receipt_missing', @@ -51,6 +55,53 @@ final class CloudDispatchLedger return self::attempts($job) === 1; } + /** + * Exact durable proof created only from the Agent's strict + * dispatch_state=not_dispatched envelope after the dispatch CAS. The + * generation remains one forever; this marker authorizes only the PHP + * business finalizer and never a replay of the original Agent mutation. + */ + public static function hasTransportNotDispatchedProof(object $job): bool + { + return self::isDispatched($job) + && (string) ($job->execution_mode ?? '') === 'agent' + && (string) ($job->phase ?? '') === 'terminal' + && (string) ($job->outcome ?? '') === 'failed' + && (string) ($job->status ?? '') === 'failed' + && (string) ($job->attempts ?? '') === '1' + && (int) ($job->dispatch_started_at ?? 0) > 0 + && (int) ($job->dispatched_at ?? 0) > 0 + && ($job->agent_accepted_at ?? null) === null + && (int) ($job->receipt_deadline_at ?? 0) > 0 + && ($job->completion_event_id ?? null) === null + && in_array( + $job->effect_may_have_applied ?? null, + [0, '0', false], + true + ) + && in_array( + $job->safe_to_resubmit ?? null, + [0, '0', false], + true + ) + && (string) ($job->effect_resolution ?? '') === 'not_applied' + && hash_equals( + self::TRANSPORT_NOT_DISPATCHED_REASON, + (string) ($job->effect_resolution_reason ?? '') + ) + && (string) ($job->error_code ?? '') + === 'local_finalization_failed' + && (bool) ($job->finalization_incident ?? false); + } + + /** Proof accepted by the PHP-only finalizer; never authorizes Agent I/O. */ + public static function isLocallyFinalizableNotDispatched( + object $job + ): bool { + return self::isNeverDispatched($job) + || self::hasTransportNotDispatchedProof($job); + } + public static function isInvalid(object $job): bool { return !in_array(self::attempts($job), [0, 1], true); diff --git a/web/app/Services/Cloud/CloudJobService.php b/web/app/Services/Cloud/CloudJobService.php index 85049efc..712b9658 100644 --- a/web/app/Services/Cloud/CloudJobService.php +++ b/web/app/Services/Cloud/CloudJobService.php @@ -3698,7 +3698,8 @@ final class CloudJobService && (string) ($job->phase ?? '') === 'terminal' && (string) ($job->outcome ?? '') === 'failed' && (string) ($job->status ?? '') === 'failed' - && CloudDispatchLedger::isNeverDispatched($job) + && CloudDispatchLedger:: + isLocallyFinalizableNotDispatched($job) && (bool) ($job->finalization_incident ?? false) && !CloudDispatchLedger::hasRepairIncident($job) && CloudOperationRecoveryPolicy::supports( diff --git a/web/app/Services/Cloud/CloudOperationCompletionService.php b/web/app/Services/Cloud/CloudOperationCompletionService.php index 179040d1..77f48bb1 100644 --- a/web/app/Services/Cloud/CloudOperationCompletionService.php +++ b/web/app/Services/Cloud/CloudOperationCompletionService.php @@ -1022,6 +1022,48 @@ final class CloudOperationCompletionService ->exists(); } + /** + * Accept either generation zero or the worker's exact persisted transport + * proof that a generation-one request never entered the Agent journal. + * This authorizes only a PHP finalizer; any completion evidence keeps the + * resource fenced and requires operator review. + */ + public static function assertLocalNotDispatchedRecoveryState( + CloudJob $job + ): void { + if (CloudDispatchLedger::isNeverDispatched($job)) { + self::assertNeverDispatchedRecoveryState($job); + return; + } + if (!CloudDispatchLedger::hasTransportNotDispatchedProof($job)) { + throw new CloudRecoveryDispatchLedgerConflict( + '任务缺少可信的未派发证明', + 409 + ); + } + if (CloudJobService::hasAgentRecoveryEnvelopeConflict($job)) { + throw new CloudRecoveryInstanceStateConflict( + '未派发传输证明与 Agent 请求账本不一致', + 409 + ); + } + $hostId = filter_var( + $job->host_id, + FILTER_VALIDATE_INT, + ['options' => ['min_range' => 1]] + ); + $operationId = $job->operation_id; + if ($hostId === false || !is_string($operationId) + || preg_match('/^[a-f0-9]{32}$/D', $operationId) !== 1 + || CloudAgentCompletionEvent::where('host_id', (int) $hostId) + ->where('operation_id', $operationId)->exists()) { + throw new CloudRecoveryInstanceStateConflict( + '未派发传输证明存在冲突的 Agent 完成证据', + 409 + ); + } + } + /** * Generation zero may run only a PHP finalizer. Any durable Agent boundary * or completion evidence means that zero is not trustworthy and must stay diff --git a/web/app/Services/Cloud/CloudWorkerService.php b/web/app/Services/Cloud/CloudWorkerService.php index 99b1d543..73f3118d 100644 --- a/web/app/Services/Cloud/CloudWorkerService.php +++ b/web/app/Services/Cloud/CloudWorkerService.php @@ -470,7 +470,7 @@ final class CloudWorkerService } catch (\Throwable $finishError) { self::forceTerminalFailure((int) $job->id, (string) $job->lease_token, !$safe, - $safe, 'local_finalization_failed'); + $safe, 'local_finalization_failed', $safe); } } return true; @@ -1002,7 +1002,8 @@ final class CloudWorkerService } private static function forceTerminalFailure(int $jobId, string $token, - bool $effect, bool $safe, string $code): void + bool $effect, bool $safe, string $code, + bool $confirmedNotDispatched = false): void { $claimed = CloudJob::where('id', $jobId) ->where('lease_token', $token)->where('phase', '<>', 'terminal') @@ -1181,17 +1182,26 @@ final class CloudWorkerService // explicit durable incident for operators. // Registered policy kinds and an order-backed create have PHP-only // retry paths that can safely publish a known not-applied effect for - // generation zero. Aggregate/other special finalizers retain the - // conservative unknown effect if their atomic fallback fails. + // generation zero or for one exact transport-level + // dispatch_state=not_dispatched proof. The immutable counter remains + // one in the latter case. Aggregate/other special finalizers retain + // the conservative unknown effect if their atomic fallback fails. $neverDispatched = $recoverableFinalizer && CloudDispatchLedger::isNeverDispatched($claimed); - CloudJob::where('id', $jobId)->where('lease_token', $token) - ->where('phase', '<>', 'terminal')->update([ + $transportNotDispatched = $recoverableFinalizer + && $confirmedNotDispatched + && !$effect + && $safe + && CloudDispatchLedger::isDispatched($claimed) + && ($claimed->agent_accepted_at ?? null) === null + && ($claimed->completion_event_id ?? null) === null; + $notDispatched = $neverDispatched || $transportNotDispatched; + $terminalUpdate = [ 'phase' => 'terminal', 'outcome' => 'failed', 'status' => 'failed', 'terminal_at' => time(), - 'effect_may_have_applied' => $neverDispatched ? 0 : 1, + 'effect_may_have_applied' => $notDispatched ? 0 : 1, 'safe_to_resubmit' => 0, - 'effect_resolution' => $neverDispatched + 'effect_resolution' => $notDispatched ? 'not_applied' : 'unknown', 'error_code' => $dispatchConflict ? CloudDispatchLedger::CONFLICT_CODE @@ -1210,7 +1220,13 @@ final class CloudWorkerService 'available_at' => ($dispatchConflict || $recoverableFinalizer) ? time() + 300 : 0, 'updated_at' => time(), - ]); + ]; + if ($transportNotDispatched) { + $terminalUpdate['effect_resolution_reason'] = + CloudDispatchLedger::TRANSPORT_NOT_DISPATCHED_REASON; + } + CloudJob::where('id', $jobId)->where('lease_token', $token) + ->where('phase', '<>', 'terminal')->update($terminalUpdate); } private static function localFinalizationIncidentCode( @@ -1927,7 +1943,7 @@ final class CloudWorkerService } /** - * Retry only the PHP ledger finalizer for a generation proven never sent. + * Retry only the PHP ledger finalizer for an operation proven not sent. * The terminal task receipt remains immutable and no Agent request occurs. */ public function retryUndispatchedFinalization( @@ -2124,7 +2140,8 @@ final class CloudWorkerService || (string) $job->phase !== 'terminal' || (string) $job->outcome !== 'failed' || (string) $job->status !== 'failed' - || !CloudDispatchLedger::isNeverDispatched($job) + || !CloudDispatchLedger:: + isLocallyFinalizableNotDispatched($job) || !(bool) $job->finalization_incident || !in_array((string) $job->effect_resolution, ['unknown', 'not_applied'], true) @@ -2146,7 +2163,7 @@ final class CloudWorkerService ); } CloudOperationCompletionService:: - assertNeverDispatchedRecoveryState($job); + assertLocalNotDispatchedRecoveryState($job); } $recoveredCreateEvent = null; if ($recoveredCreateEventId !== null) { diff --git a/web/tests/CloudUndispatchedFinalizerRecoveryTest.php b/web/tests/CloudUndispatchedFinalizerRecoveryTest.php index ceaf4025..35d3e8c3 100644 --- a/web/tests/CloudUndispatchedFinalizerRecoveryTest.php +++ b/web/tests/CloudUndispatchedFinalizerRecoveryTest.php @@ -66,9 +66,12 @@ namespace { require dirname(__DIR__) . '/vendor/autoload.php'; use App\Models\CloudJob; + use App\Models\CloudRescueTemplateImport; use App\Services\Cloud\CloudAutomaticOperationRecoveryService; use App\Services\Cloud\CloudAgentSettingsService; + use App\Services\Cloud\CloudDispatchLedger; use App\Services\Cloud\CloudJobService; + use App\Services\Cloud\CloudRescueTemplateImportService; use App\Services\Cloud\CloudSecrets; use App\Services\Cloud\CloudWorkerService; use Illuminate\Database\Capsule\Manager as Capsule; @@ -180,6 +183,21 @@ namespace { $table->integer('active_job_id')->nullable(); $table->string('state')->default('active'); }); + $schema->create('cloud_rescue_template_imports', + static function (Blueprint $table): void { + $table->string('operation_id')->primary(); + $table->integer('host_id'); + $table->string('template_id', 32); + $table->integer('job_id'); + $table->string('import_type'); + $table->string('name'); + $table->integer('size_bytes')->default(0); + $table->string('sha256', 64)->default(''); + $table->string('state'); + $table->string('error_code')->default(''); + $table->integer('created_at'); + $table->integer('updated_at'); + }); $schema->create('cloud_template_replicas', static function (Blueprint $table): void { $table->integer('host_id'); @@ -777,5 +795,164 @@ namespace { && CloudAgentSettingsService::$requests === 0, 'Undispatched create recovery repeated refund/release or contacted Agent.'); + // The dispatch CAS may already be generation one when an exact Agent + // validation response proves that its operation journal was never entered. + // If PHP's catalog finalizer then fails, preserve that proof without + // rewriting the immutable counter, recover only the local ledger, and + // allow a new same-name import generation afterward. + $transportOperationId = str_repeat('ba', 16); + $transportTemplateId = str_repeat('c', 32); + $transportName = 'transport-not-dispatched-rescue'; + $transportBytes = 2 * 1024 * 1024; + $transportPayload = [ + 'host_id' => 7, + 'import_type' => 'upload', + 'name' => $transportName, + 'source' => '', + 'sha256' => '', + 'source_size_bytes' => $transportBytes, + 'can_manage' => true, + 'rescue_template_id' => $transportTemplateId, + 'method' => 'POST', + 'request_target' => '/v1/rescue-templates', + 'body' => [ + 'operation_id' => $transportOperationId, + 'id' => $transportTemplateId, + 'name' => $transportName, + 'source_url' => 'https://panel.example.test/staging/rescue.iso', + 'source_size_bytes' => $transportBytes, + 'sha256' => '', + ], + 'owner_id' => 1, + ]; + $transportEnvelope = CloudSecrets::encrypt( + $transportPayload, + 'job-payload' + ); + Capsule::table('cloud_jobs')->insert([ + 'id' => 701, + 'operation_id' => $transportOperationId, + 'request_idempotency_key' => $transportOperationId, + 'request_fingerprint' => hash('sha256', $transportOperationId), + 'parent_job_id' => null, + 'order_id' => null, + 'invoice_id' => null, + 'instance_id' => null, + 'snapshot_id' => null, + 'user_id' => 1, + 'host_id' => 7, + 'kind' => 'rescue_template_import', + 'execution_mode' => 'agent', + 'phase' => 'dispatching', + 'outcome' => null, + 'status' => 'running', + 'sealed_payload' => $transportEnvelope, + 'attempts' => 1, + 'dispatch_attempts' => 1, + 'dispatch_deadline_at' => $now + 60, + 'dispatch_started_at' => $now, + 'agent_accepted_at' => null, + 'receipt_deadline_at' => $now + 300, + 'terminal_at' => null, + 'effect_may_have_applied' => null, + 'safe_to_resubmit' => null, + 'effect_resolution' => 'unknown', + 'effect_resolved_at' => null, + 'effect_resolved_by' => null, + 'effect_resolution_reason' => '', + 'completion_event_id' => null, + 'read_at' => null, + 'dispatched_at' => $now, + 'lease_token' => str_repeat('7', 32), + 'lease_until' => $now + 180, + 'available_at' => 0, + 'error_code' => null, + 'finalization_incident' => 0, + 'finalization_error_code' => null, + 'result_json' => null, + 'created_at' => $now - 1, + 'updated_at' => $now, + ]); + + $forceTerminalFailure->invoke( + null, + 701, + str_repeat('7', 32), + false, + true, + 'local_finalization_failed', + true + ); + $transportIncident = CloudJob::findOrFail(701); + $assert((int) $transportIncident->dispatch_attempts === 1 + && CloudDispatchLedger::hasTransportNotDispatchedProof( + $transportIncident + ) + && (string) $transportIncident->sealed_payload === $transportEnvelope + && (int) $transportIncident->available_at >= time() + 290 + && CloudAgentSettingsService::$requests === 0, + 'A strict transport not-dispatched response lost its proof or immutable generation.'); + + Capsule::table('cloud_rescue_template_imports')->insert([ + 'operation_id' => $transportOperationId, + 'host_id' => 7, + 'template_id' => $transportTemplateId, + 'job_id' => 701, + 'import_type' => 'upload', + 'name' => $transportName, + 'size_bytes' => 0, + 'sha256' => '', + 'state' => 'importing', + 'error_code' => '', + 'created_at' => $now, + 'updated_at' => $now, + ]); + Capsule::table('cloud_jobs')->where('id', 701)->update([ + 'available_at' => 0, + 'lease_token' => null, + 'lease_until' => 0, + ]); + $transportService = new CloudAutomaticOperationRecoveryService(); + $assert($transportService->tick(), + 'The strict transport not-dispatched finalizer was not claimed.'); + $transportRecovered = CloudJob::findOrFail(701); + $assert((int) $transportRecovered->dispatch_attempts === 1 + && !(bool) $transportRecovered->effect_may_have_applied + && (bool) $transportRecovered->safe_to_resubmit + && (string) $transportRecovered->effect_resolution === 'not_applied' + && (string) $transportRecovered->effect_resolution_reason + === 'automatic_local_not_dispatched' + && !(bool) $transportRecovered->finalization_incident + && $transportRecovered->finalization_error_code === null + && (string) CloudRescueTemplateImport::query() + ->where('operation_id', $transportOperationId) + ->value('state') === 'failed' + && CloudAgentSettingsService::$requests === 0, + 'Transport not-dispatched recovery did not converge the rescue ledger locally.'); + + $retryActor = [ + 'user_id' => 1, + 'is_admin' => true, + 'is_super_admin' => true, + 'can_manage' => true, + ]; + $transportRetry = CloudRescueTemplateImportService::start( + 7, + 'upload', + $transportName, + 'https://panel.example.test/staging/rescue-retry.iso', + '', + '', + $transportBytes, + $retryActor + ); + $assert((int) $transportRetry['task_id'] !== 701 + && CloudRescueTemplateImport::query() + ->where('host_id', 7) + ->where('name', $transportName) + ->count() === 2 + && CloudAgentSettingsService::$requests === 0, + 'A locally converged rescue import still blocked its same-name retry.'); + echo "Cloud undispatched finalizer recovery tests passed.\n"; }