fix(cloud): recover strict undispatched imports

go
codex 1 month ago
parent efd766afe9
commit 8f9af85681

@ -295,15 +295,41 @@ final class CloudAutomaticOperationRecoveryService
->first();
}
if ($job === null && !$receivedOnly) {
// A valid zero counter proves that no Agent mutation was
// sent. Retry only PHP's kind-specific not-applied ledger
// finalizer; never inspect or contact the host.
// A zero counter or the worker's exact persisted
// dispatch_state=not_dispatched proof establishes that no
// Agent mutation was sent. Retry only PHP's kind-specific
// not-applied ledger finalizer; never inspect or contact
// the host.
$job = CloudJob::query()
->where('execution_mode', 'agent')
->where('phase', 'terminal')
->where('outcome', 'failed')
->where('status', 'failed')
->where('dispatch_attempts', 0)
->where(static function ($proof): void {
$proof->where('dispatch_attempts', 0)
->orWhere(static function ($transport): void {
$transport->where('dispatch_attempts', 1)
->where('attempts', 1)
->where('dispatch_started_at', '>', 0)
->where('dispatched_at', '>', 0)
->where('receipt_deadline_at', '>', 0)
->whereNull('agent_accepted_at')
->whereNull('completion_event_id')
->where('effect_may_have_applied', 0)
->where('safe_to_resubmit', 0)
->where(
'effect_resolution',
'not_applied'
)->where(
'effect_resolution_reason',
CloudDispatchLedger::
TRANSPORT_NOT_DISPATCHED_REASON
)->where(
'error_code',
'local_finalization_failed'
);
});
})
->where('finalization_incident', 1)
->where(static function ($incident): void {
$incident->whereNull('finalization_error_code')
@ -317,10 +343,10 @@ final class CloudAutomaticOperationRecoveryService
'kind',
CloudOperationRecoveryPolicy::kinds()
)->orWhere(static function ($create): void {
// Generation zero is definitive no-effect
// evidence for create too. Its financial and
// placement ledgers use the specialised atomic
// local finalizer, never the Agent mutation.
// Proven local no-effect is authoritative for
// create too. Its financial and placement
// ledgers use the specialised atomic local
// finalizer, never the Agent mutation.
$create->where('kind', 'create')
->whereNotNull('order_id');
});

@ -315,7 +315,8 @@ final class CloudCreateFailureFinalizer
|| (string) $job->phase !== 'terminal'
|| (string) $job->outcome !== 'failed'
|| (string) $job->status !== 'failed'
|| !CloudDispatchLedger::isNeverDispatched($job)
|| !CloudDispatchLedger::
isLocallyFinalizableNotDispatched($job)
|| !(bool) $job->finalization_incident
|| !in_array(
(string) $job->effect_resolution,
@ -339,7 +340,7 @@ final class CloudCreateFailureFinalizer
);
}
CloudOperationCompletionService::
assertNeverDispatchedRecoveryState($job);
assertLocalNotDispatchedRecoveryState($job);
try {
$payload = CloudSecrets::decrypt(

@ -5,14 +5,18 @@ namespace App\Services\Cloud;
/**
* Fail-closed interpretation of the immutable single-dispatch counter.
*
* Zero is the only proof that no Agent mutation crossed the dispatch boundary.
* One is the only valid dispatched generation. Any other value is corruption:
* Zero is the only counter-level proof that no Agent request was attempted.
* One is the only valid attempted generation. A generation-one request may
* still carry a separate, exact Agent transport proof that its durable
* operation journal was never entered. Any other counter value is corruption:
* callers must retain the resource fence and must not consume recovery proof
* until an operator repairs the ledger.
*/
final class CloudDispatchLedger
{
public const CONFLICT_CODE = 'recovery_dispatch_ledger_conflict';
public const TRANSPORT_NOT_DISPATCHED_REASON =
'worker_transport_not_dispatched';
private const REPAIR_INCIDENT_CODES = [
self::CONFLICT_CODE,
'recovery_dispatch_repair_completion_receipt_missing',
@ -51,6 +55,53 @@ final class CloudDispatchLedger
return self::attempts($job) === 1;
}
/**
* Exact durable proof created only from the Agent's strict
* dispatch_state=not_dispatched envelope after the dispatch CAS. The
* generation remains one forever; this marker authorizes only the PHP
* business finalizer and never a replay of the original Agent mutation.
*/
public static function hasTransportNotDispatchedProof(object $job): bool
{
return self::isDispatched($job)
&& (string) ($job->execution_mode ?? '') === 'agent'
&& (string) ($job->phase ?? '') === 'terminal'
&& (string) ($job->outcome ?? '') === 'failed'
&& (string) ($job->status ?? '') === 'failed'
&& (string) ($job->attempts ?? '') === '1'
&& (int) ($job->dispatch_started_at ?? 0) > 0
&& (int) ($job->dispatched_at ?? 0) > 0
&& ($job->agent_accepted_at ?? null) === null
&& (int) ($job->receipt_deadline_at ?? 0) > 0
&& ($job->completion_event_id ?? null) === null
&& in_array(
$job->effect_may_have_applied ?? null,
[0, '0', false],
true
)
&& in_array(
$job->safe_to_resubmit ?? null,
[0, '0', false],
true
)
&& (string) ($job->effect_resolution ?? '') === 'not_applied'
&& hash_equals(
self::TRANSPORT_NOT_DISPATCHED_REASON,
(string) ($job->effect_resolution_reason ?? '')
)
&& (string) ($job->error_code ?? '')
=== 'local_finalization_failed'
&& (bool) ($job->finalization_incident ?? false);
}
/** Proof accepted by the PHP-only finalizer; never authorizes Agent I/O. */
public static function isLocallyFinalizableNotDispatched(
object $job
): bool {
return self::isNeverDispatched($job)
|| self::hasTransportNotDispatchedProof($job);
}
public static function isInvalid(object $job): bool
{
return !in_array(self::attempts($job), [0, 1], true);

@ -3698,7 +3698,8 @@ final class CloudJobService
&& (string) ($job->phase ?? '') === 'terminal'
&& (string) ($job->outcome ?? '') === 'failed'
&& (string) ($job->status ?? '') === 'failed'
&& CloudDispatchLedger::isNeverDispatched($job)
&& CloudDispatchLedger::
isLocallyFinalizableNotDispatched($job)
&& (bool) ($job->finalization_incident ?? false)
&& !CloudDispatchLedger::hasRepairIncident($job)
&& CloudOperationRecoveryPolicy::supports(

@ -1022,6 +1022,48 @@ final class CloudOperationCompletionService
->exists();
}
/**
* Accept either generation zero or the worker's exact persisted transport
* proof that a generation-one request never entered the Agent journal.
* This authorizes only a PHP finalizer; any completion evidence keeps the
* resource fenced and requires operator review.
*/
public static function assertLocalNotDispatchedRecoveryState(
CloudJob $job
): void {
if (CloudDispatchLedger::isNeverDispatched($job)) {
self::assertNeverDispatchedRecoveryState($job);
return;
}
if (!CloudDispatchLedger::hasTransportNotDispatchedProof($job)) {
throw new CloudRecoveryDispatchLedgerConflict(
'任务缺少可信的未派发证明',
409
);
}
if (CloudJobService::hasAgentRecoveryEnvelopeConflict($job)) {
throw new CloudRecoveryInstanceStateConflict(
'未派发传输证明与 Agent 请求账本不一致',
409
);
}
$hostId = filter_var(
$job->host_id,
FILTER_VALIDATE_INT,
['options' => ['min_range' => 1]]
);
$operationId = $job->operation_id;
if ($hostId === false || !is_string($operationId)
|| preg_match('/^[a-f0-9]{32}$/D', $operationId) !== 1
|| CloudAgentCompletionEvent::where('host_id', (int) $hostId)
->where('operation_id', $operationId)->exists()) {
throw new CloudRecoveryInstanceStateConflict(
'未派发传输证明存在冲突的 Agent 完成证据',
409
);
}
}
/**
* Generation zero may run only a PHP finalizer. Any durable Agent boundary
* or completion evidence means that zero is not trustworthy and must stay

@ -470,7 +470,7 @@ final class CloudWorkerService
} catch (\Throwable $finishError) {
self::forceTerminalFailure((int) $job->id,
(string) $job->lease_token, !$safe,
$safe, 'local_finalization_failed');
$safe, 'local_finalization_failed', $safe);
}
}
return true;
@ -1002,7 +1002,8 @@ final class CloudWorkerService
}
private static function forceTerminalFailure(int $jobId, string $token,
bool $effect, bool $safe, string $code): void
bool $effect, bool $safe, string $code,
bool $confirmedNotDispatched = false): void
{
$claimed = CloudJob::where('id', $jobId)
->where('lease_token', $token)->where('phase', '<>', 'terminal')
@ -1181,17 +1182,26 @@ final class CloudWorkerService
// explicit durable incident for operators.
// Registered policy kinds and an order-backed create have PHP-only
// retry paths that can safely publish a known not-applied effect for
// generation zero. Aggregate/other special finalizers retain the
// conservative unknown effect if their atomic fallback fails.
// generation zero or for one exact transport-level
// dispatch_state=not_dispatched proof. The immutable counter remains
// one in the latter case. Aggregate/other special finalizers retain
// the conservative unknown effect if their atomic fallback fails.
$neverDispatched = $recoverableFinalizer
&& CloudDispatchLedger::isNeverDispatched($claimed);
CloudJob::where('id', $jobId)->where('lease_token', $token)
->where('phase', '<>', 'terminal')->update([
$transportNotDispatched = $recoverableFinalizer
&& $confirmedNotDispatched
&& !$effect
&& $safe
&& CloudDispatchLedger::isDispatched($claimed)
&& ($claimed->agent_accepted_at ?? null) === null
&& ($claimed->completion_event_id ?? null) === null;
$notDispatched = $neverDispatched || $transportNotDispatched;
$terminalUpdate = [
'phase' => 'terminal', 'outcome' => 'failed',
'status' => 'failed', 'terminal_at' => time(),
'effect_may_have_applied' => $neverDispatched ? 0 : 1,
'effect_may_have_applied' => $notDispatched ? 0 : 1,
'safe_to_resubmit' => 0,
'effect_resolution' => $neverDispatched
'effect_resolution' => $notDispatched
? 'not_applied' : 'unknown',
'error_code' => $dispatchConflict
? CloudDispatchLedger::CONFLICT_CODE
@ -1210,7 +1220,13 @@ final class CloudWorkerService
'available_at' => ($dispatchConflict
|| $recoverableFinalizer) ? time() + 300 : 0,
'updated_at' => time(),
]);
];
if ($transportNotDispatched) {
$terminalUpdate['effect_resolution_reason'] =
CloudDispatchLedger::TRANSPORT_NOT_DISPATCHED_REASON;
}
CloudJob::where('id', $jobId)->where('lease_token', $token)
->where('phase', '<>', 'terminal')->update($terminalUpdate);
}
private static function localFinalizationIncidentCode(
@ -1927,7 +1943,7 @@ final class CloudWorkerService
}
/**
* Retry only the PHP ledger finalizer for a generation proven never sent.
* Retry only the PHP ledger finalizer for an operation proven not sent.
* The terminal task receipt remains immutable and no Agent request occurs.
*/
public function retryUndispatchedFinalization(
@ -2124,7 +2140,8 @@ final class CloudWorkerService
|| (string) $job->phase !== 'terminal'
|| (string) $job->outcome !== 'failed'
|| (string) $job->status !== 'failed'
|| !CloudDispatchLedger::isNeverDispatched($job)
|| !CloudDispatchLedger::
isLocallyFinalizableNotDispatched($job)
|| !(bool) $job->finalization_incident
|| !in_array((string) $job->effect_resolution,
['unknown', 'not_applied'], true)
@ -2146,7 +2163,7 @@ final class CloudWorkerService
);
}
CloudOperationCompletionService::
assertNeverDispatchedRecoveryState($job);
assertLocalNotDispatchedRecoveryState($job);
}
$recoveredCreateEvent = null;
if ($recoveredCreateEventId !== null) {

@ -66,9 +66,12 @@ namespace {
require dirname(__DIR__) . '/vendor/autoload.php';
use App\Models\CloudJob;
use App\Models\CloudRescueTemplateImport;
use App\Services\Cloud\CloudAutomaticOperationRecoveryService;
use App\Services\Cloud\CloudAgentSettingsService;
use App\Services\Cloud\CloudDispatchLedger;
use App\Services\Cloud\CloudJobService;
use App\Services\Cloud\CloudRescueTemplateImportService;
use App\Services\Cloud\CloudSecrets;
use App\Services\Cloud\CloudWorkerService;
use Illuminate\Database\Capsule\Manager as Capsule;
@ -180,6 +183,21 @@ namespace {
$table->integer('active_job_id')->nullable();
$table->string('state')->default('active');
});
$schema->create('cloud_rescue_template_imports',
static function (Blueprint $table): void {
$table->string('operation_id')->primary();
$table->integer('host_id');
$table->string('template_id', 32);
$table->integer('job_id');
$table->string('import_type');
$table->string('name');
$table->integer('size_bytes')->default(0);
$table->string('sha256', 64)->default('');
$table->string('state');
$table->string('error_code')->default('');
$table->integer('created_at');
$table->integer('updated_at');
});
$schema->create('cloud_template_replicas',
static function (Blueprint $table): void {
$table->integer('host_id');
@ -777,5 +795,164 @@ namespace {
&& CloudAgentSettingsService::$requests === 0,
'Undispatched create recovery repeated refund/release or contacted Agent.');
// The dispatch CAS may already be generation one when an exact Agent
// validation response proves that its operation journal was never entered.
// If PHP's catalog finalizer then fails, preserve that proof without
// rewriting the immutable counter, recover only the local ledger, and
// allow a new same-name import generation afterward.
$transportOperationId = str_repeat('ba', 16);
$transportTemplateId = str_repeat('c', 32);
$transportName = 'transport-not-dispatched-rescue';
$transportBytes = 2 * 1024 * 1024;
$transportPayload = [
'host_id' => 7,
'import_type' => 'upload',
'name' => $transportName,
'source' => '',
'sha256' => '',
'source_size_bytes' => $transportBytes,
'can_manage' => true,
'rescue_template_id' => $transportTemplateId,
'method' => 'POST',
'request_target' => '/v1/rescue-templates',
'body' => [
'operation_id' => $transportOperationId,
'id' => $transportTemplateId,
'name' => $transportName,
'source_url' => 'https://panel.example.test/staging/rescue.iso',
'source_size_bytes' => $transportBytes,
'sha256' => '',
],
'owner_id' => 1,
];
$transportEnvelope = CloudSecrets::encrypt(
$transportPayload,
'job-payload'
);
Capsule::table('cloud_jobs')->insert([
'id' => 701,
'operation_id' => $transportOperationId,
'request_idempotency_key' => $transportOperationId,
'request_fingerprint' => hash('sha256', $transportOperationId),
'parent_job_id' => null,
'order_id' => null,
'invoice_id' => null,
'instance_id' => null,
'snapshot_id' => null,
'user_id' => 1,
'host_id' => 7,
'kind' => 'rescue_template_import',
'execution_mode' => 'agent',
'phase' => 'dispatching',
'outcome' => null,
'status' => 'running',
'sealed_payload' => $transportEnvelope,
'attempts' => 1,
'dispatch_attempts' => 1,
'dispatch_deadline_at' => $now + 60,
'dispatch_started_at' => $now,
'agent_accepted_at' => null,
'receipt_deadline_at' => $now + 300,
'terminal_at' => null,
'effect_may_have_applied' => null,
'safe_to_resubmit' => null,
'effect_resolution' => 'unknown',
'effect_resolved_at' => null,
'effect_resolved_by' => null,
'effect_resolution_reason' => '',
'completion_event_id' => null,
'read_at' => null,
'dispatched_at' => $now,
'lease_token' => str_repeat('7', 32),
'lease_until' => $now + 180,
'available_at' => 0,
'error_code' => null,
'finalization_incident' => 0,
'finalization_error_code' => null,
'result_json' => null,
'created_at' => $now - 1,
'updated_at' => $now,
]);
$forceTerminalFailure->invoke(
null,
701,
str_repeat('7', 32),
false,
true,
'local_finalization_failed',
true
);
$transportIncident = CloudJob::findOrFail(701);
$assert((int) $transportIncident->dispatch_attempts === 1
&& CloudDispatchLedger::hasTransportNotDispatchedProof(
$transportIncident
)
&& (string) $transportIncident->sealed_payload === $transportEnvelope
&& (int) $transportIncident->available_at >= time() + 290
&& CloudAgentSettingsService::$requests === 0,
'A strict transport not-dispatched response lost its proof or immutable generation.');
Capsule::table('cloud_rescue_template_imports')->insert([
'operation_id' => $transportOperationId,
'host_id' => 7,
'template_id' => $transportTemplateId,
'job_id' => 701,
'import_type' => 'upload',
'name' => $transportName,
'size_bytes' => 0,
'sha256' => '',
'state' => 'importing',
'error_code' => '',
'created_at' => $now,
'updated_at' => $now,
]);
Capsule::table('cloud_jobs')->where('id', 701)->update([
'available_at' => 0,
'lease_token' => null,
'lease_until' => 0,
]);
$transportService = new CloudAutomaticOperationRecoveryService();
$assert($transportService->tick(),
'The strict transport not-dispatched finalizer was not claimed.');
$transportRecovered = CloudJob::findOrFail(701);
$assert((int) $transportRecovered->dispatch_attempts === 1
&& !(bool) $transportRecovered->effect_may_have_applied
&& (bool) $transportRecovered->safe_to_resubmit
&& (string) $transportRecovered->effect_resolution === 'not_applied'
&& (string) $transportRecovered->effect_resolution_reason
=== 'automatic_local_not_dispatched'
&& !(bool) $transportRecovered->finalization_incident
&& $transportRecovered->finalization_error_code === null
&& (string) CloudRescueTemplateImport::query()
->where('operation_id', $transportOperationId)
->value('state') === 'failed'
&& CloudAgentSettingsService::$requests === 0,
'Transport not-dispatched recovery did not converge the rescue ledger locally.');
$retryActor = [
'user_id' => 1,
'is_admin' => true,
'is_super_admin' => true,
'can_manage' => true,
];
$transportRetry = CloudRescueTemplateImportService::start(
7,
'upload',
$transportName,
'https://panel.example.test/staging/rescue-retry.iso',
'',
'',
$transportBytes,
$retryActor
);
$assert((int) $transportRetry['task_id'] !== 701
&& CloudRescueTemplateImport::query()
->where('host_id', 7)
->where('name', $transportName)
->count() === 2
&& CloudAgentSettingsService::$requests === 0,
'A locally converged rescue import still blocked its same-name retry.');
echo "Cloud undispatched finalizer recovery tests passed.\n";
}

Loading…
Cancel
Save