addConnection([ 'driver' => 'sqlite', 'database' => ':memory:', ], 'default'); $capsule->setAsGlobal(); $capsule->bootEloquent(); $schema = $capsule->schema('default'); $schema->create('user', static function (Blueprint $table): void { $table->increments('id'); $table->decimal('money', 12, 2)->default(0); $table->integer('enable')->default(1); $table->integer('is_admin')->default(0); }); $schema->create('cloud_jobs', static function (Blueprint $table): void { $table->increments('id'); $table->string('operation_id')->unique(); $table->string('request_idempotency_key')->unique(); $table->string('request_fingerprint'); $table->integer('parent_job_id')->nullable(); $table->integer('order_id')->nullable(); $table->integer('invoice_id')->nullable(); $table->integer('instance_id')->nullable(); $table->integer('snapshot_id')->nullable(); $table->integer('user_id'); $table->integer('host_id')->nullable(); $table->string('kind'); $table->string('execution_mode')->default('agent'); $table->string('phase'); $table->string('outcome')->nullable(); $table->string('status'); $table->text('sealed_payload'); $table->integer('attempts')->default(0); $table->integer('dispatch_attempts')->default(0); $table->integer('dispatch_deadline_at'); $table->integer('dispatch_started_at')->nullable(); $table->integer('agent_accepted_at')->nullable(); $table->integer('receipt_deadline_at')->nullable(); $table->integer('terminal_at')->nullable(); $table->boolean('effect_may_have_applied')->nullable(); $table->boolean('safe_to_resubmit')->nullable(); $table->string('effect_resolution')->default('not_applicable'); $table->integer('effect_resolved_at')->nullable(); $table->integer('effect_resolved_by')->nullable(); $table->string('effect_resolution_reason')->default(''); $table->string('completion_event_id')->nullable(); $table->integer('read_at')->nullable(); $table->integer('dispatched_at')->nullable(); $table->string('lease_token')->nullable(); $table->integer('lease_until')->default(0); $table->integer('available_at')->default(0); $table->string('error_code')->nullable(); $table->boolean('finalization_incident')->default(0); $table->string('finalization_error_code')->nullable(); $table->text('result_json')->nullable(); $table->integer('created_at'); $table->integer('updated_at'); }); $schema->create('cloud_agent_completion_events', static function (Blueprint $table): void { $table->string('event_id')->primary(); $table->integer('host_id'); $table->string('operation_id'); $table->string('payload_sha256'); $table->string('outcome'); $table->boolean('effect_may_have_applied'); $table->boolean('safe_to_resubmit'); $table->string('error_code')->default(''); $table->text('result_json')->nullable(); $table->string('disposition')->default('received'); $table->integer('received_at'); $table->integer('applied_at')->nullable(); $table->unique(['host_id', 'operation_id']); }); $schema->create('cloud_invoice_payments', static function (Blueprint $table): void { $table->increments('id'); $table->integer('invoice_id'); $table->integer('user_id'); $table->string('provider'); $table->string('provider_order_no'); $table->string('idempotency_key'); $table->decimal('amount', 12, 2)->default(0); $table->string('status'); $table->integer('created_at'); $table->integer('updated_at'); $table->unique(['user_id', 'idempotency_key']); }); Capsule::table('user')->insert([ ['id' => 1, 'money' => 0, 'enable' => 1, 'is_admin' => 0], ['id' => 2, 'money' => 0, 'enable' => 1, 'is_admin' => 0], ]); $now = time(); $insertJob = static function (int $userId, string $operationId, string $phase, int $dispatchAttempts) use ($now): int { return (int) Capsule::table('cloud_jobs')->insertGetId([ 'operation_id' => $operationId, 'request_idempotency_key' => $operationId, 'request_fingerprint' => hash('sha256', $operationId), 'parent_job_id' => null, 'order_id' => null, 'invoice_id' => null, 'instance_id' => null, 'snapshot_id' => null, 'user_id' => $userId, 'host_id' => 7, // An unrecognised kind exercises the generic terminal ledger path // without requiring a business-resource fixture. 'kind' => 'test_mutation', 'execution_mode' => 'agent', 'phase' => $phase, 'outcome' => null, 'status' => $phase === 'accepted' ? 'queued' : 'running', 'sealed_payload' => CloudSecrets::encrypt([ 'method' => 'POST', 'request_target' => '/v1/test', 'body' => ['operation_id' => $operationId], 'owner_id' => $userId, ], 'job-payload'), 'attempts' => $dispatchAttempts, 'dispatch_attempts' => $dispatchAttempts, 'dispatch_deadline_at' => $now - 20, 'dispatch_started_at' => $dispatchAttempts === 1 ? $now - 30 : null, 'agent_accepted_at' => null, 'receipt_deadline_at' => $dispatchAttempts === 1 ? $now - 10 : null, 'terminal_at' => null, 'effect_may_have_applied' => null, 'safe_to_resubmit' => null, 'effect_resolution' => 'not_applicable', 'effect_resolved_at' => null, 'effect_resolved_by' => null, 'effect_resolution_reason' => '', 'completion_event_id' => null, 'read_at' => null, 'dispatched_at' => $dispatchAttempts === 1 ? $now - 30 : null, // Simulate a dead daemon whose stale lease extends past deadline. 'lease_token' => str_repeat('f', 32), 'lease_until' => $now + 600, 'available_at' => 0, 'error_code' => null, 'result_json' => null, 'created_at' => $now - 60, 'updated_at' => $now - 30, ]); }; $receiptOperation = str_repeat('a', 32); $receiptJobId = $insertJob(1, $receiptOperation, 'dispatching', 1); Capsule::table('cloud_agent_completion_events')->insert([ 'event_id' => str_repeat('e', 32), 'host_id' => 7, 'operation_id' => $receiptOperation, 'payload_sha256' => str_repeat('1', 64), 'outcome' => 'succeeded', 'effect_may_have_applied' => 0, 'safe_to_resubmit' => 0, 'error_code' => '', 'result_json' => json_encode(['confirmed' => true], JSON_THROW_ON_ERROR), 'disposition' => 'received', 'received_at' => $now - 15, 'applied_at' => null, ]); $otherOperation = str_repeat('b', 32); $otherJobId = $insertJob(2, $otherOperation, 'dispatching', 1); $assert(CloudWorkerService::expireDueTasksForOwner( 1, $receiptJobId, 1 ) === 1, 'The owner-scoped fallback did not claim the due task.'); $receiptJob = CloudJob::find($receiptJobId); $otherJob = CloudJob::find($otherJobId); $event = Capsule::table('cloud_agent_completion_events') ->where('event_id', str_repeat('e', 32))->first(); $assert($receiptJob !== null && (string) $receiptJob->phase === 'terminal' && (string) $receiptJob->outcome === 'succeeded' && (int) $receiptJob->dispatch_attempts === 1 && (string) $event->disposition === 'applied', 'A receipt saved during dispatch was replaced by deadline failure.'); $assert($otherJob !== null && (string) $otherJob->phase === 'dispatching', 'Owner-scoped expiry mutated another user\'s task.'); $failureOperation = str_repeat('c', 32); $failureJobId = $insertJob(1, $failureOperation, 'dispatching', 1); $assert(CloudWorkerService::expireDueTasksForOwner( 1, $failureJobId, 1 ) === 1, 'The receipt-less due task was not closed.'); $failed = CloudJob::find($failureJobId); $assert($failed !== null && (string) $failed->phase === 'terminal' && (string) $failed->outcome === 'failed' && (bool) $failed->effect_may_have_applied && !(bool) $failed->safe_to_resubmit, 'A crossed dispatch boundary did not become a conservative failure.'); $incidentOperation = str_repeat('5', 32); $incidentJobId = $insertJob( 1, $incidentOperation, 'awaiting_receipt', 1 ); Capsule::table('cloud_jobs')->where('id', $incidentJobId)->update([ 'sealed_payload' => 'deadline-test:missing-payload', ]); $assert(CloudWorkerService::expireDueTasksForOwner( 1, $incidentJobId, 1 ) === 1, 'The finalization-incident fixture was not claimed.'); $incident = CloudJob::find($incidentJobId); $assert($incident !== null && (string) $incident->phase === 'terminal' && (string) $incident->outcome === 'failed' && (string) $incident->status === 'failed' && (bool) $incident->effect_may_have_applied && !(bool) $incident->safe_to_resubmit && (string) $incident->effect_resolution === 'unknown' && (string) $incident->error_code === 'local_finalization_failed' && (bool) $incident->finalization_incident && (string) $incident->finalization_error_code === 'completion_deadline_exceeded' && (string) $incident->sealed_payload === '', 'A local finalizer crash left the task active, leaked its payload, or concealed the incident.'); $lateOperation = str_repeat('7', 32); $lateJobId = $insertJob(1, $lateOperation, 'awaiting_receipt', 1); $lateEventId = str_repeat('6', 32); Capsule::table('cloud_agent_completion_events')->insert([ 'event_id' => $lateEventId, 'host_id' => 7, 'operation_id' => $lateOperation, 'payload_sha256' => str_repeat('2', 64), 'outcome' => 'succeeded', 'effect_may_have_applied' => 0, 'safe_to_resubmit' => 0, 'error_code' => '', 'result_json' => json_encode(['confirmed' => true], JSON_THROW_ON_ERROR), 'disposition' => 'received', // Equal to or later than the absolute boundary is audit-only. 'received_at' => $now - 9, 'applied_at' => null, ]); $lateJob = (new CloudWorkerService())->settleCompletionReceipt( $lateJobId, $lateEventId ); $lateEvent = Capsule::table('cloud_agent_completion_events') ->where('event_id', $lateEventId)->first(); $assert((string) $lateJob->phase === 'terminal' && (string) $lateJob->outcome === 'failed' && (string) $lateJob->error_code === 'operation_failed' && (bool) $lateJob->effect_may_have_applied && !(bool) $lateJob->safe_to_resubmit && !(bool) $lateJob->finalization_incident && (string) $lateEvent->disposition === 'late', 'A receipt persisted after the deadline revived or succeeded the task.'); $insertPaymentJob = static function (string $key, int $invoiceId) use ($now): int { $operationId = md5('payment-deadline:' . $key); return (int) Capsule::table('cloud_jobs')->insertGetId([ 'operation_id' => $operationId, 'request_idempotency_key' => $key, 'request_fingerprint' => hash('sha256', $key), 'parent_job_id' => null, 'order_id' => null, 'invoice_id' => $invoiceId, 'instance_id' => null, 'snapshot_id' => null, 'user_id' => 1, 'host_id' => null, 'kind' => 'payment_create', 'execution_mode' => 'local', 'phase' => 'dispatching', 'outcome' => null, 'status' => 'running', 'sealed_payload' => CloudSecrets::encrypt([ 'input' => ['idempotency_key' => $key, 'method' => 'paytaro'], ], 'job-payload'), 'attempts' => 1, 'dispatch_attempts' => 1, 'dispatch_deadline_at' => $now - 20, 'dispatch_started_at' => $now - 30, 'agent_accepted_at' => null, 'receipt_deadline_at' => $now - 10, 'terminal_at' => null, 'effect_may_have_applied' => null, 'safe_to_resubmit' => null, 'effect_resolution' => 'not_applicable', 'effect_resolved_at' => null, 'effect_resolved_by' => null, 'effect_resolution_reason' => '', 'completion_event_id' => null, 'read_at' => null, 'dispatched_at' => $now - 30, 'lease_token' => bin2hex(random_bytes(16)), 'lease_until' => $now + 600, 'available_at' => 0, 'error_code' => null, 'result_json' => null, 'created_at' => $now - 60, 'updated_at' => $now - 30, ]); }; $missingPaymentKey = 'payment-missing-0001'; $pendingPaymentKey = 'payment-pending-0001'; $unknownPaymentKey = 'payment-unknown-0001'; Capsule::table('cloud_invoice_payments')->insert([ [ 'invoice_id' => 502, 'user_id' => 1, 'provider' => 'paytaro', 'provider_order_no' => 'provider-pending-0001', 'idempotency_key' => $pendingPaymentKey, 'amount' => 10, 'status' => 'pending', 'created_at' => $now - 30, 'updated_at' => $now - 30, ], [ 'invoice_id' => 503, 'user_id' => 1, 'provider' => 'paytaro', 'provider_order_no' => 'provider-unknown-0001', 'idempotency_key' => $unknownPaymentKey, 'amount' => 10, 'status' => 'creation_unknown', 'created_at' => $now - 30, 'updated_at' => $now - 30, ], ]); $missingPaymentJobId = $insertPaymentJob($missingPaymentKey, 501); $pendingPaymentJobId = $insertPaymentJob($pendingPaymentKey, 502); $unknownPaymentJobId = $insertPaymentJob($unknownPaymentKey, 503); foreach ([$missingPaymentJobId, $pendingPaymentJobId, $unknownPaymentJobId] as $paymentJobId) { $assert(CloudWorkerService::expireDueTasksForOwner( 1, $paymentJobId, 1 ) === 1, 'A due local payment task was not terminalized.'); } $missingPaymentJob = CloudJob::find($missingPaymentJobId); $pendingPaymentJob = CloudJob::find($pendingPaymentJobId); $unknownPaymentJob = CloudJob::find($unknownPaymentJobId); $pendingPaymentResult = json_decode( (string) $pendingPaymentJob->result_json, true, flags: JSON_THROW_ON_ERROR ); $assert($missingPaymentJob->phase === 'terminal' && $missingPaymentJob->outcome === 'failed' && $missingPaymentJob->effect_resolution === 'not_applied' && !(bool) $missingPaymentJob->effect_may_have_applied && (bool) $missingPaymentJob->safe_to_resubmit && $missingPaymentJob->error_code === 'payment_order_not_dispatched' && $pendingPaymentJob->phase === 'terminal' && $pendingPaymentJob->outcome === 'succeeded' && ($pendingPaymentResult['payment_id'] ?? null) === '1' && $unknownPaymentJob->phase === 'terminal' && $unknownPaymentJob->outcome === 'failed' && $unknownPaymentJob->effect_resolution === 'unknown' && (bool) $unknownPaymentJob->effect_may_have_applied && !(bool) $unknownPaymentJob->safe_to_resubmit && $unknownPaymentJob->error_code === 'payment_creation_unknown' && Capsule::table('cloud_invoice_payments')->count() === 2, 'Payment deadline fallback retried a provider POST or misclassified its durable attempt.'); $assert(CloudJobService::receiptDeadlineSeconds('action') === 960 && CloudJobService::receiptDeadlineSeconds( 'template_import_replica') === 8100 && CloudJobService::receiptDeadlineSeconds( 'rescue_template_import') === 8100 && CloudJobService::receiptDeadlineSeconds( 'user_iso_import_replica') === 8100 && CloudJobService::aggregateDeadlineSeconds( 'template_metadata_sync') === 1560 && CloudJobService::aggregateDeadlineSeconds( 'template_import') === 8700 && CloudJobService::aggregateDeadlineSeconds( 'user_iso_import') === 8700, 'Task deadlines lost dispatch, receipt, or aggregate finalization margin.'); $parentOperation = str_repeat('d', 32); $parentId = (int) Capsule::table('cloud_jobs')->insertGetId([ 'operation_id' => $parentOperation, 'request_idempotency_key' => $parentOperation, 'request_fingerprint' => hash('sha256', $parentOperation), 'parent_job_id' => null, 'order_id' => null, 'invoice_id' => null, 'instance_id' => null, 'snapshot_id' => null, 'user_id' => 1, 'host_id' => null, 'kind' => 'test_aggregate', 'execution_mode' => 'aggregate', 'phase' => 'awaiting_receipt', 'outcome' => null, 'status' => 'running', 'sealed_payload' => CloudSecrets::encrypt([], 'job-payload'), 'attempts' => 0, 'dispatch_attempts' => 0, 'dispatch_deadline_at' => $now - 30, 'dispatch_started_at' => null, 'agent_accepted_at' => null, 'receipt_deadline_at' => $now - 1, 'terminal_at' => null, 'effect_may_have_applied' => null, 'safe_to_resubmit' => null, 'effect_resolution' => 'not_applicable', 'effect_resolved_at' => null, 'effect_resolved_by' => null, 'effect_resolution_reason' => '', 'completion_event_id' => null, 'read_at' => null, 'dispatched_at' => null, 'lease_token' => str_repeat('9', 32), 'lease_until' => $now + 600, 'available_at' => 0, 'error_code' => null, 'result_json' => null, 'created_at' => $now - 100, 'updated_at' => $now - 50, ]); $childOperation = str_repeat('8', 32); $childId = (int) Capsule::table('cloud_jobs')->insertGetId([ 'operation_id' => $childOperation, 'request_idempotency_key' => $childOperation, 'request_fingerprint' => hash('sha256', $childOperation), 'parent_job_id' => $parentId, 'order_id' => null, 'invoice_id' => null, 'instance_id' => null, 'snapshot_id' => null, 'user_id' => 1, 'host_id' => 7, 'kind' => 'test_mutation', 'execution_mode' => 'agent', 'phase' => 'accepted', 'outcome' => null, 'status' => 'queued', 'sealed_payload' => CloudSecrets::encrypt([ 'method' => 'POST', 'request_target' => '/v1/test', 'body' => ['operation_id' => $childOperation], 'owner_id' => 1, ], 'job-payload'), 'attempts' => 0, 'dispatch_attempts' => 0, 'dispatch_deadline_at' => $now + 600, 'dispatch_started_at' => null, 'agent_accepted_at' => null, 'receipt_deadline_at' => null, 'terminal_at' => null, 'effect_may_have_applied' => null, 'safe_to_resubmit' => null, 'effect_resolution' => 'not_applicable', 'effect_resolved_at' => null, 'effect_resolved_by' => null, 'effect_resolution_reason' => '', 'completion_event_id' => null, 'read_at' => null, 'dispatched_at' => null, 'lease_token' => null, 'lease_until' => 0, 'available_at' => 0, 'error_code' => null, 'result_json' => null, 'created_at' => $now - 80, 'updated_at' => $now - 40, ]); Capsule::unprepared('CREATE TRIGGER simulate_child_claim_cas_miss ' . 'BEFORE UPDATE OF lease_token ON cloud_jobs ' . 'WHEN OLD.id = ' . $childId . " AND OLD.phase <> 'terminal' " . 'BEGIN SELECT RAISE(IGNORE); END'); $assert(CloudWorkerService::expireDueTasksForOwner( 1, $parentId, 10 ) === 1, 'Owner fallback did not claim an expired aggregate parent.'); $parent = CloudJob::find($parentId); $child = CloudJob::find($childId); $assert($parent !== null && $child !== null && (string) $parent->phase === 'awaiting_receipt' && $parent->outcome === null && $parent->lease_token === null && (int) $parent->lease_until === 0 && (string) $child->phase === 'accepted', 'A child claim CAS miss hid active work behind a terminal parent.'); Capsule::unprepared('DROP TRIGGER simulate_child_claim_cas_miss'); $assert(CloudWorkerService::expireDueTasksForOwner( 1, $parentId, 10 ) === 1, 'A later fallback pass did not reclaim the aggregate parent.'); $parent = CloudJob::find($parentId); $child = CloudJob::find($childId); $assert($parent !== null && $child !== null && (string) $child->phase === 'terminal' && (string) $child->outcome === 'failed' && !(bool) $child->effect_may_have_applied && (bool) $child->safe_to_resubmit && (string) $parent->phase === 'terminal' && (string) $parent->outcome === 'failed', 'A later aggregate fallback did not terminalize children first.'); echo "Owner-scoped task deadline fallback tests passed.\n"; }