false, 1 => true, 2 => true, -1 => true, ] as $attempts => $mayHaveDispatched) { $job = (object) ['dispatch_attempts' => $attempts]; $assert( CloudDispatchLedger::mayHaveDispatched($job) === $mayHaveDispatched, 'The dispatch ledger did not treat only zero as proof of no dispatch.' ); } $capsule = new Capsule(); $capsule->addConnection([ 'driver' => 'sqlite', 'database' => ':memory:', ], 'default'); $capsule->setAsGlobal(); $capsule->bootEloquent(); $capsule->schema('default')->create( 'cloud_jobs', static function (Blueprint $table): void { $table->increments('id'); $table->string('operation_id')->default( 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa' ); $table->string('request_idempotency_key')->default( 'bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb' ); $table->string('request_fingerprint')->default( 'cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc' ); $table->integer('parent_job_id')->nullable(); $table->integer('order_id')->nullable(); $table->integer('invoice_id')->nullable(); $table->integer('instance_id')->nullable(); $table->integer('snapshot_id')->nullable(); $table->integer('user_id')->default(1); $table->integer('host_id')->nullable(); $table->string('kind')->default('template_import'); $table->string('execution_mode')->default('aggregate'); $table->integer('attempts')->default(0); $table->integer('dispatch_attempts'); $table->integer('dispatch_deadline_at')->default(1); $table->integer('dispatch_started_at')->nullable(); $table->integer('agent_accepted_at')->nullable(); $table->integer('dispatched_at')->nullable(); $table->integer('receipt_deadline_at')->nullable()->default(1); $table->string('phase'); $table->string('outcome')->nullable(); $table->string('status')->default('succeeded'); $table->integer('terminal_at')->nullable()->default(1); $table->string('completion_event_id')->nullable(); $table->boolean('effect_may_have_applied')->nullable(); $table->boolean('safe_to_resubmit')->nullable(); $table->string('effect_resolution'); $table->string('error_code')->nullable(); $table->text('result_json')->nullable(); $table->boolean('finalization_incident')->default(0); $table->string('finalization_error_code')->nullable(); } ); $agentEnvelope = [ 'host_id' => 7, 'attempts' => 1, 'dispatch_deadline_at' => 1, 'dispatch_started_at' => 1, 'agent_accepted_at' => 1, 'dispatched_at' => 1, 'receipt_deadline_at' => 1, 'terminal_at' => 1, ]; $capsule->schema('default')->create( 'cloud_templates', static function (Blueprint $table): void { $table->string('id')->primary(); $table->integer('active_job_id')->nullable(); } ); $capsule->schema('default')->create( 'cloud_user_isos', static function (Blueprint $table): void { $table->string('id')->primary(); $table->integer('user_id'); $table->integer('active_job_id')->nullable(); } ); foreach ([ [1, 0, 'terminal', 'succeeded', 'applied'], [2, 1, 'terminal', 'succeeded', 'applied'], [3, 2, 'terminal', 'succeeded', 'applied'], [4, -1, 'terminal', 'succeeded', 'applied'], [5, 1, 'awaiting_receipt', null, 'unknown'], ] as [$id, $attempts, $phase, $outcome, $resolution]) { Capsule::table('cloud_jobs')->insert([ 'id' => $id, 'parent_job_id' => null, 'dispatch_attempts' => $attempts, 'phase' => $phase, 'outcome' => $outcome, 'status' => $outcome ?? 'running', 'effect_may_have_applied' => 0, 'safe_to_resubmit' => 0, 'effect_resolution' => $resolution, ]); } foreach ([1, 2] as $parentId) { Capsule::table('cloud_jobs')->insert([ ...$agentEnvelope, 'id' => 100 + $parentId, 'parent_job_id' => $parentId, 'user_id' => 1, 'kind' => 'template_import_replica', 'execution_mode' => 'agent', 'dispatch_attempts' => 1, 'phase' => 'terminal', 'outcome' => 'succeeded', 'status' => 'succeeded', 'effect_may_have_applied' => 0, 'safe_to_resubmit' => 0, 'effect_resolution' => 'applied', 'finalization_incident' => 0, 'finalization_error_code' => null, ]); } foreach ([ [201, 0, 'terminal', 'succeeded', 'applied'], [202, 1, 'terminal', 'succeeded', 'applied'], [203, 2, 'terminal', 'succeeded', 'applied'], [204, -1, 'terminal', 'succeeded', 'applied'], [205, 1, 'awaiting_receipt', null, 'unknown'], ] as [$id, $attempts, $phase, $outcome, $resolution]) { Capsule::table('cloud_jobs')->insert([ 'id' => $id, 'parent_job_id' => null, 'user_id' => 1, 'kind' => 'user_iso_import', 'execution_mode' => 'aggregate', 'dispatch_attempts' => $attempts, 'phase' => $phase, 'outcome' => $outcome, 'status' => $outcome ?? 'running', 'effect_may_have_applied' => 0, 'safe_to_resubmit' => 0, 'effect_resolution' => $resolution, 'finalization_incident' => 0, 'finalization_error_code' => null, ]); } foreach ([201, 202] as $parentId) { Capsule::table('cloud_jobs')->insert([ ...$agentEnvelope, 'id' => 100 + $parentId, 'parent_job_id' => $parentId, 'user_id' => 1, 'kind' => 'user_iso_import_replica', 'execution_mode' => 'agent', 'dispatch_attempts' => 1, 'phase' => 'terminal', 'outcome' => 'succeeded', 'status' => 'succeeded', 'effect_may_have_applied' => 0, 'safe_to_resubmit' => 0, 'effect_resolution' => 'applied', 'finalization_incident' => 0, 'finalization_error_code' => null, ]); } $invokeBoundary = static function ( string $service, object $resource, bool $shouldBlock ) use ($assert): void { $method = new ReflectionMethod($service, 'assertMutationBoundaryLocked'); $jobId = (int) ($resource->active_job_id ?? 0); $parent = $jobId > 0 ? CloudJob::find($jobId) : null; $validKinds = $service === CloudTemplateCatalogService::class ? ['template_import', 'template_metadata_sync', 'template_delete'] : ['user_iso_import', 'user_iso_delete']; if ($parent === null || (string) $parent->execution_mode !== 'aggregate' || $parent->parent_job_id !== null || !in_array((string) $parent->kind, $validKinds, true)) { $parent = null; } try { $method->invoke(null, $resource, $parent); } catch (CloudError $error) { $assert($shouldBlock && $error->httpStatus === 409, $service . ' rejected a valid settled job boundary.'); return; } $assert(!$shouldBlock, $service . ' released an unsafe prior job boundary.'); }; // This focused fixture intentionally has no replica tables. Exercise the // parent/child dispatch-ledger predicate directly; full catalog projection // ownership is covered by CloudCatalogActiveJobPointerRecoveryTest. $parentSafelySettled = static function ( string $service, object $resource, int $jobId ): bool { $method = new ReflectionMethod($service, 'aggregateParentSafelySettled'); $parent = CloudJob::find($jobId); if ($parent === null) { return false; } return $service === CloudTemplateCatalogService::class ? $method->invoke(null, $parent, false) : $method->invoke(null, $resource, $parent, false); }; foreach ([ 1 => false, 2 => true, 3 => true, 4 => true, 5 => true, 999 => true, ] as $jobId => $shouldBlock) { $template = new CloudTemplate(); $template->active_job_id = $jobId; $assert( $parentSafelySettled( CloudTemplateCatalogService::class, $template, $jobId ) === !$shouldBlock, 'The template aggregate dispatch ledger boundary was misclassified.' ); } foreach ([ 201 => false, 202 => true, 203 => true, 204 => true, 205 => true, 999 => true, ] as $jobId => $shouldBlock) { $iso = new CloudUserIso(); $iso->active_job_id = $jobId; $iso->user_id = 1; $assert( $parentSafelySettled( CloudUserIsoService::class, $iso, $jobId ) === !$shouldBlock, 'The ISO aggregate dispatch ledger boundary was misclassified.' ); } // A fail-open aggregate parent may already say succeeded/applied while one of // its Agent children still needs recovery. Both template and ISO mutation // boundaries must follow the child fence rather than trusting the parent row. Capsule::table('cloud_jobs')->insert([ 'id' => 6, 'parent_job_id' => null, 'user_id' => 1, 'kind' => 'template_import', 'execution_mode' => 'aggregate', 'dispatch_attempts' => 0, 'phase' => 'terminal', 'outcome' => 'succeeded', 'status' => 'succeeded', 'effect_may_have_applied' => 0, 'safe_to_resubmit' => 0, 'effect_resolution' => 'applied', 'finalization_incident' => 0, 'finalization_error_code' => null, ]); Capsule::table('cloud_jobs')->insert([ ...$agentEnvelope, 'id' => 106, 'parent_job_id' => 6, 'user_id' => 1, 'kind' => 'template_import_replica', 'execution_mode' => 'agent', 'dispatch_attempts' => 2, 'phase' => 'terminal', 'outcome' => 'succeeded', 'status' => 'succeeded', 'effect_may_have_applied' => 0, 'safe_to_resubmit' => 0, 'effect_resolution' => 'applied', 'finalization_incident' => 0, 'finalization_error_code' => null, ]); Capsule::table('cloud_jobs')->insert([ 'id' => 206, 'parent_job_id' => null, 'user_id' => 1, 'kind' => 'user_iso_import', 'execution_mode' => 'aggregate', 'dispatch_attempts' => 0, 'phase' => 'terminal', 'outcome' => 'succeeded', 'status' => 'succeeded', 'effect_may_have_applied' => 0, 'safe_to_resubmit' => 0, 'effect_resolution' => 'applied', 'finalization_incident' => 0, 'finalization_error_code' => null, ]); Capsule::table('cloud_jobs')->insert([ ...$agentEnvelope, 'id' => 306, 'parent_job_id' => 206, 'user_id' => 1, 'kind' => 'user_iso_import_replica', 'execution_mode' => 'agent', 'dispatch_attempts' => 2, 'phase' => 'terminal', 'outcome' => 'succeeded', 'status' => 'succeeded', 'effect_may_have_applied' => 0, 'safe_to_resubmit' => 0, 'effect_resolution' => 'applied', 'finalization_incident' => 0, 'finalization_error_code' => null, ]); $failOpenTemplate = new CloudTemplate(); $failOpenTemplate->active_job_id = 6; $failOpenIso = new CloudUserIso(); $failOpenIso->active_job_id = 206; $failOpenIso->user_id = 1; $assertBothBoundaries = static function (bool $shouldBlock) use ( $assert, $parentSafelySettled, $failOpenTemplate, $failOpenIso ): void { $assert( $parentSafelySettled( CloudTemplateCatalogService::class, $failOpenTemplate, 6 ) === !$shouldBlock, 'The template child dispatch ledger boundary was misclassified.' ); $assert( $parentSafelySettled( CloudUserIsoService::class, $failOpenIso, 206 ) === !$shouldBlock, 'The ISO child dispatch ledger boundary was misclassified.' ); }; $assertBothBoundaries(true); Capsule::table('cloud_jobs')->whereIn('id', [106, 306])->update([ 'dispatch_attempts' => 1, 'finalization_incident' => 1, 'finalization_error_code' => 'recovery_dispatch_ledger_conflict', ]); $assertBothBoundaries(true); Capsule::table('cloud_jobs')->whereIn('id', [106, 306])->update([ 'dispatch_attempts' => 0, 'outcome' => 'failed', 'status' => 'failed', 'effect_may_have_applied' => 0, 'safe_to_resubmit' => 0, 'effect_resolution' => 'not_applied', 'finalization_incident' => 1, 'finalization_error_code' => 'recovery_ledger_conflict', ]); $assertBothBoundaries(true); Capsule::table('cloud_jobs')->whereIn('id', [106, 306])->update([ 'dispatch_attempts' => 1, 'effect_may_have_applied' => 1, 'effect_resolution' => 'unknown', 'finalization_incident' => 0, 'finalization_error_code' => null, ]); $assertBothBoundaries(true); Capsule::table('cloud_jobs')->whereIn('id', [106, 306])->update([ 'outcome' => 'succeeded', 'status' => 'succeeded', 'effect_may_have_applied' => 0, 'safe_to_resubmit' => 0, 'effect_resolution' => 'applied', ]); $assertBothBoundaries(false); // A settled-looking row is not a valid catalog boundary unless its active job // is the exact aggregate root for that resource type and identity, and exactly // one main record owns that job. foreach ([ [20, 'user_iso_import', 1], [21, 'user_iso_import', 2], [22, 'template_import', 1], [23, 'user_iso_import', 1], ] as [$id, $kind, $userId]) { Capsule::table('cloud_jobs')->insert([ 'id' => $id, 'parent_job_id' => null, 'user_id' => $userId, 'kind' => $kind, 'execution_mode' => 'aggregate', 'dispatch_attempts' => 0, 'phase' => 'terminal', 'outcome' => 'succeeded', 'status' => 'succeeded', 'effect_may_have_applied' => 1, 'safe_to_resubmit' => 0, 'effect_resolution' => 'applied', 'finalization_incident' => 0, 'finalization_error_code' => null, ]); } Capsule::table('cloud_templates')->insert([ ['id' => 'wrong-kind-template', 'active_job_id' => 20], ['id' => 'duplicate-template-a', 'active_job_id' => 22], ['id' => 'duplicate-template-b', 'active_job_id' => 22], ]); Capsule::table('cloud_user_isos')->insert([ ['id' => 'cross-user-iso', 'user_id' => 1, 'active_job_id' => 21], ['id' => 'duplicate-iso-a', 'user_id' => 1, 'active_job_id' => 23], ['id' => 'duplicate-iso-b', 'user_id' => 1, 'active_job_id' => 23], ]); $invokeBoundary( CloudTemplateCatalogService::class, CloudTemplate::findOrFail('wrong-kind-template'), true ); $invokeBoundary( CloudUserIsoService::class, CloudUserIso::findOrFail('cross-user-iso'), true ); foreach (['duplicate-template-a', 'duplicate-template-b'] as $id) { $invokeBoundary( CloudTemplateCatalogService::class, CloudTemplate::findOrFail($id), true ); } foreach (['duplicate-iso-a', 'duplicate-iso-b'] as $id) { $invokeBoundary( CloudUserIsoService::class, CloudUserIso::findOrFail($id), true ); } // The current receipt is still nonterminal while its catalog projection is // finalized. It is the sole allowed exception; sibling and current ledger // corruption remain blockers. Capsule::table('cloud_jobs')->insert([ 'id' => 7, 'parent_job_id' => null, 'user_id' => 1, 'kind' => 'template_import', 'execution_mode' => 'aggregate', 'dispatch_attempts' => 0, 'phase' => 'awaiting_receipt', 'outcome' => null, 'status' => 'running', 'effect_may_have_applied' => null, 'safe_to_resubmit' => null, 'effect_resolution' => 'not_applicable', 'terminal_at' => null, 'finalization_incident' => 0, 'finalization_error_code' => null, ]); Capsule::table('cloud_jobs')->insert([ array_replace($agentEnvelope, [ 'id' => 107, 'parent_job_id' => 7, 'user_id' => 1, 'kind' => 'template_import_replica', 'execution_mode' => 'agent', 'dispatch_attempts' => 1, 'phase' => 'awaiting_receipt', 'outcome' => null, 'status' => 'running', 'effect_may_have_applied' => null, 'safe_to_resubmit' => null, 'effect_resolution' => 'not_applicable', 'terminal_at' => null, 'finalization_incident' => 0, 'finalization_error_code' => null, ]), array_replace($agentEnvelope, [ 'id' => 108, 'parent_job_id' => 7, 'user_id' => 1, 'kind' => 'template_import_replica', 'execution_mode' => 'agent', 'dispatch_attempts' => 1, 'phase' => 'terminal', 'outcome' => 'succeeded', 'status' => 'succeeded', 'effect_may_have_applied' => 0, 'safe_to_resubmit' => 0, 'effect_resolution' => 'applied', 'finalization_incident' => 0, 'finalization_error_code' => null, ]), ]); $settlingParent = CloudJob::findOrFail(7); $assert( !CloudJobService::aggregateParentRecoveryPending( $settlingParent, 107 ) && CloudJobService::aggregateParentRecoveryPending($settlingParent), 'The normal completing child exception was not scoped to that child.' ); Capsule::table('cloud_jobs')->where('id', 107)->update([ 'dispatch_attempts' => 2, ]); $assert(CloudJobService::aggregateParentRecoveryPending( $settlingParent, 107 ), 'A corrupt currently completing child bypassed the aggregate gate.'); $root = dirname(__DIR__); $templateSource = file_get_contents( $root . '/app/Services/Cloud/CloudTemplateCatalogService.php' ); $isoSource = file_get_contents( $root . '/app/Services/Cloud/CloudUserIsoService.php' ); $assert(substr_count( $templateSource, 'CloudDispatchLedger::mayHaveDispatched($child)' ) === 1 && substr_count( $isoSource, 'CloudDispatchLedger::mayHaveDispatched($child)' ) === 1, 'A catalog aggregate can still classify a corrupt child as undispatched.'); $assert(substr_count( $isoSource, 'self::assertMutationBoundaryLocked(' ) >= 2, 'ISO import does not reuse the unresolved-child mutation boundary.'); echo "Catalog dispatch-ledger safety checks passed.\n";