You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
virt/web/tests/CloudCatalogDispatchLedgerS...

568 lines
18 KiB
PHP

<?php
require dirname(__DIR__) . '/vendor/autoload.php';
use App\Models\CloudTemplate;
use App\Models\CloudUserIso;
use App\Models\CloudJob;
use App\Services\Cloud\CloudDispatchLedger;
use App\Services\Cloud\CloudError;
use App\Services\Cloud\CloudJobService;
use App\Services\Cloud\CloudTemplateCatalogService;
use App\Services\Cloud\CloudUserIsoService;
use Illuminate\Database\Capsule\Manager as Capsule;
use Illuminate\Database\Schema\Blueprint;
$assert = static function (bool $condition, string $message): void {
if (!$condition) {
throw new RuntimeException($message);
}
};
foreach ([
0 => false,
1 => true,
2 => true,
-1 => true,
] as $attempts => $mayHaveDispatched) {
$job = (object) ['dispatch_attempts' => $attempts];
$assert(
CloudDispatchLedger::mayHaveDispatched($job) === $mayHaveDispatched,
'The dispatch ledger did not treat only zero as proof of no dispatch.'
);
}
$capsule = new Capsule();
$capsule->addConnection([
'driver' => 'sqlite',
'database' => ':memory:',
], 'default');
$capsule->setAsGlobal();
$capsule->bootEloquent();
$capsule->schema('default')->create(
'cloud_jobs',
static function (Blueprint $table): void {
$table->increments('id');
$table->string('operation_id')->default(
'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa'
);
$table->string('request_idempotency_key')->default(
'bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb'
);
$table->string('request_fingerprint')->default(
'cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc'
);
$table->integer('parent_job_id')->nullable();
$table->integer('order_id')->nullable();
$table->integer('invoice_id')->nullable();
$table->integer('instance_id')->nullable();
$table->integer('snapshot_id')->nullable();
$table->integer('user_id')->default(1);
$table->integer('host_id')->nullable();
$table->string('kind')->default('template_import');
$table->string('execution_mode')->default('aggregate');
$table->integer('attempts')->default(0);
$table->integer('dispatch_attempts');
$table->integer('dispatch_deadline_at')->default(1);
$table->integer('dispatch_started_at')->nullable();
$table->integer('agent_accepted_at')->nullable();
$table->integer('dispatched_at')->nullable();
$table->integer('receipt_deadline_at')->nullable()->default(1);
$table->string('phase');
$table->string('outcome')->nullable();
$table->string('status')->default('succeeded');
$table->integer('terminal_at')->nullable()->default(1);
$table->string('completion_event_id')->nullable();
$table->boolean('effect_may_have_applied')->nullable();
$table->boolean('safe_to_resubmit')->nullable();
$table->string('effect_resolution');
$table->string('error_code')->nullable();
$table->text('result_json')->nullable();
$table->boolean('finalization_incident')->default(0);
$table->string('finalization_error_code')->nullable();
}
);
$agentEnvelope = [
'host_id' => 7,
'attempts' => 1,
'dispatch_deadline_at' => 1,
'dispatch_started_at' => 1,
'agent_accepted_at' => 1,
'dispatched_at' => 1,
'receipt_deadline_at' => 1,
'terminal_at' => 1,
];
$capsule->schema('default')->create(
'cloud_templates',
static function (Blueprint $table): void {
$table->string('id')->primary();
$table->integer('active_job_id')->nullable();
}
);
$capsule->schema('default')->create(
'cloud_user_isos',
static function (Blueprint $table): void {
$table->string('id')->primary();
$table->integer('user_id');
$table->integer('active_job_id')->nullable();
}
);
foreach ([
[1, 0, 'terminal', 'succeeded', 'applied'],
[2, 1, 'terminal', 'succeeded', 'applied'],
[3, 2, 'terminal', 'succeeded', 'applied'],
[4, -1, 'terminal', 'succeeded', 'applied'],
[5, 1, 'awaiting_receipt', null, 'unknown'],
] as [$id, $attempts, $phase, $outcome, $resolution]) {
Capsule::table('cloud_jobs')->insert([
'id' => $id,
'parent_job_id' => null,
'dispatch_attempts' => $attempts,
'phase' => $phase,
'outcome' => $outcome,
'status' => $outcome ?? 'running',
'effect_may_have_applied' => 0,
'safe_to_resubmit' => 0,
'effect_resolution' => $resolution,
]);
}
foreach ([1, 2] as $parentId) {
Capsule::table('cloud_jobs')->insert([
...$agentEnvelope,
'id' => 100 + $parentId,
'parent_job_id' => $parentId,
'user_id' => 1,
'kind' => 'template_import_replica',
'execution_mode' => 'agent',
'dispatch_attempts' => 1,
'phase' => 'terminal',
'outcome' => 'succeeded',
'status' => 'succeeded',
'effect_may_have_applied' => 0,
'safe_to_resubmit' => 0,
'effect_resolution' => 'applied',
'finalization_incident' => 0,
'finalization_error_code' => null,
]);
}
foreach ([
[201, 0, 'terminal', 'succeeded', 'applied'],
[202, 1, 'terminal', 'succeeded', 'applied'],
[203, 2, 'terminal', 'succeeded', 'applied'],
[204, -1, 'terminal', 'succeeded', 'applied'],
[205, 1, 'awaiting_receipt', null, 'unknown'],
] as [$id, $attempts, $phase, $outcome, $resolution]) {
Capsule::table('cloud_jobs')->insert([
'id' => $id,
'parent_job_id' => null,
'user_id' => 1,
'kind' => 'user_iso_import',
'execution_mode' => 'aggregate',
'dispatch_attempts' => $attempts,
'phase' => $phase,
'outcome' => $outcome,
'status' => $outcome ?? 'running',
'effect_may_have_applied' => 0,
'safe_to_resubmit' => 0,
'effect_resolution' => $resolution,
'finalization_incident' => 0,
'finalization_error_code' => null,
]);
}
foreach ([201, 202] as $parentId) {
Capsule::table('cloud_jobs')->insert([
...$agentEnvelope,
'id' => 100 + $parentId,
'parent_job_id' => $parentId,
'user_id' => 1,
'kind' => 'user_iso_import_replica',
'execution_mode' => 'agent',
'dispatch_attempts' => 1,
'phase' => 'terminal',
'outcome' => 'succeeded',
'status' => 'succeeded',
'effect_may_have_applied' => 0,
'safe_to_resubmit' => 0,
'effect_resolution' => 'applied',
'finalization_incident' => 0,
'finalization_error_code' => null,
]);
}
$invokeBoundary = static function (
string $service,
object $resource,
bool $shouldBlock
) use ($assert): void {
$method = new ReflectionMethod($service, 'assertMutationBoundaryLocked');
$jobId = (int) ($resource->active_job_id ?? 0);
$parent = $jobId > 0 ? CloudJob::find($jobId) : null;
$validKinds = $service === CloudTemplateCatalogService::class
? ['template_import', 'template_metadata_sync', 'template_delete']
: ['user_iso_import', 'user_iso_delete'];
if ($parent === null
|| (string) $parent->execution_mode !== 'aggregate'
|| $parent->parent_job_id !== null
|| !in_array((string) $parent->kind, $validKinds, true)) {
$parent = null;
}
try {
$method->invoke(null, $resource, $parent);
} catch (CloudError $error) {
$assert($shouldBlock && $error->httpStatus === 409,
$service . ' rejected a valid settled job boundary.');
return;
}
$assert(!$shouldBlock,
$service . ' released an unsafe prior job boundary.');
};
// This focused fixture intentionally has no replica tables. Exercise the
// parent/child dispatch-ledger predicate directly; full catalog projection
// ownership is covered by CloudCatalogActiveJobPointerRecoveryTest.
$parentSafelySettled = static function (
string $service,
object $resource,
int $jobId
): bool {
$method = new ReflectionMethod($service, 'aggregateParentSafelySettled');
$parent = CloudJob::find($jobId);
if ($parent === null) {
return false;
}
return $service === CloudTemplateCatalogService::class
? $method->invoke(null, $parent, false)
: $method->invoke(null, $resource, $parent, false);
};
foreach ([
1 => false,
2 => true,
3 => true,
4 => true,
5 => true,
999 => true,
] as $jobId => $shouldBlock) {
$template = new CloudTemplate();
$template->active_job_id = $jobId;
$assert(
$parentSafelySettled(
CloudTemplateCatalogService::class,
$template,
$jobId
) === !$shouldBlock,
'The template aggregate dispatch ledger boundary was misclassified.'
);
}
foreach ([
201 => false,
202 => true,
203 => true,
204 => true,
205 => true,
999 => true,
] as $jobId => $shouldBlock) {
$iso = new CloudUserIso();
$iso->active_job_id = $jobId;
$iso->user_id = 1;
$assert(
$parentSafelySettled(
CloudUserIsoService::class,
$iso,
$jobId
) === !$shouldBlock,
'The ISO aggregate dispatch ledger boundary was misclassified.'
);
}
// A fail-open aggregate parent may already say succeeded/applied while one of
// its Agent children still needs recovery. Both template and ISO mutation
// boundaries must follow the child fence rather than trusting the parent row.
Capsule::table('cloud_jobs')->insert([
'id' => 6,
'parent_job_id' => null,
'user_id' => 1,
'kind' => 'template_import',
'execution_mode' => 'aggregate',
'dispatch_attempts' => 0,
'phase' => 'terminal',
'outcome' => 'succeeded',
'status' => 'succeeded',
'effect_may_have_applied' => 0,
'safe_to_resubmit' => 0,
'effect_resolution' => 'applied',
'finalization_incident' => 0,
'finalization_error_code' => null,
]);
Capsule::table('cloud_jobs')->insert([
...$agentEnvelope,
'id' => 106,
'parent_job_id' => 6,
'user_id' => 1,
'kind' => 'template_import_replica',
'execution_mode' => 'agent',
'dispatch_attempts' => 2,
'phase' => 'terminal',
'outcome' => 'succeeded',
'status' => 'succeeded',
'effect_may_have_applied' => 0,
'safe_to_resubmit' => 0,
'effect_resolution' => 'applied',
'finalization_incident' => 0,
'finalization_error_code' => null,
]);
Capsule::table('cloud_jobs')->insert([
'id' => 206,
'parent_job_id' => null,
'user_id' => 1,
'kind' => 'user_iso_import',
'execution_mode' => 'aggregate',
'dispatch_attempts' => 0,
'phase' => 'terminal',
'outcome' => 'succeeded',
'status' => 'succeeded',
'effect_may_have_applied' => 0,
'safe_to_resubmit' => 0,
'effect_resolution' => 'applied',
'finalization_incident' => 0,
'finalization_error_code' => null,
]);
Capsule::table('cloud_jobs')->insert([
...$agentEnvelope,
'id' => 306,
'parent_job_id' => 206,
'user_id' => 1,
'kind' => 'user_iso_import_replica',
'execution_mode' => 'agent',
'dispatch_attempts' => 2,
'phase' => 'terminal',
'outcome' => 'succeeded',
'status' => 'succeeded',
'effect_may_have_applied' => 0,
'safe_to_resubmit' => 0,
'effect_resolution' => 'applied',
'finalization_incident' => 0,
'finalization_error_code' => null,
]);
$failOpenTemplate = new CloudTemplate();
$failOpenTemplate->active_job_id = 6;
$failOpenIso = new CloudUserIso();
$failOpenIso->active_job_id = 206;
$failOpenIso->user_id = 1;
$assertBothBoundaries = static function (bool $shouldBlock) use (
$assert,
$parentSafelySettled,
$failOpenTemplate,
$failOpenIso
): void {
$assert(
$parentSafelySettled(
CloudTemplateCatalogService::class,
$failOpenTemplate,
6
) === !$shouldBlock,
'The template child dispatch ledger boundary was misclassified.'
);
$assert(
$parentSafelySettled(
CloudUserIsoService::class,
$failOpenIso,
206
) === !$shouldBlock,
'The ISO child dispatch ledger boundary was misclassified.'
);
};
$assertBothBoundaries(true);
Capsule::table('cloud_jobs')->whereIn('id', [106, 306])->update([
'dispatch_attempts' => 1,
'finalization_incident' => 1,
'finalization_error_code' => 'recovery_dispatch_ledger_conflict',
]);
$assertBothBoundaries(true);
Capsule::table('cloud_jobs')->whereIn('id', [106, 306])->update([
'dispatch_attempts' => 0,
'outcome' => 'failed',
'status' => 'failed',
'effect_may_have_applied' => 0,
'safe_to_resubmit' => 0,
'effect_resolution' => 'not_applied',
'finalization_incident' => 1,
'finalization_error_code' => 'recovery_ledger_conflict',
]);
$assertBothBoundaries(true);
Capsule::table('cloud_jobs')->whereIn('id', [106, 306])->update([
'dispatch_attempts' => 1,
'effect_may_have_applied' => 1,
'effect_resolution' => 'unknown',
'finalization_incident' => 0,
'finalization_error_code' => null,
]);
$assertBothBoundaries(true);
Capsule::table('cloud_jobs')->whereIn('id', [106, 306])->update([
'outcome' => 'succeeded',
'status' => 'succeeded',
'effect_may_have_applied' => 0,
'safe_to_resubmit' => 0,
'effect_resolution' => 'applied',
]);
$assertBothBoundaries(false);
// A settled-looking row is not a valid catalog boundary unless its active job
// is the exact aggregate root for that resource type and identity, and exactly
// one main record owns that job.
foreach ([
[20, 'user_iso_import', 1],
[21, 'user_iso_import', 2],
[22, 'template_import', 1],
[23, 'user_iso_import', 1],
] as [$id, $kind, $userId]) {
Capsule::table('cloud_jobs')->insert([
'id' => $id,
'parent_job_id' => null,
'user_id' => $userId,
'kind' => $kind,
'execution_mode' => 'aggregate',
'dispatch_attempts' => 0,
'phase' => 'terminal',
'outcome' => 'succeeded',
'status' => 'succeeded',
'effect_may_have_applied' => 1,
'safe_to_resubmit' => 0,
'effect_resolution' => 'applied',
'finalization_incident' => 0,
'finalization_error_code' => null,
]);
}
Capsule::table('cloud_templates')->insert([
['id' => 'wrong-kind-template', 'active_job_id' => 20],
['id' => 'duplicate-template-a', 'active_job_id' => 22],
['id' => 'duplicate-template-b', 'active_job_id' => 22],
]);
Capsule::table('cloud_user_isos')->insert([
['id' => 'cross-user-iso', 'user_id' => 1, 'active_job_id' => 21],
['id' => 'duplicate-iso-a', 'user_id' => 1, 'active_job_id' => 23],
['id' => 'duplicate-iso-b', 'user_id' => 1, 'active_job_id' => 23],
]);
$invokeBoundary(
CloudTemplateCatalogService::class,
CloudTemplate::findOrFail('wrong-kind-template'),
true
);
$invokeBoundary(
CloudUserIsoService::class,
CloudUserIso::findOrFail('cross-user-iso'),
true
);
foreach (['duplicate-template-a', 'duplicate-template-b'] as $id) {
$invokeBoundary(
CloudTemplateCatalogService::class,
CloudTemplate::findOrFail($id),
true
);
}
foreach (['duplicate-iso-a', 'duplicate-iso-b'] as $id) {
$invokeBoundary(
CloudUserIsoService::class,
CloudUserIso::findOrFail($id),
true
);
}
// The current receipt is still nonterminal while its catalog projection is
// finalized. It is the sole allowed exception; sibling and current ledger
// corruption remain blockers.
Capsule::table('cloud_jobs')->insert([
'id' => 7,
'parent_job_id' => null,
'user_id' => 1,
'kind' => 'template_import',
'execution_mode' => 'aggregate',
'dispatch_attempts' => 0,
'phase' => 'awaiting_receipt',
'outcome' => null,
'status' => 'running',
'effect_may_have_applied' => null,
'safe_to_resubmit' => null,
'effect_resolution' => 'not_applicable',
'terminal_at' => null,
'finalization_incident' => 0,
'finalization_error_code' => null,
]);
Capsule::table('cloud_jobs')->insert([
array_replace($agentEnvelope, [
'id' => 107,
'parent_job_id' => 7,
'user_id' => 1,
'kind' => 'template_import_replica',
'execution_mode' => 'agent',
'dispatch_attempts' => 1,
'phase' => 'awaiting_receipt',
'outcome' => null,
'status' => 'running',
'effect_may_have_applied' => null,
'safe_to_resubmit' => null,
'effect_resolution' => 'not_applicable',
'terminal_at' => null,
'finalization_incident' => 0,
'finalization_error_code' => null,
]),
array_replace($agentEnvelope, [
'id' => 108,
'parent_job_id' => 7,
'user_id' => 1,
'kind' => 'template_import_replica',
'execution_mode' => 'agent',
'dispatch_attempts' => 1,
'phase' => 'terminal',
'outcome' => 'succeeded',
'status' => 'succeeded',
'effect_may_have_applied' => 0,
'safe_to_resubmit' => 0,
'effect_resolution' => 'applied',
'finalization_incident' => 0,
'finalization_error_code' => null,
]),
]);
$settlingParent = CloudJob::findOrFail(7);
$assert(
!CloudJobService::aggregateParentRecoveryPending(
$settlingParent,
107
)
&& CloudJobService::aggregateParentRecoveryPending($settlingParent),
'The normal completing child exception was not scoped to that child.'
);
Capsule::table('cloud_jobs')->where('id', 107)->update([
'dispatch_attempts' => 2,
]);
$assert(CloudJobService::aggregateParentRecoveryPending(
$settlingParent,
107
), 'A corrupt currently completing child bypassed the aggregate gate.');
$root = dirname(__DIR__);
$templateSource = file_get_contents(
$root . '/app/Services/Cloud/CloudTemplateCatalogService.php'
);
$isoSource = file_get_contents(
$root . '/app/Services/Cloud/CloudUserIsoService.php'
);
$assert(substr_count(
$templateSource,
'CloudDispatchLedger::mayHaveDispatched($child)'
) === 1 && substr_count(
$isoSource,
'CloudDispatchLedger::mayHaveDispatched($child)'
) === 1,
'A catalog aggregate can still classify a corrupt child as undispatched.');
$assert(substr_count(
$isoSource,
'self::assertMutationBoundaryLocked('
) >= 2,
'ISO import does not reuse the unresolved-child mutation boundary.');
echo "Catalog dispatch-ledger safety checks passed.\n";