You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
virt/web/tests/CloudTaskDeadlineFallbackTe...

658 lines
30 KiB
PHTML

<?php
declare(strict_types=1);
namespace App\Services\Cloud {
/** In-memory envelope for the isolated deadline/finalization test. */
final class CloudSecrets
{
private static array $values = [];
public static function encrypt(array $value, string $purpose): string
{
$key = bin2hex(random_bytes(16));
self::$values[$key] = $value;
return 'deadline-test:' . $key;
}
public static function decrypt(string $value, string $purpose): array
{
$key = substr($value, strlen('deadline-test:'));
if (!array_key_exists($key, self::$values)) {
throw new \RuntimeException('Missing test payload');
}
return self::$values[$key];
}
}
}
namespace {
require dirname(__DIR__) . '/vendor/autoload.php';
use App\Models\CloudJob;
use App\Services\Cloud\CloudAgentClient;
use App\Services\Cloud\CloudJobService;
use App\Services\Cloud\CloudSecrets;
use App\Services\Cloud\CloudWorkerService;
use GuzzleHttp\Client;
use Illuminate\Database\Capsule\Manager as Capsule;
use Illuminate\Database\Schema\Blueprint;
$assert = static function (bool $condition, string $message): void {
if (!$condition) {
throw new RuntimeException($message);
}
};
$capsule = new Capsule();
$capsule->addConnection([
'driver' => 'sqlite',
'database' => ':memory:',
], 'default');
$capsule->setAsGlobal();
$capsule->bootEloquent();
$schema = $capsule->schema('default');
$schema->create('user', static function (Blueprint $table): void {
$table->increments('id');
$table->decimal('money', 12, 2)->default(0);
$table->integer('enable')->default(1);
$table->integer('is_admin')->default(0);
});
$schema->create('cloud_jobs', static function (Blueprint $table): void {
$table->increments('id');
$table->string('operation_id')->unique();
$table->string('request_idempotency_key')->unique();
$table->string('request_fingerprint');
$table->integer('parent_job_id')->nullable();
$table->integer('order_id')->nullable();
$table->integer('invoice_id')->nullable();
$table->integer('instance_id')->nullable();
$table->integer('snapshot_id')->nullable();
$table->integer('user_id');
$table->integer('host_id')->nullable();
$table->string('kind');
$table->string('execution_mode')->default('agent');
$table->string('phase');
$table->string('outcome')->nullable();
$table->string('status');
$table->text('sealed_payload');
$table->integer('attempts')->default(0);
$table->integer('dispatch_attempts')->default(0);
$table->integer('dispatch_deadline_at');
$table->integer('dispatch_started_at')->nullable();
$table->integer('agent_accepted_at')->nullable();
$table->integer('receipt_deadline_at')->nullable();
$table->integer('terminal_at')->nullable();
$table->boolean('effect_may_have_applied')->nullable();
$table->boolean('safe_to_resubmit')->nullable();
$table->string('effect_resolution')->default('not_applicable');
$table->integer('effect_resolved_at')->nullable();
$table->integer('effect_resolved_by')->nullable();
$table->string('effect_resolution_reason')->default('');
$table->string('completion_event_id')->nullable();
$table->integer('read_at')->nullable();
$table->integer('dispatched_at')->nullable();
$table->string('lease_token')->nullable();
$table->integer('lease_until')->default(0);
$table->integer('available_at')->default(0);
$table->string('error_code')->nullable();
$table->boolean('finalization_incident')->default(0);
$table->string('finalization_error_code')->nullable();
$table->text('result_json')->nullable();
$table->integer('created_at');
$table->integer('updated_at');
});
$schema->create('cloud_host_storage_configurations',
static function (Blueprint $table): void {
$table->integer('host_id')->primary();
$table->integer('revision')->default(0);
foreach (['managed', 'template', 'seed', 'user_iso', 'rescue']
as $role) {
$table->text($role . '_directory');
}
$table->string('state')->default('applying');
$table->string('operation_id')->nullable()->unique();
$table->integer('job_id')->nullable();
$table->integer('created_at');
$table->integer('updated_at');
});
$schema->create('cloud_agent_completion_events',
static function (Blueprint $table): void {
$table->string('event_id')->primary();
$table->integer('host_id');
$table->string('operation_id');
$table->string('payload_sha256');
$table->string('outcome');
$table->boolean('effect_may_have_applied');
$table->boolean('safe_to_resubmit');
$table->string('error_code')->default('');
$table->text('result_json')->nullable();
$table->string('disposition')->default('received');
$table->integer('received_at');
$table->integer('applied_at')->nullable();
$table->unique(['host_id', 'operation_id']);
});
$schema->create('cloud_invoice_payments',
static function (Blueprint $table): void {
$table->increments('id');
$table->integer('invoice_id');
$table->integer('user_id');
$table->string('provider');
$table->string('provider_order_no');
$table->string('idempotency_key');
$table->decimal('amount', 12, 2)->default(0);
$table->string('status');
$table->integer('created_at');
$table->integer('updated_at');
$table->unique(['user_id', 'idempotency_key']);
});
Capsule::table('user')->insert([
['id' => 1, 'money' => 0, 'enable' => 1, 'is_admin' => 0],
['id' => 2, 'money' => 0, 'enable' => 1, 'is_admin' => 0],
]);
$now = time();
$insertJob = static function (int $userId, string $operationId,
string $phase, int $dispatchAttempts) use ($now): int {
return (int) Capsule::table('cloud_jobs')->insertGetId([
'operation_id' => $operationId,
'request_idempotency_key' => $operationId,
'request_fingerprint' => hash('sha256', $operationId),
'parent_job_id' => null,
'order_id' => null,
'invoice_id' => null,
'instance_id' => null,
'snapshot_id' => null,
'user_id' => $userId,
'host_id' => 7,
// An unrecognised kind exercises the generic terminal ledger path
// without requiring a business-resource fixture.
'kind' => 'test_mutation',
'execution_mode' => 'agent',
'phase' => $phase,
'outcome' => null,
'status' => $phase === 'accepted' ? 'queued' : 'running',
'sealed_payload' => CloudSecrets::encrypt([
'method' => 'POST',
'request_target' => '/v1/test',
'body' => ['operation_id' => $operationId],
'owner_id' => $userId,
], 'job-payload'),
'attempts' => $dispatchAttempts,
'dispatch_attempts' => $dispatchAttempts,
'dispatch_deadline_at' => $now - 20,
'dispatch_started_at' => $dispatchAttempts === 1
? $now - 30 : null,
'agent_accepted_at' => null,
'receipt_deadline_at' => $dispatchAttempts === 1
? $now - 10 : null,
'terminal_at' => null,
'effect_may_have_applied' => null,
'safe_to_resubmit' => null,
'effect_resolution' => 'not_applicable',
'effect_resolved_at' => null,
'effect_resolved_by' => null,
'effect_resolution_reason' => '',
'completion_event_id' => null,
'read_at' => null,
'dispatched_at' => $dispatchAttempts === 1 ? $now - 30 : null,
// Simulate a dead daemon whose stale lease extends past deadline.
'lease_token' => str_repeat('f', 32),
'lease_until' => $now + 600,
'available_at' => 0,
'error_code' => null,
'result_json' => null,
'created_at' => $now - 60,
'updated_at' => $now - 30,
]);
};
$receiptOperation = str_repeat('a', 32);
$receiptJobId = $insertJob(1, $receiptOperation, 'dispatching', 1);
Capsule::table('cloud_agent_completion_events')->insert([
'event_id' => str_repeat('e', 32),
'host_id' => 7,
'operation_id' => $receiptOperation,
'payload_sha256' => str_repeat('1', 64),
'outcome' => 'succeeded',
'effect_may_have_applied' => 0,
'safe_to_resubmit' => 0,
'error_code' => '',
'result_json' => json_encode(['confirmed' => true], JSON_THROW_ON_ERROR),
'disposition' => 'received',
'received_at' => $now - 15,
'applied_at' => null,
]);
$otherOperation = str_repeat('b', 32);
$otherJobId = $insertJob(2, $otherOperation, 'dispatching', 1);
$assert(CloudWorkerService::expireDueTasksForOwner(
1, $receiptJobId, 1
) === 1, 'The owner-scoped fallback did not claim the due task.');
$receiptJob = CloudJob::find($receiptJobId);
$otherJob = CloudJob::find($otherJobId);
$event = Capsule::table('cloud_agent_completion_events')
->where('event_id', str_repeat('e', 32))->first();
$assert($receiptJob !== null
&& (string) $receiptJob->phase === 'terminal'
&& (string) $receiptJob->outcome === 'succeeded'
&& (int) $receiptJob->dispatch_attempts === 1
&& (string) $event->disposition === 'applied',
'A receipt saved during dispatch was replaced by deadline failure.');
$assert($otherJob !== null && (string) $otherJob->phase === 'dispatching',
'Owner-scoped expiry mutated another user\'s task.');
$failureOperation = str_repeat('c', 32);
$failureJobId = $insertJob(1, $failureOperation, 'dispatching', 1);
$assert(CloudWorkerService::expireDueTasksForOwner(
1, $failureJobId, 1
) === 1, 'The receipt-less due task was not closed.');
$failed = CloudJob::find($failureJobId);
$assert($failed !== null
&& (string) $failed->phase === 'terminal'
&& (string) $failed->outcome === 'failed'
&& (bool) $failed->effect_may_have_applied
&& !(bool) $failed->safe_to_resubmit,
'A crossed dispatch boundary did not become a conservative failure.');
// Deploying the shorter storage window must also bound tasks which already
// persisted the old 960-second receipt deadline. No data rewrite is needed:
// the effective deadline is the earlier of the two absolute boundaries.
$legacyStorageOperation = str_repeat('3', 32);
$legacyStorageJobId = $insertJob(
1, $legacyStorageOperation, 'awaiting_receipt', 1
);
Capsule::table('cloud_jobs')->where('id', $legacyStorageJobId)->update([
'kind' => 'host_storage_configure',
'dispatched_at' => $now - 181,
'dispatch_started_at' => $now - 181,
'receipt_deadline_at' => $now + 779,
]);
Capsule::table('cloud_host_storage_configurations')->insert([
'host_id' => 7, 'revision' => 1,
'managed_directory' => '/home/libvirt/kvm',
'template_directory' => '/home/libvirt/images',
'seed_directory' => '/home/libvirt/seed',
'user_iso_directory' => '/home/libvirt/iso',
'rescue_directory' => '/home/libvirt/rescue-iso',
'state' => 'applying', 'operation_id' => $legacyStorageOperation,
'job_id' => $legacyStorageJobId,
'created_at' => $now - 200, 'updated_at' => $now - 181,
]);
$legacyStorage = CloudJob::find($legacyStorageJobId);
$assert(CloudJobService::effectiveReceiptDeadline($legacyStorage)
=== $now - 1
&& CloudJobService::receiptDeadlineReached($legacyStorage, $now),
'The persisted storage deadline was not bounded by dispatch plus 180 seconds.');
$assert(CloudWorkerService::expireDueTasksForOwner(
1, $legacyStorageJobId, 1
) === 1, 'An old 960-second storage task remained active after 180 seconds.');
$legacyStorage = CloudJob::find($legacyStorageJobId);
$legacyStorageRow = Capsule::table(
'cloud_host_storage_configurations'
)->where('host_id', 7)->first();
$assert((string) $legacyStorage->phase === 'terminal'
&& (string) $legacyStorage->outcome === 'failed'
&& (string) $legacyStorageRow->state === 'failed',
'Storage deadline fallback did not close the task and its projection.');
// Receipt classification must use exactly the same effective boundary as
// the expiry query, otherwise a late success could revive a timed-out task.
$lateStorageOperation = str_repeat('2', 32);
$lateStorageJobId = $insertJob(
1, $lateStorageOperation, 'awaiting_receipt', 1
);
Capsule::table('cloud_jobs')->where('id', $lateStorageJobId)->update([
'kind' => 'host_storage_configure',
'dispatched_at' => $now - 181,
'dispatch_started_at' => $now - 181,
'receipt_deadline_at' => $now + 779,
]);
Capsule::table('cloud_host_storage_configurations')
->where('host_id', 7)->update([
'state' => 'applying', 'operation_id' => $lateStorageOperation,
'job_id' => $lateStorageJobId, 'updated_at' => $now,
]);
$lateStorageEventId = str_repeat('4', 32);
Capsule::table('cloud_agent_completion_events')->insert([
'event_id' => $lateStorageEventId, 'host_id' => 7,
'operation_id' => $lateStorageOperation,
'payload_sha256' => str_repeat('3', 64),
'outcome' => 'succeeded', 'effect_may_have_applied' => 0,
'safe_to_resubmit' => 0, 'error_code' => '',
'result_json' => json_encode(['configured' => true],
JSON_THROW_ON_ERROR),
'disposition' => 'received', 'received_at' => $now - 1,
'applied_at' => null,
]);
$lateStorage = (new CloudWorkerService())->settleCompletionReceipt(
$lateStorageJobId, $lateStorageEventId
);
$lateStorageEvent = Capsule::table('cloud_agent_completion_events')
->where('event_id', $lateStorageEventId)->first();
$assert((string) $lateStorage->phase === 'terminal'
&& (string) $lateStorage->outcome === 'failed'
&& (string) $lateStorageEvent->disposition === 'late',
'A storage receipt at the effective deadline was not classified as late.');
$acceptedStorageOperation = str_repeat('1', 32);
$acceptedStorageJobId = $insertJob(
1, $acceptedStorageOperation, 'accepted', 0
);
Capsule::table('cloud_jobs')->where('id', $acceptedStorageJobId)->update([
'kind' => 'host_storage_configure',
'created_at' => $now - 61,
'dispatch_deadline_at' => $now + 239,
]);
Capsule::table('cloud_host_storage_configurations')
->where('host_id', 7)->update([
'state' => 'applying', 'operation_id' => $acceptedStorageOperation,
'job_id' => $acceptedStorageJobId, 'updated_at' => $now,
]);
$acceptedStorage = CloudJob::find($acceptedStorageJobId);
$assert(CloudJobService::effectiveDispatchDeadline($acceptedStorage)
=== $now - 1
&& CloudJobService::activeDeadlineReached($acceptedStorage, $now),
'The persisted storage dispatch deadline was not bounded to 60 seconds.');
$assert(CloudWorkerService::expireDueTasksForOwner(
1, $acceptedStorageJobId, 1
) === 1, 'An old 300-second accepted storage task remained active.');
$acceptedStorage = CloudJob::find($acceptedStorageJobId);
$assert((string) $acceptedStorage->phase === 'terminal'
&& (string) $acceptedStorage->outcome === 'failed'
&& !(bool) $acceptedStorage->effect_may_have_applied
&& (bool) $acceptedStorage->safe_to_resubmit,
'An undispatched storage deadline did not become a safe failure.');
// The normal worker tick must run local deadline convergence before it can
// claim an old storage task. Its injected Agent transport is a tripwire:
// a request here would prove the former 300-second deadline was used.
$tickStorageOperation = '0123456789abcdef0123456789abcdef';
$tickStorageJobId = $insertJob(
1, $tickStorageOperation, 'accepted', 0
);
Capsule::table('cloud_jobs')->where('id', $tickStorageJobId)->update([
'kind' => 'host_storage_configure',
'created_at' => time() - 61,
'dispatch_deadline_at' => time() + 239,
]);
Capsule::table('cloud_host_storage_configurations')
->where('host_id', 7)->update([
'state' => 'applying', 'operation_id' => $tickStorageOperation,
'job_id' => $tickStorageJobId, 'updated_at' => time(),
]);
$agentCalls = 0;
$transport = new Client(['handler' => static function () use (
&$agentCalls
) {
$agentCalls++;
throw new RuntimeException('Expired storage task reached Agent');
}]);
$tripwireAgent = new CloudAgentClient(
'7', str_repeat('a', 32), $transport, 120.0,
'http://127.0.0.1:8001/internal/agent/request',
str_repeat('b', 32)
);
$assert((new CloudWorkerService($tripwireAgent))->tick() === true,
'Worker tick did not converge the expired storage task.');
$tickStorage = CloudJob::find($tickStorageJobId);
$assert($agentCalls === 0
&& (string) $tickStorage->phase === 'terminal'
&& (string) $tickStorage->outcome === 'failed'
&& (int) $tickStorage->dispatch_attempts === 0,
'Worker dispatched an old storage task after its effective 60-second boundary.');
$incidentOperation = str_repeat('5', 32);
$incidentJobId = $insertJob(
1, $incidentOperation, 'awaiting_receipt', 1
);
Capsule::table('cloud_jobs')->where('id', $incidentJobId)->update([
'sealed_payload' => 'deadline-test:missing-payload',
]);
$assert(CloudWorkerService::expireDueTasksForOwner(
1, $incidentJobId, 1
) === 1, 'The finalization-incident fixture was not claimed.');
$incident = CloudJob::find($incidentJobId);
$assert($incident !== null
&& (string) $incident->phase === 'terminal'
&& (string) $incident->outcome === 'failed'
&& (string) $incident->status === 'failed'
&& (bool) $incident->effect_may_have_applied
&& !(bool) $incident->safe_to_resubmit
&& (string) $incident->effect_resolution === 'unknown'
&& (string) $incident->error_code === 'local_finalization_failed'
&& (bool) $incident->finalization_incident
&& (string) $incident->finalization_error_code
=== 'completion_deadline_exceeded'
&& (string) $incident->sealed_payload === '',
'A local finalizer crash left the task active, leaked its payload, or concealed the incident.');
$lateOperation = str_repeat('7', 32);
$lateJobId = $insertJob(1, $lateOperation, 'awaiting_receipt', 1);
$lateEventId = str_repeat('6', 32);
Capsule::table('cloud_agent_completion_events')->insert([
'event_id' => $lateEventId,
'host_id' => 7,
'operation_id' => $lateOperation,
'payload_sha256' => str_repeat('2', 64),
'outcome' => 'succeeded',
'effect_may_have_applied' => 0,
'safe_to_resubmit' => 0,
'error_code' => '',
'result_json' => json_encode(['confirmed' => true],
JSON_THROW_ON_ERROR),
'disposition' => 'received',
// Equal to or later than the absolute boundary is audit-only.
'received_at' => $now - 9,
'applied_at' => null,
]);
$lateJob = (new CloudWorkerService())->settleCompletionReceipt(
$lateJobId, $lateEventId
);
$lateEvent = Capsule::table('cloud_agent_completion_events')
->where('event_id', $lateEventId)->first();
$assert((string) $lateJob->phase === 'terminal'
&& (string) $lateJob->outcome === 'failed'
&& (string) $lateJob->error_code === 'operation_failed'
&& (bool) $lateJob->effect_may_have_applied
&& !(bool) $lateJob->safe_to_resubmit
&& !(bool) $lateJob->finalization_incident
&& (string) $lateEvent->disposition === 'late',
'A receipt persisted after the deadline revived or succeeded the task.');
$insertPaymentJob = static function (string $key, int $invoiceId)
use ($now): int {
$operationId = md5('payment-deadline:' . $key);
return (int) Capsule::table('cloud_jobs')->insertGetId([
'operation_id' => $operationId,
'request_idempotency_key' => $key,
'request_fingerprint' => hash('sha256', $key),
'parent_job_id' => null, 'order_id' => null,
'invoice_id' => $invoiceId, 'instance_id' => null,
'snapshot_id' => null, 'user_id' => 1, 'host_id' => null,
'kind' => 'payment_create', 'execution_mode' => 'local',
'phase' => 'dispatching', 'outcome' => null,
'status' => 'running',
'sealed_payload' => CloudSecrets::encrypt([
'input' => ['idempotency_key' => $key,
'method' => 'paytaro'],
], 'job-payload'),
'attempts' => 1, 'dispatch_attempts' => 1,
'dispatch_deadline_at' => $now - 20,
'dispatch_started_at' => $now - 30,
'agent_accepted_at' => null,
'receipt_deadline_at' => $now - 10,
'terminal_at' => null,
'effect_may_have_applied' => null,
'safe_to_resubmit' => null,
'effect_resolution' => 'not_applicable',
'effect_resolved_at' => null, 'effect_resolved_by' => null,
'effect_resolution_reason' => '',
'completion_event_id' => null, 'read_at' => null,
'dispatched_at' => $now - 30,
'lease_token' => bin2hex(random_bytes(16)),
'lease_until' => $now + 600, 'available_at' => 0,
'error_code' => null, 'result_json' => null,
'created_at' => $now - 60, 'updated_at' => $now - 30,
]);
};
$missingPaymentKey = 'payment-missing-0001';
$pendingPaymentKey = 'payment-pending-0001';
$unknownPaymentKey = 'payment-unknown-0001';
Capsule::table('cloud_invoice_payments')->insert([
[
'invoice_id' => 502, 'user_id' => 1, 'provider' => 'paytaro',
'provider_order_no' => 'provider-pending-0001',
'idempotency_key' => $pendingPaymentKey,
'amount' => 10, 'status' => 'pending',
'created_at' => $now - 30, 'updated_at' => $now - 30,
],
[
'invoice_id' => 503, 'user_id' => 1, 'provider' => 'paytaro',
'provider_order_no' => 'provider-unknown-0001',
'idempotency_key' => $unknownPaymentKey,
'amount' => 10, 'status' => 'creation_unknown',
'created_at' => $now - 30, 'updated_at' => $now - 30,
],
]);
$missingPaymentJobId = $insertPaymentJob($missingPaymentKey, 501);
$pendingPaymentJobId = $insertPaymentJob($pendingPaymentKey, 502);
$unknownPaymentJobId = $insertPaymentJob($unknownPaymentKey, 503);
foreach ([$missingPaymentJobId, $pendingPaymentJobId,
$unknownPaymentJobId] as $paymentJobId) {
$assert(CloudWorkerService::expireDueTasksForOwner(
1, $paymentJobId, 1
) === 1, 'A due local payment task was not terminalized.');
}
$missingPaymentJob = CloudJob::find($missingPaymentJobId);
$pendingPaymentJob = CloudJob::find($pendingPaymentJobId);
$unknownPaymentJob = CloudJob::find($unknownPaymentJobId);
$pendingPaymentResult = json_decode(
(string) $pendingPaymentJob->result_json, true,
flags: JSON_THROW_ON_ERROR
);
$assert($missingPaymentJob->phase === 'terminal'
&& $missingPaymentJob->outcome === 'failed'
&& $missingPaymentJob->effect_resolution === 'not_applied'
&& !(bool) $missingPaymentJob->effect_may_have_applied
&& (bool) $missingPaymentJob->safe_to_resubmit
&& $missingPaymentJob->error_code === 'payment_order_not_dispatched'
&& $pendingPaymentJob->phase === 'terminal'
&& $pendingPaymentJob->outcome === 'succeeded'
&& ($pendingPaymentResult['payment_id'] ?? null) === '1'
&& $unknownPaymentJob->phase === 'terminal'
&& $unknownPaymentJob->outcome === 'failed'
&& $unknownPaymentJob->effect_resolution === 'unknown'
&& (bool) $unknownPaymentJob->effect_may_have_applied
&& !(bool) $unknownPaymentJob->safe_to_resubmit
&& $unknownPaymentJob->error_code === 'payment_creation_unknown'
&& Capsule::table('cloud_invoice_payments')->count() === 2,
'Payment deadline fallback retried a provider POST or misclassified its durable attempt.');
$assert(CloudJobService::dispatchDeadlineSeconds('action') === 300
&& CloudJobService::dispatchDeadlineSeconds(
'host_storage_configure') === 60
&& CloudJobService::receiptDeadlineSeconds('action') === 960
&& CloudJobService::receiptDeadlineSeconds(
'host_storage_configure') === 180
&& CloudJobService::receiptDeadlineSeconds(
'template_import_replica') === 8100
&& CloudJobService::receiptDeadlineSeconds(
'rescue_template_import') === 8100
&& CloudJobService::receiptDeadlineSeconds(
'user_iso_import_replica') === 8100
&& CloudJobService::aggregateDeadlineSeconds(
'template_metadata_sync') === 1560
&& CloudJobService::aggregateDeadlineSeconds(
'template_import') === 8700
&& CloudJobService::aggregateDeadlineSeconds(
'user_iso_import') === 8700,
'Task deadlines lost dispatch, receipt, or aggregate finalization margin.');
$parentOperation = str_repeat('d', 32);
$parentId = (int) Capsule::table('cloud_jobs')->insertGetId([
'operation_id' => $parentOperation,
'request_idempotency_key' => $parentOperation,
'request_fingerprint' => hash('sha256', $parentOperation),
'parent_job_id' => null, 'order_id' => null, 'invoice_id' => null,
'instance_id' => null, 'snapshot_id' => null,
'user_id' => 1, 'host_id' => null, 'kind' => 'test_aggregate',
'execution_mode' => 'aggregate', 'phase' => 'awaiting_receipt',
'outcome' => null, 'status' => 'running',
'sealed_payload' => CloudSecrets::encrypt([], 'job-payload'),
'attempts' => 0, 'dispatch_attempts' => 0,
'dispatch_deadline_at' => $now - 30,
'dispatch_started_at' => null, 'agent_accepted_at' => null,
'receipt_deadline_at' => $now - 1, 'terminal_at' => null,
'effect_may_have_applied' => null, 'safe_to_resubmit' => null,
'effect_resolution' => 'not_applicable',
'effect_resolved_at' => null, 'effect_resolved_by' => null,
'effect_resolution_reason' => '', 'completion_event_id' => null,
'read_at' => null, 'dispatched_at' => null,
'lease_token' => str_repeat('9', 32), 'lease_until' => $now + 600,
'available_at' => 0, 'error_code' => null, 'result_json' => null,
'created_at' => $now - 100, 'updated_at' => $now - 50,
]);
$childOperation = str_repeat('8', 32);
$childId = (int) Capsule::table('cloud_jobs')->insertGetId([
'operation_id' => $childOperation,
'request_idempotency_key' => $childOperation,
'request_fingerprint' => hash('sha256', $childOperation),
'parent_job_id' => $parentId, 'order_id' => null,
'invoice_id' => null, 'instance_id' => null, 'snapshot_id' => null,
'user_id' => 1, 'host_id' => 7, 'kind' => 'test_mutation',
'execution_mode' => 'agent', 'phase' => 'accepted',
'outcome' => null, 'status' => 'queued',
'sealed_payload' => CloudSecrets::encrypt([
'method' => 'POST', 'request_target' => '/v1/test',
'body' => ['operation_id' => $childOperation], 'owner_id' => 1,
], 'job-payload'),
'attempts' => 0, 'dispatch_attempts' => 0,
'dispatch_deadline_at' => $now + 600,
'dispatch_started_at' => null, 'agent_accepted_at' => null,
'receipt_deadline_at' => null, 'terminal_at' => null,
'effect_may_have_applied' => null, 'safe_to_resubmit' => null,
'effect_resolution' => 'not_applicable',
'effect_resolved_at' => null, 'effect_resolved_by' => null,
'effect_resolution_reason' => '', 'completion_event_id' => null,
'read_at' => null, 'dispatched_at' => null,
'lease_token' => null, 'lease_until' => 0, 'available_at' => 0,
'error_code' => null, 'result_json' => null,
'created_at' => $now - 80, 'updated_at' => $now - 40,
]);
Capsule::unprepared('CREATE TRIGGER simulate_child_claim_cas_miss '
. 'BEFORE UPDATE OF lease_token ON cloud_jobs '
. 'WHEN OLD.id = ' . $childId . " AND OLD.phase <> 'terminal' "
. 'BEGIN SELECT RAISE(IGNORE); END');
$assert(CloudWorkerService::expireDueTasksForOwner(
1, $parentId, 10
) === 1, 'Owner fallback did not claim an expired aggregate parent.');
$parent = CloudJob::find($parentId);
$child = CloudJob::find($childId);
$assert($parent !== null && $child !== null
&& (string) $parent->phase === 'awaiting_receipt'
&& $parent->outcome === null
&& $parent->lease_token === null
&& (int) $parent->lease_until === 0
&& (string) $child->phase === 'accepted',
'A child claim CAS miss hid active work behind a terminal parent.');
Capsule::unprepared('DROP TRIGGER simulate_child_claim_cas_miss');
$assert(CloudWorkerService::expireDueTasksForOwner(
1, $parentId, 10
) === 1, 'A later fallback pass did not reclaim the aggregate parent.');
$parent = CloudJob::find($parentId);
$child = CloudJob::find($childId);
$assert($parent !== null && $child !== null
&& (string) $child->phase === 'terminal'
&& (string) $child->outcome === 'failed'
&& !(bool) $child->effect_may_have_applied
&& (bool) $child->safe_to_resubmit
&& (string) $parent->phase === 'terminal'
&& (string) $parent->outcome === 'failed',
'A later aggregate fallback did not terminalize children first.');
echo "Owner-scoped task deadline fallback tests passed.\n";
}