You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
490 lines
22 KiB
PHTML
490 lines
22 KiB
PHTML
|
4 weeks ago
|
<?php
|
||
|
|
declare(strict_types=1);
|
||
|
|
|
||
|
|
namespace App\Services\Cloud {
|
||
|
|
/** In-memory envelope for the isolated deadline/finalization test. */
|
||
|
|
final class CloudSecrets
|
||
|
|
{
|
||
|
|
private static array $values = [];
|
||
|
|
|
||
|
|
public static function encrypt(array $value, string $purpose): string
|
||
|
|
{
|
||
|
|
$key = bin2hex(random_bytes(16));
|
||
|
|
self::$values[$key] = $value;
|
||
|
|
return 'deadline-test:' . $key;
|
||
|
|
}
|
||
|
|
|
||
|
|
public static function decrypt(string $value, string $purpose): array
|
||
|
|
{
|
||
|
|
$key = substr($value, strlen('deadline-test:'));
|
||
|
|
if (!array_key_exists($key, self::$values)) {
|
||
|
|
throw new \RuntimeException('Missing test payload');
|
||
|
|
}
|
||
|
|
return self::$values[$key];
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
namespace {
|
||
|
|
require dirname(__DIR__) . '/vendor/autoload.php';
|
||
|
|
|
||
|
|
use App\Models\CloudJob;
|
||
|
|
use App\Services\Cloud\CloudJobService;
|
||
|
|
use App\Services\Cloud\CloudSecrets;
|
||
|
|
use App\Services\Cloud\CloudWorkerService;
|
||
|
|
use Illuminate\Database\Capsule\Manager as Capsule;
|
||
|
|
use Illuminate\Database\Schema\Blueprint;
|
||
|
|
|
||
|
|
$assert = static function (bool $condition, string $message): void {
|
||
|
|
if (!$condition) {
|
||
|
|
throw new RuntimeException($message);
|
||
|
|
}
|
||
|
|
};
|
||
|
|
|
||
|
|
$capsule = new Capsule();
|
||
|
|
$capsule->addConnection([
|
||
|
|
'driver' => 'sqlite',
|
||
|
|
'database' => ':memory:',
|
||
|
|
], 'default');
|
||
|
|
$capsule->setAsGlobal();
|
||
|
|
$capsule->bootEloquent();
|
||
|
|
$schema = $capsule->schema('default');
|
||
|
|
$schema->create('user', static function (Blueprint $table): void {
|
||
|
|
$table->increments('id');
|
||
|
|
$table->decimal('money', 12, 2)->default(0);
|
||
|
|
$table->integer('enable')->default(1);
|
||
|
|
$table->integer('is_admin')->default(0);
|
||
|
|
});
|
||
|
|
$schema->create('cloud_jobs', static function (Blueprint $table): void {
|
||
|
|
$table->increments('id');
|
||
|
|
$table->string('operation_id')->unique();
|
||
|
|
$table->string('request_idempotency_key')->unique();
|
||
|
|
$table->string('request_fingerprint');
|
||
|
|
$table->integer('parent_job_id')->nullable();
|
||
|
|
$table->integer('order_id')->nullable();
|
||
|
|
$table->integer('invoice_id')->nullable();
|
||
|
|
$table->integer('instance_id')->nullable();
|
||
|
|
$table->integer('snapshot_id')->nullable();
|
||
|
|
$table->integer('user_id');
|
||
|
|
$table->integer('host_id')->nullable();
|
||
|
|
$table->string('kind');
|
||
|
|
$table->string('execution_mode')->default('agent');
|
||
|
|
$table->string('phase');
|
||
|
|
$table->string('outcome')->nullable();
|
||
|
|
$table->string('status');
|
||
|
|
$table->text('sealed_payload');
|
||
|
|
$table->integer('attempts')->default(0);
|
||
|
|
$table->integer('dispatch_attempts')->default(0);
|
||
|
|
$table->integer('dispatch_deadline_at');
|
||
|
|
$table->integer('dispatch_started_at')->nullable();
|
||
|
|
$table->integer('agent_accepted_at')->nullable();
|
||
|
|
$table->integer('receipt_deadline_at')->nullable();
|
||
|
|
$table->integer('terminal_at')->nullable();
|
||
|
|
$table->boolean('effect_may_have_applied')->nullable();
|
||
|
|
$table->boolean('safe_to_resubmit')->nullable();
|
||
|
|
$table->string('effect_resolution')->default('not_applicable');
|
||
|
|
$table->integer('effect_resolved_at')->nullable();
|
||
|
|
$table->integer('effect_resolved_by')->nullable();
|
||
|
|
$table->string('effect_resolution_reason')->default('');
|
||
|
|
$table->string('completion_event_id')->nullable();
|
||
|
|
$table->integer('read_at')->nullable();
|
||
|
|
$table->integer('dispatched_at')->nullable();
|
||
|
|
$table->string('lease_token')->nullable();
|
||
|
|
$table->integer('lease_until')->default(0);
|
||
|
|
$table->integer('available_at')->default(0);
|
||
|
|
$table->string('error_code')->nullable();
|
||
|
|
$table->boolean('finalization_incident')->default(0);
|
||
|
|
$table->string('finalization_error_code')->nullable();
|
||
|
|
$table->text('result_json')->nullable();
|
||
|
|
$table->integer('created_at');
|
||
|
|
$table->integer('updated_at');
|
||
|
|
});
|
||
|
|
$schema->create('cloud_agent_completion_events',
|
||
|
|
static function (Blueprint $table): void {
|
||
|
|
$table->string('event_id')->primary();
|
||
|
|
$table->integer('host_id');
|
||
|
|
$table->string('operation_id');
|
||
|
|
$table->string('payload_sha256');
|
||
|
|
$table->string('outcome');
|
||
|
|
$table->boolean('effect_may_have_applied');
|
||
|
|
$table->boolean('safe_to_resubmit');
|
||
|
|
$table->string('error_code')->default('');
|
||
|
|
$table->text('result_json')->nullable();
|
||
|
|
$table->string('disposition')->default('received');
|
||
|
|
$table->integer('received_at');
|
||
|
|
$table->integer('applied_at')->nullable();
|
||
|
|
$table->unique(['host_id', 'operation_id']);
|
||
|
|
});
|
||
|
|
$schema->create('cloud_invoice_payments',
|
||
|
|
static function (Blueprint $table): void {
|
||
|
|
$table->increments('id');
|
||
|
|
$table->integer('invoice_id');
|
||
|
|
$table->integer('user_id');
|
||
|
|
$table->string('provider');
|
||
|
|
$table->string('provider_order_no');
|
||
|
|
$table->string('idempotency_key');
|
||
|
|
$table->decimal('amount', 12, 2)->default(0);
|
||
|
|
$table->string('status');
|
||
|
|
$table->integer('created_at');
|
||
|
|
$table->integer('updated_at');
|
||
|
|
$table->unique(['user_id', 'idempotency_key']);
|
||
|
|
});
|
||
|
|
|
||
|
|
Capsule::table('user')->insert([
|
||
|
|
['id' => 1, 'money' => 0, 'enable' => 1, 'is_admin' => 0],
|
||
|
|
['id' => 2, 'money' => 0, 'enable' => 1, 'is_admin' => 0],
|
||
|
|
]);
|
||
|
|
$now = time();
|
||
|
|
$insertJob = static function (int $userId, string $operationId,
|
||
|
|
string $phase, int $dispatchAttempts) use ($now): int {
|
||
|
|
return (int) Capsule::table('cloud_jobs')->insertGetId([
|
||
|
|
'operation_id' => $operationId,
|
||
|
|
'request_idempotency_key' => $operationId,
|
||
|
|
'request_fingerprint' => hash('sha256', $operationId),
|
||
|
|
'parent_job_id' => null,
|
||
|
|
'order_id' => null,
|
||
|
|
'invoice_id' => null,
|
||
|
|
'instance_id' => null,
|
||
|
|
'snapshot_id' => null,
|
||
|
|
'user_id' => $userId,
|
||
|
|
'host_id' => 7,
|
||
|
|
// An unrecognised kind exercises the generic terminal ledger path
|
||
|
|
// without requiring a business-resource fixture.
|
||
|
|
'kind' => 'test_mutation',
|
||
|
|
'execution_mode' => 'agent',
|
||
|
|
'phase' => $phase,
|
||
|
|
'outcome' => null,
|
||
|
|
'status' => $phase === 'accepted' ? 'queued' : 'running',
|
||
|
|
'sealed_payload' => CloudSecrets::encrypt([
|
||
|
|
'method' => 'POST',
|
||
|
|
'request_target' => '/v1/test',
|
||
|
|
'body' => ['operation_id' => $operationId],
|
||
|
|
'owner_id' => $userId,
|
||
|
|
], 'job-payload'),
|
||
|
|
'attempts' => $dispatchAttempts,
|
||
|
|
'dispatch_attempts' => $dispatchAttempts,
|
||
|
|
'dispatch_deadline_at' => $now - 20,
|
||
|
|
'dispatch_started_at' => $dispatchAttempts === 1
|
||
|
|
? $now - 30 : null,
|
||
|
|
'agent_accepted_at' => null,
|
||
|
|
'receipt_deadline_at' => $dispatchAttempts === 1
|
||
|
|
? $now - 10 : null,
|
||
|
|
'terminal_at' => null,
|
||
|
|
'effect_may_have_applied' => null,
|
||
|
|
'safe_to_resubmit' => null,
|
||
|
|
'effect_resolution' => 'not_applicable',
|
||
|
|
'effect_resolved_at' => null,
|
||
|
|
'effect_resolved_by' => null,
|
||
|
|
'effect_resolution_reason' => '',
|
||
|
|
'completion_event_id' => null,
|
||
|
|
'read_at' => null,
|
||
|
|
'dispatched_at' => $dispatchAttempts === 1 ? $now - 30 : null,
|
||
|
|
// Simulate a dead daemon whose stale lease extends past deadline.
|
||
|
|
'lease_token' => str_repeat('f', 32),
|
||
|
|
'lease_until' => $now + 600,
|
||
|
|
'available_at' => 0,
|
||
|
|
'error_code' => null,
|
||
|
|
'result_json' => null,
|
||
|
|
'created_at' => $now - 60,
|
||
|
|
'updated_at' => $now - 30,
|
||
|
|
]);
|
||
|
|
};
|
||
|
|
|
||
|
|
$receiptOperation = str_repeat('a', 32);
|
||
|
|
$receiptJobId = $insertJob(1, $receiptOperation, 'dispatching', 1);
|
||
|
|
Capsule::table('cloud_agent_completion_events')->insert([
|
||
|
|
'event_id' => str_repeat('e', 32),
|
||
|
|
'host_id' => 7,
|
||
|
|
'operation_id' => $receiptOperation,
|
||
|
|
'payload_sha256' => str_repeat('1', 64),
|
||
|
|
'outcome' => 'succeeded',
|
||
|
|
'effect_may_have_applied' => 0,
|
||
|
|
'safe_to_resubmit' => 0,
|
||
|
|
'error_code' => '',
|
||
|
|
'result_json' => json_encode(['confirmed' => true], JSON_THROW_ON_ERROR),
|
||
|
|
'disposition' => 'received',
|
||
|
|
'received_at' => $now - 15,
|
||
|
|
'applied_at' => null,
|
||
|
|
]);
|
||
|
|
|
||
|
|
$otherOperation = str_repeat('b', 32);
|
||
|
|
$otherJobId = $insertJob(2, $otherOperation, 'dispatching', 1);
|
||
|
|
$assert(CloudWorkerService::expireDueTasksForOwner(
|
||
|
|
1, $receiptJobId, 1
|
||
|
|
) === 1, 'The owner-scoped fallback did not claim the due task.');
|
||
|
|
|
||
|
|
$receiptJob = CloudJob::find($receiptJobId);
|
||
|
|
$otherJob = CloudJob::find($otherJobId);
|
||
|
|
$event = Capsule::table('cloud_agent_completion_events')
|
||
|
|
->where('event_id', str_repeat('e', 32))->first();
|
||
|
|
$assert($receiptJob !== null
|
||
|
|
&& (string) $receiptJob->phase === 'terminal'
|
||
|
|
&& (string) $receiptJob->outcome === 'succeeded'
|
||
|
|
&& (int) $receiptJob->dispatch_attempts === 1
|
||
|
|
&& (string) $event->disposition === 'applied',
|
||
|
|
'A receipt saved during dispatch was replaced by deadline failure.');
|
||
|
|
$assert($otherJob !== null && (string) $otherJob->phase === 'dispatching',
|
||
|
|
'Owner-scoped expiry mutated another user\'s task.');
|
||
|
|
|
||
|
|
$failureOperation = str_repeat('c', 32);
|
||
|
|
$failureJobId = $insertJob(1, $failureOperation, 'dispatching', 1);
|
||
|
|
$assert(CloudWorkerService::expireDueTasksForOwner(
|
||
|
|
1, $failureJobId, 1
|
||
|
|
) === 1, 'The receipt-less due task was not closed.');
|
||
|
|
$failed = CloudJob::find($failureJobId);
|
||
|
|
$assert($failed !== null
|
||
|
|
&& (string) $failed->phase === 'terminal'
|
||
|
|
&& (string) $failed->outcome === 'failed'
|
||
|
|
&& (bool) $failed->effect_may_have_applied
|
||
|
|
&& !(bool) $failed->safe_to_resubmit,
|
||
|
|
'A crossed dispatch boundary did not become a conservative failure.');
|
||
|
|
|
||
|
|
$incidentOperation = str_repeat('5', 32);
|
||
|
|
$incidentJobId = $insertJob(
|
||
|
|
1, $incidentOperation, 'awaiting_receipt', 1
|
||
|
|
);
|
||
|
|
Capsule::table('cloud_jobs')->where('id', $incidentJobId)->update([
|
||
|
|
'sealed_payload' => 'deadline-test:missing-payload',
|
||
|
|
]);
|
||
|
|
$assert(CloudWorkerService::expireDueTasksForOwner(
|
||
|
|
1, $incidentJobId, 1
|
||
|
|
) === 1, 'The finalization-incident fixture was not claimed.');
|
||
|
|
$incident = CloudJob::find($incidentJobId);
|
||
|
|
$assert($incident !== null
|
||
|
|
&& (string) $incident->phase === 'terminal'
|
||
|
|
&& (string) $incident->outcome === 'failed'
|
||
|
|
&& (string) $incident->status === 'failed'
|
||
|
|
&& (bool) $incident->effect_may_have_applied
|
||
|
|
&& !(bool) $incident->safe_to_resubmit
|
||
|
|
&& (string) $incident->effect_resolution === 'unknown'
|
||
|
|
&& (string) $incident->error_code === 'local_finalization_failed'
|
||
|
|
&& (bool) $incident->finalization_incident
|
||
|
|
&& (string) $incident->finalization_error_code
|
||
|
|
=== 'completion_deadline_exceeded'
|
||
|
|
&& (string) $incident->sealed_payload === '',
|
||
|
|
'A local finalizer crash left the task active, leaked its payload, or concealed the incident.');
|
||
|
|
|
||
|
|
$lateOperation = str_repeat('7', 32);
|
||
|
|
$lateJobId = $insertJob(1, $lateOperation, 'awaiting_receipt', 1);
|
||
|
|
$lateEventId = str_repeat('6', 32);
|
||
|
|
Capsule::table('cloud_agent_completion_events')->insert([
|
||
|
|
'event_id' => $lateEventId,
|
||
|
|
'host_id' => 7,
|
||
|
|
'operation_id' => $lateOperation,
|
||
|
|
'payload_sha256' => str_repeat('2', 64),
|
||
|
|
'outcome' => 'succeeded',
|
||
|
|
'effect_may_have_applied' => 0,
|
||
|
|
'safe_to_resubmit' => 0,
|
||
|
|
'error_code' => '',
|
||
|
|
'result_json' => json_encode(['confirmed' => true],
|
||
|
|
JSON_THROW_ON_ERROR),
|
||
|
|
'disposition' => 'received',
|
||
|
|
// Equal to or later than the absolute boundary is audit-only.
|
||
|
|
'received_at' => $now - 9,
|
||
|
|
'applied_at' => null,
|
||
|
|
]);
|
||
|
|
$lateJob = (new CloudWorkerService())->settleCompletionReceipt(
|
||
|
|
$lateJobId, $lateEventId
|
||
|
|
);
|
||
|
|
$lateEvent = Capsule::table('cloud_agent_completion_events')
|
||
|
|
->where('event_id', $lateEventId)->first();
|
||
|
|
$assert((string) $lateJob->phase === 'terminal'
|
||
|
|
&& (string) $lateJob->outcome === 'failed'
|
||
|
|
&& (string) $lateJob->error_code === 'operation_failed'
|
||
|
|
&& (bool) $lateJob->effect_may_have_applied
|
||
|
|
&& !(bool) $lateJob->safe_to_resubmit
|
||
|
|
&& !(bool) $lateJob->finalization_incident
|
||
|
|
&& (string) $lateEvent->disposition === 'late',
|
||
|
|
'A receipt persisted after the deadline revived or succeeded the task.');
|
||
|
|
|
||
|
|
$insertPaymentJob = static function (string $key, int $invoiceId)
|
||
|
|
use ($now): int {
|
||
|
|
$operationId = md5('payment-deadline:' . $key);
|
||
|
|
return (int) Capsule::table('cloud_jobs')->insertGetId([
|
||
|
|
'operation_id' => $operationId,
|
||
|
|
'request_idempotency_key' => $key,
|
||
|
|
'request_fingerprint' => hash('sha256', $key),
|
||
|
|
'parent_job_id' => null, 'order_id' => null,
|
||
|
|
'invoice_id' => $invoiceId, 'instance_id' => null,
|
||
|
|
'snapshot_id' => null, 'user_id' => 1, 'host_id' => null,
|
||
|
|
'kind' => 'payment_create', 'execution_mode' => 'local',
|
||
|
|
'phase' => 'dispatching', 'outcome' => null,
|
||
|
|
'status' => 'running',
|
||
|
|
'sealed_payload' => CloudSecrets::encrypt([
|
||
|
|
'input' => ['idempotency_key' => $key,
|
||
|
|
'method' => 'paytaro'],
|
||
|
|
], 'job-payload'),
|
||
|
|
'attempts' => 1, 'dispatch_attempts' => 1,
|
||
|
|
'dispatch_deadline_at' => $now - 20,
|
||
|
|
'dispatch_started_at' => $now - 30,
|
||
|
|
'agent_accepted_at' => null,
|
||
|
|
'receipt_deadline_at' => $now - 10,
|
||
|
|
'terminal_at' => null,
|
||
|
|
'effect_may_have_applied' => null,
|
||
|
|
'safe_to_resubmit' => null,
|
||
|
|
'effect_resolution' => 'not_applicable',
|
||
|
|
'effect_resolved_at' => null, 'effect_resolved_by' => null,
|
||
|
|
'effect_resolution_reason' => '',
|
||
|
|
'completion_event_id' => null, 'read_at' => null,
|
||
|
|
'dispatched_at' => $now - 30,
|
||
|
|
'lease_token' => bin2hex(random_bytes(16)),
|
||
|
|
'lease_until' => $now + 600, 'available_at' => 0,
|
||
|
|
'error_code' => null, 'result_json' => null,
|
||
|
|
'created_at' => $now - 60, 'updated_at' => $now - 30,
|
||
|
|
]);
|
||
|
|
};
|
||
|
|
|
||
|
|
$missingPaymentKey = 'payment-missing-0001';
|
||
|
|
$pendingPaymentKey = 'payment-pending-0001';
|
||
|
|
$unknownPaymentKey = 'payment-unknown-0001';
|
||
|
|
Capsule::table('cloud_invoice_payments')->insert([
|
||
|
|
[
|
||
|
|
'invoice_id' => 502, 'user_id' => 1, 'provider' => 'paytaro',
|
||
|
|
'provider_order_no' => 'provider-pending-0001',
|
||
|
|
'idempotency_key' => $pendingPaymentKey,
|
||
|
|
'amount' => 10, 'status' => 'pending',
|
||
|
|
'created_at' => $now - 30, 'updated_at' => $now - 30,
|
||
|
|
],
|
||
|
|
[
|
||
|
|
'invoice_id' => 503, 'user_id' => 1, 'provider' => 'paytaro',
|
||
|
|
'provider_order_no' => 'provider-unknown-0001',
|
||
|
|
'idempotency_key' => $unknownPaymentKey,
|
||
|
|
'amount' => 10, 'status' => 'creation_unknown',
|
||
|
|
'created_at' => $now - 30, 'updated_at' => $now - 30,
|
||
|
|
],
|
||
|
|
]);
|
||
|
|
$missingPaymentJobId = $insertPaymentJob($missingPaymentKey, 501);
|
||
|
|
$pendingPaymentJobId = $insertPaymentJob($pendingPaymentKey, 502);
|
||
|
|
$unknownPaymentJobId = $insertPaymentJob($unknownPaymentKey, 503);
|
||
|
|
foreach ([$missingPaymentJobId, $pendingPaymentJobId,
|
||
|
|
$unknownPaymentJobId] as $paymentJobId) {
|
||
|
|
$assert(CloudWorkerService::expireDueTasksForOwner(
|
||
|
|
1, $paymentJobId, 1
|
||
|
|
) === 1, 'A due local payment task was not terminalized.');
|
||
|
|
}
|
||
|
|
$missingPaymentJob = CloudJob::find($missingPaymentJobId);
|
||
|
|
$pendingPaymentJob = CloudJob::find($pendingPaymentJobId);
|
||
|
|
$unknownPaymentJob = CloudJob::find($unknownPaymentJobId);
|
||
|
|
$pendingPaymentResult = json_decode(
|
||
|
|
(string) $pendingPaymentJob->result_json, true,
|
||
|
|
flags: JSON_THROW_ON_ERROR
|
||
|
|
);
|
||
|
|
$assert($missingPaymentJob->phase === 'terminal'
|
||
|
|
&& $missingPaymentJob->outcome === 'failed'
|
||
|
|
&& $missingPaymentJob->effect_resolution === 'not_applied'
|
||
|
|
&& !(bool) $missingPaymentJob->effect_may_have_applied
|
||
|
|
&& (bool) $missingPaymentJob->safe_to_resubmit
|
||
|
|
&& $missingPaymentJob->error_code === 'payment_order_not_dispatched'
|
||
|
|
&& $pendingPaymentJob->phase === 'terminal'
|
||
|
|
&& $pendingPaymentJob->outcome === 'succeeded'
|
||
|
|
&& ($pendingPaymentResult['payment_id'] ?? null) === '1'
|
||
|
|
&& $unknownPaymentJob->phase === 'terminal'
|
||
|
|
&& $unknownPaymentJob->outcome === 'failed'
|
||
|
|
&& $unknownPaymentJob->effect_resolution === 'unknown'
|
||
|
|
&& (bool) $unknownPaymentJob->effect_may_have_applied
|
||
|
|
&& !(bool) $unknownPaymentJob->safe_to_resubmit
|
||
|
|
&& $unknownPaymentJob->error_code === 'payment_creation_unknown'
|
||
|
|
&& Capsule::table('cloud_invoice_payments')->count() === 2,
|
||
|
|
'Payment deadline fallback retried a provider POST or misclassified its durable attempt.');
|
||
|
|
|
||
|
|
$assert(CloudJobService::receiptDeadlineSeconds('action') === 960
|
||
|
|
&& CloudJobService::receiptDeadlineSeconds(
|
||
|
|
'template_import_replica') === 8100
|
||
|
|
&& CloudJobService::receiptDeadlineSeconds(
|
||
|
|
'rescue_template_import') === 8100
|
||
|
|
&& CloudJobService::receiptDeadlineSeconds(
|
||
|
|
'user_iso_import_replica') === 8100
|
||
|
|
&& CloudJobService::aggregateDeadlineSeconds(
|
||
|
|
'template_metadata_sync') === 1560
|
||
|
|
&& CloudJobService::aggregateDeadlineSeconds(
|
||
|
|
'template_import') === 8700
|
||
|
|
&& CloudJobService::aggregateDeadlineSeconds(
|
||
|
|
'user_iso_import') === 8700,
|
||
|
|
'Task deadlines lost dispatch, receipt, or aggregate finalization margin.');
|
||
|
|
|
||
|
|
$parentOperation = str_repeat('d', 32);
|
||
|
|
$parentId = (int) Capsule::table('cloud_jobs')->insertGetId([
|
||
|
|
'operation_id' => $parentOperation,
|
||
|
|
'request_idempotency_key' => $parentOperation,
|
||
|
|
'request_fingerprint' => hash('sha256', $parentOperation),
|
||
|
|
'parent_job_id' => null, 'order_id' => null, 'invoice_id' => null,
|
||
|
|
'instance_id' => null, 'snapshot_id' => null,
|
||
|
|
'user_id' => 1, 'host_id' => null, 'kind' => 'test_aggregate',
|
||
|
|
'execution_mode' => 'aggregate', 'phase' => 'awaiting_receipt',
|
||
|
|
'outcome' => null, 'status' => 'running',
|
||
|
|
'sealed_payload' => CloudSecrets::encrypt([], 'job-payload'),
|
||
|
|
'attempts' => 0, 'dispatch_attempts' => 0,
|
||
|
|
'dispatch_deadline_at' => $now - 30,
|
||
|
|
'dispatch_started_at' => null, 'agent_accepted_at' => null,
|
||
|
|
'receipt_deadline_at' => $now - 1, 'terminal_at' => null,
|
||
|
|
'effect_may_have_applied' => null, 'safe_to_resubmit' => null,
|
||
|
|
'effect_resolution' => 'not_applicable',
|
||
|
|
'effect_resolved_at' => null, 'effect_resolved_by' => null,
|
||
|
|
'effect_resolution_reason' => '', 'completion_event_id' => null,
|
||
|
|
'read_at' => null, 'dispatched_at' => null,
|
||
|
|
'lease_token' => str_repeat('9', 32), 'lease_until' => $now + 600,
|
||
|
|
'available_at' => 0, 'error_code' => null, 'result_json' => null,
|
||
|
|
'created_at' => $now - 100, 'updated_at' => $now - 50,
|
||
|
|
]);
|
||
|
|
$childOperation = str_repeat('8', 32);
|
||
|
|
$childId = (int) Capsule::table('cloud_jobs')->insertGetId([
|
||
|
|
'operation_id' => $childOperation,
|
||
|
|
'request_idempotency_key' => $childOperation,
|
||
|
|
'request_fingerprint' => hash('sha256', $childOperation),
|
||
|
|
'parent_job_id' => $parentId, 'order_id' => null,
|
||
|
|
'invoice_id' => null, 'instance_id' => null, 'snapshot_id' => null,
|
||
|
|
'user_id' => 1, 'host_id' => 7, 'kind' => 'test_mutation',
|
||
|
|
'execution_mode' => 'agent', 'phase' => 'accepted',
|
||
|
|
'outcome' => null, 'status' => 'queued',
|
||
|
|
'sealed_payload' => CloudSecrets::encrypt([
|
||
|
|
'method' => 'POST', 'request_target' => '/v1/test',
|
||
|
|
'body' => ['operation_id' => $childOperation], 'owner_id' => 1,
|
||
|
|
], 'job-payload'),
|
||
|
|
'attempts' => 0, 'dispatch_attempts' => 0,
|
||
|
|
'dispatch_deadline_at' => $now + 600,
|
||
|
|
'dispatch_started_at' => null, 'agent_accepted_at' => null,
|
||
|
|
'receipt_deadline_at' => null, 'terminal_at' => null,
|
||
|
|
'effect_may_have_applied' => null, 'safe_to_resubmit' => null,
|
||
|
|
'effect_resolution' => 'not_applicable',
|
||
|
|
'effect_resolved_at' => null, 'effect_resolved_by' => null,
|
||
|
|
'effect_resolution_reason' => '', 'completion_event_id' => null,
|
||
|
|
'read_at' => null, 'dispatched_at' => null,
|
||
|
|
'lease_token' => null, 'lease_until' => 0, 'available_at' => 0,
|
||
|
|
'error_code' => null, 'result_json' => null,
|
||
|
|
'created_at' => $now - 80, 'updated_at' => $now - 40,
|
||
|
|
]);
|
||
|
|
Capsule::unprepared('CREATE TRIGGER simulate_child_claim_cas_miss '
|
||
|
|
. 'BEFORE UPDATE OF lease_token ON cloud_jobs '
|
||
|
|
. 'WHEN OLD.id = ' . $childId . " AND OLD.phase <> 'terminal' "
|
||
|
|
. 'BEGIN SELECT RAISE(IGNORE); END');
|
||
|
|
$assert(CloudWorkerService::expireDueTasksForOwner(
|
||
|
|
1, $parentId, 10
|
||
|
|
) === 1, 'Owner fallback did not claim an expired aggregate parent.');
|
||
|
|
$parent = CloudJob::find($parentId);
|
||
|
|
$child = CloudJob::find($childId);
|
||
|
|
$assert($parent !== null && $child !== null
|
||
|
|
&& (string) $parent->phase === 'awaiting_receipt'
|
||
|
|
&& $parent->outcome === null
|
||
|
|
&& $parent->lease_token === null
|
||
|
|
&& (int) $parent->lease_until === 0
|
||
|
|
&& (string) $child->phase === 'accepted',
|
||
|
|
'A child claim CAS miss hid active work behind a terminal parent.');
|
||
|
|
|
||
|
|
Capsule::unprepared('DROP TRIGGER simulate_child_claim_cas_miss');
|
||
|
|
$assert(CloudWorkerService::expireDueTasksForOwner(
|
||
|
|
1, $parentId, 10
|
||
|
|
) === 1, 'A later fallback pass did not reclaim the aggregate parent.');
|
||
|
|
$parent = CloudJob::find($parentId);
|
||
|
|
$child = CloudJob::find($childId);
|
||
|
|
$assert($parent !== null && $child !== null
|
||
|
|
&& (string) $child->phase === 'terminal'
|
||
|
|
&& (string) $child->outcome === 'failed'
|
||
|
|
&& !(bool) $child->effect_may_have_applied
|
||
|
|
&& (bool) $child->safe_to_resubmit
|
||
|
|
&& (string) $parent->phase === 'terminal'
|
||
|
|
&& (string) $parent->outcome === 'failed',
|
||
|
|
'A later aggregate fallback did not terminalize children first.');
|
||
|
|
|
||
|
|
echo "Owner-scoped task deadline fallback tests passed.\n";
|
||
|
|
}
|