You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
658 lines
30 KiB
PHP
658 lines
30 KiB
PHP
<?php
|
|
declare(strict_types=1);
|
|
|
|
namespace App\Services\Cloud {
|
|
/** In-memory envelope for the isolated deadline/finalization test. */
|
|
final class CloudSecrets
|
|
{
|
|
private static array $values = [];
|
|
|
|
public static function encrypt(array $value, string $purpose): string
|
|
{
|
|
$key = bin2hex(random_bytes(16));
|
|
self::$values[$key] = $value;
|
|
return 'deadline-test:' . $key;
|
|
}
|
|
|
|
public static function decrypt(string $value, string $purpose): array
|
|
{
|
|
$key = substr($value, strlen('deadline-test:'));
|
|
if (!array_key_exists($key, self::$values)) {
|
|
throw new \RuntimeException('Missing test payload');
|
|
}
|
|
return self::$values[$key];
|
|
}
|
|
}
|
|
}
|
|
|
|
namespace {
|
|
require dirname(__DIR__) . '/vendor/autoload.php';
|
|
|
|
use App\Models\CloudJob;
|
|
use App\Services\Cloud\CloudAgentClient;
|
|
use App\Services\Cloud\CloudJobService;
|
|
use App\Services\Cloud\CloudSecrets;
|
|
use App\Services\Cloud\CloudWorkerService;
|
|
use GuzzleHttp\Client;
|
|
use Illuminate\Database\Capsule\Manager as Capsule;
|
|
use Illuminate\Database\Schema\Blueprint;
|
|
|
|
$assert = static function (bool $condition, string $message): void {
|
|
if (!$condition) {
|
|
throw new RuntimeException($message);
|
|
}
|
|
};
|
|
|
|
$capsule = new Capsule();
|
|
$capsule->addConnection([
|
|
'driver' => 'sqlite',
|
|
'database' => ':memory:',
|
|
], 'default');
|
|
$capsule->setAsGlobal();
|
|
$capsule->bootEloquent();
|
|
$schema = $capsule->schema('default');
|
|
$schema->create('user', static function (Blueprint $table): void {
|
|
$table->increments('id');
|
|
$table->decimal('money', 12, 2)->default(0);
|
|
$table->integer('enable')->default(1);
|
|
$table->integer('is_admin')->default(0);
|
|
});
|
|
$schema->create('cloud_jobs', static function (Blueprint $table): void {
|
|
$table->increments('id');
|
|
$table->string('operation_id')->unique();
|
|
$table->string('request_idempotency_key')->unique();
|
|
$table->string('request_fingerprint');
|
|
$table->integer('parent_job_id')->nullable();
|
|
$table->integer('order_id')->nullable();
|
|
$table->integer('invoice_id')->nullable();
|
|
$table->integer('instance_id')->nullable();
|
|
$table->integer('snapshot_id')->nullable();
|
|
$table->integer('user_id');
|
|
$table->integer('host_id')->nullable();
|
|
$table->string('kind');
|
|
$table->string('execution_mode')->default('agent');
|
|
$table->string('phase');
|
|
$table->string('outcome')->nullable();
|
|
$table->string('status');
|
|
$table->text('sealed_payload');
|
|
$table->integer('attempts')->default(0);
|
|
$table->integer('dispatch_attempts')->default(0);
|
|
$table->integer('dispatch_deadline_at');
|
|
$table->integer('dispatch_started_at')->nullable();
|
|
$table->integer('agent_accepted_at')->nullable();
|
|
$table->integer('receipt_deadline_at')->nullable();
|
|
$table->integer('terminal_at')->nullable();
|
|
$table->boolean('effect_may_have_applied')->nullable();
|
|
$table->boolean('safe_to_resubmit')->nullable();
|
|
$table->string('effect_resolution')->default('not_applicable');
|
|
$table->integer('effect_resolved_at')->nullable();
|
|
$table->integer('effect_resolved_by')->nullable();
|
|
$table->string('effect_resolution_reason')->default('');
|
|
$table->string('completion_event_id')->nullable();
|
|
$table->integer('read_at')->nullable();
|
|
$table->integer('dispatched_at')->nullable();
|
|
$table->string('lease_token')->nullable();
|
|
$table->integer('lease_until')->default(0);
|
|
$table->integer('available_at')->default(0);
|
|
$table->string('error_code')->nullable();
|
|
$table->boolean('finalization_incident')->default(0);
|
|
$table->string('finalization_error_code')->nullable();
|
|
$table->text('result_json')->nullable();
|
|
$table->integer('created_at');
|
|
$table->integer('updated_at');
|
|
});
|
|
$schema->create('cloud_host_storage_configurations',
|
|
static function (Blueprint $table): void {
|
|
$table->integer('host_id')->primary();
|
|
$table->integer('revision')->default(0);
|
|
foreach (['managed', 'template', 'seed', 'user_iso', 'rescue']
|
|
as $role) {
|
|
$table->text($role . '_directory');
|
|
}
|
|
$table->string('state')->default('applying');
|
|
$table->string('operation_id')->nullable()->unique();
|
|
$table->integer('job_id')->nullable();
|
|
$table->integer('created_at');
|
|
$table->integer('updated_at');
|
|
});
|
|
$schema->create('cloud_agent_completion_events',
|
|
static function (Blueprint $table): void {
|
|
$table->string('event_id')->primary();
|
|
$table->integer('host_id');
|
|
$table->string('operation_id');
|
|
$table->string('payload_sha256');
|
|
$table->string('outcome');
|
|
$table->boolean('effect_may_have_applied');
|
|
$table->boolean('safe_to_resubmit');
|
|
$table->string('error_code')->default('');
|
|
$table->text('result_json')->nullable();
|
|
$table->string('disposition')->default('received');
|
|
$table->integer('received_at');
|
|
$table->integer('applied_at')->nullable();
|
|
$table->unique(['host_id', 'operation_id']);
|
|
});
|
|
$schema->create('cloud_invoice_payments',
|
|
static function (Blueprint $table): void {
|
|
$table->increments('id');
|
|
$table->integer('invoice_id');
|
|
$table->integer('user_id');
|
|
$table->string('provider');
|
|
$table->string('provider_order_no');
|
|
$table->string('idempotency_key');
|
|
$table->decimal('amount', 12, 2)->default(0);
|
|
$table->string('status');
|
|
$table->integer('created_at');
|
|
$table->integer('updated_at');
|
|
$table->unique(['user_id', 'idempotency_key']);
|
|
});
|
|
|
|
Capsule::table('user')->insert([
|
|
['id' => 1, 'money' => 0, 'enable' => 1, 'is_admin' => 0],
|
|
['id' => 2, 'money' => 0, 'enable' => 1, 'is_admin' => 0],
|
|
]);
|
|
$now = time();
|
|
$insertJob = static function (int $userId, string $operationId,
|
|
string $phase, int $dispatchAttempts) use ($now): int {
|
|
return (int) Capsule::table('cloud_jobs')->insertGetId([
|
|
'operation_id' => $operationId,
|
|
'request_idempotency_key' => $operationId,
|
|
'request_fingerprint' => hash('sha256', $operationId),
|
|
'parent_job_id' => null,
|
|
'order_id' => null,
|
|
'invoice_id' => null,
|
|
'instance_id' => null,
|
|
'snapshot_id' => null,
|
|
'user_id' => $userId,
|
|
'host_id' => 7,
|
|
// An unrecognised kind exercises the generic terminal ledger path
|
|
// without requiring a business-resource fixture.
|
|
'kind' => 'test_mutation',
|
|
'execution_mode' => 'agent',
|
|
'phase' => $phase,
|
|
'outcome' => null,
|
|
'status' => $phase === 'accepted' ? 'queued' : 'running',
|
|
'sealed_payload' => CloudSecrets::encrypt([
|
|
'method' => 'POST',
|
|
'request_target' => '/v1/test',
|
|
'body' => ['operation_id' => $operationId],
|
|
'owner_id' => $userId,
|
|
], 'job-payload'),
|
|
'attempts' => $dispatchAttempts,
|
|
'dispatch_attempts' => $dispatchAttempts,
|
|
'dispatch_deadline_at' => $now - 20,
|
|
'dispatch_started_at' => $dispatchAttempts === 1
|
|
? $now - 30 : null,
|
|
'agent_accepted_at' => null,
|
|
'receipt_deadline_at' => $dispatchAttempts === 1
|
|
? $now - 10 : null,
|
|
'terminal_at' => null,
|
|
'effect_may_have_applied' => null,
|
|
'safe_to_resubmit' => null,
|
|
'effect_resolution' => 'not_applicable',
|
|
'effect_resolved_at' => null,
|
|
'effect_resolved_by' => null,
|
|
'effect_resolution_reason' => '',
|
|
'completion_event_id' => null,
|
|
'read_at' => null,
|
|
'dispatched_at' => $dispatchAttempts === 1 ? $now - 30 : null,
|
|
// Simulate a dead daemon whose stale lease extends past deadline.
|
|
'lease_token' => str_repeat('f', 32),
|
|
'lease_until' => $now + 600,
|
|
'available_at' => 0,
|
|
'error_code' => null,
|
|
'result_json' => null,
|
|
'created_at' => $now - 60,
|
|
'updated_at' => $now - 30,
|
|
]);
|
|
};
|
|
|
|
$receiptOperation = str_repeat('a', 32);
|
|
$receiptJobId = $insertJob(1, $receiptOperation, 'dispatching', 1);
|
|
Capsule::table('cloud_agent_completion_events')->insert([
|
|
'event_id' => str_repeat('e', 32),
|
|
'host_id' => 7,
|
|
'operation_id' => $receiptOperation,
|
|
'payload_sha256' => str_repeat('1', 64),
|
|
'outcome' => 'succeeded',
|
|
'effect_may_have_applied' => 0,
|
|
'safe_to_resubmit' => 0,
|
|
'error_code' => '',
|
|
'result_json' => json_encode(['confirmed' => true], JSON_THROW_ON_ERROR),
|
|
'disposition' => 'received',
|
|
'received_at' => $now - 15,
|
|
'applied_at' => null,
|
|
]);
|
|
|
|
$otherOperation = str_repeat('b', 32);
|
|
$otherJobId = $insertJob(2, $otherOperation, 'dispatching', 1);
|
|
$assert(CloudWorkerService::expireDueTasksForOwner(
|
|
1, $receiptJobId, 1
|
|
) === 1, 'The owner-scoped fallback did not claim the due task.');
|
|
|
|
$receiptJob = CloudJob::find($receiptJobId);
|
|
$otherJob = CloudJob::find($otherJobId);
|
|
$event = Capsule::table('cloud_agent_completion_events')
|
|
->where('event_id', str_repeat('e', 32))->first();
|
|
$assert($receiptJob !== null
|
|
&& (string) $receiptJob->phase === 'terminal'
|
|
&& (string) $receiptJob->outcome === 'succeeded'
|
|
&& (int) $receiptJob->dispatch_attempts === 1
|
|
&& (string) $event->disposition === 'applied',
|
|
'A receipt saved during dispatch was replaced by deadline failure.');
|
|
$assert($otherJob !== null && (string) $otherJob->phase === 'dispatching',
|
|
'Owner-scoped expiry mutated another user\'s task.');
|
|
|
|
$failureOperation = str_repeat('c', 32);
|
|
$failureJobId = $insertJob(1, $failureOperation, 'dispatching', 1);
|
|
$assert(CloudWorkerService::expireDueTasksForOwner(
|
|
1, $failureJobId, 1
|
|
) === 1, 'The receipt-less due task was not closed.');
|
|
$failed = CloudJob::find($failureJobId);
|
|
$assert($failed !== null
|
|
&& (string) $failed->phase === 'terminal'
|
|
&& (string) $failed->outcome === 'failed'
|
|
&& (bool) $failed->effect_may_have_applied
|
|
&& !(bool) $failed->safe_to_resubmit,
|
|
'A crossed dispatch boundary did not become a conservative failure.');
|
|
|
|
// Deploying the shorter storage window must also bound tasks which already
|
|
// persisted the old 960-second receipt deadline. No data rewrite is needed:
|
|
// the effective deadline is the earlier of the two absolute boundaries.
|
|
$legacyStorageOperation = str_repeat('3', 32);
|
|
$legacyStorageJobId = $insertJob(
|
|
1, $legacyStorageOperation, 'awaiting_receipt', 1
|
|
);
|
|
Capsule::table('cloud_jobs')->where('id', $legacyStorageJobId)->update([
|
|
'kind' => 'host_storage_configure',
|
|
'dispatched_at' => $now - 181,
|
|
'dispatch_started_at' => $now - 181,
|
|
'receipt_deadline_at' => $now + 779,
|
|
]);
|
|
Capsule::table('cloud_host_storage_configurations')->insert([
|
|
'host_id' => 7, 'revision' => 1,
|
|
'managed_directory' => '/home/libvirt/kvm',
|
|
'template_directory' => '/home/libvirt/images',
|
|
'seed_directory' => '/home/libvirt/seed',
|
|
'user_iso_directory' => '/home/libvirt/iso',
|
|
'rescue_directory' => '/home/libvirt/rescue-iso',
|
|
'state' => 'applying', 'operation_id' => $legacyStorageOperation,
|
|
'job_id' => $legacyStorageJobId,
|
|
'created_at' => $now - 200, 'updated_at' => $now - 181,
|
|
]);
|
|
$legacyStorage = CloudJob::find($legacyStorageJobId);
|
|
$assert(CloudJobService::effectiveReceiptDeadline($legacyStorage)
|
|
=== $now - 1
|
|
&& CloudJobService::receiptDeadlineReached($legacyStorage, $now),
|
|
'The persisted storage deadline was not bounded by dispatch plus 180 seconds.');
|
|
$assert(CloudWorkerService::expireDueTasksForOwner(
|
|
1, $legacyStorageJobId, 1
|
|
) === 1, 'An old 960-second storage task remained active after 180 seconds.');
|
|
$legacyStorage = CloudJob::find($legacyStorageJobId);
|
|
$legacyStorageRow = Capsule::table(
|
|
'cloud_host_storage_configurations'
|
|
)->where('host_id', 7)->first();
|
|
$assert((string) $legacyStorage->phase === 'terminal'
|
|
&& (string) $legacyStorage->outcome === 'failed'
|
|
&& (string) $legacyStorageRow->state === 'failed',
|
|
'Storage deadline fallback did not close the task and its projection.');
|
|
|
|
// Receipt classification must use exactly the same effective boundary as
|
|
// the expiry query, otherwise a late success could revive a timed-out task.
|
|
$lateStorageOperation = str_repeat('2', 32);
|
|
$lateStorageJobId = $insertJob(
|
|
1, $lateStorageOperation, 'awaiting_receipt', 1
|
|
);
|
|
Capsule::table('cloud_jobs')->where('id', $lateStorageJobId)->update([
|
|
'kind' => 'host_storage_configure',
|
|
'dispatched_at' => $now - 181,
|
|
'dispatch_started_at' => $now - 181,
|
|
'receipt_deadline_at' => $now + 779,
|
|
]);
|
|
Capsule::table('cloud_host_storage_configurations')
|
|
->where('host_id', 7)->update([
|
|
'state' => 'applying', 'operation_id' => $lateStorageOperation,
|
|
'job_id' => $lateStorageJobId, 'updated_at' => $now,
|
|
]);
|
|
$lateStorageEventId = str_repeat('4', 32);
|
|
Capsule::table('cloud_agent_completion_events')->insert([
|
|
'event_id' => $lateStorageEventId, 'host_id' => 7,
|
|
'operation_id' => $lateStorageOperation,
|
|
'payload_sha256' => str_repeat('3', 64),
|
|
'outcome' => 'succeeded', 'effect_may_have_applied' => 0,
|
|
'safe_to_resubmit' => 0, 'error_code' => '',
|
|
'result_json' => json_encode(['configured' => true],
|
|
JSON_THROW_ON_ERROR),
|
|
'disposition' => 'received', 'received_at' => $now - 1,
|
|
'applied_at' => null,
|
|
]);
|
|
$lateStorage = (new CloudWorkerService())->settleCompletionReceipt(
|
|
$lateStorageJobId, $lateStorageEventId
|
|
);
|
|
$lateStorageEvent = Capsule::table('cloud_agent_completion_events')
|
|
->where('event_id', $lateStorageEventId)->first();
|
|
$assert((string) $lateStorage->phase === 'terminal'
|
|
&& (string) $lateStorage->outcome === 'failed'
|
|
&& (string) $lateStorageEvent->disposition === 'late',
|
|
'A storage receipt at the effective deadline was not classified as late.');
|
|
|
|
$acceptedStorageOperation = str_repeat('1', 32);
|
|
$acceptedStorageJobId = $insertJob(
|
|
1, $acceptedStorageOperation, 'accepted', 0
|
|
);
|
|
Capsule::table('cloud_jobs')->where('id', $acceptedStorageJobId)->update([
|
|
'kind' => 'host_storage_configure',
|
|
'created_at' => $now - 61,
|
|
'dispatch_deadline_at' => $now + 239,
|
|
]);
|
|
Capsule::table('cloud_host_storage_configurations')
|
|
->where('host_id', 7)->update([
|
|
'state' => 'applying', 'operation_id' => $acceptedStorageOperation,
|
|
'job_id' => $acceptedStorageJobId, 'updated_at' => $now,
|
|
]);
|
|
$acceptedStorage = CloudJob::find($acceptedStorageJobId);
|
|
$assert(CloudJobService::effectiveDispatchDeadline($acceptedStorage)
|
|
=== $now - 1
|
|
&& CloudJobService::activeDeadlineReached($acceptedStorage, $now),
|
|
'The persisted storage dispatch deadline was not bounded to 60 seconds.');
|
|
$assert(CloudWorkerService::expireDueTasksForOwner(
|
|
1, $acceptedStorageJobId, 1
|
|
) === 1, 'An old 300-second accepted storage task remained active.');
|
|
$acceptedStorage = CloudJob::find($acceptedStorageJobId);
|
|
$assert((string) $acceptedStorage->phase === 'terminal'
|
|
&& (string) $acceptedStorage->outcome === 'failed'
|
|
&& !(bool) $acceptedStorage->effect_may_have_applied
|
|
&& (bool) $acceptedStorage->safe_to_resubmit,
|
|
'An undispatched storage deadline did not become a safe failure.');
|
|
|
|
// The normal worker tick must run local deadline convergence before it can
|
|
// claim an old storage task. Its injected Agent transport is a tripwire:
|
|
// a request here would prove the former 300-second deadline was used.
|
|
$tickStorageOperation = '0123456789abcdef0123456789abcdef';
|
|
$tickStorageJobId = $insertJob(
|
|
1, $tickStorageOperation, 'accepted', 0
|
|
);
|
|
Capsule::table('cloud_jobs')->where('id', $tickStorageJobId)->update([
|
|
'kind' => 'host_storage_configure',
|
|
'created_at' => time() - 61,
|
|
'dispatch_deadline_at' => time() + 239,
|
|
]);
|
|
Capsule::table('cloud_host_storage_configurations')
|
|
->where('host_id', 7)->update([
|
|
'state' => 'applying', 'operation_id' => $tickStorageOperation,
|
|
'job_id' => $tickStorageJobId, 'updated_at' => time(),
|
|
]);
|
|
$agentCalls = 0;
|
|
$transport = new Client(['handler' => static function () use (
|
|
&$agentCalls
|
|
) {
|
|
$agentCalls++;
|
|
throw new RuntimeException('Expired storage task reached Agent');
|
|
}]);
|
|
$tripwireAgent = new CloudAgentClient(
|
|
'7', str_repeat('a', 32), $transport, 120.0,
|
|
'http://127.0.0.1:8001/internal/agent/request',
|
|
str_repeat('b', 32)
|
|
);
|
|
$assert((new CloudWorkerService($tripwireAgent))->tick() === true,
|
|
'Worker tick did not converge the expired storage task.');
|
|
$tickStorage = CloudJob::find($tickStorageJobId);
|
|
$assert($agentCalls === 0
|
|
&& (string) $tickStorage->phase === 'terminal'
|
|
&& (string) $tickStorage->outcome === 'failed'
|
|
&& (int) $tickStorage->dispatch_attempts === 0,
|
|
'Worker dispatched an old storage task after its effective 60-second boundary.');
|
|
|
|
$incidentOperation = str_repeat('5', 32);
|
|
$incidentJobId = $insertJob(
|
|
1, $incidentOperation, 'awaiting_receipt', 1
|
|
);
|
|
Capsule::table('cloud_jobs')->where('id', $incidentJobId)->update([
|
|
'sealed_payload' => 'deadline-test:missing-payload',
|
|
]);
|
|
$assert(CloudWorkerService::expireDueTasksForOwner(
|
|
1, $incidentJobId, 1
|
|
) === 1, 'The finalization-incident fixture was not claimed.');
|
|
$incident = CloudJob::find($incidentJobId);
|
|
$assert($incident !== null
|
|
&& (string) $incident->phase === 'terminal'
|
|
&& (string) $incident->outcome === 'failed'
|
|
&& (string) $incident->status === 'failed'
|
|
&& (bool) $incident->effect_may_have_applied
|
|
&& !(bool) $incident->safe_to_resubmit
|
|
&& (string) $incident->effect_resolution === 'unknown'
|
|
&& (string) $incident->error_code === 'local_finalization_failed'
|
|
&& (bool) $incident->finalization_incident
|
|
&& (string) $incident->finalization_error_code
|
|
=== 'completion_deadline_exceeded'
|
|
&& (string) $incident->sealed_payload === '',
|
|
'A local finalizer crash left the task active, leaked its payload, or concealed the incident.');
|
|
|
|
$lateOperation = str_repeat('7', 32);
|
|
$lateJobId = $insertJob(1, $lateOperation, 'awaiting_receipt', 1);
|
|
$lateEventId = str_repeat('6', 32);
|
|
Capsule::table('cloud_agent_completion_events')->insert([
|
|
'event_id' => $lateEventId,
|
|
'host_id' => 7,
|
|
'operation_id' => $lateOperation,
|
|
'payload_sha256' => str_repeat('2', 64),
|
|
'outcome' => 'succeeded',
|
|
'effect_may_have_applied' => 0,
|
|
'safe_to_resubmit' => 0,
|
|
'error_code' => '',
|
|
'result_json' => json_encode(['confirmed' => true],
|
|
JSON_THROW_ON_ERROR),
|
|
'disposition' => 'received',
|
|
// Equal to or later than the absolute boundary is audit-only.
|
|
'received_at' => $now - 9,
|
|
'applied_at' => null,
|
|
]);
|
|
$lateJob = (new CloudWorkerService())->settleCompletionReceipt(
|
|
$lateJobId, $lateEventId
|
|
);
|
|
$lateEvent = Capsule::table('cloud_agent_completion_events')
|
|
->where('event_id', $lateEventId)->first();
|
|
$assert((string) $lateJob->phase === 'terminal'
|
|
&& (string) $lateJob->outcome === 'failed'
|
|
&& (string) $lateJob->error_code === 'operation_failed'
|
|
&& (bool) $lateJob->effect_may_have_applied
|
|
&& !(bool) $lateJob->safe_to_resubmit
|
|
&& !(bool) $lateJob->finalization_incident
|
|
&& (string) $lateEvent->disposition === 'late',
|
|
'A receipt persisted after the deadline revived or succeeded the task.');
|
|
|
|
$insertPaymentJob = static function (string $key, int $invoiceId)
|
|
use ($now): int {
|
|
$operationId = md5('payment-deadline:' . $key);
|
|
return (int) Capsule::table('cloud_jobs')->insertGetId([
|
|
'operation_id' => $operationId,
|
|
'request_idempotency_key' => $key,
|
|
'request_fingerprint' => hash('sha256', $key),
|
|
'parent_job_id' => null, 'order_id' => null,
|
|
'invoice_id' => $invoiceId, 'instance_id' => null,
|
|
'snapshot_id' => null, 'user_id' => 1, 'host_id' => null,
|
|
'kind' => 'payment_create', 'execution_mode' => 'local',
|
|
'phase' => 'dispatching', 'outcome' => null,
|
|
'status' => 'running',
|
|
'sealed_payload' => CloudSecrets::encrypt([
|
|
'input' => ['idempotency_key' => $key,
|
|
'method' => 'paytaro'],
|
|
], 'job-payload'),
|
|
'attempts' => 1, 'dispatch_attempts' => 1,
|
|
'dispatch_deadline_at' => $now - 20,
|
|
'dispatch_started_at' => $now - 30,
|
|
'agent_accepted_at' => null,
|
|
'receipt_deadline_at' => $now - 10,
|
|
'terminal_at' => null,
|
|
'effect_may_have_applied' => null,
|
|
'safe_to_resubmit' => null,
|
|
'effect_resolution' => 'not_applicable',
|
|
'effect_resolved_at' => null, 'effect_resolved_by' => null,
|
|
'effect_resolution_reason' => '',
|
|
'completion_event_id' => null, 'read_at' => null,
|
|
'dispatched_at' => $now - 30,
|
|
'lease_token' => bin2hex(random_bytes(16)),
|
|
'lease_until' => $now + 600, 'available_at' => 0,
|
|
'error_code' => null, 'result_json' => null,
|
|
'created_at' => $now - 60, 'updated_at' => $now - 30,
|
|
]);
|
|
};
|
|
|
|
$missingPaymentKey = 'payment-missing-0001';
|
|
$pendingPaymentKey = 'payment-pending-0001';
|
|
$unknownPaymentKey = 'payment-unknown-0001';
|
|
Capsule::table('cloud_invoice_payments')->insert([
|
|
[
|
|
'invoice_id' => 502, 'user_id' => 1, 'provider' => 'paytaro',
|
|
'provider_order_no' => 'provider-pending-0001',
|
|
'idempotency_key' => $pendingPaymentKey,
|
|
'amount' => 10, 'status' => 'pending',
|
|
'created_at' => $now - 30, 'updated_at' => $now - 30,
|
|
],
|
|
[
|
|
'invoice_id' => 503, 'user_id' => 1, 'provider' => 'paytaro',
|
|
'provider_order_no' => 'provider-unknown-0001',
|
|
'idempotency_key' => $unknownPaymentKey,
|
|
'amount' => 10, 'status' => 'creation_unknown',
|
|
'created_at' => $now - 30, 'updated_at' => $now - 30,
|
|
],
|
|
]);
|
|
$missingPaymentJobId = $insertPaymentJob($missingPaymentKey, 501);
|
|
$pendingPaymentJobId = $insertPaymentJob($pendingPaymentKey, 502);
|
|
$unknownPaymentJobId = $insertPaymentJob($unknownPaymentKey, 503);
|
|
foreach ([$missingPaymentJobId, $pendingPaymentJobId,
|
|
$unknownPaymentJobId] as $paymentJobId) {
|
|
$assert(CloudWorkerService::expireDueTasksForOwner(
|
|
1, $paymentJobId, 1
|
|
) === 1, 'A due local payment task was not terminalized.');
|
|
}
|
|
$missingPaymentJob = CloudJob::find($missingPaymentJobId);
|
|
$pendingPaymentJob = CloudJob::find($pendingPaymentJobId);
|
|
$unknownPaymentJob = CloudJob::find($unknownPaymentJobId);
|
|
$pendingPaymentResult = json_decode(
|
|
(string) $pendingPaymentJob->result_json, true,
|
|
flags: JSON_THROW_ON_ERROR
|
|
);
|
|
$assert($missingPaymentJob->phase === 'terminal'
|
|
&& $missingPaymentJob->outcome === 'failed'
|
|
&& $missingPaymentJob->effect_resolution === 'not_applied'
|
|
&& !(bool) $missingPaymentJob->effect_may_have_applied
|
|
&& (bool) $missingPaymentJob->safe_to_resubmit
|
|
&& $missingPaymentJob->error_code === 'payment_order_not_dispatched'
|
|
&& $pendingPaymentJob->phase === 'terminal'
|
|
&& $pendingPaymentJob->outcome === 'succeeded'
|
|
&& ($pendingPaymentResult['payment_id'] ?? null) === '1'
|
|
&& $unknownPaymentJob->phase === 'terminal'
|
|
&& $unknownPaymentJob->outcome === 'failed'
|
|
&& $unknownPaymentJob->effect_resolution === 'unknown'
|
|
&& (bool) $unknownPaymentJob->effect_may_have_applied
|
|
&& !(bool) $unknownPaymentJob->safe_to_resubmit
|
|
&& $unknownPaymentJob->error_code === 'payment_creation_unknown'
|
|
&& Capsule::table('cloud_invoice_payments')->count() === 2,
|
|
'Payment deadline fallback retried a provider POST or misclassified its durable attempt.');
|
|
|
|
$assert(CloudJobService::dispatchDeadlineSeconds('action') === 300
|
|
&& CloudJobService::dispatchDeadlineSeconds(
|
|
'host_storage_configure') === 60
|
|
&& CloudJobService::receiptDeadlineSeconds('action') === 960
|
|
&& CloudJobService::receiptDeadlineSeconds(
|
|
'host_storage_configure') === 180
|
|
&& CloudJobService::receiptDeadlineSeconds(
|
|
'template_import_replica') === 8100
|
|
&& CloudJobService::receiptDeadlineSeconds(
|
|
'rescue_template_import') === 8100
|
|
&& CloudJobService::receiptDeadlineSeconds(
|
|
'user_iso_import_replica') === 8100
|
|
&& CloudJobService::aggregateDeadlineSeconds(
|
|
'template_metadata_sync') === 1560
|
|
&& CloudJobService::aggregateDeadlineSeconds(
|
|
'template_import') === 8700
|
|
&& CloudJobService::aggregateDeadlineSeconds(
|
|
'user_iso_import') === 8700,
|
|
'Task deadlines lost dispatch, receipt, or aggregate finalization margin.');
|
|
|
|
$parentOperation = str_repeat('d', 32);
|
|
$parentId = (int) Capsule::table('cloud_jobs')->insertGetId([
|
|
'operation_id' => $parentOperation,
|
|
'request_idempotency_key' => $parentOperation,
|
|
'request_fingerprint' => hash('sha256', $parentOperation),
|
|
'parent_job_id' => null, 'order_id' => null, 'invoice_id' => null,
|
|
'instance_id' => null, 'snapshot_id' => null,
|
|
'user_id' => 1, 'host_id' => null, 'kind' => 'test_aggregate',
|
|
'execution_mode' => 'aggregate', 'phase' => 'awaiting_receipt',
|
|
'outcome' => null, 'status' => 'running',
|
|
'sealed_payload' => CloudSecrets::encrypt([], 'job-payload'),
|
|
'attempts' => 0, 'dispatch_attempts' => 0,
|
|
'dispatch_deadline_at' => $now - 30,
|
|
'dispatch_started_at' => null, 'agent_accepted_at' => null,
|
|
'receipt_deadline_at' => $now - 1, 'terminal_at' => null,
|
|
'effect_may_have_applied' => null, 'safe_to_resubmit' => null,
|
|
'effect_resolution' => 'not_applicable',
|
|
'effect_resolved_at' => null, 'effect_resolved_by' => null,
|
|
'effect_resolution_reason' => '', 'completion_event_id' => null,
|
|
'read_at' => null, 'dispatched_at' => null,
|
|
'lease_token' => str_repeat('9', 32), 'lease_until' => $now + 600,
|
|
'available_at' => 0, 'error_code' => null, 'result_json' => null,
|
|
'created_at' => $now - 100, 'updated_at' => $now - 50,
|
|
]);
|
|
$childOperation = str_repeat('8', 32);
|
|
$childId = (int) Capsule::table('cloud_jobs')->insertGetId([
|
|
'operation_id' => $childOperation,
|
|
'request_idempotency_key' => $childOperation,
|
|
'request_fingerprint' => hash('sha256', $childOperation),
|
|
'parent_job_id' => $parentId, 'order_id' => null,
|
|
'invoice_id' => null, 'instance_id' => null, 'snapshot_id' => null,
|
|
'user_id' => 1, 'host_id' => 7, 'kind' => 'test_mutation',
|
|
'execution_mode' => 'agent', 'phase' => 'accepted',
|
|
'outcome' => null, 'status' => 'queued',
|
|
'sealed_payload' => CloudSecrets::encrypt([
|
|
'method' => 'POST', 'request_target' => '/v1/test',
|
|
'body' => ['operation_id' => $childOperation], 'owner_id' => 1,
|
|
], 'job-payload'),
|
|
'attempts' => 0, 'dispatch_attempts' => 0,
|
|
'dispatch_deadline_at' => $now + 600,
|
|
'dispatch_started_at' => null, 'agent_accepted_at' => null,
|
|
'receipt_deadline_at' => null, 'terminal_at' => null,
|
|
'effect_may_have_applied' => null, 'safe_to_resubmit' => null,
|
|
'effect_resolution' => 'not_applicable',
|
|
'effect_resolved_at' => null, 'effect_resolved_by' => null,
|
|
'effect_resolution_reason' => '', 'completion_event_id' => null,
|
|
'read_at' => null, 'dispatched_at' => null,
|
|
'lease_token' => null, 'lease_until' => 0, 'available_at' => 0,
|
|
'error_code' => null, 'result_json' => null,
|
|
'created_at' => $now - 80, 'updated_at' => $now - 40,
|
|
]);
|
|
Capsule::unprepared('CREATE TRIGGER simulate_child_claim_cas_miss '
|
|
. 'BEFORE UPDATE OF lease_token ON cloud_jobs '
|
|
. 'WHEN OLD.id = ' . $childId . " AND OLD.phase <> 'terminal' "
|
|
. 'BEGIN SELECT RAISE(IGNORE); END');
|
|
$assert(CloudWorkerService::expireDueTasksForOwner(
|
|
1, $parentId, 10
|
|
) === 1, 'Owner fallback did not claim an expired aggregate parent.');
|
|
$parent = CloudJob::find($parentId);
|
|
$child = CloudJob::find($childId);
|
|
$assert($parent !== null && $child !== null
|
|
&& (string) $parent->phase === 'awaiting_receipt'
|
|
&& $parent->outcome === null
|
|
&& $parent->lease_token === null
|
|
&& (int) $parent->lease_until === 0
|
|
&& (string) $child->phase === 'accepted',
|
|
'A child claim CAS miss hid active work behind a terminal parent.');
|
|
|
|
Capsule::unprepared('DROP TRIGGER simulate_child_claim_cas_miss');
|
|
$assert(CloudWorkerService::expireDueTasksForOwner(
|
|
1, $parentId, 10
|
|
) === 1, 'A later fallback pass did not reclaim the aggregate parent.');
|
|
$parent = CloudJob::find($parentId);
|
|
$child = CloudJob::find($childId);
|
|
$assert($parent !== null && $child !== null
|
|
&& (string) $child->phase === 'terminal'
|
|
&& (string) $child->outcome === 'failed'
|
|
&& !(bool) $child->effect_may_have_applied
|
|
&& (bool) $child->safe_to_resubmit
|
|
&& (string) $parent->phase === 'terminal'
|
|
&& (string) $parent->outcome === 'failed',
|
|
'A later aggregate fallback did not terminalize children first.');
|
|
|
|
echo "Owner-scoped task deadline fallback tests passed.\n";
|
|
}
|