You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
virt/web/tests/CloudTaskDeadlineFallbackTe...

3001 lines
125 KiB
PHP

<?php
declare(strict_types=1);
namespace App\Services\Cloud {
/** In-memory envelope for the isolated deadline/finalization test. */
final class CloudSecrets
{
private static array $values = [];
public static function encrypt(array $value, string $purpose): string
{
$key = bin2hex(random_bytes(16));
self::$values[$key] = $value;
return 'deadline-test:' . $key;
}
public static function decrypt(string $value, string $purpose): array
{
$key = substr($value, strlen('deadline-test:'));
if (!array_key_exists($key, self::$values)) {
throw new \RuntimeException('Missing test payload');
}
return self::$values[$key];
}
}
}
namespace {
require dirname(__DIR__) . '/vendor/autoload.php';
use App\Models\CloudAgentCompletionEvent;
use App\Models\CloudJob;
use App\Services\Cloud\CloudAgentClient;
use App\Services\Cloud\CloudAutomaticOperationRecoveryService;
use App\Services\Cloud\CloudJobService;
use App\Services\Cloud\CloudOperationCompletionService;
use App\Services\Cloud\CloudOperationRecoveryIdentity;
use App\Services\Cloud\CloudOperationRecoveryPolicy;
use App\Services\Cloud\CloudOperationRecoveryService;
use App\Services\Cloud\CloudSecrets;
use App\Services\Cloud\CloudWorkerService;
use GuzzleHttp\Client;
use Illuminate\Database\Capsule\Manager as Capsule;
use Illuminate\Database\Schema\Blueprint;
$assert = static function (bool $condition, string $message): void {
if (!$condition) {
throw new RuntimeException($message);
}
};
$capsule = new Capsule();
$capsule->addConnection([
'driver' => 'sqlite',
'database' => ':memory:',
], 'default');
$capsule->setAsGlobal();
$capsule->bootEloquent();
$schema = $capsule->schema('default');
$schema->create('user', static function (Blueprint $table): void {
$table->increments('id');
$table->decimal('money', 12, 2)->default(0);
$table->integer('enable')->default(1);
$table->integer('is_admin')->default(0);
});
$schema->create('cloud_jobs', static function (Blueprint $table): void {
$table->increments('id');
$table->string('operation_id')->unique();
$table->string('request_idempotency_key')->unique();
$table->string('request_fingerprint');
$table->integer('parent_job_id')->nullable();
$table->integer('order_id')->nullable();
$table->integer('invoice_id')->nullable();
$table->integer('instance_id')->nullable();
$table->integer('snapshot_id')->nullable();
$table->integer('vpc_repair_root_job_id')->nullable();
$table->integer('user_id');
$table->integer('host_id')->nullable();
$table->string('kind');
$table->string('execution_mode')->default('agent');
$table->string('phase');
$table->string('outcome')->nullable();
$table->string('status');
$table->text('sealed_payload');
$table->integer('attempts')->default(0);
$table->integer('dispatch_attempts')->default(0);
$table->integer('dispatch_deadline_at')->nullable();
$table->integer('dispatch_started_at')->nullable();
$table->integer('agent_accepted_at')->nullable();
$table->integer('receipt_deadline_at')->nullable();
$table->integer('terminal_at')->nullable();
$table->boolean('effect_may_have_applied')->nullable();
$table->boolean('safe_to_resubmit')->nullable();
$table->string('effect_resolution')->default('not_applicable');
$table->integer('effect_resolved_at')->nullable();
$table->integer('effect_resolved_by')->nullable();
$table->string('effect_resolution_reason')->default('');
$table->string('completion_event_id')->nullable();
$table->integer('read_at')->nullable();
$table->integer('dispatched_at')->nullable();
$table->string('lease_token')->nullable();
$table->integer('lease_until')->default(0);
$table->integer('available_at')->default(0);
$table->string('error_code')->nullable();
$table->boolean('finalization_incident')->default(0);
$table->string('finalization_error_code')->nullable();
$table->text('result_json')->nullable();
$table->integer('created_at');
$table->integer('updated_at');
});
$schema->create('cloud_instances', static function (Blueprint $table): void {
$table->increments('id');
$table->string('uuid');
$table->integer('host_id');
$table->integer('user_id');
$table->integer('order_id')->nullable();
$table->integer('plan_id')->nullable();
$table->integer('offer_id')->nullable();
$table->integer('resource_pool_id')->nullable();
$table->integer('reservation_id')->nullable();
$table->integer('current_period_id')->nullable();
$table->string('name');
$table->string('status');
$table->string('service_status');
$table->string('network_status');
$table->integer('expires_at')->nullable();
$table->decimal('renewal_price', 12, 2)->nullable();
$table->integer('renewal_days')->default(0);
$table->integer('renewal_traffic_bytes')->default(0);
$table->integer('traffic_quota_bytes')->default(0);
$table->integer('traffic_download_bytes')->default(0);
$table->integer('traffic_upload_bytes')->default(0);
$table->integer('lifetime_download_bytes')->default(0);
$table->integer('lifetime_upload_bytes')->default(0);
$table->integer('traffic_synced_at')->default(0);
$table->integer('suspended_at')->nullable();
$table->integer('network_suspended_at')->nullable();
$table->text('spec_json');
$table->text('sealed_credential')->nullable();
$table->integer('created_at');
$table->integer('updated_at');
});
$schema->create('cloud_host_storage_configurations',
static function (Blueprint $table): void {
$table->integer('host_id')->primary();
$table->integer('revision')->default(0);
foreach (['managed', 'template', 'seed', 'user_iso', 'rescue']
as $role) {
$table->text($role . '_directory');
}
$table->string('state')->default('applying');
$table->string('operation_id')->nullable()->unique();
$table->integer('job_id')->nullable();
$table->integer('created_at');
$table->integer('updated_at');
});
$schema->create('cloud_agent_completion_events',
static function (Blueprint $table): void {
$table->string('event_id')->primary();
$table->integer('host_id');
$table->string('operation_id');
$table->string('payload_sha256');
$table->string('outcome');
$table->boolean('effect_may_have_applied');
$table->boolean('safe_to_resubmit');
$table->string('error_code')->default('');
$table->text('result_json')->nullable();
$table->string('disposition')->default('received');
$table->integer('received_at');
$table->integer('applied_at')->nullable();
$table->unique(['host_id', 'operation_id']);
});
$schema->create('cloud_templates',
static function (Blueprint $table): void {
$table->string('id')->primary();
$table->string('catalog_key')->default('test-catalog');
$table->string('version_label')->default('test-version');
$table->boolean('is_default')->default(true);
$table->integer('active_job_id')->nullable();
$table->string('state')->default('active');
});
$schema->create('cloud_user_isos',
static function (Blueprint $table): void {
$table->string('id')->primary();
$table->integer('active_job_id')->nullable();
$table->string('state')->default('active');
});
$schema->create('cloud_template_replicas',
static function (Blueprint $table): void {
$table->integer('host_id');
$table->string('template_id');
$table->integer('revision')->default(0);
$table->string('generation_token')->default('');
$table->string('state')->default('pending');
$table->integer('image_bytes')->default(0);
$table->string('sha256')->default('');
$table->string('operation_id')->default('');
$table->integer('job_id')->nullable();
$table->string('error_code')->default('');
$table->integer('created_at')->default(0);
$table->integer('updated_at')->default(0);
$table->primary(['host_id', 'template_id']);
});
$schema->create('cloud_user_iso_replicas',
static function (Blueprint $table): void {
$table->integer('host_id');
$table->string('user_iso_id');
$table->integer('user_id');
$table->string('state')->default('pending');
$table->integer('size_bytes')->default(0);
$table->string('sha256')->default('');
$table->string('operation_id')->default('');
$table->integer('job_id')->nullable();
$table->string('error_code')->default('');
$table->integer('created_at')->default(0);
$table->integer('updated_at')->default(0);
$table->primary(['host_id', 'user_iso_id']);
});
$schema->create('cloud_invoice_payments',
static function (Blueprint $table): void {
$table->increments('id');
$table->integer('invoice_id');
$table->integer('user_id');
$table->string('provider');
$table->string('provider_order_no');
$table->string('idempotency_key');
$table->decimal('amount', 12, 2)->default(0);
$table->string('status');
$table->integer('created_at');
$table->integer('updated_at');
$table->unique(['user_id', 'idempotency_key']);
});
Capsule::table('user')->insert([
['id' => 1, 'money' => 0, 'enable' => 1, 'is_admin' => 0],
['id' => 2, 'money' => 0, 'enable' => 1, 'is_admin' => 0],
]);
$now = time();
$insertJob = static function (int $userId, string $operationId,
string $phase, int $dispatchAttempts) use ($now): int {
return (int) Capsule::table('cloud_jobs')->insertGetId([
'operation_id' => $operationId,
'request_idempotency_key' => $operationId,
'request_fingerprint' => hash('sha256', $operationId),
'parent_job_id' => null,
'order_id' => null,
'invoice_id' => null,
'instance_id' => null,
'snapshot_id' => null,
'user_id' => $userId,
'host_id' => 7,
// An unrecognised kind exercises the generic terminal ledger path
// without requiring a business-resource fixture.
'kind' => 'test_mutation',
'execution_mode' => 'agent',
'phase' => $phase,
'outcome' => null,
'status' => $phase === 'accepted' ? 'queued' : 'running',
'sealed_payload' => CloudSecrets::encrypt([
'method' => 'POST',
'request_target' => '/v1/test',
'body' => ['operation_id' => $operationId],
'owner_id' => $userId,
// Crossed-dispatch failures must retain one encrypted copy
// until their immutable Agent receipt resolves the effect.
'credential' => [
'username' => 'root',
'password' => 'deadline-recovery-secret',
],
], 'job-payload'),
'attempts' => $dispatchAttempts,
'dispatch_attempts' => $dispatchAttempts,
'dispatch_deadline_at' => $now - 20,
'dispatch_started_at' => $dispatchAttempts === 1
? $now - 30 : null,
'agent_accepted_at' => null,
'receipt_deadline_at' => $dispatchAttempts === 1
? $now - 10 : null,
'terminal_at' => null,
'effect_may_have_applied' => null,
'safe_to_resubmit' => null,
'effect_resolution' => 'not_applicable',
'effect_resolved_at' => null,
'effect_resolved_by' => null,
'effect_resolution_reason' => '',
'completion_event_id' => null,
'read_at' => null,
'dispatched_at' => $dispatchAttempts === 1 ? $now - 30 : null,
// Simulate a dead daemon whose stale lease extends past deadline.
'lease_token' => str_repeat('f', 32),
'lease_until' => $now + 600,
'available_at' => 0,
'error_code' => null,
'result_json' => null,
'created_at' => $now - 60,
'updated_at' => $now - 30,
]);
};
$insertInstance = static function (
int $id,
string $uuid,
string $name
) use ($now): void {
Capsule::table('cloud_instances')->insert([
'id' => $id,
'uuid' => $uuid,
'host_id' => 7,
'user_id' => 1,
'order_id' => null,
'plan_id' => null,
'offer_id' => null,
'resource_pool_id' => null,
'reservation_id' => null,
'current_period_id' => null,
'name' => $name,
'status' => 'active',
'service_status' => 'exempt',
'network_status' => 'active',
'expires_at' => null,
'renewal_price' => null,
'renewal_days' => 0,
'renewal_traffic_bytes' => 0,
'traffic_quota_bytes' => 0,
'traffic_download_bytes' => 0,
'traffic_upload_bytes' => 0,
'lifetime_download_bytes' => 0,
'lifetime_upload_bytes' => 0,
'traffic_synced_at' => 0,
'suspended_at' => null,
'network_suspended_at' => null,
'spec_json' => json_encode(
['managed' => true],
JSON_THROW_ON_ERROR
),
'sealed_credential' => null,
'created_at' => $now - 3600,
'updated_at' => $now - 3600,
]);
};
$storagePayload = static function (string $operationId): string {
return CloudSecrets::encrypt([
'method' => 'POST',
'request_target' => '/v1/storage/configuration',
'body' => [
'managed_directory' => '/home/libvirt/kvm',
'template_directory' => '/home/libvirt/images',
'seed_directory' => '/home/libvirt/seed',
'user_iso_directory' => '/home/libvirt/iso',
'rescue_directory' => '/home/libvirt/rescue-iso',
'operation_id' => $operationId,
],
'owner_id' => 1,
'can_manage' => true,
'storage_revision' => 1,
], 'job-payload');
};
$identityFor = static function (
string $operationId,
bool $applied
): array {
$job = CloudJob::where('operation_id', $operationId)->firstOrFail();
if (CloudOperationRecoveryIdentity::supports((string) $job->kind)) {
return CloudOperationRecoveryIdentity::expected($job, $applied);
}
return [
'action' => 'test-mutation',
'resource_id' => 'operation-' . substr($operationId, 0, 16),
'request_fingerprint' => hash('sha256', $operationId),
];
};
$withIdentity = static function (array $receipt) use (
$identityFor
): array {
return $receipt + $identityFor(
(string) $receipt['operation_id'],
($receipt['status'] ?? null) === 'succeeded'
);
};
$storedEvent = static function (
array $receipt,
int $receivedAt,
string $disposition = 'received'
): array {
$canonical = [
'event_id' => $receipt['id'],
'host_id' => (int) $receipt['agent_id'],
'operation_id' => $receipt['operation_id'],
'action' => $receipt['action'],
'resource_id' => $receipt['resource_id'],
'request_fingerprint' => $receipt['request_fingerprint'],
'status' => $receipt['status'],
'effect_may_have_applied' =>
$receipt['effect_may_have_applied'],
'safe_to_resubmit' => $receipt['safe_to_resubmit'],
'error_code' => $receipt['error_code'],
'result' => $receipt['result'],
];
return [
'event_id' => $canonical['event_id'],
'host_id' => $canonical['host_id'],
'operation_id' => $canonical['operation_id'],
'payload_sha256' => hash('sha256', json_encode(
$canonical,
JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES
| JSON_UNESCAPED_UNICODE
)),
'outcome' => $canonical['status'],
'effect_may_have_applied' =>
$canonical['effect_may_have_applied'] ? 1 : 0,
'safe_to_resubmit' => $canonical['safe_to_resubmit'] ? 1 : 0,
'error_code' => $canonical['error_code'],
'result_json' => json_encode([
'schema' => 'virt_cloud_completion_event',
'version' => 1,
'identity' => [
'action' => $canonical['action'],
'resource_id' => $canonical['resource_id'],
'request_fingerprint' =>
$canonical['request_fingerprint'],
],
'result' => $canonical['result'],
], JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES
| JSON_UNESCAPED_UNICODE),
'disposition' => $disposition,
'received_at' => $receivedAt,
'applied_at' => null,
];
};
$receiptOperation = str_repeat('a', 32);
$receiptJobId = $insertJob(1, $receiptOperation, 'dispatching', 1);
$receiptIdentity = [
'action' => 'test-mutation',
'resource_id' => 'test-resource',
'request_fingerprint' => str_repeat('1', 64),
];
$receipt = [
'event_id' => str_repeat('e', 32),
'host_id' => 7,
'operation_id' => $receiptOperation,
] + $receiptIdentity + [
'status' => 'succeeded',
'effect_may_have_applied' => false,
'safe_to_resubmit' => false,
'error_code' => '',
'result' => ['confirmed' => true],
];
Capsule::table('cloud_agent_completion_events')->insert([
'event_id' => $receipt['event_id'],
'host_id' => 7,
'operation_id' => $receiptOperation,
'payload_sha256' => hash('sha256', json_encode(
$receipt,
JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES
| JSON_UNESCAPED_UNICODE
)),
'outcome' => 'succeeded',
'effect_may_have_applied' => 0,
'safe_to_resubmit' => 0,
'error_code' => '',
'result_json' => json_encode([
'schema' => 'virt_cloud_completion_event',
'version' => 1,
'identity' => $receiptIdentity,
'result' => $receipt['result'],
], JSON_THROW_ON_ERROR),
'disposition' => 'received',
'received_at' => $now - 15,
'applied_at' => null,
]);
$otherOperation = str_repeat('b', 32);
$otherJobId = $insertJob(2, $otherOperation, 'dispatching', 1);
$assert(CloudWorkerService::expireDueTasksForOwner(
1, $receiptJobId, 1
) === 1, 'The owner-scoped fallback did not claim the due task.');
$receiptJob = CloudJob::find($receiptJobId);
$otherJob = CloudJob::find($otherJobId);
$event = Capsule::table('cloud_agent_completion_events')
->where('event_id', str_repeat('e', 32))->first();
$assert($receiptJob !== null
&& (string) $receiptJob->phase === 'terminal'
&& (string) $receiptJob->outcome === 'succeeded'
&& (int) $receiptJob->dispatch_attempts === 1
&& (string) $event->disposition === 'applied',
'A receipt saved during dispatch was replaced by deadline failure.');
$assert($otherJob !== null && (string) $otherJob->phase === 'dispatching',
'Owner-scoped expiry mutated another user\'s task.');
$failureOperation = str_repeat('c', 32);
$failureJobId = $insertJob(1, $failureOperation, 'dispatching', 1);
$assert(CloudWorkerService::expireDueTasksForOwner(
1, $failureJobId, 1
) === 1, 'The receipt-less due task was not closed.');
$failed = CloudJob::find($failureJobId);
$failedRecoveryPayload = CloudSecrets::decrypt(
(string) $failed->sealed_payload,
'job-payload'
);
$assert($failed !== null
&& (string) $failed->phase === 'terminal'
&& (string) $failed->outcome === 'failed'
&& (bool) $failed->effect_may_have_applied
&& !(bool) $failed->safe_to_resubmit
&& !array_key_exists('credential', $failedRecoveryPayload),
'An unsupported unknown-effect kind retained secret recovery escrow.');
// dispatch_attempts is a durable 0/1 fence, not a retry counter. Values
// outside that state machine must fail closed: neither deadline handling
// nor its last-resort finalizer may reinterpret them as "not dispatched".
$invalidLedgerUuid =
'00000000-0000-4000-8000-000000000301';
$insertInstance(301, $invalidLedgerUuid, 'invalid-ledger-settings');
$invalidLedgerOperation = str_repeat('cd', 16);
$invalidLedgerJobId = $insertJob(
1,
$invalidLedgerOperation,
'dispatching',
-1
);
$invalidLedgerPayload = CloudSecrets::encrypt([
'method' => 'POST',
'request_target' => '/v1/vms/' . $invalidLedgerUuid . '/settings',
'body' => [
'hostname' => 'invalid-ledger.example',
'confirm_restart' => true,
],
'owner_id' => 1,
'can_manage' => false,
'settings_fields' => ['hostname'],
], 'job-payload');
Capsule::table('cloud_jobs')->where('id', $invalidLedgerJobId)->update([
'kind' => 'instance_settings',
'instance_id' => 301,
'sealed_payload' => $invalidLedgerPayload,
'dispatch_started_at' => $now - 30,
'dispatched_at' => $now - 30,
'receipt_deadline_at' => $now - 10,
]);
$assert(CloudWorkerService::expireDueTasksForOwner(
1,
$invalidLedgerJobId,
1
) === 1, 'The invalid dispatch-ledger deadline was not claimed.');
$invalidLedgerJob = CloudJob::findOrFail($invalidLedgerJobId);
$assert((string) $invalidLedgerJob->phase === 'terminal'
&& (string) $invalidLedgerJob->outcome === 'failed'
&& (int) $invalidLedgerJob->dispatch_attempts === -1
&& (bool) $invalidLedgerJob->effect_may_have_applied
&& !(bool) $invalidLedgerJob->safe_to_resubmit
&& (string) $invalidLedgerJob->effect_resolution === 'unknown'
&& (bool) $invalidLedgerJob->finalization_incident
&& (string) $invalidLedgerJob->finalization_error_code
=== 'recovery_dispatch_ledger_conflict'
&& (string) $invalidLedgerJob->sealed_payload
=== $invalidLedgerPayload,
'A negative dispatch ledger was treated as a safe undispatched failure.');
// Exercise the same invariant through forceTerminalFailure(): the linked
// instance is intentionally absent so the ordinary policy finalizer fails
// and the last-resort path owns the durable incident write.
$forcedLedgerOperation = str_repeat('ce', 16);
$forcedLedgerJobId = $insertJob(
1,
$forcedLedgerOperation,
'dispatching',
2
);
$forcedLedgerPayload = CloudSecrets::encrypt([
'method' => 'POST',
'request_target' => '/v1/vms/'
. '00000000-0000-4000-8000-000000009999/suspend',
'body' => [],
'owner_id' => 1,
'can_manage' => false,
], 'job-payload');
Capsule::table('cloud_jobs')->where('id', $forcedLedgerJobId)->update([
'kind' => 'service_suspend',
'instance_id' => 9999,
'sealed_payload' => $forcedLedgerPayload,
'dispatch_started_at' => $now - 30,
'dispatched_at' => $now - 30,
'receipt_deadline_at' => $now - 10,
]);
$assert(CloudWorkerService::expireDueTasksForOwner(
1,
$forcedLedgerJobId,
1
) === 1, 'The forced invalid dispatch-ledger deadline was not claimed.');
$forcedLedgerJob = CloudJob::findOrFail($forcedLedgerJobId);
$assert((string) $forcedLedgerJob->phase === 'terminal'
&& (string) $forcedLedgerJob->outcome === 'failed'
&& (int) $forcedLedgerJob->dispatch_attempts === 2
&& (bool) $forcedLedgerJob->effect_may_have_applied
&& !(bool) $forcedLedgerJob->safe_to_resubmit
&& (string) $forcedLedgerJob->effect_resolution === 'unknown'
&& (bool) $forcedLedgerJob->finalization_incident
&& (string) $forcedLedgerJob->finalization_error_code
=== 'recovery_dispatch_ledger_conflict'
&& (string) $forcedLedgerJob->sealed_payload
=== $forcedLedgerPayload,
'The forced finalizer concealed or unlocked an invalid dispatch ledger.');
// Deploying the shorter storage window must also bound tasks which already
// persisted the old 960-second receipt deadline. No data rewrite is needed:
// the effective deadline is the earlier of the two absolute boundaries.
$legacyStorageOperation = str_repeat('3', 32);
$legacyStorageJobId = $insertJob(
1, $legacyStorageOperation, 'awaiting_receipt', 1
);
Capsule::table('cloud_jobs')->where('id', $legacyStorageJobId)->update([
'kind' => 'host_storage_configure',
'sealed_payload' => $storagePayload($legacyStorageOperation),
'dispatched_at' => $now - 181,
'dispatch_started_at' => $now - 181,
'receipt_deadline_at' => $now + 779,
]);
Capsule::table('cloud_host_storage_configurations')->insert([
'host_id' => 7, 'revision' => 1,
'managed_directory' => '/home/libvirt/kvm',
'template_directory' => '/home/libvirt/images',
'seed_directory' => '/home/libvirt/seed',
'user_iso_directory' => '/home/libvirt/iso',
'rescue_directory' => '/home/libvirt/rescue-iso',
'state' => 'applying', 'operation_id' => $legacyStorageOperation,
'job_id' => $legacyStorageJobId,
'created_at' => $now - 200, 'updated_at' => $now - 181,
]);
$legacyStorage = CloudJob::find($legacyStorageJobId);
$assert(CloudJobService::effectiveReceiptDeadline($legacyStorage)
=== $now - 1
&& CloudJobService::receiptDeadlineReached($legacyStorage, $now),
'The persisted storage deadline was not bounded by dispatch plus 180 seconds.');
$assert(CloudWorkerService::expireDueTasksForOwner(
1, $legacyStorageJobId, 1
) === 1, 'An old 960-second storage task remained active after 180 seconds.');
$legacyStorage = CloudJob::find($legacyStorageJobId);
$legacyStorageRow = Capsule::table(
'cloud_host_storage_configurations'
)->where('host_id', 7)->first();
$assert((string) $legacyStorage->phase === 'terminal'
&& (string) $legacyStorage->outcome === 'failed'
&& (string) $legacyStorageRow->state === 'failed',
'Storage deadline fallback did not close the task and its projection.');
// Receipt classification must use exactly the same effective boundary as
// the expiry query, otherwise a late success could revive a timed-out task.
$lateStorageOperation = str_repeat('2', 32);
$lateStorageJobId = $insertJob(
1, $lateStorageOperation, 'awaiting_receipt', 1
);
Capsule::table('cloud_jobs')->where('id', $lateStorageJobId)->update([
'kind' => 'host_storage_configure',
'sealed_payload' => $storagePayload($lateStorageOperation),
'dispatched_at' => $now - 181,
'dispatch_started_at' => $now - 181,
'receipt_deadline_at' => $now + 779,
]);
Capsule::table('cloud_host_storage_configurations')
->where('host_id', 7)->update([
'state' => 'applying', 'operation_id' => $lateStorageOperation,
'job_id' => $lateStorageJobId, 'updated_at' => $now,
]);
$lateStorageEventId = str_repeat('4', 32);
$lateStorageReceipt = $withIdentity([
'type' => 'completion',
'id' => $lateStorageEventId,
'agent_id' => '7',
'operation_id' => $lateStorageOperation,
'status' => 'succeeded',
'effect_may_have_applied' => false,
'safe_to_resubmit' => false,
'error_code' => '',
'result' => ['configured' => true],
]);
Capsule::table('cloud_agent_completion_events')->insert(
$storedEvent($lateStorageReceipt, $now - 1)
);
$lateStorage = (new CloudWorkerService())->settleCompletionReceipt(
$lateStorageJobId, $lateStorageEventId
);
$lateStorageEvent = Capsule::table('cloud_agent_completion_events')
->where('event_id', $lateStorageEventId)->first();
$assert((string) $lateStorage->phase === 'terminal'
&& (string) $lateStorage->outcome === 'failed'
&& (string) $lateStorageEvent->disposition === 'late',
'A storage receipt at the effective deadline was not classified as late.');
$acceptedStorageOperation = str_repeat('1', 32);
$acceptedStorageJobId = $insertJob(
1, $acceptedStorageOperation, 'accepted', 0
);
Capsule::table('cloud_jobs')->where('id', $acceptedStorageJobId)->update([
'kind' => 'host_storage_configure',
'sealed_payload' => $storagePayload($acceptedStorageOperation),
'created_at' => $now - 61,
'dispatch_deadline_at' => $now + 239,
]);
Capsule::table('cloud_host_storage_configurations')
->where('host_id', 7)->update([
'state' => 'applying', 'operation_id' => $acceptedStorageOperation,
'job_id' => $acceptedStorageJobId, 'updated_at' => $now,
]);
$acceptedStorage = CloudJob::find($acceptedStorageJobId);
$assert(CloudJobService::effectiveDispatchDeadline($acceptedStorage)
=== $now - 1
&& CloudJobService::activeDeadlineReached($acceptedStorage, $now),
'The persisted storage dispatch deadline was not bounded to 60 seconds.');
$assert(CloudWorkerService::expireDueTasksForOwner(
1, $acceptedStorageJobId, 1
) === 1, 'An old 300-second accepted storage task remained active.');
$acceptedStorage = CloudJob::find($acceptedStorageJobId);
$assert((string) $acceptedStorage->phase === 'terminal'
&& (string) $acceptedStorage->outcome === 'failed'
&& !(bool) $acceptedStorage->effect_may_have_applied
&& (bool) $acceptedStorage->safe_to_resubmit,
'An undispatched storage deadline did not become a safe failure.');
// The normal worker tick must run local deadline convergence before it can
// claim an old storage task. Its injected Agent transport is a tripwire:
// a request here would prove the former 300-second deadline was used.
$tickStorageOperation = '0123456789abcdef0123456789abcdef';
$tickStorageJobId = $insertJob(
1, $tickStorageOperation, 'accepted', 0
);
Capsule::table('cloud_jobs')->where('id', $tickStorageJobId)->update([
'kind' => 'host_storage_configure',
'sealed_payload' => $storagePayload($tickStorageOperation),
'created_at' => time() - 61,
'dispatch_deadline_at' => time() + 239,
]);
Capsule::table('cloud_host_storage_configurations')
->where('host_id', 7)->update([
'state' => 'applying', 'operation_id' => $tickStorageOperation,
'job_id' => $tickStorageJobId, 'updated_at' => time(),
]);
$agentCalls = 0;
$transport = new Client(['handler' => static function () use (
&$agentCalls
) {
$agentCalls++;
throw new RuntimeException('Expired storage task reached Agent');
}]);
$tripwireAgent = new CloudAgentClient(
'7', str_repeat('a', 32), $transport, 120.0,
'http://127.0.0.1:8001/internal/agent/request',
str_repeat('b', 32)
);
$assert((new CloudWorkerService($tripwireAgent))->tick() === true,
'Worker tick did not converge the expired storage task.');
$tickStorage = CloudJob::find($tickStorageJobId);
$assert($agentCalls === 0
&& (string) $tickStorage->phase === 'terminal'
&& (string) $tickStorage->outcome === 'failed'
&& (int) $tickStorage->dispatch_attempts === 0,
'Worker dispatched an old storage task after its effective 60-second boundary.');
$incidentOperation = str_repeat('5', 32);
$incidentJobId = $insertJob(
1, $incidentOperation, 'awaiting_receipt', 1
);
Capsule::table('cloud_jobs')->where('id', $incidentJobId)->update([
'sealed_payload' => 'deadline-test:missing-payload',
]);
$assert(CloudWorkerService::expireDueTasksForOwner(
1, $incidentJobId, 1
) === 1, 'The finalization-incident fixture was not claimed.');
$incident = CloudJob::find($incidentJobId);
$assert($incident !== null
&& (string) $incident->phase === 'terminal'
&& (string) $incident->outcome === 'failed'
&& (string) $incident->status === 'failed'
&& (bool) $incident->effect_may_have_applied
&& !(bool) $incident->safe_to_resubmit
&& (string) $incident->effect_resolution === 'unknown'
&& (string) $incident->error_code === 'local_finalization_failed'
&& (bool) $incident->finalization_incident
&& (string) $incident->finalization_error_code
=== 'recovery_payload_unavailable'
&& (string) $incident->sealed_payload === '',
'A local finalizer crash left the task active, leaked its payload, or concealed the incident.');
$redactedIncidentOperation = '8123456789abcdef8123456789abcdef';
$redactedIncidentJobId = $insertJob(
1,
$redactedIncidentOperation,
'awaiting_receipt',
1
);
Capsule::table('cloud_jobs')->where(
'id',
$redactedIncidentJobId
)->update(['kind' => 'network_change']);
$assert(CloudWorkerService::expireDueTasksForOwner(
1,
$redactedIncidentJobId,
1
) === 1, 'The redacted finalization-incident fixture was not claimed.');
$redactedIncident = CloudJob::find($redactedIncidentJobId);
$redactedIncidentPayload = CloudSecrets::decrypt(
(string) $redactedIncident->sealed_payload,
'job-payload'
);
$assert((bool) $redactedIncident->finalization_incident
&& (string) $redactedIncident->effect_resolution === 'unknown'
&& array_key_exists('credential', $redactedIncidentPayload),
'A recoverable finalization incident lost its encrypted recovery escrow.');
Capsule::table('cloud_jobs')->where(
'id',
$redactedIncidentJobId
)->update(['available_at' => time() + 3600]);
$unreadableEnvelope = 'deadline-test:missing-policy-payload';
$unreadableIncidentJobId = $insertJob(
1,
'823456789abcdef0823456789abcdef0',
'awaiting_receipt',
1
);
Capsule::table('cloud_jobs')->where(
'id',
$unreadableIncidentJobId
)->update([
'kind' => 'network_change',
'sealed_payload' => $unreadableEnvelope,
]);
$assert(CloudWorkerService::expireDueTasksForOwner(
1,
$unreadableIncidentJobId,
1
) === 1, 'The unreadable recovery escrow fixture was not claimed.');
$unreadableIncident = CloudJob::findOrFail($unreadableIncidentJobId);
$assert((bool) $unreadableIncident->finalization_incident
&& (string) $unreadableIncident->finalization_error_code
=== 'recovery_payload_unavailable'
&& (string) $unreadableIncident->sealed_payload
=== $unreadableEnvelope
&& (int) $unreadableIncident->available_at >= time() + 290,
'An unreadable recovery escrow was erased or left the retry queue.');
Capsule::table('cloud_jobs')->where(
'id',
$unreadableIncidentJobId
)->update(['available_at' => time() + 3600]);
$lateOperation = str_repeat('7', 32);
$lateJobId = $insertJob(1, $lateOperation, 'awaiting_receipt', 1);
$lateEventId = str_repeat('6', 32);
$lateReceipt = $withIdentity([
'type' => 'completion',
'id' => $lateEventId,
'agent_id' => '7',
'operation_id' => $lateOperation,
'status' => 'succeeded',
'effect_may_have_applied' => false,
'safe_to_resubmit' => false,
'error_code' => '',
'result' => ['confirmed' => true],
]);
Capsule::table('cloud_agent_completion_events')->insert($storedEvent(
$lateReceipt,
// Equal to or later than the absolute boundary is audit-only.
$now - 9
));
$lateJob = (new CloudWorkerService())->settleCompletionReceipt(
$lateJobId, $lateEventId
);
$lateEvent = Capsule::table('cloud_agent_completion_events')
->where('event_id', $lateEventId)->first();
$assert((string) $lateJob->phase === 'terminal'
&& (string) $lateJob->outcome === 'failed'
&& (string) $lateJob->error_code === 'operation_failed'
&& (bool) $lateJob->effect_may_have_applied
&& !(bool) $lateJob->safe_to_resubmit
&& !(bool) $lateJob->finalization_incident
&& (string) $lateEvent->disposition === 'late',
'A receipt persisted after the deadline revived or succeeded the task.');
// completion_event_id is reserved before the business finalizer commits.
// If that finalizer fails, the matching on-time receipt must stay received
// so the settings recovery worker can finish PHP without Agent replay.
$onTimeRecoveryInstanceId = 90;
$onTimeRecoveryUuid = '00000000-1111-4222-8333-444444444444';
$insertInstance(
$onTimeRecoveryInstanceId,
$onTimeRecoveryUuid,
'on-time-finalizer-recovery-vm'
);
Capsule::table('cloud_instances')->where(
'id',
$onTimeRecoveryInstanceId
)->update(['spec_json' => '{invalid-json']);
$onTimeRecoveryOperation = '0bcdef0123456789abcdef0123456789';
$onTimeRecoveryJobId = $insertJob(
1,
$onTimeRecoveryOperation,
'awaiting_receipt',
1
);
Capsule::table('cloud_jobs')->where(
'id',
$onTimeRecoveryJobId
)->update([
'instance_id' => $onTimeRecoveryInstanceId,
'kind' => 'instance_settings',
'receipt_deadline_at' => time() + 60,
'sealed_payload' => CloudSecrets::encrypt([
'method' => 'POST',
'request_target' => '/v1/vms/' . $onTimeRecoveryUuid
. '/settings',
'body' => [
'hostname' => 'on-time-recovered.example',
'confirm_restart' => true,
],
'owner_id' => 1,
'settings_fields' => ['hostname'],
'credential' => null,
], 'job-payload'),
]);
$onTimeRecoveryReceipt = $withIdentity([
'type' => 'completion',
'id' => 'settings-on-time-finalizer-0001',
'agent_id' => '7',
'operation_id' => $onTimeRecoveryOperation,
'status' => 'succeeded',
'effect_may_have_applied' => false,
'safe_to_resubmit' => false,
'error_code' => '',
'result' => [
'settings_changed' => true,
'hostname' => 'on-time-recovered.example',
],
]);
$assert(CloudOperationCompletionService::accept(
$onTimeRecoveryReceipt
) === ['accepted' => true, 'disposition' => 'received'],
'A failed on-time finalizer incorrectly acknowledged its receipt.');
$onTimeFailed = CloudJob::find($onTimeRecoveryJobId);
$onTimeEvent = Capsule::table('cloud_agent_completion_events')
->where('event_id', $onTimeRecoveryReceipt['id'])->first();
$assert((string) $onTimeFailed->phase === 'terminal'
&& (string) $onTimeFailed->outcome === 'failed'
&& (string) $onTimeFailed->effect_resolution === 'unknown'
&& (string) $onTimeFailed->completion_event_id
=== $onTimeRecoveryReceipt['id']
&& (string) $onTimeEvent->disposition === 'received'
&& CloudJobService::blockingQueryForInstance(
$onTimeRecoveryInstanceId
)->exists(),
'A failed on-time finalizer lost its recovery receipt or blocker.');
Capsule::table('cloud_instances')->where(
'id',
$onTimeRecoveryInstanceId
)->update([
'spec_json' => json_encode(
['managed' => true],
JSON_THROW_ON_ERROR
),
]);
$assert((new CloudAutomaticOperationRecoveryService())->tick(),
'The recovery worker did not resume the on-time receipt.');
$onTimeRecovered = CloudJob::find($onTimeRecoveryJobId);
$onTimeEvent = Capsule::table('cloud_agent_completion_events')
->where('event_id', $onTimeRecoveryReceipt['id'])->first();
$onTimeSpec = json_decode(
(string) Capsule::table('cloud_instances')
->where('id', $onTimeRecoveryInstanceId)
->value('spec_json'),
true,
flags: JSON_THROW_ON_ERROR
);
$assert((string) $onTimeRecovered->outcome === 'failed'
&& (string) $onTimeRecovered->status === 'failed'
&& (string) $onTimeRecovered->effect_resolution === 'applied'
&& (string) $onTimeRecovered->completion_event_id
=== $onTimeRecoveryReceipt['id']
&& (string) $onTimeEvent->disposition === 'applied'
&& ($onTimeSpec['guest_hostname'] ?? null)
=== 'on-time-recovered.example'
&& !CloudJobService::blockingQueryForInstance(
$onTimeRecoveryInstanceId
)->exists(),
'The on-time receipt did not finish PHP-only recovery exactly once.');
// A password reset can finish on the Agent after PHP has conservatively
// failed its task. The immutable late receipt must converge the instance
// ledger and release the blocker without rewriting the failed task.
$settingsInstanceId = 91;
$settingsUuid = '11111111-2222-4333-8444-555555555555';
$insertInstance($settingsInstanceId, $settingsUuid, 'late-settings-vm');
$settingsOperation = 'abcdef0123456789abcdef0123456789';
$settingsJobId = $insertJob(
1,
$settingsOperation,
'awaiting_receipt',
1
);
Capsule::table('cloud_jobs')->where('id', $settingsJobId)->update([
'instance_id' => $settingsInstanceId,
'kind' => 'instance_settings',
'sealed_payload' => CloudSecrets::encrypt([
'method' => 'POST',
'request_target' => '/v1/vms/' . $settingsUuid . '/settings',
'body' => [
'root_password' => 'New-late-password9!',
'confirm_restart' => true,
],
'owner_id' => 1,
'settings_fields' => ['password'],
'credential' => [
'password' => 'New-late-password9!',
'login_name' => 'root',
'auth_mode' => 'password',
],
], 'job-payload'),
]);
$assert(CloudWorkerService::expireDueTasksForOwner(
1,
$settingsJobId,
1
) === 1, 'The settings fixture did not cross its receipt deadline.');
$timedOutSettings = CloudJob::find($settingsJobId);
$originalTerminal = [
'operation_id' => (string) $timedOutSettings->operation_id,
'kind' => (string) $timedOutSettings->kind,
'phase' => (string) $timedOutSettings->phase,
'outcome' => (string) $timedOutSettings->outcome,
'status' => (string) $timedOutSettings->status,
'dispatch_attempts' => (int) $timedOutSettings->dispatch_attempts,
'effect_may_have_applied' =>
(bool) $timedOutSettings->effect_may_have_applied,
'safe_to_resubmit' => (bool) $timedOutSettings->safe_to_resubmit,
'error_code' => (string) $timedOutSettings->error_code,
'result_json' => $timedOutSettings->result_json,
'terminal_at' => (int) $timedOutSettings->terminal_at,
'completion_event_id' => $timedOutSettings->completion_event_id,
'read_at' => $timedOutSettings->read_at,
];
$settingsReceipt = $withIdentity([
'type' => 'completion',
'id' => 'settings-recovery-event-000001',
'agent_id' => '7',
'operation_id' => $settingsOperation,
'status' => 'succeeded',
'effect_may_have_applied' => false,
'safe_to_resubmit' => false,
'error_code' => '',
'result' => [
'settings_changed' => true,
],
]);
$settingsAccepted = CloudOperationCompletionService::accept(
$settingsReceipt
);
$recoveredSettings = CloudJob::find($settingsJobId);
$recoveredInstance = Capsule::table('cloud_instances')
->where('id', $settingsInstanceId)->first();
$recoveredSpec = json_decode(
(string) $recoveredInstance->spec_json,
true,
flags: JSON_THROW_ON_ERROR
);
$recoveredCredential = CloudSecrets::decrypt(
(string) $recoveredInstance->sealed_credential,
'instance-credential'
);
$redactedRecoveryPayload = CloudSecrets::decrypt(
(string) $recoveredSettings->sealed_payload,
'job-payload'
);
$settingsEvent = Capsule::table('cloud_agent_completion_events')
->where('event_id', $settingsReceipt['id'])->first();
$assert($settingsAccepted === [
'accepted' => true,
'disposition' => 'applied',
]
&& (string) $settingsEvent->disposition === 'applied'
&& (string) $recoveredSettings->effect_resolution === 'applied'
&& (int) $recoveredSettings->effect_resolved_at > 0
&& $originalTerminal === [
'operation_id' => (string) $recoveredSettings->operation_id,
'kind' => (string) $recoveredSettings->kind,
'phase' => (string) $recoveredSettings->phase,
'outcome' => (string) $recoveredSettings->outcome,
'status' => (string) $recoveredSettings->status,
'dispatch_attempts' =>
(int) $recoveredSettings->dispatch_attempts,
'effect_may_have_applied' =>
(bool) $recoveredSettings->effect_may_have_applied,
'safe_to_resubmit' =>
(bool) $recoveredSettings->safe_to_resubmit,
'error_code' => (string) $recoveredSettings->error_code,
'result_json' => $recoveredSettings->result_json,
'terminal_at' => (int) $recoveredSettings->terminal_at,
'completion_event_id' => $recoveredSettings->completion_event_id,
'read_at' => $recoveredSettings->read_at,
]
&& $recoveredSpec === ['managed' => true]
&& ($recoveredCredential['password'] ?? null)
=== 'New-late-password9!'
&& !array_key_exists('credential', $redactedRecoveryPayload)
&& !CloudJobService::blockingQueryForInstance(
$settingsInstanceId
)->exists(),
'A matching late settings receipt did not safely converge the ledger.');
$recoveredUpdatedAt = (int) $recoveredSettings->updated_at;
$recoveredCredentialEnvelope = (string) $recoveredInstance
->sealed_credential;
$assert(
CloudOperationCompletionService::accept($settingsReceipt) === [
'accepted' => true,
'disposition' => 'applied',
]
&& (int) CloudJob::find($settingsJobId)->updated_at
=== $recoveredUpdatedAt
&& (string) Capsule::table('cloud_instances')
->where('id', $settingsInstanceId)
->value('sealed_credential') === $recoveredCredentialEnvelope,
'An identical late receipt was not idempotent after reconciliation.'
);
$forgedSettingsReceipt = $settingsReceipt;
$forgedSettingsReceipt['id'] = 'settings-recovery-event-forged01';
$forgedConflict = false;
try {
CloudOperationCompletionService::accept($forgedSettingsReceipt);
} catch (\App\Services\Cloud\CloudError $error) {
$forgedConflict = $error->httpStatus === 409;
}
$unknownOperationReceipt = $settingsReceipt;
$unknownOperationReceipt['id'] = 'settings-recovery-event-unknown1';
$unknownOperationReceipt['operation_id'] =
'fedcba9876543210fedcba9876543210';
$assert($forgedConflict
&& CloudOperationCompletionService::accept(
$unknownOperationReceipt
) === ['accepted' => true, 'disposition' => 'rejected'],
'A different event or operation was accepted as recovery evidence.');
// An exact terminal Agent failure which explicitly says no effect was
// possible can also settle a timed-out settings task. Its transport error
// vocabulary is normalized to the settings no-effect finalizer; it never
// authorizes replay even when safe_to_resubmit=true.
$notAppliedInstanceId = 93;
$notAppliedUuid = 'bbbbbbbb-cccc-4ddd-8eee-ffffffffffff';
$insertInstance(
$notAppliedInstanceId,
$notAppliedUuid,
'not-applied-settings-vm'
);
$notAppliedOperationId = '2234567890abcdef1234567890abcdef';
$notAppliedJobId = $insertJob(
1,
$notAppliedOperationId,
'awaiting_receipt',
1
);
Capsule::table('cloud_jobs')->where('id', $notAppliedJobId)->update([
'instance_id' => $notAppliedInstanceId,
'kind' => 'instance_settings',
'sealed_payload' => CloudSecrets::encrypt([
'method' => 'POST',
'request_target' => '/v1/vms/' . $notAppliedUuid
. '/settings',
'body' => [
'hostname' => 'not-applied.example',
'confirm_restart' => true,
],
'owner_id' => 1,
'settings_fields' => ['hostname'],
'credential' => null,
], 'job-payload'),
]);
$assert(CloudWorkerService::expireDueTasksForOwner(
1,
$notAppliedJobId,
1
) === 1, 'The late no-effect fixture did not reach terminal failure.');
Capsule::table('cloud_jobs')->where('id', $notAppliedJobId)->update([
'effect_resolution_reason' => 'automatic_recovery:v1:'
. (time() - 21600) . ':12:pending',
'finalization_incident' => 1,
'finalization_error_code' => 'recovery_limit_reached',
]);
$notAppliedTerminal = CloudJob::find($notAppliedJobId);
$notAppliedAudit = [
'phase' => (string) $notAppliedTerminal->phase,
'outcome' => (string) $notAppliedTerminal->outcome,
'status' => (string) $notAppliedTerminal->status,
'error_code' => (string) $notAppliedTerminal->error_code,
'result_json' => $notAppliedTerminal->result_json,
'terminal_at' => (int) $notAppliedTerminal->terminal_at,
'completion_event_id' =>
$notAppliedTerminal->completion_event_id,
];
$notAppliedReceipt = $withIdentity([
'type' => 'completion',
'id' => 'settings-no-effect-event-000001',
'agent_id' => '7',
'operation_id' => $notAppliedOperationId,
'status' => 'failed',
'effect_may_have_applied' => false,
'safe_to_resubmit' => true,
'error_code' => 'guest_update_failed',
'result' => ['agent_detail' => 'bounded'],
]);
$assert(CloudOperationCompletionService::accept(
$notAppliedReceipt
) === ['accepted' => true, 'disposition' => 'applied'],
'An exact late no-effect settings receipt was not applied.');
$notAppliedRecovered = CloudJob::find($notAppliedJobId);
$notAppliedPayload = CloudSecrets::decrypt(
(string) $notAppliedRecovered->sealed_payload,
'job-payload'
);
$assert((string) $notAppliedRecovered->effect_resolution
=== 'not_applied'
&& $notAppliedAudit === [
'phase' => (string) $notAppliedRecovered->phase,
'outcome' => (string) $notAppliedRecovered->outcome,
'status' => (string) $notAppliedRecovered->status,
'error_code' => (string) $notAppliedRecovered->error_code,
'result_json' => $notAppliedRecovered->result_json,
'terminal_at' => (int) $notAppliedRecovered->terminal_at,
'completion_event_id' =>
$notAppliedRecovered->completion_event_id,
]
&& !array_key_exists('credential', $notAppliedPayload)
&& !(bool) $notAppliedRecovered->finalization_incident
&& $notAppliedRecovered->finalization_error_code === null
&& !CloudJobService::blockingQueryForInstance(
$notAppliedInstanceId
)->exists(),
'Late no-effect recovery rewrote task audit fields or retained its lock.');
// Receipt persistence and late-effect settlement are deliberately
// separate transactions. If PHP dies between them, the recovery-only
// worker must discover the exact received event through the job's indexed
// host+operation key, even when inspection retries were already exhausted.
$persistedEventInstanceId = 96;
$persistedEventUuid = 'eeeeeeee-ffff-4000-8111-000000000003';
$insertInstance(
$persistedEventInstanceId,
$persistedEventUuid,
'persisted-event-settings-vm'
);
$persistedEventOperationId = '5234567890abcdef1234567890abcdef';
$persistedEventJobId = $insertJob(
1,
$persistedEventOperationId,
'awaiting_receipt',
1
);
Capsule::table('cloud_jobs')->where('id', $persistedEventJobId)->update([
'instance_id' => $persistedEventInstanceId,
'kind' => 'instance_settings',
'sealed_payload' => CloudSecrets::encrypt([
'method' => 'POST',
'request_target' => '/v1/vms/' . $persistedEventUuid
. '/settings',
'body' => [
'hostname' => 'persisted-event.example',
'confirm_restart' => true,
],
'owner_id' => 1,
'settings_fields' => ['hostname'],
'credential' => null,
], 'job-payload'),
]);
$assert(CloudWorkerService::expireDueTasksForOwner(
1,
$persistedEventJobId,
1
) === 1, 'The persisted-event crash fixture did not expire.');
Capsule::table('cloud_jobs')->where('id', $persistedEventJobId)->update([
'effect_resolution_reason' => 'automatic_recovery:v1:'
. (time() - 21600) . ':12:pending',
'finalization_incident' => 1,
'finalization_error_code' => 'recovery_limit_reached',
// Exact immutable receipt evidence is stronger than an earlier
// inconclusive/offline inspection backoff.
'available_at' => time() + 600,
]);
$persistedEventId = 'settings-persisted-event-00001';
$persistedEventResult = ['agent_detail' => 'persisted-before-prepare'];
$persistedEventReceipt = $withIdentity([
'type' => 'completion',
'id' => $persistedEventId,
'agent_id' => '7',
'operation_id' => $persistedEventOperationId,
'status' => 'failed',
'effect_may_have_applied' => false,
'safe_to_resubmit' => true,
'error_code' => 'guest_update_failed',
'result' => $persistedEventResult,
]);
Capsule::table('cloud_agent_completion_events')->insert(
$storedEvent($persistedEventReceipt, time())
);
$persistedEventAudit = CloudJob::find($persistedEventJobId);
$persistedEventAudit = [
'phase' => (string) $persistedEventAudit->phase,
'outcome' => (string) $persistedEventAudit->outcome,
'status' => (string) $persistedEventAudit->status,
'error_code' => (string) $persistedEventAudit->error_code,
'result_json' => $persistedEventAudit->result_json,
'terminal_at' => (int) $persistedEventAudit->terminal_at,
];
$assert((new CloudAutomaticOperationRecoveryService())->tick(),
'The worker did not drain a persisted pre-prepare receipt.');
$persistedEventRecovered = CloudJob::find($persistedEventJobId);
$persistedEventRow = Capsule::table('cloud_agent_completion_events')
->where('event_id', $persistedEventId)->first();
$assert((string) $persistedEventRecovered->effect_resolution
=== 'not_applied'
&& (string) $persistedEventRecovered->effect_resolution_reason
=== 'late_completion_receipt:' . $persistedEventId
&& !(bool) $persistedEventRecovered->finalization_incident
&& $persistedEventRecovered->finalization_error_code === null
&& (string) $persistedEventRow->disposition === 'applied'
&& $persistedEventAudit === [
'phase' => (string) $persistedEventRecovered->phase,
'outcome' => (string) $persistedEventRecovered->outcome,
'status' => (string) $persistedEventRecovered->status,
'error_code' => (string) $persistedEventRecovered->error_code,
'result_json' => $persistedEventRecovered->result_json,
'terminal_at' => (int) $persistedEventRecovered->terminal_at,
]
&& !CloudJobService::blockingQueryForInstance(
$persistedEventInstanceId
)->exists(),
'Persisted pre-prepare receipt did not converge locally and exactly.');
// A row with the right indexed host+operation but a conflicting immutable
// payload hash is not receipt authority. Reject only that event, surface
// the concrete receipt error, and leave the job queued for fresh
// inspection without contacting the Agent in the same tick.
$conflictEventInstanceId = 97;
$conflictEventUuid = 'ffffffff-0000-4111-8222-000000000004';
$insertInstance(
$conflictEventInstanceId,
$conflictEventUuid,
'conflict-event-settings-vm'
);
$conflictEventOperationId = '6234567890abcdef1234567890abcdef';
$conflictEventJobId = $insertJob(
1,
$conflictEventOperationId,
'awaiting_receipt',
1
);
Capsule::table('cloud_jobs')->where('id', $conflictEventJobId)->update([
'instance_id' => $conflictEventInstanceId,
'kind' => 'instance_settings',
'sealed_payload' => CloudSecrets::encrypt([
'method' => 'POST',
'request_target' => '/v1/vms/' . $conflictEventUuid
. '/settings',
'body' => [
'root_password' => 'Conflict-password9!',
'confirm_restart' => true,
],
'owner_id' => 1,
'settings_fields' => ['password'],
'credential' => [
'password' => 'Conflict-password9!',
'login_name' => 'root',
'auth_mode' => 'password',
],
], 'job-payload'),
]);
$assert(CloudWorkerService::expireDueTasksForOwner(
1,
$conflictEventJobId,
1
) === 1, 'The conflicting-event fixture did not expire.');
Capsule::table('cloud_jobs')->where('id', $conflictEventJobId)->update([
'effect_resolution_reason' => 'automatic_recovery:v1:'
. (time() - 21600) . ':12:pending',
'finalization_incident' => 1,
'finalization_error_code' => 'recovery_state_invalid',
]);
$conflictEventId = 'settings-conflict-event-000001';
Capsule::table('cloud_agent_completion_events')->insert([
'event_id' => $conflictEventId,
'host_id' => 7,
'operation_id' => $conflictEventOperationId,
'payload_sha256' => str_repeat('0', 64),
'outcome' => 'failed',
'effect_may_have_applied' => 0,
'safe_to_resubmit' => 1,
'error_code' => 'guest_update_failed',
'result_json' => json_encode(
['agent_detail' => 'does-not-match-hash'],
JSON_THROW_ON_ERROR
),
'disposition' => 'received',
'received_at' => time() + 1,
'applied_at' => null,
]);
$assert((new CloudAutomaticOperationRecoveryService())->tick(),
'The worker did not examine a conflicting persisted event.');
$conflictEventJob = CloudJob::find($conflictEventJobId);
$conflictEventRow = Capsule::table('cloud_agent_completion_events')
->where('event_id', $conflictEventId)->first();
$assert((string) $conflictEventRow->disposition === 'rejected'
&& (string) $conflictEventJob->effect_resolution === 'unknown'
&& (bool) $conflictEventJob->finalization_incident
&& (string) $conflictEventJob->finalization_error_code
=== 'recovery_receipt_invalid'
&& (int) $conflictEventJob->available_at >= time() + 290
&& CloudJobService::blockingQueryForInstance(
$conflictEventInstanceId
)->exists(),
'A conflicting persisted event altered the job or released its lock.');
// A self-consistent late receipt may still contradict an already prepared
// conclusion. Such conflicts are deterministic: atomically isolate the
// event once, retain the original marker/escrow/blocker, and schedule a
// later fresh inspection without polling Agent in the same tick.
$conflictEscrow = (string) $conflictEventJob->sealed_payload;
$conflictIdentity = $identityFor($conflictEventOperationId, false);
$noEffectOperation = $conflictIdentity + [
'effect_may_have_applied' => false,
'error_code' => 'settings_not_committed',
'operation_id' => $conflictEventOperationId,
'result' => [],
'safe_to_resubmit' => false,
'status' => 'failed',
];
$canonicalNoEffectOperation = $noEffectOperation;
ksort($canonicalNoEffectOperation, SORT_STRING);
$noEffectOperationHash = hash(
'sha256',
json_encode(
$canonicalNoEffectOperation,
JSON_THROW_ON_ERROR
| JSON_UNESCAPED_SLASHES
| JSON_UNESCAPED_UNICODE
)
);
$armConflictReceipt = static function (
string $status,
bool $safeToResubmit,
string $errorCode,
array $result,
string $marker
) use (
$conflictEventId,
$conflictEventJobId,
$conflictEventOperationId,
$storedEvent,
$withIdentity
): void {
$receipt = $withIdentity([
'type' => 'completion',
'id' => $conflictEventId,
'agent_id' => '7',
'operation_id' => $conflictEventOperationId,
'status' => $status,
'effect_may_have_applied' => false,
'safe_to_resubmit' => $safeToResubmit,
'error_code' => $errorCode,
'result' => $result,
]);
Capsule::table('cloud_jobs')->where(
'id',
$conflictEventJobId
)->update([
'effect_resolution_reason' => $marker,
'finalization_incident' => 0,
'finalization_error_code' => null,
'lease_token' => null,
'lease_until' => 0,
'available_at' => 0,
]);
$eventRow = $storedEvent($receipt, time());
Capsule::table('cloud_agent_completion_events')->where(
'event_id',
$conflictEventId
)->update($eventRow);
};
$assertPreparedConflict = static function (
string $marker,
string $message
) use (
$assert,
$conflictEscrow,
$conflictEventId,
$conflictEventInstanceId,
$conflictEventJobId
): void {
$assert((new CloudAutomaticOperationRecoveryService())->tick(),
$message . ' was not examined.');
$job = CloudJob::find($conflictEventJobId);
$event = Capsule::table('cloud_agent_completion_events')
->where('event_id', $conflictEventId)->first();
$updatedAt = (int) $job->updated_at;
$assert((string) $event->disposition === 'late'
&& $event->applied_at === null
&& (string) $job->effect_resolution === 'unknown'
&& (string) $job->effect_resolution_reason === $marker
&& (bool) $job->finalization_incident
&& (string) $job->finalization_error_code
=== 'recovery_evidence_conflict'
&& (string) $job->sealed_payload === $conflictEscrow
&& $job->lease_token === null
&& (int) $job->lease_until === 0
&& (int) $job->available_at >= time() + 290
&& CloudJobService::blockingQueryForInstance(
$conflictEventInstanceId
)->exists(),
$message . ' did not retain the fail-closed evidence fence.');
// Other newly recoverable kinds may be claimed here; the isolated job
// itself must remain untouched until its own retry time arrives.
(new CloudAutomaticOperationRecoveryService())->tick();
$afterSecondTick = CloudJob::find($conflictEventJobId);
$afterSecondEvent = Capsule::table(
'cloud_agent_completion_events'
)->where('event_id', $conflictEventId)->first();
$assert((int) $afterSecondTick->updated_at === $updatedAt
&& (string) $afterSecondTick->effect_resolution === 'unknown'
&& (string) $afterSecondTick->effect_resolution_reason === $marker
&& (bool) $afterSecondTick->finalization_incident
&& (string) $afterSecondTick->finalization_error_code
=== 'recovery_evidence_conflict'
&& (string) $afterSecondTick->sealed_payload === $conflictEscrow
&& (string) $afterSecondEvent->disposition === 'late'
&& $afterSecondEvent->applied_at === null
&& CloudJobService::blockingQueryForInstance(
$conflictEventInstanceId
)->exists(),
$message . ' ignored its retry backoff.');
};
$directionConflictMarker = 'automatic_recovery:v1:'
. (time() - 300)
. ':2:prepared:automatic_applied_succeeded_nofence:'
. str_repeat('1', 64) . ':' . $noEffectOperationHash;
$armConflictReceipt(
'failed',
true,
'guest_update_failed',
['agent_detail' => 'direction-conflict'],
$directionConflictMarker
);
$assertPreparedConflict(
$directionConflictMarker,
'Opposite-direction late evidence'
);
$hashConflictMarker = 'automatic_recovery:v1:'
. (time() - 300)
. ':2:prepared:automatic_not_applied_failed_nofence:'
. str_repeat('2', 64) . ':' . str_repeat('3', 64);
$armConflictReceipt(
'failed',
true,
'guest_update_failed',
['agent_detail' => 'hash-conflict'],
$hashConflictMarker
);
$assertPreparedConflict(
$hashConflictMarker,
'Different-operation-hash late evidence'
);
$invalidStateConflictMarker = 'invalid-recovery-state';
$armConflictReceipt(
'failed',
true,
'guest_update_failed',
['agent_detail' => 'invalid-state-conflict'],
$invalidStateConflictMarker
);
$assertPreparedConflict(
$invalidStateConflictMarker,
'Invalid-state late evidence'
);
$noEffectFenceConflictMarker = 'automatic_recovery:v1:'
. (time() - 300)
. ':2:prepared:automatic_not_applied_failed_fence:'
. str_repeat('4', 64) . ':' . $noEffectOperationHash;
$armConflictReceipt(
'failed',
true,
'guest_update_failed',
['agent_detail' => 'no-effect-fence-conflict'],
$noEffectFenceConflictMarker
);
$assertPreparedConflict(
$noEffectFenceConflictMarker,
'No-effect active-fence late evidence'
);
$appliedReceiptResult = ['settings_changed' => true];
$appliedOperation = $conflictIdentity + [
'effect_may_have_applied' => false,
'error_code' => '',
'operation_id' => $conflictEventOperationId,
'result' => $appliedReceiptResult,
'safe_to_resubmit' => false,
'status' => 'succeeded',
];
$canonicalAppliedOperation = $appliedOperation;
ksort($canonicalAppliedOperation, SORT_STRING);
$appliedOperationHash = hash(
'sha256',
json_encode(
$canonicalAppliedOperation,
JSON_THROW_ON_ERROR
| JSON_UNESCAPED_SLASHES
| JSON_UNESCAPED_UNICODE
)
);
$appliedFenceConflictMarker = 'automatic_recovery:v1:'
. (time() - 300)
. ':2:prepared:automatic_applied_failed_fence:'
. str_repeat('5', 64) . ':' . $appliedOperationHash;
$armConflictReceipt(
'succeeded',
false,
'',
$appliedReceiptResult,
$appliedFenceConflictMarker
);
$assertPreparedConflict(
$appliedFenceConflictMarker,
'Applied active-fence late evidence'
);
// A receipt can become durable while an automatic worker is outside the
// database resolving an Agent fence. Its stale commit must yield before
// publishing, so the next event-first turn checks compatibility.
$pendingReceiptSource = 'automatic_not_applied_failed_nofence:'
. str_repeat('6', 64);
$pendingReceiptMarker = 'automatic_recovery:v1:'
. (time() - 300) . ':2:prepared:' . $pendingReceiptSource
. ':' . $noEffectOperationHash;
$armConflictReceipt(
'failed',
false,
'settings_not_committed',
[],
$pendingReceiptMarker
);
$pendingReceiptCommitRejected = false;
try {
CloudOperationRecoveryService::commitNotApplied(
$conflictEventJobId,
$noEffectOperation,
$pendingReceiptSource,
0
);
} catch (\App\Services\Cloud\CloudError $error) {
$pendingReceiptCommitRejected = $error->httpStatus === 409;
}
$pendingReceiptJob = CloudJob::find($conflictEventJobId);
$pendingReceiptEvent = Capsule::table(
'cloud_agent_completion_events'
)->where('event_id', $conflictEventId)->first();
$assert($pendingReceiptCommitRejected
&& (string) $pendingReceiptJob->effect_resolution === 'unknown'
&& (string) $pendingReceiptJob->effect_resolution_reason
=== $pendingReceiptMarker
&& (string) $pendingReceiptEvent->disposition === 'received',
'An automatic commit ignored a newly durable completion receipt.');
$assert((new CloudAutomaticOperationRecoveryService())->tick(),
'The compatible priority receipt was not claimed after commit yield.');
$pendingReceiptResolved = CloudJob::find($conflictEventJobId);
$pendingReceiptEvent = Capsule::table(
'cloud_agent_completion_events'
)->where('event_id', $conflictEventId)->first();
$assert((string) $pendingReceiptResolved->effect_resolution
=== 'not_applied'
&& (string) $pendingReceiptResolved->effect_resolution_reason
=== 'late_completion_receipt:' . $conflictEventId
&& (string) $pendingReceiptEvent->disposition === 'applied'
&& !CloudJobService::blockingQueryForInstance(
$conflictEventInstanceId
)->exists(),
'The priority receipt did not safely win the automatic commit race.');
// A receipt can crash after its PHP prepare transaction but before the
// effect commit/event acknowledgement. Even if it replaced an exhausted
// automatic prepared intent, the dedicated recovery worker must rebuild
// the exact immutable event and finish PHP locally without an Agent
// retransmission or host request.
$lateCrashInstanceId = 95;
$lateCrashUuid = 'dddddddd-eeee-4fff-8000-000000000002';
$insertInstance(
$lateCrashInstanceId,
$lateCrashUuid,
'late-crash-settings-vm'
);
$lateCrashOperationId = '4234567890abcdef1234567890abcdef';
$lateCrashJobId = $insertJob(
1,
$lateCrashOperationId,
'awaiting_receipt',
1
);
Capsule::table('cloud_jobs')->where('id', $lateCrashJobId)->update([
'instance_id' => $lateCrashInstanceId,
'kind' => 'instance_settings',
'sealed_payload' => CloudSecrets::encrypt([
'method' => 'POST',
'request_target' => '/v1/vms/' . $lateCrashUuid
. '/settings',
'body' => [
'hostname' => 'late-crash.example',
'confirm_restart' => true,
],
'owner_id' => 1,
'settings_fields' => ['hostname'],
'credential' => null,
], 'job-payload'),
]);
$assert(CloudWorkerService::expireDueTasksForOwner(
1,
$lateCrashJobId,
1
) === 1, 'The late-receipt crash fixture did not expire.');
$lateCrashOperation = $identityFor($lateCrashOperationId, false) + [
'effect_may_have_applied' => false,
'error_code' => 'guest_update_failed',
'operation_id' => $lateCrashOperationId,
'result' => ['agent_detail' => 'bounded'],
'safe_to_resubmit' => true,
'status' => 'failed',
];
$canonicalLateCrashOperation = $lateCrashOperation;
ksort($canonicalLateCrashOperation, SORT_STRING);
$lateCrashOperationHash = hash(
'sha256',
json_encode(
$canonicalLateCrashOperation,
JSON_THROW_ON_ERROR
| JSON_UNESCAPED_SLASHES
| JSON_UNESCAPED_UNICODE
)
);
Capsule::table('cloud_jobs')->where('id', $lateCrashJobId)->update([
'effect_resolution_reason' => 'automatic_recovery:v1:'
. (time() - 21600)
. ':12:prepared:automatic_not_applied_failed_nofence:'
. str_repeat('a', 64) . ':' . $lateCrashOperationHash,
'finalization_incident' => 1,
'finalization_error_code' => 'recovery_limit_reached',
]);
$lateCrashEventId = 'settings-late-crash-event-0001';
$lateCrashReceipt = $withIdentity([
'type' => 'completion',
'id' => $lateCrashEventId,
'agent_id' => '7',
'operation_id' => $lateCrashOperationId,
'status' => 'failed',
'effect_may_have_applied' => false,
'safe_to_resubmit' => true,
'error_code' => 'guest_update_failed',
'result' => ['agent_detail' => 'bounded'],
]);
Capsule::table('cloud_agent_completion_events')->insert(
$storedEvent($lateCrashReceipt, time())
);
CloudOperationRecoveryService::prepareNotApplied(
$lateCrashJobId,
$lateCrashOperation,
'late_completion_receipt:' . $lateCrashEventId
);
$lateCrashPrepared = CloudJob::find($lateCrashJobId);
$legacyLateCrashPreparedMarker = (string)
$lateCrashPrepared->effect_resolution_reason;
$assert(str_starts_with(
$legacyLateCrashPreparedMarker,
'prepared:late_completion_receipt:' . $lateCrashEventId . ':'
)
&& !(bool) $lateCrashPrepared->finalization_incident
&& $lateCrashPrepared->finalization_error_code === null
&& (string) $lateCrashPrepared->effect_resolution === 'unknown',
'Exact late evidence did not replace exhausted matching prepare state.');
// Older crash windows could retain the stable late-receipt marker inside
// the mutable automatic-attempt envelope. The exact same receipt must
// upgrade that legacy wrapper without weakening operation-hash matching.
Capsule::table('cloud_jobs')->where('id', $lateCrashJobId)->update([
'effect_resolution_reason' => 'automatic_recovery:v1:'
. (time() - 21600) . ':12:'
. $legacyLateCrashPreparedMarker,
]);
$lateCrashPrepared = CloudJob::find($lateCrashJobId);
$lateCrashAudit = [
'phase' => (string) $lateCrashPrepared->phase,
'outcome' => (string) $lateCrashPrepared->outcome,
'status' => (string) $lateCrashPrepared->status,
'error_code' => (string) $lateCrashPrepared->error_code,
'result_json' => $lateCrashPrepared->result_json,
'terminal_at' => (int) $lateCrashPrepared->terminal_at,
'completion_event_id' => $lateCrashPrepared->completion_event_id,
];
$assert((new CloudAutomaticOperationRecoveryService())->tick(),
'The recovery worker did not resume a prepared late receipt.');
$lateCrashRecovered = CloudJob::find($lateCrashJobId);
$lateCrashEvent = Capsule::table('cloud_agent_completion_events')
->where('event_id', $lateCrashEventId)->first();
$assert((string) $lateCrashRecovered->effect_resolution
=== 'not_applied'
&& (string) $lateCrashRecovered->effect_resolution_reason
=== 'late_completion_receipt:' . $lateCrashEventId
&& (string) $lateCrashEvent->disposition === 'applied'
&& $lateCrashAudit === [
'phase' => (string) $lateCrashRecovered->phase,
'outcome' => (string) $lateCrashRecovered->outcome,
'status' => (string) $lateCrashRecovered->status,
'error_code' => (string) $lateCrashRecovered->error_code,
'result_json' => $lateCrashRecovered->result_json,
'terminal_at' => (int) $lateCrashRecovered->terminal_at,
'completion_event_id' =>
$lateCrashRecovered->completion_event_id,
]
&& !CloudJobService::blockingQueryForInstance(
$lateCrashInstanceId
)->exists(),
'Prepared late receipt did not finish PHP-only convergence.');
// A hash-valid receipt can still violate the kind-specific finalizer
// contract. This deterministic local error must become one incident, not
// an unbounded received-event loop every five minutes.
$invalidReceiptInstanceId = 98;
$invalidReceiptUuid = '99999999-aaaa-4bbb-8ccc-dddddddddddd';
$insertInstance(
$invalidReceiptInstanceId,
$invalidReceiptUuid,
'invalid-receipt-settings-vm'
);
$invalidReceiptOperationId = '7234567890abcdef1234567890abcdef';
$invalidReceiptJobId = $insertJob(
1,
$invalidReceiptOperationId,
'awaiting_receipt',
1
);
Capsule::table('cloud_jobs')->where(
'id',
$invalidReceiptJobId
)->update([
'instance_id' => $invalidReceiptInstanceId,
'kind' => 'instance_settings',
'sealed_payload' => CloudSecrets::encrypt([
'method' => 'POST',
'request_target' => '/v1/vms/' . $invalidReceiptUuid
. '/settings',
'body' => [
'hostname' => 'invalid-receipt.example',
'confirm_restart' => true,
],
'owner_id' => 1,
'settings_fields' => ['hostname'],
'credential' => null,
], 'job-payload'),
]);
$assert(CloudWorkerService::expireDueTasksForOwner(
1,
$invalidReceiptJobId,
1
) === 1, 'The invalid-receipt fixture did not expire.');
$invalidReceipt = $withIdentity([
'type' => 'completion',
'id' => 'settings-invalid-result-event-01',
'agent_id' => '7',
'operation_id' => $invalidReceiptOperationId,
'status' => 'succeeded',
'effect_may_have_applied' => false,
'safe_to_resubmit' => false,
'error_code' => '',
// The generic receipt is valid, but instance_settings requires the
// explicit settings_changed proof before touching its PHP ledger.
'result' => ['hostname' => 'invalid-receipt.example'],
]);
$assert(CloudOperationCompletionService::accept(
$invalidReceipt
) === ['accepted' => true, 'disposition' => 'late'],
'A kind-invalid exact receipt was not terminally parked.');
$invalidReceiptJob = CloudJob::find($invalidReceiptJobId);
$invalidReceiptEvent = Capsule::table(
'cloud_agent_completion_events'
)->where('event_id', $invalidReceipt['id'])->first();
$assert((string) $invalidReceiptJob->effect_resolution === 'unknown'
&& (bool) $invalidReceiptJob->finalization_incident
&& (string) $invalidReceiptJob->finalization_error_code
=== 'recovery_receipt_invalid'
&& (string) $invalidReceiptEvent->disposition === 'late'
&& (int) $invalidReceiptJob->available_at >= time() + 290
&& CloudJobService::blockingQueryForInstance(
$invalidReceiptInstanceId
)->exists(),
'An invalid receipt altered the ledger or released its blocker.');
// The incident is diagnostic, not a permanent queue state. A later fresh
// operation-bound inspection may prepare and publish a valid conclusion.
$freshInspectionOperation = $identityFor(
$invalidReceiptOperationId,
true
) + [
'operation_id' => $invalidReceiptOperationId,
'status' => 'succeeded',
'effect_may_have_applied' => false,
'safe_to_resubmit' => false,
'error_code' => '',
'result' => [
'settings_changed' => true,
'hostname' => 'invalid-receipt.example',
],
];
$freshInspectionSource =
'agent_inspection:fresh-after-receipt-incident';
CloudOperationRecoveryService::prepareApplied(
$invalidReceiptJobId,
$freshInspectionOperation,
$freshInspectionSource
);
CloudOperationRecoveryService::commitApplied(
$invalidReceiptJobId,
$freshInspectionOperation,
$freshInspectionSource,
0
);
$invalidReceiptAfter = CloudJob::find($invalidReceiptJobId);
$invalidReceiptSpec = json_decode(
(string) Capsule::table('cloud_instances')
->where('id', $invalidReceiptInstanceId)->value('spec_json'),
true,
flags: JSON_THROW_ON_ERROR
);
$assert((string) $invalidReceiptAfter->effect_resolution === 'applied'
&& (string) $invalidReceiptAfter->effect_resolution_reason
=== $freshInspectionSource
&& !(bool) $invalidReceiptAfter->finalization_incident
&& $invalidReceiptAfter->finalization_error_code === null
&& $invalidReceiptAfter->read_at === null
&& ($invalidReceiptSpec['guest_hostname'] ?? null)
=== 'invalid-receipt.example'
&& !CloudJobService::blockingQueryForInstance(
$invalidReceiptInstanceId
)->exists(),
'A fresh inspection could not converge after an invalid receipt.');
// A valid immutable receipt may be temporarily unable to update a damaged
// local projection. Age is audit metadata only: keep it retryable until an
// operator repairs PHP state, even beyond the former six-hour window.
$receiptLimitInstanceId = 99;
$receiptLimitUuid = 'aaaaaaaa-1111-4222-8333-bbbbbbbbbbbb';
$insertInstance(
$receiptLimitInstanceId,
$receiptLimitUuid,
'receipt-limit-settings-vm'
);
Capsule::table('cloud_instances')->where(
'id',
$receiptLimitInstanceId
)->update(['spec_json' => '{invalid-json']);
$receiptLimitOperationId = '8234567890abcdef1234567890abcdef';
$receiptLimitJobId = $insertJob(
1,
$receiptLimitOperationId,
'awaiting_receipt',
1
);
Capsule::table('cloud_jobs')->where(
'id',
$receiptLimitJobId
)->update([
'instance_id' => $receiptLimitInstanceId,
'kind' => 'instance_settings',
'sealed_payload' => CloudSecrets::encrypt([
'method' => 'POST',
'request_target' => '/v1/vms/' . $receiptLimitUuid
. '/settings',
'body' => [
'hostname' => 'receipt-limit.example',
'confirm_restart' => true,
],
'owner_id' => 1,
'settings_fields' => ['hostname'],
'credential' => null,
], 'job-payload'),
]);
$assert(CloudWorkerService::expireDueTasksForOwner(
1,
$receiptLimitJobId,
1
) === 1, 'The receipt-limit fixture did not expire.');
$receiptLimitReceipt = $withIdentity([
'type' => 'completion',
'id' => 'settings-receipt-limit-event-01',
'agent_id' => '7',
'operation_id' => $receiptLimitOperationId,
'status' => 'succeeded',
'effect_may_have_applied' => false,
'safe_to_resubmit' => false,
'error_code' => '',
'result' => [
'settings_changed' => true,
'hostname' => 'receipt-limit.example',
],
]);
$assert(CloudOperationCompletionService::accept(
$receiptLimitReceipt
) === ['accepted' => true, 'disposition' => 'received'],
'A repairable local finalizer error was parked immediately.');
Capsule::table('cloud_agent_completion_events')->where(
'event_id',
$receiptLimitReceipt['id']
)->update(['received_at' => time() - 21599]);
$assert((new CloudAutomaticOperationRecoveryService())->tick(),
'The recovery worker did not claim the near-limit receipt.');
$nearLimitJob = CloudJob::find($receiptLimitJobId);
$assert((string) Capsule::table('cloud_agent_completion_events')
->where('event_id', $receiptLimitReceipt['id'])
->value('disposition') === 'received'
&& (int) $nearLimitJob->available_at >= time() + 290,
'A repairable receipt did not retain its retry backoff.');
Capsule::table('cloud_agent_completion_events')->where(
'event_id',
$receiptLimitReceipt['id']
)->update(['received_at' => time() - 21601]);
Capsule::table('cloud_jobs')->where(
'id',
$receiptLimitJobId
)->update([
'available_at' => 0,
'lease_until' => 0,
'lease_token' => null,
]);
$assert((new CloudAutomaticOperationRecoveryService())->tick(),
'The recovery worker did not claim the six-hour-old receipt.');
$receiptLimitJob = CloudJob::find($receiptLimitJobId);
$receiptLimitEvent = CloudAgentCompletionEvent::find(
$receiptLimitReceipt['id']
);
$assert((string) $receiptLimitJob->effect_resolution === 'unknown'
&& (string) $receiptLimitEvent->disposition === 'received'
&& (int) $receiptLimitJob->available_at >= time() + 290
&& CloudJobService::blockingQueryForInstance(
$receiptLimitInstanceId
)->exists(),
'A six-hour-old receipt left the recovery queue.');
Capsule::table('cloud_instances')->where(
'id',
$receiptLimitInstanceId
)->update(['spec_json' => json_encode(
['managed' => true],
JSON_THROW_ON_ERROR
)]);
Capsule::table('cloud_jobs')->where(
'id',
$receiptLimitJobId
)->update([
'available_at' => 0,
'lease_until' => 0,
'lease_token' => null,
]);
$assert((new CloudAutomaticOperationRecoveryService())->tick(),
'The repaired six-hour-old receipt was not claimed.');
$receiptLimitJob = CloudJob::find($receiptLimitJobId);
$receiptLimitEvent = CloudAgentCompletionEvent::find(
$receiptLimitReceipt['id']
);
$receiptLimitSpec = json_decode(
(string) Capsule::table('cloud_instances')
->where('id', $receiptLimitInstanceId)->value('spec_json'),
true,
flags: JSON_THROW_ON_ERROR
);
$assert((string) $receiptLimitJob->effect_resolution === 'applied'
&& (string) $receiptLimitEvent->disposition === 'applied'
&& ($receiptLimitSpec['guest_hostname'] ?? null)
=== 'receipt-limit.example'
&& !CloudJobService::blockingQueryForInstance(
$receiptLimitInstanceId
)->exists(),
'A repaired six-hour-old receipt did not converge.');
// Decryptable escrow is not enough: a password operation whose retained
// credential DTO is missing must never be treated as a complete ledger
// finalization merely because the Agent receipt says settings_changed.
$damagedEscrowInstanceId = 100;
$damagedEscrowUuid = 'aaaaaaaa-2222-4333-8444-cccccccccccc';
$insertInstance(
$damagedEscrowInstanceId,
$damagedEscrowUuid,
'damaged-escrow-settings-vm'
);
$damagedEscrowOperationId = '9234567890abcdef1234567890abcdef';
$damagedEscrowJobId = $insertJob(
1,
$damagedEscrowOperationId,
'awaiting_receipt',
1
);
Capsule::table('cloud_jobs')->where(
'id',
$damagedEscrowJobId
)->update([
'instance_id' => $damagedEscrowInstanceId,
'kind' => 'instance_settings',
'sealed_payload' => CloudSecrets::encrypt([
'method' => 'POST',
'request_target' => '/v1/vms/' . $damagedEscrowUuid
. '/settings',
'body' => [
'root_password' => 'Escrow-damaged9!',
'confirm_restart' => true,
],
'owner_id' => 1,
'settings_fields' => ['password'],
'credential' => null,
], 'job-payload'),
]);
$assert(CloudWorkerService::expireDueTasksForOwner(
1,
$damagedEscrowJobId,
1
) === 1, 'The damaged-escrow fixture did not expire.');
$damagedEscrowPayload = (string) CloudJob::find(
$damagedEscrowJobId
)->sealed_payload;
$damagedEscrowReceipt = $withIdentity([
'type' => 'completion',
'id' => 'settings-damaged-escrow-event-01',
'agent_id' => '7',
'operation_id' => $damagedEscrowOperationId,
'status' => 'succeeded',
'effect_may_have_applied' => false,
'safe_to_resubmit' => false,
'error_code' => '',
'result' => ['settings_changed' => true],
]);
$assert(CloudOperationCompletionService::accept(
$damagedEscrowReceipt
) === ['accepted' => true, 'disposition' => 'received'],
'Damaged recovery escrow falsely converged from a valid receipt.');
$damagedEscrowJob = CloudJob::find($damagedEscrowJobId);
$assert((string) $damagedEscrowJob->effect_resolution === 'unknown'
&& (string) $damagedEscrowJob->sealed_payload
=== $damagedEscrowPayload
&& CloudJobService::blockingQueryForInstance(
$damagedEscrowInstanceId
)->exists(),
'Damaged escrow changed the ledger or released its blocker.');
Capsule::table('cloud_agent_completion_events')->where(
'event_id',
$damagedEscrowReceipt['id']
)->update(['received_at' => time() - 21601]);
$assert((new CloudAutomaticOperationRecoveryService())->tick(),
'Damaged escrow was not retried beyond six hours.');
$damagedEscrowJob = CloudJob::find($damagedEscrowJobId);
$damagedEscrowEvent = CloudAgentCompletionEvent::find(
$damagedEscrowReceipt['id']
);
$assert((string) $damagedEscrowJob->effect_resolution === 'unknown'
&& (string) $damagedEscrowEvent->disposition === 'received'
&& (int) $damagedEscrowJob->available_at >= time() + 290
&& CloudJobService::blockingQueryForInstance(
$damagedEscrowInstanceId
)->exists(),
'Damaged escrow left the recovery queue or released its blocker.');
// A structurally valid credential DTO must still match the setting being
// recovered. Otherwise a late receipt could overwrite credential metadata
// for a hostname-only change or forget that a newly installed SSH key is
// available.
$sshBlob = pack('N', strlen('ssh-ed25519')) . 'ssh-ed25519'
. pack('N', 32) . str_repeat("\x02", 32);
$recoveryPublicKey = 'ssh-ed25519 ' . base64_encode($sshBlob);
foreach ([
[
'label' => 'hostname-only credential',
'instance_id' => 103,
'uuid' => 'aaaaaaaa-4444-4555-8666-eeeeeeeeeeee',
'operation_id' => 'd234567890abcdef1234567890abcdef',
'event_id' => 'settings-hostname-credential-damaged-01',
'body' => [
'hostname' => 'credential-damaged.example',
'confirm_restart' => true,
],
'settings_fields' => ['hostname'],
'credential' => [
'password' => 'Credential-damaged9!',
'login_name' => 'root',
'auth_mode' => 'password',
],
'result' => [
'settings_changed' => true,
'hostname' => 'credential-damaged.example',
],
],
[
'label' => 'SSH-key credential mode',
'instance_id' => 104,
'uuid' => 'aaaaaaaa-5555-4666-8777-ffffffffffff',
'operation_id' => 'e234567890abcdef1234567890abcdef',
'event_id' => 'settings-ssh-credential-damaged-0001',
'body' => [
'ssh_public_key' => $recoveryPublicKey,
'confirm_restart' => true,
],
'settings_fields' => ['ssh_key'],
'credential' => [
'password' => 'Credential-damaged9!',
'login_name' => 'root',
'auth_mode' => 'password',
],
'result' => ['settings_changed' => true],
],
] as $credentialCase) {
$insertInstance(
$credentialCase['instance_id'],
$credentialCase['uuid'],
'credential-semantics-settings-vm-'
. $credentialCase['instance_id']
);
$credentialJobId = $insertJob(
1,
$credentialCase['operation_id'],
'awaiting_receipt',
1
);
Capsule::table('cloud_jobs')->where('id', $credentialJobId)->update([
'instance_id' => $credentialCase['instance_id'],
'kind' => 'instance_settings',
'sealed_payload' => CloudSecrets::encrypt([
'method' => 'POST',
'request_target' => '/v1/vms/' . $credentialCase['uuid']
. '/settings',
'body' => $credentialCase['body'],
'owner_id' => 1,
'settings_fields' => $credentialCase['settings_fields'],
'credential' => $credentialCase['credential'],
], 'job-payload'),
]);
$assert(CloudWorkerService::expireDueTasksForOwner(
1,
$credentialJobId,
1
) === 1, $credentialCase['label'] . ' fixture did not expire.');
$credentialPayload = (string) CloudJob::find(
$credentialJobId
)->sealed_payload;
$credentialReceipt = $withIdentity([
'type' => 'completion',
'id' => $credentialCase['event_id'],
'agent_id' => '7',
'operation_id' => $credentialCase['operation_id'],
'status' => 'succeeded',
'effect_may_have_applied' => false,
'safe_to_resubmit' => false,
'error_code' => '',
'result' => $credentialCase['result'],
]);
$assert(CloudOperationCompletionService::accept(
$credentialReceipt
) === ['accepted' => true, 'disposition' => 'received'],
$credentialCase['label']
. ' corruption falsely converged from a valid receipt.');
$credentialJob = CloudJob::find($credentialJobId);
$credentialEvent = CloudAgentCompletionEvent::find(
$credentialCase['event_id']
);
$assert((string) $credentialJob->effect_resolution === 'unknown'
&& (string) $credentialJob->sealed_payload === $credentialPayload
&& (string) $credentialEvent->disposition === 'received'
&& Capsule::table('cloud_instances')->where(
'id',
$credentialCase['instance_id']
)->value('sealed_credential') === null
&& CloudJobService::blockingQueryForInstance(
$credentialCase['instance_id']
)->exists(),
$credentialCase['label']
. ' corruption changed the ledger or released its blocker.');
Capsule::table('cloud_agent_completion_events')->where(
'event_id',
$credentialCase['event_id']
)->update(['received_at' => time() - 21601]);
$assert((new CloudAutomaticOperationRecoveryService())->tick(),
$credentialCase['label']
. ' receipt was not retried beyond six hours.');
$credentialJob = CloudJob::find($credentialJobId);
$credentialEvent = CloudAgentCompletionEvent::find(
$credentialCase['event_id']
);
$assert((string) $credentialJob->effect_resolution === 'unknown'
&& (string) $credentialEvent->disposition === 'received'
&& (int) $credentialJob->available_at >= time() + 290
&& CloudJobService::blockingQueryForInstance(
$credentialCase['instance_id']
)->exists(),
$credentialCase['label']
. ' corruption left the recovery queue or released its blocker.');
}
// A stored event whose immutable fields no longer match its accepted hash
// must be rejected before any settings projection or prepare marker moves.
$corruptEventInstanceId = 101;
$corruptEventUuid = 'aaaaaaaa-3333-4444-8555-dddddddddddd';
$insertInstance(
$corruptEventInstanceId,
$corruptEventUuid,
'corrupt-event-settings-vm'
);
$corruptEventOperationId = 'a234567890abcdef1234567890abcdef';
$corruptEventJobId = $insertJob(
1,
$corruptEventOperationId,
'awaiting_receipt',
1
);
Capsule::table('cloud_jobs')->where(
'id',
$corruptEventJobId
)->update([
'instance_id' => $corruptEventInstanceId,
'kind' => 'instance_settings',
'sealed_payload' => CloudSecrets::encrypt([
'method' => 'POST',
'request_target' => '/v1/vms/' . $corruptEventUuid
. '/settings',
'body' => [
'hostname' => 'corrupt-event.example',
'confirm_restart' => true,
],
'owner_id' => 1,
'settings_fields' => ['hostname'],
'credential' => null,
], 'job-payload'),
]);
$assert(CloudWorkerService::expireDueTasksForOwner(
1,
$corruptEventJobId,
1
) === 1, 'The corrupt-event fixture did not expire.');
$corruptEventId = 'settings-corrupt-stored-event-01';
$corruptAccepted = [
'event_id' => $corruptEventId,
'host_id' => 7,
'operation_id' => $corruptEventOperationId,
'status' => 'succeeded',
'effect_may_have_applied' => false,
'safe_to_resubmit' => false,
'error_code' => '',
'result' => [
'settings_changed' => true,
'hostname' => 'corrupt-event.example',
],
];
Capsule::table('cloud_agent_completion_events')->insert([
'event_id' => $corruptEventId,
'host_id' => 7,
'operation_id' => $corruptEventOperationId,
'payload_sha256' => hash('sha256', json_encode(
$corruptAccepted,
JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES
| JSON_UNESCAPED_UNICODE
)),
'outcome' => 'succeeded',
'effect_may_have_applied' => 0,
'safe_to_resubmit' => 0,
'error_code' => '',
// Simulate durable-row corruption after the original receipt hash was
// accepted. This must fail the exact-event reconstruction.
'result_json' => json_encode([
'settings_changed' => true,
'hostname' => 'tampered.example',
], JSON_THROW_ON_ERROR),
'disposition' => 'received',
'received_at' => time(),
'applied_at' => null,
]);
$corruptEventJobBefore = CloudJob::find($corruptEventJobId);
$corruptEventSpecBefore = (string) Capsule::table('cloud_instances')
->where('id', $corruptEventInstanceId)->value('spec_json');
$assert(CloudOperationRecoveryService::settleLateReceipt(
$corruptEventJobBefore,
CloudAgentCompletionEvent::find($corruptEventId)
) === 'rejected',
'A hash-inconsistent stored event was not rejected.');
$corruptEventJobAfter = CloudJob::find($corruptEventJobId);
$assert((string) $corruptEventJobAfter->effect_resolution === 'unknown'
&& (string) $corruptEventJobAfter->effect_resolution_reason
=== (string) $corruptEventJobBefore->effect_resolution_reason
&& (string) $corruptEventJobAfter->sealed_payload
=== (string) $corruptEventJobBefore->sealed_payload
&& (string) Capsule::table('cloud_instances')
->where('id', $corruptEventInstanceId)->value('spec_json')
=== $corruptEventSpecBefore,
'A hash-inconsistent event changed PHP settings state.');
// Agent VM metadata may include a hostname even when the original setting
// changed only a password. Preserve that exact bounded observation in both
// recovery sources; the settings finalizer ignores it for password-only
// requests instead of rewriting the receipt into a synthetic result.
$extraHostnameInstanceId = 102;
$extraHostnameUuid = 'aaaaaaaa-4444-4555-8666-eeeeeeeeeeee';
$insertInstance(
$extraHostnameInstanceId,
$extraHostnameUuid,
'extra-hostname-settings-vm'
);
$extraHostnameOperationId = 'b234567890abcdef1234567890abcdef';
$extraHostnameJobId = $insertJob(
1,
$extraHostnameOperationId,
'awaiting_receipt',
1
);
Capsule::table('cloud_jobs')->where(
'id',
$extraHostnameJobId
)->update([
'instance_id' => $extraHostnameInstanceId,
'kind' => 'instance_settings',
'sealed_payload' => CloudSecrets::encrypt([
'method' => 'POST',
'request_target' => '/v1/vms/' . $extraHostnameUuid
. '/settings',
'body' => [
'root_password' => 'Extra-hostname9!Pass',
'confirm_restart' => true,
],
'owner_id' => 1,
'settings_fields' => ['password'],
'credential' => [
'password' => 'Extra-hostname9!Pass',
'login_name' => 'root',
'auth_mode' => 'password',
],
], 'job-payload'),
]);
$assert(CloudWorkerService::expireDueTasksForOwner(
1,
$extraHostnameJobId,
1
) === 1, 'The extra-hostname fixture did not expire.');
Capsule::table('cloud_jobs')->where(
'id',
$extraHostnameJobId
)->update([
'effect_resolution_reason' => 'automatic_recovery:v1:'
. (time() - 60) . ':1:pending',
]);
$extraHostnameOperation = $identityFor(
$extraHostnameOperationId,
true
) + [
'operation_id' => $extraHostnameOperationId,
'status' => 'succeeded',
'effect_may_have_applied' => false,
'safe_to_resubmit' => false,
'error_code' => '',
'result' => [
'uuid' => $extraHostnameUuid,
'settings_changed' => true,
'hostname' => 'metadata-only.example',
],
];
CloudOperationRecoveryService::prepareApplied(
$extraHostnameJobId,
$extraHostnameOperation,
'automatic_applied_succeeded_nofence:' . str_repeat('7', 64)
);
$extraHostnameReceipt = $withIdentity([
'type' => 'completion',
'id' => 'settings-extra-hostname-event-01',
'agent_id' => '7',
'operation_id' => $extraHostnameOperationId,
'status' => 'succeeded',
'effect_may_have_applied' => false,
'safe_to_resubmit' => false,
'error_code' => '',
'result' => [
'uuid' => $extraHostnameUuid,
'settings_changed' => true,
'hostname' => 'metadata-only.example',
],
]);
$assert(CloudOperationCompletionService::accept(
$extraHostnameReceipt
) === ['accepted' => true, 'disposition' => 'applied'],
'An unrelated receipt hostname conflicted with password recovery.');
$extraHostnameJob = CloudJob::find($extraHostnameJobId);
$assert((string) $extraHostnameJob->effect_resolution === 'applied'
&& (string) $extraHostnameJob->effect_resolution_reason
=== 'late_completion_receipt:' . $extraHostnameReceipt['id']
&& !(bool) $extraHostnameJob->finalization_incident
&& !CloudJobService::blockingQueryForInstance(
$extraHostnameInstanceId
)->exists(),
'Compatible password recovery did not release its blocker.');
// A failed receipt which still reports possible effect is not no-effect
// evidence. It remains late and the original unknown-effect fence stays.
$ambiguousInstanceId = 94;
$ambiguousUuid = 'cccccccc-dddd-4eee-8fff-000000000001';
$insertInstance(
$ambiguousInstanceId,
$ambiguousUuid,
'ambiguous-settings-vm'
);
$ambiguousOperationId = '3234567890abcdef1234567890abcdef';
$ambiguousJobId = $insertJob(
1,
$ambiguousOperationId,
'awaiting_receipt',
1
);
Capsule::table('cloud_jobs')->where('id', $ambiguousJobId)->update([
'instance_id' => $ambiguousInstanceId,
'kind' => 'instance_settings',
'sealed_payload' => CloudSecrets::encrypt([
'method' => 'POST',
'request_target' => '/v1/vms/' . $ambiguousUuid . '/settings',
'body' => [
'hostname' => 'ambiguous.example',
'confirm_restart' => true,
],
'owner_id' => 1,
'settings_fields' => ['hostname'],
'credential' => null,
], 'job-payload'),
]);
$assert(CloudWorkerService::expireDueTasksForOwner(
1,
$ambiguousJobId,
1
) === 1, 'The ambiguous late receipt fixture did not expire.');
$ambiguousReceipt = $withIdentity([
'type' => 'completion',
'id' => 'settings-ambiguous-event-0001',
'agent_id' => '7',
'operation_id' => $ambiguousOperationId,
'status' => 'failed',
'effect_may_have_applied' => true,
'safe_to_resubmit' => false,
'error_code' => 'operation_failed',
'result' => null,
]);
$assert(CloudOperationCompletionService::accept(
$ambiguousReceipt
) === ['accepted' => true, 'disposition' => 'late']
&& (string) CloudJob::find($ambiguousJobId)->effect_resolution
=== 'unknown'
&& CloudJobService::blockingQueryForInstance(
$ambiguousInstanceId
)->exists(),
'Ambiguous late failure incorrectly released the settings fence.');
// Live inspection uses the same PHP-only recovery seam in two phases:
// first converge the ledger while preserving the blocker, then publish
// the resolution only after the caller releases the Agent fence.
$preparedInstanceId = 92;
$preparedUuid = 'aaaaaaaa-bbbb-4ccc-8ddd-eeeeeeeeeeee';
$insertInstance(
$preparedInstanceId,
$preparedUuid,
'prepared-settings-vm'
);
$preparedOperationId = '1234567890abcdef1234567890abcdef';
$preparedJobId = $insertJob(
1,
$preparedOperationId,
'awaiting_receipt',
1
);
Capsule::table('cloud_jobs')->where('id', $preparedJobId)->update([
'instance_id' => $preparedInstanceId,
'kind' => 'instance_settings',
'sealed_payload' => CloudSecrets::encrypt([
'method' => 'PATCH',
'request_target' => '/v1/vms/' . $preparedUuid . '/settings',
'body' => [
'hostname' => 'prepared-recovered.example',
'root_password' => 'Prepared-password9!',
'confirm_restart' => true,
],
'owner_id' => 1,
'settings_fields' => ['hostname', 'password'],
'credential' => [
'password' => 'Prepared-password9!',
'login_name' => 'root',
'auth_mode' => 'password',
],
], 'job-payload'),
]);
$assert(CloudWorkerService::expireDueTasksForOwner(
1,
$preparedJobId,
1
) === 1, 'The two-phase recovery fixture did not reach terminal failure.');
$preparedTerminal = CloudJob::find($preparedJobId);
$preparedTerminalAt = (int) $preparedTerminal->terminal_at;
$preparedError = (string) $preparedTerminal->error_code;
$preparedResult = $preparedTerminal->result_json;
$preparedCompletionEvent = $preparedTerminal->completion_event_id;
$preparedReadAt = $preparedTerminal->read_at;
$preparedOperation = $identityFor($preparedOperationId, true) + [
'operation_id' => $preparedOperationId,
'status' => 'succeeded',
'effect_may_have_applied' => false,
'safe_to_resubmit' => false,
'error_code' => '',
'result' => [
'settings_changed' => true,
'hostname' => 'prepared-recovered.example',
],
];
$preparedEvidence =
'agent_inspection:settings-inspection-proof-0001';
$prepared = CloudOperationRecoveryService::prepareApplied(
$preparedJobId,
$preparedOperation,
$preparedEvidence
);
$preparedPayload = CloudSecrets::decrypt(
(string) $prepared->sealed_payload,
'job-payload'
);
$preparedInstance = Capsule::table('cloud_instances')
->where('id', $preparedInstanceId)->first();
$preparedSpec = json_decode(
(string) $preparedInstance->spec_json,
true,
flags: JSON_THROW_ON_ERROR
);
$preparedUpdatedAt = (int) $prepared->updated_at;
$preparedAgain = CloudOperationRecoveryService::prepareApplied(
$preparedJobId,
$preparedOperation,
$preparedEvidence
);
$assert((string) $prepared->phase === 'terminal'
&& (string) $prepared->outcome === 'failed'
&& (string) $prepared->status === 'failed'
&& (bool) $prepared->effect_may_have_applied
&& !(bool) $prepared->safe_to_resubmit
&& (string) $prepared->effect_resolution === 'unknown'
&& str_starts_with(
(string) $prepared->effect_resolution_reason,
'prepared:' . $preparedEvidence . ':'
)
&& (int) $prepared->terminal_at === $preparedTerminalAt
&& (string) $prepared->error_code === $preparedError
&& $prepared->result_json === $preparedResult
&& $prepared->completion_event_id === $preparedCompletionEvent
&& $prepared->read_at === $preparedReadAt
&& ($preparedPayload['credential']['password'] ?? null)
=== 'Prepared-password9!'
&& ($preparedSpec['guest_hostname'] ?? null)
=== 'prepared-recovered.example'
&& CloudJobService::blockingQueryForInstance(
$preparedInstanceId
)->exists()
&& (int) $preparedAgain->updated_at === $preparedUpdatedAt,
'Recovery prepare did not atomically converge while retaining fences.');
foreach (['commitApplied', 'commitNotApplied'] as $commitMethod) {
$negativeResolverRejected = false;
try {
$commit = [CloudOperationRecoveryService::class, $commitMethod];
$commit(
$preparedJobId,
$preparedOperation,
$preparedEvidence,
-1
);
} catch (\App\Services\Cloud\CloudError $error) {
$negativeResolverRejected = $error->httpStatus === 409;
}
$assert($negativeResolverRejected
&& (string) CloudJob::find($preparedJobId)->effect_resolution
=== 'unknown',
$commitMethod . ' accepted a negative recovery audit actor.');
}
$committed = CloudOperationRecoveryService::commitApplied(
$preparedJobId,
$preparedOperation,
$preparedEvidence
);
$committedPayload = CloudSecrets::decrypt(
(string) $committed->sealed_payload,
'job-payload'
);
$assert((string) $committed->phase === 'terminal'
&& (string) $committed->outcome === 'failed'
&& (string) $committed->status === 'failed'
&& (bool) $committed->effect_may_have_applied
&& !(bool) $committed->safe_to_resubmit
&& (string) $committed->effect_resolution === 'applied'
&& (string) $committed->effect_resolution_reason
=== $preparedEvidence
&& (int) $committed->effect_resolved_at > 0
&& (int) $committed->terminal_at === $preparedTerminalAt
&& (string) $committed->error_code === $preparedError
&& $committed->result_json === $preparedResult
&& $committed->completion_event_id === $preparedCompletionEvent
&& $committed->read_at === $preparedReadAt
&& !array_key_exists('credential', $committedPayload)
&& !CloudJobService::blockingQueryForInstance(
$preparedInstanceId
)->exists(),
'Recovery commit did not publish the effect and clear its escrow.');
$committedAgain = CloudOperationRecoveryService::commitApplied(
$preparedJobId,
$preparedOperation,
$preparedEvidence
);
$mismatchedPreparedOperation = $preparedOperation;
$mismatchedPreparedOperation['operation_id'] =
'00000000000000000000000000000001';
$mismatchedCommitRejected = false;
try {
CloudOperationRecoveryService::commitApplied(
$preparedJobId,
$mismatchedPreparedOperation,
$preparedEvidence
);
} catch (\App\Services\Cloud\CloudError $error) {
$mismatchedCommitRejected = $error->httpStatus === 409;
}
$assert((string) $committedAgain->effect_resolution === 'applied'
&& $mismatchedCommitRejected,
'Recovery commit was not idempotent or accepted another operation.');
$insertPaymentJob = static function (string $key, int $invoiceId)
use ($now): int {
$operationId = md5('payment-deadline:' . $key);
return (int) Capsule::table('cloud_jobs')->insertGetId([
'operation_id' => $operationId,
'request_idempotency_key' => $key,
'request_fingerprint' => hash('sha256', $key),
'parent_job_id' => null, 'order_id' => null,
'invoice_id' => $invoiceId, 'instance_id' => null,
'snapshot_id' => null, 'user_id' => 1, 'host_id' => null,
'kind' => 'payment_create', 'execution_mode' => 'local',
'phase' => 'dispatching', 'outcome' => null,
'status' => 'running',
'sealed_payload' => CloudSecrets::encrypt([
'input' => ['idempotency_key' => $key,
'method' => 'paytaro'],
], 'job-payload'),
'attempts' => 1, 'dispatch_attempts' => 1,
'dispatch_deadline_at' => $now - 20,
'dispatch_started_at' => $now - 30,
'agent_accepted_at' => null,
'receipt_deadline_at' => $now - 10,
'terminal_at' => null,
'effect_may_have_applied' => null,
'safe_to_resubmit' => null,
'effect_resolution' => 'not_applicable',
'effect_resolved_at' => null, 'effect_resolved_by' => null,
'effect_resolution_reason' => '',
'completion_event_id' => null, 'read_at' => null,
'dispatched_at' => $now - 30,
'lease_token' => bin2hex(random_bytes(16)),
'lease_until' => $now + 600, 'available_at' => 0,
'error_code' => null, 'result_json' => null,
'created_at' => $now - 60, 'updated_at' => $now - 30,
]);
};
$missingPaymentKey = 'payment-missing-0001';
$pendingPaymentKey = 'payment-pending-0001';
$unknownPaymentKey = 'payment-unknown-0001';
$expiredPaymentKey = 'payment-expired-0001';
$failedPaymentKey = 'payment-failed-0001';
Capsule::table('cloud_invoice_payments')->insert([
[
'invoice_id' => 502, 'user_id' => 1, 'provider' => 'paytaro',
'provider_order_no' => 'provider-pending-0001',
'idempotency_key' => $pendingPaymentKey,
'amount' => 10, 'status' => 'pending',
'created_at' => $now - 30, 'updated_at' => $now - 30,
],
[
'invoice_id' => 503, 'user_id' => 1, 'provider' => 'paytaro',
'provider_order_no' => 'provider-unknown-0001',
'idempotency_key' => $unknownPaymentKey,
'amount' => 10, 'status' => 'creation_unknown',
'created_at' => $now - 30, 'updated_at' => $now - 30,
],
[
'invoice_id' => 504, 'user_id' => 1, 'provider' => 'usdt',
'provider_order_no' => 'provider-expired-0001',
'idempotency_key' => $expiredPaymentKey,
'amount' => 10, 'status' => 'expired',
'created_at' => $now - 30, 'updated_at' => $now - 30,
],
[
'invoice_id' => 505, 'user_id' => 1,
'provider' => 'stripe_card',
'provider_order_no' => 'provider-failed-0001',
'idempotency_key' => $failedPaymentKey,
'amount' => 10, 'status' => 'failed',
'created_at' => $now - 30, 'updated_at' => $now - 30,
],
]);
$missingPaymentJobId = $insertPaymentJob($missingPaymentKey, 501);
$pendingPaymentJobId = $insertPaymentJob($pendingPaymentKey, 502);
$unknownPaymentJobId = $insertPaymentJob($unknownPaymentKey, 503);
$expiredPaymentJobId = $insertPaymentJob($expiredPaymentKey, 504);
$failedPaymentJobId = $insertPaymentJob($failedPaymentKey, 505);
foreach ([$missingPaymentJobId, $pendingPaymentJobId,
$unknownPaymentJobId, $expiredPaymentJobId,
$failedPaymentJobId] as $paymentJobId) {
$assert(CloudWorkerService::expireDueTasksForOwner(
1, $paymentJobId, 1
) === 1, 'A due local payment task was not terminalized.');
}
$missingPaymentJob = CloudJob::find($missingPaymentJobId);
$pendingPaymentJob = CloudJob::find($pendingPaymentJobId);
$unknownPaymentJob = CloudJob::find($unknownPaymentJobId);
$expiredPaymentJob = CloudJob::find($expiredPaymentJobId);
$failedPaymentJob = CloudJob::find($failedPaymentJobId);
$pendingPaymentResult = json_decode(
(string) $pendingPaymentJob->result_json, true,
flags: JSON_THROW_ON_ERROR
);
$assert($missingPaymentJob->phase === 'terminal'
&& $missingPaymentJob->outcome === 'failed'
&& $missingPaymentJob->effect_resolution === 'not_applied'
&& !(bool) $missingPaymentJob->effect_may_have_applied
&& (bool) $missingPaymentJob->safe_to_resubmit
&& $missingPaymentJob->error_code === 'payment_order_not_dispatched'
&& $pendingPaymentJob->phase === 'terminal'
&& $pendingPaymentJob->outcome === 'succeeded'
&& ($pendingPaymentResult['payment_id'] ?? null) === '1'
&& $unknownPaymentJob->phase === 'terminal'
&& $unknownPaymentJob->outcome === 'failed'
&& $unknownPaymentJob->effect_resolution === 'unknown'
&& (bool) $unknownPaymentJob->effect_may_have_applied
&& !(bool) $unknownPaymentJob->safe_to_resubmit
&& $unknownPaymentJob->error_code === 'payment_creation_unknown'
&& $expiredPaymentJob->phase === 'terminal'
&& $expiredPaymentJob->outcome === 'succeeded'
&& $expiredPaymentJob->effect_resolution === 'applied'
&& $failedPaymentJob->phase === 'terminal'
&& $failedPaymentJob->outcome === 'succeeded'
&& $failedPaymentJob->effect_resolution === 'applied'
&& Capsule::table('cloud_invoice_payments')->count() === 4,
'Payment deadline fallback retried a provider POST or misclassified its durable attempt.');
$nullDispatchOperation = md5('null-dispatch-deadline-fixture');
$nullDispatchJobId = $insertJob(
1,
$nullDispatchOperation,
'accepted',
0
);
Capsule::table('cloud_jobs')->where('id', $nullDispatchJobId)->update([
'dispatch_deadline_at' => null,
]);
$invalidReceiptOperation = md5('invalid-receipt-deadline-fixture');
$invalidReceiptJobId = $insertJob(
1,
$invalidReceiptOperation,
'dispatching',
1
);
Capsule::table('cloud_jobs')->where('id', $invalidReceiptJobId)->update([
'receipt_deadline_at' => 0,
]);
$assert(CloudWorkerService::expireDueTasksForOwner(
1,
$nullDispatchJobId,
1
) === 1, 'An accepted task with a NULL dispatch deadline stayed active.');
$assert(CloudWorkerService::expireDueTasksForOwner(
1,
$invalidReceiptJobId,
1
) === 1, 'A dispatched task with an invalid receipt deadline stayed active.');
$nullDispatchJob = CloudJob::find($nullDispatchJobId);
$invalidReceiptJob = CloudJob::find($invalidReceiptJobId);
$assert($nullDispatchJob !== null
&& (string) $nullDispatchJob->phase === 'terminal'
&& (string) $nullDispatchJob->outcome === 'failed'
&& !(bool) $nullDispatchJob->effect_may_have_applied
&& (bool) $nullDispatchJob->safe_to_resubmit
&& $invalidReceiptJob !== null
&& (string) $invalidReceiptJob->phase === 'terminal'
&& (string) $invalidReceiptJob->outcome === 'failed'
&& (bool) $invalidReceiptJob->effect_may_have_applied
&& !(bool) $invalidReceiptJob->safe_to_resubmit,
'Malformed active deadlines did not fail closed by dispatch evidence.');
$assert(CloudJobService::dispatchDeadlineSeconds('action') === 300
&& CloudJobService::dispatchDeadlineSeconds(
'host_storage_configure') === 60
&& CloudJobService::receiptDeadlineSeconds('action') === 960
&& CloudJobService::receiptDeadlineSeconds(
'host_storage_configure') === 180
&& CloudJobService::receiptDeadlineSeconds(
'template_import_replica') === 8100
&& CloudJobService::receiptDeadlineSeconds(
'rescue_catalog_validate_replica') === 8100
&& CloudJobService::receiptDeadlineSeconds(
'rescue_catalog_import_replica') === 8100
&& CloudJobService::receiptDeadlineSeconds(
'user_iso_import_replica') === 8100
&& CloudJobService::aggregateDeadlineSeconds(
'template_metadata_sync') === 1560
&& CloudJobService::aggregateDeadlineSeconds(
'template_import') === 8700
&& CloudJobService::aggregateDeadlineSeconds(
'user_iso_import') === 8700,
'Task deadlines lost dispatch, receipt, or aggregate finalization margin.');
$parentOperation = str_repeat('d', 32);
$parentId = (int) Capsule::table('cloud_jobs')->insertGetId([
'operation_id' => $parentOperation,
'request_idempotency_key' => $parentOperation,
'request_fingerprint' => hash('sha256', $parentOperation),
'parent_job_id' => null, 'order_id' => null, 'invoice_id' => null,
'instance_id' => null, 'snapshot_id' => null,
'user_id' => 1, 'host_id' => null, 'kind' => 'test_aggregate',
'execution_mode' => 'aggregate', 'phase' => 'awaiting_receipt',
'outcome' => null, 'status' => 'running',
'sealed_payload' => CloudSecrets::encrypt([], 'job-payload'),
'attempts' => 0, 'dispatch_attempts' => 0,
'dispatch_deadline_at' => $now - 30,
'dispatch_started_at' => null, 'agent_accepted_at' => null,
'receipt_deadline_at' => $now - 1, 'terminal_at' => null,
'effect_may_have_applied' => null, 'safe_to_resubmit' => null,
'effect_resolution' => 'not_applicable',
'effect_resolved_at' => null, 'effect_resolved_by' => null,
'effect_resolution_reason' => '', 'completion_event_id' => null,
'read_at' => null, 'dispatched_at' => null,
'lease_token' => str_repeat('9', 32), 'lease_until' => $now + 600,
'available_at' => 0, 'error_code' => null, 'result_json' => null,
'created_at' => $now - 100, 'updated_at' => $now - 50,
]);
$childOperation = str_repeat('8', 32);
$childId = (int) Capsule::table('cloud_jobs')->insertGetId([
'operation_id' => $childOperation,
'request_idempotency_key' => $childOperation,
'request_fingerprint' => hash('sha256', $childOperation),
'parent_job_id' => $parentId, 'order_id' => null,
'invoice_id' => null, 'instance_id' => null, 'snapshot_id' => null,
'user_id' => 1, 'host_id' => 7, 'kind' => 'test_mutation',
'execution_mode' => 'agent', 'phase' => 'accepted',
'outcome' => null, 'status' => 'queued',
'sealed_payload' => CloudSecrets::encrypt([
'method' => 'POST', 'request_target' => '/v1/test',
'body' => ['operation_id' => $childOperation], 'owner_id' => 1,
], 'job-payload'),
'attempts' => 0, 'dispatch_attempts' => 0,
'dispatch_deadline_at' => $now + 600,
'dispatch_started_at' => null, 'agent_accepted_at' => null,
'receipt_deadline_at' => null, 'terminal_at' => null,
'effect_may_have_applied' => null, 'safe_to_resubmit' => null,
'effect_resolution' => 'not_applicable',
'effect_resolved_at' => null, 'effect_resolved_by' => null,
'effect_resolution_reason' => '', 'completion_event_id' => null,
'read_at' => null, 'dispatched_at' => null,
'lease_token' => null, 'lease_until' => 0, 'available_at' => 0,
'error_code' => null, 'result_json' => null,
'created_at' => $now - 80, 'updated_at' => $now - 40,
]);
Capsule::unprepared('CREATE TRIGGER simulate_child_claim_cas_miss '
. 'BEFORE UPDATE OF lease_token ON cloud_jobs '
. 'WHEN OLD.id = ' . $childId . " AND OLD.phase <> 'terminal' "
. 'BEGIN SELECT RAISE(IGNORE); END');
$assert(CloudWorkerService::expireDueTasksForOwner(
1, $parentId, 10
) === 1, 'Owner fallback did not claim an expired aggregate parent.');
$parent = CloudJob::find($parentId);
$child = CloudJob::find($childId);
$assert($parent !== null && $child !== null
&& (string) $parent->phase === 'awaiting_receipt'
&& $parent->outcome === null
&& $parent->lease_token === null
&& (int) $parent->lease_until === 0
&& (string) $child->phase === 'accepted',
'A child claim CAS miss hid active work behind a terminal parent.');
Capsule::unprepared('DROP TRIGGER simulate_child_claim_cas_miss');
$assert(CloudWorkerService::expireDueTasksForOwner(
1, $parentId, 10
) === 1, 'A later fallback pass did not reclaim the aggregate parent.');
$parent = CloudJob::find($parentId);
$child = CloudJob::find($childId);
$assert($parent !== null && $child !== null
&& (string) $child->phase === 'terminal'
&& (string) $child->outcome === 'failed'
&& !(bool) $child->effect_may_have_applied
&& (bool) $child->safe_to_resubmit
&& (string) $parent->phase === 'terminal'
&& (string) $parent->outcome === 'failed',
'A later aggregate fallback did not terminalize children first.');
echo "Owner-scoped task deadline fallback tests passed.\n";
}