You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
virt/web/tests/CloudAutomaticOperationReco...

3303 lines
138 KiB
PHTML

<?php
declare(strict_types=1);
namespace App\Services\Cloud {
/** Isolated encrypted envelope for automatic-recovery tests. */
final class CloudSecrets
{
private static array $values = [];
public static function encrypt(array $value, string $purpose): string
{
$key = bin2hex(random_bytes(16));
self::$values[$key] = $value;
return 'automatic-recovery-test:' . $key;
}
public static function decrypt(string $value, string $purpose): array
{
$key = substr($value, strlen('automatic-recovery-test:'));
if (!array_key_exists($key, self::$values)) {
throw new \RuntimeException('Missing recovery test payload.');
}
return self::$values[$key];
}
}
final class AutomaticRecoveryTestAgentClient
{
public array $requests = [];
public array $responses = [];
public function systemRecoveryRequest(
string $method,
string $target,
string $operationId,
?array $payload = null
): array {
$request = compact(
'method', 'target', 'operationId', 'payload'
);
$this->requests[] = $request;
if ($this->responses === []) {
throw new \RuntimeException('Unexpected Agent request.');
}
$response = array_shift($this->responses);
if (is_callable($response)) {
$response = $response($request);
}
if ($response instanceof \Throwable) {
throw $response;
}
if (!is_array($response)) {
throw new \RuntimeException('Invalid test Agent response.');
}
return $response;
}
}
final class CloudAgentSettingsService
{
public static ?AutomaticRecoveryTestAgentClient $client = null;
public static function clientForConfiguration(
int $hostId,
float $timeoutSeconds = 120.0
): AutomaticRecoveryTestAgentClient {
if ($hostId !== 7 || $timeoutSeconds !== 30.0
|| self::$client === null) {
throw new \RuntimeException(
'Unexpected automatic recovery client request.'
);
}
return self::$client;
}
}
}
namespace {
require dirname(__DIR__) . '/vendor/autoload.php';
use App\Models\CloudJob;
use App\Services\Cloud\AutomaticRecoveryTestAgentClient;
use App\Services\Cloud\CloudAgentError;
use App\Services\Cloud\CloudAgentSettingsService;
use App\Services\Cloud\CloudAutomaticOperationRecoveryService;
use App\Services\Cloud\CloudError;
use App\Services\Cloud\CloudJobService;
use App\Services\Cloud\CloudOperationCompletionService;
use App\Services\Cloud\CloudOperationRecoveryIdentity;
use App\Services\Cloud\CloudOperationRecoveryService;
use App\Services\Cloud\CloudRecoveryInstanceStateConflict;
use App\Services\Cloud\CloudRecoveryOperationIdentityConflict;
use App\Services\Cloud\CloudSecrets;
use App\Services\Cloud\CloudWorkerService;
use Illuminate\Database\Capsule\Manager as Capsule;
use Illuminate\Database\Schema\Blueprint;
$assert = static function (bool $condition, string $message): void {
if (!$condition) {
throw new RuntimeException($message);
}
};
$capsule = new Capsule();
$capsule->addConnection([
'driver' => 'sqlite', 'database' => ':memory:',
], 'default');
$capsule->setAsGlobal();
$capsule->bootEloquent();
$schema = $capsule->schema('default');
$schema->create('user', static function (Blueprint $table): void {
$table->increments('id');
$table->integer('enable')->default(1);
$table->integer('is_admin')->default(0);
});
$schema->create('cloud_jobs', static function (Blueprint $table): void {
$table->increments('id');
$table->string('operation_id')->unique();
$table->string('request_idempotency_key')->unique();
$table->string('request_fingerprint');
$table->integer('parent_job_id')->nullable();
$table->integer('order_id')->nullable();
$table->integer('invoice_id')->nullable();
$table->integer('instance_id')->nullable();
$table->integer('snapshot_id')->nullable();
$table->integer('public_network_transition_id')->nullable();
$table->integer('vpc_repair_root_job_id')->nullable();
$table->integer('user_id');
$table->integer('host_id')->nullable();
$table->string('kind');
$table->string('execution_mode');
$table->string('phase');
$table->string('outcome')->nullable();
$table->string('status');
$table->text('sealed_payload');
$table->integer('attempts')->default(0);
$table->integer('dispatch_attempts')->default(0);
$table->integer('dispatch_deadline_at');
$table->integer('dispatch_started_at')->nullable();
$table->integer('agent_accepted_at')->nullable();
$table->integer('receipt_deadline_at')->nullable();
$table->integer('terminal_at')->nullable();
$table->boolean('effect_may_have_applied')->nullable();
$table->boolean('safe_to_resubmit')->nullable();
$table->string('effect_resolution');
$table->integer('effect_resolved_at')->nullable();
$table->integer('effect_resolved_by')->nullable();
$table->string('effect_resolution_reason')->default('');
$table->string('completion_event_id')->nullable();
$table->integer('read_at')->nullable();
$table->integer('dispatched_at')->nullable();
$table->string('lease_token')->nullable();
$table->integer('lease_until')->default(0);
$table->integer('available_at')->default(0);
$table->string('error_code')->nullable();
$table->boolean('finalization_incident')->default(0);
$table->string('finalization_error_code')->nullable();
$table->text('result_json')->nullable();
$table->integer('created_at');
$table->integer('updated_at');
});
$schema->create('cloud_public_network_transitions',
static function (Blueprint $table): void {
$table->increments('id');
$table->string('state');
$table->string('effect_resolution');
$table->string('last_error_code')->default('');
});
$schema->create('cloud_instances', static function (Blueprint $table): void {
$table->increments('id');
$table->string('uuid');
$table->integer('host_id');
$table->integer('user_id');
$table->integer('order_id')->nullable();
$table->integer('plan_id')->nullable();
$table->integer('offer_id')->nullable();
$table->integer('resource_pool_id')->nullable();
$table->integer('reservation_id')->nullable();
$table->integer('current_period_id')->nullable();
$table->string('name');
$table->string('status');
$table->string('service_status');
$table->string('network_status');
$table->integer('expires_at')->nullable();
$table->decimal('renewal_price', 12, 2)->nullable();
$table->integer('renewal_days')->default(0);
$table->integer('renewal_traffic_bytes')->default(0);
$table->integer('traffic_quota_bytes')->default(0);
$table->integer('traffic_download_bytes')->default(0);
$table->integer('traffic_upload_bytes')->default(0);
$table->integer('lifetime_download_bytes')->default(0);
$table->integer('lifetime_upload_bytes')->default(0);
$table->integer('traffic_synced_at')->default(0);
$table->integer('suspended_at')->nullable();
$table->integer('network_suspended_at')->nullable();
$table->text('spec_json');
$table->text('sealed_credential')->nullable();
$table->integer('created_at');
$table->integer('updated_at');
});
$schema->create('cloud_agent_completion_events',
static function (Blueprint $table): void {
$table->string('event_id')->primary();
$table->integer('host_id');
$table->string('operation_id');
$table->string('payload_sha256');
$table->string('outcome');
$table->boolean('effect_may_have_applied');
$table->boolean('safe_to_resubmit');
$table->string('error_code')->default('');
$table->text('result_json')->nullable();
$table->string('disposition')->default('received');
$table->integer('received_at');
$table->integer('applied_at')->nullable();
$table->unique(['host_id', 'operation_id']);
});
$schema->create('cloud_instance_snapshots',
static function (Blueprint $table): void {
$table->increments('id');
$table->integer('instance_id');
$table->string('state');
$table->string('last_operation_id')->default('');
$table->integer('deleted_at')->nullable();
$table->integer('updated_at');
});
$schema->create('cloud_templates',
static function (Blueprint $table): void {
$table->string('id')->primary();
$table->string('catalog_key')->default('test-catalog');
$table->string('version_label')->default('test-version');
$table->boolean('is_default')->default(true);
$table->integer('active_job_id')->nullable();
$table->string('state')->default('active');
$table->integer('revision')->default(1);
$table->string('generation_token')->default('');
$table->integer('image_bytes')->default(0);
$table->string('sha256')->default('');
$table->integer('updated_at')->default(0);
});
$schema->create('cloud_user_isos',
static function (Blueprint $table): void {
$table->string('id')->primary();
$table->integer('active_job_id')->nullable();
$table->string('state')->default('active');
});
$schema->create('cloud_template_replicas',
static function (Blueprint $table): void {
$table->integer('host_id');
$table->string('template_id');
$table->integer('revision')->default(0);
$table->string('generation_token')->default('');
$table->string('state')->default('pending');
$table->integer('image_bytes')->default(0);
$table->string('sha256')->default('');
$table->string('operation_id')->default('');
$table->integer('job_id')->nullable();
$table->string('error_code')->default('');
$table->integer('created_at')->default(0);
$table->integer('updated_at')->default(0);
$table->primary(['host_id', 'template_id']);
});
$schema->create('cloud_template_artifacts',
static function (Blueprint $table): void {
$table->string('template_id')->primary();
$table->integer('storage_revision');
$table->text('sealed_storage_runtime');
$table->string('object_key');
$table->string('object_version_id')->default('');
$table->string('object_etag');
$table->integer('size_bytes');
$table->string('sha256');
$table->string('state');
$table->string('error_code')->default('');
$table->string('delete_operation_id')->default('');
$table->integer('delete_generation')->default(0);
$table->integer('delete_cycle_attempts')->default(0);
$table->integer('delete_presence_confirmations')->default(0);
$table->integer('delete_last_checked_at')->default(0);
$table->integer('delete_next_check_at')->default(0);
$table->integer('delete_attention_at')->nullable();
$table->integer('verified_at')->default(0);
$table->integer('created_at');
$table->integer('updated_at');
});
$schema->create('cloud_user_iso_replicas',
static function (Blueprint $table): void {
$table->integer('host_id');
$table->string('user_iso_id');
$table->integer('user_id');
$table->string('state')->default('pending');
$table->integer('size_bytes')->default(0);
$table->string('sha256')->default('');
$table->string('operation_id')->default('');
$table->integer('job_id')->nullable();
$table->string('error_code')->default('');
$table->integer('created_at')->default(0);
$table->integer('updated_at')->default(0);
$table->primary(['host_id', 'user_iso_id']);
});
Capsule::table('user')->insert([
'id' => 1, 'enable' => 1, 'is_admin' => 0,
]);
$now = time();
$nextId = 100;
$insertSettingsJob = static function (
string $operationId,
array $body,
?array $credential = null
) use (&$nextId, $now): array {
$id = $nextId++;
$uuid = sprintf(
'00000000-0000-4000-8000-%012d',
$id
);
Capsule::table('cloud_instances')->insert([
'id' => $id,
'uuid' => $uuid,
'host_id' => 7,
'user_id' => 1,
'order_id' => null,
'plan_id' => null,
'offer_id' => null,
'resource_pool_id' => null,
'reservation_id' => null,
'current_period_id' => null,
'name' => 'settings-' . $id,
'status' => 'active',
'service_status' => 'exempt',
'network_status' => 'active',
'expires_at' => null,
'renewal_price' => null,
'renewal_days' => 0,
'renewal_traffic_bytes' => 0,
'traffic_quota_bytes' => 0,
'traffic_download_bytes' => 0,
'traffic_upload_bytes' => 0,
'lifetime_download_bytes' => 0,
'lifetime_upload_bytes' => 0,
'traffic_synced_at' => 0,
'suspended_at' => null,
'network_suspended_at' => null,
'spec_json' => json_encode(
['managed' => true],
JSON_THROW_ON_ERROR
),
'sealed_credential' => null,
'created_at' => $now - 3600,
'updated_at' => $now - 3600,
]);
$fields = [];
foreach ([
'hostname' => 'hostname',
'root_password' => 'password',
'ssh_public_key' => 'ssh_key',
] as $field => $code) {
if (array_key_exists($field, $body)) {
$fields[] = $code;
}
}
$payload = [
'method' => 'POST',
'request_target' => '/v1/vms/' . $uuid . '/settings',
'body' => $body + ['confirm_restart' => true],
'owner_id' => 1,
'can_manage' => false,
'credential' => $credential,
'settings_fields' => $fields,
];
Capsule::table('cloud_jobs')->insert([
'id' => $id,
'operation_id' => $operationId,
'request_idempotency_key' => $operationId,
'request_fingerprint' => hash('sha256', $operationId),
'parent_job_id' => null,
'order_id' => null,
'invoice_id' => null,
'instance_id' => $id,
'snapshot_id' => null,
'user_id' => 1,
'host_id' => 7,
'kind' => 'instance_settings',
'execution_mode' => 'agent',
'phase' => 'terminal',
'outcome' => 'failed',
'status' => 'failed',
'sealed_payload' => CloudSecrets::encrypt(
$payload,
'job-payload'
),
'attempts' => 1,
'dispatch_attempts' => 1,
'dispatch_deadline_at' => $now - 1000,
'dispatch_started_at' => $now - 990,
'agent_accepted_at' => $now - 980,
'receipt_deadline_at' => $now - 20,
'terminal_at' => $now - 10,
'effect_may_have_applied' => 1,
'safe_to_resubmit' => 0,
'effect_resolution' => 'unknown',
'effect_resolved_at' => null,
'effect_resolved_by' => null,
'effect_resolution_reason' => '',
'completion_event_id' => null,
'read_at' => null,
'dispatched_at' => $now - 980,
'lease_token' => null,
'lease_until' => 0,
'available_at' => 0,
'error_code' => 'operation_deadline_exceeded',
'finalization_incident' => 0,
'finalization_error_code' => null,
'result_json' => json_encode(
['cause_code' => 'operation_deadline_exceeded'],
JSON_THROW_ON_ERROR
),
'created_at' => $now - 1000,
'updated_at' => $now - 10,
]);
return [$id, $uuid];
};
$insertDispatchableJob = static function (
string $operationId
) use (&$nextId, $now): int {
$id = $nextId++;
Capsule::table('cloud_jobs')->insert([
'id' => $id,
'operation_id' => $operationId,
'request_idempotency_key' => $operationId,
'request_fingerprint' => hash('sha256', $operationId),
'parent_job_id' => null,
'order_id' => null,
'invoice_id' => null,
'instance_id' => null,
'snapshot_id' => null,
// A missing actor closes safely before dispatch. It keeps this
// fixture focused on the public worker scheduling seam rather
// than sharing an HTTP transport with system recovery.
'user_id' => 999,
'host_id' => 7,
'kind' => 'power_start',
'execution_mode' => 'agent',
'phase' => 'accepted',
'outcome' => null,
'status' => 'queued',
'sealed_payload' => CloudSecrets::encrypt([
'method' => 'POST',
'request_target' => '/v1/vms/'
. '00000000-0000-4000-8000-000000000999/start',
'body' => [],
'owner_id' => 999,
'can_manage' => false,
], 'job-payload'),
'attempts' => 0,
'dispatch_attempts' => 0,
'dispatch_deadline_at' => $now + 300,
'dispatch_started_at' => null,
'agent_accepted_at' => null,
'receipt_deadline_at' => null,
'terminal_at' => null,
'effect_may_have_applied' => null,
'safe_to_resubmit' => null,
'effect_resolution' => 'not_applicable',
'effect_resolved_at' => null,
'effect_resolved_by' => null,
'effect_resolution_reason' => '',
'completion_event_id' => null,
'read_at' => null,
'dispatched_at' => null,
'lease_token' => null,
'lease_until' => 0,
'available_at' => 0,
'error_code' => null,
'finalization_incident' => 0,
'finalization_error_code' => null,
'result_json' => null,
'created_at' => $now,
'updated_at' => $now,
]);
return $id;
};
$audit = static function (int $id): array {
$job = CloudJob::find($id);
return [
'operation_id' => (string) $job->operation_id,
'kind' => (string) $job->kind,
'phase' => (string) $job->phase,
'outcome' => (string) $job->outcome,
'status' => (string) $job->status,
'dispatch_attempts' => (int) $job->dispatch_attempts,
'effect_may_have_applied' =>
(bool) $job->effect_may_have_applied,
'safe_to_resubmit' => (bool) $job->safe_to_resubmit,
'error_code' => (string) $job->error_code,
'result_json' => $job->result_json,
'terminal_at' => (int) $job->terminal_at,
'completion_event_id' => $job->completion_event_id,
'read_at' => $job->read_at,
];
};
$inspection = static function (
string $operationId,
string $journalStatus,
string $resolution,
?array $projection = null,
?bool $fenceResolutionRequired = null,
?string $errorCode = null,
?string $fenceState = null
): array {
$applied = $resolution === 'applied';
$uncertain = in_array(
$resolution,
['partial', 'inconclusive'],
true
);
$value = [
'operation_id' => $operationId,
'operation_status' => $journalStatus,
'checked_at' => time(),
'outcome' => $uncertain
? 'uncertain' : ($applied ? 'succeeded' : 'failed'),
'summary_code' => $uncertain
? 'evidence_incomplete'
: ($applied ? 'operation_succeeded' : 'operation_failed'),
'effect_resolution' => $resolution,
'fence_resolution_required' => $fenceResolutionRequired
?? $journalStatus === 'failed',
'fence_state' => $fenceState ?? (($fenceResolutionRequired
?? $journalStatus === 'failed') ? 'active' : 'not_required'),
'abort_job' => ['available' => false],
'cleanup' => ['available' => false],
] + CloudOperationRecoveryIdentity::expected(
CloudJob::where('operation_id', $operationId)->firstOrFail(),
$applied
);
if ($projection !== null) {
$value['effect_projection'] = $projection;
}
if ($resolution === 'not_applied') {
$value['error_code'] = $errorCode ?? 'settings_not_committed';
}
return $value;
};
$notFound = static fn (): CloudAgentError => new CloudAgentError(
'not found', 404, 'fence_resolution_not_found', false, 404
);
$fenceResponse = static function (
array $request,
int $resolvedAt
): array {
$payload = $request['payload'];
return [
'operation_id' => $request['operationId'],
'resolution_id' => $payload['resolution_id'],
'resolution' => $payload['resolution'],
'inspection_checked_at' =>
$payload['inspection_checked_at'],
'reason_sha256' => $payload['reason_sha256'],
'resolved_at' => $resolvedAt,
'resolved_by' => 0,
'replayed' => false,
];
};
$client = new AutomaticRecoveryTestAgentClient();
CloudAgentSettingsService::$client = $client;
// An immutable received receipt must outrank a legacy aggregate whose
// child can remain active indefinitely. This prevents the aggregate sweep
// from starving both PHP convergence and the Agent outbox ACK.
[$receiptPriorityId, $receiptPriorityUuid] = $insertSettingsJob(
str_repeat('0a', 16),
['hostname' => 'receipt-priority.example']
);
$receiptPriorityJob = CloudJob::findOrFail($receiptPriorityId);
$receiptPriorityIdentity = CloudOperationRecoveryIdentity::expected(
$receiptPriorityJob,
true
);
$receiptPriorityResult = [
'uuid' => $receiptPriorityUuid,
'hostname' => 'receipt-priority.example',
'settings_changed' => true,
];
$receiptPriorityWire = [
'event_id' => 'automatic-priority-receipt',
'host_id' => 7,
'operation_id' => (string) $receiptPriorityJob->operation_id,
'action' => $receiptPriorityIdentity['action'],
'resource_id' => $receiptPriorityIdentity['resource_id'],
'request_fingerprint' =>
$receiptPriorityIdentity['request_fingerprint'],
'status' => 'succeeded',
'effect_may_have_applied' => false,
'safe_to_resubmit' => false,
'error_code' => '',
'result' => $receiptPriorityResult,
];
Capsule::table('cloud_agent_completion_events')->insert([
'event_id' => $receiptPriorityWire['event_id'],
'host_id' => 7,
'operation_id' => $receiptPriorityWire['operation_id'],
'payload_sha256' => hash('sha256', json_encode(
$receiptPriorityWire,
JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES
| JSON_UNESCAPED_UNICODE
)),
'outcome' => 'succeeded',
'effect_may_have_applied' => 0,
'safe_to_resubmit' => 0,
'error_code' => '',
'result_json' => json_encode([
'schema' => 'virt_cloud_completion_event',
'version' => 1,
'identity' => $receiptPriorityIdentity,
'result' => $receiptPriorityResult,
], JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES
| JSON_UNESCAPED_UNICODE),
'disposition' => 'received',
'received_at' => $now,
'applied_at' => null,
]);
[$legacyParentId] = $insertSettingsJob(
str_repeat('0b', 16),
['hostname' => 'legacy-parent.example']
);
Capsule::table('cloud_jobs')->where('id', $legacyParentId)->update([
'kind' => 'template_import',
'execution_mode' => 'aggregate',
'host_id' => null,
'instance_id' => null,
'dispatch_attempts' => 0,
'dispatched_at' => null,
'available_at' => 0,
'lease_until' => 0,
]);
[$legacyChildId] = $insertSettingsJob(
str_repeat('0c', 16),
['hostname' => 'legacy-child.example']
);
Capsule::table('cloud_jobs')->where('id', $legacyChildId)->update([
'parent_job_id' => $legacyParentId,
'kind' => 'template_import_replica',
'phase' => 'awaiting_receipt',
'outcome' => null,
'status' => 'running',
'effect_may_have_applied' => null,
'safe_to_resubmit' => null,
'effect_resolution' => 'not_applicable',
'terminal_at' => null,
]);
$beforePriorityRequests = count($client->requests);
$assert((new CloudAutomaticOperationRecoveryService())->tick(),
'An exact completion receipt was not claimed ahead of legacy sweep.');
$priorityEvent = Capsule::table('cloud_agent_completion_events')
->where('event_id', 'automatic-priority-receipt')->first();
$assert((string) CloudJob::findOrFail($receiptPriorityId)
->effect_resolution === 'applied'
&& (string) $priorityEvent->disposition === 'applied'
&& (int) CloudJob::findOrFail($legacyParentId)->available_at === 0
&& count($client->requests) === $beforePriorityRequests,
'A waiting aggregate starved or contaminated an exact receipt.');
Capsule::table('cloud_jobs')->whereIn(
'id',
[$legacyParentId, $legacyChildId]
)->delete();
Capsule::table('cloud_instances')->whereIn(
'id',
[$legacyParentId, $legacyChildId]
)->delete();
// Sustained completion traffic must not monopolize the recovery worker.
// The long-lived worker has three independently useful lanes: immutable
// receipts, ordinary Agent leaves, and PHP-only aggregate projection.
// With all three continuously ready, every lane must make progress within
// one three-tick rotation; replenishing receipts may not starve either of
// the other two lanes.
[$fairLeafId] = $insertSettingsJob(
md5('automatic-fair-leaf'),
['hostname' => 'fair-leaf.example']
);
Capsule::table('cloud_jobs')->where('id', $fairLeafId)->update([
'host_id' => null,
'available_at' => 0,
'lease_token' => null,
'lease_until' => 0,
]);
[$fairParentId] = $insertSettingsJob(
md5('automatic-fair-aggregate'),
['hostname' => 'fair-aggregate.example']
);
Capsule::table('cloud_jobs')->where('id', $fairParentId)->update([
'parent_job_id' => null,
'order_id' => null,
'invoice_id' => null,
'instance_id' => null,
'snapshot_id' => null,
'host_id' => null,
'kind' => 'template_import',
'execution_mode' => 'aggregate',
'attempts' => 0,
'dispatch_attempts' => 0,
'dispatch_started_at' => null,
'agent_accepted_at' => null,
'dispatched_at' => null,
'completion_event_id' => null,
'effect_may_have_applied' => 1,
'safe_to_resubmit' => 0,
'effect_resolution' => 'unknown',
'finalization_incident' => 1,
'finalization_error_code' => 'recovery_child_convergence',
'available_at' => 0,
'lease_token' => null,
'lease_until' => 0,
]);
[$fairChildId] = $insertSettingsJob(
md5('automatic-fair-aggregate-child'),
['hostname' => 'fair-aggregate-child.example']
);
Capsule::table('cloud_jobs')->where('id', $fairChildId)->update([
'parent_job_id' => $fairParentId,
'kind' => 'template_import_replica',
'effect_may_have_applied' => 0,
'safe_to_resubmit' => 1,
'effect_resolution' => 'not_applied',
'effect_resolved_at' => $now,
'effect_resolution_reason' => 'normal_completion',
'available_at' => $now + 3600,
'lease_token' => null,
'lease_until' => $now + 3600,
]);
$insertFairReceipt = static function (int $sequence) use (
$insertSettingsJob,
$now
): string {
$operationId = md5('automatic-fair-receipt:' . $sequence);
[$jobId, $uuid] = $insertSettingsJob(
$operationId,
['hostname' => 'fair-receipt-' . $sequence . '.example']
);
$job = CloudJob::findOrFail($jobId);
$identity = CloudOperationRecoveryIdentity::expected($job, true);
$result = [
'uuid' => $uuid,
'hostname' => 'fair-receipt-' . $sequence . '.example',
'settings_changed' => true,
];
$wire = [
'event_id' => 'automatic-fair-receipt-' . $sequence,
'host_id' => 7,
'operation_id' => $operationId,
'action' => $identity['action'],
'resource_id' => $identity['resource_id'],
'request_fingerprint' => $identity['request_fingerprint'],
'status' => 'succeeded',
'effect_may_have_applied' => false,
'safe_to_resubmit' => false,
'error_code' => '',
'result' => $result,
];
Capsule::table('cloud_agent_completion_events')->insert([
'event_id' => $wire['event_id'],
'host_id' => 7,
'operation_id' => $operationId,
'payload_sha256' => hash('sha256', json_encode(
$wire,
JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES
| JSON_UNESCAPED_UNICODE
)),
'outcome' => 'succeeded',
'effect_may_have_applied' => 0,
'safe_to_resubmit' => 0,
'error_code' => '',
'result_json' => json_encode([
'schema' => 'virt_cloud_completion_event',
'version' => 1,
'identity' => $identity,
'result' => $result,
], JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES
| JSON_UNESCAPED_UNICODE),
'disposition' => 'received',
'received_at' => $now + $sequence,
'applied_at' => null,
]);
return $wire['event_id'];
};
$fairService = new CloudAutomaticOperationRecoveryService();
$fairReceiptIds = [];
for ($sequence = 1; $sequence <= 6; $sequence++) {
$fairReceiptIds[] = $insertFairReceipt($sequence);
$assert($fairService->tick(),
'A fair recovery rotation unexpectedly found no work.');
}
$fairAppliedReceipts = Capsule::table(
'cloud_agent_completion_events'
)->whereIn('event_id', $fairReceiptIds)
->where('disposition', 'applied')->count();
$fairLeaf = CloudJob::findOrFail($fairLeafId);
$fairParent = CloudJob::findOrFail($fairParentId);
$assert($fairAppliedReceipts >= 2
&& (bool) $fairLeaf->finalization_incident
&& (string) $fairLeaf->finalization_error_code
=== 'recovery_instance_state_conflict'
&& (bool) $fairParent->finalization_incident
&& (int) $fairParent->available_at >= time() + 290,
'Continuous receipt traffic starved a leaf or aggregate lane, '
. 'or fairness starved immutable receipts.');
$fairReceiptOperations = Capsule::table(
'cloud_agent_completion_events'
)->whereIn('event_id', $fairReceiptIds)->pluck('operation_id')->all();
$fairReceiptJobs = Capsule::table('cloud_jobs')
->whereIn('operation_id', $fairReceiptOperations);
$fairReceiptJobIds = $fairReceiptJobs->pluck('id')->all();
$fairReceiptInstanceIds = Capsule::table('cloud_jobs')
->whereIn('id', $fairReceiptJobIds)->pluck('instance_id')->all();
Capsule::table('cloud_agent_completion_events')
->whereIn('event_id', $fairReceiptIds)->delete();
Capsule::table('cloud_jobs')->whereIn('id', $fairReceiptJobIds)->delete();
Capsule::table('cloud_instances')
->whereIn('id', $fairReceiptInstanceIds)->delete();
Capsule::table('cloud_jobs')->whereIn('id', [
$fairLeafId,
$fairParentId,
$fairChildId,
])->delete();
Capsule::table('cloud_instances')->whereIn('id', [
$fairLeafId,
$fairParentId,
$fairChildId,
])->delete();
// Every explicitly registered Agent leaf remains eligible. Inconclusive
// evidence keeps the exact operation fenced and schedules another check.
[$nonSettingsId, $nonSettingsUuid] = $insertSettingsJob(
str_repeat('0f', 16),
['hostname' => 'not-a-settings-recovery.example']
);
Capsule::table('cloud_jobs')->where('id', $nonSettingsId)->update([
'kind' => 'network_change',
'sealed_payload' => CloudSecrets::encrypt([
'method' => 'POST',
'request_target' => '/v1/vms/' . $nonSettingsUuid . '/network',
'body' => [
'operation_id' => str_repeat('0f', 16),
'network' => 'bridge:br0',
'public_pool_key' => str_repeat('a', 32),
'public_pool_cidr' => '203.0.113.0/24',
'public_ip_cidr' => '203.0.113.10/24',
'gateway' => '203.0.113.1',
'dns_servers' => ['1.1.1.1'],
'confirm_restart' => true,
],
'owner_id' => 1,
'can_manage' => false,
], 'job-payload'),
]);
$beforeNonSettingsRequests = count($client->requests);
$client->responses[] = $inspection(
str_repeat('0f', 16),
'failed',
'inconclusive'
);
$assert((new CloudAutomaticOperationRecoveryService())->tick(),
'The recovery worker did not claim a registered non-settings kind.');
$nonSettings = CloudJob::find($nonSettingsId);
$assert((string) $nonSettings->effect_resolution === 'unknown'
&& (string) $nonSettings->finalization_error_code
=== 'recovery_evidence_inconclusive'
&& (int) $nonSettings->available_at >= time() + 290
&& count($client->requests) === $beforeNonSettingsRequests + 1,
'Inconclusive non-settings evidence did not stay recoverable.');
// A successful read-only inspection must adapt back to the same terminal
// operation contract as an immediate Agent response. This is the public
// recovery seam for a template deletion whose original success receipt
// reached PHP but failed local catalog finalization.
$templateDeleteId = 'automatic-delete-template';
[$templateDeleteParentId] = $insertSettingsJob(
md5('automatic-template-delete-parent'),
['hostname' => 'unused-parent.example']
);
Capsule::table('cloud_jobs')->where(
'id',
$templateDeleteParentId
)->update([
'parent_job_id' => null,
'instance_id' => null,
'host_id' => null,
'kind' => 'template_delete',
'execution_mode' => 'aggregate',
'attempts' => 0,
'dispatch_attempts' => 0,
'dispatch_started_at' => null,
'agent_accepted_at' => null,
'dispatched_at' => null,
'effect_resolution' => 'unknown',
'finalization_incident' => 1,
'finalization_error_code' => 'recovery_child_convergence',
'available_at' => 0,
'lease_token' => null,
'lease_until' => 0,
]);
[$templateDeleteChildId] = $insertSettingsJob(
md5('automatic-template-delete-child'),
['hostname' => 'unused-child.example']
);
$templateDeleteOperationId = (string) CloudJob::findOrFail(
$templateDeleteChildId
)->operation_id;
Capsule::table('cloud_jobs')->where(
'id',
$templateDeleteChildId
)->update([
'parent_job_id' => $templateDeleteParentId,
'instance_id' => null,
'kind' => 'template_delete_replica',
'sealed_payload' => CloudSecrets::encrypt([
'method' => 'DELETE',
'request_target' => '/v1/templates/' . $templateDeleteId,
'body' => ['operation_id' => $templateDeleteOperationId],
'owner_id' => 1,
'can_manage' => true,
'template_id' => $templateDeleteId,
], 'job-payload'),
'finalization_incident' => 1,
'finalization_error_code' => 'recovery_receipt_invalid',
'available_at' => 0,
'lease_token' => null,
'lease_until' => 0,
]);
Capsule::table('cloud_templates')->insert([
'id' => $templateDeleteId,
'active_job_id' => $templateDeleteParentId,
'state' => 'deleting',
'revision' => 1,
'generation_token' => str_repeat('d', 32),
'image_bytes' => 4096,
'sha256' => str_repeat('e', 64),
'updated_at' => $now,
]);
Capsule::table('cloud_template_replicas')->insert([
'host_id' => 7,
'template_id' => $templateDeleteId,
'revision' => 1,
'generation_token' => str_repeat('d', 32),
'state' => 'delete_failed',
'image_bytes' => 4096,
'sha256' => str_repeat('e', 64),
'operation_id' => $templateDeleteOperationId,
'job_id' => $templateDeleteChildId,
'error_code' => 'completion_finalization_failed',
'created_at' => $now - 120,
'updated_at' => $now,
]);
Capsule::table('cloud_template_artifacts')->insert([
'template_id' => $templateDeleteId,
'storage_revision' => 1,
'sealed_storage_runtime' => CloudSecrets::encrypt([
'config' => [],
'credentials' => [],
'revision' => 1,
], 'template-artifact-storage-runtime'),
'object_key' => 'templates/' . $templateDeleteId . '/system.qcow2',
'object_version_id' => '',
'object_etag' => str_repeat('f', 32),
'size_bytes' => 4096,
'sha256' => str_repeat('e', 64),
'state' => 'ready',
'error_code' => '',
'delete_operation_id' => '',
'delete_generation' => 0,
'delete_cycle_attempts' => 0,
'delete_presence_confirmations' => 0,
'delete_last_checked_at' => 0,
'delete_next_check_at' => 0,
'delete_attention_at' => null,
'verified_at' => $now,
'created_at' => $now - 120,
'updated_at' => $now,
]);
$client->responses[] = $inspection(
$templateDeleteOperationId,
'succeeded',
'applied',
[
'resource_type' => 'template',
'id' => $templateDeleteId,
'deleted' => true,
'managed_image_absent' => true,
'managed_metadata_absent' => true,
'managed_artifacts_absent' => true,
],
false
);
$beforeTemplateDeleteRequests = count($client->requests);
$assert((new CloudAutomaticOperationRecoveryService())->tick(),
'The recovery worker did not inspect the template deletion.');
$templateDeleteChild = CloudJob::findOrFail($templateDeleteChildId);
$templateDeleteParent = CloudJob::findOrFail($templateDeleteParentId);
$assert((string) $templateDeleteChild->effect_resolution === 'applied'
&& !(bool) $templateDeleteChild->finalization_incident
&& $templateDeleteChild->finalization_error_code === null
&& (string) $templateDeleteParent->effect_resolution === 'unknown'
&& (bool) $templateDeleteParent->finalization_incident
&& (string) $templateDeleteParent->finalization_error_code
=== 'recovery_child_convergence'
&& (string) Capsule::table('cloud_templates')
->where('id', $templateDeleteId)->value('state') === 'deleting'
&& Capsule::table('cloud_template_replicas')
->where('template_id', $templateDeleteId)->exists()
&& (string) Capsule::table('cloud_template_artifacts')
->where('template_id', $templateDeleteId)->value('state')
=== 'delete_pending'
&& count($client->requests) === $beforeTemplateDeleteRequests + 1
&& str_ends_with(
$client->requests[$beforeTemplateDeleteRequests]['target'],
'/inspect'
),
'A successful template deletion inspection did not converge its host '
. 'operation into the two-phase master cleanup fence.');
// A poisoned Agent completion outbox is never evidence for either effect
// verdict. Even a syntactically complete not-applied projection remains
// fenced and the reviewed cause is visible without exposing raw data.
[$quarantinedId] = $insertSettingsJob(
str_repeat('0e', 16),
['hostname' => 'quarantined-receipt.example']
);
$quarantinedInspection = $inspection(
str_repeat('0e', 16),
'failed',
'not_applied',
null,
true,
'settings_not_committed'
);
$quarantinedInspection['completion_delivery_state'] = 'quarantined';
$quarantinedInspection['completion_delivery_error_code'] =
'completion_identity_mismatch';
$client->responses[] = $quarantinedInspection;
$beforeQuarantinedRequests = count($client->requests);
$assert((new CloudAutomaticOperationRecoveryService())->tick(),
'The recovery worker did not inspect a quarantined completion.');
$quarantined = CloudJob::findOrFail($quarantinedId);
$quarantinedUser = CloudJobService::recoveryPresentation(
$quarantined,
false
);
$quarantinedAdmin = CloudJobService::recoveryPresentation(
$quarantined,
true
);
$assert((string) $quarantined->effect_resolution === 'unknown'
&& (string) $quarantined->finalization_error_code
=== 'recovery_completion_identity_mismatch'
&& (int) $quarantined->available_at >= time() + 290
&& count($client->requests) === $beforeQuarantinedRequests + 1
&& ($quarantinedAdmin['code'] ?? null)
=== 'recovery_completion_identity_mismatch'
&& str_contains(
(string) ($quarantinedAdmin['message'] ?? ''),
'身份与原操作记录不匹配'
)
&& str_contains(
(string) ($quarantinedUser['message'] ?? ''),
'身份与原操作记录不匹配'
)
&& !array_key_exists('code', $quarantinedUser),
'A quarantined completion was trusted or its safe cause was hidden.');
// A journal success is immutable exact evidence and has no active fence.
[$succeededId, $succeededUuid] = $insertSettingsJob(
str_repeat('1', 32),
['root_password' => 'New-automatic9!Pass'],
[
'password' => 'New-automatic9!Pass',
'login_name' => 'root',
'auth_mode' => 'password',
]
);
$succeededAudit = $audit($succeededId);
$client->responses[] = $inspection(
str_repeat('1', 32),
'succeeded',
'applied',
[
'uuid' => $succeededUuid,
'status' => 'running',
'settings_changed' => true,
]
);
$dispatchId = $insertDispatchableJob(str_repeat('8', 32));
$beforeDispatchRequests = count($client->requests);
$assert((new CloudWorkerService())->tick(),
'The worker did not claim the ordinary dispatchable task first.');
$dispatch = CloudJob::find($dispatchId);
$unrecovered = CloudJob::find($succeededId);
$assert((string) $dispatch->phase === 'terminal'
&& (string) $dispatch->outcome === 'failed'
&& (int) $dispatch->dispatch_attempts === 0
&& (string) $unrecovered->effect_resolution === 'unknown'
&& count($client->requests) === $beforeDispatchRequests,
'Automatic recovery ran before an ordinary dispatchable task.');
$queuedBehindRecovery = $insertDispatchableJob(str_repeat('b', 32));
$beforeSucceededRecovery = count($client->requests);
$assert((new CloudAutomaticOperationRecoveryService())->tick(),
'The dedicated recovery service did not claim a recoverable task.');
$assert((string) CloudJob::find($queuedBehindRecovery)->phase
=== 'accepted',
'The dedicated recovery service consumed an ordinary task.');
$assert((new CloudWorkerService())->tick()
&& (string) CloudJob::find($queuedBehindRecovery)->phase
=== 'terminal',
'The ordinary worker did not keep processing its independent queue.');
$succeeded = CloudJob::find($succeededId);
$credential = CloudSecrets::decrypt(
(string) Capsule::table('cloud_instances')
->where('id', $succeededId)->value('sealed_credential'),
'instance-credential'
);
$redacted = CloudSecrets::decrypt(
(string) $succeeded->sealed_payload,
'job-payload'
);
$assert((string) $succeeded->effect_resolution === 'applied'
&& (int) $succeeded->effect_resolved_by === 0
&& $audit($succeededId) === $succeededAudit
&& ($credential['password'] ?? null)
=== 'New-automatic9!Pass'
&& !array_key_exists('credential', $redacted)
&& !CloudJobService::blockingQueryForInstance(
$succeededId
)->exists()
&& count($client->requests) === $beforeSucceededRecovery + 1
&& str_ends_with(
$client->requests[array_key_last($client->requests)]['target'],
'/inspect'
),
'Journal-success recovery did not converge without a fence write.');
// A malformed or contradictory receipt is isolated from an existing
// prepared intent. The job itself remains scheduled for a fresh Agent
// inspection instead of treating the incident as a permanent stop.
$invalidProofRequests = count($client->requests);
$invalidProofCases = [
[
'operation_id' => str_repeat('c1', 16),
'body' => ['root_password' => 'Invalid-proof9!Pass'],
'credential' => [
'password' => 'Invalid-proof9!Pass',
'login_name' => 'root',
'auth_mode' => 'password',
],
'receipt_result' => [],
'message' => 'missing settings_changed proof',
],
[
'operation_id' => str_repeat('d2', 16),
'body' => ['hostname' => 'prepared-proof.example'],
'credential' => null,
'receipt_result' => [
'settings_changed' => true,
'hostname' => 'different-proof.example',
],
'message' => 'mismatched hostname proof',
],
];
foreach ($invalidProofCases as $index => $case) {
[$invalidProofId, $invalidProofUuid] = $insertSettingsJob(
$case['operation_id'],
$case['body'],
$case['credential']
);
Capsule::table('cloud_jobs')->where(
'id',
$invalidProofId
)->update([
'effect_resolution_reason' => 'automatic_recovery:v1:'
. time() . ':1:pending',
]);
$preparedResult = [
'uuid' => $invalidProofUuid,
'settings_changed' => true,
];
if (array_key_exists('hostname', $case['body'])) {
$preparedResult['hostname'] = $case['body']['hostname'];
}
$operationIdentity = CloudOperationRecoveryIdentity::expected(
CloudJob::findOrFail($invalidProofId),
true
);
CloudOperationRecoveryService::prepareApplied(
$invalidProofId,
$operationIdentity + [
'operation_id' => $case['operation_id'],
'status' => 'succeeded',
'effect_may_have_applied' => false,
'safe_to_resubmit' => false,
'error_code' => '',
'result' => $preparedResult,
],
'automatic_applied_succeeded_nofence:'
. str_repeat((string) ($index + 3), 64)
);
$preparedInvalidProof = CloudJob::find($invalidProofId);
$preparedMarker = (string) $preparedInvalidProof
->effect_resolution_reason;
$receiptResult = ['uuid' => $invalidProofUuid]
+ $case['receipt_result'];
$eventId = 'automatic-invalid-proof-' . ($index + 1);
$accepted = CloudOperationCompletionService::accept([
'type' => 'completion',
'id' => $eventId,
'agent_id' => '7',
'operation_id' => $case['operation_id'],
'action' => $operationIdentity['action'],
'resource_id' => $operationIdentity['resource_id'],
'request_fingerprint' =>
$operationIdentity['request_fingerprint'],
'status' => 'succeeded',
'effect_may_have_applied' => false,
'safe_to_resubmit' => false,
'error_code' => '',
'result' => $receiptResult,
]);
$invalidProofJob = CloudJob::find($invalidProofId);
$invalidProofEvent = Capsule::table(
'cloud_agent_completion_events'
)->where('event_id', $eventId)->first();
$assert($accepted === [
'accepted' => true,
'disposition' => 'late',
]
&& (string) $invalidProofJob->effect_resolution === 'unknown'
&& (string) $invalidProofJob->effect_resolution_reason
=== $preparedMarker
&& (bool) $invalidProofJob->finalization_incident
&& in_array(
(string) $invalidProofJob->finalization_error_code,
[
'recovery_receipt_invalid',
'recovery_evidence_conflict',
],
true
)
&& (int) $invalidProofJob->available_at >= time() + 290
&& (string) $invalidProofEvent->disposition === 'late'
&& CloudJobService::blockingQueryForInstance(
$invalidProofId
)->exists(),
'A success receipt with ' . $case['message']
. ' was not isolated while leaving inspection retryable.');
}
$assert(count($client->requests) === $invalidProofRequests,
'Invalid late receipt proof unexpectedly contacted the Agent.');
// A well-formed completion for a different immutable Agent request is a
// bad event, not a reason to strand the already-prepared PHP convergence.
// Quarantine it, then let the normal prepared crash-resume path finish.
[$wrongIdentityId, $wrongIdentityUuid] = $insertSettingsJob(
str_repeat('e3', 16),
['hostname' => 'prepared-identity.example']
);
$expectedIdentity = CloudOperationRecoveryIdentity::expected(
CloudJob::findOrFail($wrongIdentityId),
true
);
CloudOperationRecoveryService::prepareApplied(
$wrongIdentityId,
$expectedIdentity + [
'operation_id' => str_repeat('e3', 16),
'status' => 'succeeded',
'effect_may_have_applied' => false,
'safe_to_resubmit' => false,
'error_code' => '',
'result' => [
'uuid' => $wrongIdentityUuid,
'hostname' => 'prepared-identity.example',
'settings_changed' => true,
],
],
'automatic_applied_succeeded_nofence:' . str_repeat('9', 64)
);
$preparedWrongIdentity = CloudJob::findOrFail($wrongIdentityId);
$preparedWrongIdentityMarker = (string) $preparedWrongIdentity
->effect_resolution_reason;
$wrongIdentityEventId = 'automatic-wrong-identity';
$acceptedWrongIdentity = CloudOperationCompletionService::accept([
'type' => 'completion',
'id' => $wrongIdentityEventId,
'agent_id' => '7',
'operation_id' => str_repeat('e3', 16),
'action' => $expectedIdentity['action'],
'resource_id' => '00000000-0000-4000-8000-000000009999',
'request_fingerprint' => $expectedIdentity['request_fingerprint'],
'status' => 'succeeded',
'effect_may_have_applied' => false,
'safe_to_resubmit' => false,
'error_code' => '',
'result' => [
'uuid' => $wrongIdentityUuid,
'hostname' => 'prepared-identity.example',
'settings_changed' => true,
],
]);
$preparedWrongIdentity = CloudJob::findOrFail($wrongIdentityId);
$wrongIdentityEvent = Capsule::table(
'cloud_agent_completion_events'
)->where('event_id', $wrongIdentityEventId)->first();
$assert($acceptedWrongIdentity === [
'accepted' => true,
'disposition' => 'rejected',
]
&& (string) $wrongIdentityEvent->disposition === 'rejected'
&& (string) $preparedWrongIdentity->effect_resolution === 'unknown'
&& (string) $preparedWrongIdentity->effect_resolution_reason
=== $preparedWrongIdentityMarker
&& (bool) $preparedWrongIdentity->finalization_incident
&& (string) $preparedWrongIdentity->finalization_error_code
=== 'recovery_operation_identity_conflict',
'A definitive wrong-operation receipt was not isolated from prepared recovery.');
Capsule::table('cloud_jobs')->where('id', $wrongIdentityId)->update([
'available_at' => 0,
'lease_token' => null,
'lease_until' => 0,
]);
$beforeWrongIdentityResume = count($client->requests);
$assert((new CloudAutomaticOperationRecoveryService())->tick(),
'Prepared recovery did not resume after quarantining a wrong receipt.');
$preparedWrongIdentity = CloudJob::findOrFail($wrongIdentityId);
$wrongIdentityEvent = Capsule::table(
'cloud_agent_completion_events'
)->where('event_id', $wrongIdentityEventId)->first();
$assert((string) $preparedWrongIdentity->effect_resolution === 'applied'
&& !(bool) $preparedWrongIdentity->finalization_incident
&& $preparedWrongIdentity->finalization_error_code === null
&& (string) $wrongIdentityEvent->disposition === 'rejected'
&& count($client->requests) === $beforeWrongIdentityResume,
'The quarantined wrong receipt starved or contaminated prepared convergence.');
// Failed journal + dual settings markers: prepare first, then resolve the
// fence. Simulate a lost POST response and prove one GET recovers it.
[$lostId, $lostUuid] = $insertSettingsJob(
str_repeat('2', 32),
['hostname' => 'recovered.example']
);
$lostAudit = $audit($lostId);
$lostFenceRequest = null;
$client->responses[] = $inspection(
str_repeat('2', 32),
'failed',
'applied',
[
'uuid' => $lostUuid,
'hostname' => 'recovered.example',
'status' => 'shutoff',
'settings_changed' => true,
]
);
$client->responses[] = $notFound();
$client->responses[] = static function (array $request) use (
&$lostFenceRequest
): Throwable {
$lostFenceRequest = $request;
return new RuntimeException('response lost after durable commit');
};
$client->responses[] = static function (array $request) use (
&$lostFenceRequest,
$fenceResponse
): array {
$read = $lostFenceRequest;
$read['method'] = $request['method'];
$read['target'] = $request['target'];
return $fenceResponse($read, time());
};
$beforeLostRequests = count($client->requests);
$assert((new CloudAutomaticOperationRecoveryService())->tick(),
'The worker did not recover a failed-journal applied task.');
$lost = CloudJob::find($lostId);
$lostSpec = json_decode(
(string) Capsule::table('cloud_instances')
->where('id', $lostId)->value('spec_json'),
true,
flags: JSON_THROW_ON_ERROR
);
$lostRequests = array_slice(
$client->requests,
$beforeLostRequests
);
$assert((string) $lost->effect_resolution === 'applied'
&& $audit($lostId) === $lostAudit
&& ($lostSpec['guest_hostname'] ?? null)
=== 'recovered.example'
&& count(array_filter(
$lostRequests,
static fn (array $request): bool =>
$request['method'] === 'POST'
)) === 1
&& count($lostRequests) === 4
&& !CloudJobService::blockingQueryForInstance($lostId)->exists(),
'Lost fence response recovery was not single-write and convergent.');
// Exact absence markers prove a settings request did not commit. PHP has
// no instance ledger delta, but still prepares before releasing the fence.
[$absentId] = $insertSettingsJob(
str_repeat('3', 32),
['hostname' => 'not-applied.example']
);
$absentAudit = $audit($absentId);
$absentSpec = (string) Capsule::table('cloud_instances')
->where('id', $absentId)->value('spec_json');
$client->responses[] = $inspection(
str_repeat('3', 32),
'failed',
'not_applied'
);
$client->responses[] = $notFound();
$client->responses[] = static fn (array $request): array =>
$fenceResponse($request, time());
$assert((new CloudAutomaticOperationRecoveryService())->tick(),
'The worker did not converge exact not-applied evidence.');
$absent = CloudJob::find($absentId);
$assert((string) $absent->effect_resolution === 'not_applied'
&& (int) $absent->effect_resolved_by === 0
&& $audit($absentId) === $absentAudit
&& (string) Capsule::table('cloud_instances')
->where('id', $absentId)->value('spec_json') === $absentSpec
&& !CloudJobService::blockingQueryForInstance($absentId)->exists(),
'Not-applied recovery changed the ledger or retained its blocker.');
// An immutable failed Journal row which explicitly records no possible
// effect has no Agent fence. Its bounded original error code is evidence,
// not a reason to invent a settings_not_committed fence marker.
[$exactAbsentId] = $insertSettingsJob(
str_repeat('9', 32),
['hostname' => 'exact-not-applied.example']
);
$exactAbsentAudit = $audit($exactAbsentId);
$client->responses[] = $inspection(
str_repeat('9', 32),
'failed',
'not_applied',
null,
false,
'cloudinit_validation_failed'
);
$beforeExactAbsent = count($client->requests);
$assert((new CloudAutomaticOperationRecoveryService())->tick(),
'The worker did not converge exact no-effect Journal evidence.');
$exactAbsentRequests = array_slice(
$client->requests,
$beforeExactAbsent
);
$exactAbsent = CloudJob::find($exactAbsentId);
$assert((string) $exactAbsent->effect_resolution === 'not_applied'
&& $audit($exactAbsentId) === $exactAbsentAudit
&& !CloudJobService::blockingQueryForInstance(
$exactAbsentId
)->exists()
&& count($exactAbsentRequests) === 1
&& $exactAbsentRequests[0]['method'] === 'GET'
&& str_ends_with($exactAbsentRequests[0]['target'], '/inspect'),
'Exact no-effect Journal recovery contacted a fence endpoint.');
// A Journal fence already resolved outside this coordinator is not the
// same thing as an operation which never needed a fence. Without PHP's
// matching durable prepare marker, automatic recovery must not accept the
// possibly conflicting Agent verdict.
[$externallyResolvedId] = $insertSettingsJob(
str_repeat('a', 32),
['hostname' => 'externally-resolved.example']
);
$client->responses[] = $inspection(
str_repeat('a', 32),
'failed',
'not_applied',
null,
false,
'settings_not_committed',
'resolved'
);
$beforeExternallyResolved = count($client->requests);
$assert((new CloudAutomaticOperationRecoveryService())->tick(),
'The externally resolved fence was not inspected.');
$externallyResolved = CloudJob::find($externallyResolvedId);
$externallyResolvedRequests = array_slice(
$client->requests,
$beforeExternallyResolved
);
$assert((string) $externallyResolved->effect_resolution === 'unknown'
&& (int) $externallyResolved->available_at >= time() + 290
&& CloudJobService::blockingQueryForInstance(
$externallyResolvedId
)->exists()
&& count($externallyResolvedRequests) === 1
&& str_ends_with(
$externallyResolvedRequests[0]['target'],
'/inspect'
),
'An unprepared externally resolved fence bypassed PHP recovery intent.');
// Partial and inconclusive evidence stay blocked and receive a five-minute
// retry delay. Two tasks also prove each worker tick handles only one.
[$partialId] = $insertSettingsJob(
str_repeat('4', 32),
['hostname' => 'partial.example']
);
[$inconclusiveId] = $insertSettingsJob(
str_repeat('5', 32),
['hostname' => 'inconclusive.example']
);
$client->responses[] = $inspection(
str_repeat('4', 32),
'failed',
'partial'
);
$client->responses[] = $inspection(
str_repeat('5', 32),
'failed',
'inconclusive'
);
$assert((new CloudAutomaticOperationRecoveryService())->tick(),
'The partial-evidence task was not inspected.');
$partial = CloudJob::find($partialId);
$notYetClaimed = CloudJob::find($inconclusiveId);
$assert((string) $partial->effect_resolution === 'unknown'
&& (int) $partial->available_at >= time() + 290
&& CloudJobService::blockingQueryForInstance($partialId)->exists()
&& (int) $notYetClaimed->available_at === 0,
'One worker tick did not stop after one inconclusive recovery.');
$assert((new CloudAutomaticOperationRecoveryService())->tick(),
'The inconclusive-evidence task was not inspected.');
$inconclusive = CloudJob::find($inconclusiveId);
$assert((string) $inconclusive->effect_resolution === 'unknown'
&& (int) $inconclusive->available_at >= time() + 290
&& CloudJobService::blockingQueryForInstance(
$inconclusiveId
)->exists(),
'Inconclusive evidence released its blocker or hot-looped.');
// Agent downtime backs off without changing the immutable task outcome.
[$offlineId] = $insertSettingsJob(
str_repeat('6', 32),
['hostname' => 'offline.example']
);
$offlineAudit = $audit($offlineId);
$client->responses[] = new CloudAgentError(
'offline', 503, 'agent_unavailable', true, 503
);
$assert((new CloudAutomaticOperationRecoveryService())->tick(),
'The offline recovery task was not claimed.');
$offline = CloudJob::find($offlineId);
$assert((string) $offline->effect_resolution === 'unknown'
&& $audit($offlineId) === $offlineAudit
&& $offline->lease_token === null
&& (int) $offline->available_at >= time() + 290
&& CloudJobService::blockingQueryForInstance($offlineId)->exists(),
'Agent downtime changed audit state or did not back off.');
// A retried low-ID offline host must not repeatedly jump ahead of a
// never-attempted higher-ID task. Persisted available_at age, then ID,
// defines the recovery queue order.
Capsule::table('cloud_jobs')->where('id', $offlineId)->update([
'available_at' => time() - 1,
'lease_until' => 0,
'lease_token' => null,
]);
[$fairId, $fairUuid] = $insertSettingsJob(
str_repeat('d', 32),
['hostname' => 'fair-queue.example']
);
$client->responses[] = $inspection(
str_repeat('d', 32),
'succeeded',
'applied',
[
'uuid' => $fairUuid,
'hostname' => 'fair-queue.example',
'settings_changed' => true,
]
);
$assert((new CloudAutomaticOperationRecoveryService())->tick(),
'The recovery queue did not claim a fair successor.');
$assert((string) CloudJob::find($fairId)->effect_resolution === 'applied'
&& (string) CloudJob::find($offlineId)->effect_resolution === 'unknown',
'A retried low-ID offline task starved a never-attempted task.');
Capsule::table('cloud_jobs')->where('id', $offlineId)->update([
'available_at' => time() + 300,
]);
// Attempt count is audit metadata only. The same operation remains in the
// recovery queue beyond the former 12-attempt boundary.
[$limitedId] = $insertSettingsJob(
str_repeat('c', 32),
['hostname' => 'retry-limit.example']
);
$beforeLimitedRequests = count($client->requests);
for ($attempt = 1; $attempt <= 13; $attempt++) {
Capsule::table('cloud_jobs')->where('id', $limitedId)->update([
'available_at' => 0,
'lease_until' => 0,
'lease_token' => null,
]);
$client->responses[] = new CloudAgentError(
'offline', 503, 'agent_unavailable', true, 503
);
$assert((new CloudAutomaticOperationRecoveryService())->tick(),
'A persistent recovery attempt was not claimed.');
}
$limited = CloudJob::find($limitedId);
$limitedPayloadBefore = (string) $limited->sealed_payload;
$limitedRequestCount = count($client->requests);
$assert((bool) $limited->finalization_incident
&& (string) $limited->finalization_error_code
=== 'recovery_waiting_connection'
&& preg_match(
'/^automatic_recovery:v1:[1-9][0-9]{0,10}:13:pending$/D',
(string) $limited->effect_resolution_reason
) === 1
&& (string) $limited->effect_resolution === 'unknown'
&& (string) $limited->sealed_payload === $limitedPayloadBefore
&& CloudJobService::blockingQueryForInstance($limitedId)->exists()
&& $limitedRequestCount - $beforeLimitedRequests === 13
&& (int) $limited->available_at >= time() + 290,
'Recovery did not remain queued beyond the former attempt limit.');
// First pass prepares PHP, commits the Agent fence, then loses both the
// POST response and its verification GET. A later pass must read that
// existing fence and commit PHP without depending on a new host inspect.
[$crashId, $crashUuid] = $insertSettingsJob(
str_repeat('7', 32),
['hostname' => 'crash-recovered.example']
);
Capsule::table('cloud_jobs')->where('id', $crashId)->update([
// The pending inspection budget is about to expire. A successful
// prepare must start a separate crash-resume budget instead of
// stranding the durable intent at attempt 12.
'effect_resolution_reason' => 'automatic_recovery:v1:'
. (time() - 300) . ':11:pending',
]);
$crashInspection = $inspection(
str_repeat('7', 32),
'failed',
'applied',
[
'uuid' => $crashUuid,
'hostname' => 'crash-recovered.example',
'settings_changed' => true,
]
);
$client->responses[] = $crashInspection;
$client->responses[] = $notFound();
$crashFenceRequest = null;
$client->responses[] = static function (array $request) use (
&$crashFenceRequest
): Throwable {
$crashFenceRequest = $request;
return new RuntimeException('fence committed before response loss');
};
$client->responses[] = new CloudAgentError(
'verification read offline', 503, 'agent_unavailable', true, 503
);
$assert((new CloudAutomaticOperationRecoveryService())->tick(),
'The crash-window fixture was not prepared.');
$prepared = CloudJob::find($crashId);
$assert(str_starts_with(
(string) $prepared->effect_resolution_reason,
'automatic_recovery:v1:'
)
&& preg_match(
'/^automatic_recovery:v1:[1-9][0-9]{0,10}:1:prepared:/D',
(string) $prepared->effect_resolution_reason
) === 1
&& str_contains(
(string) $prepared->effect_resolution_reason,
':prepared:automatic_applied_failed_fence:'
)
&& (string) $prepared->effect_resolution === 'unknown'
&& CloudJobService::blockingQueryForInstance($crashId)->exists(),
'Prepare failure window did not retain its blocker and intent.');
Capsule::table('cloud_jobs')->where('id', $crashId)->update([
'available_at' => 0,
'lease_until' => 0,
'lease_token' => null,
]);
$client->responses[] = static function (array $request) use (
&$crashFenceRequest,
$fenceResponse
): array {
$read = $crashFenceRequest;
$read['method'] = $request['method'];
$read['target'] = $request['target'];
return $fenceResponse($read, time());
};
$beforeCrashRetry = count($client->requests);
$assert((new CloudAutomaticOperationRecoveryService())->tick(),
'Prepared recovery did not resume after the crash window.');
$crashRetryRequests = array_slice(
$client->requests,
$beforeCrashRetry
);
$assert((string) CloudJob::find($crashId)->effect_resolution === 'applied'
&& count(array_filter(
$crashRetryRequests,
static fn (array $request): bool =>
$request['method'] === 'POST'
)) === 0
&& !CloudJobService::blockingQueryForInstance($crashId)->exists()
&& count($crashRetryRequests) === 1
&& str_ends_with(
$crashRetryRequests[0]['target'],
'/fence-resolution'
),
'Prepared recovery re-inspected/replayed or failed to commit.');
// Prepared crash-resume also remains eligible beyond the former limit.
[$preparedLimitId] = $insertSettingsJob(
str_repeat('e', 32),
['hostname' => 'prepared-limit.example']
);
$client->responses[] = $inspection(
str_repeat('e', 32),
'failed',
'not_applied'
);
$client->responses[] = $notFound();
$client->responses[] = new RuntimeException(
'fence response lost before verification'
);
$client->responses[] = new CloudAgentError(
'verification read offline',
503,
'agent_unavailable',
true,
503
);
$beforePreparedLimit = count($client->requests);
$assert((new CloudAutomaticOperationRecoveryService())->tick(),
'The prepared-limit fixture did not enter its resume stage.');
$preparedLimitFirst = CloudJob::find($preparedLimitId);
$assert(preg_match(
'/^automatic_recovery:v1:[1-9][0-9]{0,10}:1:prepared:automatic_not_applied_failed_fence:/D',
(string) $preparedLimitFirst->effect_resolution_reason
) === 1,
'Preparing did not reset the independent resume budget.');
for ($attempt = 2; $attempt <= 13; $attempt++) {
Capsule::table('cloud_jobs')->where('id', $preparedLimitId)->update([
'available_at' => 0,
'lease_until' => 0,
'lease_token' => null,
]);
$client->responses[] = new CloudAgentError(
'prepared fence read offline',
503,
'agent_unavailable',
true,
503
);
$assert((new CloudAutomaticOperationRecoveryService())->tick(),
'A persistent prepared-resume attempt was not claimed.');
}
$preparedLimited = CloudJob::find($preparedLimitId);
$preparedLimitRequestCount = count($client->requests);
$preparedLimitPayload = (string) $preparedLimited->sealed_payload;
$preparedLimitEscrow = CloudSecrets::decrypt(
$preparedLimitPayload,
'job-payload'
);
$preparedLimitRequests = array_slice(
$client->requests,
$beforePreparedLimit
);
$preparedLimitFenceWrite = null;
foreach ($preparedLimitRequests as $request) {
if ($request['method'] === 'POST'
&& str_ends_with($request['target'], '/resolve-fence')) {
$preparedLimitFenceWrite = $request;
break;
}
}
$assert((bool) $preparedLimited->finalization_incident
&& (string) $preparedLimited->finalization_error_code
=== 'recovery_waiting_connection'
&& preg_match(
'/^automatic_recovery:v1:[1-9][0-9]{0,10}:13:prepared:/D',
(string) $preparedLimited->effect_resolution_reason
) === 1
&& (string) $preparedLimited->effect_resolution === 'unknown'
&& CloudJobService::blockingQueryForInstance(
$preparedLimitId
)->exists()
&& count($client->requests) - $beforePreparedLimit === 16
&& is_array($preparedLimitEscrow['_operation_recovery'] ?? null)
&& ($preparedLimitEscrow['_operation_recovery']['operation']
['operation_id'] ?? null) === str_repeat('e', 32)
&& ($preparedLimitEscrow['_operation_recovery']['operation']
['status'] ?? null) === 'failed'
&& is_array($preparedLimitFenceWrite)
&& (int) $preparedLimited->available_at >= time() + 290,
'Prepared recovery did not remain queued beyond the former limit.');
// A fourteenth pass can still finish the original crash-resume protocol.
// The retry marker contains only hashes, so this succeeds only if the
// worker reconstructs the exact failed operation from the encrypted
// _operation_recovery envelope retained across attempts 1 through 13.
Capsule::table('cloud_jobs')->where('id', $preparedLimitId)->update([
'available_at' => 0,
'lease_until' => 0,
'lease_token' => null,
]);
$client->responses[] = static fn (): array =>
$fenceResponse($preparedLimitFenceWrite, time());
$beforePreparedLimitResume = count($client->requests);
$assert((new CloudAutomaticOperationRecoveryService())->tick(),
'Prepared recovery was not claimable after attempt thirteen.');
$preparedLimitResumeRequests = array_slice(
$client->requests,
$beforePreparedLimitResume
);
$preparedLimitResolved = CloudJob::find($preparedLimitId);
$preparedLimitFinalPayload = CloudSecrets::decrypt(
(string) $preparedLimitResolved->sealed_payload,
'job-payload'
);
$assert((string) $preparedLimitResolved->effect_resolution
=== 'not_applied'
&& !(bool) $preparedLimitResolved->finalization_incident
&& $preparedLimitResolved->finalization_error_code === null
&& !array_key_exists(
'_operation_recovery',
$preparedLimitFinalPayload
)
&& !CloudJobService::blockingQueryForInstance(
$preparedLimitId
)->exists()
&& count($preparedLimitResumeRequests) === 1
&& $preparedLimitResumeRequests[0]['method'] === 'GET'
&& str_ends_with(
$preparedLimitResumeRequests[0]['target'],
'/fence-resolution'
),
'Attempt thirteen stranded encrypted prepared recovery state.');
// A timed-out graceful delete may later be proven not applied because the
// Agent requires an explicitly authorized force generation. Recovery must
// preserve that exact business error: it is the immutable authorization
// anchor for enqueueForceDelete(), not an instance-state contradiction.
[$forceRequiredId, $forceRequiredUuid] = $insertSettingsJob(
str_repeat('ab', 16),
['hostname' => 'delete-force-required.example']
);
Capsule::table('cloud_jobs')->where('id', $forceRequiredId)->update([
'kind' => 'delete',
'sealed_payload' => CloudSecrets::encrypt([
'method' => 'DELETE',
'request_target' => '/v1/vms/' . $forceRequiredUuid,
'body' => ['remove_managed_disks' => true],
'owner_id' => 1,
'can_manage' => false,
], 'job-payload'),
]);
$beforeForceRequiredRecovery = count($client->requests);
$client->responses[] = $inspection(
str_repeat('ab', 16),
'failed',
'not_applied',
null,
false,
'force_required',
'not_required'
);
$assert((new CloudAutomaticOperationRecoveryService())->tick(),
'The force-required delete recovery was not claimed.');
$forceRequired = CloudJob::find($forceRequiredId);
$forceRequiredInstance = \App\Models\CloudInstance::find(
$forceRequiredId
);
$assert((string) $forceRequired->effect_resolution === 'not_applied'
&& (string) $forceRequired->error_code === 'force_required'
&& (string) $forceRequired->phase === 'terminal'
&& (string) $forceRequired->outcome === 'failed'
&& $forceRequired->dispatched_at !== null,
'Force-required inspection did not converge into its authorization anchor: '
. json_encode($forceRequired->toArray(), JSON_THROW_ON_ERROR));
$beforeForceJobCount = CloudJob::count();
Capsule::table('cloud_jobs')->where('id', $forceRequiredId)->update([
'finalization_incident' => 1,
'finalization_error_code' => 'recovery_ledger_conflict',
]);
try {
CloudJobService::enqueueForceDelete(
$forceRequired->fresh(),
[
'user_id' => 99,
'is_admin' => true,
'is_super_admin' => true,
'can_manage' => true,
]
);
throw new RuntimeException(
'A sticky recovery incident created a force-delete generation.'
);
} catch (CloudError $error) {
$assert($error->httpStatus === 409
&& CloudJob::count() === $beforeForceJobCount,
'Sticky force-delete rejection changed the immutable job ledger.');
}
Capsule::table('cloud_jobs')->where('id', $forceRequiredId)->update([
'finalization_incident' => 0,
'finalization_error_code' => null,
'receipt_deadline_at' => null,
]);
try {
CloudJobService::enqueueForceDelete(
$forceRequired->fresh(),
[
'user_id' => 99,
'is_admin' => true,
'is_super_admin' => true,
'can_manage' => true,
]
);
throw new RuntimeException(
'A damaged Agent envelope created a force-delete generation.'
);
} catch (CloudError $error) {
$assert($error->httpStatus === 409
&& CloudJob::count() === $beforeForceJobCount,
'Envelope force-delete rejection changed the immutable job ledger.');
}
Capsule::table('cloud_jobs')->where('id', $forceRequiredId)->update([
'receipt_deadline_at' => $now + 120,
]);
$forceRequired->refresh();
$forceDelete = CloudJobService::enqueueForceDelete(
$forceRequired,
[
'user_id' => 99,
'is_admin' => true,
'is_super_admin' => true,
'can_manage' => true,
]
);
$nextId = max($nextId, (int) $forceDelete->id + 1);
$forceDeletePayload = CloudSecrets::decrypt(
(string) $forceDelete->sealed_payload,
'job-payload'
);
$assert((string) $forceRequired->effect_resolution === 'not_applied'
&& (string) $forceRequired->error_code === 'force_required'
&& (string) $forceRequiredInstance->status === 'active'
&& !(bool) $forceRequired->finalization_incident
&& (string) $forceDelete->kind === 'delete_force'
&& ($forceDeletePayload['prior_job_id'] ?? null)
=== $forceRequiredId
&& ($forceDeletePayload['prior_operation_id'] ?? null)
=== str_repeat('ab', 16)
&& count($client->requests) === $beforeForceRequiredRecovery + 1
&& $client->requests[array_key_last($client->requests)]['method']
=== 'GET',
'Recovered force-required evidence did not preserve the separately authorized force-delete path.');
// Age is also audit metadata only; a six-hour-old operation is still
// inspected with the original operation ID.
[$agedId] = $insertSettingsJob(
str_repeat('f', 32),
['hostname' => 'age-limit.example']
);
Capsule::table('cloud_jobs')->where('id', $agedId)->update([
'effect_resolution_reason' => 'automatic_recovery:v1:'
. (time() - 21601) . ':1:pending',
]);
$beforeAged = count($client->requests);
$client->responses[] = new CloudAgentError(
'offline', 503, 'agent_unavailable', true, 503
);
$assert((new CloudAutomaticOperationRecoveryService())->tick(),
'A six-hour-old recovery was not claimed.');
$aged = CloudJob::find($agedId);
$assert((bool) $aged->finalization_incident
&& (string) $aged->finalization_error_code
=== 'recovery_waiting_connection'
&& preg_match(
'/^automatic_recovery:v1:[1-9][0-9]{0,10}:2:pending$/D',
(string) $aged->effect_resolution_reason
) === 1
&& (string) $aged->effect_resolution === 'unknown'
&& CloudJobService::blockingQueryForInstance($agedId)->exists()
&& count($client->requests) === $beforeAged + 1,
'An aged recovery stopped retrying or released safety.');
// A corrupt dispatch ledger remains a recovery candidate forever, but it
// cannot authorize either an Agent read or consumption of even an exact
// received completion. The only safe automatic action is to record the
// concrete incident and back off until an operator repairs the 0/1 fence.
[$invalidDispatchId, $invalidDispatchUuid] = $insertSettingsJob(
str_repeat('dc', 16),
['hostname' => 'invalid-dispatch-ledger.example']
);
$invalidDispatchEventId = 'automatic-invalid-dispatch-ledger';
Capsule::table('cloud_jobs')->where('id', $invalidDispatchId)->update([
'dispatch_attempts' => 2,
// Simulate a historical fail-open release which mistook the corrupt
// non-zero counter for a fully settled successful Agent mutation.
'outcome' => 'succeeded',
'status' => 'succeeded',
'error_code' => null,
'effect_may_have_applied' => 0,
'safe_to_resubmit' => 0,
'effect_resolution' => 'applied',
'completion_event_id' => $invalidDispatchEventId,
'available_at' => 0,
'lease_token' => null,
'lease_until' => 0,
'finalization_incident' => 0,
'finalization_error_code' => null,
]);
$invalidDispatchJob = CloudJob::findOrFail($invalidDispatchId);
$invalidDispatchEnvelope = (string) $invalidDispatchJob->sealed_payload;
$invalidDispatchTerminalAt = (int) $invalidDispatchJob->terminal_at;
$invalidDispatchResultJson = (string) $invalidDispatchJob->result_json;
$invalidDispatchIdentity = CloudOperationRecoveryIdentity::expected(
$invalidDispatchJob,
true
);
$invalidDispatchResult = [
'uuid' => $invalidDispatchUuid,
'hostname' => 'invalid-dispatch-ledger.example',
'settings_changed' => true,
];
$invalidDispatchWire = [
'event_id' => $invalidDispatchEventId,
'host_id' => 7,
'operation_id' => (string) $invalidDispatchJob->operation_id,
'action' => $invalidDispatchIdentity['action'],
'resource_id' => $invalidDispatchIdentity['resource_id'],
'request_fingerprint' =>
$invalidDispatchIdentity['request_fingerprint'],
'status' => 'succeeded',
'effect_may_have_applied' => false,
'safe_to_resubmit' => false,
'error_code' => '',
'result' => $invalidDispatchResult,
];
Capsule::table('cloud_agent_completion_events')->insert([
'event_id' => $invalidDispatchWire['event_id'],
'host_id' => 7,
'operation_id' => $invalidDispatchWire['operation_id'],
'payload_sha256' => hash('sha256', json_encode(
$invalidDispatchWire,
JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES
| JSON_UNESCAPED_UNICODE
)),
'outcome' => 'succeeded',
'effect_may_have_applied' => 0,
'safe_to_resubmit' => 0,
'error_code' => '',
'result_json' => json_encode([
'schema' => 'virt_cloud_completion_event',
'version' => 1,
'identity' => $invalidDispatchIdentity,
'result' => $invalidDispatchResult,
], JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES
| JSON_UNESCAPED_UNICODE),
'disposition' => 'received',
'received_at' => time(),
'applied_at' => null,
]);
$beforeInvalidDispatchRequests = count($client->requests);
$assert((new CloudAutomaticOperationRecoveryService())->tick(),
'An invalid dispatch ledger permanently left the recovery queue.');
$invalidDispatchJob = CloudJob::findOrFail($invalidDispatchId);
$invalidDispatchEvent = Capsule::table(
'cloud_agent_completion_events'
)->where(
'event_id',
$invalidDispatchWire['event_id']
)->first();
$assert((string) $invalidDispatchJob->phase === 'terminal'
&& (string) $invalidDispatchJob->outcome === 'succeeded'
&& (string) $invalidDispatchJob->status === 'succeeded'
&& $invalidDispatchJob->error_code === null
&& !(bool) $invalidDispatchJob->effect_may_have_applied
&& (string) $invalidDispatchJob->effect_resolution === 'applied'
&& !(bool) $invalidDispatchJob->safe_to_resubmit
&& (int) $invalidDispatchJob->terminal_at
=== $invalidDispatchTerminalAt
&& (string) $invalidDispatchJob->result_json
=== $invalidDispatchResultJson
&& (string) $invalidDispatchJob->completion_event_id
=== $invalidDispatchEventId
&& (bool) $invalidDispatchJob->finalization_incident
&& (string) $invalidDispatchJob->finalization_error_code
=== 'recovery_dispatch_ledger_conflict'
&& (string) $invalidDispatchJob->sealed_payload
=== $invalidDispatchEnvelope
&& $invalidDispatchJob->lease_token === null
&& (int) $invalidDispatchJob->lease_until === 0
&& (int) $invalidDispatchJob->available_at >= time() + 290
&& (string) $invalidDispatchEvent->disposition === 'received'
&& $invalidDispatchEvent->applied_at === null
&& count($client->requests) === $beforeInvalidDispatchRequests,
'Invalid dispatch recovery contacted Agent, consumed evidence, or released its fence.');
// Once the durable ledger is repaired to the only valid dispatched state,
// the same unconsumed receipt must resume convergence without waiting for
// an Agent retransmission or issuing any Agent request.
Capsule::table('cloud_jobs')->where('id', $invalidDispatchId)->update([
'dispatch_attempts' => 1,
'available_at' => 0,
'lease_token' => null,
'lease_until' => 0,
]);
$assert((new CloudAutomaticOperationRecoveryService())->tick(),
'A repaired dispatch ledger could not resume its queued receipt.');
$repairedDispatchJob = CloudJob::findOrFail($invalidDispatchId);
$repairedDispatchEvent = Capsule::table(
'cloud_agent_completion_events'
)->where(
'event_id',
$invalidDispatchWire['event_id']
)->first();
$assert((string) $repairedDispatchJob->phase === 'terminal'
&& (string) $repairedDispatchJob->outcome === 'succeeded'
&& (string) $repairedDispatchJob->status === 'succeeded'
&& (int) $repairedDispatchJob->terminal_at
=== $invalidDispatchTerminalAt
&& $repairedDispatchJob->error_code === null
&& (string) $repairedDispatchJob->result_json
=== $invalidDispatchResultJson
&& (string) $repairedDispatchJob->effect_resolution === 'applied'
&& !(bool) $repairedDispatchJob->finalization_incident
&& $repairedDispatchJob->finalization_error_code === null
&& (string) $repairedDispatchEvent->disposition === 'applied'
&& (int) $repairedDispatchEvent->applied_at > 0
&& count($client->requests) === $beforeInvalidDispatchRequests,
'Repairing the dispatch ledger did not resume the exact stored completion.');
// An exact immutable receipt cannot authorize convergence when any other
// field in the dispatched Agent envelope is damaged. Claiming must keep
// the receipt untouched and record a local-only state conflict without an
// Agent request.
[$corruptReceiptEnvelopeId, $corruptReceiptEnvelopeUuid] =
$insertSettingsJob(
str_repeat('b1', 16),
['hostname' => 'corrupt-receipt-envelope.example']
);
$corruptReceiptEnvelopeJob = CloudJob::findOrFail(
$corruptReceiptEnvelopeId
);
$corruptReceiptEnvelopeIdentity =
CloudOperationRecoveryIdentity::expected(
$corruptReceiptEnvelopeJob,
true
);
$corruptReceiptEnvelopeResult = [
'uuid' => $corruptReceiptEnvelopeUuid,
'hostname' => 'corrupt-receipt-envelope.example',
'settings_changed' => true,
];
$corruptReceiptEnvelopeWire = [
'event_id' => 'automatic-corrupt-receipt-envelope',
'host_id' => 7,
'operation_id' => (string) $corruptReceiptEnvelopeJob->operation_id,
'action' => $corruptReceiptEnvelopeIdentity['action'],
'resource_id' => $corruptReceiptEnvelopeIdentity['resource_id'],
'request_fingerprint' =>
$corruptReceiptEnvelopeIdentity['request_fingerprint'],
'status' => 'succeeded',
'effect_may_have_applied' => false,
'safe_to_resubmit' => false,
'error_code' => '',
'result' => $corruptReceiptEnvelopeResult,
];
Capsule::table('cloud_agent_completion_events')->insert([
'event_id' => $corruptReceiptEnvelopeWire['event_id'],
'host_id' => 7,
'operation_id' => $corruptReceiptEnvelopeWire['operation_id'],
'payload_sha256' => hash('sha256', json_encode(
$corruptReceiptEnvelopeWire,
JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES
| JSON_UNESCAPED_UNICODE
)),
'outcome' => 'succeeded',
'effect_may_have_applied' => 0,
'safe_to_resubmit' => 0,
'error_code' => '',
'result_json' => json_encode([
'schema' => 'virt_cloud_completion_event',
'version' => 1,
'identity' => $corruptReceiptEnvelopeIdentity,
'result' => $corruptReceiptEnvelopeResult,
], JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES
| JSON_UNESCAPED_UNICODE),
'disposition' => 'received',
'received_at' => time(),
'applied_at' => null,
]);
Capsule::table('cloud_jobs')->where(
'id',
$corruptReceiptEnvelopeId
)->update([
'request_idempotency_key' => 'bad-receipt-key',
'available_at' => 0,
'lease_token' => null,
'lease_until' => 0,
]);
Capsule::table('cloud_templates')->insert([
'id' => 'identity-priority-template',
'active_job_id' => $corruptReceiptEnvelopeId,
'state' => 'active',
]);
$beforeCorruptReceiptEnvelopeRequests = count($client->requests);
$assert((new CloudAutomaticOperationRecoveryService())->tick(),
'A damaged envelope with an exact receipt left recovery.');
$corruptReceiptEnvelopeJob = CloudJob::findOrFail(
$corruptReceiptEnvelopeId
);
$corruptReceiptEnvelopeEvent = Capsule::table(
'cloud_agent_completion_events'
)->where(
'event_id',
$corruptReceiptEnvelopeWire['event_id']
)->first();
$assert((string) $corruptReceiptEnvelopeJob->effect_resolution
=== 'unknown'
&& (bool) $corruptReceiptEnvelopeJob->finalization_incident
&& (string) $corruptReceiptEnvelopeJob->finalization_error_code
=== 'recovery_operation_identity_conflict'
&& (int) $corruptReceiptEnvelopeJob->available_at >= time() + 290
&& (int) $corruptReceiptEnvelopeJob->lease_until >= time() + 290
&& (string) $corruptReceiptEnvelopeEvent->disposition === 'received'
&& $corruptReceiptEnvelopeEvent->applied_at === null
&& (int) Capsule::table('cloud_templates')->where(
'id',
'identity-priority-template'
)->value('active_job_id') === $corruptReceiptEnvelopeId
&& CloudJobService::blockingQueryForInstance(
$corruptReceiptEnvelopeId
)->exists()
&& count($client->requests)
=== $beforeCorruptReceiptEnvelopeRequests,
'A damaged envelope consumed its receipt, contacted Agent, or falsely unlocked.');
Capsule::table('cloud_templates')->where(
'id',
'identity-priority-template'
)->delete();
// A prepared no-fence conclusion must revalidate the complete envelope at
// the locked publication boundary. This protects direct callers and the
// crash-resume path even if claim-time validation is bypassed or races.
[$preparedStateConflictId, $preparedStateConflictUuid] =
$insertSettingsJob(
str_repeat('b2', 16),
['hostname' => 'prepared-state-conflict.example']
);
$preparedStateConflictIdentity =
CloudOperationRecoveryIdentity::expected(
CloudJob::findOrFail($preparedStateConflictId),
true
);
$preparedStateConflictOperation = $preparedStateConflictIdentity + [
'operation_id' => str_repeat('b2', 16),
'status' => 'succeeded',
'effect_may_have_applied' => false,
'safe_to_resubmit' => false,
'error_code' => '',
'result' => [
'uuid' => $preparedStateConflictUuid,
'hostname' => 'prepared-state-conflict.example',
'settings_changed' => true,
],
];
$preparedStateConflictSource =
'automatic_applied_succeeded_nofence:' . str_repeat('2', 64);
CloudOperationRecoveryService::prepareApplied(
$preparedStateConflictId,
$preparedStateConflictOperation,
$preparedStateConflictSource
);
$preparedStateConflictMarker = (string) CloudJob::findOrFail(
$preparedStateConflictId
)->effect_resolution_reason;
Capsule::table('cloud_jobs')->where(
'id',
$preparedStateConflictId
)->update([
'receipt_deadline_at' => 0,
'available_at' => 0,
'lease_token' => null,
'lease_until' => 0,
]);
try {
CloudOperationRecoveryService::commitApplied(
$preparedStateConflictId,
$preparedStateConflictOperation,
$preparedStateConflictSource,
0
);
throw new RuntimeException(
'A direct applied commit published a damaged Agent envelope.'
);
} catch (CloudRecoveryInstanceStateConflict $error) {
$assert($error->httpStatus === 409,
'The applied envelope guard returned the wrong status.');
}
$preparedStateConflict = CloudJob::findOrFail(
$preparedStateConflictId
);
$assert((string) $preparedStateConflict->effect_resolution === 'unknown'
&& (string) $preparedStateConflict->effect_resolution_reason
=== $preparedStateConflictMarker,
'A rejected applied commit changed its prepared recovery intent.');
$beforePreparedStateConflictRequests = count($client->requests);
$assert((new CloudAutomaticOperationRecoveryService())->tick(),
'A prepared no-fence envelope conflict left recovery.');
$preparedStateConflict = CloudJob::findOrFail(
$preparedStateConflictId
);
$assert((string) $preparedStateConflict->effect_resolution === 'unknown'
&& (string) $preparedStateConflict->effect_resolution_reason
=== $preparedStateConflictMarker
&& (bool) $preparedStateConflict->finalization_incident
&& (string) $preparedStateConflict->finalization_error_code
=== 'recovery_instance_state_conflict'
&& CloudJobService::blockingQueryForInstance(
$preparedStateConflictId
)->exists()
&& count($client->requests)
=== $beforePreparedStateConflictRequests,
'A prepared no-fence conflict contacted Agent, published, or unlocked.');
// The not-applied publication boundary applies the same immutable identity
// guard. A prepared proof cannot rely only on its encrypted escrow after
// the durable request fingerprint column has been damaged.
[$preparedIdentityConflictId] = $insertSettingsJob(
str_repeat('b3', 16),
['hostname' => 'prepared-identity-conflict.example']
);
$preparedIdentityConflictIdentity =
CloudOperationRecoveryIdentity::expected(
CloudJob::findOrFail($preparedIdentityConflictId),
false
);
$preparedIdentityConflictOperation = $preparedIdentityConflictIdentity + [
'operation_id' => str_repeat('b3', 16),
'status' => 'failed',
'effect_may_have_applied' => false,
'safe_to_resubmit' => true,
'error_code' => 'settings_not_committed',
'result' => ['agent_detail' => 'bounded'],
];
$preparedIdentityConflictSource =
'automatic_not_applied_failed_nofence:' . str_repeat('3', 64);
CloudOperationRecoveryService::prepareNotApplied(
$preparedIdentityConflictId,
$preparedIdentityConflictOperation,
$preparedIdentityConflictSource
);
$preparedIdentityConflictMarker = (string) CloudJob::findOrFail(
$preparedIdentityConflictId
)->effect_resolution_reason;
Capsule::table('cloud_jobs')->where(
'id',
$preparedIdentityConflictId
)->update([
'request_fingerprint' => 'bad-fingerprint',
'available_at' => 0,
'lease_token' => null,
'lease_until' => 0,
]);
try {
CloudOperationRecoveryService::commitNotApplied(
$preparedIdentityConflictId,
$preparedIdentityConflictOperation,
$preparedIdentityConflictSource,
0
);
throw new RuntimeException(
'A direct not-applied commit published damaged request identity.'
);
} catch (CloudRecoveryOperationIdentityConflict $error) {
$assert($error->httpStatus === 409,
'The not-applied identity guard returned the wrong status.');
}
$preparedIdentityConflict = CloudJob::findOrFail(
$preparedIdentityConflictId
);
$assert((string) $preparedIdentityConflict->effect_resolution === 'unknown'
&& (string) $preparedIdentityConflict->effect_resolution_reason
=== $preparedIdentityConflictMarker,
'A rejected not-applied commit changed its prepared recovery intent.');
$beforePreparedIdentityConflictRequests = count($client->requests);
$assert((new CloudAutomaticOperationRecoveryService())->tick(),
'A prepared request-identity conflict left recovery.');
$preparedIdentityConflict = CloudJob::findOrFail(
$preparedIdentityConflictId
);
$assert((string) $preparedIdentityConflict->effect_resolution === 'unknown'
&& (string) $preparedIdentityConflict->effect_resolution_reason
=== $preparedIdentityConflictMarker
&& (bool) $preparedIdentityConflict->finalization_incident
&& (string) $preparedIdentityConflict->finalization_error_code
=== 'recovery_operation_identity_conflict'
&& CloudJobService::blockingQueryForInstance(
$preparedIdentityConflictId
)->exists()
&& count($client->requests)
=== $beforePreparedIdentityConflictRequests,
'A prepared identity conflict contacted Agent, published, or unlocked.');
// A previously published effect with an older incident is not exempt from
// envelope validation. The immutable terminal audit remains resolved, but
// the concrete envelope conflict becomes visible and the resource remains
// blocked without consulting Agent state.
[$resolvedEnvelopeConflictId] = $insertSettingsJob(
str_repeat('b4', 16),
['hostname' => 'resolved-envelope-conflict.example']
);
Capsule::table('cloud_jobs')->where(
'id',
$resolvedEnvelopeConflictId
)->update([
'effect_resolution' => 'applied',
'effect_resolved_at' => time() - 5,
'effect_resolved_by' => 0,
'effect_resolution_reason' => 'prior_exact_evidence',
'receipt_deadline_at' => 0,
'finalization_incident' => 1,
'finalization_error_code' => 'recovery_waiting_connection',
'available_at' => 0,
'lease_token' => null,
'lease_until' => 0,
]);
$beforeResolvedEnvelopeConflictRequests = count($client->requests);
$assert((new CloudAutomaticOperationRecoveryService())->tick(),
'A resolved task with a damaged envelope left local recovery.');
$resolvedEnvelopeConflict = CloudJob::findOrFail(
$resolvedEnvelopeConflictId
);
$assert((string) $resolvedEnvelopeConflict->effect_resolution === 'applied'
&& (string) $resolvedEnvelopeConflict->effect_resolution_reason
=== 'prior_exact_evidence'
&& (bool) $resolvedEnvelopeConflict->finalization_incident
&& (string) $resolvedEnvelopeConflict->finalization_error_code
=== 'recovery_instance_state_conflict'
&& (int) $resolvedEnvelopeConflict->available_at >= time() + 290
&& CloudJobService::blockingQueryForInstance(
$resolvedEnvelopeConflictId
)->exists()
&& count($client->requests)
=== $beforeResolvedEnvelopeConflictRequests,
'A resolved envelope conflict contacted Agent, hid its cause, or falsely unlocked.');
// Repairing a failed row to the dispatched state is not enough when the
// immutable host/operation/time envelope is still damaged. Keep it in the
// local envelope-conflict queue instead of publishing a recovery that can
// never satisfy the normal Agent claim.
[$incompleteRepairId] = $insertSettingsJob(
str_repeat('bd', 16),
['hostname' => 'incomplete-dispatch-repair.example']
);
Capsule::table('cloud_jobs')->where('id', $incompleteRepairId)->update([
'dispatch_attempts' => 2,
'available_at' => 0,
'lease_token' => null,
'lease_until' => 0,
]);
CloudOperationCompletionService::deferDispatchLedgerConflict(
CloudJob::findOrFail($incompleteRepairId)
);
Capsule::table('cloud_jobs')->where('id', $incompleteRepairId)->update([
'dispatch_attempts' => 1,
'dispatched_at' => null,
'available_at' => 0,
'lease_token' => null,
'lease_until' => 0,
]);
$beforeIncompleteRepairRequests = count($client->requests);
$assert((new CloudAutomaticOperationRecoveryService())->tick(),
'An incomplete repaired dispatch envelope was not checked.');
$incompleteRepair = CloudJob::findOrFail($incompleteRepairId);
$assert((string) $incompleteRepair->effect_resolution === 'unknown'
&& (bool) $incompleteRepair->finalization_incident
&& (string) $incompleteRepair->finalization_error_code
=== 'recovery_dispatch_repair_instance_state_conflict'
&& (int) $incompleteRepair->lease_until >= time() + 290
&& (int) $incompleteRepair->available_at >= time() + 290
&& count($client->requests) === $beforeIncompleteRepairRequests,
'An incomplete repaired dispatch envelope left its sticky queue.');
// A pre-existing dispatched/unknown row with a damaged immutable envelope
// must still be claimed forever, but without attempting an Agent request.
[$malformedEnvelopeId] = $insertSettingsJob(
str_repeat('bc', 16),
['hostname' => 'malformed-envelope.example']
);
Capsule::table('cloud_jobs')->where('id', $malformedEnvelopeId)->update([
'dispatched_at' => null,
'finalization_incident' => 0,
'finalization_error_code' => null,
'available_at' => 0,
'lease_token' => null,
'lease_until' => 0,
]);
$beforeMalformedEnvelopeRequests = count($client->requests);
$assert((new CloudAutomaticOperationRecoveryService())->tick(),
'A damaged dispatched recovery envelope fell out of the queue.');
$malformedEnvelope = CloudJob::findOrFail($malformedEnvelopeId);
$assert((bool) $malformedEnvelope->finalization_incident
&& (string) $malformedEnvelope->finalization_error_code
=== 'recovery_instance_state_conflict'
&& (string) $malformedEnvelope->effect_resolution === 'unknown'
&& (int) $malformedEnvelope->lease_until >= time() + 290
&& (int) $malformedEnvelope->available_at >= time() + 290
&& count($client->requests) === $beforeMalformedEnvelopeRequests,
'A damaged dispatched recovery envelope contacted Agent or lost its concrete incident.');
[$invalidOperationIdJob] = $insertSettingsJob(
str_repeat('G', 32),
['hostname' => 'invalid-operation-id.example']
);
Capsule::table('cloud_jobs')->where('id', $invalidOperationIdJob)->update([
'available_at' => 0,
'lease_token' => null,
'lease_until' => 0,
]);
$beforeInvalidOperationIdRequests = count($client->requests);
$assert((new CloudAutomaticOperationRecoveryService())->tick(),
'A malformed Agent operation ID fell out of recovery.');
$invalidOperationIdentity = CloudJob::findOrFail(
$invalidOperationIdJob
);
$assert((bool) $invalidOperationIdentity->finalization_incident
&& (string) $invalidOperationIdentity->finalization_error_code
=== 'recovery_operation_identity_conflict'
&& (string) $invalidOperationIdentity->effect_resolution === 'unknown'
&& (int) $invalidOperationIdentity->available_at >= time() + 290
&& count($client->requests) === $beforeInvalidOperationIdRequests,
'A malformed operation ID lost its concrete identity-conflict status or contacted Agent.');
// A historical succeeded row without a bound completion event must still
// prove the immutable dispatched envelope before its sticky repair
// incident can be cleared. Otherwise repairing only the numeric counter
// would release the instance while host/time identity remained corrupt.
[$incompleteSucceededRepairId] = $insertSettingsJob(
str_repeat('be', 16),
['hostname' => 'incomplete-succeeded-repair.example']
);
$incompleteSucceededResult = json_encode([
'hostname' => 'incomplete-succeeded-repair.example',
'settings_changed' => true,
], JSON_THROW_ON_ERROR);
Capsule::table('cloud_jobs')->where(
'id',
$incompleteSucceededRepairId
)->update([
'dispatch_attempts' => 2,
'outcome' => 'succeeded',
'status' => 'succeeded',
'error_code' => null,
'effect_may_have_applied' => 0,
'safe_to_resubmit' => 0,
'effect_resolution' => 'applied',
'completion_event_id' => null,
'result_json' => $incompleteSucceededResult,
'available_at' => 0,
'lease_token' => null,
'lease_until' => 0,
]);
$incompleteSucceededBefore = $audit($incompleteSucceededRepairId);
$incompleteSucceededBefore['dispatch_attempts'] = 1;
$incompleteSucceededEnvelope = (string) CloudJob::findOrFail(
$incompleteSucceededRepairId
)->sealed_payload;
CloudOperationCompletionService::deferDispatchLedgerConflict(
CloudJob::findOrFail($incompleteSucceededRepairId)
);
Capsule::table('cloud_jobs')->where(
'id',
$incompleteSucceededRepairId
)->update([
'dispatch_attempts' => 1,
'dispatched_at' => null,
'available_at' => 0,
'lease_token' => null,
'lease_until' => 0,
]);
$beforeIncompleteSucceededRequests = count($client->requests);
$assert((new CloudAutomaticOperationRecoveryService())->tick(),
'An incomplete succeeded dispatch envelope was not checked.');
$incompleteSucceededRepair = CloudJob::findOrFail(
$incompleteSucceededRepairId
);
$assert((bool) $incompleteSucceededRepair->finalization_incident
&& (string) $incompleteSucceededRepair->finalization_error_code
=== 'recovery_dispatch_repair_instance_state_conflict'
&& (int) $incompleteSucceededRepair->lease_until >= time() + 290
&& (int) $incompleteSucceededRepair->available_at >= time() + 290
&& count($client->requests) === $beforeIncompleteSucceededRequests,
'A succeeded task cleared its repair incident without a complete dispatched envelope.');
$assert($audit($incompleteSucceededRepairId)
=== $incompleteSucceededBefore,
'Checking an incomplete succeeded envelope rewrote terminal audit data: '
. json_encode([
'expected' => $incompleteSucceededBefore,
'actual' => $audit($incompleteSucceededRepairId),
], JSON_THROW_ON_ERROR));
Capsule::table('cloud_jobs')->where(
'id',
$incompleteSucceededRepairId
)->update([
'dispatched_at' => time() - 980,
'sealed_payload' => 'automatic-recovery-test:missing',
'available_at' => 0,
'lease_token' => null,
'lease_until' => 0,
]);
$assert((new CloudAutomaticOperationRecoveryService())->tick(),
'A succeeded repair with a damaged identity escrow was not checked.');
$damagedSucceededRepair = CloudJob::findOrFail(
$incompleteSucceededRepairId
);
$assert((bool) $damagedSucceededRepair->finalization_incident
&& (string) $damagedSucceededRepair->finalization_error_code
=== 'recovery_dispatch_repair_payload_unavailable'
&& (int) $damagedSucceededRepair->lease_until >= time() + 290
&& (int) $damagedSucceededRepair->available_at >= time() + 290,
'A succeeded repair cleared its sticky incident without validating the immutable identity.');
Capsule::table('cloud_jobs')->where(
'id',
$incompleteSucceededRepairId
)->update([
'sealed_payload' => $incompleteSucceededEnvelope,
'available_at' => 0,
'lease_token' => null,
'lease_until' => 0,
]);
$assert((new CloudAutomaticOperationRecoveryService())->tick(),
'A completed succeeded dispatch envelope did not resume repair.');
$completedSucceededRepair = CloudJob::findOrFail(
$incompleteSucceededRepairId
);
$assert(!(bool) $completedSucceededRepair->finalization_incident
&& $completedSucceededRepair->finalization_error_code === null
&& $audit($incompleteSucceededRepairId)
=== $incompleteSucceededBefore
&& count($client->requests) === $beforeIncompleteSucceededRequests,
'A complete succeeded dispatch envelope was not unlocked without rewriting its terminal audit.');
// Repairing a corrupt counter to zero cannot erase already durable Agent
// evidence. The sticky repair queue must outrank the PHP-only finalizer;
// after the operator chooses one, the same receipt resumes convergence.
[$zeroCounterConflictId, $zeroCounterConflictUuid] = $insertSettingsJob(
str_repeat('bf', 16),
['hostname' => 'zero-counter-conflict.example']
);
$zeroCounterConflictJob = CloudJob::findOrFail($zeroCounterConflictId);
$zeroCounterIdentity = CloudOperationRecoveryIdentity::expected(
$zeroCounterConflictJob,
true
);
$zeroCounterEventId = 'zero-counter-proof-01';
$zeroCounterWire = [
'type' => 'completion',
'id' => $zeroCounterEventId,
'agent_id' => '7',
'operation_id' => (string) $zeroCounterConflictJob->operation_id,
'action' => $zeroCounterIdentity['action'],
'resource_id' => $zeroCounterIdentity['resource_id'],
'request_fingerprint' =>
$zeroCounterIdentity['request_fingerprint'],
'status' => 'succeeded',
'effect_may_have_applied' => false,
'safe_to_resubmit' => false,
'error_code' => '',
'result' => [
'uuid' => $zeroCounterConflictUuid,
'hostname' => 'zero-counter-conflict.example',
'settings_changed' => true,
],
];
Capsule::table('cloud_jobs')->where(
'id',
$zeroCounterConflictId
)->update([
'dispatch_attempts' => 2,
'available_at' => 0,
'lease_token' => null,
'lease_until' => 0,
]);
$beforeZeroCounterRequests = count($client->requests);
$acceptedZeroCounter = CloudOperationCompletionService::accept(
$zeroCounterWire
);
$zeroCounterEnvelope = (string) CloudJob::findOrFail(
$zeroCounterConflictId
)->sealed_payload;
$zeroCounterTerminalAt = (int) CloudJob::findOrFail(
$zeroCounterConflictId
)->terminal_at;
Capsule::table('cloud_jobs')->where(
'id',
$zeroCounterConflictId
)->update([
'dispatch_attempts' => 0,
'available_at' => 0,
'lease_token' => null,
'lease_until' => 0,
]);
$assert((new CloudAutomaticOperationRecoveryService())->tick(),
'A repaired-zero ledger with Agent evidence was not checked.');
$zeroCounterConflict = CloudJob::findOrFail($zeroCounterConflictId);
$zeroCounterEvent = Capsule::table(
'cloud_agent_completion_events'
)->where('event_id', $zeroCounterEventId)->first();
$assert($acceptedZeroCounter === [
'accepted' => true,
'disposition' => 'received',
]
&& (bool) $zeroCounterConflict->finalization_incident
&& (string) $zeroCounterConflict->finalization_error_code
=== 'recovery_dispatch_repair_instance_state_conflict'
&& (string) $zeroCounterConflict->effect_resolution === 'unknown'
&& (string) $zeroCounterConflict->sealed_payload
=== $zeroCounterEnvelope
&& (string) $zeroCounterEvent->disposition === 'received'
&& $zeroCounterEvent->applied_at === null
&& (int) $zeroCounterConflict->lease_until >= time() + 290
&& (int) $zeroCounterConflict->available_at >= time() + 290
&& count($client->requests) === $beforeZeroCounterRequests,
'Repairing a dispatched operation to zero ran the PHP-only finalizer or consumed its receipt.');
Capsule::table('cloud_jobs')->where(
'id',
$zeroCounterConflictId
)->update([
'dispatch_attempts' => 1,
'available_at' => 0,
'lease_token' => null,
'lease_until' => 0,
]);
$assert(CloudOperationCompletionService::accept($zeroCounterWire) === [
'accepted' => true,
'disposition' => 'received',
]
&& (string) Capsule::table('cloud_agent_completion_events')
->where('event_id', $zeroCounterEventId)->value('disposition')
=== 'received',
'A retransmitted receipt bypassed the sticky dispatch-repair coordinator.');
$assert((new CloudAutomaticOperationRecoveryService())->tick(),
'The repaired dispatched ledger did not re-enter Agent recovery.');
$assert((new CloudAutomaticOperationRecoveryService())->tick(),
'The repaired dispatched ledger did not consume its retained receipt.');
$zeroCounterResolved = CloudJob::findOrFail($zeroCounterConflictId);
$zeroCounterResolvedEvent = Capsule::table(
'cloud_agent_completion_events'
)->where('event_id', $zeroCounterEventId)->first();
$assert((string) $zeroCounterResolved->phase === 'terminal'
&& (string) $zeroCounterResolved->outcome === 'failed'
&& (string) $zeroCounterResolved->status === 'failed'
&& (int) $zeroCounterResolved->terminal_at
=== $zeroCounterTerminalAt
&& (string) $zeroCounterResolved->effect_resolution === 'applied'
&& !(bool) $zeroCounterResolved->finalization_incident
&& $zeroCounterResolved->finalization_error_code === null
&& (string) $zeroCounterResolvedEvent->disposition === 'applied'
&& (int) $zeroCounterResolvedEvent->applied_at > 0
&& count($client->requests) === $beforeZeroCounterRequests,
'Repairing the counter to one did not safely resume the retained Agent receipt.');
// The same contradiction on an active generation-zero task must move the
// row to a sticky terminal fence before the ordinary worker can dispatch.
[$activeZeroConflictId] = $insertSettingsJob(
str_repeat('c0', 16),
['hostname' => 'active-zero-conflict.example']
);
Capsule::table('cloud_jobs')->where('id', $activeZeroConflictId)->update([
'phase' => 'accepted',
'outcome' => null,
'status' => 'queued',
'attempts' => 0,
'dispatch_attempts' => 0,
'dispatch_started_at' => null,
'agent_accepted_at' => null,
'receipt_deadline_at' => null,
'terminal_at' => null,
'effect_may_have_applied' => null,
'safe_to_resubmit' => null,
'effect_resolution' => 'not_applicable',
'dispatched_at' => null,
'error_code' => null,
'finalization_incident' => 0,
'finalization_error_code' => null,
'result_json' => null,
'available_at' => 0,
'lease_token' => null,
'lease_until' => 0,
]);
$activeZeroJob = CloudJob::findOrFail($activeZeroConflictId);
$activeZeroIdentity = CloudOperationRecoveryIdentity::expected(
$activeZeroJob,
true
);
$activeZeroEventId = 'active-zero-proof-01';
$activeZeroAccepted = CloudOperationCompletionService::accept([
'type' => 'completion',
'id' => $activeZeroEventId,
'agent_id' => '7',
'operation_id' => (string) $activeZeroJob->operation_id,
'action' => $activeZeroIdentity['action'],
'resource_id' => $activeZeroIdentity['resource_id'],
'request_fingerprint' => $activeZeroIdentity['request_fingerprint'],
'status' => 'succeeded',
'effect_may_have_applied' => false,
'safe_to_resubmit' => false,
'error_code' => '',
'result' => [
'hostname' => 'active-zero-conflict.example',
'settings_changed' => true,
],
]);
$activeZeroConflict = CloudJob::findOrFail($activeZeroConflictId);
$activeZeroEvent = Capsule::table(
'cloud_agent_completion_events'
)->where('event_id', $activeZeroEventId)->first();
$assert($activeZeroAccepted === [
'accepted' => true,
'disposition' => 'received',
]
&& (string) $activeZeroConflict->phase === 'terminal'
&& (string) $activeZeroConflict->outcome === 'failed'
&& (string) $activeZeroConflict->status === 'failed'
&& (string) $activeZeroConflict->effect_resolution === 'unknown'
&& (bool) $activeZeroConflict->finalization_incident
&& (string) $activeZeroConflict->finalization_error_code
=== 'recovery_dispatch_repair_instance_state_conflict'
&& (string) $activeZeroEvent->disposition === 'received'
&& !CloudJob::where('id', $activeZeroConflictId)
->where('phase', 'accepted')
->where('dispatch_attempts', 0)->exists()
&& count($client->requests) === $beforeZeroCounterRequests,
'An active zero-counter task retained a replayable mutation after Agent completion evidence arrived.');
// Defence in depth for a pre-upgrade/crash snapshot where only the event
// became durable: the ordinary worker query itself excludes any Agent job
// with same-operation completion evidence.
Capsule::table('cloud_jobs')->where('id', $activeZeroConflictId)->update([
'phase' => 'accepted',
'outcome' => null,
'status' => 'queued',
'attempts' => 0,
'dispatch_attempts' => 0,
'dispatch_deadline_at' => time() + 300,
'dispatch_started_at' => null,
'agent_accepted_at' => null,
'receipt_deadline_at' => null,
'terminal_at' => null,
'effect_may_have_applied' => null,
'safe_to_resubmit' => null,
'effect_resolution' => 'not_applicable',
'dispatched_at' => null,
'error_code' => null,
'finalization_incident' => 0,
'finalization_error_code' => null,
'available_at' => 0,
'lease_token' => null,
'lease_until' => 0,
]);
$beforeCrashWindowRequests = count($client->requests);
(new CloudWorkerService())->tick();
$crashWindowJob = CloudJob::findOrFail($activeZeroConflictId);
$assert((string) $crashWindowJob->phase === 'accepted'
&& (int) $crashWindowJob->dispatch_attempts === 0
&& count($client->requests) === $beforeCrashWindowRequests,
'The ordinary worker replayed a mutation after durable same-operation completion evidence.');
// A repaired ledger with a corrupt bound receipt stays visible but obeys
// the same retry gate on both ordinary and receipt-priority passes. Without
// the lease clock this exact event would be reclaimed on every tick.
[$repairBackoffId, $repairBackoffUuid] = $insertSettingsJob(
str_repeat('ce', 16),
['hostname' => 'dispatch-repair-backoff.example']
);
$repairBackoffEventId = 'dispatch-repair-invalid-event';
Capsule::table('cloud_jobs')->where('id', $repairBackoffId)->update([
'dispatch_attempts' => 2,
'outcome' => 'succeeded',
'status' => 'succeeded',
'error_code' => null,
'effect_may_have_applied' => 0,
'safe_to_resubmit' => 0,
'effect_resolution' => 'applied',
'completion_event_id' => $repairBackoffEventId,
'available_at' => 0,
'lease_token' => null,
'lease_until' => 0,
]);
$repairBackoffJob = CloudJob::findOrFail($repairBackoffId);
$repairBackoffIdentity = CloudOperationRecoveryIdentity::expected(
$repairBackoffJob,
true
);
Capsule::table('cloud_agent_completion_events')->insert([
'event_id' => $repairBackoffEventId,
'host_id' => 7,
'operation_id' => (string) $repairBackoffJob->operation_id,
// Valid shape but deliberately not the hash of the immutable receipt.
'payload_sha256' => str_repeat('0', 64),
'outcome' => 'succeeded',
'effect_may_have_applied' => 0,
'safe_to_resubmit' => 0,
'error_code' => '',
'result_json' => json_encode([
'schema' => 'virt_cloud_completion_event',
'version' => 1,
'identity' => $repairBackoffIdentity,
'result' => [
'uuid' => $repairBackoffUuid,
'hostname' => 'dispatch-repair-backoff.example',
'settings_changed' => true,
],
], JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES
| JSON_UNESCAPED_UNICODE),
'disposition' => 'received',
'received_at' => time(),
'applied_at' => null,
]);
CloudOperationCompletionService::deferDispatchLedgerConflict(
$repairBackoffJob
);
Capsule::table('cloud_jobs')->where('id', $repairBackoffId)->update([
'dispatch_attempts' => 1,
'available_at' => 0,
'lease_token' => null,
'lease_until' => 0,
]);
$beforeRepairBackoffRequests = count($client->requests);
$repairBackoffService = new CloudAutomaticOperationRecoveryService();
$assert($repairBackoffService->tick(),
'A repaired ledger with an invalid bound receipt was not checked.');
$repairBackoffJob = CloudJob::findOrFail($repairBackoffId);
$repairBackoffUpdatedAt = (int) $repairBackoffJob->updated_at;
$assert((bool) $repairBackoffJob->finalization_incident
&& (string) $repairBackoffJob->finalization_error_code
=== 'recovery_dispatch_repair_receipt_invalid'
&& (int) $repairBackoffJob->lease_until >= time() + 290
&& (int) $repairBackoffJob->available_at >= time() + 290
&& count($client->requests) === $beforeRepairBackoffRequests,
'An invalid repaired-ledger receipt did not enter bounded backoff.');
Capsule::table('cloud_jobs')->where('id', '<>', $repairBackoffId)
->update([
'lease_until' => time() + 3600,
'available_at' => time() + 3600,
]);
$assert(!$repairBackoffService->tick()
&& (int) CloudJob::findOrFail($repairBackoffId)->updated_at
=== $repairBackoffUpdatedAt
&& count($client->requests) === $beforeRepairBackoffRequests,
'Receipt-priority recovery ignored the repaired-ledger retry gate.');
// Contradictory terminal/unknown safety tuples are PHP-only incidents.
// They must remain scheduled forever without inspection, fence writes, or
// any replay of the original mutation. Cover both legal dispatch counters
// and each independent part of the canonical terminal tuple.
$stateCases = [
[str_repeat('f1', 16), [
'effect_may_have_applied' => 1,
'safe_to_resubmit' => 1,
]],
[str_repeat('f2', 16), [
'effect_may_have_applied' => 0,
'safe_to_resubmit' => 0,
]],
[str_repeat('f3', 16), [
'outcome' => 'succeeded',
'status' => 'succeeded',
]],
[str_repeat('f4', 16), [
'dispatch_attempts' => 0,
'dispatched_at' => null,
'dispatch_started_at' => null,
'agent_accepted_at' => null,
'receipt_deadline_at' => null,
'effect_may_have_applied' => 0,
'safe_to_resubmit' => 0,
]],
// The safety booleans look canonical for an unknown dispatched
// operation, but generation zero proves no Agent mutation crossed the
// boundary. Without an undispatched-finalizer incident this is a
// contradictory historical row, not an Agent recovery candidate.
[str_repeat('f8', 16), [
'dispatch_attempts' => 0,
'dispatched_at' => null,
'dispatch_started_at' => null,
'agent_accepted_at' => null,
'receipt_deadline_at' => null,
'effect_may_have_applied' => 1,
'safe_to_resubmit' => 0,
]],
];
foreach ($stateCases as [$operationId, $overrides]) {
[$stateId] = $insertSettingsJob(
$operationId,
['hostname' => 'damaged-state.example']
);
Capsule::table('cloud_jobs')->where('id', $stateId)->update(
$overrides + [
'finalization_incident' => 0,
'finalization_error_code' => null,
'available_at' => 0,
'lease_token' => null,
'lease_until' => 0,
]
);
$beforeStateRequests = count($client->requests);
$stateService = new CloudAutomaticOperationRecoveryService();
$assert($stateService->tick(),
'A damaged leaf terminal tuple left the recovery queue.');
$stateJob = CloudJob::findOrFail($stateId);
$assert((bool) $stateJob->finalization_incident
&& (string) $stateJob->finalization_error_code
=== 'recovery_instance_state_conflict'
&& (int) $stateJob->available_at >= time() + 290
&& (int) $stateJob->lease_until >= time() + 290
&& count($client->requests) === $beforeStateRequests,
'A damaged leaf terminal tuple contacted Agent or lost backoff.');
Capsule::table('cloud_jobs')->where('id', $stateId)->update([
'available_at' => time() + 3600,
'lease_until' => time() + 3600,
]);
}
// A completed Agent effect can still carry a sticky PHP projection
// incident. Both resolved outcomes remain in the local retry queue, retain
// their concrete code, and must never trigger another Agent request.
$resolvedIncidentCases = [
[
str_repeat('f9', 16),
'applied',
'succeeded',
'recovery_payload_unavailable',
],
[
str_repeat('fa', 16),
'not_applied',
'failed',
'recovery_operation_identity_conflict',
],
];
foreach ($resolvedIncidentCases as [
$operationId,
$resolution,
$outcome,
$incidentCode,
]) {
[$resolvedIncidentId] = $insertSettingsJob(
$operationId,
['hostname' => 'resolved-local-incident.example']
);
Capsule::table('cloud_jobs')
->where('id', $resolvedIncidentId)->update([
'outcome' => $outcome,
'status' => $outcome,
'dispatch_attempts' => 1,
'effect_may_have_applied' => 0,
'safe_to_resubmit' => 0,
'effect_resolution' => $resolution,
'effect_resolved_at' => time() - 1,
'finalization_incident' => 1,
'finalization_error_code' => $incidentCode,
'available_at' => 0,
'lease_token' => null,
'lease_until' => 0,
]);
$beforeResolvedIncidentRequests = count($client->requests);
$assert((new CloudAutomaticOperationRecoveryService())->tick(),
'A resolved sticky incident left the local recovery queue.');
$resolvedIncident = CloudJob::findOrFail($resolvedIncidentId);
$assert((bool) $resolvedIncident->finalization_incident
&& (string) $resolvedIncident->finalization_error_code
=== $incidentCode
&& (string) $resolvedIncident->effect_resolution === $resolution
&& (int) $resolvedIncident->available_at >= time() + 290
&& (int) $resolvedIncident->lease_until >= time() + 290
&& count($client->requests) === $beforeResolvedIncidentRequests,
'A resolved sticky incident contacted Agent, lost its code, '
. 'or failed to enter local backoff.');
}
// The same rule applies to a replica child below an already-resolved
// aggregate. Child ancestry must not turn a local finalization incident
// back into an Agent recovery or remove its concrete cause.
[$resolvedIncidentParentId] = $insertSettingsJob(
str_repeat('fb', 16),
['hostname' => 'resolved-incident-parent.example']
);
Capsule::table('cloud_jobs')
->where('id', $resolvedIncidentParentId)->update([
'instance_id' => null,
'host_id' => null,
'kind' => 'template_import',
'execution_mode' => 'aggregate',
'outcome' => 'succeeded',
'status' => 'succeeded',
'attempts' => 0,
'dispatch_attempts' => 0,
'dispatch_started_at' => null,
'agent_accepted_at' => null,
'dispatched_at' => null,
'completion_event_id' => null,
'effect_may_have_applied' => 0,
'safe_to_resubmit' => 0,
'effect_resolution' => 'applied',
'effect_resolved_at' => time() - 1,
'finalization_incident' => 0,
'finalization_error_code' => null,
'available_at' => time() + 3600,
'lease_token' => null,
'lease_until' => time() + 3600,
]);
[$resolvedIncidentChildId] = $insertSettingsJob(
str_repeat('fc', 16),
['hostname' => 'resolved-incident-child.example']
);
Capsule::table('cloud_jobs')
->where('id', $resolvedIncidentChildId)->update([
'parent_job_id' => $resolvedIncidentParentId,
'kind' => 'template_import_replica',
'outcome' => 'failed',
'status' => 'failed',
'dispatch_attempts' => 1,
'effect_may_have_applied' => 0,
'safe_to_resubmit' => 0,
'effect_resolution' => 'not_applied',
'effect_resolved_at' => time() - 1,
'finalization_incident' => 1,
'finalization_error_code' => 'recovery_payload_unavailable',
'available_at' => 0,
'lease_token' => null,
'lease_until' => 0,
]);
$beforeResolvedIncidentChildRequests = count($client->requests);
$assert((new CloudAutomaticOperationRecoveryService())->tick(),
'A resolved sticky aggregate child left the local recovery queue.');
$resolvedIncidentChild = CloudJob::findOrFail(
$resolvedIncidentChildId
);
$assert((bool) $resolvedIncidentChild->finalization_incident
&& (string) $resolvedIncidentChild->finalization_error_code
=== 'recovery_payload_unavailable'
&& (string) $resolvedIncidentChild->effect_resolution
=== 'not_applied'
&& (int) $resolvedIncidentChild->available_at >= time() + 290
&& (int) $resolvedIncidentChild->lease_until >= time() + 290
&& count($client->requests)
=== $beforeResolvedIncidentChildRequests,
'A resolved sticky aggregate child contacted Agent, lost its code, '
. 'or failed to enter local backoff.');
[$aggregateStateId] = $insertSettingsJob(
str_repeat('f5', 16),
['hostname' => 'aggregate-state.example']
);
Capsule::table('cloud_jobs')->where('id', $aggregateStateId)->update([
'instance_id' => null,
'host_id' => null,
'kind' => 'template_import',
'execution_mode' => 'aggregate',
'dispatch_attempts' => 0,
'dispatched_at' => null,
'dispatch_started_at' => null,
'agent_accepted_at' => null,
'receipt_deadline_at' => null,
'safe_to_resubmit' => 1,
'finalization_incident' => 0,
'finalization_error_code' => null,
'available_at' => 0,
'lease_token' => null,
'lease_until' => 0,
]);
$beforeAggregateStateRequests = count($client->requests);
$aggregateStateService = new CloudAutomaticOperationRecoveryService();
$assert($aggregateStateService->tick(),
'A damaged aggregate-parent tuple left the recovery queue.');
$aggregateStateJob = CloudJob::findOrFail($aggregateStateId);
$assert((bool) $aggregateStateJob->finalization_incident
&& (string) $aggregateStateJob->finalization_error_code
=== 'recovery_aggregate_state_conflict'
&& (int) $aggregateStateJob->available_at >= time() + 290
&& (int) $aggregateStateJob->lease_until >= time() + 290
&& count($client->requests) === $beforeAggregateStateRequests,
'A damaged aggregate-parent tuple contacted Agent or lost backoff.');
Capsule::table('cloud_jobs')->where('id', $aggregateStateId)->update([
'available_at' => time() + 3600,
'lease_until' => time() + 3600,
]);
// A resolved aggregate parent must still surface and retain an anomalous
// Agent child; only the child enters the PHP-only incident loop.
[$resolvedParentId] = $insertSettingsJob(
str_repeat('f6', 16),
['hostname' => 'resolved-parent.example']
);
Capsule::table('cloud_jobs')->where('id', $resolvedParentId)->update([
'instance_id' => null,
'host_id' => null,
'kind' => 'template_import',
'execution_mode' => 'aggregate',
'dispatch_attempts' => 0,
'dispatched_at' => null,
'outcome' => 'succeeded',
'status' => 'succeeded',
'effect_may_have_applied' => 0,
'safe_to_resubmit' => 0,
'effect_resolution' => 'applied',
'available_at' => time() + 3600,
'lease_until' => time() + 3600,
]);
[$stateChildId] = $insertSettingsJob(
str_repeat('f7', 16),
['hostname' => 'damaged-child.example']
);
Capsule::table('cloud_jobs')->where('id', $stateChildId)->update([
'parent_job_id' => $resolvedParentId,
'kind' => 'template_import_replica',
'safe_to_resubmit' => 1,
'finalization_incident' => 0,
'finalization_error_code' => null,
'available_at' => 0,
'lease_token' => null,
'lease_until' => 0,
]);
$beforeStateChildRequests = count($client->requests);
$stateChildService = new CloudAutomaticOperationRecoveryService();
$assert($stateChildService->tick(),
'An anomalous aggregate child left the recovery queue.');
$stateChild = CloudJob::findOrFail($stateChildId);
$assert((bool) $stateChild->finalization_incident
&& (string) $stateChild->finalization_error_code
=== 'recovery_instance_state_conflict'
&& count($client->requests) === $beforeStateChildRequests,
'An anomalous aggregate child contacted Agent or lost its incident.');
// A transition with independently persisted evidence is claimed through
// the PHP-only public-network finalizer path. This intentionally uses a
// minimal transition fixture so the deep finalizer rejects it; the
// coordinator must back it off locally without contacting the Agent.
[$networkFinalizerId] = $insertSettingsJob(
str_repeat('fd', 16),
['hostname' => 'network-finalizer.example']
);
Capsule::table('cloud_public_network_transitions')->insert([
'id' => 1,
'state' => 'finalization_pending',
'effect_resolution' => 'applied',
'last_error_code' => 'local_finalization_failed',
]);
Capsule::table('cloud_jobs')->where('id', '<>', $networkFinalizerId)
->update([
'available_at' => time() + 3600,
'lease_until' => time() + 3600,
]);
Capsule::table('cloud_jobs')->where('id', $networkFinalizerId)->update([
'kind' => 'network_change',
'public_network_transition_id' => 1,
'finalization_incident' => 1,
'finalization_error_code' => 'recovery_internal_error',
'available_at' => 0,
'lease_token' => null,
'lease_until' => 0,
]);
$beforeNetworkFinalizerRequests = count($client->requests);
$networkFinalizerService = new CloudAutomaticOperationRecoveryService();
$assert($networkFinalizerService->tick(),
'A pending public-network finalizer left the recovery queue.');
$networkFinalizer = CloudJob::findOrFail($networkFinalizerId);
$assert($networkFinalizer->lease_token === null
&& (int) $networkFinalizer->available_at >= time() + 290
&& (int) $networkFinalizer->lease_until >= time() + 290
&& (bool) $networkFinalizer->finalization_incident
&& count($client->requests) === $beforeNetworkFinalizerRequests,
'Public-network local finalizer contacted Agent or lost backoff.');
foreach ($client->requests as $request) {
$allowed = preg_match(
'#^/v1/operations/[a-f0-9]{32}/(?:inspect|fence-resolution|resolve-fence)$#D',
$request['target']
) === 1;
$assert($allowed
&& ($request['method'] === 'GET'
|| ($request['method'] === 'POST'
&& str_ends_with(
$request['target'],
'/resolve-fence'
))),
'Automatic recovery called an Agent mutation endpoint.');
}
echo "Cloud automatic operation recovery tests passed.\n";
}