|
|
|
|
<?php
|
|
|
|
|
declare(strict_types=1);
|
|
|
|
|
|
|
|
|
|
namespace App\Services\Cloud {
|
|
|
|
|
/** Isolated encrypted envelope for automatic-recovery tests. */
|
|
|
|
|
final class CloudSecrets
|
|
|
|
|
{
|
|
|
|
|
private static array $values = [];
|
|
|
|
|
|
|
|
|
|
public static function encrypt(array $value, string $purpose): string
|
|
|
|
|
{
|
|
|
|
|
$key = bin2hex(random_bytes(16));
|
|
|
|
|
self::$values[$key] = $value;
|
|
|
|
|
return 'automatic-recovery-test:' . $key;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
public static function decrypt(string $value, string $purpose): array
|
|
|
|
|
{
|
|
|
|
|
$key = substr($value, strlen('automatic-recovery-test:'));
|
|
|
|
|
if (!array_key_exists($key, self::$values)) {
|
|
|
|
|
throw new \RuntimeException('Missing recovery test payload.');
|
|
|
|
|
}
|
|
|
|
|
return self::$values[$key];
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
final class AutomaticRecoveryTestAgentClient
|
|
|
|
|
{
|
|
|
|
|
public array $requests = [];
|
|
|
|
|
public array $responses = [];
|
|
|
|
|
|
|
|
|
|
public function systemRecoveryRequest(
|
|
|
|
|
string $method,
|
|
|
|
|
string $target,
|
|
|
|
|
string $operationId,
|
|
|
|
|
?array $payload = null
|
|
|
|
|
): array {
|
|
|
|
|
$request = compact(
|
|
|
|
|
'method', 'target', 'operationId', 'payload'
|
|
|
|
|
);
|
|
|
|
|
$this->requests[] = $request;
|
|
|
|
|
if ($this->responses === []) {
|
|
|
|
|
throw new \RuntimeException('Unexpected Agent request.');
|
|
|
|
|
}
|
|
|
|
|
$response = array_shift($this->responses);
|
|
|
|
|
if (is_callable($response)) {
|
|
|
|
|
$response = $response($request);
|
|
|
|
|
}
|
|
|
|
|
if ($response instanceof \Throwable) {
|
|
|
|
|
throw $response;
|
|
|
|
|
}
|
|
|
|
|
if (!is_array($response)) {
|
|
|
|
|
throw new \RuntimeException('Invalid test Agent response.');
|
|
|
|
|
}
|
|
|
|
|
return $response;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
final class CloudAgentSettingsService
|
|
|
|
|
{
|
|
|
|
|
public static ?AutomaticRecoveryTestAgentClient $client = null;
|
|
|
|
|
|
|
|
|
|
public static function clientForConfiguration(
|
|
|
|
|
int $hostId,
|
|
|
|
|
float $timeoutSeconds = 120.0
|
|
|
|
|
): AutomaticRecoveryTestAgentClient {
|
|
|
|
|
if ($hostId !== 7 || $timeoutSeconds !== 30.0
|
|
|
|
|
|| self::$client === null) {
|
|
|
|
|
throw new \RuntimeException(
|
|
|
|
|
'Unexpected automatic recovery client request.'
|
|
|
|
|
);
|
|
|
|
|
}
|
|
|
|
|
return self::$client;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
namespace {
|
|
|
|
|
require dirname(__DIR__) . '/vendor/autoload.php';
|
|
|
|
|
|
|
|
|
|
use App\Models\CloudJob;
|
|
|
|
|
use App\Services\Cloud\AutomaticRecoveryTestAgentClient;
|
|
|
|
|
use App\Services\Cloud\CloudAgentError;
|
|
|
|
|
use App\Services\Cloud\CloudAgentSettingsService;
|
|
|
|
|
use App\Services\Cloud\CloudAutomaticOperationRecoveryService;
|
|
|
|
|
use App\Services\Cloud\CloudError;
|
|
|
|
|
use App\Services\Cloud\CloudJobService;
|
|
|
|
|
use App\Services\Cloud\CloudOperationCompletionService;
|
|
|
|
|
use App\Services\Cloud\CloudOperationRecoveryIdentity;
|
|
|
|
|
use App\Services\Cloud\CloudOperationRecoveryService;
|
|
|
|
|
use App\Services\Cloud\CloudRecoveryInstanceStateConflict;
|
|
|
|
|
use App\Services\Cloud\CloudRecoveryOperationIdentityConflict;
|
|
|
|
|
use App\Services\Cloud\CloudSecrets;
|
|
|
|
|
use App\Services\Cloud\CloudWorkerService;
|
|
|
|
|
use Illuminate\Database\Capsule\Manager as Capsule;
|
|
|
|
|
use Illuminate\Database\Schema\Blueprint;
|
|
|
|
|
|
|
|
|
|
$assert = static function (bool $condition, string $message): void {
|
|
|
|
|
if (!$condition) {
|
|
|
|
|
throw new RuntimeException($message);
|
|
|
|
|
}
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
$capsule = new Capsule();
|
|
|
|
|
$capsule->addConnection([
|
|
|
|
|
'driver' => 'sqlite', 'database' => ':memory:',
|
|
|
|
|
], 'default');
|
|
|
|
|
$capsule->setAsGlobal();
|
|
|
|
|
$capsule->bootEloquent();
|
|
|
|
|
$schema = $capsule->schema('default');
|
|
|
|
|
$schema->create('user', static function (Blueprint $table): void {
|
|
|
|
|
$table->increments('id');
|
|
|
|
|
$table->integer('enable')->default(1);
|
|
|
|
|
$table->integer('is_admin')->default(0);
|
|
|
|
|
});
|
|
|
|
|
$schema->create('cloud_jobs', static function (Blueprint $table): void {
|
|
|
|
|
$table->increments('id');
|
|
|
|
|
$table->string('operation_id')->unique();
|
|
|
|
|
$table->string('request_idempotency_key')->unique();
|
|
|
|
|
$table->string('request_fingerprint');
|
|
|
|
|
$table->integer('parent_job_id')->nullable();
|
|
|
|
|
$table->integer('order_id')->nullable();
|
|
|
|
|
$table->integer('invoice_id')->nullable();
|
|
|
|
|
$table->integer('instance_id')->nullable();
|
|
|
|
|
$table->integer('snapshot_id')->nullable();
|
|
|
|
|
$table->integer('public_network_transition_id')->nullable();
|
|
|
|
|
$table->integer('vpc_repair_root_job_id')->nullable();
|
|
|
|
|
$table->integer('user_id');
|
|
|
|
|
$table->integer('host_id')->nullable();
|
|
|
|
|
$table->string('kind');
|
|
|
|
|
$table->string('execution_mode');
|
|
|
|
|
$table->string('phase');
|
|
|
|
|
$table->string('outcome')->nullable();
|
|
|
|
|
$table->string('status');
|
|
|
|
|
$table->text('sealed_payload');
|
|
|
|
|
$table->integer('attempts')->default(0);
|
|
|
|
|
$table->integer('dispatch_attempts')->default(0);
|
|
|
|
|
$table->integer('dispatch_deadline_at');
|
|
|
|
|
$table->integer('dispatch_started_at')->nullable();
|
|
|
|
|
$table->integer('agent_accepted_at')->nullable();
|
|
|
|
|
$table->integer('receipt_deadline_at')->nullable();
|
|
|
|
|
$table->integer('terminal_at')->nullable();
|
|
|
|
|
$table->boolean('effect_may_have_applied')->nullable();
|
|
|
|
|
$table->boolean('safe_to_resubmit')->nullable();
|
|
|
|
|
$table->string('effect_resolution');
|
|
|
|
|
$table->integer('effect_resolved_at')->nullable();
|
|
|
|
|
$table->integer('effect_resolved_by')->nullable();
|
|
|
|
|
$table->string('effect_resolution_reason')->default('');
|
|
|
|
|
$table->string('completion_event_id')->nullable();
|
|
|
|
|
$table->integer('read_at')->nullable();
|
|
|
|
|
$table->integer('dispatched_at')->nullable();
|
|
|
|
|
$table->string('lease_token')->nullable();
|
|
|
|
|
$table->integer('lease_until')->default(0);
|
|
|
|
|
$table->integer('available_at')->default(0);
|
|
|
|
|
$table->string('error_code')->nullable();
|
|
|
|
|
$table->boolean('finalization_incident')->default(0);
|
|
|
|
|
$table->string('finalization_error_code')->nullable();
|
|
|
|
|
$table->text('result_json')->nullable();
|
|
|
|
|
$table->integer('created_at');
|
|
|
|
|
$table->integer('updated_at');
|
|
|
|
|
});
|
|
|
|
|
$schema->create('cloud_backup_restore_attempts',
|
|
|
|
|
static function (Blueprint $table): void {
|
|
|
|
|
$table->increments('id');
|
|
|
|
|
$table->integer('job_id');
|
|
|
|
|
$table->string('state');
|
|
|
|
|
$table->string('effect_resolution');
|
|
|
|
|
$table->integer('active_attempt')->nullable();
|
|
|
|
|
$table->string('last_error_code')->default('');
|
|
|
|
|
});
|
|
|
|
|
$schema->create('cloud_public_network_transitions',
|
|
|
|
|
static function (Blueprint $table): void {
|
|
|
|
|
$table->increments('id');
|
|
|
|
|
$table->string('state');
|
|
|
|
|
$table->string('effect_resolution');
|
|
|
|
|
$table->string('last_error_code')->default('');
|
|
|
|
|
});
|
|
|
|
|
$schema->create('cloud_instances', static function (Blueprint $table): void {
|
|
|
|
|
$table->increments('id');
|
|
|
|
|
$table->string('uuid');
|
|
|
|
|
$table->integer('host_id');
|
|
|
|
|
$table->integer('user_id');
|
|
|
|
|
$table->integer('order_id')->nullable();
|
|
|
|
|
$table->integer('plan_id')->nullable();
|
|
|
|
|
$table->integer('offer_id')->nullable();
|
|
|
|
|
$table->integer('resource_pool_id')->nullable();
|
|
|
|
|
$table->integer('reservation_id')->nullable();
|
|
|
|
|
$table->integer('current_period_id')->nullable();
|
|
|
|
|
$table->string('name');
|
|
|
|
|
$table->string('status');
|
|
|
|
|
$table->string('service_status');
|
|
|
|
|
$table->string('network_status');
|
|
|
|
|
$table->integer('expires_at')->nullable();
|
|
|
|
|
$table->decimal('renewal_price', 12, 2)->nullable();
|
|
|
|
|
$table->integer('renewal_days')->default(0);
|
|
|
|
|
$table->integer('renewal_traffic_bytes')->default(0);
|
|
|
|
|
$table->integer('traffic_quota_bytes')->default(0);
|
|
|
|
|
$table->integer('traffic_download_bytes')->default(0);
|
|
|
|
|
$table->integer('traffic_upload_bytes')->default(0);
|
|
|
|
|
$table->integer('lifetime_download_bytes')->default(0);
|
|
|
|
|
$table->integer('lifetime_upload_bytes')->default(0);
|
|
|
|
|
$table->integer('traffic_synced_at')->default(0);
|
|
|
|
|
$table->integer('suspended_at')->nullable();
|
|
|
|
|
$table->integer('network_suspended_at')->nullable();
|
|
|
|
|
$table->text('spec_json');
|
|
|
|
|
$table->text('sealed_credential')->nullable();
|
|
|
|
|
$table->integer('created_at');
|
|
|
|
|
$table->integer('updated_at');
|
|
|
|
|
});
|
|
|
|
|
$schema->create('cloud_agent_completion_events',
|
|
|
|
|
static function (Blueprint $table): void {
|
|
|
|
|
$table->string('event_id')->primary();
|
|
|
|
|
$table->integer('host_id');
|
|
|
|
|
$table->string('operation_id');
|
|
|
|
|
$table->string('payload_sha256');
|
|
|
|
|
$table->string('outcome');
|
|
|
|
|
$table->boolean('effect_may_have_applied');
|
|
|
|
|
$table->boolean('safe_to_resubmit');
|
|
|
|
|
$table->string('error_code')->default('');
|
|
|
|
|
$table->text('result_json')->nullable();
|
|
|
|
|
$table->string('disposition')->default('received');
|
|
|
|
|
$table->integer('received_at');
|
|
|
|
|
$table->integer('applied_at')->nullable();
|
|
|
|
|
$table->unique(['host_id', 'operation_id']);
|
|
|
|
|
});
|
|
|
|
|
$schema->create('cloud_instance_snapshots',
|
|
|
|
|
static function (Blueprint $table): void {
|
|
|
|
|
$table->increments('id');
|
|
|
|
|
$table->integer('instance_id');
|
|
|
|
|
$table->string('state');
|
|
|
|
|
$table->string('last_operation_id')->default('');
|
|
|
|
|
$table->integer('deleted_at')->nullable();
|
|
|
|
|
$table->integer('updated_at');
|
|
|
|
|
});
|
|
|
|
|
$schema->create('cloud_templates',
|
|
|
|
|
static function (Blueprint $table): void {
|
|
|
|
|
$table->string('id')->primary();
|
|
|
|
|
$table->string('catalog_key')->default('test-catalog');
|
|
|
|
|
$table->string('version_label')->default('test-version');
|
|
|
|
|
$table->boolean('is_default')->default(true);
|
|
|
|
|
$table->integer('active_job_id')->nullable();
|
|
|
|
|
$table->string('state')->default('active');
|
|
|
|
|
$table->integer('revision')->default(1);
|
|
|
|
|
$table->string('generation_token')->default('');
|
|
|
|
|
$table->integer('image_bytes')->default(0);
|
|
|
|
|
$table->string('sha256')->default('');
|
|
|
|
|
$table->integer('updated_at')->default(0);
|
|
|
|
|
});
|
|
|
|
|
$schema->create('cloud_user_isos',
|
|
|
|
|
static function (Blueprint $table): void {
|
|
|
|
|
$table->string('id')->primary();
|
|
|
|
|
$table->integer('active_job_id')->nullable();
|
|
|
|
|
$table->string('state')->default('active');
|
|
|
|
|
});
|
|
|
|
|
$schema->create('cloud_template_replicas',
|
|
|
|
|
static function (Blueprint $table): void {
|
|
|
|
|
$table->integer('host_id');
|
|
|
|
|
$table->string('template_id');
|
|
|
|
|
$table->integer('revision')->default(0);
|
|
|
|
|
$table->string('generation_token')->default('');
|
|
|
|
|
$table->string('state')->default('pending');
|
|
|
|
|
$table->integer('image_bytes')->default(0);
|
|
|
|
|
$table->string('sha256')->default('');
|
|
|
|
|
$table->string('operation_id')->default('');
|
|
|
|
|
$table->integer('job_id')->nullable();
|
|
|
|
|
$table->string('error_code')->default('');
|
|
|
|
|
$table->integer('created_at')->default(0);
|
|
|
|
|
$table->integer('updated_at')->default(0);
|
|
|
|
|
$table->primary(['host_id', 'template_id']);
|
|
|
|
|
});
|
|
|
|
|
$schema->create('cloud_template_artifacts',
|
|
|
|
|
static function (Blueprint $table): void {
|
|
|
|
|
$table->string('template_id')->primary();
|
|
|
|
|
$table->integer('storage_revision');
|
|
|
|
|
$table->text('sealed_storage_runtime');
|
|
|
|
|
$table->string('object_key');
|
|
|
|
|
$table->string('object_version_id')->default('');
|
|
|
|
|
$table->string('object_etag');
|
|
|
|
|
$table->integer('size_bytes');
|
|
|
|
|
$table->string('sha256');
|
|
|
|
|
$table->string('state');
|
|
|
|
|
$table->string('error_code')->default('');
|
|
|
|
|
$table->string('delete_operation_id')->default('');
|
|
|
|
|
$table->integer('delete_generation')->default(0);
|
|
|
|
|
$table->integer('delete_cycle_attempts')->default(0);
|
|
|
|
|
$table->integer('delete_presence_confirmations')->default(0);
|
|
|
|
|
$table->integer('delete_last_checked_at')->default(0);
|
|
|
|
|
$table->integer('delete_next_check_at')->default(0);
|
|
|
|
|
$table->integer('delete_attention_at')->nullable();
|
|
|
|
|
$table->integer('verified_at')->default(0);
|
|
|
|
|
$table->integer('created_at');
|
|
|
|
|
$table->integer('updated_at');
|
|
|
|
|
});
|
|
|
|
|
$schema->create('cloud_user_iso_replicas',
|
|
|
|
|
static function (Blueprint $table): void {
|
|
|
|
|
$table->integer('host_id');
|
|
|
|
|
$table->string('user_iso_id');
|
|
|
|
|
$table->integer('user_id');
|
|
|
|
|
$table->string('state')->default('pending');
|
|
|
|
|
$table->integer('size_bytes')->default(0);
|
|
|
|
|
$table->string('sha256')->default('');
|
|
|
|
|
$table->string('operation_id')->default('');
|
|
|
|
|
$table->integer('job_id')->nullable();
|
|
|
|
|
$table->string('error_code')->default('');
|
|
|
|
|
$table->integer('created_at')->default(0);
|
|
|
|
|
$table->integer('updated_at')->default(0);
|
|
|
|
|
$table->primary(['host_id', 'user_iso_id']);
|
|
|
|
|
});
|
|
|
|
|
Capsule::table('user')->insert([
|
|
|
|
|
'id' => 1, 'enable' => 1, 'is_admin' => 0,
|
|
|
|
|
]);
|
|
|
|
|
|
|
|
|
|
$now = time();
|
|
|
|
|
$nextId = 100;
|
|
|
|
|
$insertSettingsJob = static function (
|
|
|
|
|
string $operationId,
|
|
|
|
|
array $body,
|
|
|
|
|
?array $credential = null
|
|
|
|
|
) use (&$nextId, $now): array {
|
|
|
|
|
$id = $nextId++;
|
|
|
|
|
$uuid = sprintf(
|
|
|
|
|
'00000000-0000-4000-8000-%012d',
|
|
|
|
|
$id
|
|
|
|
|
);
|
|
|
|
|
Capsule::table('cloud_instances')->insert([
|
|
|
|
|
'id' => $id,
|
|
|
|
|
'uuid' => $uuid,
|
|
|
|
|
'host_id' => 7,
|
|
|
|
|
'user_id' => 1,
|
|
|
|
|
'order_id' => null,
|
|
|
|
|
'plan_id' => null,
|
|
|
|
|
'offer_id' => null,
|
|
|
|
|
'resource_pool_id' => null,
|
|
|
|
|
'reservation_id' => null,
|
|
|
|
|
'current_period_id' => null,
|
|
|
|
|
'name' => 'settings-' . $id,
|
|
|
|
|
'status' => 'active',
|
|
|
|
|
'service_status' => 'exempt',
|
|
|
|
|
'network_status' => 'active',
|
|
|
|
|
'expires_at' => null,
|
|
|
|
|
'renewal_price' => null,
|
|
|
|
|
'renewal_days' => 0,
|
|
|
|
|
'renewal_traffic_bytes' => 0,
|
|
|
|
|
'traffic_quota_bytes' => 0,
|
|
|
|
|
'traffic_download_bytes' => 0,
|
|
|
|
|
'traffic_upload_bytes' => 0,
|
|
|
|
|
'lifetime_download_bytes' => 0,
|
|
|
|
|
'lifetime_upload_bytes' => 0,
|
|
|
|
|
'traffic_synced_at' => 0,
|
|
|
|
|
'suspended_at' => null,
|
|
|
|
|
'network_suspended_at' => null,
|
|
|
|
|
'spec_json' => json_encode(
|
|
|
|
|
['managed' => true],
|
|
|
|
|
JSON_THROW_ON_ERROR
|
|
|
|
|
),
|
|
|
|
|
'sealed_credential' => null,
|
|
|
|
|
'created_at' => $now - 3600,
|
|
|
|
|
'updated_at' => $now - 3600,
|
|
|
|
|
]);
|
|
|
|
|
$fields = [];
|
|
|
|
|
foreach ([
|
|
|
|
|
'hostname' => 'hostname',
|
|
|
|
|
'root_password' => 'password',
|
|
|
|
|
'ssh_public_key' => 'ssh_key',
|
|
|
|
|
] as $field => $code) {
|
|
|
|
|
if (array_key_exists($field, $body)) {
|
|
|
|
|
$fields[] = $code;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
$payload = [
|
|
|
|
|
'method' => 'POST',
|
|
|
|
|
'request_target' => '/v1/vms/' . $uuid . '/settings',
|
|
|
|
|
'body' => $body + ['confirm_restart' => true],
|
|
|
|
|
'owner_id' => 1,
|
|
|
|
|
'can_manage' => false,
|
|
|
|
|
'credential' => $credential,
|
|
|
|
|
'settings_fields' => $fields,
|
|
|
|
|
];
|
|
|
|
|
Capsule::table('cloud_jobs')->insert([
|
|
|
|
|
'id' => $id,
|
|
|
|
|
'operation_id' => $operationId,
|
|
|
|
|
'request_idempotency_key' => $operationId,
|
|
|
|
|
'request_fingerprint' => hash('sha256', $operationId),
|
|
|
|
|
'parent_job_id' => null,
|
|
|
|
|
'order_id' => null,
|
|
|
|
|
'invoice_id' => null,
|
|
|
|
|
'instance_id' => $id,
|
|
|
|
|
'snapshot_id' => null,
|
|
|
|
|
'user_id' => 1,
|
|
|
|
|
'host_id' => 7,
|
|
|
|
|
'kind' => 'instance_settings',
|
|
|
|
|
'execution_mode' => 'agent',
|
|
|
|
|
'phase' => 'terminal',
|
|
|
|
|
'outcome' => 'failed',
|
|
|
|
|
'status' => 'failed',
|
|
|
|
|
'sealed_payload' => CloudSecrets::encrypt(
|
|
|
|
|
$payload,
|
|
|
|
|
'job-payload'
|
|
|
|
|
),
|
|
|
|
|
'attempts' => 1,
|
|
|
|
|
'dispatch_attempts' => 1,
|
|
|
|
|
'dispatch_deadline_at' => $now - 1000,
|
|
|
|
|
'dispatch_started_at' => $now - 990,
|
|
|
|
|
'agent_accepted_at' => $now - 980,
|
|
|
|
|
'receipt_deadline_at' => $now - 20,
|
|
|
|
|
'terminal_at' => $now - 10,
|
|
|
|
|
'effect_may_have_applied' => 1,
|
|
|
|
|
'safe_to_resubmit' => 0,
|
|
|
|
|
'effect_resolution' => 'unknown',
|
|
|
|
|
'effect_resolved_at' => null,
|
|
|
|
|
'effect_resolved_by' => null,
|
|
|
|
|
'effect_resolution_reason' => '',
|
|
|
|
|
'completion_event_id' => null,
|
|
|
|
|
'read_at' => null,
|
|
|
|
|
'dispatched_at' => $now - 980,
|
|
|
|
|
'lease_token' => null,
|
|
|
|
|
'lease_until' => 0,
|
|
|
|
|
'available_at' => 0,
|
|
|
|
|
'error_code' => 'operation_deadline_exceeded',
|
|
|
|
|
'finalization_incident' => 0,
|
|
|
|
|
'finalization_error_code' => null,
|
|
|
|
|
'result_json' => json_encode(
|
|
|
|
|
['cause_code' => 'operation_deadline_exceeded'],
|
|
|
|
|
JSON_THROW_ON_ERROR
|
|
|
|
|
),
|
|
|
|
|
'created_at' => $now - 1000,
|
|
|
|
|
'updated_at' => $now - 10,
|
|
|
|
|
]);
|
|
|
|
|
return [$id, $uuid];
|
|
|
|
|
};
|
|
|
|
|
$insertDispatchableJob = static function (
|
|
|
|
|
string $operationId
|
|
|
|
|
) use (&$nextId, $now): int {
|
|
|
|
|
$id = $nextId++;
|
|
|
|
|
Capsule::table('cloud_jobs')->insert([
|
|
|
|
|
'id' => $id,
|
|
|
|
|
'operation_id' => $operationId,
|
|
|
|
|
'request_idempotency_key' => $operationId,
|
|
|
|
|
'request_fingerprint' => hash('sha256', $operationId),
|
|
|
|
|
'parent_job_id' => null,
|
|
|
|
|
'order_id' => null,
|
|
|
|
|
'invoice_id' => null,
|
|
|
|
|
'instance_id' => null,
|
|
|
|
|
'snapshot_id' => null,
|
|
|
|
|
// A missing actor closes safely before dispatch. It keeps this
|
|
|
|
|
// fixture focused on the public worker scheduling seam rather
|
|
|
|
|
// than sharing an HTTP transport with system recovery.
|
|
|
|
|
'user_id' => 999,
|
|
|
|
|
'host_id' => 7,
|
|
|
|
|
'kind' => 'power_start',
|
|
|
|
|
'execution_mode' => 'agent',
|
|
|
|
|
'phase' => 'accepted',
|
|
|
|
|
'outcome' => null,
|
|
|
|
|
'status' => 'queued',
|
|
|
|
|
'sealed_payload' => CloudSecrets::encrypt([
|
|
|
|
|
'method' => 'POST',
|
|
|
|
|
'request_target' => '/v1/vms/'
|
|
|
|
|
. '00000000-0000-4000-8000-000000000999/start',
|
|
|
|
|
'body' => [],
|
|
|
|
|
'owner_id' => 999,
|
|
|
|
|
'can_manage' => false,
|
|
|
|
|
], 'job-payload'),
|
|
|
|
|
'attempts' => 0,
|
|
|
|
|
'dispatch_attempts' => 0,
|
|
|
|
|
'dispatch_deadline_at' => $now + 300,
|
|
|
|
|
'dispatch_started_at' => null,
|
|
|
|
|
'agent_accepted_at' => null,
|
|
|
|
|
'receipt_deadline_at' => null,
|
|
|
|
|
'terminal_at' => null,
|
|
|
|
|
'effect_may_have_applied' => null,
|
|
|
|
|
'safe_to_resubmit' => null,
|
|
|
|
|
'effect_resolution' => 'not_applicable',
|
|
|
|
|
'effect_resolved_at' => null,
|
|
|
|
|
'effect_resolved_by' => null,
|
|
|
|
|
'effect_resolution_reason' => '',
|
|
|
|
|
'completion_event_id' => null,
|
|
|
|
|
'read_at' => null,
|
|
|
|
|
'dispatched_at' => null,
|
|
|
|
|
'lease_token' => null,
|
|
|
|
|
'lease_until' => 0,
|
|
|
|
|
'available_at' => 0,
|
|
|
|
|
'error_code' => null,
|
|
|
|
|
'finalization_incident' => 0,
|
|
|
|
|
'finalization_error_code' => null,
|
|
|
|
|
'result_json' => null,
|
|
|
|
|
'created_at' => $now,
|
|
|
|
|
'updated_at' => $now,
|
|
|
|
|
]);
|
|
|
|
|
return $id;
|
|
|
|
|
};
|
|
|
|
|
$audit = static function (int $id): array {
|
|
|
|
|
$job = CloudJob::find($id);
|
|
|
|
|
return [
|
|
|
|
|
'operation_id' => (string) $job->operation_id,
|
|
|
|
|
'kind' => (string) $job->kind,
|
|
|
|
|
'phase' => (string) $job->phase,
|
|
|
|
|
'outcome' => (string) $job->outcome,
|
|
|
|
|
'status' => (string) $job->status,
|
|
|
|
|
'dispatch_attempts' => (int) $job->dispatch_attempts,
|
|
|
|
|
'effect_may_have_applied' =>
|
|
|
|
|
(bool) $job->effect_may_have_applied,
|
|
|
|
|
'safe_to_resubmit' => (bool) $job->safe_to_resubmit,
|
|
|
|
|
'error_code' => (string) $job->error_code,
|
|
|
|
|
'result_json' => $job->result_json,
|
|
|
|
|
'terminal_at' => (int) $job->terminal_at,
|
|
|
|
|
'completion_event_id' => $job->completion_event_id,
|
|
|
|
|
'read_at' => $job->read_at,
|
|
|
|
|
];
|
|
|
|
|
};
|
|
|
|
|
$inspection = static function (
|
|
|
|
|
string $operationId,
|
|
|
|
|
string $journalStatus,
|
|
|
|
|
string $resolution,
|
|
|
|
|
?array $projection = null,
|
|
|
|
|
?bool $fenceResolutionRequired = null,
|
|
|
|
|
?string $errorCode = null,
|
|
|
|
|
?string $fenceState = null
|
|
|
|
|
): array {
|
|
|
|
|
$applied = $resolution === 'applied';
|
|
|
|
|
$uncertain = in_array(
|
|
|
|
|
$resolution,
|
|
|
|
|
['partial', 'inconclusive'],
|
|
|
|
|
true
|
|
|
|
|
);
|
|
|
|
|
$value = [
|
|
|
|
|
'operation_id' => $operationId,
|
|
|
|
|
'operation_status' => $journalStatus,
|
|
|
|
|
'checked_at' => time(),
|
|
|
|
|
'outcome' => $uncertain
|
|
|
|
|
? 'uncertain' : ($applied ? 'succeeded' : 'failed'),
|
|
|
|
|
'summary_code' => $uncertain
|
|
|
|
|
? 'evidence_incomplete'
|
|
|
|
|
: ($applied ? 'operation_succeeded' : 'operation_failed'),
|
|
|
|
|
'effect_resolution' => $resolution,
|
|
|
|
|
'fence_resolution_required' => $fenceResolutionRequired
|
|
|
|
|
?? $journalStatus === 'failed',
|
|
|
|
|
'fence_state' => $fenceState ?? (($fenceResolutionRequired
|
|
|
|
|
?? $journalStatus === 'failed') ? 'active' : 'not_required'),
|
|
|
|
|
'abort_job' => ['available' => false],
|
|
|
|
|
'cleanup' => ['available' => false],
|
|
|
|
|
] + CloudOperationRecoveryIdentity::expected(
|
|
|
|
|
CloudJob::where('operation_id', $operationId)->firstOrFail(),
|
|
|
|
|
$applied
|
|
|
|
|
);
|
|
|
|
|
if ($projection !== null) {
|
|
|
|
|
$value['effect_projection'] = $projection;
|
|
|
|
|
}
|
|
|
|
|
if ($resolution === 'not_applied') {
|
|
|
|
|
$value['error_code'] = $errorCode ?? 'settings_not_committed';
|
|
|
|
|
}
|
|
|
|
|
return $value;
|
|
|
|
|
};
|
|
|
|
|
$notFound = static fn (): CloudAgentError => new CloudAgentError(
|
|
|
|
|
'not found', 404, 'fence_resolution_not_found', false, 404
|
|
|
|
|
);
|
|
|
|
|
$fenceResponse = static function (
|
|
|
|
|
array $request,
|
|
|
|
|
int $resolvedAt
|
|
|
|
|
): array {
|
|
|
|
|
$payload = $request['payload'];
|
|
|
|
|
return [
|
|
|
|
|
'operation_id' => $request['operationId'],
|
|
|
|
|
'resolution_id' => $payload['resolution_id'],
|
|
|
|
|
'resolution' => $payload['resolution'],
|
|
|
|
|
'inspection_checked_at' =>
|
|
|
|
|
$payload['inspection_checked_at'],
|
|
|
|
|
'reason_sha256' => $payload['reason_sha256'],
|
|
|
|
|
'resolved_at' => $resolvedAt,
|
|
|
|
|
'resolved_by' => 0,
|
|
|
|
|
'replayed' => false,
|
|
|
|
|
];
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
$client = new AutomaticRecoveryTestAgentClient();
|
|
|
|
|
CloudAgentSettingsService::$client = $client;
|
|
|
|
|
|
|
|
|
|
// An immutable received receipt must outrank a legacy aggregate whose
|
|
|
|
|
// child can remain active indefinitely. This prevents the aggregate sweep
|
|
|
|
|
// from starving both PHP convergence and the Agent outbox ACK.
|
|
|
|
|
[$receiptPriorityId, $receiptPriorityUuid] = $insertSettingsJob(
|
|
|
|
|
str_repeat('0a', 16),
|
|
|
|
|
['hostname' => 'receipt-priority.example']
|
|
|
|
|
);
|
|
|
|
|
$receiptPriorityJob = CloudJob::findOrFail($receiptPriorityId);
|
|
|
|
|
$receiptPriorityIdentity = CloudOperationRecoveryIdentity::expected(
|
|
|
|
|
$receiptPriorityJob,
|
|
|
|
|
true
|
|
|
|
|
);
|
|
|
|
|
$receiptPriorityResult = [
|
|
|
|
|
'uuid' => $receiptPriorityUuid,
|
|
|
|
|
'hostname' => 'receipt-priority.example',
|
|
|
|
|
'settings_changed' => true,
|
|
|
|
|
];
|
|
|
|
|
$receiptPriorityWire = [
|
|
|
|
|
'event_id' => 'automatic-priority-receipt',
|
|
|
|
|
'host_id' => 7,
|
|
|
|
|
'operation_id' => (string) $receiptPriorityJob->operation_id,
|
|
|
|
|
'action' => $receiptPriorityIdentity['action'],
|
|
|
|
|
'resource_id' => $receiptPriorityIdentity['resource_id'],
|
|
|
|
|
'request_fingerprint' =>
|
|
|
|
|
$receiptPriorityIdentity['request_fingerprint'],
|
|
|
|
|
'status' => 'succeeded',
|
|
|
|
|
'effect_may_have_applied' => false,
|
|
|
|
|
'safe_to_resubmit' => false,
|
|
|
|
|
'error_code' => '',
|
|
|
|
|
'result' => $receiptPriorityResult,
|
|
|
|
|
];
|
|
|
|
|
Capsule::table('cloud_agent_completion_events')->insert([
|
|
|
|
|
'event_id' => $receiptPriorityWire['event_id'],
|
|
|
|
|
'host_id' => 7,
|
|
|
|
|
'operation_id' => $receiptPriorityWire['operation_id'],
|
|
|
|
|
'payload_sha256' => hash('sha256', json_encode(
|
|
|
|
|
$receiptPriorityWire,
|
|
|
|
|
JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES
|
|
|
|
|
| JSON_UNESCAPED_UNICODE
|
|
|
|
|
)),
|
|
|
|
|
'outcome' => 'succeeded',
|
|
|
|
|
'effect_may_have_applied' => 0,
|
|
|
|
|
'safe_to_resubmit' => 0,
|
|
|
|
|
'error_code' => '',
|
|
|
|
|
'result_json' => json_encode([
|
|
|
|
|
'schema' => 'virt_cloud_completion_event',
|
|
|
|
|
'version' => 1,
|
|
|
|
|
'identity' => $receiptPriorityIdentity,
|
|
|
|
|
'result' => $receiptPriorityResult,
|
|
|
|
|
], JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES
|
|
|
|
|
| JSON_UNESCAPED_UNICODE),
|
|
|
|
|
'disposition' => 'received',
|
|
|
|
|
'received_at' => $now,
|
|
|
|
|
'applied_at' => null,
|
|
|
|
|
]);
|
|
|
|
|
[$legacyParentId] = $insertSettingsJob(
|
|
|
|
|
str_repeat('0b', 16),
|
|
|
|
|
['hostname' => 'legacy-parent.example']
|
|
|
|
|
);
|
|
|
|
|
Capsule::table('cloud_jobs')->where('id', $legacyParentId)->update([
|
|
|
|
|
'kind' => 'template_import',
|
|
|
|
|
'execution_mode' => 'aggregate',
|
|
|
|
|
'host_id' => null,
|
|
|
|
|
'instance_id' => null,
|
|
|
|
|
'dispatch_attempts' => 0,
|
|
|
|
|
'dispatched_at' => null,
|
|
|
|
|
'available_at' => 0,
|
|
|
|
|
'lease_until' => 0,
|
|
|
|
|
]);
|
|
|
|
|
[$legacyChildId] = $insertSettingsJob(
|
|
|
|
|
str_repeat('0c', 16),
|
|
|
|
|
['hostname' => 'legacy-child.example']
|
|
|
|
|
);
|
|
|
|
|
Capsule::table('cloud_jobs')->where('id', $legacyChildId)->update([
|
|
|
|
|
'parent_job_id' => $legacyParentId,
|
|
|
|
|
'kind' => 'template_import_replica',
|
|
|
|
|
'phase' => 'awaiting_receipt',
|
|
|
|
|
'outcome' => null,
|
|
|
|
|
'status' => 'running',
|
|
|
|
|
'effect_may_have_applied' => null,
|
|
|
|
|
'safe_to_resubmit' => null,
|
|
|
|
|
'effect_resolution' => 'not_applicable',
|
|
|
|
|
'terminal_at' => null,
|
|
|
|
|
]);
|
|
|
|
|
$beforePriorityRequests = count($client->requests);
|
|
|
|
|
$assert((new CloudAutomaticOperationRecoveryService())->tick(),
|
|
|
|
|
'An exact completion receipt was not claimed ahead of legacy sweep.');
|
|
|
|
|
$priorityEvent = Capsule::table('cloud_agent_completion_events')
|
|
|
|
|
->where('event_id', 'automatic-priority-receipt')->first();
|
|
|
|
|
$assert((string) CloudJob::findOrFail($receiptPriorityId)
|
|
|
|
|
->effect_resolution === 'applied'
|
|
|
|
|
&& (string) $priorityEvent->disposition === 'applied'
|
|
|
|
|
&& (int) CloudJob::findOrFail($legacyParentId)->available_at === 0
|
|
|
|
|
&& count($client->requests) === $beforePriorityRequests,
|
|
|
|
|
'A waiting aggregate starved or contaminated an exact receipt.');
|
|
|
|
|
Capsule::table('cloud_jobs')->whereIn(
|
|
|
|
|
'id',
|
|
|
|
|
[$legacyParentId, $legacyChildId]
|
|
|
|
|
)->delete();
|
|
|
|
|
Capsule::table('cloud_instances')->whereIn(
|
|
|
|
|
'id',
|
|
|
|
|
[$legacyParentId, $legacyChildId]
|
|
|
|
|
)->delete();
|
|
|
|
|
|
|
|
|
|
// Sustained completion traffic must not monopolize the recovery worker.
|
|
|
|
|
// The long-lived worker has three independently useful lanes: immutable
|
|
|
|
|
// receipts, ordinary Agent leaves, and PHP-only aggregate projection.
|
|
|
|
|
// With all three continuously ready, every lane must make progress within
|
|
|
|
|
// one three-tick rotation; replenishing receipts may not starve either of
|
|
|
|
|
// the other two lanes.
|
|
|
|
|
[$fairLeafId] = $insertSettingsJob(
|
|
|
|
|
md5('automatic-fair-leaf'),
|
|
|
|
|
['hostname' => 'fair-leaf.example']
|
|
|
|
|
);
|
|
|
|
|
Capsule::table('cloud_jobs')->where('id', $fairLeafId)->update([
|
|
|
|
|
'host_id' => null,
|
|
|
|
|
'available_at' => 0,
|
|
|
|
|
'lease_token' => null,
|
|
|
|
|
'lease_until' => 0,
|
|
|
|
|
]);
|
|
|
|
|
|
|
|
|
|
[$fairParentId] = $insertSettingsJob(
|
|
|
|
|
md5('automatic-fair-aggregate'),
|
|
|
|
|
['hostname' => 'fair-aggregate.example']
|
|
|
|
|
);
|
|
|
|
|
Capsule::table('cloud_jobs')->where('id', $fairParentId)->update([
|
|
|
|
|
'parent_job_id' => null,
|
|
|
|
|
'order_id' => null,
|
|
|
|
|
'invoice_id' => null,
|
|
|
|
|
'instance_id' => null,
|
|
|
|
|
'snapshot_id' => null,
|
|
|
|
|
'host_id' => null,
|
|
|
|
|
'kind' => 'template_import',
|
|
|
|
|
'execution_mode' => 'aggregate',
|
|
|
|
|
'attempts' => 0,
|
|
|
|
|
'dispatch_attempts' => 0,
|
|
|
|
|
'dispatch_started_at' => null,
|
|
|
|
|
'agent_accepted_at' => null,
|
|
|
|
|
'dispatched_at' => null,
|
|
|
|
|
'completion_event_id' => null,
|
|
|
|
|
'effect_may_have_applied' => 1,
|
|
|
|
|
'safe_to_resubmit' => 0,
|
|
|
|
|
'effect_resolution' => 'unknown',
|
|
|
|
|
'finalization_incident' => 1,
|
|
|
|
|
'finalization_error_code' => 'recovery_child_convergence',
|
|
|
|
|
'available_at' => 0,
|
|
|
|
|
'lease_token' => null,
|
|
|
|
|
'lease_until' => 0,
|
|
|
|
|
]);
|
|
|
|
|
[$fairChildId] = $insertSettingsJob(
|
|
|
|
|
md5('automatic-fair-aggregate-child'),
|
|
|
|
|
['hostname' => 'fair-aggregate-child.example']
|
|
|
|
|
);
|
|
|
|
|
Capsule::table('cloud_jobs')->where('id', $fairChildId)->update([
|
|
|
|
|
'parent_job_id' => $fairParentId,
|
|
|
|
|
'kind' => 'template_import_replica',
|
|
|
|
|
'effect_may_have_applied' => 0,
|
|
|
|
|
'safe_to_resubmit' => 1,
|
|
|
|
|
'effect_resolution' => 'not_applied',
|
|
|
|
|
'effect_resolved_at' => $now,
|
|
|
|
|
'effect_resolution_reason' => 'normal_completion',
|
|
|
|
|
'available_at' => $now + 3600,
|
|
|
|
|
'lease_token' => null,
|
|
|
|
|
'lease_until' => $now + 3600,
|
|
|
|
|
]);
|
|
|
|
|
|
|
|
|
|
$insertFairReceipt = static function (int $sequence) use (
|
|
|
|
|
$insertSettingsJob,
|
|
|
|
|
$now
|
|
|
|
|
): string {
|
|
|
|
|
$operationId = md5('automatic-fair-receipt:' . $sequence);
|
|
|
|
|
[$jobId, $uuid] = $insertSettingsJob(
|
|
|
|
|
$operationId,
|
|
|
|
|
['hostname' => 'fair-receipt-' . $sequence . '.example']
|
|
|
|
|
);
|
|
|
|
|
$job = CloudJob::findOrFail($jobId);
|
|
|
|
|
$identity = CloudOperationRecoveryIdentity::expected($job, true);
|
|
|
|
|
$result = [
|
|
|
|
|
'uuid' => $uuid,
|
|
|
|
|
'hostname' => 'fair-receipt-' . $sequence . '.example',
|
|
|
|
|
'settings_changed' => true,
|
|
|
|
|
];
|
|
|
|
|
$wire = [
|
|
|
|
|
'event_id' => 'automatic-fair-receipt-' . $sequence,
|
|
|
|
|
'host_id' => 7,
|
|
|
|
|
'operation_id' => $operationId,
|
|
|
|
|
'action' => $identity['action'],
|
|
|
|
|
'resource_id' => $identity['resource_id'],
|
|
|
|
|
'request_fingerprint' => $identity['request_fingerprint'],
|
|
|
|
|
'status' => 'succeeded',
|
|
|
|
|
'effect_may_have_applied' => false,
|
|
|
|
|
'safe_to_resubmit' => false,
|
|
|
|
|
'error_code' => '',
|
|
|
|
|
'result' => $result,
|
|
|
|
|
];
|
|
|
|
|
Capsule::table('cloud_agent_completion_events')->insert([
|
|
|
|
|
'event_id' => $wire['event_id'],
|
|
|
|
|
'host_id' => 7,
|
|
|
|
|
'operation_id' => $operationId,
|
|
|
|
|
'payload_sha256' => hash('sha256', json_encode(
|
|
|
|
|
$wire,
|
|
|
|
|
JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES
|
|
|
|
|
| JSON_UNESCAPED_UNICODE
|
|
|
|
|
)),
|
|
|
|
|
'outcome' => 'succeeded',
|
|
|
|
|
'effect_may_have_applied' => 0,
|
|
|
|
|
'safe_to_resubmit' => 0,
|
|
|
|
|
'error_code' => '',
|
|
|
|
|
'result_json' => json_encode([
|
|
|
|
|
'schema' => 'virt_cloud_completion_event',
|
|
|
|
|
'version' => 1,
|
|
|
|
|
'identity' => $identity,
|
|
|
|
|
'result' => $result,
|
|
|
|
|
], JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES
|
|
|
|
|
| JSON_UNESCAPED_UNICODE),
|
|
|
|
|
'disposition' => 'received',
|
|
|
|
|
'received_at' => $now + $sequence,
|
|
|
|
|
'applied_at' => null,
|
|
|
|
|
]);
|
|
|
|
|
return $wire['event_id'];
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
$fairService = new CloudAutomaticOperationRecoveryService();
|
|
|
|
|
$fairReceiptIds = [];
|
|
|
|
|
for ($sequence = 1; $sequence <= 6; $sequence++) {
|
|
|
|
|
$fairReceiptIds[] = $insertFairReceipt($sequence);
|
|
|
|
|
$assert($fairService->tick(),
|
|
|
|
|
'A fair recovery rotation unexpectedly found no work.');
|
|
|
|
|
}
|
|
|
|
|
$fairAppliedReceipts = Capsule::table(
|
|
|
|
|
'cloud_agent_completion_events'
|
|
|
|
|
)->whereIn('event_id', $fairReceiptIds)
|
|
|
|
|
->where('disposition', 'applied')->count();
|
|
|
|
|
$fairLeaf = CloudJob::findOrFail($fairLeafId);
|
|
|
|
|
$fairParent = CloudJob::findOrFail($fairParentId);
|
|
|
|
|
$assert($fairAppliedReceipts >= 2
|
|
|
|
|
&& (bool) $fairLeaf->finalization_incident
|
|
|
|
|
&& (string) $fairLeaf->finalization_error_code
|
|
|
|
|
=== 'recovery_instance_state_conflict'
|
|
|
|
|
&& (bool) $fairParent->finalization_incident
|
|
|
|
|
&& (int) $fairParent->available_at >= time() + 290,
|
|
|
|
|
'Continuous receipt traffic starved a leaf or aggregate lane, '
|
|
|
|
|
. 'or fairness starved immutable receipts.');
|
|
|
|
|
$fairReceiptOperations = Capsule::table(
|
|
|
|
|
'cloud_agent_completion_events'
|
|
|
|
|
)->whereIn('event_id', $fairReceiptIds)->pluck('operation_id')->all();
|
|
|
|
|
$fairReceiptJobs = Capsule::table('cloud_jobs')
|
|
|
|
|
->whereIn('operation_id', $fairReceiptOperations);
|
|
|
|
|
$fairReceiptJobIds = $fairReceiptJobs->pluck('id')->all();
|
|
|
|
|
$fairReceiptInstanceIds = Capsule::table('cloud_jobs')
|
|
|
|
|
->whereIn('id', $fairReceiptJobIds)->pluck('instance_id')->all();
|
|
|
|
|
Capsule::table('cloud_agent_completion_events')
|
|
|
|
|
->whereIn('event_id', $fairReceiptIds)->delete();
|
|
|
|
|
Capsule::table('cloud_jobs')->whereIn('id', $fairReceiptJobIds)->delete();
|
|
|
|
|
Capsule::table('cloud_instances')
|
|
|
|
|
->whereIn('id', $fairReceiptInstanceIds)->delete();
|
|
|
|
|
Capsule::table('cloud_jobs')->whereIn('id', [
|
|
|
|
|
$fairLeafId,
|
|
|
|
|
$fairParentId,
|
|
|
|
|
$fairChildId,
|
|
|
|
|
])->delete();
|
|
|
|
|
Capsule::table('cloud_instances')->whereIn('id', [
|
|
|
|
|
$fairLeafId,
|
|
|
|
|
$fairParentId,
|
|
|
|
|
$fairChildId,
|
|
|
|
|
])->delete();
|
|
|
|
|
|
|
|
|
|
// Every explicitly registered Agent leaf remains eligible. Inconclusive
|
|
|
|
|
// evidence keeps the exact operation fenced and schedules another check.
|
|
|
|
|
[$nonSettingsId, $nonSettingsUuid] = $insertSettingsJob(
|
|
|
|
|
str_repeat('0f', 16),
|
|
|
|
|
['hostname' => 'not-a-settings-recovery.example']
|
|
|
|
|
);
|
|
|
|
|
Capsule::table('cloud_jobs')->where('id', $nonSettingsId)->update([
|
|
|
|
|
'kind' => 'network_change',
|
|
|
|
|
'sealed_payload' => CloudSecrets::encrypt([
|
|
|
|
|
'method' => 'POST',
|
|
|
|
|
'request_target' => '/v1/vms/' . $nonSettingsUuid . '/network',
|
|
|
|
|
'body' => [
|
|
|
|
|
'operation_id' => str_repeat('0f', 16),
|
|
|
|
|
'network' => 'bridge:br0',
|
|
|
|
|
'public_pool_key' => str_repeat('a', 32),
|
|
|
|
|
'public_pool_cidr' => '203.0.113.0/24',
|
|
|
|
|
'public_ip_cidr' => '203.0.113.10/24',
|
|
|
|
|
'gateway' => '203.0.113.1',
|
|
|
|
|
'dns_servers' => ['1.1.1.1'],
|
|
|
|
|
'confirm_restart' => true,
|
|
|
|
|
],
|
|
|
|
|
'owner_id' => 1,
|
|
|
|
|
'can_manage' => false,
|
|
|
|
|
], 'job-payload'),
|
|
|
|
|
]);
|
|
|
|
|
$beforeNonSettingsRequests = count($client->requests);
|
|
|
|
|
$client->responses[] = $inspection(
|
|
|
|
|
str_repeat('0f', 16),
|
|
|
|
|
'failed',
|
|
|
|
|
'inconclusive'
|
|
|
|
|
);
|
|
|
|
|
$assert((new CloudAutomaticOperationRecoveryService())->tick(),
|
|
|
|
|
'The recovery worker did not claim a registered non-settings kind.');
|
|
|
|
|
$nonSettings = CloudJob::find($nonSettingsId);
|
|
|
|
|
$assert((string) $nonSettings->effect_resolution === 'unknown'
|
|
|
|
|
&& (string) $nonSettings->finalization_error_code
|
|
|
|
|
=== 'recovery_evidence_inconclusive'
|
|
|
|
|
&& (int) $nonSettings->available_at >= time() + 290
|
|
|
|
|
&& count($client->requests) === $beforeNonSettingsRequests + 1,
|
|
|
|
|
'Inconclusive non-settings evidence did not stay recoverable.');
|
|
|
|
|
|
|
|
|
|
// A successful read-only inspection must adapt back to the same terminal
|
|
|
|
|
// operation contract as an immediate Agent response. This is the public
|
|
|
|
|
// recovery seam for a template deletion whose original success receipt
|
|
|
|
|
// reached PHP but failed local catalog finalization.
|
|
|
|
|
$templateDeleteId = 'automatic-delete-template';
|
|
|
|
|
[$templateDeleteParentId] = $insertSettingsJob(
|
|
|
|
|
md5('automatic-template-delete-parent'),
|
|
|
|
|
['hostname' => 'unused-parent.example']
|
|
|
|
|
);
|
|
|
|
|
Capsule::table('cloud_jobs')->where(
|
|
|
|
|
'id',
|
|
|
|
|
$templateDeleteParentId
|
|
|
|
|
)->update([
|
|
|
|
|
'parent_job_id' => null,
|
|
|
|
|
'instance_id' => null,
|
|
|
|
|
'host_id' => null,
|
|
|
|
|
'kind' => 'template_delete',
|
|
|
|
|
'execution_mode' => 'aggregate',
|
|
|
|
|
'attempts' => 0,
|
|
|
|
|
'dispatch_attempts' => 0,
|
|
|
|
|
'dispatch_started_at' => null,
|
|
|
|
|
'agent_accepted_at' => null,
|
|
|
|
|
'dispatched_at' => null,
|
|
|
|
|
'effect_resolution' => 'unknown',
|
|
|
|
|
'finalization_incident' => 1,
|
|
|
|
|
'finalization_error_code' => 'recovery_child_convergence',
|
|
|
|
|
'available_at' => 0,
|
|
|
|
|
'lease_token' => null,
|
|
|
|
|
'lease_until' => 0,
|
|
|
|
|
]);
|
|
|
|
|
[$templateDeleteChildId] = $insertSettingsJob(
|
|
|
|
|
md5('automatic-template-delete-child'),
|
|
|
|
|
['hostname' => 'unused-child.example']
|
|
|
|
|
);
|
|
|
|
|
$templateDeleteOperationId = (string) CloudJob::findOrFail(
|
|
|
|
|
$templateDeleteChildId
|
|
|
|
|
)->operation_id;
|
|
|
|
|
Capsule::table('cloud_jobs')->where(
|
|
|
|
|
'id',
|
|
|
|
|
$templateDeleteChildId
|
|
|
|
|
)->update([
|
|
|
|
|
'parent_job_id' => $templateDeleteParentId,
|
|
|
|
|
'instance_id' => null,
|
|
|
|
|
'kind' => 'template_delete_replica',
|
|
|
|
|
'sealed_payload' => CloudSecrets::encrypt([
|
|
|
|
|
'method' => 'DELETE',
|
|
|
|
|
'request_target' => '/v1/templates/' . $templateDeleteId,
|
|
|
|
|
'body' => ['operation_id' => $templateDeleteOperationId],
|
|
|
|
|
'owner_id' => 1,
|
|
|
|
|
'can_manage' => true,
|
|
|
|
|
'template_id' => $templateDeleteId,
|
|
|
|
|
], 'job-payload'),
|
|
|
|
|
'finalization_incident' => 1,
|
|
|
|
|
'finalization_error_code' => 'recovery_receipt_invalid',
|
|
|
|
|
'available_at' => 0,
|
|
|
|
|
'lease_token' => null,
|
|
|
|
|
'lease_until' => 0,
|
|
|
|
|
]);
|
|
|
|
|
Capsule::table('cloud_templates')->insert([
|
|
|
|
|
'id' => $templateDeleteId,
|
|
|
|
|
'active_job_id' => $templateDeleteParentId,
|
|
|
|
|
'state' => 'deleting',
|
|
|
|
|
'revision' => 1,
|
|
|
|
|
'generation_token' => str_repeat('d', 32),
|
|
|
|
|
'image_bytes' => 4096,
|
|
|
|
|
'sha256' => str_repeat('e', 64),
|
|
|
|
|
'updated_at' => $now,
|
|
|
|
|
]);
|
|
|
|
|
Capsule::table('cloud_template_replicas')->insert([
|
|
|
|
|
'host_id' => 7,
|
|
|
|
|
'template_id' => $templateDeleteId,
|
|
|
|
|
'revision' => 1,
|
|
|
|
|
'generation_token' => str_repeat('d', 32),
|
|
|
|
|
'state' => 'delete_failed',
|
|
|
|
|
'image_bytes' => 4096,
|
|
|
|
|
'sha256' => str_repeat('e', 64),
|
|
|
|
|
'operation_id' => $templateDeleteOperationId,
|
|
|
|
|
'job_id' => $templateDeleteChildId,
|
|
|
|
|
'error_code' => 'completion_finalization_failed',
|
|
|
|
|
'created_at' => $now - 120,
|
|
|
|
|
'updated_at' => $now,
|
|
|
|
|
]);
|
|
|
|
|
Capsule::table('cloud_template_artifacts')->insert([
|
|
|
|
|
'template_id' => $templateDeleteId,
|
|
|
|
|
'storage_revision' => 1,
|
|
|
|
|
'sealed_storage_runtime' => CloudSecrets::encrypt([
|
|
|
|
|
'config' => [],
|
|
|
|
|
'credentials' => [],
|
|
|
|
|
'revision' => 1,
|
|
|
|
|
], 'template-artifact-storage-runtime'),
|
|
|
|
|
'object_key' => 'templates/' . $templateDeleteId . '/system.qcow2',
|
|
|
|
|
'object_version_id' => '',
|
|
|
|
|
'object_etag' => str_repeat('f', 32),
|
|
|
|
|
'size_bytes' => 4096,
|
|
|
|
|
'sha256' => str_repeat('e', 64),
|
|
|
|
|
'state' => 'ready',
|
|
|
|
|
'error_code' => '',
|
|
|
|
|
'delete_operation_id' => '',
|
|
|
|
|
'delete_generation' => 0,
|
|
|
|
|
'delete_cycle_attempts' => 0,
|
|
|
|
|
'delete_presence_confirmations' => 0,
|
|
|
|
|
'delete_last_checked_at' => 0,
|
|
|
|
|
'delete_next_check_at' => 0,
|
|
|
|
|
'delete_attention_at' => null,
|
|
|
|
|
'verified_at' => $now,
|
|
|
|
|
'created_at' => $now - 120,
|
|
|
|
|
'updated_at' => $now,
|
|
|
|
|
]);
|
|
|
|
|
$client->responses[] = $inspection(
|
|
|
|
|
$templateDeleteOperationId,
|
|
|
|
|
'succeeded',
|
|
|
|
|
'applied',
|
|
|
|
|
[
|
|
|
|
|
'resource_type' => 'template',
|
|
|
|
|
'id' => $templateDeleteId,
|
|
|
|
|
'deleted' => true,
|
|
|
|
|
'managed_image_absent' => true,
|
|
|
|
|
'managed_metadata_absent' => true,
|
|
|
|
|
'managed_artifacts_absent' => true,
|
|
|
|
|
],
|
|
|
|
|
false
|
|
|
|
|
);
|
|
|
|
|
$beforeTemplateDeleteRequests = count($client->requests);
|
|
|
|
|
$assert((new CloudAutomaticOperationRecoveryService())->tick(),
|
|
|
|
|
'The recovery worker did not inspect the template deletion.');
|
|
|
|
|
$templateDeleteChild = CloudJob::findOrFail($templateDeleteChildId);
|
|
|
|
|
$templateDeleteParent = CloudJob::findOrFail($templateDeleteParentId);
|
|
|
|
|
$assert((string) $templateDeleteChild->effect_resolution === 'applied'
|
|
|
|
|
&& !(bool) $templateDeleteChild->finalization_incident
|
|
|
|
|
&& $templateDeleteChild->finalization_error_code === null
|
|
|
|
|
&& (string) $templateDeleteParent->effect_resolution === 'unknown'
|
|
|
|
|
&& (bool) $templateDeleteParent->finalization_incident
|
|
|
|
|
&& (string) $templateDeleteParent->finalization_error_code
|
|
|
|
|
=== 'recovery_child_convergence'
|
|
|
|
|
&& (string) Capsule::table('cloud_templates')
|
|
|
|
|
->where('id', $templateDeleteId)->value('state') === 'deleting'
|
|
|
|
|
&& Capsule::table('cloud_template_replicas')
|
|
|
|
|
->where('template_id', $templateDeleteId)->exists()
|
|
|
|
|
&& (string) Capsule::table('cloud_template_artifacts')
|
|
|
|
|
->where('template_id', $templateDeleteId)->value('state')
|
|
|
|
|
=== 'delete_pending'
|
|
|
|
|
&& count($client->requests) === $beforeTemplateDeleteRequests + 1
|
|
|
|
|
&& str_ends_with(
|
|
|
|
|
$client->requests[$beforeTemplateDeleteRequests]['target'],
|
|
|
|
|
'/inspect'
|
|
|
|
|
),
|
|
|
|
|
'A successful template deletion inspection did not converge its host '
|
|
|
|
|
. 'operation into the two-phase master cleanup fence.');
|
|
|
|
|
|
|
|
|
|
// A poisoned Agent completion outbox is never evidence for either effect
|
|
|
|
|
// verdict. Even a syntactically complete not-applied projection remains
|
|
|
|
|
// fenced and the reviewed cause is visible without exposing raw data.
|
|
|
|
|
[$quarantinedId] = $insertSettingsJob(
|
|
|
|
|
str_repeat('0e', 16),
|
|
|
|
|
['hostname' => 'quarantined-receipt.example']
|
|
|
|
|
);
|
|
|
|
|
$quarantinedInspection = $inspection(
|
|
|
|
|
str_repeat('0e', 16),
|
|
|
|
|
'failed',
|
|
|
|
|
'not_applied',
|
|
|
|
|
null,
|
|
|
|
|
true,
|
|
|
|
|
'settings_not_committed'
|
|
|
|
|
);
|
|
|
|
|
$quarantinedInspection['completion_delivery_state'] = 'quarantined';
|
|
|
|
|
$quarantinedInspection['completion_delivery_error_code'] =
|
|
|
|
|
'completion_identity_mismatch';
|
|
|
|
|
$client->responses[] = $quarantinedInspection;
|
|
|
|
|
$beforeQuarantinedRequests = count($client->requests);
|
|
|
|
|
$assert((new CloudAutomaticOperationRecoveryService())->tick(),
|
|
|
|
|
'The recovery worker did not inspect a quarantined completion.');
|
|
|
|
|
$quarantined = CloudJob::findOrFail($quarantinedId);
|
|
|
|
|
$quarantinedUser = CloudJobService::recoveryPresentation(
|
|
|
|
|
$quarantined,
|
|
|
|
|
false
|
|
|
|
|
);
|
|
|
|
|
$quarantinedAdmin = CloudJobService::recoveryPresentation(
|
|
|
|
|
$quarantined,
|
|
|
|
|
true
|
|
|
|
|
);
|
|
|
|
|
$assert((string) $quarantined->effect_resolution === 'unknown'
|
|
|
|
|
&& (string) $quarantined->finalization_error_code
|
|
|
|
|
=== 'recovery_completion_identity_mismatch'
|
|
|
|
|
&& (int) $quarantined->available_at >= time() + 290
|
|
|
|
|
&& count($client->requests) === $beforeQuarantinedRequests + 1
|
|
|
|
|
&& ($quarantinedAdmin['code'] ?? null)
|
|
|
|
|
=== 'recovery_completion_identity_mismatch'
|
|
|
|
|
&& str_contains(
|
|
|
|
|
(string) ($quarantinedAdmin['message'] ?? ''),
|
|
|
|
|
'身份与原操作记录不匹配'
|
|
|
|
|
)
|
|
|
|
|
&& str_contains(
|
|
|
|
|
(string) ($quarantinedUser['message'] ?? ''),
|
|
|
|
|
'身份与原操作记录不匹配'
|
|
|
|
|
)
|
|
|
|
|
&& !array_key_exists('code', $quarantinedUser),
|
|
|
|
|
'A quarantined completion was trusted or its safe cause was hidden.');
|
|
|
|
|
|
|
|
|
|
// A journal success is immutable exact evidence and has no active fence.
|
|
|
|
|
[$succeededId, $succeededUuid] = $insertSettingsJob(
|
|
|
|
|
str_repeat('1', 32),
|
|
|
|
|
['root_password' => 'New-automatic9!Pass'],
|
|
|
|
|
[
|
|
|
|
|
'password' => 'New-automatic9!Pass',
|
|
|
|
|
'login_name' => 'root',
|
|
|
|
|
'auth_mode' => 'password',
|
|
|
|
|
]
|
|
|
|
|
);
|
|
|
|
|
$succeededAudit = $audit($succeededId);
|
|
|
|
|
$client->responses[] = $inspection(
|
|
|
|
|
str_repeat('1', 32),
|
|
|
|
|
'succeeded',
|
|
|
|
|
'applied',
|
|
|
|
|
[
|
|
|
|
|
'uuid' => $succeededUuid,
|
|
|
|
|
'status' => 'running',
|
|
|
|
|
'settings_changed' => true,
|
|
|
|
|
]
|
|
|
|
|
);
|
|
|
|
|
$dispatchId = $insertDispatchableJob(str_repeat('8', 32));
|
|
|
|
|
$beforeDispatchRequests = count($client->requests);
|
|
|
|
|
$assert((new CloudWorkerService())->tick(),
|
|
|
|
|
'The worker did not claim the ordinary dispatchable task first.');
|
|
|
|
|
$dispatch = CloudJob::find($dispatchId);
|
|
|
|
|
$unrecovered = CloudJob::find($succeededId);
|
|
|
|
|
$assert((string) $dispatch->phase === 'terminal'
|
|
|
|
|
&& (string) $dispatch->outcome === 'failed'
|
|
|
|
|
&& (int) $dispatch->dispatch_attempts === 0
|
|
|
|
|
&& (string) $unrecovered->effect_resolution === 'unknown'
|
|
|
|
|
&& count($client->requests) === $beforeDispatchRequests,
|
|
|
|
|
'Automatic recovery ran before an ordinary dispatchable task.');
|
|
|
|
|
$queuedBehindRecovery = $insertDispatchableJob(str_repeat('b', 32));
|
|
|
|
|
$beforeSucceededRecovery = count($client->requests);
|
|
|
|
|
$assert((new CloudAutomaticOperationRecoveryService())->tick(),
|
|
|
|
|
'The dedicated recovery service did not claim a recoverable task.');
|
|
|
|
|
$assert((string) CloudJob::find($queuedBehindRecovery)->phase
|
|
|
|
|
=== 'accepted',
|
|
|
|
|
'The dedicated recovery service consumed an ordinary task.');
|
|
|
|
|
$assert((new CloudWorkerService())->tick()
|
|
|
|
|
&& (string) CloudJob::find($queuedBehindRecovery)->phase
|
|
|
|
|
=== 'terminal',
|
|
|
|
|
'The ordinary worker did not keep processing its independent queue.');
|
|
|
|
|
$succeeded = CloudJob::find($succeededId);
|
|
|
|
|
$credential = CloudSecrets::decrypt(
|
|
|
|
|
(string) Capsule::table('cloud_instances')
|
|
|
|
|
->where('id', $succeededId)->value('sealed_credential'),
|
|
|
|
|
'instance-credential'
|
|
|
|
|
);
|
|
|
|
|
$redacted = CloudSecrets::decrypt(
|
|
|
|
|
(string) $succeeded->sealed_payload,
|
|
|
|
|
'job-payload'
|
|
|
|
|
);
|
|
|
|
|
$assert((string) $succeeded->effect_resolution === 'applied'
|
|
|
|
|
&& (int) $succeeded->effect_resolved_by === 0
|
|
|
|
|
&& $audit($succeededId) === $succeededAudit
|
|
|
|
|
&& ($credential['password'] ?? null)
|
|
|
|
|
=== 'New-automatic9!Pass'
|
|
|
|
|
&& !array_key_exists('credential', $redacted)
|
|
|
|
|
&& !CloudJobService::blockingQueryForInstance(
|
|
|
|
|
$succeededId
|
|
|
|
|
)->exists()
|
|
|
|
|
&& count($client->requests) === $beforeSucceededRecovery + 1
|
|
|
|
|
&& str_ends_with(
|
|
|
|
|
$client->requests[array_key_last($client->requests)]['target'],
|
|
|
|
|
'/inspect'
|
|
|
|
|
),
|
|
|
|
|
'Journal-success recovery did not converge without a fence write.');
|
|
|
|
|
|
|
|
|
|
// A malformed or contradictory receipt is isolated from an existing
|
|
|
|
|
// prepared intent. The job itself remains scheduled for a fresh Agent
|
|
|
|
|
// inspection instead of treating the incident as a permanent stop.
|
|
|
|
|
$invalidProofRequests = count($client->requests);
|
|
|
|
|
$invalidProofCases = [
|
|
|
|
|
[
|
|
|
|
|
'operation_id' => str_repeat('c1', 16),
|
|
|
|
|
'body' => ['root_password' => 'Invalid-proof9!Pass'],
|
|
|
|
|
'credential' => [
|
|
|
|
|
'password' => 'Invalid-proof9!Pass',
|
|
|
|
|
'login_name' => 'root',
|
|
|
|
|
'auth_mode' => 'password',
|
|
|
|
|
],
|
|
|
|
|
'receipt_result' => [],
|
|
|
|
|
'message' => 'missing settings_changed proof',
|
|
|
|
|
],
|
|
|
|
|
[
|
|
|
|
|
'operation_id' => str_repeat('d2', 16),
|
|
|
|
|
'body' => ['hostname' => 'prepared-proof.example'],
|
|
|
|
|
'credential' => null,
|
|
|
|
|
'receipt_result' => [
|
|
|
|
|
'settings_changed' => true,
|
|
|
|
|
'hostname' => 'different-proof.example',
|
|
|
|
|
],
|
|
|
|
|
'message' => 'mismatched hostname proof',
|
|
|
|
|
],
|
|
|
|
|
];
|
|
|
|
|
foreach ($invalidProofCases as $index => $case) {
|
|
|
|
|
[$invalidProofId, $invalidProofUuid] = $insertSettingsJob(
|
|
|
|
|
$case['operation_id'],
|
|
|
|
|
$case['body'],
|
|
|
|
|
$case['credential']
|
|
|
|
|
);
|
|
|
|
|
Capsule::table('cloud_jobs')->where(
|
|
|
|
|
'id',
|
|
|
|
|
$invalidProofId
|
|
|
|
|
)->update([
|
|
|
|
|
'effect_resolution_reason' => 'automatic_recovery:v1:'
|
|
|
|
|
. time() . ':1:pending',
|
|
|
|
|
]);
|
|
|
|
|
$preparedResult = [
|
|
|
|
|
'uuid' => $invalidProofUuid,
|
|
|
|
|
'settings_changed' => true,
|
|
|
|
|
];
|
|
|
|
|
if (array_key_exists('hostname', $case['body'])) {
|
|
|
|
|
$preparedResult['hostname'] = $case['body']['hostname'];
|
|
|
|
|
}
|
|
|
|
|
$operationIdentity = CloudOperationRecoveryIdentity::expected(
|
|
|
|
|
CloudJob::findOrFail($invalidProofId),
|
|
|
|
|
true
|
|
|
|
|
);
|
|
|
|
|
CloudOperationRecoveryService::prepareApplied(
|
|
|
|
|
$invalidProofId,
|
|
|
|
|
$operationIdentity + [
|
|
|
|
|
'operation_id' => $case['operation_id'],
|
|
|
|
|
'status' => 'succeeded',
|
|
|
|
|
'effect_may_have_applied' => false,
|
|
|
|
|
'safe_to_resubmit' => false,
|
|
|
|
|
'error_code' => '',
|
|
|
|
|
'result' => $preparedResult,
|
|
|
|
|
],
|
|
|
|
|
'automatic_applied_succeeded_nofence:'
|
|
|
|
|
. str_repeat((string) ($index + 3), 64)
|
|
|
|
|
);
|
|
|
|
|
$preparedInvalidProof = CloudJob::find($invalidProofId);
|
|
|
|
|
$preparedMarker = (string) $preparedInvalidProof
|
|
|
|
|
->effect_resolution_reason;
|
|
|
|
|
$receiptResult = ['uuid' => $invalidProofUuid]
|
|
|
|
|
+ $case['receipt_result'];
|
|
|
|
|
$eventId = 'automatic-invalid-proof-' . ($index + 1);
|
|
|
|
|
$accepted = CloudOperationCompletionService::accept([
|
|
|
|
|
'type' => 'completion',
|
|
|
|
|
'id' => $eventId,
|
|
|
|
|
'agent_id' => '7',
|
|
|
|
|
'operation_id' => $case['operation_id'],
|
|
|
|
|
'action' => $operationIdentity['action'],
|
|
|
|
|
'resource_id' => $operationIdentity['resource_id'],
|
|
|
|
|
'request_fingerprint' =>
|
|
|
|
|
$operationIdentity['request_fingerprint'],
|
|
|
|
|
'status' => 'succeeded',
|
|
|
|
|
'effect_may_have_applied' => false,
|
|
|
|
|
'safe_to_resubmit' => false,
|
|
|
|
|
'error_code' => '',
|
|
|
|
|
'result' => $receiptResult,
|
|
|
|
|
]);
|
|
|
|
|
$invalidProofJob = CloudJob::find($invalidProofId);
|
|
|
|
|
$invalidProofEvent = Capsule::table(
|
|
|
|
|
'cloud_agent_completion_events'
|
|
|
|
|
)->where('event_id', $eventId)->first();
|
|
|
|
|
$assert($accepted === [
|
|
|
|
|
'accepted' => true,
|
|
|
|
|
'disposition' => 'late',
|
|
|
|
|
]
|
|
|
|
|
&& (string) $invalidProofJob->effect_resolution === 'unknown'
|
|
|
|
|
&& (string) $invalidProofJob->effect_resolution_reason
|
|
|
|
|
=== $preparedMarker
|
|
|
|
|
&& (bool) $invalidProofJob->finalization_incident
|
|
|
|
|
&& in_array(
|
|
|
|
|
(string) $invalidProofJob->finalization_error_code,
|
|
|
|
|
[
|
|
|
|
|
'recovery_receipt_invalid',
|
|
|
|
|
'recovery_evidence_conflict',
|
|
|
|
|
],
|
|
|
|
|
true
|
|
|
|
|
)
|
|
|
|
|
&& (int) $invalidProofJob->available_at >= time() + 290
|
|
|
|
|
&& (string) $invalidProofEvent->disposition === 'late'
|
|
|
|
|
&& CloudJobService::blockingQueryForInstance(
|
|
|
|
|
$invalidProofId
|
|
|
|
|
)->exists(),
|
|
|
|
|
'A success receipt with ' . $case['message']
|
|
|
|
|
. ' was not isolated while leaving inspection retryable.');
|
|
|
|
|
}
|
|
|
|
|
$assert(count($client->requests) === $invalidProofRequests,
|
|
|
|
|
'Invalid late receipt proof unexpectedly contacted the Agent.');
|
|
|
|
|
|
|
|
|
|
// A well-formed completion for a different immutable Agent request is a
|
|
|
|
|
// bad event, not a reason to strand the already-prepared PHP convergence.
|
|
|
|
|
// Quarantine it, then let the normal prepared crash-resume path finish.
|
|
|
|
|
[$wrongIdentityId, $wrongIdentityUuid] = $insertSettingsJob(
|
|
|
|
|
str_repeat('e3', 16),
|
|
|
|
|
['hostname' => 'prepared-identity.example']
|
|
|
|
|
);
|
|
|
|
|
$expectedIdentity = CloudOperationRecoveryIdentity::expected(
|
|
|
|
|
CloudJob::findOrFail($wrongIdentityId),
|
|
|
|
|
true
|
|
|
|
|
);
|
|
|
|
|
CloudOperationRecoveryService::prepareApplied(
|
|
|
|
|
$wrongIdentityId,
|
|
|
|
|
$expectedIdentity + [
|
|
|
|
|
'operation_id' => str_repeat('e3', 16),
|
|
|
|
|
'status' => 'succeeded',
|
|
|
|
|
'effect_may_have_applied' => false,
|
|
|
|
|
'safe_to_resubmit' => false,
|
|
|
|
|
'error_code' => '',
|
|
|
|
|
'result' => [
|
|
|
|
|
'uuid' => $wrongIdentityUuid,
|
|
|
|
|
'hostname' => 'prepared-identity.example',
|
|
|
|
|
'settings_changed' => true,
|
|
|
|
|
],
|
|
|
|
|
],
|
|
|
|
|
'automatic_applied_succeeded_nofence:' . str_repeat('9', 64)
|
|
|
|
|
);
|
|
|
|
|
$preparedWrongIdentity = CloudJob::findOrFail($wrongIdentityId);
|
|
|
|
|
$preparedWrongIdentityMarker = (string) $preparedWrongIdentity
|
|
|
|
|
->effect_resolution_reason;
|
|
|
|
|
$wrongIdentityEventId = 'automatic-wrong-identity';
|
|
|
|
|
$acceptedWrongIdentity = CloudOperationCompletionService::accept([
|
|
|
|
|
'type' => 'completion',
|
|
|
|
|
'id' => $wrongIdentityEventId,
|
|
|
|
|
'agent_id' => '7',
|
|
|
|
|
'operation_id' => str_repeat('e3', 16),
|
|
|
|
|
'action' => $expectedIdentity['action'],
|
|
|
|
|
'resource_id' => '00000000-0000-4000-8000-000000009999',
|
|
|
|
|
'request_fingerprint' => $expectedIdentity['request_fingerprint'],
|
|
|
|
|
'status' => 'succeeded',
|
|
|
|
|
'effect_may_have_applied' => false,
|
|
|
|
|
'safe_to_resubmit' => false,
|
|
|
|
|
'error_code' => '',
|
|
|
|
|
'result' => [
|
|
|
|
|
'uuid' => $wrongIdentityUuid,
|
|
|
|
|
'hostname' => 'prepared-identity.example',
|
|
|
|
|
'settings_changed' => true,
|
|
|
|
|
],
|
|
|
|
|
]);
|
|
|
|
|
$preparedWrongIdentity = CloudJob::findOrFail($wrongIdentityId);
|
|
|
|
|
$wrongIdentityEvent = Capsule::table(
|
|
|
|
|
'cloud_agent_completion_events'
|
|
|
|
|
)->where('event_id', $wrongIdentityEventId)->first();
|
|
|
|
|
$assert($acceptedWrongIdentity === [
|
|
|
|
|
'accepted' => true,
|
|
|
|
|
'disposition' => 'rejected',
|
|
|
|
|
]
|
|
|
|
|
&& (string) $wrongIdentityEvent->disposition === 'rejected'
|
|
|
|
|
&& (string) $preparedWrongIdentity->effect_resolution === 'unknown'
|
|
|
|
|
&& (string) $preparedWrongIdentity->effect_resolution_reason
|
|
|
|
|
=== $preparedWrongIdentityMarker
|
|
|
|
|
&& (bool) $preparedWrongIdentity->finalization_incident
|
|
|
|
|
&& (string) $preparedWrongIdentity->finalization_error_code
|
|
|
|
|
=== 'recovery_operation_identity_conflict',
|
|
|
|
|
'A definitive wrong-operation receipt was not isolated from prepared recovery.');
|
|
|
|
|
Capsule::table('cloud_jobs')->where('id', $wrongIdentityId)->update([
|
|
|
|
|
'available_at' => 0,
|
|
|
|
|
'lease_token' => null,
|
|
|
|
|
'lease_until' => 0,
|
|
|
|
|
]);
|
|
|
|
|
$beforeWrongIdentityResume = count($client->requests);
|
|
|
|
|
$assert((new CloudAutomaticOperationRecoveryService())->tick(),
|
|
|
|
|
'Prepared recovery did not resume after quarantining a wrong receipt.');
|
|
|
|
|
$preparedWrongIdentity = CloudJob::findOrFail($wrongIdentityId);
|
|
|
|
|
$wrongIdentityEvent = Capsule::table(
|
|
|
|
|
'cloud_agent_completion_events'
|
|
|
|
|
)->where('event_id', $wrongIdentityEventId)->first();
|
|
|
|
|
$assert((string) $preparedWrongIdentity->effect_resolution === 'applied'
|
|
|
|
|
&& !(bool) $preparedWrongIdentity->finalization_incident
|
|
|
|
|
&& $preparedWrongIdentity->finalization_error_code === null
|
|
|
|
|
&& (string) $wrongIdentityEvent->disposition === 'rejected'
|
|
|
|
|
&& count($client->requests) === $beforeWrongIdentityResume,
|
|
|
|
|
'The quarantined wrong receipt starved or contaminated prepared convergence.');
|
|
|
|
|
|
|
|
|
|
// Failed journal + dual settings markers: prepare first, then resolve the
|
|
|
|
|
// fence. Simulate a lost POST response and prove one GET recovers it.
|
|
|
|
|
[$lostId, $lostUuid] = $insertSettingsJob(
|
|
|
|
|
str_repeat('2', 32),
|
|
|
|
|
['hostname' => 'recovered.example']
|
|
|
|
|
);
|
|
|
|
|
$lostAudit = $audit($lostId);
|
|
|
|
|
$lostFenceRequest = null;
|
|
|
|
|
$client->responses[] = $inspection(
|
|
|
|
|
str_repeat('2', 32),
|
|
|
|
|
'failed',
|
|
|
|
|
'applied',
|
|
|
|
|
[
|
|
|
|
|
'uuid' => $lostUuid,
|
|
|
|
|
'hostname' => 'recovered.example',
|
|
|
|
|
'status' => 'shutoff',
|
|
|
|
|
'settings_changed' => true,
|
|
|
|
|
]
|
|
|
|
|
);
|
|
|
|
|
$client->responses[] = $notFound();
|
|
|
|
|
$client->responses[] = static function (array $request) use (
|
|
|
|
|
&$lostFenceRequest
|
|
|
|
|
): Throwable {
|
|
|
|
|
$lostFenceRequest = $request;
|
|
|
|
|
return new RuntimeException('response lost after durable commit');
|
|
|
|
|
};
|
|
|
|
|
$client->responses[] = static function (array $request) use (
|
|
|
|
|
&$lostFenceRequest,
|
|
|
|
|
$fenceResponse
|
|
|
|
|
): array {
|
|
|
|
|
$read = $lostFenceRequest;
|
|
|
|
|
$read['method'] = $request['method'];
|
|
|
|
|
$read['target'] = $request['target'];
|
|
|
|
|
return $fenceResponse($read, time());
|
|
|
|
|
};
|
|
|
|
|
$beforeLostRequests = count($client->requests);
|
|
|
|
|
$assert((new CloudAutomaticOperationRecoveryService())->tick(),
|
|
|
|
|
'The worker did not recover a failed-journal applied task.');
|
|
|
|
|
$lost = CloudJob::find($lostId);
|
|
|
|
|
$lostSpec = json_decode(
|
|
|
|
|
(string) Capsule::table('cloud_instances')
|
|
|
|
|
->where('id', $lostId)->value('spec_json'),
|
|
|
|
|
true,
|
|
|
|
|
flags: JSON_THROW_ON_ERROR
|
|
|
|
|
);
|
|
|
|
|
$lostRequests = array_slice(
|
|
|
|
|
$client->requests,
|
|
|
|
|
$beforeLostRequests
|
|
|
|
|
);
|
|
|
|
|
$assert((string) $lost->effect_resolution === 'applied'
|
|
|
|
|
&& $audit($lostId) === $lostAudit
|
|
|
|
|
&& ($lostSpec['guest_hostname'] ?? null)
|
|
|
|
|
=== 'recovered.example'
|
|
|
|
|
&& count(array_filter(
|
|
|
|
|
$lostRequests,
|
|
|
|
|
static fn (array $request): bool =>
|
|
|
|
|
$request['method'] === 'POST'
|
|
|
|
|
)) === 1
|
|
|
|
|
&& count($lostRequests) === 4
|
|
|
|
|
&& !CloudJobService::blockingQueryForInstance($lostId)->exists(),
|
|
|
|
|
'Lost fence response recovery was not single-write and convergent.');
|
|
|
|
|
|
|
|
|
|
// Exact absence markers prove a settings request did not commit. PHP has
|
|
|
|
|
// no instance ledger delta, but still prepares before releasing the fence.
|
|
|
|
|
[$absentId] = $insertSettingsJob(
|
|
|
|
|
str_repeat('3', 32),
|
|
|
|
|
['hostname' => 'not-applied.example']
|
|
|
|
|
);
|
|
|
|
|
$absentAudit = $audit($absentId);
|
|
|
|
|
$absentSpec = (string) Capsule::table('cloud_instances')
|
|
|
|
|
->where('id', $absentId)->value('spec_json');
|
|
|
|
|
$client->responses[] = $inspection(
|
|
|
|
|
str_repeat('3', 32),
|
|
|
|
|
'failed',
|
|
|
|
|
'not_applied'
|
|
|
|
|
);
|
|
|
|
|
$client->responses[] = $notFound();
|
|
|
|
|
$client->responses[] = static fn (array $request): array =>
|
|
|
|
|
$fenceResponse($request, time());
|
|
|
|
|
$assert((new CloudAutomaticOperationRecoveryService())->tick(),
|
|
|
|
|
'The worker did not converge exact not-applied evidence.');
|
|
|
|
|
$absent = CloudJob::find($absentId);
|
|
|
|
|
$assert((string) $absent->effect_resolution === 'not_applied'
|
|
|
|
|
&& (int) $absent->effect_resolved_by === 0
|
|
|
|
|
&& $audit($absentId) === $absentAudit
|
|
|
|
|
&& (string) Capsule::table('cloud_instances')
|
|
|
|
|
->where('id', $absentId)->value('spec_json') === $absentSpec
|
|
|
|
|
&& !CloudJobService::blockingQueryForInstance($absentId)->exists(),
|
|
|
|
|
'Not-applied recovery changed the ledger or retained its blocker.');
|
|
|
|
|
|
|
|
|
|
// An immutable failed Journal row which explicitly records no possible
|
|
|
|
|
// effect has no Agent fence. Its bounded original error code is evidence,
|
|
|
|
|
// not a reason to invent a settings_not_committed fence marker.
|
|
|
|
|
[$exactAbsentId] = $insertSettingsJob(
|
|
|
|
|
str_repeat('9', 32),
|
|
|
|
|
['hostname' => 'exact-not-applied.example']
|
|
|
|
|
);
|
|
|
|
|
$exactAbsentAudit = $audit($exactAbsentId);
|
|
|
|
|
$client->responses[] = $inspection(
|
|
|
|
|
str_repeat('9', 32),
|
|
|
|
|
'failed',
|
|
|
|
|
'not_applied',
|
|
|
|
|
null,
|
|
|
|
|
false,
|
|
|
|
|
'cloudinit_validation_failed'
|
|
|
|
|
);
|
|
|
|
|
$beforeExactAbsent = count($client->requests);
|
|
|
|
|
$assert((new CloudAutomaticOperationRecoveryService())->tick(),
|
|
|
|
|
'The worker did not converge exact no-effect Journal evidence.');
|
|
|
|
|
$exactAbsentRequests = array_slice(
|
|
|
|
|
$client->requests,
|
|
|
|
|
$beforeExactAbsent
|
|
|
|
|
);
|
|
|
|
|
$exactAbsent = CloudJob::find($exactAbsentId);
|
|
|
|
|
$assert((string) $exactAbsent->effect_resolution === 'not_applied'
|
|
|
|
|
&& $audit($exactAbsentId) === $exactAbsentAudit
|
|
|
|
|
&& !CloudJobService::blockingQueryForInstance(
|
|
|
|
|
$exactAbsentId
|
|
|
|
|
)->exists()
|
|
|
|
|
&& count($exactAbsentRequests) === 1
|
|
|
|
|
&& $exactAbsentRequests[0]['method'] === 'GET'
|
|
|
|
|
&& str_ends_with($exactAbsentRequests[0]['target'], '/inspect'),
|
|
|
|
|
'Exact no-effect Journal recovery contacted a fence endpoint.');
|
|
|
|
|
|
|
|
|
|
// A Journal fence already resolved outside this coordinator is not the
|
|
|
|
|
// same thing as an operation which never needed a fence. Without PHP's
|
|
|
|
|
// matching durable prepare marker, automatic recovery must not accept the
|
|
|
|
|
// possibly conflicting Agent verdict.
|
|
|
|
|
[$externallyResolvedId] = $insertSettingsJob(
|
|
|
|
|
str_repeat('a', 32),
|
|
|
|
|
['hostname' => 'externally-resolved.example']
|
|
|
|
|
);
|
|
|
|
|
$client->responses[] = $inspection(
|
|
|
|
|
str_repeat('a', 32),
|
|
|
|
|
'failed',
|
|
|
|
|
'not_applied',
|
|
|
|
|
null,
|
|
|
|
|
false,
|
|
|
|
|
'settings_not_committed',
|
|
|
|
|
'resolved'
|
|
|
|
|
);
|
|
|
|
|
$beforeExternallyResolved = count($client->requests);
|
|
|
|
|
$assert((new CloudAutomaticOperationRecoveryService())->tick(),
|
|
|
|
|
'The externally resolved fence was not inspected.');
|
|
|
|
|
$externallyResolved = CloudJob::find($externallyResolvedId);
|
|
|
|
|
$externallyResolvedRequests = array_slice(
|
|
|
|
|
$client->requests,
|
|
|
|
|
$beforeExternallyResolved
|
|
|
|
|
);
|
|
|
|
|
$assert((string) $externallyResolved->effect_resolution === 'unknown'
|
|
|
|
|
&& (int) $externallyResolved->available_at >= time() + 290
|
|
|
|
|
&& CloudJobService::blockingQueryForInstance(
|
|
|
|
|
$externallyResolvedId
|
|
|
|
|
)->exists()
|
|
|
|
|
&& count($externallyResolvedRequests) === 1
|
|
|
|
|
&& str_ends_with(
|
|
|
|
|
$externallyResolvedRequests[0]['target'],
|
|
|
|
|
'/inspect'
|
|
|
|
|
),
|
|
|
|
|
'An unprepared externally resolved fence bypassed PHP recovery intent.');
|
|
|
|
|
|
|
|
|
|
// Partial and inconclusive evidence stay blocked and receive a five-minute
|
|
|
|
|
// retry delay. Two tasks also prove each worker tick handles only one.
|
|
|
|
|
[$partialId] = $insertSettingsJob(
|
|
|
|
|
str_repeat('4', 32),
|
|
|
|
|
['hostname' => 'partial.example']
|
|
|
|
|
);
|
|
|
|
|
[$inconclusiveId] = $insertSettingsJob(
|
|
|
|
|
str_repeat('5', 32),
|
|
|
|
|
['hostname' => 'inconclusive.example']
|
|
|
|
|
);
|
|
|
|
|
$client->responses[] = $inspection(
|
|
|
|
|
str_repeat('4', 32),
|
|
|
|
|
'failed',
|
|
|
|
|
'partial'
|
|
|
|
|
);
|
|
|
|
|
$client->responses[] = $inspection(
|
|
|
|
|
str_repeat('5', 32),
|
|
|
|
|
'failed',
|
|
|
|
|
'inconclusive'
|
|
|
|
|
);
|
|
|
|
|
$assert((new CloudAutomaticOperationRecoveryService())->tick(),
|
|
|
|
|
'The partial-evidence task was not inspected.');
|
|
|
|
|
$partial = CloudJob::find($partialId);
|
|
|
|
|
$notYetClaimed = CloudJob::find($inconclusiveId);
|
|
|
|
|
$assert((string) $partial->effect_resolution === 'unknown'
|
|
|
|
|
&& (int) $partial->available_at >= time() + 290
|
|
|
|
|
&& CloudJobService::blockingQueryForInstance($partialId)->exists()
|
|
|
|
|
&& (int) $notYetClaimed->available_at === 0,
|
|
|
|
|
'One worker tick did not stop after one inconclusive recovery.');
|
|
|
|
|
$assert((new CloudAutomaticOperationRecoveryService())->tick(),
|
|
|
|
|
'The inconclusive-evidence task was not inspected.');
|
|
|
|
|
$inconclusive = CloudJob::find($inconclusiveId);
|
|
|
|
|
$assert((string) $inconclusive->effect_resolution === 'unknown'
|
|
|
|
|
&& (int) $inconclusive->available_at >= time() + 290
|
|
|
|
|
&& CloudJobService::blockingQueryForInstance(
|
|
|
|
|
$inconclusiveId
|
|
|
|
|
)->exists(),
|
|
|
|
|
'Inconclusive evidence released its blocker or hot-looped.');
|
|
|
|
|
|
|
|
|
|
// Agent downtime backs off without changing the immutable task outcome.
|
|
|
|
|
[$offlineId] = $insertSettingsJob(
|
|
|
|
|
str_repeat('6', 32),
|
|
|
|
|
['hostname' => 'offline.example']
|
|
|
|
|
);
|
|
|
|
|
$offlineAudit = $audit($offlineId);
|
|
|
|
|
$client->responses[] = new CloudAgentError(
|
|
|
|
|
'offline', 503, 'agent_unavailable', true, 503
|
|
|
|
|
);
|
|
|
|
|
$assert((new CloudAutomaticOperationRecoveryService())->tick(),
|
|
|
|
|
'The offline recovery task was not claimed.');
|
|
|
|
|
$offline = CloudJob::find($offlineId);
|
|
|
|
|
$assert((string) $offline->effect_resolution === 'unknown'
|
|
|
|
|
&& $audit($offlineId) === $offlineAudit
|
|
|
|
|
&& $offline->lease_token === null
|
|
|
|
|
&& (int) $offline->available_at >= time() + 290
|
|
|
|
|
&& CloudJobService::blockingQueryForInstance($offlineId)->exists(),
|
|
|
|
|
'Agent downtime changed audit state or did not back off.');
|
|
|
|
|
|
|
|
|
|
// A retried low-ID offline host must not repeatedly jump ahead of a
|
|
|
|
|
// never-attempted higher-ID task. Persisted available_at age, then ID,
|
|
|
|
|
// defines the recovery queue order.
|
|
|
|
|
Capsule::table('cloud_jobs')->where('id', $offlineId)->update([
|
|
|
|
|
'available_at' => time() - 1,
|
|
|
|
|
'lease_until' => 0,
|
|
|
|
|
'lease_token' => null,
|
|
|
|
|
]);
|
|
|
|
|
[$fairId, $fairUuid] = $insertSettingsJob(
|
|
|
|
|
str_repeat('d', 32),
|
|
|
|
|
['hostname' => 'fair-queue.example']
|
|
|
|
|
);
|
|
|
|
|
$client->responses[] = $inspection(
|
|
|
|
|
str_repeat('d', 32),
|
|
|
|
|
'succeeded',
|
|
|
|
|
'applied',
|
|
|
|
|
[
|
|
|
|
|
'uuid' => $fairUuid,
|
|
|
|
|
'hostname' => 'fair-queue.example',
|
|
|
|
|
'settings_changed' => true,
|
|
|
|
|
]
|
|
|
|
|
);
|
|
|
|
|
$assert((new CloudAutomaticOperationRecoveryService())->tick(),
|
|
|
|
|
'The recovery queue did not claim a fair successor.');
|
|
|
|
|
$assert((string) CloudJob::find($fairId)->effect_resolution === 'applied'
|
|
|
|
|
&& (string) CloudJob::find($offlineId)->effect_resolution === 'unknown',
|
|
|
|
|
'A retried low-ID offline task starved a never-attempted task.');
|
|
|
|
|
Capsule::table('cloud_jobs')->where('id', $offlineId)->update([
|
|
|
|
|
'available_at' => time() + 300,
|
|
|
|
|
]);
|
|
|
|
|
|
|
|
|
|
// Attempt count is audit metadata only. The same operation remains in the
|
|
|
|
|
// recovery queue beyond the former 12-attempt boundary.
|
|
|
|
|
[$limitedId] = $insertSettingsJob(
|
|
|
|
|
str_repeat('c', 32),
|
|
|
|
|
['hostname' => 'retry-limit.example']
|
|
|
|
|
);
|
|
|
|
|
$beforeLimitedRequests = count($client->requests);
|
|
|
|
|
for ($attempt = 1; $attempt <= 13; $attempt++) {
|
|
|
|
|
Capsule::table('cloud_jobs')->where('id', $limitedId)->update([
|
|
|
|
|
'available_at' => 0,
|
|
|
|
|
'lease_until' => 0,
|
|
|
|
|
'lease_token' => null,
|
|
|
|
|
]);
|
|
|
|
|
$client->responses[] = new CloudAgentError(
|
|
|
|
|
'offline', 503, 'agent_unavailable', true, 503
|
|
|
|
|
);
|
|
|
|
|
$assert((new CloudAutomaticOperationRecoveryService())->tick(),
|
|
|
|
|
'A persistent recovery attempt was not claimed.');
|
|
|
|
|
}
|
|
|
|
|
$limited = CloudJob::find($limitedId);
|
|
|
|
|
$limitedPayloadBefore = (string) $limited->sealed_payload;
|
|
|
|
|
$limitedRequestCount = count($client->requests);
|
|
|
|
|
$assert((bool) $limited->finalization_incident
|
|
|
|
|
&& (string) $limited->finalization_error_code
|
|
|
|
|
=== 'recovery_waiting_connection'
|
|
|
|
|
&& preg_match(
|
|
|
|
|
'/^automatic_recovery:v1:[1-9][0-9]{0,10}:13:pending$/D',
|
|
|
|
|
(string) $limited->effect_resolution_reason
|
|
|
|
|
) === 1
|
|
|
|
|
&& (string) $limited->effect_resolution === 'unknown'
|
|
|
|
|
&& (string) $limited->sealed_payload === $limitedPayloadBefore
|
|
|
|
|
&& CloudJobService::blockingQueryForInstance($limitedId)->exists()
|
|
|
|
|
&& $limitedRequestCount - $beforeLimitedRequests === 13
|
|
|
|
|
&& (int) $limited->available_at >= time() + 290,
|
|
|
|
|
'Recovery did not remain queued beyond the former attempt limit.');
|
|
|
|
|
|
|
|
|
|
// First pass prepares PHP, commits the Agent fence, then loses both the
|
|
|
|
|
// POST response and its verification GET. A later pass must read that
|
|
|
|
|
// existing fence and commit PHP without depending on a new host inspect.
|
|
|
|
|
[$crashId, $crashUuid] = $insertSettingsJob(
|
|
|
|
|
str_repeat('7', 32),
|
|
|
|
|
['hostname' => 'crash-recovered.example']
|
|
|
|
|
);
|
|
|
|
|
Capsule::table('cloud_jobs')->where('id', $crashId)->update([
|
|
|
|
|
// The pending inspection budget is about to expire. A successful
|
|
|
|
|
// prepare must start a separate crash-resume budget instead of
|
|
|
|
|
// stranding the durable intent at attempt 12.
|
|
|
|
|
'effect_resolution_reason' => 'automatic_recovery:v1:'
|
|
|
|
|
. (time() - 300) . ':11:pending',
|
|
|
|
|
]);
|
|
|
|
|
$crashInspection = $inspection(
|
|
|
|
|
str_repeat('7', 32),
|
|
|
|
|
'failed',
|
|
|
|
|
'applied',
|
|
|
|
|
[
|
|
|
|
|
'uuid' => $crashUuid,
|
|
|
|
|
'hostname' => 'crash-recovered.example',
|
|
|
|
|
'settings_changed' => true,
|
|
|
|
|
]
|
|
|
|
|
);
|
|
|
|
|
$client->responses[] = $crashInspection;
|
|
|
|
|
$client->responses[] = $notFound();
|
|
|
|
|
$crashFenceRequest = null;
|
|
|
|
|
$client->responses[] = static function (array $request) use (
|
|
|
|
|
&$crashFenceRequest
|
|
|
|
|
): Throwable {
|
|
|
|
|
$crashFenceRequest = $request;
|
|
|
|
|
return new RuntimeException('fence committed before response loss');
|
|
|
|
|
};
|
|
|
|
|
$client->responses[] = new CloudAgentError(
|
|
|
|
|
'verification read offline', 503, 'agent_unavailable', true, 503
|
|
|
|
|
);
|
|
|
|
|
$assert((new CloudAutomaticOperationRecoveryService())->tick(),
|
|
|
|
|
'The crash-window fixture was not prepared.');
|
|
|
|
|
$prepared = CloudJob::find($crashId);
|
|
|
|
|
$assert(str_starts_with(
|
|
|
|
|
(string) $prepared->effect_resolution_reason,
|
|
|
|
|
'automatic_recovery:v1:'
|
|
|
|
|
)
|
|
|
|
|
&& preg_match(
|
|
|
|
|
'/^automatic_recovery:v1:[1-9][0-9]{0,10}:1:prepared:/D',
|
|
|
|
|
(string) $prepared->effect_resolution_reason
|
|
|
|
|
) === 1
|
|
|
|
|
&& str_contains(
|
|
|
|
|
(string) $prepared->effect_resolution_reason,
|
|
|
|
|
':prepared:automatic_applied_failed_fence:'
|
|
|
|
|
)
|
|
|
|
|
&& (string) $prepared->effect_resolution === 'unknown'
|
|
|
|
|
&& CloudJobService::blockingQueryForInstance($crashId)->exists(),
|
|
|
|
|
'Prepare failure window did not retain its blocker and intent.');
|
|
|
|
|
Capsule::table('cloud_jobs')->where('id', $crashId)->update([
|
|
|
|
|
'available_at' => 0,
|
|
|
|
|
'lease_until' => 0,
|
|
|
|
|
'lease_token' => null,
|
|
|
|
|
]);
|
|
|
|
|
$client->responses[] = static function (array $request) use (
|
|
|
|
|
&$crashFenceRequest,
|
|
|
|
|
$fenceResponse
|
|
|
|
|
): array {
|
|
|
|
|
$read = $crashFenceRequest;
|
|
|
|
|
$read['method'] = $request['method'];
|
|
|
|
|
$read['target'] = $request['target'];
|
|
|
|
|
return $fenceResponse($read, time());
|
|
|
|
|
};
|
|
|
|
|
$beforeCrashRetry = count($client->requests);
|
|
|
|
|
$assert((new CloudAutomaticOperationRecoveryService())->tick(),
|
|
|
|
|
'Prepared recovery did not resume after the crash window.');
|
|
|
|
|
$crashRetryRequests = array_slice(
|
|
|
|
|
$client->requests,
|
|
|
|
|
$beforeCrashRetry
|
|
|
|
|
);
|
|
|
|
|
$assert((string) CloudJob::find($crashId)->effect_resolution === 'applied'
|
|
|
|
|
&& count(array_filter(
|
|
|
|
|
$crashRetryRequests,
|
|
|
|
|
static fn (array $request): bool =>
|
|
|
|
|
$request['method'] === 'POST'
|
|
|
|
|
)) === 0
|
|
|
|
|
&& !CloudJobService::blockingQueryForInstance($crashId)->exists()
|
|
|
|
|
&& count($crashRetryRequests) === 1
|
|
|
|
|
&& str_ends_with(
|
|
|
|
|
$crashRetryRequests[0]['target'],
|
|
|
|
|
'/fence-resolution'
|
|
|
|
|
),
|
|
|
|
|
'Prepared recovery re-inspected/replayed or failed to commit.');
|
|
|
|
|
|
|
|
|
|
// Prepared crash-resume also remains eligible beyond the former limit.
|
|
|
|
|
[$preparedLimitId] = $insertSettingsJob(
|
|
|
|
|
str_repeat('e', 32),
|
|
|
|
|
['hostname' => 'prepared-limit.example']
|
|
|
|
|
);
|
|
|
|
|
$client->responses[] = $inspection(
|
|
|
|
|
str_repeat('e', 32),
|
|
|
|
|
'failed',
|
|
|
|
|
'not_applied'
|
|
|
|
|
);
|
|
|
|
|
$client->responses[] = $notFound();
|
|
|
|
|
$client->responses[] = new RuntimeException(
|
|
|
|
|
'fence response lost before verification'
|
|
|
|
|
);
|
|
|
|
|
$client->responses[] = new CloudAgentError(
|
|
|
|
|
'verification read offline',
|
|
|
|
|
503,
|
|
|
|
|
'agent_unavailable',
|
|
|
|
|
true,
|
|
|
|
|
503
|
|
|
|
|
);
|
|
|
|
|
$beforePreparedLimit = count($client->requests);
|
|
|
|
|
$assert((new CloudAutomaticOperationRecoveryService())->tick(),
|
|
|
|
|
'The prepared-limit fixture did not enter its resume stage.');
|
|
|
|
|
$preparedLimitFirst = CloudJob::find($preparedLimitId);
|
|
|
|
|
$assert(preg_match(
|
|
|
|
|
'/^automatic_recovery:v1:[1-9][0-9]{0,10}:1:prepared:automatic_not_applied_failed_fence:/D',
|
|
|
|
|
(string) $preparedLimitFirst->effect_resolution_reason
|
|
|
|
|
) === 1,
|
|
|
|
|
'Preparing did not reset the independent resume budget.');
|
|
|
|
|
for ($attempt = 2; $attempt <= 13; $attempt++) {
|
|
|
|
|
Capsule::table('cloud_jobs')->where('id', $preparedLimitId)->update([
|
|
|
|
|
'available_at' => 0,
|
|
|
|
|
'lease_until' => 0,
|
|
|
|
|
'lease_token' => null,
|
|
|
|
|
]);
|
|
|
|
|
$client->responses[] = new CloudAgentError(
|
|
|
|
|
'prepared fence read offline',
|
|
|
|
|
503,
|
|
|
|
|
'agent_unavailable',
|
|
|
|
|
true,
|
|
|
|
|
503
|
|
|
|
|
);
|
|
|
|
|
$assert((new CloudAutomaticOperationRecoveryService())->tick(),
|
|
|
|
|
'A persistent prepared-resume attempt was not claimed.');
|
|
|
|
|
}
|
|
|
|
|
$preparedLimited = CloudJob::find($preparedLimitId);
|
|
|
|
|
$preparedLimitRequestCount = count($client->requests);
|
|
|
|
|
$preparedLimitPayload = (string) $preparedLimited->sealed_payload;
|
|
|
|
|
$preparedLimitEscrow = CloudSecrets::decrypt(
|
|
|
|
|
$preparedLimitPayload,
|
|
|
|
|
'job-payload'
|
|
|
|
|
);
|
|
|
|
|
$preparedLimitRequests = array_slice(
|
|
|
|
|
$client->requests,
|
|
|
|
|
$beforePreparedLimit
|
|
|
|
|
);
|
|
|
|
|
$preparedLimitFenceWrite = null;
|
|
|
|
|
foreach ($preparedLimitRequests as $request) {
|
|
|
|
|
if ($request['method'] === 'POST'
|
|
|
|
|
&& str_ends_with($request['target'], '/resolve-fence')) {
|
|
|
|
|
$preparedLimitFenceWrite = $request;
|
|
|
|
|
break;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
$assert((bool) $preparedLimited->finalization_incident
|
|
|
|
|
&& (string) $preparedLimited->finalization_error_code
|
|
|
|
|
=== 'recovery_waiting_connection'
|
|
|
|
|
&& preg_match(
|
|
|
|
|
'/^automatic_recovery:v1:[1-9][0-9]{0,10}:13:prepared:/D',
|
|
|
|
|
(string) $preparedLimited->effect_resolution_reason
|
|
|
|
|
) === 1
|
|
|
|
|
&& (string) $preparedLimited->effect_resolution === 'unknown'
|
|
|
|
|
&& CloudJobService::blockingQueryForInstance(
|
|
|
|
|
$preparedLimitId
|
|
|
|
|
)->exists()
|
|
|
|
|
&& count($client->requests) - $beforePreparedLimit === 16
|
|
|
|
|
&& is_array($preparedLimitEscrow['_operation_recovery'] ?? null)
|
|
|
|
|
&& ($preparedLimitEscrow['_operation_recovery']['operation']
|
|
|
|
|
['operation_id'] ?? null) === str_repeat('e', 32)
|
|
|
|
|
&& ($preparedLimitEscrow['_operation_recovery']['operation']
|
|
|
|
|
['status'] ?? null) === 'failed'
|
|
|
|
|
&& is_array($preparedLimitFenceWrite)
|
|
|
|
|
&& (int) $preparedLimited->available_at >= time() + 290,
|
|
|
|
|
'Prepared recovery did not remain queued beyond the former limit.');
|
|
|
|
|
|
|
|
|
|
// A fourteenth pass can still finish the original crash-resume protocol.
|
|
|
|
|
// The retry marker contains only hashes, so this succeeds only if the
|
|
|
|
|
// worker reconstructs the exact failed operation from the encrypted
|
|
|
|
|
// _operation_recovery envelope retained across attempts 1 through 13.
|
|
|
|
|
Capsule::table('cloud_jobs')->where('id', $preparedLimitId)->update([
|
|
|
|
|
'available_at' => 0,
|
|
|
|
|
'lease_until' => 0,
|
|
|
|
|
'lease_token' => null,
|
|
|
|
|
]);
|
|
|
|
|
$client->responses[] = static fn (): array =>
|
|
|
|
|
$fenceResponse($preparedLimitFenceWrite, time());
|
|
|
|
|
$beforePreparedLimitResume = count($client->requests);
|
|
|
|
|
$assert((new CloudAutomaticOperationRecoveryService())->tick(),
|
|
|
|
|
'Prepared recovery was not claimable after attempt thirteen.');
|
|
|
|
|
$preparedLimitResumeRequests = array_slice(
|
|
|
|
|
$client->requests,
|
|
|
|
|
$beforePreparedLimitResume
|
|
|
|
|
);
|
|
|
|
|
$preparedLimitResolved = CloudJob::find($preparedLimitId);
|
|
|
|
|
$preparedLimitFinalPayload = CloudSecrets::decrypt(
|
|
|
|
|
(string) $preparedLimitResolved->sealed_payload,
|
|
|
|
|
'job-payload'
|
|
|
|
|
);
|
|
|
|
|
$assert((string) $preparedLimitResolved->effect_resolution
|
|
|
|
|
=== 'not_applied'
|
|
|
|
|
&& !(bool) $preparedLimitResolved->finalization_incident
|
|
|
|
|
&& $preparedLimitResolved->finalization_error_code === null
|
|
|
|
|
&& !array_key_exists(
|
|
|
|
|
'_operation_recovery',
|
|
|
|
|
$preparedLimitFinalPayload
|
|
|
|
|
)
|
|
|
|
|
&& !CloudJobService::blockingQueryForInstance(
|
|
|
|
|
$preparedLimitId
|
|
|
|
|
)->exists()
|
|
|
|
|
&& count($preparedLimitResumeRequests) === 1
|
|
|
|
|
&& $preparedLimitResumeRequests[0]['method'] === 'GET'
|
|
|
|
|
&& str_ends_with(
|
|
|
|
|
$preparedLimitResumeRequests[0]['target'],
|
|
|
|
|
'/fence-resolution'
|
|
|
|
|
),
|
|
|
|
|
'Attempt thirteen stranded encrypted prepared recovery state.');
|
|
|
|
|
|
|
|
|
|
// A timed-out graceful delete may later be proven not applied because the
|
|
|
|
|
// Agent requires an explicitly authorized force generation. Recovery must
|
|
|
|
|
// preserve that exact business error: it is the immutable authorization
|
|
|
|
|
// anchor for enqueueForceDelete(), not an instance-state contradiction.
|
|
|
|
|
[$forceRequiredId, $forceRequiredUuid] = $insertSettingsJob(
|
|
|
|
|
str_repeat('ab', 16),
|
|
|
|
|
['hostname' => 'delete-force-required.example']
|
|
|
|
|
);
|
|
|
|
|
Capsule::table('cloud_jobs')->where('id', $forceRequiredId)->update([
|
|
|
|
|
'kind' => 'delete',
|
|
|
|
|
'sealed_payload' => CloudSecrets::encrypt([
|
|
|
|
|
'method' => 'DELETE',
|
|
|
|
|
'request_target' => '/v1/vms/' . $forceRequiredUuid,
|
|
|
|
|
'body' => ['remove_managed_disks' => true],
|
|
|
|
|
'owner_id' => 1,
|
|
|
|
|
'can_manage' => false,
|
|
|
|
|
], 'job-payload'),
|
|
|
|
|
]);
|
|
|
|
|
$beforeForceRequiredRecovery = count($client->requests);
|
|
|
|
|
$client->responses[] = $inspection(
|
|
|
|
|
str_repeat('ab', 16),
|
|
|
|
|
'failed',
|
|
|
|
|
'not_applied',
|
|
|
|
|
null,
|
|
|
|
|
false,
|
|
|
|
|
'force_required',
|
|
|
|
|
'not_required'
|
|
|
|
|
);
|
|
|
|
|
$assert((new CloudAutomaticOperationRecoveryService())->tick(),
|
|
|
|
|
'The force-required delete recovery was not claimed.');
|
|
|
|
|
$forceRequired = CloudJob::find($forceRequiredId);
|
|
|
|
|
$forceRequiredInstance = \App\Models\CloudInstance::find(
|
|
|
|
|
$forceRequiredId
|
|
|
|
|
);
|
|
|
|
|
$assert((string) $forceRequired->effect_resolution === 'not_applied'
|
|
|
|
|
&& (string) $forceRequired->error_code === 'force_required'
|
|
|
|
|
&& (string) $forceRequired->phase === 'terminal'
|
|
|
|
|
&& (string) $forceRequired->outcome === 'failed'
|
|
|
|
|
&& $forceRequired->dispatched_at !== null,
|
|
|
|
|
'Force-required inspection did not converge into its authorization anchor: '
|
|
|
|
|
. json_encode($forceRequired->toArray(), JSON_THROW_ON_ERROR));
|
|
|
|
|
$beforeForceJobCount = CloudJob::count();
|
|
|
|
|
Capsule::table('cloud_jobs')->where('id', $forceRequiredId)->update([
|
|
|
|
|
'finalization_incident' => 1,
|
|
|
|
|
'finalization_error_code' => 'recovery_ledger_conflict',
|
|
|
|
|
]);
|
|
|
|
|
try {
|
|
|
|
|
CloudJobService::enqueueForceDelete(
|
|
|
|
|
$forceRequired->fresh(),
|
|
|
|
|
[
|
|
|
|
|
'user_id' => 99,
|
|
|
|
|
'is_admin' => true,
|
|
|
|
|
'is_super_admin' => true,
|
|
|
|
|
'can_manage' => true,
|
|
|
|
|
]
|
|
|
|
|
);
|
|
|
|
|
throw new RuntimeException(
|
|
|
|
|
'A sticky recovery incident created a force-delete generation.'
|
|
|
|
|
);
|
|
|
|
|
} catch (CloudError $error) {
|
|
|
|
|
$assert($error->httpStatus === 409
|
|
|
|
|
&& CloudJob::count() === $beforeForceJobCount,
|
|
|
|
|
'Sticky force-delete rejection changed the immutable job ledger.');
|
|
|
|
|
}
|
|
|
|
|
Capsule::table('cloud_jobs')->where('id', $forceRequiredId)->update([
|
|
|
|
|
'finalization_incident' => 0,
|
|
|
|
|
'finalization_error_code' => null,
|
|
|
|
|
'receipt_deadline_at' => null,
|
|
|
|
|
]);
|
|
|
|
|
try {
|
|
|
|
|
CloudJobService::enqueueForceDelete(
|
|
|
|
|
$forceRequired->fresh(),
|
|
|
|
|
[
|
|
|
|
|
'user_id' => 99,
|
|
|
|
|
'is_admin' => true,
|
|
|
|
|
'is_super_admin' => true,
|
|
|
|
|
'can_manage' => true,
|
|
|
|
|
]
|
|
|
|
|
);
|
|
|
|
|
throw new RuntimeException(
|
|
|
|
|
'A damaged Agent envelope created a force-delete generation.'
|
|
|
|
|
);
|
|
|
|
|
} catch (CloudError $error) {
|
|
|
|
|
$assert($error->httpStatus === 409
|
|
|
|
|
&& CloudJob::count() === $beforeForceJobCount,
|
|
|
|
|
'Envelope force-delete rejection changed the immutable job ledger.');
|
|
|
|
|
}
|
|
|
|
|
Capsule::table('cloud_jobs')->where('id', $forceRequiredId)->update([
|
|
|
|
|
'receipt_deadline_at' => $now + 120,
|
|
|
|
|
]);
|
|
|
|
|
$forceRequired->refresh();
|
|
|
|
|
$forceDelete = CloudJobService::enqueueForceDelete(
|
|
|
|
|
$forceRequired,
|
|
|
|
|
[
|
|
|
|
|
'user_id' => 99,
|
|
|
|
|
'is_admin' => true,
|
|
|
|
|
'is_super_admin' => true,
|
|
|
|
|
'can_manage' => true,
|
|
|
|
|
]
|
|
|
|
|
);
|
|
|
|
|
$nextId = max($nextId, (int) $forceDelete->id + 1);
|
|
|
|
|
$forceDeletePayload = CloudSecrets::decrypt(
|
|
|
|
|
(string) $forceDelete->sealed_payload,
|
|
|
|
|
'job-payload'
|
|
|
|
|
);
|
|
|
|
|
$assert((string) $forceRequired->effect_resolution === 'not_applied'
|
|
|
|
|
&& (string) $forceRequired->error_code === 'force_required'
|
|
|
|
|
&& (string) $forceRequiredInstance->status === 'active'
|
|
|
|
|
&& !(bool) $forceRequired->finalization_incident
|
|
|
|
|
&& (string) $forceDelete->kind === 'delete_force'
|
|
|
|
|
&& ($forceDeletePayload['prior_job_id'] ?? null)
|
|
|
|
|
=== $forceRequiredId
|
|
|
|
|
&& ($forceDeletePayload['prior_operation_id'] ?? null)
|
|
|
|
|
=== str_repeat('ab', 16)
|
|
|
|
|
&& count($client->requests) === $beforeForceRequiredRecovery + 1
|
|
|
|
|
&& $client->requests[array_key_last($client->requests)]['method']
|
|
|
|
|
=== 'GET',
|
|
|
|
|
'Recovered force-required evidence did not preserve the separately authorized force-delete path.');
|
|
|
|
|
|
|
|
|
|
// Age is also audit metadata only; a six-hour-old operation is still
|
|
|
|
|
// inspected with the original operation ID.
|
|
|
|
|
[$agedId] = $insertSettingsJob(
|
|
|
|
|
str_repeat('f', 32),
|
|
|
|
|
['hostname' => 'age-limit.example']
|
|
|
|
|
);
|
|
|
|
|
Capsule::table('cloud_jobs')->where('id', $agedId)->update([
|
|
|
|
|
'effect_resolution_reason' => 'automatic_recovery:v1:'
|
|
|
|
|
. (time() - 21601) . ':1:pending',
|
|
|
|
|
]);
|
|
|
|
|
$beforeAged = count($client->requests);
|
|
|
|
|
$client->responses[] = new CloudAgentError(
|
|
|
|
|
'offline', 503, 'agent_unavailable', true, 503
|
|
|
|
|
);
|
|
|
|
|
$assert((new CloudAutomaticOperationRecoveryService())->tick(),
|
|
|
|
|
'A six-hour-old recovery was not claimed.');
|
|
|
|
|
$aged = CloudJob::find($agedId);
|
|
|
|
|
$assert((bool) $aged->finalization_incident
|
|
|
|
|
&& (string) $aged->finalization_error_code
|
|
|
|
|
=== 'recovery_waiting_connection'
|
|
|
|
|
&& preg_match(
|
|
|
|
|
'/^automatic_recovery:v1:[1-9][0-9]{0,10}:2:pending$/D',
|
|
|
|
|
(string) $aged->effect_resolution_reason
|
|
|
|
|
) === 1
|
|
|
|
|
&& (string) $aged->effect_resolution === 'unknown'
|
|
|
|
|
&& CloudJobService::blockingQueryForInstance($agedId)->exists()
|
|
|
|
|
&& count($client->requests) === $beforeAged + 1,
|
|
|
|
|
'An aged recovery stopped retrying or released safety.');
|
|
|
|
|
|
|
|
|
|
// A corrupt dispatch ledger remains a recovery candidate forever, but it
|
|
|
|
|
// cannot authorize either an Agent read or consumption of even an exact
|
|
|
|
|
// received completion. The only safe automatic action is to record the
|
|
|
|
|
// concrete incident and back off until an operator repairs the 0/1 fence.
|
|
|
|
|
[$invalidDispatchId, $invalidDispatchUuid] = $insertSettingsJob(
|
|
|
|
|
str_repeat('dc', 16),
|
|
|
|
|
['hostname' => 'invalid-dispatch-ledger.example']
|
|
|
|
|
);
|
|
|
|
|
$invalidDispatchEventId = 'automatic-invalid-dispatch-ledger';
|
|
|
|
|
Capsule::table('cloud_jobs')->where('id', $invalidDispatchId)->update([
|
|
|
|
|
'dispatch_attempts' => 2,
|
|
|
|
|
// Simulate a historical fail-open release which mistook the corrupt
|
|
|
|
|
// non-zero counter for a fully settled successful Agent mutation.
|
|
|
|
|
'outcome' => 'succeeded',
|
|
|
|
|
'status' => 'succeeded',
|
|
|
|
|
'error_code' => null,
|
|
|
|
|
'effect_may_have_applied' => 0,
|
|
|
|
|
'safe_to_resubmit' => 0,
|
|
|
|
|
'effect_resolution' => 'applied',
|
|
|
|
|
'completion_event_id' => $invalidDispatchEventId,
|
|
|
|
|
'available_at' => 0,
|
|
|
|
|
'lease_token' => null,
|
|
|
|
|
'lease_until' => 0,
|
|
|
|
|
'finalization_incident' => 0,
|
|
|
|
|
'finalization_error_code' => null,
|
|
|
|
|
]);
|
|
|
|
|
$invalidDispatchJob = CloudJob::findOrFail($invalidDispatchId);
|
|
|
|
|
$invalidDispatchEnvelope = (string) $invalidDispatchJob->sealed_payload;
|
|
|
|
|
$invalidDispatchTerminalAt = (int) $invalidDispatchJob->terminal_at;
|
|
|
|
|
$invalidDispatchResultJson = (string) $invalidDispatchJob->result_json;
|
|
|
|
|
$invalidDispatchIdentity = CloudOperationRecoveryIdentity::expected(
|
|
|
|
|
$invalidDispatchJob,
|
|
|
|
|
true
|
|
|
|
|
);
|
|
|
|
|
$invalidDispatchResult = [
|
|
|
|
|
'uuid' => $invalidDispatchUuid,
|
|
|
|
|
'hostname' => 'invalid-dispatch-ledger.example',
|
|
|
|
|
'settings_changed' => true,
|
|
|
|
|
];
|
|
|
|
|
$invalidDispatchWire = [
|
|
|
|
|
'event_id' => $invalidDispatchEventId,
|
|
|
|
|
'host_id' => 7,
|
|
|
|
|
'operation_id' => (string) $invalidDispatchJob->operation_id,
|
|
|
|
|
'action' => $invalidDispatchIdentity['action'],
|
|
|
|
|
'resource_id' => $invalidDispatchIdentity['resource_id'],
|
|
|
|
|
'request_fingerprint' =>
|
|
|
|
|
$invalidDispatchIdentity['request_fingerprint'],
|
|
|
|
|
'status' => 'succeeded',
|
|
|
|
|
'effect_may_have_applied' => false,
|
|
|
|
|
'safe_to_resubmit' => false,
|
|
|
|
|
'error_code' => '',
|
|
|
|
|
'result' => $invalidDispatchResult,
|
|
|
|
|
];
|
|
|
|
|
Capsule::table('cloud_agent_completion_events')->insert([
|
|
|
|
|
'event_id' => $invalidDispatchWire['event_id'],
|
|
|
|
|
'host_id' => 7,
|
|
|
|
|
'operation_id' => $invalidDispatchWire['operation_id'],
|
|
|
|
|
'payload_sha256' => hash('sha256', json_encode(
|
|
|
|
|
$invalidDispatchWire,
|
|
|
|
|
JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES
|
|
|
|
|
| JSON_UNESCAPED_UNICODE
|
|
|
|
|
)),
|
|
|
|
|
'outcome' => 'succeeded',
|
|
|
|
|
'effect_may_have_applied' => 0,
|
|
|
|
|
'safe_to_resubmit' => 0,
|
|
|
|
|
'error_code' => '',
|
|
|
|
|
'result_json' => json_encode([
|
|
|
|
|
'schema' => 'virt_cloud_completion_event',
|
|
|
|
|
'version' => 1,
|
|
|
|
|
'identity' => $invalidDispatchIdentity,
|
|
|
|
|
'result' => $invalidDispatchResult,
|
|
|
|
|
], JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES
|
|
|
|
|
| JSON_UNESCAPED_UNICODE),
|
|
|
|
|
'disposition' => 'received',
|
|
|
|
|
'received_at' => time(),
|
|
|
|
|
'applied_at' => null,
|
|
|
|
|
]);
|
|
|
|
|
$beforeInvalidDispatchRequests = count($client->requests);
|
|
|
|
|
$assert((new CloudAutomaticOperationRecoveryService())->tick(),
|
|
|
|
|
'An invalid dispatch ledger permanently left the recovery queue.');
|
|
|
|
|
$invalidDispatchJob = CloudJob::findOrFail($invalidDispatchId);
|
|
|
|
|
$invalidDispatchEvent = Capsule::table(
|
|
|
|
|
'cloud_agent_completion_events'
|
|
|
|
|
)->where(
|
|
|
|
|
'event_id',
|
|
|
|
|
$invalidDispatchWire['event_id']
|
|
|
|
|
)->first();
|
|
|
|
|
$assert((string) $invalidDispatchJob->phase === 'terminal'
|
|
|
|
|
&& (string) $invalidDispatchJob->outcome === 'succeeded'
|
|
|
|
|
&& (string) $invalidDispatchJob->status === 'succeeded'
|
|
|
|
|
&& $invalidDispatchJob->error_code === null
|
|
|
|
|
&& !(bool) $invalidDispatchJob->effect_may_have_applied
|
|
|
|
|
&& (string) $invalidDispatchJob->effect_resolution === 'applied'
|
|
|
|
|
&& !(bool) $invalidDispatchJob->safe_to_resubmit
|
|
|
|
|
&& (int) $invalidDispatchJob->terminal_at
|
|
|
|
|
=== $invalidDispatchTerminalAt
|
|
|
|
|
&& (string) $invalidDispatchJob->result_json
|
|
|
|
|
=== $invalidDispatchResultJson
|
|
|
|
|
&& (string) $invalidDispatchJob->completion_event_id
|
|
|
|
|
=== $invalidDispatchEventId
|
|
|
|
|
&& (bool) $invalidDispatchJob->finalization_incident
|
|
|
|
|
&& (string) $invalidDispatchJob->finalization_error_code
|
|
|
|
|
=== 'recovery_dispatch_ledger_conflict'
|
|
|
|
|
&& (string) $invalidDispatchJob->sealed_payload
|
|
|
|
|
=== $invalidDispatchEnvelope
|
|
|
|
|
&& $invalidDispatchJob->lease_token === null
|
|
|
|
|
&& (int) $invalidDispatchJob->lease_until === 0
|
|
|
|
|
&& (int) $invalidDispatchJob->available_at >= time() + 290
|
|
|
|
|
&& (string) $invalidDispatchEvent->disposition === 'received'
|
|
|
|
|
&& $invalidDispatchEvent->applied_at === null
|
|
|
|
|
&& count($client->requests) === $beforeInvalidDispatchRequests,
|
|
|
|
|
'Invalid dispatch recovery contacted Agent, consumed evidence, or released its fence.');
|
|
|
|
|
|
|
|
|
|
// Once the durable ledger is repaired to the only valid dispatched state,
|
|
|
|
|
// the same unconsumed receipt must resume convergence without waiting for
|
|
|
|
|
// an Agent retransmission or issuing any Agent request.
|
|
|
|
|
Capsule::table('cloud_jobs')->where('id', $invalidDispatchId)->update([
|
|
|
|
|
'dispatch_attempts' => 1,
|
|
|
|
|
'available_at' => 0,
|
|
|
|
|
'lease_token' => null,
|
|
|
|
|
'lease_until' => 0,
|
|
|
|
|
]);
|
|
|
|
|
$assert((new CloudAutomaticOperationRecoveryService())->tick(),
|
|
|
|
|
'A repaired dispatch ledger could not resume its queued receipt.');
|
|
|
|
|
$repairedDispatchJob = CloudJob::findOrFail($invalidDispatchId);
|
|
|
|
|
$repairedDispatchEvent = Capsule::table(
|
|
|
|
|
'cloud_agent_completion_events'
|
|
|
|
|
)->where(
|
|
|
|
|
'event_id',
|
|
|
|
|
$invalidDispatchWire['event_id']
|
|
|
|
|
)->first();
|
|
|
|
|
$assert((string) $repairedDispatchJob->phase === 'terminal'
|
|
|
|
|
&& (string) $repairedDispatchJob->outcome === 'succeeded'
|
|
|
|
|
&& (string) $repairedDispatchJob->status === 'succeeded'
|
|
|
|
|
&& (int) $repairedDispatchJob->terminal_at
|
|
|
|
|
=== $invalidDispatchTerminalAt
|
|
|
|
|
&& $repairedDispatchJob->error_code === null
|
|
|
|
|
&& (string) $repairedDispatchJob->result_json
|
|
|
|
|
=== $invalidDispatchResultJson
|
|
|
|
|
&& (string) $repairedDispatchJob->effect_resolution === 'applied'
|
|
|
|
|
&& !(bool) $repairedDispatchJob->finalization_incident
|
|
|
|
|
&& $repairedDispatchJob->finalization_error_code === null
|
|
|
|
|
&& (string) $repairedDispatchEvent->disposition === 'applied'
|
|
|
|
|
&& (int) $repairedDispatchEvent->applied_at > 0
|
|
|
|
|
&& count($client->requests) === $beforeInvalidDispatchRequests,
|
|
|
|
|
'Repairing the dispatch ledger did not resume the exact stored completion.');
|
|
|
|
|
|
|
|
|
|
// An exact immutable receipt cannot authorize convergence when any other
|
|
|
|
|
// field in the dispatched Agent envelope is damaged. Claiming must keep
|
|
|
|
|
// the receipt untouched and record a local-only state conflict without an
|
|
|
|
|
// Agent request.
|
|
|
|
|
[$corruptReceiptEnvelopeId, $corruptReceiptEnvelopeUuid] =
|
|
|
|
|
$insertSettingsJob(
|
|
|
|
|
str_repeat('b1', 16),
|
|
|
|
|
['hostname' => 'corrupt-receipt-envelope.example']
|
|
|
|
|
);
|
|
|
|
|
$corruptReceiptEnvelopeJob = CloudJob::findOrFail(
|
|
|
|
|
$corruptReceiptEnvelopeId
|
|
|
|
|
);
|
|
|
|
|
$corruptReceiptEnvelopeIdentity =
|
|
|
|
|
CloudOperationRecoveryIdentity::expected(
|
|
|
|
|
$corruptReceiptEnvelopeJob,
|
|
|
|
|
true
|
|
|
|
|
);
|
|
|
|
|
$corruptReceiptEnvelopeResult = [
|
|
|
|
|
'uuid' => $corruptReceiptEnvelopeUuid,
|
|
|
|
|
'hostname' => 'corrupt-receipt-envelope.example',
|
|
|
|
|
'settings_changed' => true,
|
|
|
|
|
];
|
|
|
|
|
$corruptReceiptEnvelopeWire = [
|
|
|
|
|
'event_id' => 'automatic-corrupt-receipt-envelope',
|
|
|
|
|
'host_id' => 7,
|
|
|
|
|
'operation_id' => (string) $corruptReceiptEnvelopeJob->operation_id,
|
|
|
|
|
'action' => $corruptReceiptEnvelopeIdentity['action'],
|
|
|
|
|
'resource_id' => $corruptReceiptEnvelopeIdentity['resource_id'],
|
|
|
|
|
'request_fingerprint' =>
|
|
|
|
|
$corruptReceiptEnvelopeIdentity['request_fingerprint'],
|
|
|
|
|
'status' => 'succeeded',
|
|
|
|
|
'effect_may_have_applied' => false,
|
|
|
|
|
'safe_to_resubmit' => false,
|
|
|
|
|
'error_code' => '',
|
|
|
|
|
'result' => $corruptReceiptEnvelopeResult,
|
|
|
|
|
];
|
|
|
|
|
Capsule::table('cloud_agent_completion_events')->insert([
|
|
|
|
|
'event_id' => $corruptReceiptEnvelopeWire['event_id'],
|
|
|
|
|
'host_id' => 7,
|
|
|
|
|
'operation_id' => $corruptReceiptEnvelopeWire['operation_id'],
|
|
|
|
|
'payload_sha256' => hash('sha256', json_encode(
|
|
|
|
|
$corruptReceiptEnvelopeWire,
|
|
|
|
|
JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES
|
|
|
|
|
| JSON_UNESCAPED_UNICODE
|
|
|
|
|
)),
|
|
|
|
|
'outcome' => 'succeeded',
|
|
|
|
|
'effect_may_have_applied' => 0,
|
|
|
|
|
'safe_to_resubmit' => 0,
|
|
|
|
|
'error_code' => '',
|
|
|
|
|
'result_json' => json_encode([
|
|
|
|
|
'schema' => 'virt_cloud_completion_event',
|
|
|
|
|
'version' => 1,
|
|
|
|
|
'identity' => $corruptReceiptEnvelopeIdentity,
|
|
|
|
|
'result' => $corruptReceiptEnvelopeResult,
|
|
|
|
|
], JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES
|
|
|
|
|
| JSON_UNESCAPED_UNICODE),
|
|
|
|
|
'disposition' => 'received',
|
|
|
|
|
'received_at' => time(),
|
|
|
|
|
'applied_at' => null,
|
|
|
|
|
]);
|
|
|
|
|
Capsule::table('cloud_jobs')->where(
|
|
|
|
|
'id',
|
|
|
|
|
$corruptReceiptEnvelopeId
|
|
|
|
|
)->update([
|
|
|
|
|
'request_idempotency_key' => 'bad-receipt-key',
|
|
|
|
|
'available_at' => 0,
|
|
|
|
|
'lease_token' => null,
|
|
|
|
|
'lease_until' => 0,
|
|
|
|
|
]);
|
|
|
|
|
Capsule::table('cloud_templates')->insert([
|
|
|
|
|
'id' => 'identity-priority-template',
|
|
|
|
|
'active_job_id' => $corruptReceiptEnvelopeId,
|
|
|
|
|
'state' => 'active',
|
|
|
|
|
]);
|
|
|
|
|
$beforeCorruptReceiptEnvelopeRequests = count($client->requests);
|
|
|
|
|
$assert((new CloudAutomaticOperationRecoveryService())->tick(),
|
|
|
|
|
'A damaged envelope with an exact receipt left recovery.');
|
|
|
|
|
$corruptReceiptEnvelopeJob = CloudJob::findOrFail(
|
|
|
|
|
$corruptReceiptEnvelopeId
|
|
|
|
|
);
|
|
|
|
|
$corruptReceiptEnvelopeEvent = Capsule::table(
|
|
|
|
|
'cloud_agent_completion_events'
|
|
|
|
|
)->where(
|
|
|
|
|
'event_id',
|
|
|
|
|
$corruptReceiptEnvelopeWire['event_id']
|
|
|
|
|
)->first();
|
|
|
|
|
$assert((string) $corruptReceiptEnvelopeJob->effect_resolution
|
|
|
|
|
=== 'unknown'
|
|
|
|
|
&& (bool) $corruptReceiptEnvelopeJob->finalization_incident
|
|
|
|
|
&& (string) $corruptReceiptEnvelopeJob->finalization_error_code
|
|
|
|
|
=== 'recovery_operation_identity_conflict'
|
|
|
|
|
&& (int) $corruptReceiptEnvelopeJob->available_at >= time() + 290
|
|
|
|
|
&& (int) $corruptReceiptEnvelopeJob->lease_until >= time() + 290
|
|
|
|
|
&& (string) $corruptReceiptEnvelopeEvent->disposition === 'received'
|
|
|
|
|
&& $corruptReceiptEnvelopeEvent->applied_at === null
|
|
|
|
|
&& (int) Capsule::table('cloud_templates')->where(
|
|
|
|
|
'id',
|
|
|
|
|
'identity-priority-template'
|
|
|
|
|
)->value('active_job_id') === $corruptReceiptEnvelopeId
|
|
|
|
|
&& CloudJobService::blockingQueryForInstance(
|
|
|
|
|
$corruptReceiptEnvelopeId
|
|
|
|
|
)->exists()
|
|
|
|
|
&& count($client->requests)
|
|
|
|
|
=== $beforeCorruptReceiptEnvelopeRequests,
|
|
|
|
|
'A damaged envelope consumed its receipt, contacted Agent, or falsely unlocked.');
|
|
|
|
|
Capsule::table('cloud_templates')->where(
|
|
|
|
|
'id',
|
|
|
|
|
'identity-priority-template'
|
|
|
|
|
)->delete();
|
|
|
|
|
|
|
|
|
|
// A prepared no-fence conclusion must revalidate the complete envelope at
|
|
|
|
|
// the locked publication boundary. This protects direct callers and the
|
|
|
|
|
// crash-resume path even if claim-time validation is bypassed or races.
|
|
|
|
|
[$preparedStateConflictId, $preparedStateConflictUuid] =
|
|
|
|
|
$insertSettingsJob(
|
|
|
|
|
str_repeat('b2', 16),
|
|
|
|
|
['hostname' => 'prepared-state-conflict.example']
|
|
|
|
|
);
|
|
|
|
|
$preparedStateConflictIdentity =
|
|
|
|
|
CloudOperationRecoveryIdentity::expected(
|
|
|
|
|
CloudJob::findOrFail($preparedStateConflictId),
|
|
|
|
|
true
|
|
|
|
|
);
|
|
|
|
|
$preparedStateConflictOperation = $preparedStateConflictIdentity + [
|
|
|
|
|
'operation_id' => str_repeat('b2', 16),
|
|
|
|
|
'status' => 'succeeded',
|
|
|
|
|
'effect_may_have_applied' => false,
|
|
|
|
|
'safe_to_resubmit' => false,
|
|
|
|
|
'error_code' => '',
|
|
|
|
|
'result' => [
|
|
|
|
|
'uuid' => $preparedStateConflictUuid,
|
|
|
|
|
'hostname' => 'prepared-state-conflict.example',
|
|
|
|
|
'settings_changed' => true,
|
|
|
|
|
],
|
|
|
|
|
];
|
|
|
|
|
$preparedStateConflictSource =
|
|
|
|
|
'automatic_applied_succeeded_nofence:' . str_repeat('2', 64);
|
|
|
|
|
CloudOperationRecoveryService::prepareApplied(
|
|
|
|
|
$preparedStateConflictId,
|
|
|
|
|
$preparedStateConflictOperation,
|
|
|
|
|
$preparedStateConflictSource
|
|
|
|
|
);
|
|
|
|
|
$preparedStateConflictMarker = (string) CloudJob::findOrFail(
|
|
|
|
|
$preparedStateConflictId
|
|
|
|
|
)->effect_resolution_reason;
|
|
|
|
|
Capsule::table('cloud_jobs')->where(
|
|
|
|
|
'id',
|
|
|
|
|
$preparedStateConflictId
|
|
|
|
|
)->update([
|
|
|
|
|
'receipt_deadline_at' => 0,
|
|
|
|
|
'available_at' => 0,
|
|
|
|
|
'lease_token' => null,
|
|
|
|
|
'lease_until' => 0,
|
|
|
|
|
]);
|
|
|
|
|
try {
|
|
|
|
|
CloudOperationRecoveryService::commitApplied(
|
|
|
|
|
$preparedStateConflictId,
|
|
|
|
|
$preparedStateConflictOperation,
|
|
|
|
|
$preparedStateConflictSource,
|
|
|
|
|
0
|
|
|
|
|
);
|
|
|
|
|
throw new RuntimeException(
|
|
|
|
|
'A direct applied commit published a damaged Agent envelope.'
|
|
|
|
|
);
|
|
|
|
|
} catch (CloudRecoveryInstanceStateConflict $error) {
|
|
|
|
|
$assert($error->httpStatus === 409,
|
|
|
|
|
'The applied envelope guard returned the wrong status.');
|
|
|
|
|
}
|
|
|
|
|
$preparedStateConflict = CloudJob::findOrFail(
|
|
|
|
|
$preparedStateConflictId
|
|
|
|
|
);
|
|
|
|
|
$assert((string) $preparedStateConflict->effect_resolution === 'unknown'
|
|
|
|
|
&& (string) $preparedStateConflict->effect_resolution_reason
|
|
|
|
|
=== $preparedStateConflictMarker,
|
|
|
|
|
'A rejected applied commit changed its prepared recovery intent.');
|
|
|
|
|
$beforePreparedStateConflictRequests = count($client->requests);
|
|
|
|
|
$assert((new CloudAutomaticOperationRecoveryService())->tick(),
|
|
|
|
|
'A prepared no-fence envelope conflict left recovery.');
|
|
|
|
|
$preparedStateConflict = CloudJob::findOrFail(
|
|
|
|
|
$preparedStateConflictId
|
|
|
|
|
);
|
|
|
|
|
$assert((string) $preparedStateConflict->effect_resolution === 'unknown'
|
|
|
|
|
&& (string) $preparedStateConflict->effect_resolution_reason
|
|
|
|
|
=== $preparedStateConflictMarker
|
|
|
|
|
&& (bool) $preparedStateConflict->finalization_incident
|
|
|
|
|
&& (string) $preparedStateConflict->finalization_error_code
|
|
|
|
|
=== 'recovery_instance_state_conflict'
|
|
|
|
|
&& CloudJobService::blockingQueryForInstance(
|
|
|
|
|
$preparedStateConflictId
|
|
|
|
|
)->exists()
|
|
|
|
|
&& count($client->requests)
|
|
|
|
|
=== $beforePreparedStateConflictRequests,
|
|
|
|
|
'A prepared no-fence conflict contacted Agent, published, or unlocked.');
|
|
|
|
|
|
|
|
|
|
// The not-applied publication boundary applies the same immutable identity
|
|
|
|
|
// guard. A prepared proof cannot rely only on its encrypted escrow after
|
|
|
|
|
// the durable request fingerprint column has been damaged.
|
|
|
|
|
[$preparedIdentityConflictId] = $insertSettingsJob(
|
|
|
|
|
str_repeat('b3', 16),
|
|
|
|
|
['hostname' => 'prepared-identity-conflict.example']
|
|
|
|
|
);
|
|
|
|
|
$preparedIdentityConflictIdentity =
|
|
|
|
|
CloudOperationRecoveryIdentity::expected(
|
|
|
|
|
CloudJob::findOrFail($preparedIdentityConflictId),
|
|
|
|
|
false
|
|
|
|
|
);
|
|
|
|
|
$preparedIdentityConflictOperation = $preparedIdentityConflictIdentity + [
|
|
|
|
|
'operation_id' => str_repeat('b3', 16),
|
|
|
|
|
'status' => 'failed',
|
|
|
|
|
'effect_may_have_applied' => false,
|
|
|
|
|
'safe_to_resubmit' => true,
|
|
|
|
|
'error_code' => 'settings_not_committed',
|
|
|
|
|
'result' => ['agent_detail' => 'bounded'],
|
|
|
|
|
];
|
|
|
|
|
$preparedIdentityConflictSource =
|
|
|
|
|
'automatic_not_applied_failed_nofence:' . str_repeat('3', 64);
|
|
|
|
|
CloudOperationRecoveryService::prepareNotApplied(
|
|
|
|
|
$preparedIdentityConflictId,
|
|
|
|
|
$preparedIdentityConflictOperation,
|
|
|
|
|
$preparedIdentityConflictSource
|
|
|
|
|
);
|
|
|
|
|
$preparedIdentityConflictMarker = (string) CloudJob::findOrFail(
|
|
|
|
|
$preparedIdentityConflictId
|
|
|
|
|
)->effect_resolution_reason;
|
|
|
|
|
Capsule::table('cloud_jobs')->where(
|
|
|
|
|
'id',
|
|
|
|
|
$preparedIdentityConflictId
|
|
|
|
|
)->update([
|
|
|
|
|
'request_fingerprint' => 'bad-fingerprint',
|
|
|
|
|
'available_at' => 0,
|
|
|
|
|
'lease_token' => null,
|
|
|
|
|
'lease_until' => 0,
|
|
|
|
|
]);
|
|
|
|
|
try {
|
|
|
|
|
CloudOperationRecoveryService::commitNotApplied(
|
|
|
|
|
$preparedIdentityConflictId,
|
|
|
|
|
$preparedIdentityConflictOperation,
|
|
|
|
|
$preparedIdentityConflictSource,
|
|
|
|
|
0
|
|
|
|
|
);
|
|
|
|
|
throw new RuntimeException(
|
|
|
|
|
'A direct not-applied commit published damaged request identity.'
|
|
|
|
|
);
|
|
|
|
|
} catch (CloudRecoveryOperationIdentityConflict $error) {
|
|
|
|
|
$assert($error->httpStatus === 409,
|
|
|
|
|
'The not-applied identity guard returned the wrong status.');
|
|
|
|
|
}
|
|
|
|
|
$preparedIdentityConflict = CloudJob::findOrFail(
|
|
|
|
|
$preparedIdentityConflictId
|
|
|
|
|
);
|
|
|
|
|
$assert((string) $preparedIdentityConflict->effect_resolution === 'unknown'
|
|
|
|
|
&& (string) $preparedIdentityConflict->effect_resolution_reason
|
|
|
|
|
=== $preparedIdentityConflictMarker,
|
|
|
|
|
'A rejected not-applied commit changed its prepared recovery intent.');
|
|
|
|
|
$beforePreparedIdentityConflictRequests = count($client->requests);
|
|
|
|
|
$assert((new CloudAutomaticOperationRecoveryService())->tick(),
|
|
|
|
|
'A prepared request-identity conflict left recovery.');
|
|
|
|
|
$preparedIdentityConflict = CloudJob::findOrFail(
|
|
|
|
|
$preparedIdentityConflictId
|
|
|
|
|
);
|
|
|
|
|
$assert((string) $preparedIdentityConflict->effect_resolution === 'unknown'
|
|
|
|
|
&& (string) $preparedIdentityConflict->effect_resolution_reason
|
|
|
|
|
=== $preparedIdentityConflictMarker
|
|
|
|
|
&& (bool) $preparedIdentityConflict->finalization_incident
|
|
|
|
|
&& (string) $preparedIdentityConflict->finalization_error_code
|
|
|
|
|
=== 'recovery_operation_identity_conflict'
|
|
|
|
|
&& CloudJobService::blockingQueryForInstance(
|
|
|
|
|
$preparedIdentityConflictId
|
|
|
|
|
)->exists()
|
|
|
|
|
&& count($client->requests)
|
|
|
|
|
=== $beforePreparedIdentityConflictRequests,
|
|
|
|
|
'A prepared identity conflict contacted Agent, published, or unlocked.');
|
|
|
|
|
|
|
|
|
|
// A previously published effect with an older incident is not exempt from
|
|
|
|
|
// envelope validation. The immutable terminal audit remains resolved, but
|
|
|
|
|
// the concrete envelope conflict becomes visible and the resource remains
|
|
|
|
|
// blocked without consulting Agent state.
|
|
|
|
|
[$resolvedEnvelopeConflictId] = $insertSettingsJob(
|
|
|
|
|
str_repeat('b4', 16),
|
|
|
|
|
['hostname' => 'resolved-envelope-conflict.example']
|
|
|
|
|
);
|
|
|
|
|
Capsule::table('cloud_jobs')->where(
|
|
|
|
|
'id',
|
|
|
|
|
$resolvedEnvelopeConflictId
|
|
|
|
|
)->update([
|
|
|
|
|
'effect_resolution' => 'applied',
|
|
|
|
|
'effect_resolved_at' => time() - 5,
|
|
|
|
|
'effect_resolved_by' => 0,
|
|
|
|
|
'effect_resolution_reason' => 'prior_exact_evidence',
|
|
|
|
|
'receipt_deadline_at' => 0,
|
|
|
|
|
'finalization_incident' => 1,
|
|
|
|
|
'finalization_error_code' => 'recovery_waiting_connection',
|
|
|
|
|
'available_at' => 0,
|
|
|
|
|
'lease_token' => null,
|
|
|
|
|
'lease_until' => 0,
|
|
|
|
|
]);
|
|
|
|
|
$beforeResolvedEnvelopeConflictRequests = count($client->requests);
|
|
|
|
|
$assert((new CloudAutomaticOperationRecoveryService())->tick(),
|
|
|
|
|
'A resolved task with a damaged envelope left local recovery.');
|
|
|
|
|
$resolvedEnvelopeConflict = CloudJob::findOrFail(
|
|
|
|
|
$resolvedEnvelopeConflictId
|
|
|
|
|
);
|
|
|
|
|
$assert((string) $resolvedEnvelopeConflict->effect_resolution === 'applied'
|
|
|
|
|
&& (string) $resolvedEnvelopeConflict->effect_resolution_reason
|
|
|
|
|
=== 'prior_exact_evidence'
|
|
|
|
|
&& (bool) $resolvedEnvelopeConflict->finalization_incident
|
|
|
|
|
&& (string) $resolvedEnvelopeConflict->finalization_error_code
|
|
|
|
|
=== 'recovery_instance_state_conflict'
|
|
|
|
|
&& (int) $resolvedEnvelopeConflict->available_at >= time() + 290
|
|
|
|
|
&& CloudJobService::blockingQueryForInstance(
|
|
|
|
|
$resolvedEnvelopeConflictId
|
|
|
|
|
)->exists()
|
|
|
|
|
&& count($client->requests)
|
|
|
|
|
=== $beforeResolvedEnvelopeConflictRequests,
|
|
|
|
|
'A resolved envelope conflict contacted Agent, hid its cause, or falsely unlocked.');
|
|
|
|
|
|
|
|
|
|
// Repairing a failed row to the dispatched state is not enough when the
|
|
|
|
|
// immutable host/operation/time envelope is still damaged. Keep it in the
|
|
|
|
|
// local envelope-conflict queue instead of publishing a recovery that can
|
|
|
|
|
// never satisfy the normal Agent claim.
|
|
|
|
|
[$incompleteRepairId] = $insertSettingsJob(
|
|
|
|
|
str_repeat('bd', 16),
|
|
|
|
|
['hostname' => 'incomplete-dispatch-repair.example']
|
|
|
|
|
);
|
|
|
|
|
Capsule::table('cloud_jobs')->where('id', $incompleteRepairId)->update([
|
|
|
|
|
'dispatch_attempts' => 2,
|
|
|
|
|
'available_at' => 0,
|
|
|
|
|
'lease_token' => null,
|
|
|
|
|
'lease_until' => 0,
|
|
|
|
|
]);
|
|
|
|
|
CloudOperationCompletionService::deferDispatchLedgerConflict(
|
|
|
|
|
CloudJob::findOrFail($incompleteRepairId)
|
|
|
|
|
);
|
|
|
|
|
Capsule::table('cloud_jobs')->where('id', $incompleteRepairId)->update([
|
|
|
|
|
'dispatch_attempts' => 1,
|
|
|
|
|
'dispatched_at' => null,
|
|
|
|
|
'available_at' => 0,
|
|
|
|
|
'lease_token' => null,
|
|
|
|
|
'lease_until' => 0,
|
|
|
|
|
]);
|
|
|
|
|
$beforeIncompleteRepairRequests = count($client->requests);
|
|
|
|
|
$assert((new CloudAutomaticOperationRecoveryService())->tick(),
|
|
|
|
|
'An incomplete repaired dispatch envelope was not checked.');
|
|
|
|
|
$incompleteRepair = CloudJob::findOrFail($incompleteRepairId);
|
|
|
|
|
$assert((string) $incompleteRepair->effect_resolution === 'unknown'
|
|
|
|
|
&& (bool) $incompleteRepair->finalization_incident
|
|
|
|
|
&& (string) $incompleteRepair->finalization_error_code
|
|
|
|
|
=== 'recovery_dispatch_repair_instance_state_conflict'
|
|
|
|
|
&& (int) $incompleteRepair->lease_until >= time() + 290
|
|
|
|
|
&& (int) $incompleteRepair->available_at >= time() + 290
|
|
|
|
|
&& count($client->requests) === $beforeIncompleteRepairRequests,
|
|
|
|
|
'An incomplete repaired dispatch envelope left its sticky queue.');
|
|
|
|
|
|
|
|
|
|
// A pre-existing dispatched/unknown row with a damaged immutable envelope
|
|
|
|
|
// must still be claimed forever, but without attempting an Agent request.
|
|
|
|
|
[$malformedEnvelopeId] = $insertSettingsJob(
|
|
|
|
|
str_repeat('bc', 16),
|
|
|
|
|
['hostname' => 'malformed-envelope.example']
|
|
|
|
|
);
|
|
|
|
|
Capsule::table('cloud_jobs')->where('id', $malformedEnvelopeId)->update([
|
|
|
|
|
'dispatched_at' => null,
|
|
|
|
|
'finalization_incident' => 0,
|
|
|
|
|
'finalization_error_code' => null,
|
|
|
|
|
'available_at' => 0,
|
|
|
|
|
'lease_token' => null,
|
|
|
|
|
'lease_until' => 0,
|
|
|
|
|
]);
|
|
|
|
|
$beforeMalformedEnvelopeRequests = count($client->requests);
|
|
|
|
|
$assert((new CloudAutomaticOperationRecoveryService())->tick(),
|
|
|
|
|
'A damaged dispatched recovery envelope fell out of the queue.');
|
|
|
|
|
$malformedEnvelope = CloudJob::findOrFail($malformedEnvelopeId);
|
|
|
|
|
$assert((bool) $malformedEnvelope->finalization_incident
|
|
|
|
|
&& (string) $malformedEnvelope->finalization_error_code
|
|
|
|
|
=== 'recovery_instance_state_conflict'
|
|
|
|
|
&& (string) $malformedEnvelope->effect_resolution === 'unknown'
|
|
|
|
|
&& (int) $malformedEnvelope->lease_until >= time() + 290
|
|
|
|
|
&& (int) $malformedEnvelope->available_at >= time() + 290
|
|
|
|
|
&& count($client->requests) === $beforeMalformedEnvelopeRequests,
|
|
|
|
|
'A damaged dispatched recovery envelope contacted Agent or lost its concrete incident.');
|
|
|
|
|
|
|
|
|
|
[$invalidOperationIdJob] = $insertSettingsJob(
|
|
|
|
|
str_repeat('G', 32),
|
|
|
|
|
['hostname' => 'invalid-operation-id.example']
|
|
|
|
|
);
|
|
|
|
|
Capsule::table('cloud_jobs')->where('id', $invalidOperationIdJob)->update([
|
|
|
|
|
'available_at' => 0,
|
|
|
|
|
'lease_token' => null,
|
|
|
|
|
'lease_until' => 0,
|
|
|
|
|
]);
|
|
|
|
|
$beforeInvalidOperationIdRequests = count($client->requests);
|
|
|
|
|
$assert((new CloudAutomaticOperationRecoveryService())->tick(),
|
|
|
|
|
'A malformed Agent operation ID fell out of recovery.');
|
|
|
|
|
$invalidOperationIdentity = CloudJob::findOrFail(
|
|
|
|
|
$invalidOperationIdJob
|
|
|
|
|
);
|
|
|
|
|
$assert((bool) $invalidOperationIdentity->finalization_incident
|
|
|
|
|
&& (string) $invalidOperationIdentity->finalization_error_code
|
|
|
|
|
=== 'recovery_operation_identity_conflict'
|
|
|
|
|
&& (string) $invalidOperationIdentity->effect_resolution === 'unknown'
|
|
|
|
|
&& (int) $invalidOperationIdentity->available_at >= time() + 290
|
|
|
|
|
&& count($client->requests) === $beforeInvalidOperationIdRequests,
|
|
|
|
|
'A malformed operation ID lost its concrete identity-conflict status or contacted Agent.');
|
|
|
|
|
|
|
|
|
|
// A historical succeeded row without a bound completion event must still
|
|
|
|
|
// prove the immutable dispatched envelope before its sticky repair
|
|
|
|
|
// incident can be cleared. Otherwise repairing only the numeric counter
|
|
|
|
|
// would release the instance while host/time identity remained corrupt.
|
|
|
|
|
[$incompleteSucceededRepairId] = $insertSettingsJob(
|
|
|
|
|
str_repeat('be', 16),
|
|
|
|
|
['hostname' => 'incomplete-succeeded-repair.example']
|
|
|
|
|
);
|
|
|
|
|
$incompleteSucceededResult = json_encode([
|
|
|
|
|
'hostname' => 'incomplete-succeeded-repair.example',
|
|
|
|
|
'settings_changed' => true,
|
|
|
|
|
], JSON_THROW_ON_ERROR);
|
|
|
|
|
Capsule::table('cloud_jobs')->where(
|
|
|
|
|
'id',
|
|
|
|
|
$incompleteSucceededRepairId
|
|
|
|
|
)->update([
|
|
|
|
|
'dispatch_attempts' => 2,
|
|
|
|
|
'outcome' => 'succeeded',
|
|
|
|
|
'status' => 'succeeded',
|
|
|
|
|
'error_code' => null,
|
|
|
|
|
'effect_may_have_applied' => 0,
|
|
|
|
|
'safe_to_resubmit' => 0,
|
|
|
|
|
'effect_resolution' => 'applied',
|
|
|
|
|
'completion_event_id' => null,
|
|
|
|
|
'result_json' => $incompleteSucceededResult,
|
|
|
|
|
'available_at' => 0,
|
|
|
|
|
'lease_token' => null,
|
|
|
|
|
'lease_until' => 0,
|
|
|
|
|
]);
|
|
|
|
|
$incompleteSucceededBefore = $audit($incompleteSucceededRepairId);
|
|
|
|
|
$incompleteSucceededBefore['dispatch_attempts'] = 1;
|
|
|
|
|
$incompleteSucceededEnvelope = (string) CloudJob::findOrFail(
|
|
|
|
|
$incompleteSucceededRepairId
|
|
|
|
|
)->sealed_payload;
|
|
|
|
|
CloudOperationCompletionService::deferDispatchLedgerConflict(
|
|
|
|
|
CloudJob::findOrFail($incompleteSucceededRepairId)
|
|
|
|
|
);
|
|
|
|
|
Capsule::table('cloud_jobs')->where(
|
|
|
|
|
'id',
|
|
|
|
|
$incompleteSucceededRepairId
|
|
|
|
|
)->update([
|
|
|
|
|
'dispatch_attempts' => 1,
|
|
|
|
|
'dispatched_at' => null,
|
|
|
|
|
'available_at' => 0,
|
|
|
|
|
'lease_token' => null,
|
|
|
|
|
'lease_until' => 0,
|
|
|
|
|
]);
|
|
|
|
|
$beforeIncompleteSucceededRequests = count($client->requests);
|
|
|
|
|
$assert((new CloudAutomaticOperationRecoveryService())->tick(),
|
|
|
|
|
'An incomplete succeeded dispatch envelope was not checked.');
|
|
|
|
|
$incompleteSucceededRepair = CloudJob::findOrFail(
|
|
|
|
|
$incompleteSucceededRepairId
|
|
|
|
|
);
|
|
|
|
|
$assert((bool) $incompleteSucceededRepair->finalization_incident
|
|
|
|
|
&& (string) $incompleteSucceededRepair->finalization_error_code
|
|
|
|
|
=== 'recovery_dispatch_repair_instance_state_conflict'
|
|
|
|
|
&& (int) $incompleteSucceededRepair->lease_until >= time() + 290
|
|
|
|
|
&& (int) $incompleteSucceededRepair->available_at >= time() + 290
|
|
|
|
|
&& count($client->requests) === $beforeIncompleteSucceededRequests,
|
|
|
|
|
'A succeeded task cleared its repair incident without a complete dispatched envelope.');
|
|
|
|
|
$assert($audit($incompleteSucceededRepairId)
|
|
|
|
|
=== $incompleteSucceededBefore,
|
|
|
|
|
'Checking an incomplete succeeded envelope rewrote terminal audit data: '
|
|
|
|
|
. json_encode([
|
|
|
|
|
'expected' => $incompleteSucceededBefore,
|
|
|
|
|
'actual' => $audit($incompleteSucceededRepairId),
|
|
|
|
|
], JSON_THROW_ON_ERROR));
|
|
|
|
|
Capsule::table('cloud_jobs')->where(
|
|
|
|
|
'id',
|
|
|
|
|
$incompleteSucceededRepairId
|
|
|
|
|
)->update([
|
|
|
|
|
'dispatched_at' => time() - 980,
|
|
|
|
|
'sealed_payload' => 'automatic-recovery-test:missing',
|
|
|
|
|
'available_at' => 0,
|
|
|
|
|
'lease_token' => null,
|
|
|
|
|
'lease_until' => 0,
|
|
|
|
|
]);
|
|
|
|
|
$assert((new CloudAutomaticOperationRecoveryService())->tick(),
|
|
|
|
|
'A succeeded repair with a damaged identity escrow was not checked.');
|
|
|
|
|
$damagedSucceededRepair = CloudJob::findOrFail(
|
|
|
|
|
$incompleteSucceededRepairId
|
|
|
|
|
);
|
|
|
|
|
$assert((bool) $damagedSucceededRepair->finalization_incident
|
|
|
|
|
&& (string) $damagedSucceededRepair->finalization_error_code
|
|
|
|
|
=== 'recovery_dispatch_repair_payload_unavailable'
|
|
|
|
|
&& (int) $damagedSucceededRepair->lease_until >= time() + 290
|
|
|
|
|
&& (int) $damagedSucceededRepair->available_at >= time() + 290,
|
|
|
|
|
'A succeeded repair cleared its sticky incident without validating the immutable identity.');
|
|
|
|
|
Capsule::table('cloud_jobs')->where(
|
|
|
|
|
'id',
|
|
|
|
|
$incompleteSucceededRepairId
|
|
|
|
|
)->update([
|
|
|
|
|
'sealed_payload' => $incompleteSucceededEnvelope,
|
|
|
|
|
'available_at' => 0,
|
|
|
|
|
'lease_token' => null,
|
|
|
|
|
'lease_until' => 0,
|
|
|
|
|
]);
|
|
|
|
|
$assert((new CloudAutomaticOperationRecoveryService())->tick(),
|
|
|
|
|
'A completed succeeded dispatch envelope did not resume repair.');
|
|
|
|
|
$completedSucceededRepair = CloudJob::findOrFail(
|
|
|
|
|
$incompleteSucceededRepairId
|
|
|
|
|
);
|
|
|
|
|
$assert(!(bool) $completedSucceededRepair->finalization_incident
|
|
|
|
|
&& $completedSucceededRepair->finalization_error_code === null
|
|
|
|
|
&& $audit($incompleteSucceededRepairId)
|
|
|
|
|
=== $incompleteSucceededBefore
|
|
|
|
|
&& count($client->requests) === $beforeIncompleteSucceededRequests,
|
|
|
|
|
'A complete succeeded dispatch envelope was not unlocked without rewriting its terminal audit.');
|
|
|
|
|
|
|
|
|
|
// Repairing a corrupt counter to zero cannot erase already durable Agent
|
|
|
|
|
// evidence. The sticky repair queue must outrank the PHP-only finalizer;
|
|
|
|
|
// after the operator chooses one, the same receipt resumes convergence.
|
|
|
|
|
[$zeroCounterConflictId, $zeroCounterConflictUuid] = $insertSettingsJob(
|
|
|
|
|
str_repeat('bf', 16),
|
|
|
|
|
['hostname' => 'zero-counter-conflict.example']
|
|
|
|
|
);
|
|
|
|
|
$zeroCounterConflictJob = CloudJob::findOrFail($zeroCounterConflictId);
|
|
|
|
|
$zeroCounterIdentity = CloudOperationRecoveryIdentity::expected(
|
|
|
|
|
$zeroCounterConflictJob,
|
|
|
|
|
true
|
|
|
|
|
);
|
|
|
|
|
$zeroCounterEventId = 'zero-counter-proof-01';
|
|
|
|
|
$zeroCounterWire = [
|
|
|
|
|
'type' => 'completion',
|
|
|
|
|
'id' => $zeroCounterEventId,
|
|
|
|
|
'agent_id' => '7',
|
|
|
|
|
'operation_id' => (string) $zeroCounterConflictJob->operation_id,
|
|
|
|
|
'action' => $zeroCounterIdentity['action'],
|
|
|
|
|
'resource_id' => $zeroCounterIdentity['resource_id'],
|
|
|
|
|
'request_fingerprint' =>
|
|
|
|
|
$zeroCounterIdentity['request_fingerprint'],
|
|
|
|
|
'status' => 'succeeded',
|
|
|
|
|
'effect_may_have_applied' => false,
|
|
|
|
|
'safe_to_resubmit' => false,
|
|
|
|
|
'error_code' => '',
|
|
|
|
|
'result' => [
|
|
|
|
|
'uuid' => $zeroCounterConflictUuid,
|
|
|
|
|
'hostname' => 'zero-counter-conflict.example',
|
|
|
|
|
'settings_changed' => true,
|
|
|
|
|
],
|
|
|
|
|
];
|
|
|
|
|
Capsule::table('cloud_jobs')->where(
|
|
|
|
|
'id',
|
|
|
|
|
$zeroCounterConflictId
|
|
|
|
|
)->update([
|
|
|
|
|
'dispatch_attempts' => 2,
|
|
|
|
|
'available_at' => 0,
|
|
|
|
|
'lease_token' => null,
|
|
|
|
|
'lease_until' => 0,
|
|
|
|
|
]);
|
|
|
|
|
$beforeZeroCounterRequests = count($client->requests);
|
|
|
|
|
$acceptedZeroCounter = CloudOperationCompletionService::accept(
|
|
|
|
|
$zeroCounterWire
|
|
|
|
|
);
|
|
|
|
|
$zeroCounterEnvelope = (string) CloudJob::findOrFail(
|
|
|
|
|
$zeroCounterConflictId
|
|
|
|
|
)->sealed_payload;
|
|
|
|
|
$zeroCounterTerminalAt = (int) CloudJob::findOrFail(
|
|
|
|
|
$zeroCounterConflictId
|
|
|
|
|
)->terminal_at;
|
|
|
|
|
Capsule::table('cloud_jobs')->where(
|
|
|
|
|
'id',
|
|
|
|
|
$zeroCounterConflictId
|
|
|
|
|
)->update([
|
|
|
|
|
'dispatch_attempts' => 0,
|
|
|
|
|
'available_at' => 0,
|
|
|
|
|
'lease_token' => null,
|
|
|
|
|
'lease_until' => 0,
|
|
|
|
|
]);
|
|
|
|
|
$assert((new CloudAutomaticOperationRecoveryService())->tick(),
|
|
|
|
|
'A repaired-zero ledger with Agent evidence was not checked.');
|
|
|
|
|
$zeroCounterConflict = CloudJob::findOrFail($zeroCounterConflictId);
|
|
|
|
|
$zeroCounterEvent = Capsule::table(
|
|
|
|
|
'cloud_agent_completion_events'
|
|
|
|
|
)->where('event_id', $zeroCounterEventId)->first();
|
|
|
|
|
$assert($acceptedZeroCounter === [
|
|
|
|
|
'accepted' => true,
|
|
|
|
|
'disposition' => 'received',
|
|
|
|
|
]
|
|
|
|
|
&& (bool) $zeroCounterConflict->finalization_incident
|
|
|
|
|
&& (string) $zeroCounterConflict->finalization_error_code
|
|
|
|
|
=== 'recovery_dispatch_repair_instance_state_conflict'
|
|
|
|
|
&& (string) $zeroCounterConflict->effect_resolution === 'unknown'
|
|
|
|
|
&& (string) $zeroCounterConflict->sealed_payload
|
|
|
|
|
=== $zeroCounterEnvelope
|
|
|
|
|
&& (string) $zeroCounterEvent->disposition === 'received'
|
|
|
|
|
&& $zeroCounterEvent->applied_at === null
|
|
|
|
|
&& (int) $zeroCounterConflict->lease_until >= time() + 290
|
|
|
|
|
&& (int) $zeroCounterConflict->available_at >= time() + 290
|
|
|
|
|
&& count($client->requests) === $beforeZeroCounterRequests,
|
|
|
|
|
'Repairing a dispatched operation to zero ran the PHP-only finalizer or consumed its receipt.');
|
|
|
|
|
Capsule::table('cloud_jobs')->where(
|
|
|
|
|
'id',
|
|
|
|
|
$zeroCounterConflictId
|
|
|
|
|
)->update([
|
|
|
|
|
'dispatch_attempts' => 1,
|
|
|
|
|
'available_at' => 0,
|
|
|
|
|
'lease_token' => null,
|
|
|
|
|
'lease_until' => 0,
|
|
|
|
|
]);
|
|
|
|
|
$assert(CloudOperationCompletionService::accept($zeroCounterWire) === [
|
|
|
|
|
'accepted' => true,
|
|
|
|
|
'disposition' => 'received',
|
|
|
|
|
]
|
|
|
|
|
&& (string) Capsule::table('cloud_agent_completion_events')
|
|
|
|
|
->where('event_id', $zeroCounterEventId)->value('disposition')
|
|
|
|
|
=== 'received',
|
|
|
|
|
'A retransmitted receipt bypassed the sticky dispatch-repair coordinator.');
|
|
|
|
|
$assert((new CloudAutomaticOperationRecoveryService())->tick(),
|
|
|
|
|
'The repaired dispatched ledger did not re-enter Agent recovery.');
|
|
|
|
|
$assert((new CloudAutomaticOperationRecoveryService())->tick(),
|
|
|
|
|
'The repaired dispatched ledger did not consume its retained receipt.');
|
|
|
|
|
$zeroCounterResolved = CloudJob::findOrFail($zeroCounterConflictId);
|
|
|
|
|
$zeroCounterResolvedEvent = Capsule::table(
|
|
|
|
|
'cloud_agent_completion_events'
|
|
|
|
|
)->where('event_id', $zeroCounterEventId)->first();
|
|
|
|
|
$assert((string) $zeroCounterResolved->phase === 'terminal'
|
|
|
|
|
&& (string) $zeroCounterResolved->outcome === 'failed'
|
|
|
|
|
&& (string) $zeroCounterResolved->status === 'failed'
|
|
|
|
|
&& (int) $zeroCounterResolved->terminal_at
|
|
|
|
|
=== $zeroCounterTerminalAt
|
|
|
|
|
&& (string) $zeroCounterResolved->effect_resolution === 'applied'
|
|
|
|
|
&& !(bool) $zeroCounterResolved->finalization_incident
|
|
|
|
|
&& $zeroCounterResolved->finalization_error_code === null
|
|
|
|
|
&& (string) $zeroCounterResolvedEvent->disposition === 'applied'
|
|
|
|
|
&& (int) $zeroCounterResolvedEvent->applied_at > 0
|
|
|
|
|
&& count($client->requests) === $beforeZeroCounterRequests,
|
|
|
|
|
'Repairing the counter to one did not safely resume the retained Agent receipt.');
|
|
|
|
|
|
|
|
|
|
// The same contradiction on an active generation-zero task must move the
|
|
|
|
|
// row to a sticky terminal fence before the ordinary worker can dispatch.
|
|
|
|
|
[$activeZeroConflictId] = $insertSettingsJob(
|
|
|
|
|
str_repeat('c0', 16),
|
|
|
|
|
['hostname' => 'active-zero-conflict.example']
|
|
|
|
|
);
|
|
|
|
|
Capsule::table('cloud_jobs')->where('id', $activeZeroConflictId)->update([
|
|
|
|
|
'phase' => 'accepted',
|
|
|
|
|
'outcome' => null,
|
|
|
|
|
'status' => 'queued',
|
|
|
|
|
'attempts' => 0,
|
|
|
|
|
'dispatch_attempts' => 0,
|
|
|
|
|
'dispatch_started_at' => null,
|
|
|
|
|
'agent_accepted_at' => null,
|
|
|
|
|
'receipt_deadline_at' => null,
|
|
|
|
|
'terminal_at' => null,
|
|
|
|
|
'effect_may_have_applied' => null,
|
|
|
|
|
'safe_to_resubmit' => null,
|
|
|
|
|
'effect_resolution' => 'not_applicable',
|
|
|
|
|
'dispatched_at' => null,
|
|
|
|
|
'error_code' => null,
|
|
|
|
|
'finalization_incident' => 0,
|
|
|
|
|
'finalization_error_code' => null,
|
|
|
|
|
'result_json' => null,
|
|
|
|
|
'available_at' => 0,
|
|
|
|
|
'lease_token' => null,
|
|
|
|
|
'lease_until' => 0,
|
|
|
|
|
]);
|
|
|
|
|
$activeZeroJob = CloudJob::findOrFail($activeZeroConflictId);
|
|
|
|
|
$activeZeroIdentity = CloudOperationRecoveryIdentity::expected(
|
|
|
|
|
$activeZeroJob,
|
|
|
|
|
true
|
|
|
|
|
);
|
|
|
|
|
$activeZeroEventId = 'active-zero-proof-01';
|
|
|
|
|
$activeZeroAccepted = CloudOperationCompletionService::accept([
|
|
|
|
|
'type' => 'completion',
|
|
|
|
|
'id' => $activeZeroEventId,
|
|
|
|
|
'agent_id' => '7',
|
|
|
|
|
'operation_id' => (string) $activeZeroJob->operation_id,
|
|
|
|
|
'action' => $activeZeroIdentity['action'],
|
|
|
|
|
'resource_id' => $activeZeroIdentity['resource_id'],
|
|
|
|
|
'request_fingerprint' => $activeZeroIdentity['request_fingerprint'],
|
|
|
|
|
'status' => 'succeeded',
|
|
|
|
|
'effect_may_have_applied' => false,
|
|
|
|
|
'safe_to_resubmit' => false,
|
|
|
|
|
'error_code' => '',
|
|
|
|
|
'result' => [
|
|
|
|
|
'hostname' => 'active-zero-conflict.example',
|
|
|
|
|
'settings_changed' => true,
|
|
|
|
|
],
|
|
|
|
|
]);
|
|
|
|
|
$activeZeroConflict = CloudJob::findOrFail($activeZeroConflictId);
|
|
|
|
|
$activeZeroEvent = Capsule::table(
|
|
|
|
|
'cloud_agent_completion_events'
|
|
|
|
|
)->where('event_id', $activeZeroEventId)->first();
|
|
|
|
|
$assert($activeZeroAccepted === [
|
|
|
|
|
'accepted' => true,
|
|
|
|
|
'disposition' => 'received',
|
|
|
|
|
]
|
|
|
|
|
&& (string) $activeZeroConflict->phase === 'terminal'
|
|
|
|
|
&& (string) $activeZeroConflict->outcome === 'failed'
|
|
|
|
|
&& (string) $activeZeroConflict->status === 'failed'
|
|
|
|
|
&& (string) $activeZeroConflict->effect_resolution === 'unknown'
|
|
|
|
|
&& (bool) $activeZeroConflict->finalization_incident
|
|
|
|
|
&& (string) $activeZeroConflict->finalization_error_code
|
|
|
|
|
=== 'recovery_dispatch_repair_instance_state_conflict'
|
|
|
|
|
&& (string) $activeZeroEvent->disposition === 'received'
|
|
|
|
|
&& !CloudJob::where('id', $activeZeroConflictId)
|
|
|
|
|
->where('phase', 'accepted')
|
|
|
|
|
->where('dispatch_attempts', 0)->exists()
|
|
|
|
|
&& count($client->requests) === $beforeZeroCounterRequests,
|
|
|
|
|
'An active zero-counter task retained a replayable mutation after Agent completion evidence arrived.');
|
|
|
|
|
// Defence in depth for a pre-upgrade/crash snapshot where only the event
|
|
|
|
|
// became durable: the ordinary worker query itself excludes any Agent job
|
|
|
|
|
// with same-operation completion evidence.
|
|
|
|
|
Capsule::table('cloud_jobs')->where('id', $activeZeroConflictId)->update([
|
|
|
|
|
'phase' => 'accepted',
|
|
|
|
|
'outcome' => null,
|
|
|
|
|
'status' => 'queued',
|
|
|
|
|
'attempts' => 0,
|
|
|
|
|
'dispatch_attempts' => 0,
|
|
|
|
|
'dispatch_deadline_at' => time() + 300,
|
|
|
|
|
'dispatch_started_at' => null,
|
|
|
|
|
'agent_accepted_at' => null,
|
|
|
|
|
'receipt_deadline_at' => null,
|
|
|
|
|
'terminal_at' => null,
|
|
|
|
|
'effect_may_have_applied' => null,
|
|
|
|
|
'safe_to_resubmit' => null,
|
|
|
|
|
'effect_resolution' => 'not_applicable',
|
|
|
|
|
'dispatched_at' => null,
|
|
|
|
|
'error_code' => null,
|
|
|
|
|
'finalization_incident' => 0,
|
|
|
|
|
'finalization_error_code' => null,
|
|
|
|
|
'available_at' => 0,
|
|
|
|
|
'lease_token' => null,
|
|
|
|
|
'lease_until' => 0,
|
|
|
|
|
]);
|
|
|
|
|
$beforeCrashWindowRequests = count($client->requests);
|
|
|
|
|
(new CloudWorkerService())->tick();
|
|
|
|
|
$crashWindowJob = CloudJob::findOrFail($activeZeroConflictId);
|
|
|
|
|
$assert((string) $crashWindowJob->phase === 'accepted'
|
|
|
|
|
&& (int) $crashWindowJob->dispatch_attempts === 0
|
|
|
|
|
&& count($client->requests) === $beforeCrashWindowRequests,
|
|
|
|
|
'The ordinary worker replayed a mutation after durable same-operation completion evidence.');
|
|
|
|
|
|
|
|
|
|
// A repaired ledger with a corrupt bound receipt stays visible but obeys
|
|
|
|
|
// the same retry gate on both ordinary and receipt-priority passes. Without
|
|
|
|
|
// the lease clock this exact event would be reclaimed on every tick.
|
|
|
|
|
[$repairBackoffId, $repairBackoffUuid] = $insertSettingsJob(
|
|
|
|
|
str_repeat('ce', 16),
|
|
|
|
|
['hostname' => 'dispatch-repair-backoff.example']
|
|
|
|
|
);
|
|
|
|
|
$repairBackoffEventId = 'dispatch-repair-invalid-event';
|
|
|
|
|
Capsule::table('cloud_jobs')->where('id', $repairBackoffId)->update([
|
|
|
|
|
'dispatch_attempts' => 2,
|
|
|
|
|
'outcome' => 'succeeded',
|
|
|
|
|
'status' => 'succeeded',
|
|
|
|
|
'error_code' => null,
|
|
|
|
|
'effect_may_have_applied' => 0,
|
|
|
|
|
'safe_to_resubmit' => 0,
|
|
|
|
|
'effect_resolution' => 'applied',
|
|
|
|
|
'completion_event_id' => $repairBackoffEventId,
|
|
|
|
|
'available_at' => 0,
|
|
|
|
|
'lease_token' => null,
|
|
|
|
|
'lease_until' => 0,
|
|
|
|
|
]);
|
|
|
|
|
$repairBackoffJob = CloudJob::findOrFail($repairBackoffId);
|
|
|
|
|
$repairBackoffIdentity = CloudOperationRecoveryIdentity::expected(
|
|
|
|
|
$repairBackoffJob,
|
|
|
|
|
true
|
|
|
|
|
);
|
|
|
|
|
Capsule::table('cloud_agent_completion_events')->insert([
|
|
|
|
|
'event_id' => $repairBackoffEventId,
|
|
|
|
|
'host_id' => 7,
|
|
|
|
|
'operation_id' => (string) $repairBackoffJob->operation_id,
|
|
|
|
|
// Valid shape but deliberately not the hash of the immutable receipt.
|
|
|
|
|
'payload_sha256' => str_repeat('0', 64),
|
|
|
|
|
'outcome' => 'succeeded',
|
|
|
|
|
'effect_may_have_applied' => 0,
|
|
|
|
|
'safe_to_resubmit' => 0,
|
|
|
|
|
'error_code' => '',
|
|
|
|
|
'result_json' => json_encode([
|
|
|
|
|
'schema' => 'virt_cloud_completion_event',
|
|
|
|
|
'version' => 1,
|
|
|
|
|
'identity' => $repairBackoffIdentity,
|
|
|
|
|
'result' => [
|
|
|
|
|
'uuid' => $repairBackoffUuid,
|
|
|
|
|
'hostname' => 'dispatch-repair-backoff.example',
|
|
|
|
|
'settings_changed' => true,
|
|
|
|
|
],
|
|
|
|
|
], JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES
|
|
|
|
|
| JSON_UNESCAPED_UNICODE),
|
|
|
|
|
'disposition' => 'received',
|
|
|
|
|
'received_at' => time(),
|
|
|
|
|
'applied_at' => null,
|
|
|
|
|
]);
|
|
|
|
|
CloudOperationCompletionService::deferDispatchLedgerConflict(
|
|
|
|
|
$repairBackoffJob
|
|
|
|
|
);
|
|
|
|
|
Capsule::table('cloud_jobs')->where('id', $repairBackoffId)->update([
|
|
|
|
|
'dispatch_attempts' => 1,
|
|
|
|
|
'available_at' => 0,
|
|
|
|
|
'lease_token' => null,
|
|
|
|
|
'lease_until' => 0,
|
|
|
|
|
]);
|
|
|
|
|
$beforeRepairBackoffRequests = count($client->requests);
|
|
|
|
|
$repairBackoffService = new CloudAutomaticOperationRecoveryService();
|
|
|
|
|
$assert($repairBackoffService->tick(),
|
|
|
|
|
'A repaired ledger with an invalid bound receipt was not checked.');
|
|
|
|
|
$repairBackoffJob = CloudJob::findOrFail($repairBackoffId);
|
|
|
|
|
$repairBackoffUpdatedAt = (int) $repairBackoffJob->updated_at;
|
|
|
|
|
$assert((bool) $repairBackoffJob->finalization_incident
|
|
|
|
|
&& (string) $repairBackoffJob->finalization_error_code
|
|
|
|
|
=== 'recovery_dispatch_repair_receipt_invalid'
|
|
|
|
|
&& (int) $repairBackoffJob->lease_until >= time() + 290
|
|
|
|
|
&& (int) $repairBackoffJob->available_at >= time() + 290
|
|
|
|
|
&& count($client->requests) === $beforeRepairBackoffRequests,
|
|
|
|
|
'An invalid repaired-ledger receipt did not enter bounded backoff.');
|
|
|
|
|
Capsule::table('cloud_jobs')->where('id', '<>', $repairBackoffId)
|
|
|
|
|
->update([
|
|
|
|
|
'lease_until' => time() + 3600,
|
|
|
|
|
'available_at' => time() + 3600,
|
|
|
|
|
]);
|
|
|
|
|
$assert(!$repairBackoffService->tick()
|
|
|
|
|
&& (int) CloudJob::findOrFail($repairBackoffId)->updated_at
|
|
|
|
|
=== $repairBackoffUpdatedAt
|
|
|
|
|
&& count($client->requests) === $beforeRepairBackoffRequests,
|
|
|
|
|
'Receipt-priority recovery ignored the repaired-ledger retry gate.');
|
|
|
|
|
|
|
|
|
|
// Contradictory terminal/unknown safety tuples are PHP-only incidents.
|
|
|
|
|
// They must remain scheduled forever without inspection, fence writes, or
|
|
|
|
|
// any replay of the original mutation. Cover both legal dispatch counters
|
|
|
|
|
// and each independent part of the canonical terminal tuple.
|
|
|
|
|
$stateCases = [
|
|
|
|
|
[str_repeat('f1', 16), [
|
|
|
|
|
'effect_may_have_applied' => 1,
|
|
|
|
|
'safe_to_resubmit' => 1,
|
|
|
|
|
]],
|
|
|
|
|
[str_repeat('f2', 16), [
|
|
|
|
|
'effect_may_have_applied' => 0,
|
|
|
|
|
'safe_to_resubmit' => 0,
|
|
|
|
|
]],
|
|
|
|
|
[str_repeat('f3', 16), [
|
|
|
|
|
'outcome' => 'succeeded',
|
|
|
|
|
'status' => 'succeeded',
|
|
|
|
|
]],
|
|
|
|
|
[str_repeat('f4', 16), [
|
|
|
|
|
'dispatch_attempts' => 0,
|
|
|
|
|
'dispatched_at' => null,
|
|
|
|
|
'dispatch_started_at' => null,
|
|
|
|
|
'agent_accepted_at' => null,
|
|
|
|
|
'receipt_deadline_at' => null,
|
|
|
|
|
'effect_may_have_applied' => 0,
|
|
|
|
|
'safe_to_resubmit' => 0,
|
|
|
|
|
]],
|
|
|
|
|
// The safety booleans look canonical for an unknown dispatched
|
|
|
|
|
// operation, but generation zero proves no Agent mutation crossed the
|
|
|
|
|
// boundary. Without an undispatched-finalizer incident this is a
|
|
|
|
|
// contradictory historical row, not an Agent recovery candidate.
|
|
|
|
|
[str_repeat('f8', 16), [
|
|
|
|
|
'dispatch_attempts' => 0,
|
|
|
|
|
'dispatched_at' => null,
|
|
|
|
|
'dispatch_started_at' => null,
|
|
|
|
|
'agent_accepted_at' => null,
|
|
|
|
|
'receipt_deadline_at' => null,
|
|
|
|
|
'effect_may_have_applied' => 1,
|
|
|
|
|
'safe_to_resubmit' => 0,
|
|
|
|
|
]],
|
|
|
|
|
];
|
|
|
|
|
foreach ($stateCases as [$operationId, $overrides]) {
|
|
|
|
|
[$stateId] = $insertSettingsJob(
|
|
|
|
|
$operationId,
|
|
|
|
|
['hostname' => 'damaged-state.example']
|
|
|
|
|
);
|
|
|
|
|
Capsule::table('cloud_jobs')->where('id', $stateId)->update(
|
|
|
|
|
$overrides + [
|
|
|
|
|
'finalization_incident' => 0,
|
|
|
|
|
'finalization_error_code' => null,
|
|
|
|
|
'available_at' => 0,
|
|
|
|
|
'lease_token' => null,
|
|
|
|
|
'lease_until' => 0,
|
|
|
|
|
]
|
|
|
|
|
);
|
|
|
|
|
$beforeStateRequests = count($client->requests);
|
|
|
|
|
$stateService = new CloudAutomaticOperationRecoveryService();
|
|
|
|
|
$assert($stateService->tick(),
|
|
|
|
|
'A damaged leaf terminal tuple left the recovery queue.');
|
|
|
|
|
$stateJob = CloudJob::findOrFail($stateId);
|
|
|
|
|
$assert((bool) $stateJob->finalization_incident
|
|
|
|
|
&& (string) $stateJob->finalization_error_code
|
|
|
|
|
=== 'recovery_instance_state_conflict'
|
|
|
|
|
&& (int) $stateJob->available_at >= time() + 290
|
|
|
|
|
&& (int) $stateJob->lease_until >= time() + 290
|
|
|
|
|
&& count($client->requests) === $beforeStateRequests,
|
|
|
|
|
'A damaged leaf terminal tuple contacted Agent or lost backoff.');
|
|
|
|
|
Capsule::table('cloud_jobs')->where('id', $stateId)->update([
|
|
|
|
|
'available_at' => time() + 3600,
|
|
|
|
|
'lease_until' => time() + 3600,
|
|
|
|
|
]);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// A completed Agent effect can still carry a sticky PHP projection
|
|
|
|
|
// incident. Both resolved outcomes remain in the local retry queue, retain
|
|
|
|
|
// their concrete code, and must never trigger another Agent request.
|
|
|
|
|
$resolvedIncidentCases = [
|
|
|
|
|
[
|
|
|
|
|
str_repeat('f9', 16),
|
|
|
|
|
'applied',
|
|
|
|
|
'succeeded',
|
|
|
|
|
'recovery_payload_unavailable',
|
|
|
|
|
],
|
|
|
|
|
[
|
|
|
|
|
str_repeat('fa', 16),
|
|
|
|
|
'not_applied',
|
|
|
|
|
'failed',
|
|
|
|
|
'recovery_operation_identity_conflict',
|
|
|
|
|
],
|
|
|
|
|
];
|
|
|
|
|
foreach ($resolvedIncidentCases as [
|
|
|
|
|
$operationId,
|
|
|
|
|
$resolution,
|
|
|
|
|
$outcome,
|
|
|
|
|
$incidentCode,
|
|
|
|
|
]) {
|
|
|
|
|
[$resolvedIncidentId] = $insertSettingsJob(
|
|
|
|
|
$operationId,
|
|
|
|
|
['hostname' => 'resolved-local-incident.example']
|
|
|
|
|
);
|
|
|
|
|
Capsule::table('cloud_jobs')
|
|
|
|
|
->where('id', $resolvedIncidentId)->update([
|
|
|
|
|
'outcome' => $outcome,
|
|
|
|
|
'status' => $outcome,
|
|
|
|
|
'dispatch_attempts' => 1,
|
|
|
|
|
'effect_may_have_applied' => 0,
|
|
|
|
|
'safe_to_resubmit' => 0,
|
|
|
|
|
'effect_resolution' => $resolution,
|
|
|
|
|
'effect_resolved_at' => time() - 1,
|
|
|
|
|
'finalization_incident' => 1,
|
|
|
|
|
'finalization_error_code' => $incidentCode,
|
|
|
|
|
'available_at' => 0,
|
|
|
|
|
'lease_token' => null,
|
|
|
|
|
'lease_until' => 0,
|
|
|
|
|
]);
|
|
|
|
|
$beforeResolvedIncidentRequests = count($client->requests);
|
|
|
|
|
$assert((new CloudAutomaticOperationRecoveryService())->tick(),
|
|
|
|
|
'A resolved sticky incident left the local recovery queue.');
|
|
|
|
|
$resolvedIncident = CloudJob::findOrFail($resolvedIncidentId);
|
|
|
|
|
$assert((bool) $resolvedIncident->finalization_incident
|
|
|
|
|
&& (string) $resolvedIncident->finalization_error_code
|
|
|
|
|
=== $incidentCode
|
|
|
|
|
&& (string) $resolvedIncident->effect_resolution === $resolution
|
|
|
|
|
&& (int) $resolvedIncident->available_at >= time() + 290
|
|
|
|
|
&& (int) $resolvedIncident->lease_until >= time() + 290
|
|
|
|
|
&& count($client->requests) === $beforeResolvedIncidentRequests,
|
|
|
|
|
'A resolved sticky incident contacted Agent, lost its code, '
|
|
|
|
|
. 'or failed to enter local backoff.');
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// The same rule applies to a replica child below an already-resolved
|
|
|
|
|
// aggregate. Child ancestry must not turn a local finalization incident
|
|
|
|
|
// back into an Agent recovery or remove its concrete cause.
|
|
|
|
|
[$resolvedIncidentParentId] = $insertSettingsJob(
|
|
|
|
|
str_repeat('fb', 16),
|
|
|
|
|
['hostname' => 'resolved-incident-parent.example']
|
|
|
|
|
);
|
|
|
|
|
Capsule::table('cloud_jobs')
|
|
|
|
|
->where('id', $resolvedIncidentParentId)->update([
|
|
|
|
|
'instance_id' => null,
|
|
|
|
|
'host_id' => null,
|
|
|
|
|
'kind' => 'template_import',
|
|
|
|
|
'execution_mode' => 'aggregate',
|
|
|
|
|
'outcome' => 'succeeded',
|
|
|
|
|
'status' => 'succeeded',
|
|
|
|
|
'attempts' => 0,
|
|
|
|
|
'dispatch_attempts' => 0,
|
|
|
|
|
'dispatch_started_at' => null,
|
|
|
|
|
'agent_accepted_at' => null,
|
|
|
|
|
'dispatched_at' => null,
|
|
|
|
|
'completion_event_id' => null,
|
|
|
|
|
'effect_may_have_applied' => 0,
|
|
|
|
|
'safe_to_resubmit' => 0,
|
|
|
|
|
'effect_resolution' => 'applied',
|
|
|
|
|
'effect_resolved_at' => time() - 1,
|
|
|
|
|
'finalization_incident' => 0,
|
|
|
|
|
'finalization_error_code' => null,
|
|
|
|
|
'available_at' => time() + 3600,
|
|
|
|
|
'lease_token' => null,
|
|
|
|
|
'lease_until' => time() + 3600,
|
|
|
|
|
]);
|
|
|
|
|
[$resolvedIncidentChildId] = $insertSettingsJob(
|
|
|
|
|
str_repeat('fc', 16),
|
|
|
|
|
['hostname' => 'resolved-incident-child.example']
|
|
|
|
|
);
|
|
|
|
|
Capsule::table('cloud_jobs')
|
|
|
|
|
->where('id', $resolvedIncidentChildId)->update([
|
|
|
|
|
'parent_job_id' => $resolvedIncidentParentId,
|
|
|
|
|
'kind' => 'template_import_replica',
|
|
|
|
|
'outcome' => 'failed',
|
|
|
|
|
'status' => 'failed',
|
|
|
|
|
'dispatch_attempts' => 1,
|
|
|
|
|
'effect_may_have_applied' => 0,
|
|
|
|
|
'safe_to_resubmit' => 0,
|
|
|
|
|
'effect_resolution' => 'not_applied',
|
|
|
|
|
'effect_resolved_at' => time() - 1,
|
|
|
|
|
'finalization_incident' => 1,
|
|
|
|
|
'finalization_error_code' => 'recovery_payload_unavailable',
|
|
|
|
|
'available_at' => 0,
|
|
|
|
|
'lease_token' => null,
|
|
|
|
|
'lease_until' => 0,
|
|
|
|
|
]);
|
|
|
|
|
$beforeResolvedIncidentChildRequests = count($client->requests);
|
|
|
|
|
$assert((new CloudAutomaticOperationRecoveryService())->tick(),
|
|
|
|
|
'A resolved sticky aggregate child left the local recovery queue.');
|
|
|
|
|
$resolvedIncidentChild = CloudJob::findOrFail(
|
|
|
|
|
$resolvedIncidentChildId
|
|
|
|
|
);
|
|
|
|
|
$assert((bool) $resolvedIncidentChild->finalization_incident
|
|
|
|
|
&& (string) $resolvedIncidentChild->finalization_error_code
|
|
|
|
|
=== 'recovery_payload_unavailable'
|
|
|
|
|
&& (string) $resolvedIncidentChild->effect_resolution
|
|
|
|
|
=== 'not_applied'
|
|
|
|
|
&& (int) $resolvedIncidentChild->available_at >= time() + 290
|
|
|
|
|
&& (int) $resolvedIncidentChild->lease_until >= time() + 290
|
|
|
|
|
&& count($client->requests)
|
|
|
|
|
=== $beforeResolvedIncidentChildRequests,
|
|
|
|
|
'A resolved sticky aggregate child contacted Agent, lost its code, '
|
|
|
|
|
. 'or failed to enter local backoff.');
|
|
|
|
|
|
|
|
|
|
[$aggregateStateId] = $insertSettingsJob(
|
|
|
|
|
str_repeat('f5', 16),
|
|
|
|
|
['hostname' => 'aggregate-state.example']
|
|
|
|
|
);
|
|
|
|
|
Capsule::table('cloud_jobs')->where('id', $aggregateStateId)->update([
|
|
|
|
|
'instance_id' => null,
|
|
|
|
|
'host_id' => null,
|
|
|
|
|
'kind' => 'template_import',
|
|
|
|
|
'execution_mode' => 'aggregate',
|
|
|
|
|
'dispatch_attempts' => 0,
|
|
|
|
|
'dispatched_at' => null,
|
|
|
|
|
'dispatch_started_at' => null,
|
|
|
|
|
'agent_accepted_at' => null,
|
|
|
|
|
'receipt_deadline_at' => null,
|
|
|
|
|
'safe_to_resubmit' => 1,
|
|
|
|
|
'finalization_incident' => 0,
|
|
|
|
|
'finalization_error_code' => null,
|
|
|
|
|
'available_at' => 0,
|
|
|
|
|
'lease_token' => null,
|
|
|
|
|
'lease_until' => 0,
|
|
|
|
|
]);
|
|
|
|
|
$beforeAggregateStateRequests = count($client->requests);
|
|
|
|
|
$aggregateStateService = new CloudAutomaticOperationRecoveryService();
|
|
|
|
|
$assert($aggregateStateService->tick(),
|
|
|
|
|
'A damaged aggregate-parent tuple left the recovery queue.');
|
|
|
|
|
$aggregateStateJob = CloudJob::findOrFail($aggregateStateId);
|
|
|
|
|
$assert((bool) $aggregateStateJob->finalization_incident
|
|
|
|
|
&& (string) $aggregateStateJob->finalization_error_code
|
|
|
|
|
=== 'recovery_aggregate_state_conflict'
|
|
|
|
|
&& (int) $aggregateStateJob->available_at >= time() + 290
|
|
|
|
|
&& (int) $aggregateStateJob->lease_until >= time() + 290
|
|
|
|
|
&& count($client->requests) === $beforeAggregateStateRequests,
|
|
|
|
|
'A damaged aggregate-parent tuple contacted Agent or lost backoff.');
|
|
|
|
|
Capsule::table('cloud_jobs')->where('id', $aggregateStateId)->update([
|
|
|
|
|
'available_at' => time() + 3600,
|
|
|
|
|
'lease_until' => time() + 3600,
|
|
|
|
|
]);
|
|
|
|
|
|
|
|
|
|
// A resolved aggregate parent must still surface and retain an anomalous
|
|
|
|
|
// Agent child; only the child enters the PHP-only incident loop.
|
|
|
|
|
[$resolvedParentId] = $insertSettingsJob(
|
|
|
|
|
str_repeat('f6', 16),
|
|
|
|
|
['hostname' => 'resolved-parent.example']
|
|
|
|
|
);
|
|
|
|
|
Capsule::table('cloud_jobs')->where('id', $resolvedParentId)->update([
|
|
|
|
|
'instance_id' => null,
|
|
|
|
|
'host_id' => null,
|
|
|
|
|
'kind' => 'template_import',
|
|
|
|
|
'execution_mode' => 'aggregate',
|
|
|
|
|
'dispatch_attempts' => 0,
|
|
|
|
|
'dispatched_at' => null,
|
|
|
|
|
'outcome' => 'succeeded',
|
|
|
|
|
'status' => 'succeeded',
|
|
|
|
|
'effect_may_have_applied' => 0,
|
|
|
|
|
'safe_to_resubmit' => 0,
|
|
|
|
|
'effect_resolution' => 'applied',
|
|
|
|
|
'available_at' => time() + 3600,
|
|
|
|
|
'lease_until' => time() + 3600,
|
|
|
|
|
]);
|
|
|
|
|
[$stateChildId] = $insertSettingsJob(
|
|
|
|
|
str_repeat('f7', 16),
|
|
|
|
|
['hostname' => 'damaged-child.example']
|
|
|
|
|
);
|
|
|
|
|
Capsule::table('cloud_jobs')->where('id', $stateChildId)->update([
|
|
|
|
|
'parent_job_id' => $resolvedParentId,
|
|
|
|
|
'kind' => 'template_import_replica',
|
|
|
|
|
'safe_to_resubmit' => 1,
|
|
|
|
|
'finalization_incident' => 0,
|
|
|
|
|
'finalization_error_code' => null,
|
|
|
|
|
'available_at' => 0,
|
|
|
|
|
'lease_token' => null,
|
|
|
|
|
'lease_until' => 0,
|
|
|
|
|
]);
|
|
|
|
|
$beforeStateChildRequests = count($client->requests);
|
|
|
|
|
$stateChildService = new CloudAutomaticOperationRecoveryService();
|
|
|
|
|
$assert($stateChildService->tick(),
|
|
|
|
|
'An anomalous aggregate child left the recovery queue.');
|
|
|
|
|
$stateChild = CloudJob::findOrFail($stateChildId);
|
|
|
|
|
$assert((bool) $stateChild->finalization_incident
|
|
|
|
|
&& (string) $stateChild->finalization_error_code
|
|
|
|
|
=== 'recovery_instance_state_conflict'
|
|
|
|
|
&& count($client->requests) === $beforeStateChildRequests,
|
|
|
|
|
'An anomalous aggregate child contacted Agent or lost its incident.');
|
|
|
|
|
|
|
|
|
|
// A transition with independently persisted evidence is claimed through
|
|
|
|
|
// the PHP-only public-network finalizer path. This intentionally uses a
|
|
|
|
|
// minimal transition fixture so the deep finalizer rejects it; the
|
|
|
|
|
// coordinator must back it off locally without contacting the Agent.
|
|
|
|
|
[$networkFinalizerId] = $insertSettingsJob(
|
|
|
|
|
str_repeat('fd', 16),
|
|
|
|
|
['hostname' => 'network-finalizer.example']
|
|
|
|
|
);
|
|
|
|
|
Capsule::table('cloud_public_network_transitions')->insert([
|
|
|
|
|
'id' => 1,
|
|
|
|
|
'state' => 'finalization_pending',
|
|
|
|
|
'effect_resolution' => 'applied',
|
|
|
|
|
'last_error_code' => 'local_finalization_failed',
|
|
|
|
|
]);
|
|
|
|
|
Capsule::table('cloud_jobs')->where('id', '<>', $networkFinalizerId)
|
|
|
|
|
->update([
|
|
|
|
|
'available_at' => time() + 3600,
|
|
|
|
|
'lease_until' => time() + 3600,
|
|
|
|
|
]);
|
|
|
|
|
Capsule::table('cloud_jobs')->where('id', $networkFinalizerId)->update([
|
|
|
|
|
'kind' => 'network_change',
|
|
|
|
|
'public_network_transition_id' => 1,
|
|
|
|
|
'finalization_incident' => 1,
|
|
|
|
|
'finalization_error_code' => 'recovery_internal_error',
|
|
|
|
|
'available_at' => 0,
|
|
|
|
|
'lease_token' => null,
|
|
|
|
|
'lease_until' => 0,
|
|
|
|
|
]);
|
|
|
|
|
$beforeNetworkFinalizerRequests = count($client->requests);
|
|
|
|
|
$networkFinalizerService = new CloudAutomaticOperationRecoveryService();
|
|
|
|
|
$assert($networkFinalizerService->tick(),
|
|
|
|
|
'A pending public-network finalizer left the recovery queue.');
|
|
|
|
|
$networkFinalizer = CloudJob::findOrFail($networkFinalizerId);
|
|
|
|
|
$assert($networkFinalizer->lease_token === null
|
|
|
|
|
&& (int) $networkFinalizer->available_at >= time() + 290
|
|
|
|
|
&& (int) $networkFinalizer->lease_until >= time() + 290
|
|
|
|
|
&& (bool) $networkFinalizer->finalization_incident
|
|
|
|
|
&& count($client->requests) === $beforeNetworkFinalizerRequests,
|
|
|
|
|
'Public-network local finalizer contacted Agent or lost backoff.');
|
|
|
|
|
|
|
|
|
|
// Portable restore evidence has its own PHP-only finalizer lane. A
|
|
|
|
|
// deliberately incomplete fixture makes the deep finalizer reject the
|
|
|
|
|
// attempt, but the coordinator must still back it off locally and must
|
|
|
|
|
// never inspect or replay the Agent restore mutation.
|
|
|
|
|
[$portableFinalizerId] = $insertSettingsJob(
|
|
|
|
|
str_repeat('fe', 16),
|
|
|
|
|
['hostname' => 'portable-restore-finalizer.example']
|
|
|
|
|
);
|
|
|
|
|
Capsule::table('cloud_jobs')->where('id', '<>', $portableFinalizerId)
|
|
|
|
|
->update([
|
|
|
|
|
'available_at' => time() + 3600,
|
|
|
|
|
'lease_until' => time() + 3600,
|
|
|
|
|
]);
|
|
|
|
|
Capsule::table('cloud_jobs')->where('id', $portableFinalizerId)->update([
|
|
|
|
|
'kind' => 'backup_restore',
|
|
|
|
|
'finalization_incident' => 1,
|
|
|
|
|
'finalization_error_code' => 'local_finalization_failed',
|
|
|
|
|
'available_at' => 0,
|
|
|
|
|
'lease_token' => null,
|
|
|
|
|
'lease_until' => 0,
|
|
|
|
|
]);
|
|
|
|
|
Capsule::table('cloud_backup_restore_attempts')->insert([
|
|
|
|
|
'job_id' => $portableFinalizerId,
|
|
|
|
|
'state' => 'finalization_pending',
|
|
|
|
|
'effect_resolution' => 'applied',
|
|
|
|
|
'active_attempt' => 1,
|
|
|
|
|
'last_error_code' => '',
|
|
|
|
|
]);
|
|
|
|
|
$beforePortableFinalizerRequests = count($client->requests);
|
|
|
|
|
$portableFinalizerService = new CloudAutomaticOperationRecoveryService();
|
|
|
|
|
$assert($portableFinalizerService->tick(),
|
|
|
|
|
'A pending portable restore finalizer left the recovery queue.');
|
|
|
|
|
$portableFinalizer = CloudJob::findOrFail($portableFinalizerId);
|
|
|
|
|
$assert($portableFinalizer->lease_token === null
|
|
|
|
|
&& (int) $portableFinalizer->available_at >= time() + 290
|
|
|
|
|
&& (int) $portableFinalizer->lease_until >= time() + 290
|
|
|
|
|
&& (bool) $portableFinalizer->finalization_incident
|
|
|
|
|
&& count($client->requests) === $beforePortableFinalizerRequests,
|
|
|
|
|
'Portable restore local finalizer contacted Agent or lost backoff.');
|
|
|
|
|
|
|
|
|
|
foreach ($client->requests as $request) {
|
|
|
|
|
$allowed = preg_match(
|
|
|
|
|
'#^/v1/operations/[a-f0-9]{32}/(?:inspect|fence-resolution|resolve-fence)$#D',
|
|
|
|
|
$request['target']
|
|
|
|
|
) === 1;
|
|
|
|
|
$assert($allowed
|
|
|
|
|
&& ($request['method'] === 'GET'
|
|
|
|
|
|| ($request['method'] === 'POST'
|
|
|
|
|
&& str_ends_with(
|
|
|
|
|
$request['target'],
|
|
|
|
|
'/resolve-fence'
|
|
|
|
|
))),
|
|
|
|
|
'Automatic recovery called an Agent mutation endpoint.');
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
echo "Cloud automatic operation recovery tests passed.\n";
|
|
|
|
|
}
|