You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
virt/web/tests/CloudAdminOperationRecovery...

854 lines
35 KiB
PHP

This file contains ambiguous Unicode characters!

This file contains ambiguous Unicode characters that may be confused with others in your current locale. If your use case is intentional and legitimate, you can safely ignore this warning. Use the Escape button to highlight these characters.

<?php
namespace App\Services\Cloud {
final class CloudSecrets
{
private static array $values = [];
public static function encrypt(array $value, string $purpose): string
{
$key = bin2hex(random_bytes(16));
self::$values[$key] = $value;
return 'admin-recovery-test:' . $key;
}
public static function decrypt(string $value, string $purpose): array
{
$key = substr($value, strlen('admin-recovery-test:'));
if (!array_key_exists($key, self::$values)) {
throw new \RuntimeException('Missing admin recovery payload.');
}
return self::$values[$key];
}
}
final class RecoveryTestAgentClient
{
public array $requests = [];
public array $responses = [];
public function request(
string $method,
string $target,
array $actor,
?array $payload = null
): array {
$request = compact('method', 'target', 'actor', 'payload');
$this->requests[] = $request;
return $this->respond($request);
}
public function systemRecoveryRequest(
string $method,
string $target,
string $operationId,
?array $payload = null
): array {
$actor = [
'user_id' => 0,
'is_admin' => false,
'is_super_admin' => false,
'can_manage' => false,
'system_recovery' => true,
'operation_id' => $operationId,
];
$request = compact('method', 'target', 'actor', 'payload');
$request['system_recovery'] = true;
$this->requests[] = $request;
return $this->respond($request);
}
private function respond(array $request): array
{
if ($this->responses === []) {
throw new \RuntimeException('Unexpected Agent request.');
}
$response = array_shift($this->responses);
if (is_callable($response)) {
$response = $response($request);
}
if ($response instanceof \Throwable) {
throw $response;
}
if (!is_array($response)) {
throw new \RuntimeException('Invalid recovery test response.');
}
return $response;
}
}
final class CloudAgentSettingsService
{
public static ?RecoveryTestAgentClient $client = null;
public static array $calls = [];
public static function clientForConfiguration(
int $hostId,
float $timeoutSeconds = 120.0
): RecoveryTestAgentClient {
self::$calls[] = [$hostId, $timeoutSeconds];
return self::$client
?? throw new \RuntimeException('Missing recovery test client.');
}
}
}
namespace {
require dirname(__DIR__) . '/vendor/autoload.php';
use App\Services\Cloud\CloudAdminOperationRecoveryService as Recovery;
use App\Services\Cloud\CloudAgentError;
use App\Services\Cloud\CloudAgentSettingsService;
use App\Services\Cloud\CloudError;
use App\Services\Cloud\CloudOperationRecoveryIdentity;
use App\Services\Cloud\CloudSecrets;
use App\Services\Cloud\RecoveryTestAgentClient;
use App\Models\CloudJob;
use Illuminate\Database\Capsule\Manager as Capsule;
use Illuminate\Database\Schema\Blueprint;
$assert = static function (bool $condition, string $message): void {
if (!$condition) {
throw new RuntimeException($message);
}
};
$reject = static function (callable $callback, int $status,
?string $code = null) use ($assert): void {
try {
$callback();
} catch (CloudError $error) {
$assert($error->httpStatus === $status,
'Unexpected HTTP status: ' . $error->httpStatus);
if ($code !== null) {
$assert($error instanceof CloudAgentError
&& $error->agentCode === $code,
'Unexpected Agent error code.');
}
return;
}
throw new RuntimeException('Invalid recovery request was accepted.');
};
$capsule = new Capsule();
$capsule->addConnection([
'driver' => 'sqlite', 'database' => ':memory:',
], 'default');
$capsule->setAsGlobal();
$capsule->bootEloquent();
$capsule->schema('default')->create('cloud_jobs',
static function (Blueprint $table): void {
$table->increments('id');
$table->string('operation_id');
$table->string('request_idempotency_key');
$table->string('request_fingerprint');
$table->integer('host_id');
$table->string('kind');
$table->string('execution_mode')->default('agent');
$table->string('phase');
$table->string('outcome')->nullable();
$table->string('status');
$table->integer('user_id')->default(99);
$table->integer('instance_id')->nullable();
$table->integer('order_id')->nullable();
$table->integer('parent_job_id')->nullable();
$table->integer('snapshot_id')->nullable();
$table->integer('attempts')->default(1);
$table->integer('dispatch_attempts')->default(1);
$table->integer('dispatch_deadline_at')->default(1);
$table->integer('dispatch_started_at')->nullable();
$table->integer('agent_accepted_at')->nullable();
$table->integer('dispatched_at')->nullable();
$table->integer('receipt_deadline_at')->nullable();
$table->string('completion_event_id')->nullable();
$table->integer('terminal_at')->nullable();
$table->boolean('effect_may_have_applied')->nullable();
$table->boolean('safe_to_resubmit')->nullable();
$table->string('effect_resolution')->default('not_applicable');
$table->string('effect_resolution_reason')->nullable();
$table->integer('available_at')->default(0);
$table->integer('lease_until')->default(0);
$table->string('error_code')->nullable();
$table->boolean('finalization_incident')->default(0);
$table->string('finalization_error_code')->nullable();
$table->text('result_json')->nullable();
$table->text('sealed_payload')->default('forbidden-payload');
$table->string('lease_token')->nullable();
$table->integer('created_at')->nullable();
$table->integer('updated_at')->nullable();
});
$capsule->schema('default')->create('cloud_instances',
static function (Blueprint $table): void {
$table->increments('id');
$table->string('uuid');
$table->integer('host_id');
$table->integer('user_id');
$table->string('status');
});
$now = time();
foreach ([
41 => 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa',
42 => 'bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb',
43 => 'cccccccccccccccccccccccccccccccc',
44 => 'dddddddddddddddddddddddddddddddd',
] as $id => $operationId) {
Capsule::table('cloud_jobs')->insert([
'id' => $id, 'operation_id' => $operationId, 'host_id' => 7,
'request_idempotency_key' => $operationId,
'request_fingerprint' => hash('sha256', $operationId),
'kind' => 'create', 'order_id' => 100,
'phase' => 'terminal', 'outcome' => 'failed',
'status' => 'failed', 'dispatched_at' => $now - 90,
'dispatch_started_at' => $now - 100,
'agent_accepted_at' => $now - 95,
'receipt_deadline_at' => $now + 30,
'terminal_at' => $now - 30,
'effect_may_have_applied' => true,
'safe_to_resubmit' => false,
'effect_resolution' => 'unknown',
'sealed_payload' => CloudSecrets::encrypt([
'method' => 'POST',
'request_target' => '/v1/vms',
'body' => [
'source' => 'cloud',
'name' => 'vm-recovery-' . $id,
'vcpu' => 2,
'memory_mib' => 2048,
'disk_gib' => 40,
'bandwidth_mbps' => 100,
'network' => 'bridge:br0',
'template_id' => 'debian-13',
'template_revision' => 1,
'template_generation_token' => str_repeat('1', 32),
'root_password' => 'StrongPassword!123',
'public_pool_key' => str_repeat('2', 32),
'public_pool_cidr' => '203.0.113.0/24',
'public_ip_cidr' => '203.0.113.41/24',
'gateway' => '203.0.113.1',
'dns_servers' => ['1.1.1.1'],
'platform_network_policy_snapshot' => [
'schema_version' => 1,
],
],
'template_metadata' => [
'template_id' => 'debian-13',
'os_source' => 'cloud_template',
'os_name' => 'Debian 13',
'login_name' => 'root',
'debian_12_network_fix' => false,
],
], 'job-payload'),
'created_at' => $now - 120, 'updated_at' => $now - 30,
]);
}
Capsule::table('cloud_instances')->insert([
'id' => 45,
'uuid' => '123e4567-e89b-42d3-a456-426614174000',
'host_id' => 7,
'user_id' => 99,
'status' => 'active',
]);
Capsule::table('cloud_jobs')->insert([
'id' => 45,
'operation_id' => 'eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee',
'request_idempotency_key' =>
'eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee',
'request_fingerprint' => hash(
'sha256',
'eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee'
),
'host_id' => 7,
'kind' => 'instance_settings',
'phase' => 'terminal',
'outcome' => 'failed',
'status' => 'failed',
'dispatched_at' => $now - 90,
'dispatch_started_at' => $now - 100,
'agent_accepted_at' => $now - 95,
'receipt_deadline_at' => $now + 30,
'terminal_at' => $now - 30,
'effect_may_have_applied' => true,
'safe_to_resubmit' => false,
'effect_resolution' => 'unknown',
'effect_resolution_reason' => 'automatic_recovery:v1:'
. ($now - 60)
. ':1:prepared:automatic_applied_failed_fence:'
. str_repeat('a', 64) . ':' . str_repeat('b', 64),
'instance_id' => 45,
'sealed_payload' => CloudSecrets::encrypt([
'method' => 'POST',
'request_target' => '/v1/vms/'
. '123e4567-e89b-42d3-a456-426614174000/settings',
'body' => [
'operation_id' => 'eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee',
'hostname' => 'safe.example',
'confirm_restart' => true,
],
'owner_id' => 99,
'can_manage' => false,
'settings_fields' => ['hostname'],
'credential' => null,
], 'job-payload'),
'created_at' => $now - 120,
'updated_at' => $now - 30,
]);
$identityA = CloudOperationRecoveryIdentity::expected(CloudJob::find(41));
$identityB = CloudOperationRecoveryIdentity::expected(CloudJob::find(42));
$identityC = CloudOperationRecoveryIdentity::expected(CloudJob::find(43));
$identityD = CloudOperationRecoveryIdentity::expected(CloudJob::find(44));
$identityE = CloudOperationRecoveryIdentity::expected(CloudJob::find(45));
$admin = ['user_id' => 99, 'is_admin' => true,
'is_super_admin' => true, 'can_manage' => true];
$readonly = ['user_id' => 98, 'is_admin' => true,
'is_super_admin' => false, 'can_manage' => false];
$client = new RecoveryTestAgentClient();
CloudAgentSettingsService::$client = $client;
$before = count($client->requests);
$beforeClientBuilds = count(CloudAgentSettingsService::$calls);
Capsule::table('cloud_jobs')->where('id', 41)->update([
'execution_mode' => 'local',
'phase' => 'accepted',
'outcome' => null,
'status' => 'queued',
]);
$reject(static fn () => Recovery::inspect($admin, '41'), 409);
$assert(count($client->requests) === $before
&& count(CloudAgentSettingsService::$calls) === $beforeClientBuilds,
'A nonterminal supported Agent kind with a damaged execution mode reached the Agent client.');
Capsule::table('cloud_jobs')->where('id', 41)->update([
'kind' => 'template_import',
'execution_mode' => 'agent',
]);
$reject(static fn () => Recovery::inspect($admin, '41'), 409);
$assert(count($client->requests) === $before
&& count(CloudAgentSettingsService::$calls) === $beforeClientBuilds,
'A malformed aggregate identity reached the Agent client.');
Capsule::table('cloud_jobs')->where('id', 41)->update([
'kind' => 'create',
'execution_mode' => 'agent',
'phase' => 'terminal',
'outcome' => 'failed',
'status' => 'failed',
]);
Capsule::table('cloud_jobs')->where('id', 45)->update([
'receipt_deadline_at' => null,
]);
$reject(static fn () => Recovery::inspect($admin, '45'), 409);
$assert(count($client->requests) === $before
&& count(CloudAgentSettingsService::$calls) === $beforeClientBuilds,
'A terminal Agent task with a damaged immutable envelope reached the Agent client.');
Capsule::table('cloud_jobs')->where('id', 45)->update([
'receipt_deadline_at' => $now + 30,
'outcome' => 'succeeded',
'status' => 'succeeded',
'effect_may_have_applied' => false,
'safe_to_resubmit' => false,
'effect_resolution' => 'applied',
'finalization_incident' => true,
'finalization_error_code' => 'recovery_ledger_conflict',
]);
$reject(static fn () => Recovery::inspect($admin, '45'), 409);
$assert(count($client->requests) === $before
&& count(CloudAgentSettingsService::$calls) === $beforeClientBuilds,
'A resolved sticky recovery incident reached the Agent client.');
Capsule::table('cloud_jobs')->where('id', 45)->update([
'outcome' => 'failed',
'status' => 'failed',
'effect_may_have_applied' => true,
'safe_to_resubmit' => false,
'effect_resolution' => 'unknown',
'finalization_incident' => false,
'finalization_error_code' => null,
]);
$before = count($client->requests);
Capsule::table('cloud_jobs')->where('id', 41)->update([
'kind' => 'host_storage_configure',
]);
$reject(static fn () => Recovery::inspectForRecovery('41'), 409);
$reject(static fn () => Recovery::resolveFenceForRecovery(
'41',
'not_applied',
'automatic recovery must reject unregistered task kinds',
[
'operation_id' => 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa',
'checked_at' => $now,
'effect_resolution' => 'not_applied',
'fence_resolution_required' => true,
'fence_state' => 'active',
]
), 409);
$reject(static fn () => Recovery::preparedFenceResolutionForRecovery(
'41',
'not_applied',
'automatic recovery must reject unregistered task kinds'
), 409);
$assert(count($client->requests) === $before,
'An unregistered task reached the system recovery Agent client.');
Capsule::table('cloud_jobs')->where('id', 41)->update([
'kind' => 'create',
]);
$operationA = 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa';
$client->responses[] = $createInspectionResponse = [
'operation_id' => $operationA,
'operation_status' => 'failed',
'checked_at' => $now,
'outcome' => 'failed',
'summary_code' => 'operation_failed',
'error_code' => 'creation_absent',
'effect_resolution' => 'not_applied',
'fence_resolution_required' => true,
'fence_state' => 'active',
'effect_projection' => [
'creation_absent' => true,
'managed_artifacts_absent' => true,
],
'abort_job' => ['available' => false],
'cleanup' => ['available' => false],
] + $identityA;
$inspection = Recovery::inspect($admin, '41');
$assert($inspection === [
'operation_id' => $operationA,
'action' => $identityA['action'],
'resource_id' => $identityA['resource_id'],
'request_fingerprint' => $identityA['request_fingerprint'],
'journal_status' => 'failed',
'checked_at' => $now,
'outcome' => 'failed',
'effect_resolution' => 'not_applied',
'fence_state' => 'active',
'fence_resolution_required' => true,
'summary_code' => 'operation_failed',
'summary' => '只读证据与宿主机失败结果一致',
'error_code' => 'creation_absent',
'error' => '宿主机未发现虚拟机或面板管理资源',
] && !array_key_exists('effect_projection', $inspection),
'Read-only inspection was not reduced to the bounded DTO.');
$assert(CloudAgentSettingsService::$calls === [[7, 30.0]]
&& $client->requests[0]['method'] === 'GET'
&& $client->requests[0]['target']
=== '/v1/operations/' . $operationA . '/inspect'
&& $client->requests[0]['payload'] === null,
'Inspection did not use exactly one signed read-only request.');
$assert(Capsule::table('cloud_jobs')->where('id', 41)
->value('effect_resolution') === 'unknown',
'Read-only inspection changed the task or its safety fence.');
$client->responses[] = $createInspectionResponse;
$createInspection = Recovery::inspectForRecovery('41');
$createRequest = $client->requests[array_key_last($client->requests)];
$assert(($createInspection['error_code'] ?? null) === 'creation_absent'
&& ($createInspection['effect_projection']['creation_absent'] ?? null) === true
&& ($createInspection['effect_projection']['managed_artifacts_absent'] ?? null) === true
&& $createRequest['method'] === 'GET'
&& ($createRequest['system_recovery'] ?? false) === true
&& $createRequest['actor']['operation_id'] === $operationA
&& Capsule::table('cloud_jobs')->where('id', 41)
->value('effect_resolution') === 'unknown',
'Create inspection did not remain operation-bound and read-only.');
$before = count($client->requests);
$reject(static fn () => Recovery::resolveFenceForRecovery(
'41', 'not_applied',
'automatic create recovery requires durable financial intent',
$createInspection
), 409);
$reject(static fn () => Recovery::resolveFenceForRecovery(
'41', 'applied',
'automatic create recovery cannot infer a successful instance',
$createInspection
), 409);
$reject(static fn () => Recovery::preparedFenceResolutionForRecovery(
'41', 'not_applied',
'automatic create recovery requires durable financial intent'
), 409);
$assert(count($client->requests) === $before,
'An unprepared create reached the system Agent fence routes.');
$client->responses[] = ['operation_id' => $operationA,
'operation_status' => 'uncertain', 'checked_at' => $now,
'outcome' => 'uncertain', 'summary_code' => 'evidence_incomplete'];
$reject(static fn () => Recovery::inspect($admin, '41'), 502,
'agent_invalid_inspection');
$client->responses[] = [
'operation_id' => $operationA,
'operation_status' => 'failed',
'checked_at' => $now,
'outcome' => 'failed',
'summary_code' => 'operation_failed',
'error_code' => 'creation_absent',
'effect_resolution' => 'not_applied',
'fence_resolution_required' => true,
'fence_state' => 'active',
'effect_projection' => [
'creation_absent' => true,
'managed_artifacts_absent' => true,
],
'abort_job' => ['available' => false],
'cleanup' => ['available' => false],
'raw_error' => '/srv/private/secret.qcow2',
] + $identityA;
$reject(static fn () => Recovery::inspect($admin, '41'), 502,
'agent_invalid_inspection');
$operationE = 'eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee';
$safeProjection = [
'uuid' => '123e4567-e89b-42d3-a456-426614174000',
'hostname' => 'safe.example',
'settings_changed' => true,
];
$client->responses[] = [
'operation_id' => $operationE,
'operation_status' => 'failed',
'checked_at' => $now,
'outcome' => 'succeeded',
'summary_code' => 'operation_succeeded',
'effect_resolution' => 'applied',
'fence_resolution_required' => true,
'fence_state' => 'active',
'effect_projection' => $safeProjection,
'abort_job' => ['available' => false],
'cleanup' => ['available' => false],
] + $identityE;
$before = count($client->requests);
$systemInspection = Recovery::inspectForRecovery('45');
$requests = array_slice($client->requests, $before);
$assert(($systemInspection['effect_resolution'] ?? null) === 'applied'
&& ($systemInspection['effect_projection'] ?? null)
=== $safeProjection
&& count($requests) === 1
&& ($requests[0]['system_recovery'] ?? false) === true
&& $requests[0]['actor'] === [
'user_id' => 0,
'is_admin' => false,
'is_super_admin' => false,
'can_manage' => false,
'system_recovery' => true,
'operation_id' => $operationE,
],
'Internal inspection lost its bounded recovery evidence or actor.');
$client->responses[] = [
'operation_id' => $operationE,
'operation_status' => 'failed',
'checked_at' => $now,
'outcome' => 'succeeded',
'summary_code' => 'operation_succeeded',
'effect_resolution' => 'applied',
'fence_resolution_required' => true,
'fence_state' => 'active',
'completion_delivery_state' => 'quarantined',
'completion_delivery_error_code' =>
'completion_receipt_invalid',
'effect_projection' => $safeProjection,
'abort_job' => ['available' => false],
'cleanup' => ['available' => false],
] + $identityE;
$poisonedInspection = Recovery::inspectForRecovery('45');
$assert(($poisonedInspection['completion_delivery_state'] ?? null)
=== 'quarantined'
&& ($poisonedInspection['completion_delivery_error_code'] ?? null)
=== 'completion_receipt_invalid'
&& str_contains(
(string) ($poisonedInspection['completion_delivery_error'] ?? ''),
'完成回执内容无效'
),
'A quarantined completion did not retain its bounded Chinese cause.');
$client->responses[] = [
'operation_id' => $operationE,
'operation_status' => 'failed',
'checked_at' => $now,
'outcome' => 'succeeded',
'summary_code' => 'operation_succeeded',
'effect_resolution' => 'applied',
'fence_state' => 'resolved',
'fence_resolution_required' => false,
'effect_projection' => $safeProjection,
'abort_job' => ['available' => false],
'cleanup' => ['available' => false],
] + $identityE;
$resolvedInspection = Recovery::inspectForRecovery('45');
$assert(($resolvedInspection['effect_resolution'] ?? null) === 'applied'
&& ($resolvedInspection['fence_state'] ?? null) === 'resolved'
&& ($resolvedInspection['fence_resolution_required'] ?? null)
=== false,
'A resolved Agent fence was collapsed into a never-fenced DTO.');
foreach ([
['effect_resolution' => 'trusted',
'effect_projection' => $safeProjection],
['effect_resolution' => 'applied', 'effect_projection' => [
'uuid' => '123e4567-e89b-42d3-a456-426614174000',
'root_password' => 'must-not-cross',
'settings_changed' => true,
]],
['effect_resolution' => 'applied', 'effect_projection' => [
'uuid' => '123e4567-e89b-42d3-a456-426614174000',
'future_field' => true,
'settings_changed' => true,
]],
['effect_resolution' => 'applied', 'effect_projection' => [
'uuid' => '123e4567-e89b-42d3-a456-426614174000',
'hostname' => str_repeat('x', 65537),
'settings_changed' => true,
]],
['effect_resolution' => 'applied',
'effect_projection' => $safeProjection,
'completion_delivery_state' => 'quarantined',
'completion_delivery_error_code' => 'future_poison_reason'],
['effect_resolution' => 'applied',
'effect_projection' => $safeProjection,
'completion_delivery_state' => 'quarantined'],
] as $unsafeEvidence) {
$client->responses[] = [
'operation_id' => $operationE,
'operation_status' => 'failed',
'checked_at' => $now,
'outcome' => 'succeeded',
'summary_code' => 'operation_succeeded',
'fence_resolution_required' => true,
'fence_state' => 'active',
'abort_job' => ['available' => false],
'cleanup' => ['available' => false],
] + $unsafeEvidence + $identityE;
$reject(static fn () => Recovery::inspectForRecovery('45'), 502,
'agent_invalid_inspection');
}
$reason = '已人工只读核对虚拟机、磁盘与网络状态,确认原操作效果已经发生。';
$reasonDigest = hash('sha256', $reason);
$poisonedManualInspection = $inspection + [
'completion_delivery_state' => 'quarantined',
'completion_delivery_error_code' =>
'completion_operation_invalid',
'completion_delivery_error' =>
'Agent 完成回执对应的原操作记录无效,当前结果不能采信',
];
$before = count($client->requests);
$reject(static fn () => Recovery::resolveFence(
$admin,
'41',
'not_applied',
$reason,
$poisonedManualInspection
), 409);
$assert(count($client->requests) === $before,
'A quarantined completion reached the Agent fence resolver.');
$oldReceipt = [
'operation_id' => $operationA,
'resolution_id' => str_repeat('1', 32),
'resolution' => 'not_applied',
'inspection_checked_at' => $now - 60,
'reason_sha256' => $reasonDigest,
'resolved_at' => $now - 50,
'resolved_by' => 7,
'replayed' => false,
];
$before = count($client->requests);
$client->responses[] = $oldReceipt;
$recovered = Recovery::resolveFence(
$admin, '41', 'not_applied', $reason, $inspection
);
$requests = array_slice($client->requests, $before);
$assert($recovered['resolution_id'] === str_repeat('1', 32)
&& count($requests) === 1 && $requests[0]['method'] === 'GET'
&& str_ends_with($requests[0]['target'], '/fence-resolution')
&& ($requests[0]['actor']['operation_id'] ?? null) === $operationA,
'A prior Agent commit was not recovered read-only after a PHP crash.');
$notFound = static fn () => new CloudAgentError(
'missing', 404, 'fence_resolution_not_found', false, 404
);
$automaticReason = 'Agent 实时只读检查确认实例设置已经提交,PHP 账本预提交完成。';
$client->responses[] = [
'operation_id' => $operationE,
'resolution_id' => str_repeat('f', 32),
'resolution' => 'applied',
'inspection_checked_at' => $now,
'reason_sha256' => hash('sha256', $automaticReason),
'resolved_at' => $now,
'resolved_by' => 7,
'replayed' => false,
];
$before = count($client->requests);
$reject(static fn () => Recovery::resolveFenceForRecovery(
'45',
'applied',
$automaticReason,
$systemInspection
), 503, 'fence_resolution_read_failed');
$assert(count($client->requests) === $before + 1,
'System recovery accepted a positive human resolved_by value.');
$client->responses[] = $notFound();
$client->responses[] = new CloudAgentError(
'lost response', 503, 'agent_disconnected', true, 503
);
$client->responses[] = static function (array $request) use (
$client, $operationE, $now
): array {
$post = $client->requests[count($client->requests) - 2];
return [
'operation_id' => $operationE,
'resolution_id' => $post['payload']['resolution_id'],
'resolution' => $post['payload']['resolution'],
'inspection_checked_at' =>
$post['payload']['inspection_checked_at'],
'reason_sha256' => $post['payload']['reason_sha256'],
'resolved_at' => $now,
'resolved_by' => 0,
'replayed' => false,
];
};
$before = count($client->requests);
$automaticFence = Recovery::resolveFenceForRecovery(
'45',
'applied',
$automaticReason,
$systemInspection
);
$requests = array_slice($client->requests, $before);
$assert(array_column($requests, 'method') === ['GET', 'POST', 'GET']
&& count(array_filter($requests,
static fn (array $request): bool =>
$request['method'] === 'POST')) === 1
&& count(array_filter($requests,
static fn (array $request): bool =>
($request['system_recovery'] ?? false) === true)) === 3
&& $automaticFence['resolution'] === 'applied'
&& $automaticFence['resolved_by'] === '0',
'System recovery did not use one exact-operation fence write and read verification.');
$operationB = 'bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb';
$inspectionB = [
'operation_id' => $operationB, 'checked_at' => $now,
'outcome' => 'failed', 'summary_code' => 'operation_failed',
'error_code' => 'creation_absent',
] + $identityB;
$client->responses[] = $notFound();
$client->responses[] = static function (array $request) use (
$operationB, $now
): array {
return [
'operation_id' => $operationB,
'resolution_id' => $request['payload']['resolution_id'],
'resolution' => $request['payload']['resolution'],
'inspection_checked_at' =>
$request['payload']['inspection_checked_at'],
'reason_sha256' => $request['payload']['reason_sha256'],
'resolved_at' => $now,
'resolved_by' => '7',
'replayed' => false,
];
};
$before = count($client->requests);
$created = Recovery::resolveFence(
$admin, '42', 'not_applied', $reason, $inspectionB
);
$requests = array_slice($client->requests, $before);
$assert(count($requests) === 2
&& array_column($requests, 'method') === ['GET', 'POST']
&& str_ends_with($requests[1]['target'], '/resolve-fence')
&& preg_match('/^[a-f0-9]{32}$/D',
$requests[1]['payload']['resolution_id']) === 1
&& $created['resolution'] === 'not_applied',
'New fence resolution was not dispatched exactly once.');
$operationC = 'cccccccccccccccccccccccccccccccc';
$inspectionC = [
'operation_id' => $operationC, 'checked_at' => $now,
'outcome' => 'failed', 'summary_code' => 'operation_failed',
'error_code' => 'creation_absent',
] + $identityC;
$client->responses[] = $notFound();
$client->responses[] = new CloudAgentError(
'lost response', 503, 'agent_disconnected', true, 503
);
$client->responses[] = static function (array $request) use (
$client, $operationC, $now
): array {
$post = $client->requests[count($client->requests) - 2];
return [
'operation_id' => $operationC,
'resolution_id' => $post['payload']['resolution_id'],
'resolution' => $post['payload']['resolution'],
'inspection_checked_at' =>
$post['payload']['inspection_checked_at'],
'reason_sha256' => $post['payload']['reason_sha256'],
'resolved_at' => $now, 'resolved_by' => 7,
'replayed' => false,
];
};
$before = count($client->requests);
$verified = Recovery::resolveFence(
$admin, '43', 'not_applied', $reason, $inspectionC
);
$requests = array_slice($client->requests, $before);
$assert(array_column($requests, 'method') === ['GET', 'POST', 'GET']
&& count(array_filter($requests,
static fn (array $request): bool => $request['method'] === 'POST'))
=== 1
&& $verified['resolution'] === 'not_applied',
'Lost fence response caused a POST retry instead of one GET verification.');
$operationD = 'dddddddddddddddddddddddddddddddd';
$inspectionD = [
'operation_id' => $operationD, 'checked_at' => $now,
'outcome' => 'failed', 'summary_code' => 'operation_failed',
'error_code' => 'creation_absent',
] + $identityD;
$client->responses[] = array_replace($oldReceipt, [
'operation_id' => $operationD,
'resolved_by' => 0,
]);
$before = count($client->requests);
$reject(static fn () => Recovery::resolveFence(
$admin, '44', 'not_applied', $reason, $inspectionD
), 503, 'fence_resolution_read_failed');
$assert(count($client->requests) === $before + 1,
'A forged system actor ID reached the human fence write path.');
$client->responses[] = array_replace($oldReceipt, [
'operation_id' => $operationD,
'resolution' => 'applied',
]);
$before = count($client->requests);
$reject(static fn () => Recovery::resolveFence(
$admin, '44', 'not_applied', $reason, $inspectionD
), 409);
$assert(count($client->requests) === $before + 1,
'Conflicting durable fence receipt triggered an Agent write.');
$before = count($client->requests);
$reject(static fn () => Recovery::resolveFence(
$readonly, '41', 'not_applied', $reason, $inspection
), 403);
$reject(static fn () => Recovery::resolveFence(
$admin, '41', 'not_applied', $reason,
[
'operation_id' => $operationA, 'checked_at' => $now - 121,
'outcome' => 'failed', 'summary_code' => 'operation_failed',
'error_code' => 'creation_absent',
] + $identityA
), 409);
$assert(count($client->requests) === $before,
'Unauthorized or stale resolution reached the Agent.');
$source = file_get_contents(dirname(__DIR__)
. '/app/Services/Cloud/CloudAdminOperationRecoveryService.php');
$assert(is_string($source)
&& !str_contains($source, 'function abortJob')
&& !str_contains($source, 'function cleanup')
&& !str_contains($source, 'confirm-inspection')
&& preg_match_all('/\$call\(\s*[\r\n ]*\'POST\'/m',
$source) === 1,
'Retired recovery mutations or a second Agent POST remain reachable.');
echo "Administrator read-only inspection and single-write fence recovery tests passed.\n";
}