|
|
<?php
|
|
|
|
|
|
namespace App\Services\Cloud {
|
|
|
final class CloudSecrets
|
|
|
{
|
|
|
private static array $values = [];
|
|
|
|
|
|
public static function encrypt(array $value, string $purpose): string
|
|
|
{
|
|
|
$key = bin2hex(random_bytes(16));
|
|
|
self::$values[$key] = $value;
|
|
|
return 'admin-recovery-test:' . $key;
|
|
|
}
|
|
|
|
|
|
public static function decrypt(string $value, string $purpose): array
|
|
|
{
|
|
|
$key = substr($value, strlen('admin-recovery-test:'));
|
|
|
if (!array_key_exists($key, self::$values)) {
|
|
|
throw new \RuntimeException('Missing admin recovery payload.');
|
|
|
}
|
|
|
return self::$values[$key];
|
|
|
}
|
|
|
}
|
|
|
|
|
|
final class RecoveryTestAgentClient
|
|
|
{
|
|
|
public array $requests = [];
|
|
|
public array $responses = [];
|
|
|
|
|
|
public function request(
|
|
|
string $method,
|
|
|
string $target,
|
|
|
array $actor,
|
|
|
?array $payload = null
|
|
|
): array {
|
|
|
$request = compact('method', 'target', 'actor', 'payload');
|
|
|
$this->requests[] = $request;
|
|
|
return $this->respond($request);
|
|
|
}
|
|
|
|
|
|
public function systemRecoveryRequest(
|
|
|
string $method,
|
|
|
string $target,
|
|
|
string $operationId,
|
|
|
?array $payload = null
|
|
|
): array {
|
|
|
$actor = [
|
|
|
'user_id' => 0,
|
|
|
'is_admin' => false,
|
|
|
'is_super_admin' => false,
|
|
|
'can_manage' => false,
|
|
|
'system_recovery' => true,
|
|
|
'operation_id' => $operationId,
|
|
|
];
|
|
|
$request = compact('method', 'target', 'actor', 'payload');
|
|
|
$request['system_recovery'] = true;
|
|
|
$this->requests[] = $request;
|
|
|
return $this->respond($request);
|
|
|
}
|
|
|
|
|
|
private function respond(array $request): array
|
|
|
{
|
|
|
if ($this->responses === []) {
|
|
|
throw new \RuntimeException('Unexpected Agent request.');
|
|
|
}
|
|
|
$response = array_shift($this->responses);
|
|
|
if (is_callable($response)) {
|
|
|
$response = $response($request);
|
|
|
}
|
|
|
if ($response instanceof \Throwable) {
|
|
|
throw $response;
|
|
|
}
|
|
|
if (!is_array($response)) {
|
|
|
throw new \RuntimeException('Invalid recovery test response.');
|
|
|
}
|
|
|
return $response;
|
|
|
}
|
|
|
}
|
|
|
|
|
|
final class CloudAgentSettingsService
|
|
|
{
|
|
|
public static ?RecoveryTestAgentClient $client = null;
|
|
|
public static array $calls = [];
|
|
|
|
|
|
public static function clientForConfiguration(
|
|
|
int $hostId,
|
|
|
float $timeoutSeconds = 120.0
|
|
|
): RecoveryTestAgentClient {
|
|
|
self::$calls[] = [$hostId, $timeoutSeconds];
|
|
|
return self::$client
|
|
|
?? throw new \RuntimeException('Missing recovery test client.');
|
|
|
}
|
|
|
}
|
|
|
}
|
|
|
|
|
|
namespace {
|
|
|
require dirname(__DIR__) . '/vendor/autoload.php';
|
|
|
|
|
|
use App\Services\Cloud\CloudAdminOperationRecoveryService as Recovery;
|
|
|
use App\Services\Cloud\CloudAgentError;
|
|
|
use App\Services\Cloud\CloudAgentSettingsService;
|
|
|
use App\Services\Cloud\CloudError;
|
|
|
use App\Services\Cloud\CloudOperationRecoveryIdentity;
|
|
|
use App\Services\Cloud\CloudSecrets;
|
|
|
use App\Services\Cloud\RecoveryTestAgentClient;
|
|
|
use App\Models\CloudJob;
|
|
|
use Illuminate\Database\Capsule\Manager as Capsule;
|
|
|
use Illuminate\Database\Schema\Blueprint;
|
|
|
|
|
|
$assert = static function (bool $condition, string $message): void {
|
|
|
if (!$condition) {
|
|
|
throw new RuntimeException($message);
|
|
|
}
|
|
|
};
|
|
|
$reject = static function (callable $callback, int $status,
|
|
|
?string $code = null) use ($assert): void {
|
|
|
try {
|
|
|
$callback();
|
|
|
} catch (CloudError $error) {
|
|
|
$assert($error->httpStatus === $status,
|
|
|
'Unexpected HTTP status: ' . $error->httpStatus);
|
|
|
if ($code !== null) {
|
|
|
$assert($error instanceof CloudAgentError
|
|
|
&& $error->agentCode === $code,
|
|
|
'Unexpected Agent error code.');
|
|
|
}
|
|
|
return;
|
|
|
}
|
|
|
throw new RuntimeException('Invalid recovery request was accepted.');
|
|
|
};
|
|
|
|
|
|
$capsule = new Capsule();
|
|
|
$capsule->addConnection([
|
|
|
'driver' => 'sqlite', 'database' => ':memory:',
|
|
|
], 'default');
|
|
|
$capsule->setAsGlobal();
|
|
|
$capsule->bootEloquent();
|
|
|
$capsule->schema('default')->create('cloud_jobs',
|
|
|
static function (Blueprint $table): void {
|
|
|
$table->increments('id');
|
|
|
$table->string('operation_id');
|
|
|
$table->string('request_idempotency_key');
|
|
|
$table->string('request_fingerprint');
|
|
|
$table->integer('host_id');
|
|
|
$table->string('kind');
|
|
|
$table->string('execution_mode')->default('agent');
|
|
|
$table->string('phase');
|
|
|
$table->string('outcome')->nullable();
|
|
|
$table->string('status');
|
|
|
$table->integer('user_id')->default(99);
|
|
|
$table->integer('instance_id')->nullable();
|
|
|
$table->integer('order_id')->nullable();
|
|
|
$table->integer('parent_job_id')->nullable();
|
|
|
$table->integer('snapshot_id')->nullable();
|
|
|
$table->integer('attempts')->default(1);
|
|
|
$table->integer('dispatch_attempts')->default(1);
|
|
|
$table->integer('dispatch_deadline_at')->default(1);
|
|
|
$table->integer('dispatch_started_at')->nullable();
|
|
|
$table->integer('agent_accepted_at')->nullable();
|
|
|
$table->integer('dispatched_at')->nullable();
|
|
|
$table->integer('receipt_deadline_at')->nullable();
|
|
|
$table->string('completion_event_id')->nullable();
|
|
|
$table->integer('terminal_at')->nullable();
|
|
|
$table->boolean('effect_may_have_applied')->nullable();
|
|
|
$table->boolean('safe_to_resubmit')->nullable();
|
|
|
$table->string('effect_resolution')->default('not_applicable');
|
|
|
$table->string('effect_resolution_reason')->nullable();
|
|
|
$table->integer('available_at')->default(0);
|
|
|
$table->integer('lease_until')->default(0);
|
|
|
$table->string('error_code')->nullable();
|
|
|
$table->boolean('finalization_incident')->default(0);
|
|
|
$table->string('finalization_error_code')->nullable();
|
|
|
$table->text('result_json')->nullable();
|
|
|
$table->text('sealed_payload')->default('forbidden-payload');
|
|
|
$table->string('lease_token')->nullable();
|
|
|
$table->integer('created_at')->nullable();
|
|
|
$table->integer('updated_at')->nullable();
|
|
|
});
|
|
|
$capsule->schema('default')->create('cloud_instances',
|
|
|
static function (Blueprint $table): void {
|
|
|
$table->increments('id');
|
|
|
$table->string('uuid');
|
|
|
$table->integer('host_id');
|
|
|
$table->integer('user_id');
|
|
|
$table->string('status');
|
|
|
});
|
|
|
$now = time();
|
|
|
foreach ([
|
|
|
41 => 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa',
|
|
|
42 => 'bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb',
|
|
|
43 => 'cccccccccccccccccccccccccccccccc',
|
|
|
44 => 'dddddddddddddddddddddddddddddddd',
|
|
|
] as $id => $operationId) {
|
|
|
Capsule::table('cloud_jobs')->insert([
|
|
|
'id' => $id, 'operation_id' => $operationId, 'host_id' => 7,
|
|
|
'request_idempotency_key' => $operationId,
|
|
|
'request_fingerprint' => hash('sha256', $operationId),
|
|
|
'kind' => 'create', 'order_id' => 100,
|
|
|
'phase' => 'terminal', 'outcome' => 'failed',
|
|
|
'status' => 'failed', 'dispatched_at' => $now - 90,
|
|
|
'dispatch_started_at' => $now - 100,
|
|
|
'agent_accepted_at' => $now - 95,
|
|
|
'receipt_deadline_at' => $now + 30,
|
|
|
'terminal_at' => $now - 30,
|
|
|
'effect_may_have_applied' => true,
|
|
|
'safe_to_resubmit' => false,
|
|
|
'effect_resolution' => 'unknown',
|
|
|
'sealed_payload' => CloudSecrets::encrypt([
|
|
|
'method' => 'POST',
|
|
|
'request_target' => '/v1/vms',
|
|
|
'body' => [
|
|
|
'source' => 'cloud',
|
|
|
'name' => 'vm-recovery-' . $id,
|
|
|
'vcpu' => 2,
|
|
|
'memory_mib' => 2048,
|
|
|
'disk_gib' => 40,
|
|
|
'bandwidth_mbps' => 100,
|
|
|
'network' => 'bridge:br0',
|
|
|
'template_id' => 'debian-13',
|
|
|
'template_revision' => 1,
|
|
|
'template_generation_token' => str_repeat('1', 32),
|
|
|
'root_password' => 'StrongPassword!123',
|
|
|
'public_pool_key' => str_repeat('2', 32),
|
|
|
'public_pool_cidr' => '203.0.113.0/24',
|
|
|
'public_ip_cidr' => '203.0.113.41/24',
|
|
|
'gateway' => '203.0.113.1',
|
|
|
'dns_servers' => ['1.1.1.1'],
|
|
|
'platform_network_policy_snapshot' => [
|
|
|
'schema_version' => 1,
|
|
|
],
|
|
|
],
|
|
|
'template_metadata' => [
|
|
|
'template_id' => 'debian-13',
|
|
|
'os_source' => 'cloud_template',
|
|
|
'os_name' => 'Debian 13',
|
|
|
'login_name' => 'root',
|
|
|
],
|
|
|
], 'job-payload'),
|
|
|
'created_at' => $now - 120, 'updated_at' => $now - 30,
|
|
|
]);
|
|
|
}
|
|
|
Capsule::table('cloud_instances')->insert([
|
|
|
'id' => 45,
|
|
|
'uuid' => '123e4567-e89b-42d3-a456-426614174000',
|
|
|
'host_id' => 7,
|
|
|
'user_id' => 99,
|
|
|
'status' => 'active',
|
|
|
]);
|
|
|
Capsule::table('cloud_jobs')->insert([
|
|
|
'id' => 45,
|
|
|
'operation_id' => 'eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee',
|
|
|
'request_idempotency_key' =>
|
|
|
'eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee',
|
|
|
'request_fingerprint' => hash(
|
|
|
'sha256',
|
|
|
'eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee'
|
|
|
),
|
|
|
'host_id' => 7,
|
|
|
'kind' => 'instance_settings',
|
|
|
'phase' => 'terminal',
|
|
|
'outcome' => 'failed',
|
|
|
'status' => 'failed',
|
|
|
'dispatched_at' => $now - 90,
|
|
|
'dispatch_started_at' => $now - 100,
|
|
|
'agent_accepted_at' => $now - 95,
|
|
|
'receipt_deadline_at' => $now + 30,
|
|
|
'terminal_at' => $now - 30,
|
|
|
'effect_may_have_applied' => true,
|
|
|
'safe_to_resubmit' => false,
|
|
|
'effect_resolution' => 'unknown',
|
|
|
'effect_resolution_reason' => 'automatic_recovery:v1:'
|
|
|
. ($now - 60)
|
|
|
. ':1:prepared:automatic_applied_failed_fence:'
|
|
|
. str_repeat('a', 64) . ':' . str_repeat('b', 64),
|
|
|
'instance_id' => 45,
|
|
|
'sealed_payload' => CloudSecrets::encrypt([
|
|
|
'method' => 'POST',
|
|
|
'request_target' => '/v1/vms/'
|
|
|
. '123e4567-e89b-42d3-a456-426614174000/settings',
|
|
|
'body' => [
|
|
|
'operation_id' => 'eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee',
|
|
|
'hostname' => 'safe.example',
|
|
|
'confirm_restart' => true,
|
|
|
],
|
|
|
'owner_id' => 99,
|
|
|
'can_manage' => false,
|
|
|
'settings_fields' => ['hostname'],
|
|
|
'credential' => null,
|
|
|
], 'job-payload'),
|
|
|
'created_at' => $now - 120,
|
|
|
'updated_at' => $now - 30,
|
|
|
]);
|
|
|
|
|
|
$identityA = CloudOperationRecoveryIdentity::expected(CloudJob::find(41));
|
|
|
$identityB = CloudOperationRecoveryIdentity::expected(CloudJob::find(42));
|
|
|
$identityC = CloudOperationRecoveryIdentity::expected(CloudJob::find(43));
|
|
|
$identityD = CloudOperationRecoveryIdentity::expected(CloudJob::find(44));
|
|
|
$identityE = CloudOperationRecoveryIdentity::expected(CloudJob::find(45));
|
|
|
|
|
|
$admin = ['user_id' => 99, 'is_admin' => true,
|
|
|
'is_super_admin' => true, 'can_manage' => true];
|
|
|
$readonly = ['user_id' => 98, 'is_admin' => true,
|
|
|
'is_super_admin' => false, 'can_manage' => false];
|
|
|
$client = new RecoveryTestAgentClient();
|
|
|
CloudAgentSettingsService::$client = $client;
|
|
|
|
|
|
$before = count($client->requests);
|
|
|
$beforeClientBuilds = count(CloudAgentSettingsService::$calls);
|
|
|
Capsule::table('cloud_jobs')->where('id', 41)->update([
|
|
|
'execution_mode' => 'local',
|
|
|
'phase' => 'accepted',
|
|
|
'outcome' => null,
|
|
|
'status' => 'queued',
|
|
|
]);
|
|
|
$reject(static fn () => Recovery::inspect($admin, '41'), 409);
|
|
|
$assert(count($client->requests) === $before
|
|
|
&& count(CloudAgentSettingsService::$calls) === $beforeClientBuilds,
|
|
|
'A nonterminal supported Agent kind with a damaged execution mode reached the Agent client.');
|
|
|
Capsule::table('cloud_jobs')->where('id', 41)->update([
|
|
|
'kind' => 'template_import',
|
|
|
'execution_mode' => 'agent',
|
|
|
]);
|
|
|
$reject(static fn () => Recovery::inspect($admin, '41'), 409);
|
|
|
$assert(count($client->requests) === $before
|
|
|
&& count(CloudAgentSettingsService::$calls) === $beforeClientBuilds,
|
|
|
'A malformed aggregate identity reached the Agent client.');
|
|
|
Capsule::table('cloud_jobs')->where('id', 41)->update([
|
|
|
'kind' => 'create',
|
|
|
'execution_mode' => 'agent',
|
|
|
'phase' => 'terminal',
|
|
|
'outcome' => 'failed',
|
|
|
'status' => 'failed',
|
|
|
]);
|
|
|
Capsule::table('cloud_jobs')->where('id', 45)->update([
|
|
|
'receipt_deadline_at' => null,
|
|
|
]);
|
|
|
$reject(static fn () => Recovery::inspect($admin, '45'), 409);
|
|
|
$assert(count($client->requests) === $before
|
|
|
&& count(CloudAgentSettingsService::$calls) === $beforeClientBuilds,
|
|
|
'A terminal Agent task with a damaged immutable envelope reached the Agent client.');
|
|
|
Capsule::table('cloud_jobs')->where('id', 45)->update([
|
|
|
'receipt_deadline_at' => $now + 30,
|
|
|
'outcome' => 'succeeded',
|
|
|
'status' => 'succeeded',
|
|
|
'effect_may_have_applied' => false,
|
|
|
'safe_to_resubmit' => false,
|
|
|
'effect_resolution' => 'applied',
|
|
|
'finalization_incident' => true,
|
|
|
'finalization_error_code' => 'recovery_ledger_conflict',
|
|
|
]);
|
|
|
$reject(static fn () => Recovery::inspect($admin, '45'), 409);
|
|
|
$assert(count($client->requests) === $before
|
|
|
&& count(CloudAgentSettingsService::$calls) === $beforeClientBuilds,
|
|
|
'A resolved sticky recovery incident reached the Agent client.');
|
|
|
Capsule::table('cloud_jobs')->where('id', 45)->update([
|
|
|
'outcome' => 'failed',
|
|
|
'status' => 'failed',
|
|
|
'effect_may_have_applied' => true,
|
|
|
'safe_to_resubmit' => false,
|
|
|
'effect_resolution' => 'unknown',
|
|
|
'finalization_incident' => false,
|
|
|
'finalization_error_code' => null,
|
|
|
]);
|
|
|
|
|
|
$before = count($client->requests);
|
|
|
Capsule::table('cloud_jobs')->where('id', 41)->update([
|
|
|
'kind' => 'host_storage_configure',
|
|
|
]);
|
|
|
$reject(static fn () => Recovery::inspectForRecovery('41'), 409);
|
|
|
$reject(static fn () => Recovery::resolveFenceForRecovery(
|
|
|
'41',
|
|
|
'not_applied',
|
|
|
'automatic recovery must reject unregistered task kinds',
|
|
|
[
|
|
|
'operation_id' => 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa',
|
|
|
'checked_at' => $now,
|
|
|
'effect_resolution' => 'not_applied',
|
|
|
'fence_resolution_required' => true,
|
|
|
'fence_state' => 'active',
|
|
|
]
|
|
|
), 409);
|
|
|
$reject(static fn () => Recovery::preparedFenceResolutionForRecovery(
|
|
|
'41',
|
|
|
'not_applied',
|
|
|
'automatic recovery must reject unregistered task kinds'
|
|
|
), 409);
|
|
|
$assert(count($client->requests) === $before,
|
|
|
'An unregistered task reached the system recovery Agent client.');
|
|
|
Capsule::table('cloud_jobs')->where('id', 41)->update([
|
|
|
'kind' => 'create',
|
|
|
]);
|
|
|
|
|
|
$operationA = 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa';
|
|
|
$client->responses[] = $createInspectionResponse = [
|
|
|
'operation_id' => $operationA,
|
|
|
'operation_status' => 'failed',
|
|
|
'checked_at' => $now,
|
|
|
'outcome' => 'failed',
|
|
|
'summary_code' => 'operation_failed',
|
|
|
'error_code' => 'creation_absent',
|
|
|
'effect_resolution' => 'not_applied',
|
|
|
'fence_resolution_required' => true,
|
|
|
'fence_state' => 'active',
|
|
|
'effect_projection' => [
|
|
|
'creation_absent' => true,
|
|
|
'managed_artifacts_absent' => true,
|
|
|
],
|
|
|
'abort_job' => ['available' => false],
|
|
|
'cleanup' => ['available' => false],
|
|
|
] + $identityA;
|
|
|
$inspection = Recovery::inspect($admin, '41');
|
|
|
$assert($inspection === [
|
|
|
'operation_id' => $operationA,
|
|
|
'action' => $identityA['action'],
|
|
|
'resource_id' => $identityA['resource_id'],
|
|
|
'request_fingerprint' => $identityA['request_fingerprint'],
|
|
|
'journal_status' => 'failed',
|
|
|
'checked_at' => $now,
|
|
|
'outcome' => 'failed',
|
|
|
'effect_resolution' => 'not_applied',
|
|
|
'fence_state' => 'active',
|
|
|
'fence_resolution_required' => true,
|
|
|
'summary_code' => 'operation_failed',
|
|
|
'summary' => '只读证据与宿主机失败结果一致',
|
|
|
'error_code' => 'creation_absent',
|
|
|
'error' => '宿主机未发现虚拟机或面板管理资源',
|
|
|
] && !array_key_exists('effect_projection', $inspection),
|
|
|
'Read-only inspection was not reduced to the bounded DTO.');
|
|
|
$assert(CloudAgentSettingsService::$calls === [[7, 30.0]]
|
|
|
&& $client->requests[0]['method'] === 'GET'
|
|
|
&& $client->requests[0]['target']
|
|
|
=== '/v1/operations/' . $operationA . '/inspect'
|
|
|
&& $client->requests[0]['payload'] === null,
|
|
|
'Inspection did not use exactly one signed read-only request.');
|
|
|
$assert(Capsule::table('cloud_jobs')->where('id', 41)
|
|
|
->value('effect_resolution') === 'unknown',
|
|
|
'Read-only inspection changed the task or its safety fence.');
|
|
|
|
|
|
$client->responses[] = $createInspectionResponse;
|
|
|
$createInspection = Recovery::inspectForRecovery('41');
|
|
|
$createRequest = $client->requests[array_key_last($client->requests)];
|
|
|
$assert(($createInspection['error_code'] ?? null) === 'creation_absent'
|
|
|
&& ($createInspection['effect_projection']['creation_absent'] ?? null) === true
|
|
|
&& ($createInspection['effect_projection']['managed_artifacts_absent'] ?? null) === true
|
|
|
&& $createRequest['method'] === 'GET'
|
|
|
&& ($createRequest['system_recovery'] ?? false) === true
|
|
|
&& $createRequest['actor']['operation_id'] === $operationA
|
|
|
&& Capsule::table('cloud_jobs')->where('id', 41)
|
|
|
->value('effect_resolution') === 'unknown',
|
|
|
'Create inspection did not remain operation-bound and read-only.');
|
|
|
$before = count($client->requests);
|
|
|
$reject(static fn () => Recovery::resolveFenceForRecovery(
|
|
|
'41', 'not_applied',
|
|
|
'automatic create recovery requires durable financial intent',
|
|
|
$createInspection
|
|
|
), 409);
|
|
|
$reject(static fn () => Recovery::resolveFenceForRecovery(
|
|
|
'41', 'applied',
|
|
|
'automatic create recovery cannot infer a successful instance',
|
|
|
$createInspection
|
|
|
), 409);
|
|
|
$reject(static fn () => Recovery::preparedFenceResolutionForRecovery(
|
|
|
'41', 'not_applied',
|
|
|
'automatic create recovery requires durable financial intent'
|
|
|
), 409);
|
|
|
$assert(count($client->requests) === $before,
|
|
|
'An unprepared create reached the system Agent fence routes.');
|
|
|
|
|
|
$client->responses[] = ['operation_id' => $operationA,
|
|
|
'operation_status' => 'uncertain', 'checked_at' => $now,
|
|
|
'outcome' => 'uncertain', 'summary_code' => 'evidence_incomplete'];
|
|
|
$reject(static fn () => Recovery::inspect($admin, '41'), 502,
|
|
|
'agent_invalid_inspection');
|
|
|
|
|
|
$client->responses[] = [
|
|
|
'operation_id' => $operationA,
|
|
|
'operation_status' => 'failed',
|
|
|
'checked_at' => $now,
|
|
|
'outcome' => 'failed',
|
|
|
'summary_code' => 'operation_failed',
|
|
|
'error_code' => 'creation_absent',
|
|
|
'effect_resolution' => 'not_applied',
|
|
|
'fence_resolution_required' => true,
|
|
|
'fence_state' => 'active',
|
|
|
'effect_projection' => [
|
|
|
'creation_absent' => true,
|
|
|
'managed_artifacts_absent' => true,
|
|
|
],
|
|
|
'abort_job' => ['available' => false],
|
|
|
'cleanup' => ['available' => false],
|
|
|
'raw_error' => '/srv/private/secret.qcow2',
|
|
|
] + $identityA;
|
|
|
$reject(static fn () => Recovery::inspect($admin, '41'), 502,
|
|
|
'agent_invalid_inspection');
|
|
|
|
|
|
$operationE = 'eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee';
|
|
|
$safeProjection = [
|
|
|
'uuid' => '123e4567-e89b-42d3-a456-426614174000',
|
|
|
'hostname' => 'safe.example',
|
|
|
'settings_changed' => true,
|
|
|
];
|
|
|
$client->responses[] = [
|
|
|
'operation_id' => $operationE,
|
|
|
'operation_status' => 'failed',
|
|
|
'checked_at' => $now,
|
|
|
'outcome' => 'succeeded',
|
|
|
'summary_code' => 'operation_succeeded',
|
|
|
'effect_resolution' => 'applied',
|
|
|
'fence_resolution_required' => true,
|
|
|
'fence_state' => 'active',
|
|
|
'effect_projection' => $safeProjection,
|
|
|
'abort_job' => ['available' => false],
|
|
|
'cleanup' => ['available' => false],
|
|
|
] + $identityE;
|
|
|
$before = count($client->requests);
|
|
|
$systemInspection = Recovery::inspectForRecovery('45');
|
|
|
$requests = array_slice($client->requests, $before);
|
|
|
$assert(($systemInspection['effect_resolution'] ?? null) === 'applied'
|
|
|
&& ($systemInspection['effect_projection'] ?? null)
|
|
|
=== $safeProjection
|
|
|
&& count($requests) === 1
|
|
|
&& ($requests[0]['system_recovery'] ?? false) === true
|
|
|
&& $requests[0]['actor'] === [
|
|
|
'user_id' => 0,
|
|
|
'is_admin' => false,
|
|
|
'is_super_admin' => false,
|
|
|
'can_manage' => false,
|
|
|
'system_recovery' => true,
|
|
|
'operation_id' => $operationE,
|
|
|
],
|
|
|
'Internal inspection lost its bounded recovery evidence or actor.');
|
|
|
|
|
|
$client->responses[] = [
|
|
|
'operation_id' => $operationE,
|
|
|
'operation_status' => 'failed',
|
|
|
'checked_at' => $now,
|
|
|
'outcome' => 'succeeded',
|
|
|
'summary_code' => 'operation_succeeded',
|
|
|
'effect_resolution' => 'applied',
|
|
|
'fence_resolution_required' => true,
|
|
|
'fence_state' => 'active',
|
|
|
'completion_delivery_state' => 'quarantined',
|
|
|
'completion_delivery_error_code' =>
|
|
|
'completion_receipt_invalid',
|
|
|
'effect_projection' => $safeProjection,
|
|
|
'abort_job' => ['available' => false],
|
|
|
'cleanup' => ['available' => false],
|
|
|
] + $identityE;
|
|
|
$poisonedInspection = Recovery::inspectForRecovery('45');
|
|
|
$assert(($poisonedInspection['completion_delivery_state'] ?? null)
|
|
|
=== 'quarantined'
|
|
|
&& ($poisonedInspection['completion_delivery_error_code'] ?? null)
|
|
|
=== 'completion_receipt_invalid'
|
|
|
&& str_contains(
|
|
|
(string) ($poisonedInspection['completion_delivery_error'] ?? ''),
|
|
|
'完成回执内容无效'
|
|
|
),
|
|
|
'A quarantined completion did not retain its bounded Chinese cause.');
|
|
|
|
|
|
$client->responses[] = [
|
|
|
'operation_id' => $operationE,
|
|
|
'operation_status' => 'failed',
|
|
|
'checked_at' => $now,
|
|
|
'outcome' => 'succeeded',
|
|
|
'summary_code' => 'operation_succeeded',
|
|
|
'effect_resolution' => 'applied',
|
|
|
'fence_state' => 'resolved',
|
|
|
'fence_resolution_required' => false,
|
|
|
'effect_projection' => $safeProjection,
|
|
|
'abort_job' => ['available' => false],
|
|
|
'cleanup' => ['available' => false],
|
|
|
] + $identityE;
|
|
|
$resolvedInspection = Recovery::inspectForRecovery('45');
|
|
|
$assert(($resolvedInspection['effect_resolution'] ?? null) === 'applied'
|
|
|
&& ($resolvedInspection['fence_state'] ?? null) === 'resolved'
|
|
|
&& ($resolvedInspection['fence_resolution_required'] ?? null)
|
|
|
=== false,
|
|
|
'A resolved Agent fence was collapsed into a never-fenced DTO.');
|
|
|
|
|
|
foreach ([
|
|
|
['effect_resolution' => 'trusted',
|
|
|
'effect_projection' => $safeProjection],
|
|
|
['effect_resolution' => 'applied', 'effect_projection' => [
|
|
|
'uuid' => '123e4567-e89b-42d3-a456-426614174000',
|
|
|
'root_password' => 'must-not-cross',
|
|
|
'settings_changed' => true,
|
|
|
]],
|
|
|
['effect_resolution' => 'applied', 'effect_projection' => [
|
|
|
'uuid' => '123e4567-e89b-42d3-a456-426614174000',
|
|
|
'future_field' => true,
|
|
|
'settings_changed' => true,
|
|
|
]],
|
|
|
['effect_resolution' => 'applied', 'effect_projection' => [
|
|
|
'uuid' => '123e4567-e89b-42d3-a456-426614174000',
|
|
|
'hostname' => str_repeat('x', 65537),
|
|
|
'settings_changed' => true,
|
|
|
]],
|
|
|
['effect_resolution' => 'applied',
|
|
|
'effect_projection' => $safeProjection,
|
|
|
'completion_delivery_state' => 'quarantined',
|
|
|
'completion_delivery_error_code' => 'future_poison_reason'],
|
|
|
['effect_resolution' => 'applied',
|
|
|
'effect_projection' => $safeProjection,
|
|
|
'completion_delivery_state' => 'quarantined'],
|
|
|
] as $unsafeEvidence) {
|
|
|
$client->responses[] = [
|
|
|
'operation_id' => $operationE,
|
|
|
'operation_status' => 'failed',
|
|
|
'checked_at' => $now,
|
|
|
'outcome' => 'succeeded',
|
|
|
'summary_code' => 'operation_succeeded',
|
|
|
'fence_resolution_required' => true,
|
|
|
'fence_state' => 'active',
|
|
|
'abort_job' => ['available' => false],
|
|
|
'cleanup' => ['available' => false],
|
|
|
] + $unsafeEvidence + $identityE;
|
|
|
$reject(static fn () => Recovery::inspectForRecovery('45'), 502,
|
|
|
'agent_invalid_inspection');
|
|
|
}
|
|
|
|
|
|
$reason = '已人工只读核对虚拟机、磁盘与网络状态,确认原操作效果已经发生。';
|
|
|
$reasonDigest = hash('sha256', $reason);
|
|
|
$poisonedManualInspection = $inspection + [
|
|
|
'completion_delivery_state' => 'quarantined',
|
|
|
'completion_delivery_error_code' =>
|
|
|
'completion_operation_invalid',
|
|
|
'completion_delivery_error' =>
|
|
|
'Agent 完成回执对应的原操作记录无效,当前结果不能采信',
|
|
|
];
|
|
|
$before = count($client->requests);
|
|
|
$reject(static fn () => Recovery::resolveFence(
|
|
|
$admin,
|
|
|
'41',
|
|
|
'not_applied',
|
|
|
$reason,
|
|
|
$poisonedManualInspection
|
|
|
), 409);
|
|
|
$assert(count($client->requests) === $before,
|
|
|
'A quarantined completion reached the Agent fence resolver.');
|
|
|
$oldReceipt = [
|
|
|
'operation_id' => $operationA,
|
|
|
'resolution_id' => str_repeat('1', 32),
|
|
|
'resolution' => 'not_applied',
|
|
|
'inspection_checked_at' => $now - 60,
|
|
|
'reason_sha256' => $reasonDigest,
|
|
|
'resolved_at' => $now - 50,
|
|
|
'resolved_by' => 7,
|
|
|
'replayed' => false,
|
|
|
];
|
|
|
$before = count($client->requests);
|
|
|
$client->responses[] = $oldReceipt;
|
|
|
$recovered = Recovery::resolveFence(
|
|
|
$admin, '41', 'not_applied', $reason, $inspection
|
|
|
);
|
|
|
$requests = array_slice($client->requests, $before);
|
|
|
$assert($recovered['resolution_id'] === str_repeat('1', 32)
|
|
|
&& count($requests) === 1 && $requests[0]['method'] === 'GET'
|
|
|
&& str_ends_with($requests[0]['target'], '/fence-resolution')
|
|
|
&& ($requests[0]['actor']['operation_id'] ?? null) === $operationA,
|
|
|
'A prior Agent commit was not recovered read-only after a PHP crash.');
|
|
|
|
|
|
$notFound = static fn () => new CloudAgentError(
|
|
|
'missing', 404, 'fence_resolution_not_found', false, 404
|
|
|
);
|
|
|
|
|
|
$automaticReason = 'Agent 实时只读检查确认实例设置已经提交,PHP 账本预提交完成。';
|
|
|
$client->responses[] = [
|
|
|
'operation_id' => $operationE,
|
|
|
'resolution_id' => str_repeat('f', 32),
|
|
|
'resolution' => 'applied',
|
|
|
'inspection_checked_at' => $now,
|
|
|
'reason_sha256' => hash('sha256', $automaticReason),
|
|
|
'resolved_at' => $now,
|
|
|
'resolved_by' => 7,
|
|
|
'replayed' => false,
|
|
|
];
|
|
|
$before = count($client->requests);
|
|
|
$reject(static fn () => Recovery::resolveFenceForRecovery(
|
|
|
'45',
|
|
|
'applied',
|
|
|
$automaticReason,
|
|
|
$systemInspection
|
|
|
), 503, 'fence_resolution_read_failed');
|
|
|
$assert(count($client->requests) === $before + 1,
|
|
|
'System recovery accepted a positive human resolved_by value.');
|
|
|
|
|
|
$client->responses[] = $notFound();
|
|
|
$client->responses[] = new CloudAgentError(
|
|
|
'lost response', 503, 'agent_disconnected', true, 503
|
|
|
);
|
|
|
$client->responses[] = static function (array $request) use (
|
|
|
$client, $operationE, $now
|
|
|
): array {
|
|
|
$post = $client->requests[count($client->requests) - 2];
|
|
|
return [
|
|
|
'operation_id' => $operationE,
|
|
|
'resolution_id' => $post['payload']['resolution_id'],
|
|
|
'resolution' => $post['payload']['resolution'],
|
|
|
'inspection_checked_at' =>
|
|
|
$post['payload']['inspection_checked_at'],
|
|
|
'reason_sha256' => $post['payload']['reason_sha256'],
|
|
|
'resolved_at' => $now,
|
|
|
'resolved_by' => 0,
|
|
|
'replayed' => false,
|
|
|
];
|
|
|
};
|
|
|
$before = count($client->requests);
|
|
|
$automaticFence = Recovery::resolveFenceForRecovery(
|
|
|
'45',
|
|
|
'applied',
|
|
|
$automaticReason,
|
|
|
$systemInspection
|
|
|
);
|
|
|
$requests = array_slice($client->requests, $before);
|
|
|
$assert(array_column($requests, 'method') === ['GET', 'POST', 'GET']
|
|
|
&& count(array_filter($requests,
|
|
|
static fn (array $request): bool =>
|
|
|
$request['method'] === 'POST')) === 1
|
|
|
&& count(array_filter($requests,
|
|
|
static fn (array $request): bool =>
|
|
|
($request['system_recovery'] ?? false) === true)) === 3
|
|
|
&& $automaticFence['resolution'] === 'applied'
|
|
|
&& $automaticFence['resolved_by'] === '0',
|
|
|
'System recovery did not use one exact-operation fence write and read verification.');
|
|
|
|
|
|
$operationB = 'bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb';
|
|
|
$inspectionB = [
|
|
|
'operation_id' => $operationB, 'checked_at' => $now,
|
|
|
'outcome' => 'failed', 'summary_code' => 'operation_failed',
|
|
|
'error_code' => 'creation_absent',
|
|
|
] + $identityB;
|
|
|
$client->responses[] = $notFound();
|
|
|
$client->responses[] = static function (array $request) use (
|
|
|
$operationB, $now
|
|
|
): array {
|
|
|
return [
|
|
|
'operation_id' => $operationB,
|
|
|
'resolution_id' => $request['payload']['resolution_id'],
|
|
|
'resolution' => $request['payload']['resolution'],
|
|
|
'inspection_checked_at' =>
|
|
|
$request['payload']['inspection_checked_at'],
|
|
|
'reason_sha256' => $request['payload']['reason_sha256'],
|
|
|
'resolved_at' => $now,
|
|
|
'resolved_by' => '7',
|
|
|
'replayed' => false,
|
|
|
];
|
|
|
};
|
|
|
$before = count($client->requests);
|
|
|
$created = Recovery::resolveFence(
|
|
|
$admin, '42', 'not_applied', $reason, $inspectionB
|
|
|
);
|
|
|
$requests = array_slice($client->requests, $before);
|
|
|
$assert(count($requests) === 2
|
|
|
&& array_column($requests, 'method') === ['GET', 'POST']
|
|
|
&& str_ends_with($requests[1]['target'], '/resolve-fence')
|
|
|
&& preg_match('/^[a-f0-9]{32}$/D',
|
|
|
$requests[1]['payload']['resolution_id']) === 1
|
|
|
&& $created['resolution'] === 'not_applied',
|
|
|
'New fence resolution was not dispatched exactly once.');
|
|
|
|
|
|
$operationC = 'cccccccccccccccccccccccccccccccc';
|
|
|
$inspectionC = [
|
|
|
'operation_id' => $operationC, 'checked_at' => $now,
|
|
|
'outcome' => 'failed', 'summary_code' => 'operation_failed',
|
|
|
'error_code' => 'creation_absent',
|
|
|
] + $identityC;
|
|
|
$client->responses[] = $notFound();
|
|
|
$client->responses[] = new CloudAgentError(
|
|
|
'lost response', 503, 'agent_disconnected', true, 503
|
|
|
);
|
|
|
$client->responses[] = static function (array $request) use (
|
|
|
$client, $operationC, $now
|
|
|
): array {
|
|
|
$post = $client->requests[count($client->requests) - 2];
|
|
|
return [
|
|
|
'operation_id' => $operationC,
|
|
|
'resolution_id' => $post['payload']['resolution_id'],
|
|
|
'resolution' => $post['payload']['resolution'],
|
|
|
'inspection_checked_at' =>
|
|
|
$post['payload']['inspection_checked_at'],
|
|
|
'reason_sha256' => $post['payload']['reason_sha256'],
|
|
|
'resolved_at' => $now, 'resolved_by' => 7,
|
|
|
'replayed' => false,
|
|
|
];
|
|
|
};
|
|
|
$before = count($client->requests);
|
|
|
$verified = Recovery::resolveFence(
|
|
|
$admin, '43', 'not_applied', $reason, $inspectionC
|
|
|
);
|
|
|
$requests = array_slice($client->requests, $before);
|
|
|
$assert(array_column($requests, 'method') === ['GET', 'POST', 'GET']
|
|
|
&& count(array_filter($requests,
|
|
|
static fn (array $request): bool => $request['method'] === 'POST'))
|
|
|
=== 1
|
|
|
&& $verified['resolution'] === 'not_applied',
|
|
|
'Lost fence response caused a POST retry instead of one GET verification.');
|
|
|
|
|
|
$operationD = 'dddddddddddddddddddddddddddddddd';
|
|
|
$inspectionD = [
|
|
|
'operation_id' => $operationD, 'checked_at' => $now,
|
|
|
'outcome' => 'failed', 'summary_code' => 'operation_failed',
|
|
|
'error_code' => 'creation_absent',
|
|
|
] + $identityD;
|
|
|
$client->responses[] = array_replace($oldReceipt, [
|
|
|
'operation_id' => $operationD,
|
|
|
'resolved_by' => 0,
|
|
|
]);
|
|
|
$before = count($client->requests);
|
|
|
$reject(static fn () => Recovery::resolveFence(
|
|
|
$admin, '44', 'not_applied', $reason, $inspectionD
|
|
|
), 503, 'fence_resolution_read_failed');
|
|
|
$assert(count($client->requests) === $before + 1,
|
|
|
'A forged system actor ID reached the human fence write path.');
|
|
|
|
|
|
$client->responses[] = array_replace($oldReceipt, [
|
|
|
'operation_id' => $operationD,
|
|
|
'resolution' => 'applied',
|
|
|
]);
|
|
|
$before = count($client->requests);
|
|
|
$reject(static fn () => Recovery::resolveFence(
|
|
|
$admin, '44', 'not_applied', $reason, $inspectionD
|
|
|
), 409);
|
|
|
$assert(count($client->requests) === $before + 1,
|
|
|
'Conflicting durable fence receipt triggered an Agent write.');
|
|
|
|
|
|
$before = count($client->requests);
|
|
|
$reject(static fn () => Recovery::resolveFence(
|
|
|
$readonly, '41', 'not_applied', $reason, $inspection
|
|
|
), 403);
|
|
|
$reject(static fn () => Recovery::resolveFence(
|
|
|
$admin, '41', 'not_applied', $reason,
|
|
|
[
|
|
|
'operation_id' => $operationA, 'checked_at' => $now - 121,
|
|
|
'outcome' => 'failed', 'summary_code' => 'operation_failed',
|
|
|
'error_code' => 'creation_absent',
|
|
|
] + $identityA
|
|
|
), 409);
|
|
|
$assert(count($client->requests) === $before,
|
|
|
'Unauthorized or stale resolution reached the Agent.');
|
|
|
|
|
|
$source = file_get_contents(dirname(__DIR__)
|
|
|
. '/app/Services/Cloud/CloudAdminOperationRecoveryService.php');
|
|
|
$assert(is_string($source)
|
|
|
&& !str_contains($source, 'function abortJob')
|
|
|
&& !str_contains($source, 'function cleanup')
|
|
|
&& !str_contains($source, 'confirm-inspection')
|
|
|
&& preg_match_all('/\$call\(\s*[\r\n ]*\'POST\'/m',
|
|
|
$source) === 1,
|
|
|
'Retired recovery mutations or a second Agent POST remain reachable.');
|
|
|
|
|
|
echo "Administrator read-only inspection and single-write fence recovery tests passed.\n";
|
|
|
}
|