You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
578 lines
23 KiB
PHP
578 lines
23 KiB
PHP
<?php
|
|
|
|
namespace App\Services\Cloud {
|
|
final class CloudSecrets
|
|
{
|
|
public static function decrypt(string $value, string $purpose): array
|
|
{
|
|
return [
|
|
'method' => 'POST',
|
|
'request_target' => '/v1/vms/test/actions',
|
|
'body' => ['action' => 'reboot'],
|
|
'owner_id' => 99,
|
|
];
|
|
}
|
|
|
|
public static function encrypt(array $value, string $purpose): string
|
|
{
|
|
return 'sealed-terminal-audit';
|
|
}
|
|
}
|
|
|
|
final class RecoveryTestAgentClient
|
|
{
|
|
public array $requests = [];
|
|
public array $responses = [];
|
|
|
|
public function request(
|
|
string $method,
|
|
string $target,
|
|
array $actor,
|
|
?array $payload = null
|
|
): array {
|
|
$this->requests[] = compact('method', 'target', 'actor', 'payload');
|
|
if ($this->responses === []) {
|
|
throw new \RuntimeException('Unexpected Agent request.');
|
|
}
|
|
$response = array_shift($this->responses);
|
|
if ($response instanceof \Throwable) {
|
|
throw $response;
|
|
}
|
|
if (!is_array($response)) {
|
|
throw new \RuntimeException('Invalid recovery test response.');
|
|
}
|
|
return $response;
|
|
}
|
|
}
|
|
|
|
final class CloudAgentSettingsService
|
|
{
|
|
public static ?RecoveryTestAgentClient $client = null;
|
|
public static array $calls = [];
|
|
|
|
public static function clientForConfiguration(
|
|
int $hostId,
|
|
float $timeoutSeconds = 120.0
|
|
): RecoveryTestAgentClient {
|
|
self::$calls[] = [$hostId, $timeoutSeconds];
|
|
return self::$client
|
|
?? throw new \RuntimeException('Missing recovery test client.');
|
|
}
|
|
}
|
|
}
|
|
|
|
namespace {
|
|
require dirname(__DIR__) . '/vendor/autoload.php';
|
|
|
|
use App\Services\Cloud\CloudAdminOperationRecoveryService as Recovery;
|
|
use App\Services\Cloud\CloudAgentError;
|
|
use App\Services\Cloud\CloudAgentSettingsService;
|
|
use App\Services\Cloud\CloudError;
|
|
use App\Services\Cloud\RecoveryTestAgentClient;
|
|
use Illuminate\Database\Capsule\Manager as Capsule;
|
|
use Illuminate\Database\Schema\Blueprint;
|
|
|
|
$assert = static function ($condition, string $message): void {
|
|
if (!$condition) {
|
|
throw new \RuntimeException($message);
|
|
}
|
|
};
|
|
$reject = static function (
|
|
callable $callback,
|
|
int $status,
|
|
?string $agentCode = null
|
|
) use ($assert): void {
|
|
try {
|
|
$callback();
|
|
} catch (CloudError $error) {
|
|
$assert($error->httpStatus === $status,
|
|
'Unexpected rejection status: ' . $error->httpStatus);
|
|
if ($agentCode !== null) {
|
|
$assert($error instanceof CloudAgentError
|
|
&& $error->agentCode === $agentCode,
|
|
'Unexpected Agent rejection code.');
|
|
}
|
|
return;
|
|
}
|
|
throw new \RuntimeException('Invalid recovery request was accepted.');
|
|
};
|
|
$capture = static function (callable $callback) use ($assert): CloudAgentError {
|
|
try {
|
|
$callback();
|
|
} catch (CloudAgentError $error) {
|
|
return $error;
|
|
}
|
|
$assert(false, 'Expected a controlled Agent error.');
|
|
throw new \RuntimeException('Unreachable.');
|
|
};
|
|
|
|
$capsule = new Capsule();
|
|
$capsule->addConnection(['driver' => 'sqlite', 'database' => ':memory:']);
|
|
$capsule->setAsGlobal();
|
|
$capsule->bootEloquent();
|
|
$capsule->schema()->create('cloud_jobs', static function (Blueprint $table): void {
|
|
$table->increments('id');
|
|
$table->string('operation_id');
|
|
$table->integer('host_id');
|
|
$table->string('kind');
|
|
$table->string('status');
|
|
$table->integer('user_id')->default(99);
|
|
$table->integer('instance_id')->nullable();
|
|
$table->integer('order_id')->nullable();
|
|
$table->integer('snapshot_id')->nullable();
|
|
$table->integer('attempts')->default(1);
|
|
$table->integer('dispatched_at')->nullable();
|
|
$table->integer('available_at')->default(0);
|
|
$table->integer('lease_until')->default(0);
|
|
$table->string('error_code')->nullable();
|
|
$table->text('result_json')->nullable();
|
|
$table->text('sealed_payload')->default('forbidden-payload');
|
|
$table->string('lease_token')->nullable();
|
|
$table->integer('created_at')->nullable();
|
|
$table->integer('updated_at')->nullable();
|
|
});
|
|
$capsule->schema()->create('user', static function (Blueprint $table): void {
|
|
$table->increments('id');
|
|
$table->integer('enable')->default(1);
|
|
});
|
|
Capsule::table('user')->insert(['id' => 99, 'enable' => 1]);
|
|
Capsule::table('cloud_jobs')->insert([
|
|
'id' => 41,
|
|
'operation_id' => 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa',
|
|
'host_id' => 7,
|
|
'kind' => 'action',
|
|
'status' => 'uncertain',
|
|
'dispatched_at' => 1_799_999_900,
|
|
]);
|
|
Capsule::table('cloud_jobs')->insert([
|
|
'id' => 42,
|
|
'operation_id' => 'dddddddddddddddddddddddddddddddd',
|
|
'host_id' => 7,
|
|
'kind' => 'action',
|
|
'status' => 'needs_attention',
|
|
'error_code' => 'reconciliation_limit_reached',
|
|
'dispatched_at' => 1_799_999_900,
|
|
]);
|
|
|
|
$admin = [
|
|
'user_id' => 99,
|
|
'is_admin' => true,
|
|
'is_super_admin' => true,
|
|
'can_manage' => true,
|
|
];
|
|
$readonly = [
|
|
'user_id' => 98,
|
|
'is_admin' => true,
|
|
'is_super_admin' => false,
|
|
'can_manage' => false,
|
|
];
|
|
$operationId = 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa';
|
|
$token = str_repeat('b', 64);
|
|
$planId = str_repeat('c', 64);
|
|
$client = new RecoveryTestAgentClient();
|
|
CloudAgentSettingsService::$client = $client;
|
|
$capsule->getConnection()->enableQueryLog();
|
|
$capsule->getConnection()->flushQueryLog();
|
|
|
|
$client->responses[] = [
|
|
'operation_id' => $operationId,
|
|
'operation_status' => 'uncertain',
|
|
'checked_at' => 1_800_000_000,
|
|
'outcome' => 'uncertain',
|
|
'summary_code' => 'abortable_job_active',
|
|
// Unknown codes and unrelated fields must not reach the browser DTO.
|
|
'error_code' => 'raw_libvirt_secret_path',
|
|
'raw_error' => '/srv/private/secret.qcow2',
|
|
'abort_job' => [
|
|
'available' => true,
|
|
'kind' => 'backup',
|
|
'inspection_token' => $token,
|
|
],
|
|
'cleanup' => ['available' => false],
|
|
'confirmation' => ['available' => false],
|
|
];
|
|
$inspection = Recovery::inspect($admin, '41');
|
|
$inspectQueries = $capsule->getConnection()->getQueryLog();
|
|
$assert($inspection['operation_id'] === $operationId
|
|
&& $inspection['summary_code'] === 'abortable_job_active'
|
|
&& $inspection['abort_job'] === [
|
|
'available' => true,
|
|
'kind' => 'backup',
|
|
'label' => '中止匹配的备份作业',
|
|
'inspection_token' => $token,
|
|
]
|
|
&& $inspection['cleanup']['available'] === false
|
|
&& !isset($inspection['error_code'])
|
|
&& !str_contains(json_encode($inspection), '/srv/private'),
|
|
'Read-only inspection was not strictly normalized.');
|
|
$assert(count($inspectQueries) === 1
|
|
&& !str_contains(strtolower($inspectQueries[0]['query']), 'sealed_payload')
|
|
&& !str_contains(strtolower($inspectQueries[0]['query']), 'lease_token'),
|
|
'Explicit recovery read selected task secrets or lease tokens.');
|
|
$assert(CloudAgentSettingsService::$calls === [[7, 30.0]]
|
|
&& $client->requests[0]['method'] === 'GET'
|
|
&& $client->requests[0]['target']
|
|
=== '/v1/operations/' . $operationId . '/inspect'
|
|
&& $client->requests[0]['payload'] === null,
|
|
'Inspection did not use the signed read-only Agent endpoint.');
|
|
|
|
$client->responses[] = [
|
|
'operation_id' => $operationId,
|
|
'operation_status' => 'uncertain',
|
|
'checked_at' => 1_800_000_001,
|
|
'outcome' => 'uncertain',
|
|
'summary_code' => 'evidence_incomplete',
|
|
'abort_job' => ['available' => false],
|
|
'cleanup' => ['available' => false],
|
|
'confirmation' => ['available' => false],
|
|
];
|
|
$afterAbort = Recovery::abortJob($admin, '41', [
|
|
'inspection_token' => $token,
|
|
]);
|
|
$abortRequest = $client->requests[1];
|
|
$assert($abortRequest['method'] === 'POST'
|
|
&& str_ends_with($abortRequest['target'], '/abort-job')
|
|
&& $abortRequest['payload'] === ['inspection_token' => $token]
|
|
&& CloudAgentSettingsService::$calls[1] === [7, 60.0]
|
|
&& !$afterAbort['abort_job']['available'],
|
|
'Abort was not bound to the inspected job token.');
|
|
|
|
$client->responses[] = [
|
|
'operation_id' => $operationId,
|
|
'operation_status' => 'uncertain',
|
|
'checked_at' => 1_800_000_002,
|
|
'outcome' => 'uncertain',
|
|
'summary_code' => 'cleanup_required',
|
|
'abort_job' => ['available' => false],
|
|
'cleanup' => [
|
|
'available' => true,
|
|
'plan_id' => $planId,
|
|
'steps' => ['destroy_domain', 'delete_managed_volumes',
|
|
'remove_traffic_state', 'persist_cleanup_metadata'],
|
|
],
|
|
'confirmation' => ['available' => false],
|
|
];
|
|
$cleanup = Recovery::cleanup($admin, '41', ['plan_id' => $planId]);
|
|
$cleanupRequest = $client->requests[2];
|
|
$assert($cleanupRequest['method'] === 'POST'
|
|
&& str_ends_with($cleanupRequest['target'], '/cleanup')
|
|
&& $cleanupRequest['payload'] === ['plan_id' => $planId]
|
|
&& CloudAgentSettingsService::$calls[2] === [7, 900.0]
|
|
&& $cleanup['cleanup']['step_codes'] === [
|
|
'destroy_domain', 'delete_managed_volumes', 'remove_traffic_state',
|
|
'persist_cleanup_metadata',
|
|
]
|
|
&& $cleanup['cleanup']['steps'] === [
|
|
'停止残留虚拟机', '删除面板管理的残留磁盘', '删除残留流量计量状态',
|
|
'清除已完成磁盘切换的待清理标记',
|
|
], 'Cleanup was not bound to the exact inspected plan.');
|
|
|
|
// Recovery errors arrive as CloudAgentError rather than operation records.
|
|
// Only stable codes may survive; the Agent's free-form message, cause and
|
|
// result must never be copied into the browser-facing exception.
|
|
$rawAgentDetail = 'raw message cause=/srv/private/domain.xml '
|
|
. 'result={"secret":"never-expose"}';
|
|
$errorCases = [
|
|
[
|
|
'operation_inspection_stale', false,
|
|
static fn () => Recovery::abortJob($admin, '41', [
|
|
'inspection_token' => $token,
|
|
]),
|
|
'核对凭证已失效,请重新立即核对',
|
|
],
|
|
[
|
|
'cleanup_plan_stale', false,
|
|
static fn () => Recovery::cleanup($admin, '41', [
|
|
'plan_id' => $planId,
|
|
]),
|
|
'宿主机状态已变化,清理计划已失效,请重新立即核对',
|
|
],
|
|
[
|
|
'job_abort_unconfirmed', true,
|
|
static fn () => Recovery::abortJob($admin, '41', [
|
|
'inspection_token' => $token,
|
|
]),
|
|
'libvirt 作业中止后的终态仍未确认,请稍后再次立即核对',
|
|
],
|
|
[
|
|
'operation_uncertain', true,
|
|
static fn () => Recovery::cleanup($admin, '41', [
|
|
'plan_id' => $planId,
|
|
]),
|
|
'宿主机操作结果仍未确认,请稍后再次立即核对',
|
|
],
|
|
[
|
|
'external_snapshot_cleanup_required', false,
|
|
static fn () => Recovery::inspect($admin, '41'),
|
|
'检测到原生外部快照或回滚元数据,请先在宿主机安全完成清理,再重新立即核对',
|
|
],
|
|
[
|
|
'native_snapshot_delete_in_progress', false,
|
|
static fn () => Recovery::inspect($admin, '41'),
|
|
'原生快照删除尚未完成,请先在宿主机继续完成该删除,再重新立即核对',
|
|
],
|
|
[
|
|
'managed_disk_shared', false,
|
|
static fn () => Recovery::inspect($admin, '41'),
|
|
'目标受管磁盘仍被另一台虚拟机的当前配置、快照或保存状态引用,请先解除引用,再重新立即核对',
|
|
],
|
|
[
|
|
'managed_filter_shared', false,
|
|
static fn () => Recovery::inspect($admin, '41'),
|
|
'目标网络过滤器仍被另一台虚拟机或过滤器引用,请先解除引用,再重新立即核对',
|
|
],
|
|
];
|
|
foreach ($errorCases as [$code, $retryable, $call, $message]) {
|
|
$client->responses[] = new CloudAgentError(
|
|
$rawAgentDetail, 409, $code, $retryable, 409, true, true
|
|
);
|
|
$mapped = $capture($call);
|
|
$assert($mapped->httpStatus === 409
|
|
&& $mapped->agentCode === $code
|
|
&& $mapped->retryable === $retryable
|
|
&& $mapped->getMessage() === $message
|
|
&& $mapped->safeValidationFailure === false
|
|
&& $mapped->safePreDispatchFailure === false
|
|
&& !str_contains($mapped->getMessage(), '/srv/private')
|
|
&& !str_contains($mapped->getMessage(), 'never-expose'),
|
|
'Recovery error was not safely mapped: ' . $code);
|
|
}
|
|
|
|
$client->responses[] = new CloudAgentError(
|
|
$rawAgentDetail, 500, 'unexpected_private_backend_code', true, 500
|
|
);
|
|
$unknown = $capture(static fn () => Recovery::inspect($admin, '41'));
|
|
$assert($unknown->httpStatus === 503
|
|
&& $unknown->agentCode === 'operation_recovery_unconfirmed'
|
|
&& $unknown->retryable === true
|
|
&& $unknown->getMessage()
|
|
=== '宿主机状态暂时无法确认,请稍后重试'
|
|
&& !str_contains($unknown->getMessage(), '/srv/private')
|
|
&& !str_contains($unknown->getMessage(), 'never-expose'),
|
|
'Unknown Agent recovery error was exposed or misclassified.');
|
|
|
|
$beforeInvalid = count($client->requests);
|
|
$reject(static fn () => Recovery::abortJob($admin, '41', [
|
|
'inspection_token' => strtoupper($token),
|
|
]), 409);
|
|
$reject(static fn () => Recovery::cleanup($admin, '41', [
|
|
'plan_id' => $planId, 'confirm' => true,
|
|
]), 422);
|
|
$reject(static fn () => Recovery::forceDelete($admin, '41', [
|
|
'ack_data_loss' => false,
|
|
]), 409);
|
|
$reject(static fn () => Recovery::forceDelete($admin, '41', [
|
|
'ack_data_loss' => true, 'confirm' => true,
|
|
]), 422);
|
|
$reject(static fn () => Recovery::forceDelete([
|
|
'user_id' => 97, 'is_admin' => true, 'is_super_admin' => false,
|
|
'can_manage' => true,
|
|
], '41', ['ack_data_loss' => true]), 403);
|
|
$reject(static fn () => Recovery::inspect($readonly, '41'), 403);
|
|
$assert(count($client->requests) === $beforeInvalid,
|
|
'Invalid or unauthorized recovery request reached the Agent.');
|
|
$routeSource = file_get_contents(
|
|
dirname(__DIR__) . '/config/routes_cloud.php'
|
|
);
|
|
$assert(is_string($routeSource) && str_contains($routeSource,
|
|
"/admin/operations/jobs/{id:[0-9]+}/force-delete"),
|
|
'Independent force-delete administrator route is missing.');
|
|
$controllerSource = file_get_contents(
|
|
dirname(__DIR__) . '/app/Controllers/CloudAdminOperationsController.php'
|
|
);
|
|
$assert(is_string($controllerSource)
|
|
&& preg_match(
|
|
'/public function inspect\\(.*?public function abortJob/s',
|
|
$controllerSource,
|
|
$inspectController
|
|
) === 1
|
|
&& strpos($inspectController[0],
|
|
'$inspection = CloudAdminOperationRecoveryService::inspect(') !== false
|
|
&& strpos($inspectController[0],
|
|
'CloudAdminOperationsService::detail($actor, $id)') !== false
|
|
&& strpos($inspectController[0],
|
|
'$inspection = CloudAdminOperationRecoveryService::inspect(')
|
|
< strpos($inspectController[0],
|
|
'CloudAdminOperationsService::detail($actor, $id)'),
|
|
'Inspect response read stale panel state before token-confirmed settlement.');
|
|
|
|
$client->responses[] = [
|
|
'operation_id' => $operationId,
|
|
'operation_status' => 'uncertain',
|
|
'checked_at' => 1_800_000_003,
|
|
'outcome' => 'uncertain',
|
|
'summary_code' => 'cleanup_required',
|
|
'abort_job' => [
|
|
'available' => true,
|
|
'kind' => 'backup',
|
|
'inspection_token' => $token,
|
|
],
|
|
'cleanup' => [
|
|
'available' => true,
|
|
'plan_id' => $planId,
|
|
'steps' => ['destroy_domain'],
|
|
],
|
|
'confirmation' => ['available' => false],
|
|
];
|
|
$reject(static fn () => Recovery::inspect($admin, '41'), 502,
|
|
'agent_invalid_inspection');
|
|
|
|
// A terminal observation must survive a second fresh read-only check
|
|
// before the Agent journal and panel ledger can be settled.
|
|
$confirmationToken = str_repeat('e', 64);
|
|
$client->responses[] = [
|
|
'operation_id' => $operationId,
|
|
'operation_status' => 'uncertain',
|
|
'checked_at' => 1_800_000_004,
|
|
'outcome' => 'failed',
|
|
'summary_code' => 'operation_failed',
|
|
'error_code' => 'snapshot_not_created',
|
|
'abort_job' => ['available' => false],
|
|
'cleanup' => ['available' => false],
|
|
'confirmation' => [
|
|
'available' => true,
|
|
'inspection_token' => $confirmationToken,
|
|
],
|
|
];
|
|
$client->responses[] = [
|
|
'operation' => [
|
|
'operation_id' => $operationId,
|
|
'status' => 'failed',
|
|
'error' => ['code' => 'snapshot_not_created'],
|
|
'result' => [],
|
|
],
|
|
'inspection' => [
|
|
'operation_id' => $operationId,
|
|
'operation_status' => 'failed',
|
|
'checked_at' => 1_800_000_005,
|
|
'outcome' => 'failed',
|
|
'summary_code' => 'operation_failed',
|
|
'error_code' => 'snapshot_not_created',
|
|
'abort_job' => ['available' => false],
|
|
'cleanup' => ['available' => false],
|
|
'confirmation' => [
|
|
'available' => true,
|
|
'inspection_token' => $confirmationToken,
|
|
],
|
|
],
|
|
];
|
|
$failed = Recovery::inspect($admin, '41');
|
|
$confirmRequest = $client->requests[array_key_last($client->requests)];
|
|
$assert(($failed['error_code'] ?? null) === 'snapshot_not_created'
|
|
&& ($failed['error'] ?? null) === '宿主机证据确认快照没有创建'
|
|
&& !str_contains(json_encode($failed), $confirmationToken),
|
|
'Known inspection error code was not mapped to a safe Chinese action message.');
|
|
$assert($confirmRequest['method'] === 'POST'
|
|
&& str_ends_with($confirmRequest['target'], '/confirm-inspection')
|
|
&& $confirmRequest['payload'] === [
|
|
'inspection_token' => $confirmationToken,
|
|
]
|
|
&& Capsule::table('cloud_jobs')->where('id', 41)->value('status')
|
|
=== 'failed',
|
|
'Confirmed failed inspection did not close the panel blocker.');
|
|
|
|
$successOperationId = 'dddddddddddddddddddddddddddddddd';
|
|
$successToken = str_repeat('f', 64);
|
|
$successInspection = [
|
|
'operation_id' => $successOperationId,
|
|
'operation_status' => 'needs_attention',
|
|
'checked_at' => 1_800_000_006,
|
|
'outcome' => 'succeeded',
|
|
'summary_code' => 'operation_succeeded',
|
|
'abort_job' => ['available' => false],
|
|
'cleanup' => ['available' => false],
|
|
'confirmation' => [
|
|
'available' => true,
|
|
'inspection_token' => $successToken,
|
|
],
|
|
];
|
|
$client->responses[] = $successInspection;
|
|
$client->responses[] = [
|
|
'operation' => [
|
|
'operation_id' => $successOperationId,
|
|
'status' => 'succeeded',
|
|
'result' => ['state' => 'running'],
|
|
],
|
|
'inspection' => array_replace($successInspection, [
|
|
'operation_status' => 'succeeded',
|
|
'checked_at' => 1_800_000_007,
|
|
]),
|
|
];
|
|
$succeeded = Recovery::inspect($admin, '42');
|
|
$assert($succeeded['summary_code'] === 'operation_succeeded'
|
|
&& Capsule::table('cloud_jobs')->where('id', 42)->value('status')
|
|
=== 'succeeded',
|
|
'Confirmed successful inspection did not close needs_attention.');
|
|
|
|
Capsule::table('cloud_jobs')->where('id', 42)->update([
|
|
'status' => 'needs_attention',
|
|
'error_code' => 'reconciliation_limit_reached',
|
|
'sealed_payload' => 'forbidden-payload',
|
|
'lease_token' => null,
|
|
'lease_until' => 0,
|
|
]);
|
|
$client->responses[] = $successInspection;
|
|
$client->responses[] = new CloudAgentError(
|
|
'raw changed host evidence', 409, 'inspection_inconclusive', false, 409
|
|
);
|
|
$inconclusive = $capture(static fn () => Recovery::inspect($admin, '42'));
|
|
$assert($inconclusive->agentCode === 'inspection_inconclusive'
|
|
&& $inconclusive->getMessage()
|
|
=== '宿主机状态在两次只读核对之间发生变化,请重新立即核对'
|
|
&& Capsule::table('cloud_jobs')->where('id', 42)->value('status')
|
|
=== 'needs_attention',
|
|
'Inconclusive fresh evidence changed the panel blocker.');
|
|
|
|
$terminalResponse = static function (int $checkedAt) use (
|
|
$successInspection, $successOperationId, $successToken
|
|
): array {
|
|
return [
|
|
'operation' => [
|
|
'operation_id' => $successOperationId,
|
|
'status' => 'succeeded',
|
|
'result' => ['state' => 'running'],
|
|
],
|
|
'inspection' => array_replace($successInspection, [
|
|
'operation_status' => 'succeeded',
|
|
'checked_at' => $checkedAt,
|
|
'confirmation' => [
|
|
'available' => true,
|
|
'inspection_token' => $successToken,
|
|
],
|
|
]),
|
|
];
|
|
};
|
|
$client->responses[] = array_replace($successInspection, [
|
|
'checked_at' => 1_800_000_008,
|
|
]);
|
|
$client->responses[] = $terminalResponse(1_800_000_009);
|
|
Recovery::abortJob($admin, '42', ['inspection_token' => $token]);
|
|
$lastTwo = array_slice($client->requests, -2);
|
|
$assert(str_ends_with($lastTwo[0]['target'], '/abort-job')
|
|
&& str_ends_with($lastTwo[1]['target'], '/confirm-inspection')
|
|
&& Capsule::table('cloud_jobs')->where('id', 42)->value('status')
|
|
=== 'succeeded',
|
|
'Abort terminal response did not converge the panel ledger.');
|
|
|
|
Capsule::table('cloud_jobs')->where('id', 42)->update([
|
|
'status' => 'needs_attention',
|
|
'error_code' => 'reconciliation_limit_reached',
|
|
'sealed_payload' => 'forbidden-payload',
|
|
'lease_token' => null,
|
|
'lease_until' => 0,
|
|
]);
|
|
$client->responses[] = array_replace($successInspection, [
|
|
'checked_at' => 1_800_000_010,
|
|
]);
|
|
$client->responses[] = $terminalResponse(1_800_000_011);
|
|
Recovery::cleanup($admin, '42', ['plan_id' => $planId]);
|
|
$lastTwo = array_slice($client->requests, -2);
|
|
$assert(str_ends_with($lastTwo[0]['target'], '/cleanup')
|
|
&& str_ends_with($lastTwo[1]['target'], '/confirm-inspection')
|
|
&& Capsule::table('cloud_jobs')->where('id', 42)->value('status')
|
|
=== 'succeeded',
|
|
'Cleanup terminal response did not converge the panel ledger.');
|
|
|
|
echo "Administrator inspect, token-confirmed settlement, recovery-error, abort and cleanup contracts passed.\n";
|
|
}
|