You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
1718 lines
67 KiB
PHP
1718 lines
67 KiB
PHP
<?php
|
|
|
|
namespace App\Services\Cloud;
|
|
|
|
use App\Models\CloudJob;
|
|
|
|
/** Read-only host inspection plus a separately authorized force-delete task. */
|
|
final class CloudAdminOperationRecoveryService
|
|
{
|
|
private const OPERATION_STATUSES = [
|
|
'reserved', 'dispatching', 'succeeded', 'failed',
|
|
];
|
|
private const OUTCOMES = [
|
|
'succeeded' => 'succeeded',
|
|
'failed' => 'failed',
|
|
'uncertain' => 'inconclusive',
|
|
];
|
|
private const EFFECT_RESOLUTIONS = [
|
|
'applied', 'not_applied', 'partial', 'inconclusive',
|
|
];
|
|
private const INSPECTION_FIELDS = [
|
|
'operation_id', 'action', 'resource_id', 'request_fingerprint',
|
|
'operation_status', 'checked_at', 'outcome',
|
|
'summary_code', 'error_code', 'effect_resolution',
|
|
'fence_state', 'fence_resolution_required',
|
|
'completion_delivery_state', 'completion_delivery_error_code',
|
|
'effect_projection', 'abort_job',
|
|
'cleanup',
|
|
];
|
|
private const COMPLETION_DELIVERY_ERRORS = [
|
|
'completion_outbox_invalid' =>
|
|
'Agent 完成回执队列状态损坏,当前结果不能采信',
|
|
'completion_receipt_missing' =>
|
|
'Agent 完成回执内容缺失,当前结果不能采信',
|
|
'completion_receipt_invalid' =>
|
|
'Agent 完成回执内容无效,当前结果不能采信',
|
|
'completion_operation_missing' =>
|
|
'Agent 完成回执缺少对应的原操作记录,当前结果不能采信',
|
|
'completion_operation_invalid' =>
|
|
'Agent 完成回执对应的原操作记录无效,当前结果不能采信',
|
|
'completion_identity_mismatch' =>
|
|
'Agent 完成回执身份与原操作记录不匹配,当前结果不能采信',
|
|
];
|
|
private const MAX_INSPECTION_BYTES = 96 * 1024;
|
|
private const MAX_PROJECTION_BYTES = 64 * 1024;
|
|
private const MAX_PROJECTION_NODES = 1024;
|
|
private const MAX_PROJECTION_DEPTH = 6;
|
|
|
|
/** Agent VMMetadata fields explicitly projected by operation kind. */
|
|
private const VM_PROJECTION = [
|
|
'uuid' => 'string',
|
|
'name' => 'string',
|
|
'status' => 'string',
|
|
'vcpu' => 'integer',
|
|
'memory_mib' => 'integer',
|
|
'disk_gib' => 'integer',
|
|
'bandwidth_mbps' => 'integer',
|
|
'hostname' => 'string',
|
|
'public_network' => 'public_network',
|
|
'public_pool_cidr' => 'string',
|
|
'gateway' => 'string',
|
|
'dns_servers' => 'string_list',
|
|
'network' => 'string',
|
|
'template_id' => 'string',
|
|
'machine' => 'string',
|
|
'firmware' => 'string',
|
|
'secure_boot' => 'boolean',
|
|
'vpc_port' => 'vpc_port',
|
|
'settings_changed' => 'boolean',
|
|
'network_changed' => 'boolean',
|
|
'reinstall_committed' => 'boolean',
|
|
'cleanup_pending' => 'boolean',
|
|
];
|
|
private const DELETE_PROJECTION = [
|
|
'deleted' => 'boolean',
|
|
'managed_artifacts_absent' => 'boolean',
|
|
'forced' => 'boolean',
|
|
];
|
|
private const SNAPSHOT_PROJECTION = [
|
|
'id' => 'string',
|
|
'snapshot_key' => 'string',
|
|
'name' => 'string',
|
|
'created_at' => 'string',
|
|
'state' => 'string',
|
|
'consistency' => 'string',
|
|
'size_bytes' => 'integer',
|
|
'created' => 'boolean',
|
|
];
|
|
private const SNAPSHOT_MUTATION_PROJECTION = [
|
|
'snapshot_key' => 'string',
|
|
'restored' => 'boolean',
|
|
'deleted' => 'boolean',
|
|
'power_state' => 'string',
|
|
'cleanup_pending' => 'boolean',
|
|
];
|
|
private const BACKUP_PROJECTION = [
|
|
'backup_key' => 'string',
|
|
'size_bytes' => 'integer',
|
|
'virtual_size_bytes' => 'integer',
|
|
'sha256' => 'string',
|
|
'consistency' => 'string',
|
|
'cleanup_pending' => 'boolean',
|
|
];
|
|
private const CATALOG_IMPORT_PROJECTION = [
|
|
'id' => 'string',
|
|
'owner_id' => 'integer',
|
|
'name' => 'string',
|
|
'os_family' => 'string',
|
|
'version' => 'string',
|
|
'architecture' => 'string',
|
|
'size_bytes' => 'integer',
|
|
'virtual_size_bytes' => 'integer',
|
|
'sha256' => 'string',
|
|
'revision' => 'integer',
|
|
'generation_token' => 'string',
|
|
'created_at' => 'string',
|
|
];
|
|
private const CATALOG_DELETE_PROJECTION = [
|
|
'resource_type' => 'string',
|
|
'id' => 'string',
|
|
'owner_id' => 'integer',
|
|
'deleted' => 'boolean',
|
|
'managed_image_absent' => 'boolean',
|
|
'managed_metadata_absent' => 'boolean',
|
|
'managed_artifacts_absent' => 'boolean',
|
|
];
|
|
private const SUMMARY_LABELS = [
|
|
'operation_succeeded' => '只读证据与宿主机成功结果一致',
|
|
'operation_failed' => '只读证据与宿主机失败结果一致',
|
|
'evidence_incomplete' => '只读证据不足;原任务终态保持不变',
|
|
'abortable_job_active' => '检测到活动作业;本接口不会中止或续作',
|
|
'cleanup_required' => '检测到残留;本接口不会自动清理',
|
|
];
|
|
private const INSPECTION_ERROR_LABELS = [
|
|
'creation_absent' => '宿主机未发现虚拟机或面板管理资源',
|
|
'expected_state_resource_absent' => '目标虚拟机不存在',
|
|
'settings_not_committed' => '实例设置没有提交',
|
|
'network_not_committed' => '网络变更没有提交',
|
|
'network_state_not_applied' => '虚拟机网卡未达到请求状态',
|
|
'service_state_not_applied' => '虚拟机服务未达到请求状态',
|
|
'firewall_not_committed' => '防火墙策略没有提交',
|
|
'rescue_not_committed' => '救援系统没有启用',
|
|
'rescue_force_start_incomplete' => '救援系统已准备但未成功启动',
|
|
'rescue_exit_incomplete' => '救援系统仍处于准备或运行状态',
|
|
'reinstall_not_committed' => '系统盘没有切换',
|
|
'snapshot_not_created' => '快照没有创建',
|
|
'snapshot_not_deleted' => '快照仍然存在',
|
|
'snapshot_not_restored' => '系统盘没有切换到快照',
|
|
'backup_workspace_cleanup_not_applied' =>
|
|
'备份临时工作区仍完整存在,清理没有生效',
|
|
'storage_configuration_mismatch' => '固定存储池与面板配置不一致',
|
|
'storage_configuration_incomplete' => '存储池配置尚未完成',
|
|
'import_not_completed' => '目录资源导入未完成',
|
|
'delete_not_completed' => '目录资源仍然存在',
|
|
'force_required' => '正常关机等待结束后虚拟机仍在运行',
|
|
'force_delete_unconfirmed_active' => '强制删除没有确认完成且虚拟机仍在运行',
|
|
];
|
|
|
|
public static function inspect(array $actor, string $id): array
|
|
{
|
|
[$job, $client] = self::context($actor, $id, 30.0);
|
|
try {
|
|
$value = $client->request(
|
|
'GET', self::agentPath($job, '/inspect'), $actor
|
|
);
|
|
} catch (CloudAgentError $error) {
|
|
throw new CloudAgentError(
|
|
'宿主机只读检查暂不可用',
|
|
$error->retryable ? 503 : 502,
|
|
'operation_inspection_unavailable',
|
|
false,
|
|
$error->agentHttpStatus
|
|
);
|
|
}
|
|
$inspection = self::inspection(
|
|
$value,
|
|
(string) $job->operation_id,
|
|
(string) $job->kind,
|
|
false
|
|
);
|
|
self::assertInspectionIdentity($job, $inspection);
|
|
return $inspection;
|
|
}
|
|
|
|
/**
|
|
* Read bounded host evidence for the automatic recovery coordinator.
|
|
*
|
|
* Unlike inspect(), this does not borrow a human administrator identity.
|
|
* CloudAgentClient constructs a user_id=0 capability bound to this exact
|
|
* operation, and the Agent permits it only on recovery inspection routes.
|
|
*/
|
|
public static function inspectForRecovery(string $id): array
|
|
{
|
|
[$job, $client] = self::jobContext(
|
|
$id,
|
|
30.0,
|
|
self::automaticRecoveryKinds()
|
|
);
|
|
if (!self::supportsAutomaticRecovery($job)
|
|
|| (string) $job->execution_mode !== 'agent'
|
|
|| (string) $job->phase !== 'terminal'
|
|
|| (string) $job->outcome !== 'failed'
|
|
|| (string) $job->status !== 'failed'
|
|
|| (int) $job->dispatch_attempts !== 1
|
|
|| !(bool) $job->effect_may_have_applied
|
|
|| (bool) $job->safe_to_resubmit
|
|
|| (string) $job->effect_resolution !== 'unknown') {
|
|
throw new CloudError('任务不处于可自动核对的影响未知状态', 409);
|
|
}
|
|
try {
|
|
$value = $client->systemRecoveryRequest(
|
|
'GET',
|
|
self::agentPath($job, '/inspect'),
|
|
(string) $job->operation_id
|
|
);
|
|
} catch (CloudAgentError $error) {
|
|
throw new CloudAgentError(
|
|
'宿主机自动只读核对暂不可用',
|
|
$error->retryable ? 503 : 502,
|
|
'operation_inspection_unavailable',
|
|
false,
|
|
$error->agentHttpStatus
|
|
);
|
|
}
|
|
return self::inspection(
|
|
$value,
|
|
(string) $job->operation_id,
|
|
(string) $job->kind,
|
|
true
|
|
);
|
|
}
|
|
|
|
/**
|
|
* Release the exact Agent fence after PHP has durably prepared its ledger.
|
|
* The original host mutation is never sent by this path.
|
|
*/
|
|
public static function resolveFenceForRecovery(
|
|
string $id,
|
|
string $resolution,
|
|
string $reason,
|
|
array $inspection
|
|
): array {
|
|
if (!in_array($resolution, ['applied', 'not_applied'], true)
|
|
|| mb_strlen($reason) < 20 || mb_strlen($reason) > 500) {
|
|
throw new CloudError('自动恢复结论或审计说明无效', 400);
|
|
}
|
|
self::assertCompletionDeliveryUsable($inspection);
|
|
[$job, $client] = self::jobContext(
|
|
$id,
|
|
30.0,
|
|
self::automaticRecoveryKinds()
|
|
);
|
|
if ((string) $job->kind === 'create') {
|
|
CloudCreateFailureRecoveryService::assertPreparedForFence(
|
|
$job,
|
|
$resolution,
|
|
$reason
|
|
);
|
|
CloudCreateFailureFinalizer::assertResolutionSupported(
|
|
$job,
|
|
$resolution,
|
|
$inspection
|
|
);
|
|
}
|
|
$checkedAt = $inspection['checked_at'] ?? null;
|
|
$identity = CloudOperationRecoveryIdentity::expected(
|
|
$job,
|
|
$resolution === 'applied'
|
|
);
|
|
CloudOperationRecoveryIdentity::assertMatches($identity, [
|
|
'action' => $inspection['action'] ?? null,
|
|
'resource_id' => $inspection['resource_id'] ?? null,
|
|
'request_fingerprint' =>
|
|
$inspection['request_fingerprint'] ?? null,
|
|
]);
|
|
if (!self::supportsAutomaticRecovery($job)
|
|
|| !CloudOperationRecoveryPolicy::allowsSystemResolution(
|
|
(string) $job->kind,
|
|
$resolution
|
|
)
|
|
|| (string) $job->execution_mode !== 'agent'
|
|
|| (string) $job->phase !== 'terminal'
|
|
|| (string) $job->outcome !== 'failed'
|
|
|| (string) $job->status !== 'failed'
|
|
|| (int) $job->dispatch_attempts !== 1
|
|
|| !(bool) $job->effect_may_have_applied
|
|
|| (bool) $job->safe_to_resubmit
|
|
|| (string) $job->effect_resolution !== 'unknown'
|
|
|| ((string) $job->kind !== 'create'
|
|
&& !self::automaticPreparedReason(
|
|
(string) $job->effect_resolution_reason
|
|
))
|
|
|| ($inspection['operation_id'] ?? null)
|
|
!== (string) $job->operation_id
|
|
|| ($inspection['effect_resolution'] ?? null) !== $resolution
|
|
|| ($inspection['fence_state'] ?? null) !== 'active'
|
|
|| ($inspection['fence_resolution_required'] ?? null) !== true
|
|
|| !is_int($checkedAt)
|
|
|| $checkedAt < time() - 120 || $checkedAt > time() + 5) {
|
|
throw new CloudError('任务尚未完成可验证的账本预提交', 409);
|
|
}
|
|
if (array_key_exists('effect_projection', $inspection)) {
|
|
self::effectProjection(
|
|
$inspection['effect_projection'],
|
|
(string) $job->kind,
|
|
$resolution
|
|
);
|
|
}
|
|
if ($resolution === 'applied'
|
|
&& (string) $job->kind === 'instance_migration_prepare'
|
|
&& CloudInstanceMigrationService::ownsPrepareJob((int) $job->id)
|
|
&& (!is_array($inspection['effect_projection'] ?? null)
|
|
|| ($inspection['effect_projection']['source_stopped']
|
|
?? null) !== true
|
|
|| ($inspection['effect_projection'][
|
|
'source_autostart_disabled'] ?? null) !== true
|
|
|| !is_bool($inspection['effect_projection'][
|
|
'source_was_running'] ?? null))) {
|
|
throw new CloudError(
|
|
'维护迁移源端检查缺少实例已停止证据',
|
|
409
|
|
);
|
|
}
|
|
if ($resolution === 'applied'
|
|
&& (string) $job->kind === 'instance_settings'
|
|
&& (!is_array($inspection['effect_projection'] ?? null)
|
|
|| ($inspection['effect_projection']['settings_changed']
|
|
?? null) !== true)) {
|
|
throw new CloudError('实例设置检查缺少已提交证据', 409);
|
|
}
|
|
return self::resolveFenceOnAgent(
|
|
$client,
|
|
$job,
|
|
$resolution,
|
|
$reason,
|
|
$checkedAt,
|
|
null,
|
|
$identity
|
|
);
|
|
}
|
|
|
|
/**
|
|
* Read an already-committed system fence resolution after a PHP crash.
|
|
*
|
|
* This recovery seam is intentionally GET-only. It can finish a durable
|
|
* PHP prepared intent even when current host inspection is unavailable or
|
|
* has changed, but it can never create a new Agent conclusion.
|
|
*/
|
|
public static function preparedFenceResolutionForRecovery(
|
|
string $id,
|
|
string $resolution,
|
|
string $reason
|
|
): array {
|
|
if (!in_array($resolution, ['applied', 'not_applied'], true)
|
|
|| mb_strlen($reason) < 20 || mb_strlen($reason) > 500) {
|
|
throw new CloudError('自动恢复结论或审计说明无效', 400);
|
|
}
|
|
[$job, $client] = self::jobContext(
|
|
$id,
|
|
30.0,
|
|
self::automaticRecoveryKinds()
|
|
);
|
|
if ((string) $job->kind === 'create') {
|
|
CloudCreateFailureRecoveryService::assertPreparedForFence(
|
|
$job,
|
|
$resolution,
|
|
$reason
|
|
);
|
|
}
|
|
if (!self::supportsAutomaticRecovery($job)
|
|
|| !CloudOperationRecoveryPolicy::allowsSystemResolution(
|
|
(string) $job->kind,
|
|
$resolution
|
|
)
|
|
|| (string) $job->execution_mode !== 'agent'
|
|
|| (string) $job->phase !== 'terminal'
|
|
|| (string) $job->outcome !== 'failed'
|
|
|| (string) $job->status !== 'failed'
|
|
|| (int) $job->dispatch_attempts !== 1
|
|
|| !(bool) $job->effect_may_have_applied
|
|
|| (bool) $job->safe_to_resubmit
|
|
|| (string) $job->effect_resolution !== 'unknown'
|
|
|| ((string) $job->kind !== 'create'
|
|
&& !self::automaticPreparedReason(
|
|
(string) $job->effect_resolution_reason
|
|
))) {
|
|
throw new CloudError('任务尚未完成可验证的账本预提交', 409);
|
|
}
|
|
// A prepared escrow persists the identity validated before the PHP
|
|
// finalizer changed any catalog or instance ledger rows. Verify that
|
|
// immutable binding before accepting an existing Agent fence verdict.
|
|
CloudOperationRecoveryIdentity::expected(
|
|
$job,
|
|
$resolution === 'applied'
|
|
);
|
|
$value = $client->systemRecoveryRequest(
|
|
'GET',
|
|
self::agentPath($job, '/fence-resolution'),
|
|
(string) $job->operation_id
|
|
);
|
|
$existing = self::fenceResolution(
|
|
$value,
|
|
(string) $job->operation_id,
|
|
true
|
|
);
|
|
if (!hash_equals($resolution, $existing['resolution'])
|
|
|| !hash_equals(
|
|
hash('sha256', $reason),
|
|
$existing['reason_sha256']
|
|
)) {
|
|
throw new CloudError(
|
|
'宿主机栅栏已有不同的审计结论,不能覆盖',
|
|
409
|
|
);
|
|
}
|
|
return $existing;
|
|
}
|
|
|
|
/**
|
|
* Resolve only the Agent's durable resource fence.
|
|
*
|
|
* This is journal metadata, not a libvirt mutation. We first read any
|
|
* previously committed resolution so a PHP crash after the Agent commit
|
|
* can be completed without replaying the POST. A new POST is attempted
|
|
* at most once; a lost response is followed by one read-only verification.
|
|
*/
|
|
public static function resolveFence(
|
|
array $actor,
|
|
string $id,
|
|
string $resolution,
|
|
string $reason,
|
|
array $inspection
|
|
): array {
|
|
if (!($actor['is_admin'] ?? false)
|
|
|| !($actor['is_super_admin'] ?? false)
|
|
|| !($actor['can_manage'] ?? false)) {
|
|
throw new CloudError('只有超级管理员可以确认任务影响', 403);
|
|
}
|
|
CloudAccess::manage($actor);
|
|
if (!in_array($resolution, ['applied', 'not_applied'], true)
|
|
|| mb_strlen($reason) < 20 || mb_strlen($reason) > 500) {
|
|
throw new CloudError('影响结论或审计说明无效', 400);
|
|
}
|
|
self::assertCompletionDeliveryUsable($inspection);
|
|
[$job, $client] = self::context($actor, $id, 30.0);
|
|
$checkedAt = $inspection['checked_at'] ?? null;
|
|
if (($inspection['operation_id'] ?? null)
|
|
!== (string) $job->operation_id
|
|
|| !is_int($checkedAt)
|
|
|| $checkedAt < time() - 120 || $checkedAt > time() + 5) {
|
|
throw new CloudError('宿主机只读检查已经失效', 409);
|
|
}
|
|
$current = CloudJob::where('id', (int) $job->id)
|
|
->where('operation_id', (string) $job->operation_id)
|
|
->where('dispatch_attempts', 1)
|
|
->where('phase', 'terminal')->where('outcome', 'failed')
|
|
->where('effect_resolution', 'unknown')
|
|
->where('effect_may_have_applied', 1)->first();
|
|
if ($current === null) {
|
|
throw new CloudError('任务不处于可确认的影响未知状态', 409);
|
|
}
|
|
if ((string) $current->kind !== 'create') {
|
|
// Never clear the Agent fence first and discover afterwards that
|
|
// PHP cannot atomically converge the corresponding business
|
|
// ledger (network IPs, snapshots, firewall, catalog, lifecycle,
|
|
// credentials, and so on). Each such kind needs its own finalizer
|
|
// before it can be enabled here.
|
|
throw new CloudError(
|
|
'该任务类型尚无可原子收敛业务账本的解除流程',
|
|
409
|
|
);
|
|
}
|
|
// A create needs kind-specific financial/placement settlement. Reject
|
|
// unsupported "applied" conclusions and incomplete absence evidence
|
|
// before reading or writing Agent fence metadata.
|
|
CloudCreateFailureFinalizer::assertResolutionSupported(
|
|
$current,
|
|
$resolution,
|
|
$inspection
|
|
);
|
|
$identity = CloudOperationRecoveryIdentity::expected(
|
|
$current,
|
|
$resolution === 'applied'
|
|
);
|
|
CloudOperationRecoveryIdentity::assertMatches($identity, [
|
|
'action' => $inspection['action'] ?? null,
|
|
'resource_id' => $inspection['resource_id'] ?? null,
|
|
'request_fingerprint' =>
|
|
$inspection['request_fingerprint'] ?? null,
|
|
]);
|
|
|
|
$agentActor = $actor;
|
|
$agentActor['operation_id'] = (string) $job->operation_id;
|
|
return self::resolveFenceOnAgent(
|
|
$client,
|
|
$job,
|
|
$resolution,
|
|
$reason,
|
|
$checkedAt,
|
|
$agentActor,
|
|
$identity
|
|
);
|
|
}
|
|
|
|
/** Queue a new force phase; the failed graceful task remains immutable. */
|
|
public static function forceDelete(
|
|
array $actor,
|
|
string $id,
|
|
array $input
|
|
): array {
|
|
CloudInput::only($input, ['ack_data_loss']);
|
|
if (($input['ack_data_loss'] ?? null) !== true) {
|
|
throw new CloudError('必须明确确认强制断电可能造成未刷盘数据丢失', 409);
|
|
}
|
|
if (!($actor['is_admin'] ?? false)
|
|
|| !($actor['is_super_admin'] ?? false)
|
|
|| !($actor['can_manage'] ?? false)) {
|
|
throw new CloudError('只有超级管理员可以确认强制删除', 403);
|
|
}
|
|
CloudAccess::manage($actor);
|
|
if (preg_match('/^[1-9][0-9]{0,17}$/D', $id) !== 1) {
|
|
throw new CloudError('编号无效', 400);
|
|
}
|
|
$prior = CloudJob::where('id', (int) $id)->first();
|
|
if ($prior === null) {
|
|
throw new CloudError('任务不存在', 404);
|
|
}
|
|
$job = CloudJobService::enqueueForceDelete($prior, $actor);
|
|
return [
|
|
'task_id' => (string) $job->id,
|
|
'status_url' => '/api/v1/tasks/' . (int) $job->id,
|
|
];
|
|
}
|
|
|
|
/** Queue a new VPC convergence operation after one fresh Agent inspection. */
|
|
public static function repairVpc(
|
|
array $actor,
|
|
string $id,
|
|
array $input
|
|
): array {
|
|
CloudInput::only($input, ['acknowledge_repair']);
|
|
if (($input['acknowledge_repair'] ?? null) !== true) {
|
|
throw new CloudError('必须明确确认将创建新的 VPC 修复任务', 409);
|
|
}
|
|
if (!($actor['is_admin'] ?? false)
|
|
|| !($actor['is_super_admin'] ?? false)
|
|
|| !($actor['can_manage'] ?? false)) {
|
|
throw new CloudError('只有超级管理员可以修复 VPC 状态', 403);
|
|
}
|
|
CloudAccess::manage($actor);
|
|
if (preg_match('/^[1-9][0-9]{0,17}$/D', $id) !== 1) {
|
|
throw new CloudError('编号无效', 400);
|
|
}
|
|
$inspection = self::inspect($actor, $id);
|
|
$job = CloudVpcPortService::repairUnknownEffect(
|
|
$actor,
|
|
(int) $id,
|
|
$inspection
|
|
);
|
|
return [
|
|
'task_id' => (string) $job->id,
|
|
'status_url' => '/api/v1/tasks/' . (int) $job->id,
|
|
];
|
|
}
|
|
|
|
/** Resolve one exact VPC removal after a physical host is retired. */
|
|
public static function retireVpcHost(
|
|
array $actor,
|
|
string $id,
|
|
array $input
|
|
): array {
|
|
CloudInput::only($input, [
|
|
'acknowledge_permanent_retirement', 'reason',
|
|
]);
|
|
if (($input['acknowledge_permanent_retirement'] ?? null) !== true) {
|
|
throw new CloudError('必须明确确认宿主机已永久退役', 409);
|
|
}
|
|
$reason = $input['reason'] ?? null;
|
|
if (!is_string($reason)) {
|
|
throw new CloudError('必须填写可审计的宿主机退役依据');
|
|
}
|
|
if (preg_match('/^[1-9][0-9]{0,17}$/D', $id) !== 1) {
|
|
throw new CloudError('编号无效', 400);
|
|
}
|
|
return CloudVpcHostRetirementService::retire(
|
|
$actor, (int) $id, $reason
|
|
);
|
|
}
|
|
|
|
/** @return array{0:CloudJob,1:CloudAgentClient} */
|
|
private static function context(
|
|
array $actor,
|
|
string $id,
|
|
float $timeoutSeconds
|
|
): array {
|
|
if (!($actor['is_admin'] ?? false)) {
|
|
throw new CloudError('需要管理员权限', 403);
|
|
}
|
|
CloudAccess::manage($actor);
|
|
return self::jobContext($id, $timeoutSeconds);
|
|
}
|
|
|
|
/** @return array{0:CloudJob,1:CloudAgentClient} */
|
|
private static function jobContext(
|
|
string $id,
|
|
float $timeoutSeconds,
|
|
?array $requiredKinds = null
|
|
): array {
|
|
if (preg_match('/^[1-9][0-9]{0,17}$/D', $id) !== 1) {
|
|
throw new CloudError('编号无效', 400);
|
|
}
|
|
$job = CloudJob::query()->select([
|
|
'id', 'operation_id', 'request_idempotency_key',
|
|
'request_fingerprint', 'host_id', 'kind', 'execution_mode',
|
|
'phase', 'outcome', 'status', 'attempts', 'dispatch_attempts',
|
|
'dispatch_deadline_at', 'dispatch_started_at',
|
|
'agent_accepted_at', 'receipt_deadline_at',
|
|
'completion_event_id', 'terminal_at', 'error_code',
|
|
'effect_may_have_applied', 'safe_to_resubmit',
|
|
'effect_resolution', 'effect_resolution_reason', 'dispatched_at',
|
|
'finalization_incident', 'finalization_error_code', 'result_json',
|
|
'sealed_payload', 'instance_id', 'snapshot_id', 'backup_id',
|
|
'backup_restore_id', 'backup_recovery_id', 'parent_job_id',
|
|
'public_network_transition_id',
|
|
'supersedes_job_id', 'vpc_repair_root_job_id',
|
|
'vpc_id', 'vpc_port_id', 'vpc_batch_id',
|
|
'user_id', 'order_id',
|
|
])->where('id', (int) $id)->first();
|
|
if ($job === null) {
|
|
throw new CloudError('任务不存在', 404);
|
|
}
|
|
if ($requiredKinds !== null
|
|
&& !in_array((string) $job->kind, $requiredKinds, true)) {
|
|
throw new CloudError('任务类型不支持自动效果恢复', 409);
|
|
}
|
|
if ((string) $job->kind === 'backup_restore'
|
|
&& !CloudPortableRestoreFinalizer::owns($job)) {
|
|
throw new CloudError('旧备份恢复任务不支持效果恢复', 409);
|
|
}
|
|
if ((string) $job->kind === 'network_change'
|
|
&& (int) ($job->public_network_transition_id ?? 0) < 1) {
|
|
throw new CloudError('公网网络任务缺少转换账本', 409);
|
|
}
|
|
if ((string) $job->execution_mode !== 'agent'
|
|
|| CloudJobService::isAggregateKind((string) $job->kind)) {
|
|
throw new CloudError(
|
|
'任务执行模式账本异常,只能由 PHP 本地恢复核对',
|
|
409
|
|
);
|
|
}
|
|
if (CloudJobService::hasTerminalRecoveryStateConflict($job)
|
|
|| ((string) $job->phase === 'terminal'
|
|
&& (bool) $job->finalization_incident
|
|
&& (in_array(
|
|
(string) $job->effect_resolution,
|
|
['applied', 'not_applied'],
|
|
true
|
|
) || CloudDispatchLedger::hasRepairIncident($job)))) {
|
|
throw new CloudError(
|
|
'任务终态、派发信封或恢复账本尚未安全收敛,只能由 PHP 本地恢复核对',
|
|
409
|
|
);
|
|
}
|
|
if (preg_match('/^[a-f0-9]{32}$/D',
|
|
(string) $job->operation_id) !== 1
|
|
|| (int) $job->host_id < 1) {
|
|
throw new CloudError('任务缺少可检查的 Agent 操作标识', 409);
|
|
}
|
|
CloudDispatchLedger::assertDispatched($job);
|
|
if ($job->dispatched_at === null || (int) $job->dispatched_at < 1) {
|
|
throw new CloudError('任务没有跨越 Agent 派发边界', 409);
|
|
}
|
|
return [$job, CloudAgentSettingsService::clientForConfiguration(
|
|
(int) $job->host_id, $timeoutSeconds
|
|
)];
|
|
}
|
|
|
|
private static function agentPath(CloudJob $job, string $suffix): string
|
|
{
|
|
return '/v1/operations/'
|
|
. rawurlencode((string) $job->operation_id) . $suffix;
|
|
}
|
|
|
|
private static function automaticPreparedReason(string $reason): bool
|
|
{
|
|
return preg_match(
|
|
'/^(?:automatic_recovery:v1:[1-9][0-9]{0,10}:[1-9][0-9]{0,9}:)?prepared:automatic_(?:applied|not_applied)_(?:succeeded|failed)_(?:fence|nofence):[a-f0-9]{64}:[a-f0-9]{64}$/D',
|
|
$reason
|
|
) === 1;
|
|
}
|
|
|
|
/** Create is inspectable, but never enters generic effect finalization. */
|
|
private static function automaticRecoveryKinds(): array
|
|
{
|
|
return CloudOperationRecoveryPolicy::automaticKinds();
|
|
}
|
|
|
|
private static function supportsAutomaticRecovery(CloudJob $job): bool
|
|
{
|
|
return CloudOperationRecoveryPolicy::supportsLedgerFinalizer(
|
|
(string) $job->kind
|
|
) || ((string) $job->kind === 'create'
|
|
&& $job->order_id !== null);
|
|
}
|
|
|
|
private static function assertCompletionDeliveryUsable(
|
|
array $inspection
|
|
): void {
|
|
if (($inspection['completion_delivery_state'] ?? null)
|
|
=== 'quarantined') {
|
|
throw new CloudError(
|
|
'Agent 完成回执处于隔离状态,修复回执证据前不能确认操作影响',
|
|
409
|
|
);
|
|
}
|
|
}
|
|
|
|
private static function assertInspectionIdentity(
|
|
CloudJob $job,
|
|
array $inspection
|
|
): void {
|
|
if (!CloudOperationRecoveryIdentity::supports((string) $job->kind)) {
|
|
return;
|
|
}
|
|
$identity = CloudOperationRecoveryIdentity::expected(
|
|
$job,
|
|
($inspection['effect_resolution'] ?? null) === 'applied'
|
|
);
|
|
CloudOperationRecoveryIdentity::assertMatches($identity, [
|
|
'action' => $inspection['action'] ?? null,
|
|
'resource_id' => $inspection['resource_id'] ?? null,
|
|
'request_fingerprint' =>
|
|
$inspection['request_fingerprint'] ?? null,
|
|
]);
|
|
}
|
|
|
|
/** Convert the fixed Agent inspection schema into a bounded DTO. */
|
|
private static function inspection(
|
|
array $value,
|
|
string $operationId,
|
|
string $kind,
|
|
bool $includeProjection
|
|
): array {
|
|
try {
|
|
$encoded = json_encode(
|
|
$value,
|
|
JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES
|
|
| JSON_UNESCAPED_UNICODE
|
|
);
|
|
} catch (\Throwable $error) {
|
|
self::invalidInspection();
|
|
}
|
|
$status = $value['operation_status'] ?? null;
|
|
$outcome = $value['outcome'] ?? null;
|
|
$summaryCode = $value['summary_code'] ?? null;
|
|
$checkedAt = $value['checked_at'] ?? null;
|
|
$effectResolution = $value['effect_resolution'] ?? null;
|
|
$fenceState = $value['fence_state'] ?? null;
|
|
$fenceResolutionRequired =
|
|
$value['fence_resolution_required'] ?? null;
|
|
$hasCompletionDeliveryState = array_key_exists(
|
|
'completion_delivery_state',
|
|
$value
|
|
);
|
|
$hasCompletionDeliveryError = array_key_exists(
|
|
'completion_delivery_error_code',
|
|
$value
|
|
);
|
|
$completionDeliveryState =
|
|
$value['completion_delivery_state'] ?? null;
|
|
$completionDeliveryErrorCode =
|
|
$value['completion_delivery_error_code'] ?? null;
|
|
$identity = [
|
|
'action' => $value['action'] ?? null,
|
|
'resource_id' => $value['resource_id'] ?? null,
|
|
'request_fingerprint' => $value['request_fingerprint'] ?? null,
|
|
];
|
|
if (array_is_list($value)
|
|
|| strlen($encoded) > self::MAX_INSPECTION_BYTES
|
|
|| array_diff(array_keys($value), self::INSPECTION_FIELDS) !== []
|
|
|| ($value['operation_id'] ?? null) !== $operationId
|
|
|| !is_string($identity['action'])
|
|
|| strlen($identity['action']) > 128
|
|
|| preg_match(
|
|
'/^[a-z][a-z0-9]*(?:[-:][a-z0-9]+)*$/D',
|
|
$identity['action']
|
|
) !== 1
|
|
|| !is_string($identity['resource_id'])
|
|
|| strlen($identity['resource_id']) > 255
|
|
|| preg_match('/^[A-Za-z0-9][A-Za-z0-9_.:-]*$/D',
|
|
$identity['resource_id']) !== 1
|
|
|| !is_string($identity['request_fingerprint'])
|
|
|| preg_match('/^[a-f0-9]{64}$/D',
|
|
$identity['request_fingerprint']) !== 1
|
|
|| !is_string($status)
|
|
|| !in_array($status, self::OPERATION_STATUSES, true)
|
|
|| !is_string($outcome)
|
|
|| !array_key_exists($outcome, self::OUTCOMES)
|
|
|| !is_string($summaryCode)
|
|
|| !array_key_exists($summaryCode, self::SUMMARY_LABELS)
|
|
|| !is_int($checkedAt) || $checkedAt < 1
|
|
|| !is_string($effectResolution)
|
|
|| !in_array($effectResolution,
|
|
self::EFFECT_RESOLUTIONS, true)
|
|
|| !is_string($fenceState)
|
|
|| !in_array($fenceState,
|
|
['not_required', 'active', 'resolved'], true)
|
|
|| !is_bool($fenceResolutionRequired)
|
|
|| $fenceResolutionRequired !== ($fenceState === 'active')
|
|
|| $hasCompletionDeliveryState !== $hasCompletionDeliveryError
|
|
|| ($hasCompletionDeliveryState
|
|
&& ($completionDeliveryState !== 'quarantined'
|
|
|| !is_string($completionDeliveryErrorCode)
|
|
|| !array_key_exists(
|
|
$completionDeliveryErrorCode,
|
|
self::COMPLETION_DELIVERY_ERRORS
|
|
)))
|
|
|| ($status === 'succeeded'
|
|
&& $fenceState !== 'not_required')
|
|
|| ($status === 'failed' && $effectResolution === 'applied'
|
|
&& $fenceState === 'not_required')
|
|
|| ($includeProjection && $kind === 'instance_upgrade'
|
|
&& $effectResolution === 'not_applied')
|
|
|| !self::effectMatchesOutcome($effectResolution, $outcome)
|
|
|| !self::validAbortInspection($value['abort_job'] ?? null)
|
|
|| !self::validCleanupInspection($value['cleanup'] ?? null)) {
|
|
self::invalidInspection();
|
|
}
|
|
|
|
$errorCode = null;
|
|
if (array_key_exists('error_code', $value)) {
|
|
$candidate = $value['error_code'];
|
|
if (!is_string($candidate)
|
|
|| preg_match('/^[a-z][a-z0-9_]{0,63}$/D',
|
|
$candidate) !== 1) {
|
|
self::invalidInspection();
|
|
}
|
|
$errorCode = $candidate;
|
|
}
|
|
|
|
$projection = null;
|
|
if (array_key_exists('effect_projection', $value)) {
|
|
$projection = self::effectProjection(
|
|
$value['effect_projection'],
|
|
$kind,
|
|
$effectResolution
|
|
);
|
|
}
|
|
$dto = [
|
|
'operation_id' => $operationId,
|
|
'action' => $identity['action'],
|
|
'resource_id' => $identity['resource_id'],
|
|
'request_fingerprint' => $identity['request_fingerprint'],
|
|
'journal_status' => $status,
|
|
'checked_at' => $checkedAt,
|
|
'outcome' => self::OUTCOMES[$outcome],
|
|
'effect_resolution' => $effectResolution,
|
|
'fence_state' => $fenceState,
|
|
'fence_resolution_required' => $fenceResolutionRequired,
|
|
'summary_code' => $summaryCode,
|
|
'summary' => self::SUMMARY_LABELS[$summaryCode],
|
|
];
|
|
if ($hasCompletionDeliveryState) {
|
|
$dto['completion_delivery_state'] =
|
|
$completionDeliveryState;
|
|
$dto['completion_delivery_error_code'] =
|
|
$completionDeliveryErrorCode;
|
|
$dto['completion_delivery_error'] =
|
|
self::COMPLETION_DELIVERY_ERRORS[
|
|
$completionDeliveryErrorCode
|
|
];
|
|
}
|
|
if ($errorCode !== null) {
|
|
$dto['error_code'] = $errorCode;
|
|
$dto['error'] = self::INSPECTION_ERROR_LABELS[$errorCode]
|
|
?? '宿主机操作失败(错误码已记录)';
|
|
}
|
|
if (($value['cleanup']['available'] ?? false) === true) {
|
|
$dto['cleanup'] = $value['cleanup'];
|
|
}
|
|
if ($includeProjection && $projection !== null) {
|
|
$dto['effect_projection'] = $projection;
|
|
}
|
|
return $dto;
|
|
}
|
|
|
|
private static function effectMatchesOutcome(
|
|
string $resolution,
|
|
string $outcome
|
|
): bool {
|
|
return match ($resolution) {
|
|
'applied' => $outcome === 'succeeded',
|
|
'not_applied' => $outcome === 'failed',
|
|
'partial', 'inconclusive' => $outcome === 'uncertain',
|
|
default => false,
|
|
};
|
|
}
|
|
|
|
private static function validAbortInspection(mixed $value): bool
|
|
{
|
|
if (!is_array($value) || array_is_list($value)
|
|
|| array_diff(array_keys($value), [
|
|
'available', 'kind', 'inspection_token',
|
|
]) !== []
|
|
|| !is_bool($value['available'] ?? null)) {
|
|
return false;
|
|
}
|
|
if ($value['available'] === false) {
|
|
return count($value) === 1;
|
|
}
|
|
return count($value) === 3
|
|
&& ($value['kind'] ?? null) === 'backup'
|
|
&& is_string($value['inspection_token'] ?? null)
|
|
&& preg_match('/^[a-f0-9]{64}$/D',
|
|
$value['inspection_token']) === 1;
|
|
}
|
|
|
|
private static function validCleanupInspection(mixed $value): bool
|
|
{
|
|
if (!is_array($value) || array_is_list($value)
|
|
|| array_diff(array_keys($value), [
|
|
'available', 'plan_id', 'steps',
|
|
]) !== []
|
|
|| !is_bool($value['available'] ?? null)) {
|
|
return false;
|
|
}
|
|
if ($value['available'] === false) {
|
|
return count($value) === 1;
|
|
}
|
|
$steps = $value['steps'] ?? null;
|
|
$allowedSteps = [
|
|
'destroy_domain', 'undefine_domain',
|
|
'delete_managed_volumes', 'delete_seed_volume',
|
|
'undefine_network_filter', 'delete_nvram',
|
|
'remove_traffic_state', 'delete_snapshot_volume',
|
|
'remove_snapshot_metadata', 'delete_previous_root',
|
|
'persist_cleanup_metadata',
|
|
];
|
|
return count($value) === 3
|
|
&& is_string($value['plan_id'] ?? null)
|
|
&& preg_match('/^[a-f0-9]{64}$/D', $value['plan_id']) === 1
|
|
&& is_array($steps) && array_is_list($steps)
|
|
&& $steps !== [] && count($steps) <= count($allowedSteps)
|
|
&& count(array_unique($steps, SORT_STRING)) === count($steps)
|
|
&& array_diff($steps, $allowedSteps) === [];
|
|
}
|
|
|
|
/** Return a validated projection rebuilt from a per-kind field schema. */
|
|
private static function effectProjection(
|
|
mixed $value,
|
|
string $kind,
|
|
string $resolution
|
|
): array {
|
|
if (!is_array($value) || $value === [] || array_is_list($value)) {
|
|
self::invalidInspection();
|
|
}
|
|
try {
|
|
$encoded = json_encode(
|
|
$value,
|
|
JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES
|
|
| JSON_UNESCAPED_UNICODE
|
|
);
|
|
} catch (\Throwable $error) {
|
|
self::invalidInspection();
|
|
}
|
|
$nodes = 0;
|
|
if (strlen($encoded) > self::MAX_PROJECTION_BYTES
|
|
|| !self::projectionWithinBounds($value, 1, $nodes,
|
|
in_array($kind, [
|
|
'template_import_replica', 'template_metadata_replica',
|
|
'rescue_catalog_validate_replica',
|
|
'rescue_catalog_import_replica',
|
|
'rescue',
|
|
], true))) {
|
|
self::invalidInspection();
|
|
}
|
|
$schema = self::projectionSchema($kind, $resolution);
|
|
if ($schema === null
|
|
|| array_diff(array_keys($value), array_keys($schema)) !== []) {
|
|
self::invalidInspection();
|
|
}
|
|
foreach ($value as $field => $item) {
|
|
if (!self::validProjectionValue($item, $schema[$field])) {
|
|
self::invalidInspection();
|
|
}
|
|
}
|
|
foreach (self::requiredProjectionFields($kind, $resolution)
|
|
as $required) {
|
|
if (!array_key_exists($required, $value)) {
|
|
self::invalidInspection();
|
|
}
|
|
}
|
|
if ($kind === 'instance_upgrade' && $resolution === 'applied'
|
|
&& (($value['power_state_restored'] ?? null) !== true
|
|
|| (($value['originally_running'] ?? null) === true
|
|
&& !in_array($value['status'] ?? null,
|
|
['running', 'blocked'], true))
|
|
|| (($value['originally_running'] ?? null) === false
|
|
&& !in_array($value['status'] ?? null,
|
|
['shutoff', 'crashed'], true)))) {
|
|
self::invalidInspection();
|
|
}
|
|
return $value;
|
|
}
|
|
|
|
private static function projectionSchema(
|
|
string $kind,
|
|
string $resolution
|
|
): ?array {
|
|
if ((CloudOperationRecoveryPolicy::isCreateLike($kind)
|
|
|| $kind === 'instance_migration_transfer')
|
|
&& $resolution === 'not_applied') {
|
|
return [
|
|
'creation_absent' => 'boolean',
|
|
'managed_artifacts_absent' => 'boolean',
|
|
];
|
|
}
|
|
if ($resolution !== 'applied') {
|
|
return null;
|
|
}
|
|
if (in_array($kind, [
|
|
'create', 'instance_settings', 'network_change', 'reinstall',
|
|
], true)) {
|
|
return self::VM_PROJECTION;
|
|
}
|
|
if ($kind === 'backup_restore') {
|
|
return array_merge(self::VM_PROJECTION, [
|
|
'backup_key' => 'string',
|
|
'sha256' => 'string',
|
|
]);
|
|
}
|
|
if ($kind === 'instance_migration_prepare') {
|
|
return [
|
|
'migration_id' => 'string',
|
|
'manifest_digest' => 'string',
|
|
'artifacts' => 'peer_migration_artifacts',
|
|
'source_stopped' => 'boolean',
|
|
'source_was_running' => 'boolean',
|
|
'source_autostart_enabled' => 'boolean',
|
|
'source_autostart_disabled' => 'boolean',
|
|
];
|
|
}
|
|
if ($kind === 'instance_migration_transfer') {
|
|
return array_merge(self::VM_PROJECTION, [
|
|
'migration_id' => 'string',
|
|
'manifest_digest' => 'string',
|
|
'artifacts' => 'peer_migration_artifacts',
|
|
'staged_powered_off' => 'boolean',
|
|
]);
|
|
}
|
|
if ($kind === 'instance_upgrade') {
|
|
return [
|
|
'uuid' => 'string',
|
|
'from_vcpu' => 'integer',
|
|
'target_vcpu' => 'integer',
|
|
'applied_vcpu' => 'integer',
|
|
'from_memory_mib' => 'integer',
|
|
'target_memory_mib' => 'integer',
|
|
'applied_memory_mib' => 'integer',
|
|
'from_disk_gib' => 'integer',
|
|
'target_disk_gib' => 'integer',
|
|
'applied_disk_gib' => 'integer',
|
|
'from_bandwidth_mbps' => 'integer',
|
|
'target_bandwidth_mbps' => 'integer',
|
|
'applied_bandwidth_mbps' => 'integer',
|
|
'status' => 'string',
|
|
'originally_running' => 'boolean',
|
|
'power_state_restored' => 'boolean',
|
|
'resources_resized' => 'boolean',
|
|
];
|
|
}
|
|
if (in_array($kind,
|
|
['delete', 'delete_force', 'expiry_delete',
|
|
'instance_migration_cleanup',
|
|
'instance_migration_target_clean'], true)) {
|
|
return self::DELETE_PROJECTION;
|
|
}
|
|
if (in_array($kind, [
|
|
'instance_migration_rollback', 'instance_migration_activate',
|
|
], true)) {
|
|
return [
|
|
'uuid' => 'string',
|
|
'state' => 'string',
|
|
'running' => 'boolean',
|
|
'autostart_enabled' => 'boolean',
|
|
];
|
|
}
|
|
if (in_array($kind, [
|
|
'action', 'service_suspend',
|
|
], true)) {
|
|
return ['uuid' => 'string', 'state' => 'string'];
|
|
}
|
|
if (in_array($kind,
|
|
['network_suspend', 'network_resume'], true)) {
|
|
return ['network_enabled' => 'boolean'];
|
|
}
|
|
if (in_array($kind,
|
|
['vpc_host_apply', 'vpc_host_remove'], true)) {
|
|
return [
|
|
'vpc_key' => 'string',
|
|
'desired_revision' => 'integer',
|
|
'desired_digest' => 'string',
|
|
'bridge' => 'string',
|
|
'applied' => 'boolean',
|
|
'removed' => 'boolean',
|
|
];
|
|
}
|
|
if (in_array($kind,
|
|
['vpc_port_attach', 'vpc_port_detach'], true)) {
|
|
return [
|
|
'uuid' => 'string',
|
|
'port_key' => 'string',
|
|
'vpc_key' => 'string',
|
|
'vpc_port' => 'vpc_port',
|
|
'status' => 'string',
|
|
'attached' => 'boolean',
|
|
'removed' => 'boolean',
|
|
];
|
|
}
|
|
if ($kind === 'snapshot_create') {
|
|
return self::SNAPSHOT_PROJECTION;
|
|
}
|
|
if ($kind === 'backup_create') {
|
|
return self::BACKUP_PROJECTION;
|
|
}
|
|
if ($kind === 'backup_workspace_cleanup') {
|
|
return [
|
|
'backup_key' => 'string',
|
|
'cleaned' => 'boolean',
|
|
'workspace_absent' => 'boolean',
|
|
'metadata_absent' => 'boolean',
|
|
];
|
|
}
|
|
if ($kind === 'backup_recover') {
|
|
return [
|
|
'backup_key' => 'string',
|
|
'sha256' => 'string',
|
|
'restored' => 'boolean',
|
|
'power_state' => 'string',
|
|
];
|
|
}
|
|
if (in_array($kind,
|
|
['snapshot_restore', 'snapshot_delete'], true)) {
|
|
return self::SNAPSHOT_MUTATION_PROJECTION;
|
|
}
|
|
if ($kind === 'firewall_apply') {
|
|
return [
|
|
'revision' => 'integer',
|
|
'default_in' => 'string',
|
|
'default_out' => 'string',
|
|
'policy_digest' => 'string',
|
|
'rules' => 'firewall_rules',
|
|
];
|
|
}
|
|
if ($kind === 'rescue') {
|
|
return [
|
|
'active' => 'boolean',
|
|
'staged' => 'boolean',
|
|
'pending_manual_restart' => 'boolean',
|
|
'template_id' => 'string',
|
|
'revision' => 'integer',
|
|
'generation_token' => 'string',
|
|
'network_suspended' => 'boolean',
|
|
];
|
|
}
|
|
if (in_array($kind, [
|
|
'template_import_replica', 'template_metadata_replica',
|
|
'rescue_catalog_validate_replica',
|
|
'rescue_catalog_import_replica',
|
|
'user_iso_import_replica',
|
|
], true)) {
|
|
return self::CATALOG_IMPORT_PROJECTION;
|
|
}
|
|
if (in_array($kind, [
|
|
'template_delete_replica',
|
|
'rescue_catalog_delete_replica',
|
|
'user_iso_delete_replica',
|
|
], true)) {
|
|
return self::CATALOG_DELETE_PROJECTION;
|
|
}
|
|
if ($kind === 'host_storage_configure') {
|
|
return ['configured' => 'boolean'];
|
|
}
|
|
return null;
|
|
}
|
|
|
|
private static function requiredProjectionFields(
|
|
string $kind,
|
|
string $resolution
|
|
): array {
|
|
if ((CloudOperationRecoveryPolicy::isCreateLike($kind)
|
|
|| $kind === 'instance_migration_transfer')
|
|
&& $resolution === 'not_applied') {
|
|
return ['creation_absent', 'managed_artifacts_absent'];
|
|
}
|
|
return match ($kind) {
|
|
'create' => ['uuid'],
|
|
'instance_settings' => ['uuid', 'settings_changed'],
|
|
'instance_upgrade' => [
|
|
'uuid', 'from_vcpu', 'target_vcpu', 'applied_vcpu',
|
|
'from_memory_mib', 'target_memory_mib',
|
|
'applied_memory_mib', 'from_disk_gib', 'target_disk_gib',
|
|
'applied_disk_gib', 'from_bandwidth_mbps',
|
|
'target_bandwidth_mbps', 'applied_bandwidth_mbps',
|
|
'status', 'originally_running', 'power_state_restored',
|
|
'resources_resized',
|
|
],
|
|
'network_change' => [
|
|
'uuid', 'network_changed', 'public_network',
|
|
],
|
|
'reinstall' => ['uuid', 'reinstall_committed'],
|
|
'delete', 'delete_force', 'expiry_delete',
|
|
'instance_migration_cleanup',
|
|
'instance_migration_target_clean' => [
|
|
'deleted', 'managed_artifacts_absent', 'forced',
|
|
],
|
|
'instance_migration_rollback', 'instance_migration_activate' =>
|
|
['uuid', 'state', 'running', 'autostart_enabled'],
|
|
'action', 'service_suspend' =>
|
|
['uuid', 'state'],
|
|
'network_suspend', 'network_resume' => ['network_enabled'],
|
|
'vpc_host_apply' => [
|
|
'vpc_key', 'desired_revision', 'desired_digest', 'bridge',
|
|
'applied',
|
|
],
|
|
'vpc_host_remove' => [
|
|
'vpc_key', 'desired_revision', 'desired_digest', 'removed',
|
|
],
|
|
'vpc_port_attach' => [
|
|
'uuid', 'port_key', 'vpc_key', 'vpc_port', 'status',
|
|
'attached',
|
|
],
|
|
'vpc_port_detach' => [
|
|
'uuid', 'port_key', 'vpc_key', 'status', 'removed',
|
|
],
|
|
'snapshot_create' => [
|
|
'id', 'snapshot_key', 'created_at', 'state', 'consistency',
|
|
],
|
|
'backup_create' => [
|
|
'backup_key', 'size_bytes', 'virtual_size_bytes', 'sha256',
|
|
'consistency',
|
|
],
|
|
'backup_workspace_cleanup' => [
|
|
'backup_key', 'cleaned', 'workspace_absent',
|
|
'metadata_absent',
|
|
],
|
|
'backup_recover' => [
|
|
'backup_key', 'sha256', 'restored', 'power_state',
|
|
],
|
|
'backup_restore' => [
|
|
'uuid', 'name', 'vcpu', 'memory_mib', 'disk_gib',
|
|
'bandwidth_mbps', 'public_network', 'machine', 'firmware',
|
|
'secure_boot', 'backup_key', 'sha256',
|
|
],
|
|
'instance_migration_prepare' => [
|
|
'migration_id', 'manifest_digest', 'artifacts',
|
|
'source_stopped', 'source_was_running',
|
|
'source_autostart_enabled', 'source_autostart_disabled',
|
|
],
|
|
'instance_migration_transfer' => [
|
|
'uuid', 'status', 'vcpu', 'memory_mib', 'disk_gib',
|
|
'bandwidth_mbps', 'public_network', 'migration_id',
|
|
'manifest_digest', 'artifacts', 'staged_powered_off',
|
|
],
|
|
'snapshot_restore' => ['snapshot_key', 'restored'],
|
|
'snapshot_delete' => ['snapshot_key', 'deleted'],
|
|
'firewall_apply' => [
|
|
'revision', 'default_in', 'default_out', 'rules',
|
|
],
|
|
'rescue' => ['active', 'staged', 'network_suspended'],
|
|
'template_import_replica',
|
|
'user_iso_import_replica' => ['id', 'created_at'],
|
|
'rescue_catalog_validate_replica',
|
|
'rescue_catalog_import_replica' => [
|
|
'id', 'name', 'size_bytes', 'sha256', 'revision',
|
|
'generation_token', 'created_at',
|
|
],
|
|
'template_metadata_replica' => [
|
|
'id', 'name', 'revision', 'generation_token',
|
|
],
|
|
'template_delete_replica',
|
|
'rescue_catalog_delete_replica',
|
|
'user_iso_delete_replica' => [
|
|
'resource_type', 'id', 'deleted',
|
|
'managed_image_absent', 'managed_metadata_absent',
|
|
'managed_artifacts_absent',
|
|
],
|
|
'host_storage_configure' => ['configured'],
|
|
default => [],
|
|
};
|
|
}
|
|
|
|
private static function validProjectionValue(
|
|
mixed $value,
|
|
string $type
|
|
): bool {
|
|
return match ($type) {
|
|
'boolean' => is_bool($value),
|
|
'integer' => is_int($value) && $value >= 0,
|
|
'string' => is_string($value) && strlen($value) <= 4096
|
|
&& preg_match('/[\x00-\x1f\x7f]/', $value) !== 1,
|
|
'string_list' => self::validStringList($value),
|
|
'firewall_rules' => self::validFirewallRules($value),
|
|
'vpc_port' => self::validVpcPortProjection($value),
|
|
'peer_migration_artifacts' =>
|
|
self::validPeerMigrationArtifacts($value),
|
|
'public_network' => self::validPublicNetwork($value),
|
|
default => false,
|
|
};
|
|
}
|
|
|
|
private static function validPublicNetwork(mixed $value): bool
|
|
{
|
|
try {
|
|
return CloudPublicNetworkSnapshot::fromArray($value)->toArray()
|
|
=== $value;
|
|
} catch (\Throwable) {
|
|
return false;
|
|
}
|
|
}
|
|
|
|
private static function validStringList(mixed $value): bool
|
|
{
|
|
if (!is_array($value) || !array_is_list($value)
|
|
|| count($value) > 64) {
|
|
return false;
|
|
}
|
|
foreach ($value as $item) {
|
|
if (!self::validProjectionValue($item, 'string')) {
|
|
return false;
|
|
}
|
|
}
|
|
return true;
|
|
}
|
|
|
|
private static function validVpcPortProjection(mixed $value): bool
|
|
{
|
|
if (!is_array($value) || array_is_list($value)) {
|
|
return false;
|
|
}
|
|
$required = [
|
|
'port_key', 'vpc_key', 'private_ip', 'cidr', 'gateway', 'mac',
|
|
'bridge', 'host_revision', 'host_digest',
|
|
];
|
|
$actual = array_keys($value);
|
|
sort($required, SORT_STRING);
|
|
sort($actual, SORT_STRING);
|
|
if ($actual !== $required
|
|
|| preg_match('/^[a-f0-9]{32}$/D',
|
|
(string) ($value['port_key'] ?? '')) !== 1
|
|
|| preg_match('/^[a-f0-9]{32}$/D',
|
|
(string) ($value['vpc_key'] ?? '')) !== 1
|
|
|| filter_var($value['private_ip'] ?? null,
|
|
FILTER_VALIDATE_IP, FILTER_FLAG_IPV4) === false
|
|
|| preg_match('/^(?:\d{1,3}\.){3}\d{1,3}\/\d{1,2}$/D',
|
|
(string) ($value['cidr'] ?? '')) !== 1
|
|
|| filter_var($value['gateway'] ?? null,
|
|
FILTER_VALIDATE_IP, FILTER_FLAG_IPV4) === false
|
|
|| preg_match('/^[0-9a-f]{2}(?::[0-9a-f]{2}){5}$/D',
|
|
(string) ($value['mac'] ?? '')) !== 1
|
|
|| preg_match('/^vc[a-f0-9]{12}$/D',
|
|
(string) ($value['bridge'] ?? '')) !== 1
|
|
|| !is_int($value['host_revision'] ?? null)
|
|
|| (int) $value['host_revision'] < 1
|
|
|| preg_match('/^[a-f0-9]{64}$/D',
|
|
(string) ($value['host_digest'] ?? '')) !== 1) {
|
|
return false;
|
|
}
|
|
return true;
|
|
}
|
|
|
|
private static function validPeerMigrationArtifacts(mixed $value): bool
|
|
{
|
|
if (!is_array($value) || !array_is_list($value)
|
|
|| count($value) < 1
|
|
|| count($value) > CloudSnapshotService::MAX_PER_INSTANCE + 1) {
|
|
return false;
|
|
}
|
|
$seen = [];
|
|
$roots = 0;
|
|
foreach ($value as $item) {
|
|
if (!is_array($item) || array_is_list($item)) {
|
|
return false;
|
|
}
|
|
$role = $item['role'] ?? null;
|
|
$required = [
|
|
'id', 'role', 'consistency', 'size_bytes',
|
|
'virtual_size_bytes', 'sha256', 'created_at',
|
|
];
|
|
if ($role === 'snapshot') {
|
|
$required[] = 'snapshot_key';
|
|
$required[] = 'name';
|
|
} elseif ($role === 'root') {
|
|
$roots++;
|
|
} else {
|
|
return false;
|
|
}
|
|
sort($required);
|
|
$actual = array_keys($item);
|
|
sort($actual);
|
|
$id = $item['id'] ?? null;
|
|
if ($actual !== $required
|
|
|| !is_string($id)
|
|
|| preg_match('/^[a-f0-9]{32}$/D', $id) !== 1
|
|
|| isset($seen[$id])
|
|
|| !in_array($item['consistency'] ?? null,
|
|
['filesystem', 'crash'], true)
|
|
|| !is_int($item['size_bytes'] ?? null)
|
|
|| (int) $item['size_bytes'] < 1
|
|
|| !is_int($item['virtual_size_bytes'] ?? null)
|
|
|| (int) $item['virtual_size_bytes'] < 1
|
|
|| !is_string($item['sha256'] ?? null)
|
|
|| preg_match('/^[a-f0-9]{64}$/D',
|
|
(string) $item['sha256']) !== 1
|
|
|| !is_int($item['created_at'] ?? null)
|
|
|| (int) $item['created_at'] < 1) {
|
|
return false;
|
|
}
|
|
if ($role === 'snapshot'
|
|
&& (!is_string($item['snapshot_key'] ?? null)
|
|
|| preg_match('/^[a-f0-9]{32}$/D',
|
|
(string) $item['snapshot_key']) !== 1
|
|
|| !self::validProjectionValue(
|
|
$item['name'] ?? null, 'string'
|
|
))) {
|
|
return false;
|
|
}
|
|
$seen[$id] = true;
|
|
}
|
|
return $roots === 1;
|
|
}
|
|
|
|
private static function validFirewallRules(mixed $value): bool
|
|
{
|
|
if (!is_array($value) || !array_is_list($value)
|
|
|| count($value) > 512) {
|
|
return false;
|
|
}
|
|
$schema = [
|
|
'direction' => 'string', 'action' => 'string',
|
|
'protocol' => 'string', 'cidr' => 'string',
|
|
'port_from' => 'integer', 'port_to' => 'integer',
|
|
];
|
|
foreach ($value as $rule) {
|
|
if (!is_array($rule) || array_is_list($rule)
|
|
|| array_diff(array_keys($rule), array_keys($schema)) !== []) {
|
|
return false;
|
|
}
|
|
foreach (['direction', 'action', 'protocol', 'cidr']
|
|
as $required) {
|
|
if (!array_key_exists($required, $rule)) {
|
|
return false;
|
|
}
|
|
}
|
|
foreach ($rule as $field => $item) {
|
|
if (!self::validProjectionValue($item, $schema[$field])) {
|
|
return false;
|
|
}
|
|
}
|
|
}
|
|
return true;
|
|
}
|
|
|
|
private static function projectionWithinBounds(
|
|
mixed $value,
|
|
int $depth,
|
|
int &$nodes,
|
|
bool $allowTemplateGeneration = false
|
|
): bool {
|
|
$nodes++;
|
|
if ($nodes > self::MAX_PROJECTION_NODES
|
|
|| $depth > self::MAX_PROJECTION_DEPTH) {
|
|
return false;
|
|
}
|
|
if (!is_array($value)) {
|
|
return true;
|
|
}
|
|
foreach ($value as $key => $item) {
|
|
// This is a public template revision identifier, not a bearer
|
|
// credential. Only its exact top-level, bounded form is exempt;
|
|
// recursive values and every other token remain forbidden.
|
|
$templateGeneration = $allowTemplateGeneration && $depth === 1
|
|
&& $key === 'generation_token' && is_string($item)
|
|
&& preg_match('/^[a-f0-9]{32}$/D', $item) === 1;
|
|
if (is_string($key) && self::sensitiveProjectionKey($key)
|
|
&& !$templateGeneration) {
|
|
return false;
|
|
}
|
|
if (!self::projectionWithinBounds(
|
|
$item,
|
|
$depth + 1,
|
|
$nodes
|
|
)) {
|
|
return false;
|
|
}
|
|
}
|
|
return true;
|
|
}
|
|
|
|
private static function sensitiveProjectionKey(string $key): bool
|
|
{
|
|
$key = strtolower(str_replace('-', '_', $key));
|
|
return in_array($key, [
|
|
'password', 'root_password', 'credential', 'sealed_credential',
|
|
'secret', 'api_secret', 'api_key', 'private_key',
|
|
'private_keys', 'ssh_private_key', 'ssh_public_key',
|
|
'guest_ssh_public_key', 'public_key', 'public_keys', 'seed',
|
|
'seed_data', 'user_data', 'userdata', 'startup_script',
|
|
'token', 'access_token', 'auth_token', 'xml', 'domain_xml',
|
|
'raw_error', 'path', 'host_path', 'loader', 'nvram_path',
|
|
], true) || preg_match(
|
|
'/(?:^|_)(?:password|credential|secret|private_key|public_key|seed|user_data|startup_script|token|xml|path)$/D',
|
|
$key
|
|
) === 1 || str_starts_with($key, 'sealed_');
|
|
}
|
|
|
|
private static function invalidInspection(): void
|
|
{
|
|
throw new CloudAgentError(
|
|
'宿主机 Agent 只读检查响应格式无效', 502,
|
|
'agent_invalid_inspection', false, 200
|
|
);
|
|
}
|
|
|
|
/**
|
|
* Read-before-write and single-write Agent fence protocol. A null actor
|
|
* selects the exact-operation system recovery capability; a human actor
|
|
* remains on the ordinary signed request path.
|
|
*/
|
|
private static function resolveFenceOnAgent(
|
|
$client,
|
|
CloudJob $job,
|
|
string $resolution,
|
|
string $reason,
|
|
int $checkedAt,
|
|
?array $actor,
|
|
array $identity
|
|
): array {
|
|
$operationId = (string) $job->operation_id;
|
|
$system = $actor === null;
|
|
$call = static function (
|
|
string $method,
|
|
string $path,
|
|
?array $payload = null
|
|
) use ($client, $operationId, $actor, $system): array {
|
|
if ($system) {
|
|
return $client->systemRecoveryRequest(
|
|
$method,
|
|
$path,
|
|
$operationId,
|
|
$payload
|
|
);
|
|
}
|
|
return $client->request($method, $path, $actor, $payload);
|
|
};
|
|
$reasonDigest = hash('sha256', $reason);
|
|
$path = self::agentPath($job, '/fence-resolution');
|
|
|
|
// Recover an Agent commit left behind by a prior PHP failure without
|
|
// issuing another write request.
|
|
try {
|
|
$existing = self::fenceResolution(
|
|
$call('GET', $path),
|
|
$operationId,
|
|
$system
|
|
);
|
|
if (!hash_equals($resolution, $existing['resolution'])
|
|
|| !hash_equals($reasonDigest, $existing['reason_sha256'])) {
|
|
throw new CloudError(
|
|
'宿主机栅栏已有不同的审计结论,不能覆盖', 409
|
|
);
|
|
}
|
|
return $existing;
|
|
} catch (CloudAgentError $error) {
|
|
if ($error->agentHttpStatus !== 404) {
|
|
throw new CloudAgentError(
|
|
'无法确认宿主机资源栅栏状态', 503,
|
|
'fence_resolution_read_failed', false,
|
|
$error->agentHttpStatus
|
|
);
|
|
}
|
|
}
|
|
|
|
$request = [
|
|
'resolution_id' => bin2hex(random_bytes(16)),
|
|
'resolution' => $resolution,
|
|
'inspection_checked_at' => $checkedAt,
|
|
'reason_sha256' => $reasonDigest,
|
|
'action' => $identity['action'] ?? null,
|
|
'resource_id' => $identity['resource_id'] ?? null,
|
|
'request_fingerprint' =>
|
|
$identity['request_fingerprint'] ?? null,
|
|
];
|
|
try {
|
|
return self::matchingFenceResolution(
|
|
$call(
|
|
'POST',
|
|
self::agentPath($job, '/resolve-fence'),
|
|
$request
|
|
),
|
|
$operationId,
|
|
$request,
|
|
$system
|
|
);
|
|
} catch (\Throwable $writeError) {
|
|
// Never retry the metadata POST. A single GET can prove that the
|
|
// first write committed before its response was lost.
|
|
try {
|
|
return self::matchingFenceResolution(
|
|
$call('GET', $path),
|
|
$operationId,
|
|
$request,
|
|
$system
|
|
);
|
|
} catch (\Throwable $readError) {
|
|
if ($writeError instanceof CloudError) {
|
|
throw $writeError;
|
|
}
|
|
throw new CloudError('宿主机资源栅栏未确认解除', 503);
|
|
}
|
|
}
|
|
}
|
|
|
|
private static function matchingFenceResolution(
|
|
array $value,
|
|
string $operationId,
|
|
array $expected,
|
|
bool $allowSystemActor = false
|
|
): array {
|
|
$resolution = self::fenceResolution(
|
|
$value,
|
|
$operationId,
|
|
$allowSystemActor
|
|
);
|
|
foreach (['resolution_id', 'resolution', 'reason_sha256'] as $key) {
|
|
if (!hash_equals((string) $expected[$key],
|
|
(string) $resolution[$key])) {
|
|
throw new CloudAgentError(
|
|
'宿主机资源栅栏回执与请求不匹配', 502,
|
|
'agent_invalid_fence_resolution', false, 200
|
|
);
|
|
}
|
|
}
|
|
if ((int) $resolution['inspection_checked_at']
|
|
!== (int) $expected['inspection_checked_at']) {
|
|
throw new CloudAgentError(
|
|
'宿主机资源栅栏回执与检查时间不匹配', 502,
|
|
'agent_invalid_fence_resolution', false, 200
|
|
);
|
|
}
|
|
return $resolution;
|
|
}
|
|
|
|
/** Validate the bounded Agent audit DTO before it influences panel state. */
|
|
private static function fenceResolution(
|
|
array $value,
|
|
string $operationId,
|
|
bool $allowSystemActor = false
|
|
): array {
|
|
$allowed = ['operation_id', 'resolution_id', 'resolution',
|
|
'inspection_checked_at', 'reason_sha256', 'resolved_at',
|
|
'resolved_by', 'replayed'];
|
|
if (array_is_list($value)
|
|
|| array_diff(array_keys($value), $allowed) !== []
|
|
|| ($value['operation_id'] ?? null) !== $operationId
|
|
|| !is_string($value['resolution_id'] ?? null)
|
|
|| preg_match('/^[a-f0-9]{32}$/D',
|
|
$value['resolution_id']) !== 1
|
|
|| !in_array($value['resolution'] ?? null,
|
|
['applied', 'not_applied'], true)
|
|
|| !is_int($value['inspection_checked_at'] ?? null)
|
|
|| (int) $value['inspection_checked_at'] < 1
|
|
|| !is_string($value['reason_sha256'] ?? null)
|
|
|| preg_match('/^[a-f0-9]{64}$/D',
|
|
$value['reason_sha256']) !== 1
|
|
|| !is_int($value['resolved_at'] ?? null)
|
|
|| (int) $value['resolved_at'] < 1
|
|
|| !self::validResolvedBy(
|
|
$value['resolved_by'] ?? null,
|
|
$allowSystemActor
|
|
)
|
|
|| (array_key_exists('replayed', $value)
|
|
&& !is_bool($value['replayed']))) {
|
|
throw new CloudAgentError(
|
|
'宿主机 Agent 栅栏回执格式无效', 502,
|
|
'agent_invalid_fence_resolution', false, 200
|
|
);
|
|
}
|
|
return [
|
|
'operation_id' => $operationId,
|
|
'resolution_id' => $value['resolution_id'],
|
|
'resolution' => $value['resolution'],
|
|
'inspection_checked_at' => $value['inspection_checked_at'],
|
|
'reason_sha256' => $value['reason_sha256'],
|
|
'resolved_at' => $value['resolved_at'],
|
|
'resolved_by' => (string) $value['resolved_by'],
|
|
];
|
|
}
|
|
|
|
private static function validResolvedBy(
|
|
mixed $value,
|
|
bool $allowSystemActor
|
|
): bool {
|
|
if ($allowSystemActor) {
|
|
// The automatic path is a distinct audit identity. It must not
|
|
// accept a positive human ID or the string form "0".
|
|
return is_int($value) && $value === 0;
|
|
}
|
|
if (is_int($value)) {
|
|
return $value >= 1;
|
|
}
|
|
// Positive human IDs retain the legacy string compatibility. The
|
|
// reserved system actor is accepted only as the numeric JSON value 0.
|
|
return is_string($value)
|
|
&& preg_match('/^[1-9][0-9]{0,18}$/D', $value) === 1;
|
|
}
|
|
}
|