You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
virt/web/app/Services/Cloud/CloudVpcHostRetirementServi...

348 lines
15 KiB
PHP

<?php
namespace App\Services\Cloud;
use App\Models\CloudHost;
use App\Models\CloudHostAgent;
use App\Models\CloudHostResourcePoolMembership;
use App\Models\CloudInstance;
use App\Models\CloudInstanceMigration;
use App\Models\CloudJob;
use App\Models\CloudResourceReservation;
use App\Models\CloudVpc;
use App\Models\CloudVpcHostState;
use App\Models\CloudVpcPort;
/**
* Resolve one unreachable host's exact VPC-removal fence after the physical
* compute node has been permanently retired from every panel resource ledger.
* This is not an Agent retry and never authorizes retiring a host that still
* owns an instance, reservation, migration, or VPC port.
*/
final class CloudVpcHostRetirementService
{
private const MAX_REASON_LENGTH = 500;
private const MIN_REASON_LENGTH = 20;
private const MAX_SUPERSESSION_DEPTH = 32;
public static function retire(
array $actor,
int $jobId,
string $reason
): array {
self::authorize($actor);
$reason = self::reason($reason);
$reference = CloudJob::query()->where('id', $jobId)->first();
if ($reference === null) {
throw new CloudError('任务不存在', 404);
}
self::assertReference($reference);
return (new CloudJob())->getConnection()->transaction(
static function () use ($actor, $reference, $reason): array {
$hostInstanceIds = CloudInstance::query()
->where('host_id', (int) $reference->host_id)
->where('status', '<>', 'deleted')->pluck('id')
->map(static fn ($id): int => (int) $id)->all();
$instanceIds = array_values(array_unique(array_merge(
$hostInstanceIds,
[(int) $reference->instance_id]
)));
$instances = CloudVpcLockOrder::instances($instanceIds);
$instance = $instances->get((int) $reference->instance_id);
$vpc = CloudVpcLockOrder::vpcById(
(int) $reference->vpc_id
);
$port = CloudVpcLockOrder::portById(
(int) $reference->vpc_port_id
);
$state = CloudVpcHostState::query()
->where('vpc_id', (int) $reference->vpc_id)
->where('host_id', (int) $reference->host_id)
->lockForUpdate()->first();
if ($instance === null || $vpc === null || $port === null
|| $state === null) {
throw new CloudError('VPC 宿主机退役账本不完整', 409);
}
$host = CloudHost::query()
->where('id', (int) $reference->host_id)
->lockForUpdate()->first();
if ($host === null) {
throw new CloudError('宿主机不存在', 404);
}
self::assertHostEmptyLocked($host);
// Coordination rows are last in the canonical VPC lock order.
// The complete root catalog is acquired in ascending id order
// so a retirement cannot invert a concurrent repair lock.
$chain = self::repairChainLocked($reference);
$job = $chain[0];
self::assertCurrent($job, $instance, $vpc, $port, $state);
$now = time();
$auditReason = substr(
'host_permanently_retired:' . $reason,
0,
self::MAX_REASON_LENGTH
);
foreach ($chain as $generation) {
$generation->effect_resolution = 'applied';
$generation->effect_resolved_at = $now;
$generation->effect_resolved_by =
(int) ($actor['user_id'] ?? 0);
$generation->effect_resolution_reason = $auditReason;
$generation->available_at = 0;
$generation->read_at = null;
$generation->updated_at = $now;
$generation->save();
}
$state->delete();
CloudVpcPortService::finishRetiredHostRemovalLocked(
$job, $vpc, $port, $instance
);
return [
'retired' => true,
'task_id' => (string) $job->id,
'host_id' => (string) $host->id,
'vpc_id' => (string) $vpc->id,
'resolved_generations' => count($chain),
];
},
3
);
}
private static function authorize(array $actor): void
{
if (!($actor['is_admin'] ?? false)
|| !($actor['is_super_admin'] ?? false)
|| !($actor['can_manage'] ?? false)
|| (int) ($actor['user_id'] ?? 0) < 1) {
throw new CloudError(
'只有超级管理员可以确认宿主机永久退役', 403
);
}
CloudAccess::manage($actor);
}
private static function reason(string $reason): string
{
$reason = trim($reason);
$length = mb_strlen($reason);
if ($length < self::MIN_REASON_LENGTH
|| $length > self::MAX_REASON_LENGTH) {
throw new CloudError('退役依据需为 20 至 500 个字符');
}
return $reason;
}
private static function assertReference(CloudJob $job): void
{
if ((string) $job->kind !== 'vpc_host_remove'
|| (int) ($job->host_id ?? 0) < 1
|| (int) ($job->instance_id ?? 0) < 1
|| (int) ($job->vpc_id ?? 0) < 1
|| (int) ($job->vpc_port_id ?? 0) < 1) {
throw new CloudError('该任务不是可退役的 VPC 宿主机清理', 409);
}
}
private static function assertCurrent(
CloudJob $job,
CloudInstance $instance,
CloudVpc $vpc,
CloudVpcPort $port,
CloudVpcHostState $state
): void {
self::assertReference($job);
if ((string) $job->execution_mode !== 'agent'
|| (string) $job->phase !== 'terminal'
|| (string) $job->outcome !== 'failed'
|| (string) $job->status !== 'failed'
|| !(bool) $job->effect_may_have_applied
|| (bool) $job->safe_to_resubmit
|| (string) $job->effect_resolution !== 'unknown'
|| !CloudDispatchLedger::isDispatched($job)
|| (int) ($job->dispatched_at ?? 0) < 1
|| CloudJobService::hasTerminalRecoveryStateConflict($job)
|| (bool) $job->finalization_incident
|| CloudDispatchLedger::isInvalid($job)
|| CloudDispatchLedger::hasRepairIncident($job)
|| (int) $job->instance_id !== (int) $instance->id
|| (int) $job->vpc_id !== (int) $vpc->id
|| (int) $job->vpc_port_id !== (int) $port->id
|| (int) $job->host_id !== (int) $state->host_id
|| (int) $port->vpc_id !== (int) $vpc->id
|| !(bool) $port->active_claim
|| (string) $port->state !== 'release_pending'
|| !hash_equals(
(string) $state->last_operation_id,
(string) $job->operation_id
)
|| !in_array((string) $state->sync_state,
['failed', 'unknown'], true)) {
throw new CloudError('VPC 宿主机清理代次已变化或不可退役', 409);
}
}
private static function assertHostEmptyLocked(
CloudHost $host
): void {
$hostId = (int) $host->id;
if ((bool) $host->enabled) {
throw new CloudError('请先停用宿主机,再确认永久退役', 409);
}
$agent = CloudHostAgent::query()->where('host_id', $hostId)
->lockForUpdate()->first();
if ($agent !== null && (bool) $agent->enabled) {
throw new CloudError('请先停用该宿主机的 Agent 配置', 409);
}
$memberships = CloudHostResourcePoolMembership::query()
->where('host_id', $hostId)->orderBy('id')
->lockForUpdate()->get();
if ($memberships->contains(static fn ($row): bool =>
(string) $row->scheduler_state !== 'disabled')) {
throw new CloudError('请先停用宿主机的全部资源池调度', 409);
}
// Placement serializes on the host row. This second current/locking
// read therefore catches a placement that committed after the
// discovery read but before this transaction acquired the host.
$currentInstances = CloudInstance::query()
->where('host_id', $hostId)
->where('status', '<>', 'deleted')->orderBy('id')
->lockForUpdate()->get();
if (!$currentInstances->isEmpty()) {
throw new CloudError('宿主机仍有未删除实例,不能确认永久退役', 409);
}
$reservations = CloudResourceReservation::query()
->where('host_id', $hostId)
->whereIn('state', ['held', 'committed', 'uncertain'])
->orderBy('id')->lockForUpdate()->get();
if (!$reservations->isEmpty()) {
throw new CloudError('宿主机仍占用计算资源,不能确认永久退役', 409);
}
$migrations = CloudInstanceMigration::query()
->where(static function ($query) use ($hostId): void {
$query->where('source_host_id', $hostId)
->orWhere('target_host_id', $hostId);
})->orderBy('id')->lockForUpdate()->get();
if ($migrations->contains(static fn ($row): bool =>
CloudInstanceMigrationWorkflow::blocksNewMigration(
(string) $row->state
))) {
throw new CloudError('宿主机仍有关联维护迁移,不能确认永久退役', 409);
}
$ports = CloudVpcPort::query()
->leftJoin(
'cloud_instances as retirement_instance',
'retirement_instance.id', '=', 'cloud_vpc_ports.instance_id'
)
->where('cloud_vpc_ports.active_claim', 1)
->where('cloud_vpc_ports.state', '<>', 'release_pending')
->where(static function ($query) use ($hostId): void {
$query->where(
'cloud_vpc_ports.migration_host_id', $hostId
)->orWhere('retirement_instance.host_id', $hostId);
})
->orderBy('cloud_vpc_ports.id')
->lockForUpdate()->get(['cloud_vpc_ports.id']);
if (!$ports->isEmpty()) {
throw new CloudError('宿主机仍承载 VPC 网卡,不能确认永久退役', 409);
}
}
/** @return list<CloudJob> current generation first */
private static function repairChainLocked(CloudJob $reference): array
{
$chain = [];
$seen = [];
$rootId = (int) ($reference->vpc_repair_root_job_id
?? $reference->id);
if ($rootId < 1) {
throw new CloudError('VPC 宿主机清理继任链缺少根任务', 409);
}
$catalog = CloudJob::query()
->where(static function ($query) use ($rootId): void {
$query->where('id', $rootId)
->orWhere('vpc_repair_root_job_id', $rootId);
})->orderBy('id')->lockForUpdate()->get()->keyBy('id');
$current = $catalog->get((int) $reference->id);
if (!$current instanceof CloudJob) {
throw new CloudError('VPC 宿主机清理当前代次已丢失', 409);
}
for ($depth = 0; $depth < self::MAX_SUPERSESSION_DEPTH; $depth++) {
$id = (int) $current->id;
if ($id < 1 || isset($seen[$id])) {
throw new CloudError('VPC 宿主机清理继任链包含循环', 409);
}
$seen[$id] = true;
if ((string) $current->kind !== 'vpc_host_remove'
|| (int) $current->host_id < 1
|| (string) $current->phase !== 'terminal'
|| (string) $current->outcome !== 'failed'
|| !(bool) $current->effect_may_have_applied
|| (bool) $current->safe_to_resubmit
|| (string) $current->effect_resolution !== 'unknown'
|| !CloudDispatchLedger::isDispatched($current)
|| (int) ($current->dispatched_at ?? 0) < 1
|| CloudJobService::hasTerminalRecoveryStateConflict(
$current
)
|| (bool) $current->finalization_incident
|| CloudDispatchLedger::isInvalid($current)
|| CloudDispatchLedger::hasRepairIncident($current)
|| (int) ($current->vpc_repair_root_job_id
?? $current->id) !== $rootId) {
throw new CloudError('VPC 宿主机清理继任链已变化', 409);
}
if ($chain !== []) {
$tip = $chain[0];
if ((int) $current->host_id !== (int) $tip->host_id
|| (int) $current->instance_id
!== (int) $tip->instance_id
|| (int) $current->vpc_id !== (int) $tip->vpc_id
|| (int) $current->vpc_port_id
!== (int) $tip->vpc_port_id) {
throw new CloudError('VPC 宿主机清理继任链资源不一致', 409);
}
}
$chain[] = $current;
$priorId = (int) ($current->supersedes_job_id ?? 0);
if ($priorId < 1) {
if ((int) $current->id !== $rootId) {
throw new CloudError(
'VPC 宿主机清理根任务身份不一致', 409
);
}
foreach ($catalog as $candidate) {
if ((int) $candidate->instance_id
!== (int) $reference->instance_id
|| (int) $candidate->vpc_id
!== (int) $reference->vpc_id
|| (int) $candidate->vpc_port_id
!== (int) $reference->vpc_port_id
|| (int) $candidate->host_id
!== (int) $reference->host_id
|| (string) $candidate->kind !== 'vpc_host_remove'
|| (string) $candidate->phase !== 'terminal'
|| ((string) $candidate->effect_resolution
=== 'unknown'
&& !isset($seen[(int) $candidate->id]))) {
throw new CloudError(
'VPC 宿主机清理根链仍有其他活动代次', 409
);
}
}
return $chain;
}
$prior = $catalog->get($priorId);
if ($prior === null) {
throw new CloudError('VPC 宿主机清理前序任务已丢失', 409);
}
$current = $prior;
}
throw new CloudError('VPC 宿主机清理继任链超过安全上限', 409);
}
}