You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
348 lines
15 KiB
PHP
348 lines
15 KiB
PHP
<?php
|
|
|
|
namespace App\Services\Cloud;
|
|
|
|
use App\Models\CloudHost;
|
|
use App\Models\CloudHostAgent;
|
|
use App\Models\CloudHostResourcePoolMembership;
|
|
use App\Models\CloudInstance;
|
|
use App\Models\CloudInstanceMigration;
|
|
use App\Models\CloudJob;
|
|
use App\Models\CloudResourceReservation;
|
|
use App\Models\CloudVpc;
|
|
use App\Models\CloudVpcHostState;
|
|
use App\Models\CloudVpcPort;
|
|
|
|
/**
|
|
* Resolve one unreachable host's exact VPC-removal fence after the physical
|
|
* compute node has been permanently retired from every panel resource ledger.
|
|
* This is not an Agent retry and never authorizes retiring a host that still
|
|
* owns an instance, reservation, migration, or VPC port.
|
|
*/
|
|
final class CloudVpcHostRetirementService
|
|
{
|
|
private const MAX_REASON_LENGTH = 500;
|
|
private const MIN_REASON_LENGTH = 20;
|
|
private const MAX_SUPERSESSION_DEPTH = 32;
|
|
|
|
public static function retire(
|
|
array $actor,
|
|
int $jobId,
|
|
string $reason
|
|
): array {
|
|
self::authorize($actor);
|
|
$reason = self::reason($reason);
|
|
$reference = CloudJob::query()->where('id', $jobId)->first();
|
|
if ($reference === null) {
|
|
throw new CloudError('任务不存在', 404);
|
|
}
|
|
self::assertReference($reference);
|
|
|
|
return (new CloudJob())->getConnection()->transaction(
|
|
static function () use ($actor, $reference, $reason): array {
|
|
$hostInstanceIds = CloudInstance::query()
|
|
->where('host_id', (int) $reference->host_id)
|
|
->where('status', '<>', 'deleted')->pluck('id')
|
|
->map(static fn ($id): int => (int) $id)->all();
|
|
$instanceIds = array_values(array_unique(array_merge(
|
|
$hostInstanceIds,
|
|
[(int) $reference->instance_id]
|
|
)));
|
|
$instances = CloudVpcLockOrder::instances($instanceIds);
|
|
$instance = $instances->get((int) $reference->instance_id);
|
|
$vpc = CloudVpcLockOrder::vpcById(
|
|
(int) $reference->vpc_id
|
|
);
|
|
$port = CloudVpcLockOrder::portById(
|
|
(int) $reference->vpc_port_id
|
|
);
|
|
$state = CloudVpcHostState::query()
|
|
->where('vpc_id', (int) $reference->vpc_id)
|
|
->where('host_id', (int) $reference->host_id)
|
|
->lockForUpdate()->first();
|
|
if ($instance === null || $vpc === null || $port === null
|
|
|| $state === null) {
|
|
throw new CloudError('VPC 宿主机退役账本不完整', 409);
|
|
}
|
|
|
|
$host = CloudHost::query()
|
|
->where('id', (int) $reference->host_id)
|
|
->lockForUpdate()->first();
|
|
if ($host === null) {
|
|
throw new CloudError('宿主机不存在', 404);
|
|
}
|
|
self::assertHostEmptyLocked($host);
|
|
|
|
// Coordination rows are last in the canonical VPC lock order.
|
|
// The complete root catalog is acquired in ascending id order
|
|
// so a retirement cannot invert a concurrent repair lock.
|
|
$chain = self::repairChainLocked($reference);
|
|
$job = $chain[0];
|
|
self::assertCurrent($job, $instance, $vpc, $port, $state);
|
|
$now = time();
|
|
$auditReason = substr(
|
|
'host_permanently_retired:' . $reason,
|
|
0,
|
|
self::MAX_REASON_LENGTH
|
|
);
|
|
foreach ($chain as $generation) {
|
|
$generation->effect_resolution = 'applied';
|
|
$generation->effect_resolved_at = $now;
|
|
$generation->effect_resolved_by =
|
|
(int) ($actor['user_id'] ?? 0);
|
|
$generation->effect_resolution_reason = $auditReason;
|
|
$generation->available_at = 0;
|
|
$generation->read_at = null;
|
|
$generation->updated_at = $now;
|
|
$generation->save();
|
|
}
|
|
|
|
$state->delete();
|
|
CloudVpcPortService::finishRetiredHostRemovalLocked(
|
|
$job, $vpc, $port, $instance
|
|
);
|
|
return [
|
|
'retired' => true,
|
|
'task_id' => (string) $job->id,
|
|
'host_id' => (string) $host->id,
|
|
'vpc_id' => (string) $vpc->id,
|
|
'resolved_generations' => count($chain),
|
|
];
|
|
},
|
|
3
|
|
);
|
|
}
|
|
|
|
private static function authorize(array $actor): void
|
|
{
|
|
if (!($actor['is_admin'] ?? false)
|
|
|| !($actor['is_super_admin'] ?? false)
|
|
|| !($actor['can_manage'] ?? false)
|
|
|| (int) ($actor['user_id'] ?? 0) < 1) {
|
|
throw new CloudError(
|
|
'只有超级管理员可以确认宿主机永久退役', 403
|
|
);
|
|
}
|
|
CloudAccess::manage($actor);
|
|
}
|
|
|
|
private static function reason(string $reason): string
|
|
{
|
|
$reason = trim($reason);
|
|
$length = mb_strlen($reason);
|
|
if ($length < self::MIN_REASON_LENGTH
|
|
|| $length > self::MAX_REASON_LENGTH) {
|
|
throw new CloudError('退役依据需为 20 至 500 个字符');
|
|
}
|
|
return $reason;
|
|
}
|
|
|
|
private static function assertReference(CloudJob $job): void
|
|
{
|
|
if ((string) $job->kind !== 'vpc_host_remove'
|
|
|| (int) ($job->host_id ?? 0) < 1
|
|
|| (int) ($job->instance_id ?? 0) < 1
|
|
|| (int) ($job->vpc_id ?? 0) < 1
|
|
|| (int) ($job->vpc_port_id ?? 0) < 1) {
|
|
throw new CloudError('该任务不是可退役的 VPC 宿主机清理', 409);
|
|
}
|
|
}
|
|
|
|
private static function assertCurrent(
|
|
CloudJob $job,
|
|
CloudInstance $instance,
|
|
CloudVpc $vpc,
|
|
CloudVpcPort $port,
|
|
CloudVpcHostState $state
|
|
): void {
|
|
self::assertReference($job);
|
|
if ((string) $job->execution_mode !== 'agent'
|
|
|| (string) $job->phase !== 'terminal'
|
|
|| (string) $job->outcome !== 'failed'
|
|
|| (string) $job->status !== 'failed'
|
|
|| !(bool) $job->effect_may_have_applied
|
|
|| (bool) $job->safe_to_resubmit
|
|
|| (string) $job->effect_resolution !== 'unknown'
|
|
|| !CloudDispatchLedger::isDispatched($job)
|
|
|| (int) ($job->dispatched_at ?? 0) < 1
|
|
|| CloudJobService::hasTerminalRecoveryStateConflict($job)
|
|
|| (bool) $job->finalization_incident
|
|
|| CloudDispatchLedger::isInvalid($job)
|
|
|| CloudDispatchLedger::hasRepairIncident($job)
|
|
|| (int) $job->instance_id !== (int) $instance->id
|
|
|| (int) $job->vpc_id !== (int) $vpc->id
|
|
|| (int) $job->vpc_port_id !== (int) $port->id
|
|
|| (int) $job->host_id !== (int) $state->host_id
|
|
|| (int) $port->vpc_id !== (int) $vpc->id
|
|
|| !(bool) $port->active_claim
|
|
|| (string) $port->state !== 'release_pending'
|
|
|| !hash_equals(
|
|
(string) $state->last_operation_id,
|
|
(string) $job->operation_id
|
|
)
|
|
|| !in_array((string) $state->sync_state,
|
|
['failed', 'unknown'], true)) {
|
|
throw new CloudError('VPC 宿主机清理代次已变化或不可退役', 409);
|
|
}
|
|
}
|
|
|
|
private static function assertHostEmptyLocked(
|
|
CloudHost $host
|
|
): void {
|
|
$hostId = (int) $host->id;
|
|
if ((bool) $host->enabled) {
|
|
throw new CloudError('请先停用宿主机,再确认永久退役', 409);
|
|
}
|
|
$agent = CloudHostAgent::query()->where('host_id', $hostId)
|
|
->lockForUpdate()->first();
|
|
if ($agent !== null && (bool) $agent->enabled) {
|
|
throw new CloudError('请先停用该宿主机的 Agent 配置', 409);
|
|
}
|
|
$memberships = CloudHostResourcePoolMembership::query()
|
|
->where('host_id', $hostId)->orderBy('id')
|
|
->lockForUpdate()->get();
|
|
if ($memberships->contains(static fn ($row): bool =>
|
|
(string) $row->scheduler_state !== 'disabled')) {
|
|
throw new CloudError('请先停用宿主机的全部资源池调度', 409);
|
|
}
|
|
// Placement serializes on the host row. This second current/locking
|
|
// read therefore catches a placement that committed after the
|
|
// discovery read but before this transaction acquired the host.
|
|
$currentInstances = CloudInstance::query()
|
|
->where('host_id', $hostId)
|
|
->where('status', '<>', 'deleted')->orderBy('id')
|
|
->lockForUpdate()->get();
|
|
if (!$currentInstances->isEmpty()) {
|
|
throw new CloudError('宿主机仍有未删除实例,不能确认永久退役', 409);
|
|
}
|
|
$reservations = CloudResourceReservation::query()
|
|
->where('host_id', $hostId)
|
|
->whereIn('state', ['held', 'committed', 'uncertain'])
|
|
->orderBy('id')->lockForUpdate()->get();
|
|
if (!$reservations->isEmpty()) {
|
|
throw new CloudError('宿主机仍占用计算资源,不能确认永久退役', 409);
|
|
}
|
|
$migrations = CloudInstanceMigration::query()
|
|
->where(static function ($query) use ($hostId): void {
|
|
$query->where('source_host_id', $hostId)
|
|
->orWhere('target_host_id', $hostId);
|
|
})->orderBy('id')->lockForUpdate()->get();
|
|
if ($migrations->contains(static fn ($row): bool =>
|
|
CloudInstanceMigrationWorkflow::blocksNewMigration(
|
|
(string) $row->state
|
|
))) {
|
|
throw new CloudError('宿主机仍有关联维护迁移,不能确认永久退役', 409);
|
|
}
|
|
$ports = CloudVpcPort::query()
|
|
->leftJoin(
|
|
'cloud_instances as retirement_instance',
|
|
'retirement_instance.id', '=', 'cloud_vpc_ports.instance_id'
|
|
)
|
|
->where('cloud_vpc_ports.active_claim', 1)
|
|
->where('cloud_vpc_ports.state', '<>', 'release_pending')
|
|
->where(static function ($query) use ($hostId): void {
|
|
$query->where(
|
|
'cloud_vpc_ports.migration_host_id', $hostId
|
|
)->orWhere('retirement_instance.host_id', $hostId);
|
|
})
|
|
->orderBy('cloud_vpc_ports.id')
|
|
->lockForUpdate()->get(['cloud_vpc_ports.id']);
|
|
if (!$ports->isEmpty()) {
|
|
throw new CloudError('宿主机仍承载 VPC 网卡,不能确认永久退役', 409);
|
|
}
|
|
}
|
|
|
|
/** @return list<CloudJob> current generation first */
|
|
private static function repairChainLocked(CloudJob $reference): array
|
|
{
|
|
$chain = [];
|
|
$seen = [];
|
|
$rootId = (int) ($reference->vpc_repair_root_job_id
|
|
?? $reference->id);
|
|
if ($rootId < 1) {
|
|
throw new CloudError('VPC 宿主机清理继任链缺少根任务', 409);
|
|
}
|
|
$catalog = CloudJob::query()
|
|
->where(static function ($query) use ($rootId): void {
|
|
$query->where('id', $rootId)
|
|
->orWhere('vpc_repair_root_job_id', $rootId);
|
|
})->orderBy('id')->lockForUpdate()->get()->keyBy('id');
|
|
$current = $catalog->get((int) $reference->id);
|
|
if (!$current instanceof CloudJob) {
|
|
throw new CloudError('VPC 宿主机清理当前代次已丢失', 409);
|
|
}
|
|
for ($depth = 0; $depth < self::MAX_SUPERSESSION_DEPTH; $depth++) {
|
|
$id = (int) $current->id;
|
|
if ($id < 1 || isset($seen[$id])) {
|
|
throw new CloudError('VPC 宿主机清理继任链包含循环', 409);
|
|
}
|
|
$seen[$id] = true;
|
|
if ((string) $current->kind !== 'vpc_host_remove'
|
|
|| (int) $current->host_id < 1
|
|
|| (string) $current->phase !== 'terminal'
|
|
|| (string) $current->outcome !== 'failed'
|
|
|| !(bool) $current->effect_may_have_applied
|
|
|| (bool) $current->safe_to_resubmit
|
|
|| (string) $current->effect_resolution !== 'unknown'
|
|
|| !CloudDispatchLedger::isDispatched($current)
|
|
|| (int) ($current->dispatched_at ?? 0) < 1
|
|
|| CloudJobService::hasTerminalRecoveryStateConflict(
|
|
$current
|
|
)
|
|
|| (bool) $current->finalization_incident
|
|
|| CloudDispatchLedger::isInvalid($current)
|
|
|| CloudDispatchLedger::hasRepairIncident($current)
|
|
|| (int) ($current->vpc_repair_root_job_id
|
|
?? $current->id) !== $rootId) {
|
|
throw new CloudError('VPC 宿主机清理继任链已变化', 409);
|
|
}
|
|
if ($chain !== []) {
|
|
$tip = $chain[0];
|
|
if ((int) $current->host_id !== (int) $tip->host_id
|
|
|| (int) $current->instance_id
|
|
!== (int) $tip->instance_id
|
|
|| (int) $current->vpc_id !== (int) $tip->vpc_id
|
|
|| (int) $current->vpc_port_id
|
|
!== (int) $tip->vpc_port_id) {
|
|
throw new CloudError('VPC 宿主机清理继任链资源不一致', 409);
|
|
}
|
|
}
|
|
$chain[] = $current;
|
|
$priorId = (int) ($current->supersedes_job_id ?? 0);
|
|
if ($priorId < 1) {
|
|
if ((int) $current->id !== $rootId) {
|
|
throw new CloudError(
|
|
'VPC 宿主机清理根任务身份不一致', 409
|
|
);
|
|
}
|
|
foreach ($catalog as $candidate) {
|
|
if ((int) $candidate->instance_id
|
|
!== (int) $reference->instance_id
|
|
|| (int) $candidate->vpc_id
|
|
!== (int) $reference->vpc_id
|
|
|| (int) $candidate->vpc_port_id
|
|
!== (int) $reference->vpc_port_id
|
|
|| (int) $candidate->host_id
|
|
!== (int) $reference->host_id
|
|
|| (string) $candidate->kind !== 'vpc_host_remove'
|
|
|| (string) $candidate->phase !== 'terminal'
|
|
|| ((string) $candidate->effect_resolution
|
|
=== 'unknown'
|
|
&& !isset($seen[(int) $candidate->id]))) {
|
|
throw new CloudError(
|
|
'VPC 宿主机清理根链仍有其他活动代次', 409
|
|
);
|
|
}
|
|
}
|
|
return $chain;
|
|
}
|
|
$prior = $catalog->get($priorId);
|
|
if ($prior === null) {
|
|
throw new CloudError('VPC 宿主机清理前序任务已丢失', 409);
|
|
}
|
|
$current = $prior;
|
|
}
|
|
throw new CloudError('VPC 宿主机清理继任链超过安全上限', 409);
|
|
}
|
|
}
|