fix(vpc): add permanent host retirement exit
parent
8e3a6dc607
commit
dcb90cad73
@ -0,0 +1,347 @@
|
||||
<?php
|
||||
|
||||
namespace App\Services\Cloud;
|
||||
|
||||
use App\Models\CloudHost;
|
||||
use App\Models\CloudHostAgent;
|
||||
use App\Models\CloudHostResourcePoolMembership;
|
||||
use App\Models\CloudInstance;
|
||||
use App\Models\CloudInstanceMigration;
|
||||
use App\Models\CloudJob;
|
||||
use App\Models\CloudResourceReservation;
|
||||
use App\Models\CloudVpc;
|
||||
use App\Models\CloudVpcHostState;
|
||||
use App\Models\CloudVpcPort;
|
||||
|
||||
/**
|
||||
* Resolve one unreachable host's exact VPC-removal fence after the physical
|
||||
* compute node has been permanently retired from every panel resource ledger.
|
||||
* This is not an Agent retry and never authorizes retiring a host that still
|
||||
* owns an instance, reservation, migration, or VPC port.
|
||||
*/
|
||||
final class CloudVpcHostRetirementService
|
||||
{
|
||||
private const MAX_REASON_LENGTH = 500;
|
||||
private const MIN_REASON_LENGTH = 20;
|
||||
private const MAX_SUPERSESSION_DEPTH = 32;
|
||||
|
||||
public static function retire(
|
||||
array $actor,
|
||||
int $jobId,
|
||||
string $reason
|
||||
): array {
|
||||
self::authorize($actor);
|
||||
$reason = self::reason($reason);
|
||||
$reference = CloudJob::query()->where('id', $jobId)->first();
|
||||
if ($reference === null) {
|
||||
throw new CloudError('任务不存在', 404);
|
||||
}
|
||||
self::assertReference($reference);
|
||||
|
||||
return (new CloudJob())->getConnection()->transaction(
|
||||
static function () use ($actor, $reference, $reason): array {
|
||||
$hostInstanceIds = CloudInstance::query()
|
||||
->where('host_id', (int) $reference->host_id)
|
||||
->where('status', '<>', 'deleted')->pluck('id')
|
||||
->map(static fn ($id): int => (int) $id)->all();
|
||||
$instanceIds = array_values(array_unique(array_merge(
|
||||
$hostInstanceIds,
|
||||
[(int) $reference->instance_id]
|
||||
)));
|
||||
$instances = CloudVpcLockOrder::instances($instanceIds);
|
||||
$instance = $instances->get((int) $reference->instance_id);
|
||||
$vpc = CloudVpcLockOrder::vpcById(
|
||||
(int) $reference->vpc_id
|
||||
);
|
||||
$port = CloudVpcLockOrder::portById(
|
||||
(int) $reference->vpc_port_id
|
||||
);
|
||||
$state = CloudVpcHostState::query()
|
||||
->where('vpc_id', (int) $reference->vpc_id)
|
||||
->where('host_id', (int) $reference->host_id)
|
||||
->lockForUpdate()->first();
|
||||
if ($instance === null || $vpc === null || $port === null
|
||||
|| $state === null) {
|
||||
throw new CloudError('VPC 宿主机退役账本不完整', 409);
|
||||
}
|
||||
|
||||
$host = CloudHost::query()
|
||||
->where('id', (int) $reference->host_id)
|
||||
->lockForUpdate()->first();
|
||||
if ($host === null) {
|
||||
throw new CloudError('宿主机不存在', 404);
|
||||
}
|
||||
self::assertHostEmptyLocked($host);
|
||||
|
||||
// Coordination rows are last in the canonical VPC lock order.
|
||||
// The complete root catalog is acquired in ascending id order
|
||||
// so a retirement cannot invert a concurrent repair lock.
|
||||
$chain = self::repairChainLocked($reference);
|
||||
$job = $chain[0];
|
||||
self::assertCurrent($job, $instance, $vpc, $port, $state);
|
||||
$now = time();
|
||||
$auditReason = substr(
|
||||
'host_permanently_retired:' . $reason,
|
||||
0,
|
||||
self::MAX_REASON_LENGTH
|
||||
);
|
||||
foreach ($chain as $generation) {
|
||||
$generation->effect_resolution = 'applied';
|
||||
$generation->effect_resolved_at = $now;
|
||||
$generation->effect_resolved_by =
|
||||
(int) ($actor['user_id'] ?? 0);
|
||||
$generation->effect_resolution_reason = $auditReason;
|
||||
$generation->available_at = 0;
|
||||
$generation->read_at = null;
|
||||
$generation->updated_at = $now;
|
||||
$generation->save();
|
||||
}
|
||||
|
||||
$state->delete();
|
||||
CloudVpcPortService::finishRetiredHostRemovalLocked(
|
||||
$job, $vpc, $port, $instance
|
||||
);
|
||||
return [
|
||||
'retired' => true,
|
||||
'task_id' => (string) $job->id,
|
||||
'host_id' => (string) $host->id,
|
||||
'vpc_id' => (string) $vpc->id,
|
||||
'resolved_generations' => count($chain),
|
||||
];
|
||||
},
|
||||
3
|
||||
);
|
||||
}
|
||||
|
||||
private static function authorize(array $actor): void
|
||||
{
|
||||
if (!($actor['is_admin'] ?? false)
|
||||
|| !($actor['is_super_admin'] ?? false)
|
||||
|| !($actor['can_manage'] ?? false)
|
||||
|| (int) ($actor['user_id'] ?? 0) < 1) {
|
||||
throw new CloudError(
|
||||
'只有超级管理员可以确认宿主机永久退役', 403
|
||||
);
|
||||
}
|
||||
CloudAccess::manage($actor);
|
||||
}
|
||||
|
||||
private static function reason(string $reason): string
|
||||
{
|
||||
$reason = trim($reason);
|
||||
$length = mb_strlen($reason);
|
||||
if ($length < self::MIN_REASON_LENGTH
|
||||
|| $length > self::MAX_REASON_LENGTH) {
|
||||
throw new CloudError('退役依据需为 20 至 500 个字符');
|
||||
}
|
||||
return $reason;
|
||||
}
|
||||
|
||||
private static function assertReference(CloudJob $job): void
|
||||
{
|
||||
if ((string) $job->kind !== 'vpc_host_remove'
|
||||
|| (int) ($job->host_id ?? 0) < 1
|
||||
|| (int) ($job->instance_id ?? 0) < 1
|
||||
|| (int) ($job->vpc_id ?? 0) < 1
|
||||
|| (int) ($job->vpc_port_id ?? 0) < 1) {
|
||||
throw new CloudError('该任务不是可退役的 VPC 宿主机清理', 409);
|
||||
}
|
||||
}
|
||||
|
||||
private static function assertCurrent(
|
||||
CloudJob $job,
|
||||
CloudInstance $instance,
|
||||
CloudVpc $vpc,
|
||||
CloudVpcPort $port,
|
||||
CloudVpcHostState $state
|
||||
): void {
|
||||
self::assertReference($job);
|
||||
if ((string) $job->execution_mode !== 'agent'
|
||||
|| (string) $job->phase !== 'terminal'
|
||||
|| (string) $job->outcome !== 'failed'
|
||||
|| (string) $job->status !== 'failed'
|
||||
|| !(bool) $job->effect_may_have_applied
|
||||
|| (bool) $job->safe_to_resubmit
|
||||
|| (string) $job->effect_resolution !== 'unknown'
|
||||
|| !CloudDispatchLedger::isDispatched($job)
|
||||
|| (int) ($job->dispatched_at ?? 0) < 1
|
||||
|| CloudJobService::hasTerminalRecoveryStateConflict($job)
|
||||
|| (bool) $job->finalization_incident
|
||||
|| CloudDispatchLedger::isInvalid($job)
|
||||
|| CloudDispatchLedger::hasRepairIncident($job)
|
||||
|| (int) $job->instance_id !== (int) $instance->id
|
||||
|| (int) $job->vpc_id !== (int) $vpc->id
|
||||
|| (int) $job->vpc_port_id !== (int) $port->id
|
||||
|| (int) $job->host_id !== (int) $state->host_id
|
||||
|| (int) $port->vpc_id !== (int) $vpc->id
|
||||
|| !(bool) $port->active_claim
|
||||
|| (string) $port->state !== 'release_pending'
|
||||
|| !hash_equals(
|
||||
(string) $state->last_operation_id,
|
||||
(string) $job->operation_id
|
||||
)
|
||||
|| !in_array((string) $state->sync_state,
|
||||
['failed', 'unknown'], true)) {
|
||||
throw new CloudError('VPC 宿主机清理代次已变化或不可退役', 409);
|
||||
}
|
||||
}
|
||||
|
||||
private static function assertHostEmptyLocked(
|
||||
CloudHost $host
|
||||
): void {
|
||||
$hostId = (int) $host->id;
|
||||
if ((bool) $host->enabled) {
|
||||
throw new CloudError('请先停用宿主机,再确认永久退役', 409);
|
||||
}
|
||||
$agent = CloudHostAgent::query()->where('host_id', $hostId)
|
||||
->lockForUpdate()->first();
|
||||
if ($agent !== null && (bool) $agent->enabled) {
|
||||
throw new CloudError('请先停用该宿主机的 Agent 配置', 409);
|
||||
}
|
||||
$memberships = CloudHostResourcePoolMembership::query()
|
||||
->where('host_id', $hostId)->orderBy('id')
|
||||
->lockForUpdate()->get();
|
||||
if ($memberships->contains(static fn ($row): bool =>
|
||||
(string) $row->scheduler_state !== 'disabled')) {
|
||||
throw new CloudError('请先停用宿主机的全部资源池调度', 409);
|
||||
}
|
||||
// Placement serializes on the host row. This second current/locking
|
||||
// read therefore catches a placement that committed after the
|
||||
// discovery read but before this transaction acquired the host.
|
||||
$currentInstances = CloudInstance::query()
|
||||
->where('host_id', $hostId)
|
||||
->where('status', '<>', 'deleted')->orderBy('id')
|
||||
->lockForUpdate()->get();
|
||||
if (!$currentInstances->isEmpty()) {
|
||||
throw new CloudError('宿主机仍有未删除实例,不能确认永久退役', 409);
|
||||
}
|
||||
$reservations = CloudResourceReservation::query()
|
||||
->where('host_id', $hostId)
|
||||
->whereIn('state', ['held', 'committed', 'uncertain'])
|
||||
->orderBy('id')->lockForUpdate()->get();
|
||||
if (!$reservations->isEmpty()) {
|
||||
throw new CloudError('宿主机仍占用计算资源,不能确认永久退役', 409);
|
||||
}
|
||||
$migrations = CloudInstanceMigration::query()
|
||||
->where(static function ($query) use ($hostId): void {
|
||||
$query->where('source_host_id', $hostId)
|
||||
->orWhere('target_host_id', $hostId);
|
||||
})->orderBy('id')->lockForUpdate()->get();
|
||||
if ($migrations->contains(static fn ($row): bool =>
|
||||
CloudInstanceMigrationWorkflow::blocksNewMigration(
|
||||
(string) $row->state
|
||||
))) {
|
||||
throw new CloudError('宿主机仍有关联维护迁移,不能确认永久退役', 409);
|
||||
}
|
||||
$ports = CloudVpcPort::query()
|
||||
->leftJoin(
|
||||
'cloud_instances as retirement_instance',
|
||||
'retirement_instance.id', '=', 'cloud_vpc_ports.instance_id'
|
||||
)
|
||||
->where('cloud_vpc_ports.active_claim', 1)
|
||||
->where('cloud_vpc_ports.state', '<>', 'release_pending')
|
||||
->where(static function ($query) use ($hostId): void {
|
||||
$query->where(
|
||||
'cloud_vpc_ports.migration_host_id', $hostId
|
||||
)->orWhere('retirement_instance.host_id', $hostId);
|
||||
})
|
||||
->orderBy('cloud_vpc_ports.id')
|
||||
->lockForUpdate()->get(['cloud_vpc_ports.id']);
|
||||
if (!$ports->isEmpty()) {
|
||||
throw new CloudError('宿主机仍承载 VPC 网卡,不能确认永久退役', 409);
|
||||
}
|
||||
}
|
||||
|
||||
/** @return list<CloudJob> current generation first */
|
||||
private static function repairChainLocked(CloudJob $reference): array
|
||||
{
|
||||
$chain = [];
|
||||
$seen = [];
|
||||
$rootId = (int) ($reference->vpc_repair_root_job_id
|
||||
?? $reference->id);
|
||||
if ($rootId < 1) {
|
||||
throw new CloudError('VPC 宿主机清理继任链缺少根任务', 409);
|
||||
}
|
||||
$catalog = CloudJob::query()
|
||||
->where(static function ($query) use ($rootId): void {
|
||||
$query->where('id', $rootId)
|
||||
->orWhere('vpc_repair_root_job_id', $rootId);
|
||||
})->orderBy('id')->lockForUpdate()->get()->keyBy('id');
|
||||
$current = $catalog->get((int) $reference->id);
|
||||
if (!$current instanceof CloudJob) {
|
||||
throw new CloudError('VPC 宿主机清理当前代次已丢失', 409);
|
||||
}
|
||||
for ($depth = 0; $depth < self::MAX_SUPERSESSION_DEPTH; $depth++) {
|
||||
$id = (int) $current->id;
|
||||
if ($id < 1 || isset($seen[$id])) {
|
||||
throw new CloudError('VPC 宿主机清理继任链包含循环', 409);
|
||||
}
|
||||
$seen[$id] = true;
|
||||
if ((string) $current->kind !== 'vpc_host_remove'
|
||||
|| (int) $current->host_id < 1
|
||||
|| (string) $current->phase !== 'terminal'
|
||||
|| (string) $current->outcome !== 'failed'
|
||||
|| !(bool) $current->effect_may_have_applied
|
||||
|| (bool) $current->safe_to_resubmit
|
||||
|| (string) $current->effect_resolution !== 'unknown'
|
||||
|| !CloudDispatchLedger::isDispatched($current)
|
||||
|| (int) ($current->dispatched_at ?? 0) < 1
|
||||
|| CloudJobService::hasTerminalRecoveryStateConflict(
|
||||
$current
|
||||
)
|
||||
|| (bool) $current->finalization_incident
|
||||
|| CloudDispatchLedger::isInvalid($current)
|
||||
|| CloudDispatchLedger::hasRepairIncident($current)
|
||||
|| (int) ($current->vpc_repair_root_job_id
|
||||
?? $current->id) !== $rootId) {
|
||||
throw new CloudError('VPC 宿主机清理继任链已变化', 409);
|
||||
}
|
||||
if ($chain !== []) {
|
||||
$tip = $chain[0];
|
||||
if ((int) $current->host_id !== (int) $tip->host_id
|
||||
|| (int) $current->instance_id
|
||||
!== (int) $tip->instance_id
|
||||
|| (int) $current->vpc_id !== (int) $tip->vpc_id
|
||||
|| (int) $current->vpc_port_id
|
||||
!== (int) $tip->vpc_port_id) {
|
||||
throw new CloudError('VPC 宿主机清理继任链资源不一致', 409);
|
||||
}
|
||||
}
|
||||
$chain[] = $current;
|
||||
$priorId = (int) ($current->supersedes_job_id ?? 0);
|
||||
if ($priorId < 1) {
|
||||
if ((int) $current->id !== $rootId) {
|
||||
throw new CloudError(
|
||||
'VPC 宿主机清理根任务身份不一致', 409
|
||||
);
|
||||
}
|
||||
foreach ($catalog as $candidate) {
|
||||
if ((int) $candidate->instance_id
|
||||
!== (int) $reference->instance_id
|
||||
|| (int) $candidate->vpc_id
|
||||
!== (int) $reference->vpc_id
|
||||
|| (int) $candidate->vpc_port_id
|
||||
!== (int) $reference->vpc_port_id
|
||||
|| (int) $candidate->host_id
|
||||
!== (int) $reference->host_id
|
||||
|| (string) $candidate->kind !== 'vpc_host_remove'
|
||||
|| (string) $candidate->phase !== 'terminal'
|
||||
|| ((string) $candidate->effect_resolution
|
||||
=== 'unknown'
|
||||
&& !isset($seen[(int) $candidate->id]))) {
|
||||
throw new CloudError(
|
||||
'VPC 宿主机清理根链仍有其他活动代次', 409
|
||||
);
|
||||
}
|
||||
}
|
||||
return $chain;
|
||||
}
|
||||
$prior = $catalog->get($priorId);
|
||||
if ($prior === null) {
|
||||
throw new CloudError('VPC 宿主机清理前序任务已丢失', 409);
|
||||
}
|
||||
$current = $prior;
|
||||
}
|
||||
throw new CloudError('VPC 宿主机清理继任链超过安全上限', 409);
|
||||
}
|
||||
}
|
||||
@ -1,2 +1,2 @@
|
||||
{* Shared Venix CSS/JS cache key. Increment whenever a shared browser asset changes. *}
|
||||
{assign var=venix_asset_version value='2026102907' scope='global'}
|
||||
{assign var=venix_asset_version value='2026102908' scope='global'}
|
||||
|
||||
@ -0,0 +1,580 @@
|
||||
<?php
|
||||
|
||||
namespace App\Services\Cloud {
|
||||
final class CloudSecrets
|
||||
{
|
||||
public static function encrypt(array $value, string $purpose): string
|
||||
{
|
||||
return 'retirement-test:' . hash('sha256', json_encode(
|
||||
$value,
|
||||
JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES
|
||||
));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
namespace {
|
||||
require dirname(__DIR__) . '/vendor/autoload.php';
|
||||
|
||||
use App\Models\CloudJob;
|
||||
use App\Services\Cloud\CloudError;
|
||||
use App\Services\Cloud\CloudVpcHostRetirementService;
|
||||
use App\Services\Cloud\CloudVpcPortService;
|
||||
use Illuminate\Database\Capsule\Manager as Capsule;
|
||||
use Illuminate\Database\Schema\Blueprint;
|
||||
|
||||
$assert = static function (bool $condition, string $message): void {
|
||||
if (!$condition) {
|
||||
throw new RuntimeException($message);
|
||||
}
|
||||
};
|
||||
$reject = static function (callable $callback, string $message) use (
|
||||
$assert
|
||||
): void {
|
||||
try {
|
||||
$callback();
|
||||
} catch (CloudError $error) {
|
||||
$assert($error->httpStatus === 409,
|
||||
$message . ': HTTP ' . $error->httpStatus);
|
||||
return;
|
||||
}
|
||||
throw new RuntimeException($message . ': request was accepted');
|
||||
};
|
||||
|
||||
$capsule = new Capsule();
|
||||
$capsule->addConnection([
|
||||
'driver' => 'sqlite', 'database' => ':memory:',
|
||||
], 'default');
|
||||
$capsule->setAsGlobal();
|
||||
$capsule->bootEloquent();
|
||||
$schema = $capsule->schema('default');
|
||||
|
||||
$schema->create('cloud_hosts', static function (Blueprint $table): void {
|
||||
$table->increments('id');
|
||||
$table->string('name');
|
||||
$table->string('region')->default('');
|
||||
$table->boolean('enabled')->default(false);
|
||||
$table->integer('created_at');
|
||||
$table->integer('updated_at');
|
||||
});
|
||||
$schema->create('cloud_host_agents', static function (Blueprint $table): void {
|
||||
$table->integer('host_id')->primary();
|
||||
$table->boolean('enabled')->default(false);
|
||||
});
|
||||
$schema->create('cloud_host_resource_pool_memberships', static function (
|
||||
Blueprint $table
|
||||
): void {
|
||||
$table->increments('id');
|
||||
$table->integer('host_id');
|
||||
$table->integer('resource_pool_id');
|
||||
$table->string('scheduler_state');
|
||||
$table->integer('priority')->default(0);
|
||||
$table->integer('weight')->default(100);
|
||||
$table->integer('created_at');
|
||||
$table->integer('updated_at');
|
||||
});
|
||||
$schema->create('cloud_instances', static function (Blueprint $table): void {
|
||||
$table->increments('id');
|
||||
$table->string('uuid');
|
||||
$table->integer('host_id');
|
||||
$table->integer('user_id');
|
||||
$table->string('name');
|
||||
$table->string('status');
|
||||
$table->string('service_status');
|
||||
$table->string('network_status');
|
||||
$table->text('spec_json');
|
||||
$table->integer('created_at');
|
||||
$table->integer('updated_at');
|
||||
});
|
||||
$schema->create('cloud_resource_reservations', static function (
|
||||
Blueprint $table
|
||||
): void {
|
||||
$table->increments('id');
|
||||
$table->integer('host_id');
|
||||
$table->string('state');
|
||||
});
|
||||
$schema->create('cloud_instance_migrations', static function (
|
||||
Blueprint $table
|
||||
): void {
|
||||
$table->increments('id');
|
||||
$table->integer('instance_id');
|
||||
$table->integer('source_host_id');
|
||||
$table->integer('target_host_id');
|
||||
$table->string('state');
|
||||
});
|
||||
$schema->create('cloud_vpcs', static function (Blueprint $table): void {
|
||||
$table->increments('id');
|
||||
$table->string('vpc_key');
|
||||
$table->integer('user_id');
|
||||
$table->integer('region_id');
|
||||
$table->integer('cidr_pool_id');
|
||||
$table->string('name');
|
||||
$table->string('cidr');
|
||||
$table->integer('prefix_length');
|
||||
$table->string('gateway');
|
||||
$table->integer('active_claim')->nullable();
|
||||
$table->string('state');
|
||||
$table->integer('desired_revision');
|
||||
$table->integer('deleted_at')->nullable();
|
||||
$table->integer('created_at');
|
||||
$table->integer('updated_at');
|
||||
});
|
||||
$schema->create('cloud_vpc_ports', static function (Blueprint $table): void {
|
||||
$table->increments('id');
|
||||
$table->string('port_key');
|
||||
$table->integer('vpc_id');
|
||||
$table->integer('instance_id');
|
||||
$table->integer('migration_host_id')->nullable();
|
||||
$table->integer('user_id');
|
||||
$table->string('private_ip');
|
||||
$table->string('mac_address');
|
||||
$table->integer('active_claim')->nullable();
|
||||
$table->string('state');
|
||||
$table->integer('desired_revision');
|
||||
$table->integer('applied_revision');
|
||||
$table->string('last_operation_id')->nullable();
|
||||
$table->string('last_error_code')->default('');
|
||||
$table->integer('attached_at')->nullable();
|
||||
$table->integer('released_at')->nullable();
|
||||
$table->integer('created_at');
|
||||
$table->integer('updated_at');
|
||||
});
|
||||
$schema->create('cloud_vpc_host_states', static function (
|
||||
Blueprint $table
|
||||
): void {
|
||||
$table->increments('id');
|
||||
$table->integer('vpc_id');
|
||||
$table->integer('host_id');
|
||||
$table->integer('desired_revision');
|
||||
$table->integer('applied_revision');
|
||||
$table->string('desired_digest');
|
||||
$table->string('applied_digest')->nullable();
|
||||
$table->string('sync_state');
|
||||
$table->string('last_operation_id')->nullable();
|
||||
$table->string('last_error_code')->default('');
|
||||
$table->integer('last_success_at')->default(0);
|
||||
$table->integer('last_failure_at')->default(0);
|
||||
$table->integer('created_at');
|
||||
$table->integer('updated_at');
|
||||
});
|
||||
$schema->create('cloud_vpc_membership_batches', static function (
|
||||
Blueprint $table
|
||||
): void {
|
||||
$table->increments('id');
|
||||
$table->string('batch_key');
|
||||
$table->integer('vpc_id');
|
||||
$table->integer('user_id');
|
||||
$table->integer('parent_job_id');
|
||||
$table->string('kind');
|
||||
$table->string('state');
|
||||
$table->integer('desired_revision');
|
||||
$table->integer('active_claim')->nullable();
|
||||
$table->string('last_error_code')->default('');
|
||||
$table->integer('terminal_at')->nullable();
|
||||
$table->integer('created_at');
|
||||
$table->integer('updated_at');
|
||||
});
|
||||
$schema->create('cloud_vpc_membership_batch_items', static function (
|
||||
Blueprint $table
|
||||
): void {
|
||||
$table->increments('id');
|
||||
$table->integer('batch_id');
|
||||
$table->integer('user_id');
|
||||
$table->integer('instance_id');
|
||||
$table->integer('vpc_port_id');
|
||||
$table->integer('host_id');
|
||||
$table->integer('child_job_id')->nullable();
|
||||
$table->string('state');
|
||||
$table->string('last_error_code')->default('');
|
||||
$table->integer('created_at');
|
||||
$table->integer('updated_at');
|
||||
});
|
||||
$schema->create('cloud_jobs', static function (Blueprint $table): void {
|
||||
$table->increments('id');
|
||||
$table->string('operation_id');
|
||||
$table->string('request_idempotency_key');
|
||||
$table->string('request_fingerprint');
|
||||
$table->integer('parent_job_id')->nullable();
|
||||
$table->integer('supersedes_job_id')->nullable();
|
||||
$table->integer('vpc_repair_root_job_id')->nullable();
|
||||
$table->integer('order_id')->nullable();
|
||||
$table->integer('invoice_id')->nullable();
|
||||
$table->integer('instance_id')->nullable();
|
||||
$table->integer('snapshot_id')->nullable();
|
||||
$table->integer('backup_id')->nullable();
|
||||
$table->integer('backup_restore_id')->nullable();
|
||||
$table->integer('backup_recovery_id')->nullable();
|
||||
$table->integer('vpc_id')->nullable();
|
||||
$table->integer('vpc_port_id')->nullable();
|
||||
$table->integer('vpc_batch_id')->nullable();
|
||||
$table->integer('user_id');
|
||||
$table->integer('host_id')->nullable();
|
||||
$table->string('kind');
|
||||
$table->string('execution_mode');
|
||||
$table->string('phase');
|
||||
$table->string('outcome')->nullable();
|
||||
$table->string('status');
|
||||
$table->text('sealed_payload');
|
||||
$table->integer('attempts');
|
||||
$table->integer('dispatch_attempts');
|
||||
$table->integer('dispatch_deadline_at');
|
||||
$table->integer('dispatch_started_at')->nullable();
|
||||
$table->integer('agent_accepted_at')->nullable();
|
||||
$table->integer('receipt_deadline_at')->nullable();
|
||||
$table->integer('terminal_at')->nullable();
|
||||
$table->boolean('effect_may_have_applied')->nullable();
|
||||
$table->boolean('safe_to_resubmit')->nullable();
|
||||
$table->string('effect_resolution');
|
||||
$table->integer('effect_resolved_at')->nullable();
|
||||
$table->integer('effect_resolved_by')->nullable();
|
||||
$table->text('effect_resolution_reason')->default('');
|
||||
$table->boolean('finalization_incident')->default(false);
|
||||
$table->string('finalization_error_code')->nullable();
|
||||
$table->string('completion_event_id')->nullable();
|
||||
$table->integer('read_at')->nullable();
|
||||
$table->integer('dispatched_at')->nullable();
|
||||
$table->string('lease_token')->nullable();
|
||||
$table->integer('lease_until')->default(0);
|
||||
$table->integer('available_at');
|
||||
$table->string('error_code')->nullable();
|
||||
$table->text('result_json')->nullable();
|
||||
$table->integer('created_at');
|
||||
$table->integer('updated_at');
|
||||
});
|
||||
|
||||
$now = time();
|
||||
$actor = [
|
||||
'user_id' => 99, 'is_admin' => true,
|
||||
'is_super_admin' => true, 'can_manage' => true,
|
||||
];
|
||||
Capsule::table('cloud_hosts')->insert([
|
||||
['id' => 1, 'name' => 'retiring-host', 'region' => 'hk',
|
||||
'enabled' => 1, 'created_at' => $now, 'updated_at' => $now],
|
||||
['id' => 2, 'name' => 'surviving-host', 'region' => 'hk',
|
||||
'enabled' => 1, 'created_at' => $now, 'updated_at' => $now],
|
||||
]);
|
||||
Capsule::table('cloud_host_agents')->insert([
|
||||
'host_id' => 1, 'enabled' => 1,
|
||||
]);
|
||||
Capsule::table('cloud_host_resource_pool_memberships')->insert([
|
||||
'id' => 1, 'host_id' => 1, 'resource_pool_id' => 1,
|
||||
'scheduler_state' => 'active', 'priority' => 0, 'weight' => 100,
|
||||
'created_at' => $now, 'updated_at' => $now,
|
||||
]);
|
||||
$instanceRows = [
|
||||
['id' => 10, 'uuid' => '11111111-1111-4111-8111-111111111110',
|
||||
'host_id' => 1, 'user_id' => 7, 'name' => 'retired-reference',
|
||||
'status' => 'deleted'],
|
||||
['id' => 11, 'uuid' => '11111111-1111-4111-8111-111111111111',
|
||||
'host_id' => 1, 'user_id' => 7, 'name' => 'host-blocker',
|
||||
'status' => 'active'],
|
||||
];
|
||||
foreach ($instanceRows as &$row) {
|
||||
$row += ['service_status' => 'active', 'network_status' => 'active',
|
||||
'spec_json' => '{}', 'created_at' => $now, 'updated_at' => $now];
|
||||
}
|
||||
unset($row);
|
||||
Capsule::table('cloud_instances')->insert($instanceRows);
|
||||
Capsule::table('cloud_resource_reservations')->insert([
|
||||
'id' => 1, 'host_id' => 1, 'state' => 'held',
|
||||
]);
|
||||
Capsule::table('cloud_instance_migrations')->insert([
|
||||
'id' => 1, 'instance_id' => 11, 'source_host_id' => 1,
|
||||
'target_host_id' => 2, 'state' => 'preparing_source',
|
||||
]);
|
||||
Capsule::table('cloud_vpcs')->insert([
|
||||
'id' => 20, 'vpc_key' => str_repeat('a', 32), 'user_id' => 7,
|
||||
'region_id' => 1, 'cidr_pool_id' => 1, 'name' => 'retire-test',
|
||||
'cidr' => '10.10.0.0/24', 'prefix_length' => 24,
|
||||
'gateway' => '10.10.0.1', 'active_claim' => 1,
|
||||
'state' => 'provisioning', 'desired_revision' => 3,
|
||||
'deleted_at' => null, 'created_at' => $now, 'updated_at' => $now,
|
||||
]);
|
||||
Capsule::table('cloud_vpc_ports')->insert([
|
||||
[
|
||||
'id' => 30, 'port_key' => str_repeat('b', 32), 'vpc_id' => 20,
|
||||
'instance_id' => 10, 'migration_host_id' => null, 'user_id' => 7,
|
||||
'private_ip' => '10.10.0.2', 'mac_address' => '02:00:00:00:00:02',
|
||||
'active_claim' => 1, 'state' => 'release_pending',
|
||||
'desired_revision' => 3, 'applied_revision' => 2,
|
||||
'last_operation_id' => null, 'last_error_code' => '',
|
||||
'attached_at' => $now - 100, 'released_at' => null,
|
||||
'created_at' => $now, 'updated_at' => $now,
|
||||
],
|
||||
[
|
||||
'id' => 31, 'port_key' => str_repeat('c', 32), 'vpc_id' => 20,
|
||||
'instance_id' => 11, 'migration_host_id' => null, 'user_id' => 7,
|
||||
'private_ip' => '10.10.0.3', 'mac_address' => '02:00:00:00:00:03',
|
||||
'active_claim' => 1, 'state' => 'attached',
|
||||
'desired_revision' => 2, 'applied_revision' => 2,
|
||||
'last_operation_id' => null, 'last_error_code' => '',
|
||||
'attached_at' => $now - 100, 'released_at' => null,
|
||||
'created_at' => $now, 'updated_at' => $now,
|
||||
],
|
||||
]);
|
||||
$rootOperation = str_repeat('d', 32);
|
||||
$tipOperation = str_repeat('e', 32);
|
||||
Capsule::table('cloud_vpc_host_states')->insert([
|
||||
'id' => 40, 'vpc_id' => 20, 'host_id' => 1,
|
||||
'desired_revision' => 3, 'applied_revision' => 2,
|
||||
'desired_digest' => str_repeat('f', 64),
|
||||
'applied_digest' => str_repeat('1', 64), 'sync_state' => 'unknown',
|
||||
'last_operation_id' => $tipOperation,
|
||||
'last_error_code' => 'operation_effect_unknown',
|
||||
'last_success_at' => $now - 100, 'last_failure_at' => $now - 10,
|
||||
'created_at' => $now, 'updated_at' => $now,
|
||||
]);
|
||||
$job = static function (
|
||||
int $id,
|
||||
string $operationId,
|
||||
?int $supersedes,
|
||||
?int $root
|
||||
) use ($now): array {
|
||||
return [
|
||||
'id' => $id, 'operation_id' => $operationId,
|
||||
'request_idempotency_key' => $operationId,
|
||||
'request_fingerprint' => hash('sha256', $operationId),
|
||||
'parent_job_id' => null, 'supersedes_job_id' => $supersedes,
|
||||
'vpc_repair_root_job_id' => $root, 'order_id' => null,
|
||||
'invoice_id' => null, 'instance_id' => 10, 'snapshot_id' => null,
|
||||
'backup_id' => null, 'backup_restore_id' => null,
|
||||
'backup_recovery_id' => null, 'vpc_id' => 20,
|
||||
'vpc_port_id' => 30, 'vpc_batch_id' => null,
|
||||
'user_id' => 7, 'host_id' => 1, 'kind' => 'vpc_host_remove',
|
||||
'execution_mode' => 'agent', 'phase' => 'terminal',
|
||||
'outcome' => 'failed', 'status' => 'failed', 'sealed_payload' => '{}',
|
||||
'attempts' => 1, 'dispatch_attempts' => 1,
|
||||
'dispatch_deadline_at' => $now - 200,
|
||||
'dispatch_started_at' => $now - 190,
|
||||
'agent_accepted_at' => $now - 185,
|
||||
'receipt_deadline_at' => $now - 100, 'terminal_at' => $now - 20,
|
||||
'effect_may_have_applied' => 1, 'safe_to_resubmit' => 0,
|
||||
'effect_resolution' => 'unknown', 'effect_resolved_at' => null,
|
||||
'effect_resolved_by' => null, 'effect_resolution_reason' => '',
|
||||
'finalization_incident' => 0, 'finalization_error_code' => null,
|
||||
'completion_event_id' => null, 'read_at' => null,
|
||||
'dispatched_at' => $now - 185, 'lease_token' => null,
|
||||
'lease_until' => 0, 'available_at' => 0,
|
||||
'error_code' => 'operation_effect_unknown', 'result_json' => null,
|
||||
'created_at' => $now - 200, 'updated_at' => $now - 20,
|
||||
];
|
||||
};
|
||||
Capsule::table('cloud_jobs')->insert([
|
||||
$job(100, $rootOperation, null, null),
|
||||
$job(101, $tipOperation, 100, 100),
|
||||
array_replace(
|
||||
$job(102, str_repeat('9', 32), 101, 100),
|
||||
[
|
||||
'effect_may_have_applied' => 0,
|
||||
'safe_to_resubmit' => 1,
|
||||
'effect_resolution' => 'not_applied',
|
||||
'error_code' => 'agent_rejected',
|
||||
]
|
||||
),
|
||||
]);
|
||||
|
||||
$retire = static fn (): array => CloudVpcHostRetirementService::retire(
|
||||
$actor,
|
||||
101,
|
||||
'The physical node was decommissioned after all tenant data was removed.'
|
||||
);
|
||||
|
||||
$reject($retire, 'enabled host must block permanent retirement');
|
||||
Capsule::table('cloud_hosts')->where('id', 1)->update(['enabled' => 0]);
|
||||
$reject($retire, 'enabled Agent must block permanent retirement');
|
||||
Capsule::table('cloud_host_agents')->where('host_id', 1)
|
||||
->update(['enabled' => 0]);
|
||||
$reject($retire, 'active scheduler membership must block retirement');
|
||||
Capsule::table('cloud_host_resource_pool_memberships')->where('id', 1)
|
||||
->update(['scheduler_state' => 'disabled']);
|
||||
$reject($retire, 'nondeleted instance must block permanent retirement');
|
||||
Capsule::table('cloud_instances')->where('id', 11)
|
||||
->update(['status' => 'deleted']);
|
||||
$reject($retire, 'active resource reservation must block retirement');
|
||||
Capsule::table('cloud_resource_reservations')->where('id', 1)
|
||||
->update(['state' => 'released']);
|
||||
$reject($retire, 'active maintenance migration must block retirement');
|
||||
Capsule::table('cloud_instance_migrations')->where('id', 1)
|
||||
->update(['state' => 'completed']);
|
||||
$reject($retire, 'attached VPC port must block permanent retirement');
|
||||
Capsule::table('cloud_vpc_ports')->where('id', 31)->update([
|
||||
'active_claim' => null, 'state' => 'released', 'released_at' => $now,
|
||||
]);
|
||||
|
||||
Capsule::table('cloud_jobs')->where('id', 102)->update([
|
||||
'effect_may_have_applied' => 1, 'safe_to_resubmit' => 0,
|
||||
'effect_resolution' => 'unknown',
|
||||
]);
|
||||
$reject($retire, 'parallel unknown repair generation must block retirement');
|
||||
Capsule::table('cloud_jobs')->where('id', 102)->update([
|
||||
'effect_may_have_applied' => 0, 'safe_to_resubmit' => 1,
|
||||
'effect_resolution' => 'not_applied',
|
||||
]);
|
||||
|
||||
Capsule::table('cloud_jobs')->where('id', 101)->update([
|
||||
'effect_may_have_applied' => 0, 'safe_to_resubmit' => 1,
|
||||
'effect_resolution' => 'not_applied',
|
||||
]);
|
||||
$reject($retire, 'known no-effect job must not be rewritten as applied');
|
||||
Capsule::table('cloud_jobs')->where('id', 101)->update([
|
||||
'effect_may_have_applied' => 1, 'safe_to_resubmit' => 0,
|
||||
'effect_resolution' => 'unknown',
|
||||
]);
|
||||
|
||||
$result = $retire();
|
||||
$assert($result['retired'] === true && $result['task_id'] === '101'
|
||||
&& $result['host_id'] === '1' && $result['vpc_id'] === '20'
|
||||
&& $result['resolved_generations'] === 2,
|
||||
'retirement result must identify the exact converged chain');
|
||||
$assert(Capsule::table('cloud_vpc_host_states')->where('id', 40)
|
||||
->doesntExist(), 'retired host projection must be deleted');
|
||||
|
||||
$resolved = CloudJob::query()->whereIn('id', [100, 101])
|
||||
->orderBy('id')->get();
|
||||
$assert($resolved->count() === 2, 'repair chain audit rows must be retained');
|
||||
foreach ($resolved as $generation) {
|
||||
$assert((string) $generation->outcome === 'failed'
|
||||
&& (string) $generation->status === 'failed'
|
||||
&& (string) $generation->effect_resolution === 'applied'
|
||||
&& (int) $generation->effect_resolved_by === 99
|
||||
&& str_starts_with(
|
||||
(string) $generation->effect_resolution_reason,
|
||||
'host_permanently_retired:'
|
||||
), 'each failed generation must retain outcome and gain audit proof');
|
||||
}
|
||||
$knownNoEffectSibling = CloudJob::find(102);
|
||||
$assert($knownNoEffectSibling !== null
|
||||
&& (string) $knownNoEffectSibling->effect_resolution === 'not_applied'
|
||||
&& $knownNoEffectSibling->effect_resolved_by === null,
|
||||
'known no-effect repair sibling must retain its original conclusion');
|
||||
$port = Capsule::table('cloud_vpc_ports')->where('id', 30)->first();
|
||||
$vpc = Capsule::table('cloud_vpcs')->where('id', 20)->first();
|
||||
$assert($port !== null && $port->active_claim === null
|
||||
&& $port->state === 'released' && (int) $port->released_at > 0,
|
||||
'reference VPC port must finish its pending release');
|
||||
$assert($vpc !== null && $vpc->state === 'active',
|
||||
'VPC must converge after its last host projection is retired');
|
||||
|
||||
Capsule::table('cloud_instances')->insert([
|
||||
'id' => 12, 'uuid' => '11111111-1111-4111-8111-111111111112',
|
||||
'host_id' => 1, 'user_id' => 7, 'name' => 'batch-reference',
|
||||
'status' => 'deleted', 'service_status' => 'active',
|
||||
'network_status' => 'active', 'spec_json' => '{}',
|
||||
'created_at' => $now, 'updated_at' => $now,
|
||||
]);
|
||||
Capsule::table('cloud_vpcs')->insert([
|
||||
'id' => 21, 'vpc_key' => str_repeat('4', 32), 'user_id' => 7,
|
||||
'region_id' => 1, 'cidr_pool_id' => 1, 'name' => 'batch-retire-test',
|
||||
'cidr' => '10.20.0.0/24', 'prefix_length' => 24,
|
||||
'gateway' => '10.20.0.1', 'active_claim' => 1,
|
||||
'state' => 'degraded', 'desired_revision' => 4,
|
||||
'deleted_at' => null, 'created_at' => $now, 'updated_at' => $now,
|
||||
]);
|
||||
Capsule::table('cloud_vpc_ports')->insert([
|
||||
'id' => 32, 'port_key' => str_repeat('5', 32), 'vpc_id' => 21,
|
||||
'instance_id' => 12, 'migration_host_id' => null, 'user_id' => 7,
|
||||
'private_ip' => '10.20.0.2', 'mac_address' => '02:00:00:00:00:04',
|
||||
'active_claim' => 1, 'state' => 'release_pending',
|
||||
'desired_revision' => 4, 'applied_revision' => 3,
|
||||
'last_operation_id' => null, 'last_error_code' => '',
|
||||
'attached_at' => $now - 100, 'released_at' => null,
|
||||
'created_at' => $now, 'updated_at' => $now,
|
||||
]);
|
||||
$batchOperation = str_repeat('6', 32);
|
||||
Capsule::table('cloud_vpc_host_states')->insert([
|
||||
'id' => 41, 'vpc_id' => 21, 'host_id' => 1,
|
||||
'desired_revision' => 4, 'applied_revision' => 3,
|
||||
'desired_digest' => str_repeat('7', 64),
|
||||
'applied_digest' => str_repeat('8', 64), 'sync_state' => 'unknown',
|
||||
'last_operation_id' => $batchOperation,
|
||||
'last_error_code' => 'operation_effect_unknown',
|
||||
'last_success_at' => $now - 100, 'last_failure_at' => $now - 10,
|
||||
'created_at' => $now, 'updated_at' => $now,
|
||||
]);
|
||||
$batchPrefix = CloudVpcPortService::batchProjectionRequestPrefix(50, 4);
|
||||
$parent = array_replace($job(200, str_repeat('2', 32), null, null), [
|
||||
'request_idempotency_key' => str_repeat('2', 32),
|
||||
'instance_id' => null, 'vpc_id' => 21, 'vpc_port_id' => null,
|
||||
'vpc_batch_id' => null, 'host_id' => null,
|
||||
'kind' => 'vpc_membership_batch', 'execution_mode' => 'local',
|
||||
'phase' => 'accepted', 'outcome' => null, 'status' => 'queued',
|
||||
'attempts' => 0, 'dispatch_attempts' => 0,
|
||||
'dispatch_deadline_at' => $now + 3600,
|
||||
'dispatch_started_at' => null, 'agent_accepted_at' => null,
|
||||
'receipt_deadline_at' => null, 'terminal_at' => null,
|
||||
'effect_may_have_applied' => null, 'safe_to_resubmit' => null,
|
||||
'effect_resolution' => 'not_applicable', 'dispatched_at' => null,
|
||||
'error_code' => null,
|
||||
]);
|
||||
$batchChild = array_replace(
|
||||
$job(201, $batchOperation, null, null),
|
||||
[
|
||||
'request_idempotency_key' => $batchPrefix
|
||||
. 'vpc_host_remove-1-' . $batchOperation,
|
||||
'instance_id' => 12, 'vpc_id' => 21, 'vpc_port_id' => 32,
|
||||
'vpc_batch_id' => 50,
|
||||
]
|
||||
);
|
||||
Capsule::table('cloud_jobs')->insert([$parent, $batchChild]);
|
||||
Capsule::table('cloud_vpc_membership_batches')->insert([
|
||||
'id' => 50, 'batch_key' => str_repeat('3', 32), 'vpc_id' => 21,
|
||||
'user_id' => 7, 'parent_job_id' => 200, 'kind' => 'detach',
|
||||
'state' => 'blocked', 'desired_revision' => 4, 'active_claim' => 1,
|
||||
'last_error_code' => 'vpc_result_uncertain', 'terminal_at' => null,
|
||||
'created_at' => $now, 'updated_at' => $now,
|
||||
]);
|
||||
Capsule::table('cloud_vpc_membership_batch_items')->insert([
|
||||
'id' => 60, 'batch_id' => 50, 'user_id' => 7, 'instance_id' => 12,
|
||||
'vpc_port_id' => 32, 'host_id' => 1, 'child_job_id' => null,
|
||||
'state' => 'succeeded', 'last_error_code' => '',
|
||||
'created_at' => $now, 'updated_at' => $now,
|
||||
]);
|
||||
$batchResult = CloudVpcHostRetirementService::retire(
|
||||
$actor,
|
||||
201,
|
||||
'The second physical node projection was retired after the batch was audited.'
|
||||
);
|
||||
$retiredBatch = Capsule::table('cloud_vpc_membership_batches')
|
||||
->where('id', 50)->first();
|
||||
$retiredParent = CloudJob::find(200);
|
||||
$retiredBatchPort = Capsule::table('cloud_vpc_ports')
|
||||
->where('id', 32)->first();
|
||||
$assert($batchResult['retired'] === true
|
||||
&& $batchResult['resolved_generations'] === 1,
|
||||
'blocked batch host retirement must resolve one exact child generation');
|
||||
$assert($retiredBatch !== null && $retiredBatch->state === 'succeeded'
|
||||
&& $retiredBatch->active_claim === null
|
||||
&& (int) $retiredBatch->terminal_at > 0,
|
||||
'blocked membership batch must reach a terminal success state');
|
||||
$assert($retiredParent !== null
|
||||
&& (string) $retiredParent->phase === 'terminal'
|
||||
&& (string) $retiredParent->outcome === 'succeeded',
|
||||
'membership batch parent task must converge with the retired child');
|
||||
$assert($retiredBatchPort !== null
|
||||
&& $retiredBatchPort->active_claim === null
|
||||
&& $retiredBatchPort->state === 'released',
|
||||
'batch release-pending port must be released after host retirement');
|
||||
|
||||
$root = dirname(__DIR__);
|
||||
$routes = file_get_contents($root . '/config/routes_cloud.php');
|
||||
$controller = file_get_contents(
|
||||
$root . '/app/Controllers/CloudAdminOperationsController.php'
|
||||
);
|
||||
$recovery = file_get_contents(
|
||||
$root . '/app/Services/Cloud/CloudAdminOperationRecoveryService.php'
|
||||
);
|
||||
$script = file_get_contents($root . '/public/cloud/admin-operations.js');
|
||||
$view = file_get_contents(
|
||||
$root . '/resources/views/venix/admin/cloud/operations.tpl'
|
||||
);
|
||||
$assert(is_string($routes) && is_string($controller)
|
||||
&& is_string($recovery) && is_string($script) && is_string($view)
|
||||
&& str_contains($routes, '/retire-vpc-host')
|
||||
&& str_contains($controller, 'function retireVpcHost(')
|
||||
&& str_contains($recovery, 'function retireVpcHost(')
|
||||
&& str_contains($script, 'async function retireVpcHost(')
|
||||
&& str_contains($view, 'operation-detail-retire-vpc-host'),
|
||||
'permanent host-retirement route, controller, UI, or service wiring is incomplete');
|
||||
|
||||
echo "CloudVpcHostRetirementServiceTest passed\n";
|
||||
}
|
||||
Loading…
Reference in New Issue