@ -10,6 +10,8 @@ use App\Models\CloudJob;
use App\Models\CloudResourcePool;
use App\Models\CloudVpc;
use App\Models\CloudVpcHostState;
use App\Models\CloudVpcMembershipBatch;
use App\Models\CloudVpcMembershipBatchItem;
use App\Models\CloudVpcPort;
/**
@ -802,6 +804,40 @@ final class CloudVpcPortService
|| ($body['operation_id'] ?? null) !== (string) $job->operation_id) {
throw new CloudError('VPC 任务与资源账本不一致', 409);
}
$repair = $job->supersedes_job_id !== null;
$prior = null;
if ($repair) {
$prior = CloudJob::query()->where(
'id', (int) $job->supersedes_job_id
)->first();
$rootId = (int) ($job->vpc_repair_root_job_id ?? 0);
if ($prior === null || $rootId < 1
|| ($payload['vpc_repair'] ?? null) !== true
|| (int) ($payload['vpc_repair_root_job_id'] ?? 0)
!== $rootId
|| ($body['vpc_repair'] ?? null) !== true
|| ($body['supersedes_operation_id'] ?? null)
!== (string) $prior->operation_id
|| (int) $prior->instance_id !== (int) $job->instance_id
|| (int) $prior->vpc_id !== (int) $job->vpc_id
|| (int) $prior->vpc_port_id !== (int) $job->vpc_port_id
|| (int) $prior->host_id !== (int) $job->host_id
|| (string) $prior->kind !== $kind
|| (string) $prior->phase !== 'terminal'
|| (string) $prior->outcome !== 'failed'
|| !(bool) $prior->effect_may_have_applied
|| (bool) $prior->safe_to_resubmit
|| (string) $prior->effect_resolution !== 'unknown'
|| (bool) $prior->finalization_incident
|| (int) ($prior->vpc_repair_root_job_id ?? $prior->id)
!== $rootId) {
throw new CloudError('VPC 修复任务继任账本不一致', 409);
}
} elseif (($payload['vpc_repair'] ?? false) === true
|| array_key_exists('vpc_repair', $body)
|| array_key_exists('supersedes_operation_id', $body)) {
throw new CloudError('普通 VPC 任务包含修复字段', 409);
}
if (in_array($kind, ['vpc_host_apply', 'vpc_host_remove'], true)) {
$state = CloudVpcHostState::query()
->where('vpc_id', (int) $vpc->id)
@ -830,12 +866,18 @@ final class CloudVpcPortService
}
$expected = self::portBody($job, $vpc, $port, $state);
}
if ($repair) {
$expected['vpc_repair'] = true;
$expected['supersedes_operation_id'] =
(string) $prior->operation_id;
}
$expectedTarget = in_array($kind,
['vpc_host_apply', 'vpc_host_remove'], true)
? '/v1/vpc/networks/' . rawurlencode((string) $vpc->vpc_key)
. ($kind === 'vpc_host_apply' ? '/apply' : '/remove')
. ($repair ? '/repair' : '')
: '/v1/vms/' . rawurlencode(strtolower((string) $instance->uuid))
. '/vpc-port';
. '/vpc-port' . ($repair ? '/repair' : '') ;
$expectedMethod = match ($kind) {
'vpc_host_apply', 'vpc_host_remove', 'vpc_port_attach' => 'POST',
'vpc_port_detach' => 'DELETE',
@ -882,6 +924,16 @@ final class CloudVpcPortService
if ($port === null || $vpc === null || $instance === null) {
throw new CloudError('VPC 任务资源账本已丢失', 409);
}
if ($job->supersedes_job_id !== null) {
if ($status === 'succeeded') {
self::settleRepairChainLocked($job);
} elseif ($status === 'failed') {
self::restoreRepairPredecessorLocked(
$job, $vpc, $port
);
return;
}
}
if ($job->vpc_batch_id !== null) {
CloudVpcMembershipBatchService::finishChildLocked(
$job, $status, $result, $operation, $vpc, $port, $instance
@ -904,6 +956,134 @@ final class CloudVpcPortService
);
}
/** Resolve only VPC safety metadata after the successor proves convergence. */
private static function settleRepairChainLocked(CloudJob $job): void
{
$rootId = (int) ($job->vpc_repair_root_job_id ?? 0);
$priorId = (int) ($job->supersedes_job_id ?? 0);
if ($rootId < 1 | | $ priorId < 1 ) {
throw new CloudError('VPC 修复继任链缺失', 409);
}
$seen = [(int) $job->id => true];
$now = time();
for ($depth = 0; $depth < 32 ; $ depth + + ) {
if ($priorId < 1 | | isset ( $ seen [ $ priorId ] ) ) {
throw new CloudError('VPC 修复继任链包含循环', 409);
}
$seen[$priorId] = true;
$prior = CloudJob::query()->where('id', $priorId)
->lockForUpdate()->first();
if ($prior === null
|| (string) $prior->kind !== (string) $job->kind
|| (int) $prior->instance_id !== (int) $job->instance_id
|| (int) $prior->vpc_id !== (int) $job->vpc_id
|| (int) $prior->vpc_port_id !== (int) $job->vpc_port_id
|| (int) $prior->host_id !== (int) $job->host_id
|| (string) $prior->phase !== 'terminal'
|| (string) $prior->outcome !== 'failed'
|| !(bool) $prior->effect_may_have_applied
|| (bool) $prior->safe_to_resubmit
|| (string) $prior->effect_resolution !== 'unknown'
|| (bool) $prior->finalization_incident
|| (int) ($prior->vpc_repair_root_job_id ?? $prior->id)
!== $rootId) {
throw new CloudError('VPC 修复继任链已变化', 409);
}
$nextId = (int) ($prior->supersedes_job_id ?? 0);
$prior->effect_resolution = 'applied';
$prior->effect_resolved_at = $now;
$prior->effect_resolved_by = null;
$prior->effect_resolution_reason = substr(
'superseded_by_vpc_repair:' . (int) $job->id . ':'
. (string) $job->operation_id,
0,
500
);
$prior->available_at = 0;
$prior->read_at = null;
$prior->updated_at = $now;
$prior->save();
if ($nextId < 1 ) {
if ((int) $prior->id !== $rootId) {
throw new CloudError('VPC 修复根任务身份不一致', 409);
}
return;
}
$priorId = $nextId;
}
throw new CloudError('VPC 修复继任链超过安全上限', 409);
}
/** A proved no-effect repair restores the still-fenced predecessor tip. */
private static function restoreRepairPredecessorLocked(
CloudJob $job,
CloudVpc $vpc,
CloudVpcPort $port
): void {
$prior = CloudJob::query()->where(
'id', (int) $job->supersedes_job_id
)->lockForUpdate()->first();
if ($prior === null
|| (string) $prior->kind !== (string) $job->kind
|| (string) $prior->effect_resolution !== 'unknown'
|| (string) $prior->phase !== 'terminal'
|| (string) $prior->outcome !== 'failed') {
throw new CloudError('VPC 修复前序任务已变化', 409);
}
$now = time();
if (in_array((string) $job->kind,
['vpc_host_apply', 'vpc_host_remove'], true)) {
$state = CloudVpcHostState::query()
->where('vpc_id', (int) $job->vpc_id)
->where('host_id', (int) $job->host_id)
->lockForUpdate()->first();
if ($state === null || !hash_equals(
(string) $state->last_operation_id,
(string) $job->operation_id
)) {
throw new CloudError('VPC 宿主机修复回退代次不一致', 409);
}
$state->last_operation_id = (string) $prior->operation_id;
$state->sync_state = 'unknown';
$state->last_error_code = 'vpc_result_uncertain';
$state->last_failure_at = $now;
$state->updated_at = $now;
$state->save();
} else {
if (!hash_equals(
(string) $port->last_operation_id,
(string) $job->operation_id
)) {
throw new CloudError('VPC 网卡修复回退代次不一致', 409);
}
$port->last_operation_id = (string) $prior->operation_id;
$port->last_error_code = 'vpc_result_uncertain';
$port->updated_at = $now;
$port->save();
}
if ($job->vpc_batch_id !== null
& & in_array((string) $job->kind,
['vpc_port_attach', 'vpc_port_detach'], true)) {
$item = CloudVpcMembershipBatchItem::query()
->where('batch_id', (int) $job->vpc_batch_id)
->where('instance_id', (int) $job->instance_id)
->where('vpc_port_id', (int) $job->vpc_port_id)
->lockForUpdate()->first();
if ($item === null
|| (int) $item->child_job_id !== (int) $job->id) {
throw new CloudError('VPC 批次修复回退代次不一致', 409);
}
$item->child_job_id = (int) $prior->id;
$item->state = 'unknown';
$item->last_error_code = 'vpc_result_uncertain';
$item->updated_at = $now;
$item->save();
}
$vpc->state = 'degraded';
$vpc->updated_at = $now;
$vpc->save();
}
public static function markJobUncertain(CloudJob $job): void
{
$kind = (string) $job->kind;
@ -946,6 +1126,296 @@ final class CloudVpcPortService
}
}
/**
* Queue one new, explicitly linked repair generation for an exact VPC
* operation whose Agent journal is reachable but still inconclusive.
* The failed operation is never replayed.
*/
public static function repairUnknownEffect(
array $actor,
int $jobId,
array $inspection
): CloudJob {
CloudAccess::manage($actor);
$reference = CloudJob::query()->where('id', $jobId)->first();
if ($reference === null) {
throw new CloudError('任务不存在', 404);
}
if (!in_array((string) $reference->kind, self::JOB_KINDS, true)
|| (int) ($reference->instance_id ?? 0) < 1
|| (int) ($reference->vpc_id ?? 0) < 1
|| (int) ($reference->vpc_port_id ?? 0) < 1
|| (int) ($reference->host_id ?? 0) < 1 ) {
throw new CloudError('该任务不是可修复的 VPC 操作', 409);
}
self::assertRepairInspection($reference, $inspection);
return (new CloudJob())->getConnection()->transaction(
function () use ($reference, $inspection): CloudJob {
$instance = CloudVpcLockOrder::instance(
(int) $reference->instance_id
);
$vpc = CloudVpcLockOrder::vpcById(
(int) $reference->vpc_id
);
$port = CloudVpcLockOrder::portById(
(int) $reference->vpc_port_id
);
$state = CloudVpcHostState::query()
->where('vpc_id', (int) $reference->vpc_id)
->where('host_id', (int) $reference->host_id)
->lockForUpdate()->first();
if ($instance === null || $vpc === null || $port === null
|| $state === null
|| (int) $port->instance_id !== (int) $instance->id
|| (int) $port->vpc_id !== (int) $vpc->id
|| (int) $reference->host_id
!== (int) $state->host_id) {
throw new CloudError('VPC 修复资源账本不完整', 409);
}
$batch = null;
$item = null;
if ($reference->vpc_batch_id !== null) {
$batch = CloudVpcMembershipBatch::query()
->where('id', (int) $reference->vpc_batch_id)
->lockForUpdate()->first();
if ($batch === null
|| (string) $batch->state !== 'blocked'
|| (int) $batch->vpc_id !== (int) $vpc->id) {
throw new CloudError('VPC 批次不处于可修复状态', 409);
}
if (in_array((string) $reference->kind,
['vpc_port_attach', 'vpc_port_detach'], true)) {
$item = CloudVpcMembershipBatchItem::query()
->where('batch_id', (int) $batch->id)
->where('instance_id', (int) $instance->id)
->where('vpc_port_id', (int) $port->id)
->lockForUpdate()->first();
}
}
$prior = CloudJob::query()->where(
'id', (int) $reference->id
)->lockForUpdate()->first();
if ($prior === null) {
throw new CloudError('待修复任务已不存在', 409);
}
self::assertRepairCandidateLocked(
$prior, $instance, $vpc, $port, $state,
$batch, $item, $inspection
);
$rootId = (int) ($prior->vpc_repair_root_job_id
?? $prior->id);
$chain = CloudJob::query()
->where(static function ($query) use ($rootId): void {
$query->where('id', $rootId)
->orWhere('vpc_repair_root_job_id', $rootId);
})->orderBy('id')->lockForUpdate()->get();
$unknownGenerations = $chain->filter(
static fn (CloudJob $candidate): bool =>
(string) $candidate->effect_resolution === 'unknown'
);
$latestUnknown = $unknownGenerations->last();
if ($chain->isEmpty() || $unknownGenerations->count() > 32
|| $chain->contains(static fn (CloudJob $candidate): bool =>
(string) $candidate->phase !== 'terminal')
|| !$latestUnknown instanceof CloudJob
|| (int) $latestUnknown->id !== (int) $prior->id) {
throw new CloudError('VPC 修复链已有其他活动代次', 409);
}
foreach ($chain as $candidate) {
if ((int) $candidate->instance_id !== (int) $instance->id
|| (int) $candidate->vpc_id !== (int) $vpc->id
|| (int) $candidate->vpc_port_id !== (int) $port->id
|| (int) $candidate->host_id !== (int) $prior->host_id
|| (string) $candidate->kind
!== (string) $prior->kind) {
throw new CloudError('VPC 修复链资源身份不一致', 409);
}
}
$payload = CloudSecrets::decrypt(
(string) $prior->sealed_payload, 'job-payload'
);
self::assertJobRunnable($prior, $payload);
$job = self::baseJob(
(string) $prior->kind,
$vpc,
$port,
(int) $prior->host_id,
$prior->vpc_batch_id === null
? null : (int) $prior->vpc_batch_id
);
$job->supersedes_job_id = (int) $prior->id;
$job->vpc_repair_root_job_id = $rootId;
$extra = [];
foreach ([
'maintenance_migration_id', 'vpc_migration_direction',
'vpc_batch_id', 'vpc_batch_revision',
] as $key) {
if (array_key_exists($key, $payload)) {
$extra[$key] = $payload[$key];
}
}
if ((string) $prior->kind === 'vpc_host_apply'
& & is_array($payload['host_projection'] ?? null)
& & !array_is_list($payload['host_projection'])) {
$extra['host_projection'] = $payload['host_projection'];
}
if ($job->vpc_batch_id !== null
& & in_array((string) $job->kind,
['vpc_host_apply', 'vpc_host_remove'], true)) {
$revision = (int) ($extra['vpc_batch_revision']
?? $batch?->desired_revision ?? 0);
if ($revision < 1 ) {
throw new CloudError('VPC 批次修复缺少投影代次', 409);
}
$job->request_idempotency_key = substr(
self::batchProjectionRequestPrefix(
(int) $job->vpc_batch_id,
$revision
) . (string) $job->kind . '-'
. (int) $job->host_id . '-repair-'
. (string) $job->operation_id,
0,
128
);
}
if (in_array((string) $job->kind,
['vpc_host_apply', 'vpc_host_remove'], true)) {
$body = (string) $job->kind === 'vpc_host_apply'
? self::hostApplyBody($job, $vpc, $state, $payload)
: self::hostRemoveBody($job, $vpc, $state);
$target = '/v1/vpc/networks/'
. rawurlencode((string) $vpc->vpc_key)
. ((string) $job->kind === 'vpc_host_apply'
? '/apply/repair' : '/remove/repair');
$method = 'POST';
} else {
if (!self::hostReady($state)) {
throw new CloudError(
'VPC 网卡修复所需宿主机路由未就绪', 409
);
}
$body = self::portBody($job, $vpc, $port, $state);
$target = '/v1/vms/' . rawurlencode(
strtolower((string) $instance->uuid)
) . '/vpc-port/repair';
$method = (string) $job->kind === 'vpc_port_attach'
? 'POST' : 'DELETE';
}
$body['vpc_repair'] = true;
$body['supersedes_operation_id'] =
(string) $prior->operation_id;
self::sealJob(
$job, $vpc, $port, $method, $target, $body,
$extra + [
'vpc_repair' => true,
'vpc_repair_root_job_id' => $rootId,
]
);
if (in_array((string) $job->kind,
['vpc_host_apply', 'vpc_host_remove'], true)) {
$state->last_operation_id = (string) $job->operation_id;
$state->sync_state = 'pending';
$state->last_error_code = '';
$state->updated_at = time();
$state->save();
} else {
$port->last_operation_id = (string) $job->operation_id;
$port->last_error_code = '';
$port->updated_at = time();
$port->save();
if ($item !== null) {
$item->child_job_id = (int) $job->id;
$item->state = 'queued';
$item->last_error_code = '';
$item->updated_at = time();
$item->save();
}
}
return $job;
},
3
);
}
private static function assertRepairInspection(
CloudJob $job,
array $inspection
): void {
if (($inspection['operation_id'] ?? null)
!== (string) $job->operation_id
|| ($inspection['journal_status'] ?? null) !== 'failed'
|| ($inspection['outcome'] ?? null) !== 'inconclusive'
|| ($inspection['effect_resolution'] ?? null) !== 'inconclusive'
|| ($inspection['fence_state'] ?? null) !== 'active'
|| ($inspection['fence_resolution_required'] ?? null) !== true
|| array_key_exists('completion_delivery_state', $inspection)
|| !is_int($inspection['checked_at'] ?? null)
|| $inspection['checked_at'] < time ( ) - 120
|| $inspection['checked_at'] > time() + 5) {
throw new CloudError(
'宿主机没有提供可执行 VPC 修复的新鲜不一致证据', 409
);
}
}
private static function assertRepairCandidateLocked(
CloudJob $job,
CloudInstance $instance,
CloudVpc $vpc,
CloudVpcPort $port,
CloudVpcHostState $state,
?CloudVpcMembershipBatch $batch,
?CloudVpcMembershipBatchItem $item,
array $inspection
): void {
self::assertRepairInspection($job, $inspection);
if (!in_array((string) $job->kind, self::JOB_KINDS, true)
|| (string) $job->execution_mode !== 'agent'
|| (string) $job->phase !== 'terminal'
|| (string) $job->outcome !== 'failed'
|| (string) $job->status !== 'failed'
|| (int) $job->dispatch_attempts !== 1
|| !(bool) $job->effect_may_have_applied
|| (bool) $job->safe_to_resubmit
|| (string) $job->effect_resolution !== 'unknown'
|| (bool) $job->finalization_incident
|| (int) $job->instance_id !== (int) $instance->id
|| (int) $job->vpc_id !== (int) $vpc->id
|| (int) $job->vpc_port_id !== (int) $port->id
|| (int) $job->host_id !== (int) $state->host_id
|| (int) $job->user_id !== (int) $port->user_id) {
throw new CloudError('VPC 任务已不处于可修复状态', 409);
}
if (in_array((string) $job->kind,
['vpc_host_apply', 'vpc_host_remove'], true)) {
if (!hash_equals(
(string) $state->last_operation_id,
(string) $job->operation_id
) || (string) $state->sync_state !== 'unknown') {
throw new CloudError('VPC 宿主机修复代次已过期', 409);
}
} elseif (!hash_equals(
(string) $port->last_operation_id,
(string) $job->operation_id
) || !in_array((string) $port->state,
['preparing', 'detaching'], true)) {
throw new CloudError('VPC 网卡修复代次已过期', 409);
}
if ($job->vpc_batch_id !== null
& & ($batch === null || (string) $batch->state !== 'blocked'
|| (in_array((string) $job->kind,
['vpc_port_attach', 'vpc_port_detach'], true)
& & ($item === null
|| (int) $item->child_job_id !== (int) $job->id
|| (string) $item->state !== 'unknown')))) {
throw new CloudError('VPC 批次修复子任务已过期', 409);
}
}
private static function finishHostJobLocked(CloudJob $job,
string $status, $result, array $operation, CloudVpc $vpc,
CloudVpcPort $port, CloudInstance $instance): void