diff --git a/web/app/Services/Cloud/CloudAdminInstanceService.php b/web/app/Services/Cloud/CloudAdminInstanceService.php index 22d96828..10ef27da 100644 --- a/web/app/Services/Cloud/CloudAdminInstanceService.php +++ b/web/app/Services/Cloud/CloudAdminInstanceService.php @@ -175,6 +175,8 @@ final class CloudAdminInstanceService && ($spec['os_source'] ?? null) === 'cloud_template'; $canMutate = $row['instance_status'] === 'active' && $instance->serviceAvailable($now); + $networkMutation = + CloudPublicNetworkMutationAdmission::inspect($instance); $row += [ 'public_ip_cidr' => $publicIpCidr, 'public_ipv4' => self::ip($spec['public_ipv4'] ?? null) ?? '', @@ -190,7 +192,7 @@ final class CloudAdminInstanceService 'bandwidth_mbps' => self::positiveOrNull($spec['bandwidth_mbps'] ?? null), 'os_source' => self::text($spec['os_source'] ?? null, 32), 'login_name' => self::text($spec['login_name'] ?? null, 32), - 'network_change_supported' => $canMutate + 'network_change_supported' => $networkMutation->allowed() && $managedCloudTemplate && $networkValid, 'instance_settings_supported' => $canMutate && $managedCloudTemplate && $networkValid, diff --git a/web/app/Services/Cloud/CloudNetworkChangeOptionsService.php b/web/app/Services/Cloud/CloudNetworkChangeOptionsService.php index 2e01681f..accc8169 100644 --- a/web/app/Services/Cloud/CloudNetworkChangeOptionsService.php +++ b/web/app/Services/Cloud/CloudNetworkChangeOptionsService.php @@ -6,10 +6,8 @@ use App\Models\CloudHost; use App\Models\CloudHostAgent; use App\Models\CloudHostNetworkAttachment; use App\Models\CloudInstance; -use App\Models\CloudJob; use App\Models\CloudPublicIp; use App\Models\CloudPublicIpPool; -use App\Models\CloudPublicNetworkTransition; use App\Models\CloudResourcePool; /** Read-only, dual-stack network-change capability projection. */ @@ -383,36 +381,19 @@ final class CloudNetworkChangeOptionsService private static function availability(CloudInstance $instance, string $configurationState): array { - $activeJob = CloudJobService::blockingQueryForInstance( - (int) $instance->id - )->orderByDesc('id')->first([ - 'id', 'kind', 'status', 'updated_at', - ]); - $activeTransition = CloudPublicNetworkTransition::where( - 'instance_id', - (int) $instance->id - )->where('active_transition', 1)->exists(); - $active = (string) $instance->status === 'active' - && $instance->serviceAvailable(); - $canChange = $active && $activeJob === null && !$activeTransition + $decision = CloudPublicNetworkMutationAdmission::inspect($instance); + $canChange = $decision->allowed() && $configurationState === 'ready'; $message = match (true) { $configurationState === 'invalid' => '面板公网地址池配置异常,请先修复双栈地址账本。', - !$active => '实例服务不可用,暂不能更换公网地址。', - $activeJob !== null || $activeTransition => - '实例已有任务,请等待完成。', + !$decision->allowed() => $decision->message, $configurationState !== 'ready' => '当前公网地址池没有可用目标地址。', default => '', }; return [ - 'active_job' => $activeJob instanceof CloudJob ? [ - 'id' => (string) $activeJob->id, - 'kind' => (string) $activeJob->kind, - 'status' => (string) $activeJob->status, - 'updated_at' => max(0, (int) $activeJob->updated_at), - ] : null, + 'active_job' => $decision->activeJob, 'can_change' => $canChange, 'message' => $message, ]; diff --git a/web/app/Services/Cloud/CloudPublicIpPoolReplacementBatchService.php b/web/app/Services/Cloud/CloudPublicIpPoolReplacementBatchService.php index 46d87f91..8097b2f1 100644 --- a/web/app/Services/Cloud/CloudPublicIpPoolReplacementBatchService.php +++ b/web/app/Services/Cloud/CloudPublicIpPoolReplacementBatchService.php @@ -53,7 +53,7 @@ final class CloudPublicIpPoolReplacementBatchService || (int) $sourceAttachment->host_id !== $hostId) { throw new CloudError('源公网地址池不存在', 404); } - $assignedCount = CloudResourceReservationIp::query() + $sourceInstanceIds = CloudResourceReservationIp::query() ->join( 'cloud_resource_reservations as reservation', 'reservation.id', @@ -69,7 +69,31 @@ final class CloudPublicIpPoolReplacementBatchService )->where('cloud_resource_reservation_ips.active_claim', 1) ->where('cloud_resource_reservation_ips.state', 'committed') ->where('reservation.state', 'committed') - ->whereNotNull('reservation.instance_id')->count(); + ->whereNotNull('reservation.instance_id') + ->orderBy('reservation.instance_id') + ->pluck('reservation.instance_id')->map( + static fn ($id): int => (int) $id + )->unique()->values()->all(); + $assignedCount = count($sourceInstanceIds); + $admissionMessage = ''; + if ($sourceInstanceIds !== []) { + $instances = CloudInstance::whereIn('id', $sourceInstanceIds) + ->orderBy('id')->get()->keyBy('id'); + if ($instances->count() !== $assignedCount) { + $admissionMessage = + '公网网络变更运行态目录不可用,请联系管理员。'; + } else { + foreach ( + CloudPublicNetworkMutationAdmission::inspectMany($instances) + as $decision + ) { + if (!$decision->allowed()) { + $admissionMessage = $decision->message; + break; + } + } + } + } $activeBatches = CloudPublicIpPoolReplacementBatch::where( 'host_id', $hostId )->where('active_claim', 1)->where(function ($query) use ( @@ -89,6 +113,21 @@ final class CloudPublicIpPoolReplacementBatchService (int) $batch->source_public_ip_pool_id, (int) $batch->target_public_ip_pool_id, ])->unique()->values()->all(); + $sourceReady = $agent instanceof CloudHostAgent + && in_array((string) $source->state, + ['active', 'draining'], true) + && (string) $source->sync_state === 'applied' + && (int) $source->observed_revision + === (int) $source->desired_revision + && CloudPlacementNetworkService::attachmentReadyForAgent( + $sourceAttachment, + $agent + ) + && CloudAgentSettingsService::publicNetworkReady( + $agent, + (string) $sourceAttachment->network_selector, + (int) $sourceAttachment->verified_agent_revision + ); $targets = []; $pools = CloudPublicIpPool::where('host_id', $hostId) ->where('id', '<>', (int) $source->id) @@ -98,7 +137,7 @@ final class CloudPublicIpPoolReplacementBatchService (int) $source->network_attachment_id ) ->where('state', 'active')->where('sync_state', 'applied') - ->whereColumn('observed_revision', '>=', 'desired_revision') + ->whereColumn('observed_revision', '=', 'desired_revision') ->orderBy('priority')->orderBy('id')->get(); foreach ($pools as $pool) { $attachment = CloudHostNetworkAttachment::find( @@ -110,7 +149,12 @@ final class CloudPublicIpPoolReplacementBatchService !== (int) $sourceAttachment->line_id || (int) $attachment->id !== (int) $sourceAttachment->id || !CloudPlacementNetworkService:: - attachmentReadyForAgent($attachment, $agent)) { + attachmentReadyForAgent($attachment, $agent) + || !CloudAgentSettingsService::publicNetworkReady( + $agent, + (string) $attachment->network_selector, + (int) $attachment->verified_agent_revision + )) { continue; } $available = self::availableCount($pool, $host); @@ -119,16 +163,15 @@ final class CloudPublicIpPoolReplacementBatchService 'name' => (string) $pool->name, 'available_count' => $available, 'network_selector' => (string) $attachment->network_selector, - 'expected_revision' => max( - (int) $pool->desired_revision, - (int) $pool->observed_revision - ), + 'expected_revision' => (int) $pool->desired_revision, 'replacement_locked' => in_array( (int) $pool->id, $lockedPoolIds, true ), 'eligible' => $assignedCount > 0 + && $sourceReady + && $admissionMessage === '' && $available >= $assignedCount && !in_array((int) $pool->id, $lockedPoolIds, true), ]; @@ -152,8 +195,8 @@ final class CloudPublicIpPoolReplacementBatchService } $canStart = $active === null && $assignedCount > 0 - && in_array((string) $source->state, - ['active', 'draining'], true) + && $sourceReady + && $admissionMessage === '' && count(array_filter($targets, static fn (array $target): bool => ($target['eligible'] ?? false) === true)) > 0; return [ @@ -167,10 +210,7 @@ final class CloudPublicIpPoolReplacementBatchService 'state' => (string) $source->state, 'network_selector' => (string) $sourceAttachment->network_selector, - 'expected_revision' => max( - (int) $source->desired_revision, - (int) $source->observed_revision - ), + 'expected_revision' => (int) $source->desired_revision, 'assigned_count' => $assignedCount, 'replacement_locked' => $active !== null, 'replacement_history_count' => $historyCount, @@ -186,7 +226,11 @@ final class CloudPublicIpPoolReplacementBatchService ? '该地址池已参与执行中的迁移批次' : ($assignedCount < 1 ? '源地址池没有需要迁移的实例' - : '当前没有容量足够的同线路目标地址池')), + : (!$sourceReady + ? '源公网地址池或网络接入尚未完成严格同步验证' + : ($admissionMessage !== '' + ? $admissionMessage + : '当前没有容量足够的同线路目标地址池')))), ]; } @@ -340,21 +384,9 @@ final class CloudPublicIpPoolReplacementBatchService if (count($reservationIds) !== count($instanceIds)) { throw new CloudError('源地址池实例缺少资源预留', 409); } - foreach ($instances as $instance) { - CloudInstanceMigrationService:: - assertInstanceMutationAvailable( - $instance, - 'network_change' - ); - if (CloudJobService::blockingQueryForInstance( - (int) $instance->id - )->exists()) { - throw new CloudError( - '源地址池中有实例正在执行其他任务', - 409 - ); - } - } + CloudPublicNetworkMutationAdmission::assertCanBeginMany( + $instances + ); $host = CloudHost::where('id', $hostId) ->lockForUpdate()->first(); $agent = CloudHostAgent::where('host_id', $hostId) @@ -455,19 +487,15 @@ final class CloudPublicIpPoolReplacementBatchService ['active', 'draining'], true) || (string) $sourcePool->sync_state !== 'applied' || (int) $sourcePool->observed_revision - < (int) $sourcePool->desired_revision + !== (int) $sourcePool->desired_revision || (string) $targetPool->state !== 'active' || (string) $targetPool->sync_state !== 'applied' || (int) $targetPool->observed_revision - < (int) $targetPool->desired_revision - || max( - (int) $sourcePool->desired_revision, - (int) $sourcePool->observed_revision - ) !== (int) $expectedSourceRevision - || max( - (int) $targetPool->desired_revision, - (int) $targetPool->observed_revision - ) !== (int) $expectedTargetRevision + !== (int) $targetPool->desired_revision + || (int) $sourcePool->desired_revision + !== (int) $expectedSourceRevision + || (int) $targetPool->desired_revision + !== (int) $expectedTargetRevision || (int) $sourceAttachment->host_id !== $hostId || (int) $targetAttachment->host_id !== $hostId || (int) $sourcePool->address_family @@ -479,7 +507,19 @@ final class CloudPublicIpPoolReplacementBatchService || (int) $sourceAttachment->line_id !== (int) $targetAttachment->line_id || !CloudPlacementNetworkService:: - attachmentReadyForAgent($targetAttachment, $agent)) { + attachmentReadyForAgent($sourceAttachment, $agent) + || !CloudPlacementNetworkService:: + attachmentReadyForAgent($targetAttachment, $agent) + || !CloudAgentSettingsService::publicNetworkReady( + $agent, + (string) $sourceAttachment->network_selector, + (int) $sourceAttachment->verified_agent_revision + ) + || !CloudAgentSettingsService::publicNetworkReady( + $agent, + (string) $targetAttachment->network_selector, + (int) $targetAttachment->verified_agent_revision + )) { throw new CloudError( '源或目标公网地址池不满足同地址族同网络接入的迁移条件', 409 @@ -557,8 +597,6 @@ final class CloudPublicIpPoolReplacementBatchService ); if ($instance === null || $reservation === null || $resourcePool === null - || (string) $instance->status !== 'active' - || !$instance->serviceAvailable() || (int) $instance->host_id !== $hostId || !is_array($spec) || array_is_list($spec) || ($spec['managed'] ?? false) !== true diff --git a/web/app/Services/Cloud/CloudPublicNetworkMutationAdmission.php b/web/app/Services/Cloud/CloudPublicNetworkMutationAdmission.php new file mode 100644 index 00000000..d5318e37 --- /dev/null +++ b/web/app/Services/Cloud/CloudPublicNetworkMutationAdmission.php @@ -0,0 +1,190 @@ +id]; + } + + /** + * Inspect an arbitrary number of instances with a fixed four-query + * catalog projection. Results are keyed by instance id. + * + * @param iterable $instances + * @return array + */ + public static function inspectMany(iterable $instances): array + { + $byId = []; + foreach ($instances as $instance) { + if (!$instance instanceof CloudInstance) { + throw new \InvalidArgumentException( + 'Public-network mutation admission requires instances' + ); + } + $byId[(int) ($instance->id ?? 0)] = $instance; + } + ksort($byId, SORT_NUMERIC); + + $decisions = []; + $candidateIds = []; + foreach ($byId as $instanceId => $instance) { + if ($instanceId < 1 + || (string) ($instance->status ?? '') !== 'active') { + $decisions[$instanceId] = self::deny( + CloudPublicNetworkMutationDecision::INSTANCE_INACTIVE, + '只有处于活动状态的实例可以更换公网地址。' + ); + } elseif (!$instance->serviceAvailable()) { + $decisions[$instanceId] = self::deny( + CloudPublicNetworkMutationDecision::SERVICE_UNAVAILABLE, + '实例服务不可用,暂不能更换公网地址。' + ); + } else { + $candidateIds[] = $instanceId; + } + } + if ($candidateIds === []) { + return $decisions; + } + + try { + $migrations = CloudInstanceMigration::whereIn( + 'instance_id', $candidateIds + )->whereIn( + 'state', + CloudInstanceMigrationWorkflow::blockingStates() + )->pluck('instance_id')->mapWithKeys( + static fn ($id): array => [(int) $id => true] + )->all(); + $recoveries = CloudBackupRecovery::whereIn( + 'instance_id', $candidateIds + )->whereIn('state', ['recovering', 'uncertain']) + ->pluck('instance_id')->mapWithKeys( + static fn ($id): array => [(int) $id => true] + )->all(); + + $latestJobIds = CloudJobService::blockingQueryForInstances( + $candidateIds + )->selectRaw('MAX(id) AS id')->groupBy('instance_id')->toBase(); + $jobRows = DB::table('cloud_jobs as admission_job')->joinSub( + $latestJobIds, + 'latest_public_network_blocking_job', + 'latest_public_network_blocking_job.id', + '=', + 'admission_job.id' + )->get([ + 'admission_job.id', + 'admission_job.instance_id', + 'admission_job.kind', + 'admission_job.status', + 'admission_job.updated_at', + ]); + $jobs = []; + foreach ($jobRows as $job) { + $jobs[(int) $job->instance_id] = $job; + } + + $transitions = CloudPublicNetworkTransition::whereIn( + 'instance_id', $candidateIds + )->where('active_transition', 1)->pluck('instance_id') + ->mapWithKeys( + static fn ($id): array => [(int) $id => true] + )->all(); + } catch (\Throwable) { + foreach ($candidateIds as $instanceId) { + $decisions[$instanceId] = self::catalogUnavailable(); + } + ksort($decisions, SORT_NUMERIC); + return $decisions; + } + + foreach ($candidateIds as $instanceId) { + $job = $jobs[$instanceId] ?? null; + $decisions[$instanceId] = match (true) { + isset($migrations[$instanceId]) => self::deny( + CloudPublicNetworkMutationDecision::MIGRATION_ACTIVE, + '实例正在维护迁移,请先等待迁移收敛。' + ), + isset($recoveries[$instanceId]) => self::deny( + CloudPublicNetworkMutationDecision::BACKUP_RECOVERY_ACTIVE, + '虚拟机正在恢复备份,请等待完成。' + ), + $job !== null => new CloudPublicNetworkMutationDecision( + CloudPublicNetworkMutationDecision::JOB_ACTIVE, + '实例已有任务,请等待完成。', + [ + 'id' => (string) $job->id, + 'kind' => (string) $job->kind, + 'status' => (string) $job->status, + 'updated_at' => max(0, (int) $job->updated_at), + ] + ), + isset($transitions[$instanceId]) => self::deny( + CloudPublicNetworkMutationDecision::TRANSITION_ACTIVE, + '虚拟机已有公网网络变更正在处理。' + ), + default => new CloudPublicNetworkMutationDecision( + CloudPublicNetworkMutationDecision::READY, + '' + ), + }; + } + ksort($decisions, SORT_NUMERIC); + return $decisions; + } + + public static function assertCanBegin(CloudInstance $instance): void + { + $decision = self::inspect($instance); + if (!$decision->allowed()) { + throw new CloudError($decision->message, 409); + } + } + + /** @param iterable $instances */ + public static function assertCanBeginMany(iterable $instances): void + { + foreach (self::inspectMany($instances) as $decision) { + if (!$decision->allowed()) { + throw new CloudError($decision->message, 409); + } + } + } + + private static function deny( + string $reasonCode, + string $message + ): CloudPublicNetworkMutationDecision { + return new CloudPublicNetworkMutationDecision( + $reasonCode, + $message + ); + } + + private static function catalogUnavailable(): CloudPublicNetworkMutationDecision + { + return self::deny( + CloudPublicNetworkMutationDecision::CATALOG_UNAVAILABLE, + '公网网络变更运行态目录不可用,请联系管理员。' + ); + } +} diff --git a/web/app/Services/Cloud/CloudPublicNetworkMutationDecision.php b/web/app/Services/Cloud/CloudPublicNetworkMutationDecision.php new file mode 100644 index 00000000..ee974af9 --- /dev/null +++ b/web/app/Services/Cloud/CloudPublicNetworkMutationDecision.php @@ -0,0 +1,57 @@ +reasonCode === self::READY; + } +} diff --git a/web/app/Services/Cloud/CloudPublicNetworkTransitionWorkflow.php b/web/app/Services/Cloud/CloudPublicNetworkTransitionWorkflow.php index 65f213b6..03f23c15 100644 --- a/web/app/Services/Cloud/CloudPublicNetworkTransitionWorkflow.php +++ b/web/app/Services/Cloud/CloudPublicNetworkTransitionWorkflow.php @@ -286,6 +286,8 @@ final class CloudPublicNetworkTransitionWorkflow ); } + CloudPublicNetworkMutationAdmission::assertCanBegin($instance); + try { $spec = json_decode( (string) $instance->spec_json, @@ -296,9 +298,7 @@ final class CloudPublicNetworkTransitionWorkflow } catch (\Throwable) { $spec = null; } - if ((string) $instance->status !== 'active' - || !$instance->serviceAvailable() - || (int) ($instance->host_id ?? 0) < 1 + if ((int) ($instance->host_id ?? 0) < 1 || (int) ($instance->reservation_id ?? 0) < 1 || (int) ($instance->resource_pool_id ?? 0) < 1 || !is_array($spec) || array_is_list($spec) @@ -306,23 +306,6 @@ final class CloudPublicNetworkTransitionWorkflow || ($spec['os_source'] ?? null) !== 'cloud_template') { throw new CloudError('虚拟机不满足公网网络变更条件', 409); } - CloudInstanceMigrationService::assertInstanceMutationAvailable( - $instance - ); - if (DB::table('cloud_backup_recoveries') - ->where('instance_id', (int) $instance->id) - ->whereIn('state', ['recovering', 'uncertain'])->exists()) { - throw new CloudError('虚拟机正在恢复备份,请等待完成', 409); - } - // The instance row serializes all supported mutation writers. - // This is intentionally an unlocked predicate: Worker finalizers - // lock job before instance, so taking job locks here would reverse - // that global order and create a job/instance deadlock. - if (CloudJobService::blockingQueryForInstance( - (int) $instance->id - )->exists()) { - throw new CloudError('虚拟机已有任务,请等待完成', 409); - } [$host, $agent] = self::lockHostBoundaryLocked( (int) $instance->host_id diff --git a/web/tests/CloudAdminInstanceServiceTest.php b/web/tests/CloudAdminInstanceServiceTest.php index 3acd1b9d..164a324f 100644 --- a/web/tests/CloudAdminInstanceServiceTest.php +++ b/web/tests/CloudAdminInstanceServiceTest.php @@ -210,6 +210,27 @@ namespace { $table->integer('created_at'); $table->integer('updated_at'); }); + $schema->create('cloud_instance_migrations', static function ( + Blueprint $table + ): void { + $table->increments('id'); + $table->integer('instance_id'); + $table->string('state'); + }); + $schema->create('cloud_backup_recoveries', static function ( + Blueprint $table + ): void { + $table->increments('id'); + $table->integer('instance_id'); + $table->string('state'); + }); + $schema->create('cloud_public_network_transitions', static function ( + Blueprint $table + ): void { + $table->increments('id'); + $table->integer('instance_id'); + $table->integer('active_transition')->nullable(); + }); $schema->create('cloud_invoices', static function (Blueprint $table): void { $table->increments('id'); $table->string('invoice_no'); @@ -518,9 +539,9 @@ namespace { && $detail['instance']['active_job']['completed'] === false && $detail['instance']['active_job']['status'] === 'active' && $detail['instance']['can_manage'] === true - && $detail['instance']['network_change_supported'] === true + && $detail['instance']['network_change_supported'] === false && $detail['instance']['instance_settings_supported'] === true, - 'An operation submitted by a super-admin was lost because the job actor differs from the instance owner.'); + 'The administrator network control ignored the canonical mutation fence or changed unrelated settings capability.'); $insertJob(31, [ 'instance_id' => 213, 'outcome' => 'failed', 'effect_may_have_applied' => true, 'safe_to_resubmit' => false, @@ -716,6 +737,30 @@ namespace { && $unsupported['instance']['network_status'] === 'configuration_invalid', 'Incomplete legacy metadata exposed administrator mutation controls.'); + Capsule::table('cloud_instance_migrations')->insert([ + 'id' => 1, 'instance_id' => 205, 'state' => 'preparing_source', + ]); + $migrationFenced = CloudAdminInstanceService::detail($admin, '205'); + $assert($migrationFenced['instance']['network_change_supported'] === false + && $migrationFenced['instance']['instance_settings_supported'] === true, + 'Migration fence leaked into unrelated settings capability.'); + Capsule::table('cloud_instance_migrations')->delete(); + Capsule::table('cloud_backup_recoveries')->insert([ + 'id' => 1, 'instance_id' => 205, 'state' => 'uncertain', + ]); + $recoveryFenced = CloudAdminInstanceService::detail($admin, '205'); + $assert($recoveryFenced['instance']['network_change_supported'] === false + && $recoveryFenced['instance']['instance_settings_supported'] === true, + 'Backup recovery fence leaked into unrelated settings capability.'); + Capsule::table('cloud_backup_recoveries')->delete(); + Capsule::table('cloud_public_network_transitions')->insert([ + 'id' => 1, 'instance_id' => 205, 'active_transition' => 1, + ]); + $transitionFenced = CloudAdminInstanceService::detail($admin, '205'); + $assert($transitionFenced['instance']['network_change_supported'] === false + && $transitionFenced['instance']['instance_settings_supported'] === true, + 'Active transition fence leaked into unrelated settings capability.'); + Capsule::table('cloud_public_network_transitions')->delete(); Capsule::table('cloud_instances')->where('id', 205)->update(['status' => 'deleting']); $inactive = CloudAdminInstanceService::detail($admin, '205'); $assert($inactive['instance']['network_change_supported'] === false diff --git a/web/tests/CloudNetworkChangeOptionsServiceTest.php b/web/tests/CloudNetworkChangeOptionsServiceTest.php index 861208b0..4ca872be 100644 --- a/web/tests/CloudNetworkChangeOptionsServiceTest.php +++ b/web/tests/CloudNetworkChangeOptionsServiceTest.php @@ -1,31 +1,6 @@ increments('id'); $t->integer('instance_id'); $t->integer('active_transition')->nullable(); }); + $schema->create('cloud_instance_migrations', static function ( + Blueprint $t + ): void { + $t->increments('id'); $t->integer('instance_id'); + $t->string('state'); + }); + $schema->create('cloud_backup_recoveries', static function ( + Blueprint $t + ): void { + $t->increments('id'); $t->integer('instance_id'); + $t->string('state'); + }); + $schema->create('cloud_jobs', static function (Blueprint $t): void { + $t->increments('id'); + $t->string('operation_id')->nullable(); + $t->string('request_idempotency_key')->nullable(); + $t->string('request_fingerprint')->nullable(); + $t->integer('instance_id')->nullable(); + $t->integer('parent_job_id')->nullable(); + $t->integer('host_id')->nullable(); + $t->string('kind'); + $t->string('execution_mode')->nullable(); + $t->string('phase'); + $t->string('outcome')->nullable(); + $t->string('status'); + $t->text('sealed_payload')->nullable(); + $t->integer('attempts')->nullable(); + $t->integer('dispatch_attempts')->nullable(); + $t->integer('dispatch_deadline_at')->nullable(); + $t->integer('dispatch_started_at')->nullable(); + $t->integer('agent_accepted_at')->nullable(); + $t->integer('dispatched_at')->nullable(); + $t->integer('receipt_deadline_at')->nullable(); + $t->integer('terminal_at')->nullable(); + $t->boolean('effect_may_have_applied')->nullable(); + $t->boolean('safe_to_resubmit')->nullable(); + $t->string('effect_resolution')->nullable(); + $t->integer('effect_resolved_at')->nullable(); + $t->boolean('finalization_incident')->default(false); + $t->string('finalization_error_code')->nullable(); + $t->string('completion_event_id')->nullable(); + $t->integer('read_at')->nullable(); + $t->string('error_code')->nullable(); + $t->text('result_json')->nullable(); + $t->integer('available_at')->default(0); + $t->integer('created_at')->default(0); + $t->integer('updated_at'); + }); $now = time(); Capsule::table('cloud_hosts')->insert([ @@ -563,15 +585,47 @@ namespace { 'state' => 'available', ]); + Capsule::table('cloud_instance_migrations')->insert([ + 'id' => 1, 'instance_id' => 30, 'state' => 'preparing_source', + ]); + $migrationBusy = CloudNetworkChangeOptionsService::tenantQuote( + $instance->fresh() + ); + $assert(($migrationBusy['can_change'] ?? null) === false + && str_contains( + (string) ($migrationBusy['message'] ?? ''), + '维护迁移' + ), 'An active migration did not disable the network quote.'); + Capsule::table('cloud_instance_migrations')->delete(); + + Capsule::table('cloud_backup_recoveries')->insert([ + 'id' => 1, 'instance_id' => 30, 'state' => 'recovering', + ]); + $recoveryBusy = CloudNetworkChangeOptionsService::tenantQuote( + $instance->fresh() + ); + $assert(($recoveryBusy['can_change'] ?? null) === false + && str_contains( + (string) ($recoveryBusy['message'] ?? ''), + '恢复备份' + ), 'An active backup recovery did not disable the network quote.'); + Capsule::table('cloud_backup_recoveries')->delete(); + $job = new CloudJob(); - $job->id = 88; $job->kind = 'action'; $job->status = 'running'; - $job->updated_at = $now; - NetworkOptionsJobState::$job = $job; + $job->id = 88; + $job->instance_id = 30; + $job->host_id = 1; + $job->kind = 'action'; + $job->execution_mode = 'agent'; + $job->sealed_payload = '{}'; + $job->available_at = $now; + $job->created_at = $job->updated_at = $now; + $job->save(); $busy = CloudNetworkChangeOptionsService::tenantQuote($instance->fresh()); $assert(($busy['can_change'] ?? null) === false && ($busy['active_job']['id'] ?? null) === '88', 'An active instance job did not disable the network quote.'); - NetworkOptionsJobState::$job = null; + $job->delete(); Capsule::table('cloud_public_network_transitions')->insert([ 'id' => 1, 'instance_id' => 30, 'active_transition' => 1, ]); @@ -595,6 +649,22 @@ namespace { 'address_binary' => inet_pton('8.8.8.20'), ]); + $schema->drop('cloud_backup_recoveries'); + $missingCatalog = CloudNetworkChangeOptionsService::tenantQuote( + $instance->fresh() + ); + $assert(($missingCatalog['can_change'] ?? null) === false + && str_contains( + (string) ($missingCatalog['message'] ?? ''), + '运行态目录不可用' + ), 'A missing runtime catalog failed open in tenant options.'); + $schema->create('cloud_backup_recoveries', static function ( + Blueprint $t + ): void { + $t->increments('id'); $t->integer('instance_id'); + $t->string('state'); + }); + $broken = $publicNetwork; $broken['ipv4']['address_cidr'] = '8.8.8.99/24'; Capsule::table('cloud_instances')->where('id', 30)->update([ diff --git a/web/tests/CloudNetworkIpLedgerContractTest.php b/web/tests/CloudNetworkIpLedgerContractTest.php index 3f29ea9a..2ec343a6 100644 --- a/web/tests/CloudNetworkIpLedgerContractTest.php +++ b/web/tests/CloudNetworkIpLedgerContractTest.php @@ -20,6 +20,12 @@ $allocation = file_get_contents( $adminInstanceService = file_get_contents( $root . '/app/Services/Cloud/CloudAdminInstanceService.php' ); +$admission = file_get_contents( + $root . '/app/Services/Cloud/CloudPublicNetworkMutationAdmission.php' +); +$replacementBatch = file_get_contents( + $root . '/app/Services/Cloud/CloudPublicIpPoolReplacementBatchService.php' +); $worker = file_get_contents($root . '/app/Services/Cloud/CloudWorkerService.php'); $input = file_get_contents($root . '/app/Services/Cloud/CloudInput.php'); $adminInstances = file_get_contents($root . '/public/cloud/admin-instances.js') @@ -75,20 +81,47 @@ $assert(is_string($controller) && is_string($options) && str_contains($options, "'allocation_end'") && str_contains($options, "'excluded_addresses'") && str_contains($options, - 'CloudJobService::blockingQueryForInstance(') + 'CloudPublicNetworkMutationAdmission::inspect($instance)') && str_contains($controller, "CloudInput::only(\$body, ['targets', 'confirm_restart'])"), 'Dual-stack network options or administrator transition command drifted.'); $assert(is_string($adminInstanceService) && str_contains( $adminInstanceService, - "'network_change_supported' => \$canMutate" + "'network_change_supported' => \$networkMutation->allowed()" ) && str_contains($adminInstanceService, '$managedCloudTemplate') && str_contains($adminInstanceService, '$networkValid') && str_contains($adminInstanceService, '$instance->serviceAvailable($now)'), 'Administrator network admission is not tied to active canonical managed instances.'); +$assert(is_string($admission) + && str_contains($admission, 'public static function inspect(') + && str_contains($admission, 'public static function inspectMany(') + && str_contains($admission, 'public static function assertCanBegin(') + && str_contains($admission, 'public static function assertCanBeginMany(') + && str_contains($admission, + 'CloudJobService::blockingQueryForInstances(') + && str_contains($admission, 'CloudInstanceMigration::whereIn(') + && str_contains($admission, 'CloudBackupRecovery::whereIn(') + && str_contains($admission, 'CloudPublicNetworkTransition::whereIn(') + && !str_contains($admission, 'lockForUpdate(') + && !str_contains($admission, 'transaction(') + && !str_contains($admission, 'hasTable(') + && !str_contains($admission, 'CloudPublicIp'), + 'Shared public-network admission became shallow or took catalog ownership.'); +$assert(is_string($replacementBatch) + && str_contains($replacementBatch, + 'CloudPublicNetworkMutationAdmission::inspectMany($instances)') + && str_contains($replacementBatch, + 'CloudPublicNetworkMutationAdmission::assertCanBeginMany(') + && str_contains($replacementBatch, + "->whereColumn('observed_revision', '=', 'desired_revision')") + && substr_count( + $replacementBatch, + 'CloudAgentSettingsService::publicNetworkReady(' + ) >= 3, + 'Pool replacement bypassed bulk admission or strict network evidence.'); $assert(is_string($adminInstances) && str_contains($adminInstances, 'buildAdminNetworkTargets(') && str_contains($adminInstances, 'targets: target.targets') diff --git a/web/tests/CloudPaidIpChangeContractTest.php b/web/tests/CloudPaidIpChangeContractTest.php index 752ba946..8191c325 100644 --- a/web/tests/CloudPaidIpChangeContractTest.php +++ b/web/tests/CloudPaidIpChangeContractTest.php @@ -111,9 +111,11 @@ $enqueue = $enqueueStart !== false && $enqueueEnd !== false ? substr($jobs, $enqueueStart, $enqueueEnd - $enqueueStart) : ''; $walletLock = strpos($workflow, "User::where('id', \$command->actorUserId())"); $instanceLock = strpos($workflow, - "CloudInstance::where('id', \$command->instanceId())"); + "CloudInstance::where('id', \$lockedInstanceId)"); $replayLookup = strpos($workflow, "'transition_key', \$command->transitionKey()"); +$admission = strpos($workflow, + 'CloudPublicNetworkMutationAdmission::assertCanBegin($instance)'); $priceLookup = strpos($workflow, 'CloudIpChangeBillingService::priceCents()'); $charge = strpos($workflow, @@ -129,8 +131,10 @@ $assert($enqueue !== '' && str_contains($workflow, "->where('address_family', 4)") && $walletLock !== false && $instanceLock !== false && $walletLock < $instanceLock - && $replayLookup !== false && $priceLookup !== false + && $replayLookup !== false && $admission !== false + && $priceLookup !== false && $charge !== false && $replayLookup < $priceLookup + && $replayLookup < $admission && $admission < $priceLookup && $priceLookup < $charge, 'Paid transition lost automatic same-attachment allocation, lock order, ' . 'or replay-before-charge semantics.'); diff --git a/web/tests/CloudPublicIpPoolReplacementBatchServiceTest.php b/web/tests/CloudPublicIpPoolReplacementBatchServiceTest.php index 20b1177c..b0377c79 100644 --- a/web/tests/CloudPublicIpPoolReplacementBatchServiceTest.php +++ b/web/tests/CloudPublicIpPoolReplacementBatchServiceTest.php @@ -26,14 +26,6 @@ namespace App\Services\Cloud { } } - final class CloudInstanceMigrationService - { - public static function assertInstanceMutationAvailable( - \App\Models\CloudInstance $instance, - ?string $operationKind = null - ): void { - } - } } namespace { @@ -366,6 +358,20 @@ namespace { $t->integer('created_at'); $t->integer('updated_at'); }); + $schema->create('cloud_instance_migrations', static function ( + Blueprint $t + ): void { + $t->increments('id'); + $t->integer('instance_id'); + $t->string('state'); + }); + $schema->create('cloud_backup_recoveries', static function ( + Blueprint $t + ): void { + $t->increments('id'); + $t->integer('instance_id'); + $t->string('state'); + }); DB::statement(<<<'SQL' CREATE TABLE cloud_public_ip_pool_replacement_batch_items ( id INTEGER PRIMARY KEY AUTOINCREMENT, @@ -732,6 +738,245 @@ SQL); ]; }; + foreach ([10 => 'behind', 12 => 'ahead'] as $observed => $label) { + DB::table('cloud_public_ip_pools')->where('id', 11)->update([ + 'observed_revision' => $observed, + ]); + $driftOverview = CloudPublicIpPoolReplacementBatchService::overview( + 1, + $poolKey('a'), + $readAdmin + ); + $driftTargets = array_column( + $driftOverview['target_options'], + null, + 'pool_key' + ); + $assert(($driftOverview['can_start'] ?? null) === false + && ($driftOverview['source_pool']['expected_revision'] ?? null) + === 11 + && ($driftTargets[$poolKey('c')]['eligible'] ?? null) === false, + 'Source pool revision ' . $label . ' drift was advertised.'); + $beforeRevisionReject = $snapshot(); + $rejects(static fn () => + CloudPublicIpPoolReplacementBatchService::start( + 1, + $poolKey('a'), + $admin, + $input( + $poolKey('c'), + 11, + 13, + 'replacement-source-revision-' . $label + ) + ), 'Source pool revision ' . $label . ' drift was accepted.'); + $assert($snapshot() === $beforeRevisionReject, + 'Source revision rejection left durable writes: ' . $label); + } + DB::table('cloud_public_ip_pools')->where('id', 11)->update([ + 'observed_revision' => 11, + ]); + + foreach ([12 => 'behind', 14 => 'ahead'] as $observed => $label) { + DB::table('cloud_public_ip_pools')->where('id', 13)->update([ + 'observed_revision' => $observed, + ]); + $driftOverview = CloudPublicIpPoolReplacementBatchService::overview( + 1, + $poolKey('a'), + $readAdmin + ); + $driftTargets = array_column( + $driftOverview['target_options'], + null, + 'pool_key' + ); + $assert(!isset($driftTargets[$poolKey('c')]), + 'Target pool revision ' . $label . ' drift was advertised.'); + $beforeRevisionReject = $snapshot(); + $rejects(static fn () => + CloudPublicIpPoolReplacementBatchService::start( + 1, + $poolKey('a'), + $admin, + $input( + $poolKey('c'), + 11, + 13, + 'replacement-target-revision-' . $label + ) + ), 'Target pool revision ' . $label . ' drift was accepted.'); + $assert($snapshot() === $beforeRevisionReject, + 'Target revision rejection left durable writes: ' . $label); + } + DB::table('cloud_public_ip_pools')->where('id', 13)->update([ + 'observed_revision' => 13, + ]); + + DB::table('cloud_host_agents')->where('host_id', 1)->update([ + 'public_network_capabilities_json' => '{}', + ]); + $capabilityOverview = CloudPublicIpPoolReplacementBatchService::overview( + 1, + $poolKey('a'), + $readAdmin + ); + $assert(($capabilityOverview['can_start'] ?? null) === false + && ($capabilityOverview['target_options'] ?? []) === [], + 'Missing Agent public-network capability was advertised.'); + $beforeCapabilityReject = $snapshot(); + $rejects(static fn () => + CloudPublicIpPoolReplacementBatchService::start( + 1, + $poolKey('a'), + $admin, + $input( + $poolKey('c'), + 11, + 13, + 'replacement-agent-capability-0001' + ) + ), 'Missing Agent public-network capability was accepted.'); + $assert($snapshot() === $beforeCapabilityReject, + 'Agent-capability rejection left durable writes.'); + DB::table('cloud_host_agents')->where('host_id', 1)->update([ + 'public_network_capabilities_json' => $capabilities, + ]); + + foreach ([ + 'migration' => static function (): void { + DB::table('cloud_instance_migrations')->insert([ + 'id' => 1, + 'instance_id' => 101, + 'state' => 'preparing_source', + ]); + }, + 'backup-recovery' => static function (): void { + DB::table('cloud_backup_recoveries')->insert([ + 'id' => 1, + 'instance_id' => 101, + 'state' => 'recovering', + ]); + }, + ] as $label => $arrange) { + $arrange(); + $blockedOverview = + CloudPublicIpPoolReplacementBatchService::overview( + 1, + $poolKey('a'), + $readAdmin + ); + $assert(($blockedOverview['can_start'] ?? null) === false, + 'Batch overview ignored active ' . $label . '.'); + $beforeAdmissionReject = $snapshot(); + $rejects(static fn () => + CloudPublicIpPoolReplacementBatchService::start( + 1, + $poolKey('a'), + $admin, + $input( + $poolKey('c'), + 11, + 13, + 'replacement-admission-' . $label + ) + ), 'Batch start ignored active ' . $label . '.'); + $assert($snapshot() === $beforeAdmissionReject, + 'Admission rejection left durable writes: ' . $label); + DB::table('cloud_instance_migrations')->delete(); + DB::table('cloud_backup_recoveries')->delete(); + } + + $blockingJob = new CloudJob(); + $blockingJob->instance_id = 101; + $blockingJob->user_id = 501; + $blockingJob->host_id = 1; + $blockingJob->kind = 'action'; + $blockingJob->execution_mode = 'agent'; + $blockingJob->sealed_payload = '{}'; + $blockingJob->available_at = $now; + $blockingJob->created_at = $blockingJob->updated_at = $now; + $blockingJob->save(); + $jobBlockedOverview = CloudPublicIpPoolReplacementBatchService::overview( + 1, + $poolKey('a'), + $readAdmin + ); + $assert(($jobBlockedOverview['can_start'] ?? null) === false, + 'Batch overview ignored a blocking instance job.'); + $beforeJobReject = $snapshot(); + $rejects(static fn () => + CloudPublicIpPoolReplacementBatchService::start( + 1, + $poolKey('a'), + $admin, + $input( + $poolKey('c'), + 11, + 13, + 'replacement-admission-job-0001' + ) + ), 'Batch start ignored a blocking instance job.'); + $assert($snapshot() === $beforeJobReject, + 'Blocking-job rejection left durable writes.'); + $blockingJob->delete(); + + $frozenNetwork = json_encode( + $publicNetwork(101), + JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES + ); + DB::table('cloud_public_network_transitions')->insert([ + 'transition_key' => str_repeat('8', 32), + 'operation_id' => str_repeat('9', 32), + 'kind' => 'admin_exact', + 'instance_id' => 101, + 'host_id' => 1, + 'reservation_id' => 1001, + 'invoice_id' => null, + 'pool_replacement_batch_item_id' => null, + 'requested_by_user_id' => 99, + 'source_network_attachment_id' => 11, + 'target_network_attachment_id' => 11, + 'source_claim_set_sha256' => str_repeat('a', 64), + 'target_claim_set_sha256' => str_repeat('b', 64), + 'source_public_network_json' => $frozenNetwork, + 'source_public_network_sha256' => hash('sha256', $frozenNetwork), + 'target_public_network_json' => $frozenNetwork, + 'target_public_network_sha256' => hash('sha256', $frozenNetwork), + 'state' => 'queued', + 'effect_resolution' => 'not_applicable', + 'active_transition' => 1, + 'last_error_code' => '', + 'effect_resolved_at' => null, + 'terminal_at' => null, + 'created_at' => $now, + 'updated_at' => $now, + ]); + $transitionBlockedOverview = + CloudPublicIpPoolReplacementBatchService::overview( + 1, + $poolKey('a'), + $readAdmin + ); + $assert(($transitionBlockedOverview['can_start'] ?? null) === false, + 'Batch overview ignored an active public-network transition.'); + $beforeTransitionReject = $snapshot(); + $rejects(static fn () => + CloudPublicIpPoolReplacementBatchService::start( + 1, + $poolKey('a'), + $admin, + $input( + $poolKey('c'), + 11, + 13, + 'replacement-admission-transition-0001' + ) + ), 'Batch start ignored an active public-network transition.'); + $assert($snapshot() === $beforeTransitionReject, + 'Active-transition rejection left durable writes.'); + DB::table('cloud_public_network_transitions')->delete(); + $v4Overview = CloudPublicIpPoolReplacementBatchService::overview( 1, $poolKey('a'), diff --git a/web/tests/CloudPublicNetworkMutationAdmissionTest.php b/web/tests/CloudPublicNetworkMutationAdmissionTest.php new file mode 100644 index 00000000..17b69324 --- /dev/null +++ b/web/tests/CloudPublicNetworkMutationAdmissionTest.php @@ -0,0 +1,298 @@ +addConnection([ + 'driver' => 'sqlite', 'database' => ':memory:', +], 'default'); +$capsule->setAsGlobal(); +$capsule->bootEloquent(); +$schema = $capsule->schema('default'); +$assert((new ReflectionClass( + CloudPublicNetworkMutationDecision::class +))->isReadOnly(), 'Admission decision is not immutable.'); + +$schema->create('cloud_instances', static function (Blueprint $table): void { + $table->increments('id'); + $table->string('status'); + $table->string('service_status'); + $table->integer('expires_at')->nullable(); +}); +$schema->create('cloud_instance_migrations', static function ( + Blueprint $table +): void { + $table->increments('id'); + $table->integer('instance_id'); + $table->string('state'); +}); +$schema->create('cloud_backup_recoveries', static function ( + Blueprint $table +): void { + $table->increments('id'); + $table->integer('instance_id'); + $table->string('state'); +}); +$schema->create('cloud_public_network_transitions', static function ( + Blueprint $table +): void { + $table->increments('id'); + $table->integer('instance_id'); + $table->integer('active_transition')->nullable(); +}); +$schema->create('cloud_jobs', static function (Blueprint $table): void { + $table->increments('id'); + $table->string('operation_id')->nullable(); + $table->string('request_idempotency_key')->nullable(); + $table->string('request_fingerprint')->nullable(); + $table->integer('instance_id')->nullable(); + $table->integer('parent_job_id')->nullable(); + $table->integer('host_id')->nullable(); + $table->string('kind'); + $table->string('execution_mode')->nullable(); + $table->string('phase'); + $table->string('outcome')->nullable(); + $table->string('status'); + $table->integer('attempts')->nullable(); + $table->integer('dispatch_attempts')->nullable(); + $table->integer('dispatch_deadline_at')->nullable(); + $table->integer('dispatch_started_at')->nullable(); + $table->integer('agent_accepted_at')->nullable(); + $table->integer('dispatched_at')->nullable(); + $table->integer('receipt_deadline_at')->nullable(); + $table->integer('terminal_at')->nullable(); + $table->boolean('effect_may_have_applied')->nullable(); + $table->boolean('safe_to_resubmit')->nullable(); + $table->string('effect_resolution')->nullable(); + $table->integer('effect_resolved_at')->nullable(); + $table->boolean('finalization_incident')->default(false); + $table->string('finalization_error_code')->nullable(); + $table->string('completion_event_id')->nullable(); + $table->string('error_code')->nullable(); + $table->text('result_json')->nullable(); + $table->integer('updated_at'); +}); + +$now = time(); +Capsule::table('cloud_instances')->insert([ + 'id' => 1, + 'status' => 'active', + 'service_status' => 'active', + 'expires_at' => $now + 86400, +]); +$instance = static fn (): CloudInstance => CloudInstance::findOrFail(1); +$reason = static fn (): string => + CloudPublicNetworkMutationAdmission::inspect($instance())->reasonCode; + +// Exercise the same catalog reads directly so the fixture fails with the +// actual SQL error instead of being reduced to the public fail-closed code. +\App\Models\CloudInstanceMigration::where('instance_id', 1) + ->whereIn( + 'state', + \App\Services\Cloud\CloudInstanceMigrationWorkflow::blockingStates() + )->exists(); +\App\Models\CloudBackupRecovery::where('instance_id', 1) + ->whereIn('state', ['recovering', 'uncertain'])->exists(); +\App\Services\Cloud\CloudJobService::blockingQueryForInstance(1) + ->orderByDesc('id')->first(['id', 'kind', 'status', 'updated_at']); + +$ready = CloudPublicNetworkMutationAdmission::inspect($instance()); +$assert($ready->allowed() + && $ready->reasonCode === CloudPublicNetworkMutationDecision::READY + && $ready->message === '' && $ready->activeJob === null, + 'A healthy instance was not admitted: ' . $ready->reasonCode); + +Capsule::table('cloud_instances')->where('id', 1)->update([ + 'status' => 'stopped', +]); +$assert($reason() === CloudPublicNetworkMutationDecision::INSTANCE_INACTIVE, + 'Inactive instance reason drifted.'); +Capsule::table('cloud_instances')->where('id', 1)->update([ + 'status' => 'active', 'expires_at' => $now - 1, +]); +$assert($reason() === CloudPublicNetworkMutationDecision::SERVICE_UNAVAILABLE, + 'Unavailable service reason drifted.'); +Capsule::table('cloud_instances')->where('id', 1)->update([ + 'expires_at' => $now + 86400, +]); + +Capsule::table('cloud_instance_migrations')->insert([ + 'id' => 1, 'instance_id' => 1, 'state' => 'preparing_source', +]); +$assert($reason() === CloudPublicNetworkMutationDecision::MIGRATION_ACTIVE, + 'Maintenance migration did not fence public-network mutation.'); +Capsule::table('cloud_instance_migrations')->delete(); + +Capsule::table('cloud_backup_recoveries')->insert([ + 'id' => 1, 'instance_id' => 1, 'state' => 'uncertain', +]); +$assert($reason() + === CloudPublicNetworkMutationDecision::BACKUP_RECOVERY_ACTIVE, + 'Backup recovery did not fence public-network mutation.'); +Capsule::table('cloud_backup_recoveries')->delete(); + +Capsule::table('cloud_jobs')->insert([ + 'id' => 1, + 'operation_id' => str_repeat('a', 32), + 'request_idempotency_key' => 'admission-job-0001', + 'request_fingerprint' => str_repeat('b', 64), + 'instance_id' => 1, + 'parent_job_id' => null, + 'host_id' => 1, + 'kind' => 'restart', + 'execution_mode' => 'agent', + 'phase' => 'active', + 'outcome' => null, + 'status' => 'running', + 'attempts' => 1, + 'dispatch_attempts' => 1, + 'dispatch_deadline_at' => $now + 60, + 'dispatch_started_at' => $now, + 'agent_accepted_at' => $now, + 'dispatched_at' => $now, + 'receipt_deadline_at' => $now + 60, + 'terminal_at' => null, + 'effect_may_have_applied' => null, + 'safe_to_resubmit' => null, + 'effect_resolution' => 'not_applicable', + 'effect_resolved_at' => null, + 'finalization_incident' => 0, + 'finalization_error_code' => null, + 'completion_event_id' => null, + 'error_code' => null, + 'result_json' => null, + 'updated_at' => $now, +]); +$job = CloudPublicNetworkMutationAdmission::inspect($instance()); +$assert($job->reasonCode === CloudPublicNetworkMutationDecision::JOB_ACTIVE + && ($job->activeJob['id'] ?? null) === '1', + 'Blocking job did not project its immutable admission reason.'); +Capsule::table('cloud_jobs')->delete(); + +Capsule::table('cloud_public_network_transitions')->insert([ + 'id' => 1, 'instance_id' => 1, 'active_transition' => 1, +]); +$assert($reason() === CloudPublicNetworkMutationDecision::TRANSITION_ACTIVE, + 'Active transition did not fence public-network mutation.'); +Capsule::table('cloud_public_network_transitions')->delete(); + +foreach ([2, 3, 4, 5] as $instanceId) { + Capsule::table('cloud_instances')->insert([ + 'id' => $instanceId, + 'status' => 'active', + 'service_status' => 'active', + 'expires_at' => $now + 86400, + ]); +} +Capsule::table('cloud_instance_migrations')->insert([ + 'id' => 2, 'instance_id' => 2, 'state' => 'preparing_source', +]); +Capsule::table('cloud_backup_recoveries')->insert([ + ['id' => 2, 'instance_id' => 2, 'state' => 'recovering'], + ['id' => 3, 'instance_id' => 3, 'state' => 'uncertain'], +]); +$jobRow = static function (int $id, int $instanceId) use ($now): array { + return [ + 'id' => $id, + 'operation_id' => str_pad(dechex($id), 32, '0', STR_PAD_LEFT), + 'request_idempotency_key' => 'admission-job-' . str_pad( + (string) $id, + 4, + '0', + STR_PAD_LEFT + ), + 'request_fingerprint' => hash('sha256', 'admission-' . $id), + 'instance_id' => $instanceId, + 'parent_job_id' => null, + 'host_id' => 1, + 'kind' => 'restart', + 'execution_mode' => 'agent', + 'phase' => 'active', + 'outcome' => null, + 'status' => 'running', + 'attempts' => 1, + 'dispatch_attempts' => 1, + 'dispatch_deadline_at' => $now + 60, + 'dispatch_started_at' => $now, + 'agent_accepted_at' => $now, + 'dispatched_at' => $now, + 'receipt_deadline_at' => $now + 60, + 'terminal_at' => null, + 'effect_may_have_applied' => null, + 'safe_to_resubmit' => null, + 'effect_resolution' => 'not_applicable', + 'effect_resolved_at' => null, + 'finalization_incident' => 0, + 'finalization_error_code' => null, + 'completion_event_id' => null, + 'error_code' => null, + 'result_json' => null, + 'updated_at' => $now + $id, + ]; +}; +Capsule::table('cloud_jobs')->insert([ + $jobRow(20, 2), + $jobRow(30, 3), + $jobRow(40, 4), + $jobRow(41, 4), +]); +Capsule::table('cloud_public_network_transitions')->insert([ + ['id' => 2, 'instance_id' => 2, 'active_transition' => 1], + ['id' => 3, 'instance_id' => 3, 'active_transition' => 1], + ['id' => 4, 'instance_id' => 4, 'active_transition' => 1], + ['id' => 5, 'instance_id' => 5, 'active_transition' => 1], +]); +$manyInstances = CloudInstance::whereIn('id', [1, 2, 3, 4, 5]) + ->orderBy('id')->get(); +$connection = Capsule::connection('default'); +$connection->flushQueryLog(); +$connection->enableQueryLog(); +$many = CloudPublicNetworkMutationAdmission::inspectMany($manyInstances); +$queries = $connection->getQueryLog(); +$connection->disableQueryLog(); +$assert(count($queries) === 4, + 'Multi-instance admission introduced N+1 catalog queries: ' + . count($queries)); +$assert($many[1]->reasonCode === CloudPublicNetworkMutationDecision::READY + && $many[2]->reasonCode + === CloudPublicNetworkMutationDecision::MIGRATION_ACTIVE + && $many[3]->reasonCode + === CloudPublicNetworkMutationDecision::BACKUP_RECOVERY_ACTIVE + && $many[4]->reasonCode === CloudPublicNetworkMutationDecision::JOB_ACTIVE + && ($many[4]->activeJob['id'] ?? null) === '41' + && $many[5]->reasonCode + === CloudPublicNetworkMutationDecision::TRANSITION_ACTIVE, + 'Multi-instance admission priority or latest-job projection drifted.'); +Capsule::table('cloud_instance_migrations')->delete(); +Capsule::table('cloud_backup_recoveries')->delete(); +Capsule::table('cloud_jobs')->delete(); +Capsule::table('cloud_public_network_transitions')->delete(); + +$schema->drop('cloud_backup_recoveries'); +$unavailable = CloudPublicNetworkMutationAdmission::inspect($instance()); +$assert(!$unavailable->allowed() + && $unavailable->reasonCode + === CloudPublicNetworkMutationDecision::CATALOG_UNAVAILABLE, + 'A missing runtime catalog failed open.'); +try { + CloudPublicNetworkMutationAdmission::assertCanBegin($instance()); + throw new RuntimeException('Missing catalog was accepted by writer seam.'); +} catch (CloudError $error) { + $assert($error->httpStatus === 409, + 'Admission writer seam returned the wrong status.'); +} + +echo "Cloud public-network mutation admission behavior passed.\n"; diff --git a/web/tests/CloudPublicNetworkTransitionWorkflowContractTest.php b/web/tests/CloudPublicNetworkTransitionWorkflowContractTest.php index 7792ba16..f6f25a7c 100644 --- a/web/tests/CloudPublicNetworkTransitionWorkflowContractTest.php +++ b/web/tests/CloudPublicNetworkTransitionWorkflowContractTest.php @@ -186,12 +186,20 @@ $assert(str_contains($source, "\$transition->state = 'reserved';") && strpos($source, "\$transition->state = 'reserved';") < strpos($source, "\$transition->state = 'queued';"), 'Transition must fence targets before atomically becoming queued.'); -$assert(str_contains($source, - 'CloudInstanceMigrationService::assertInstanceMutationAvailable(') - && str_contains($source, "DB::table('cloud_backup_recoveries')") - && str_contains($source, - 'CloudJobService::blockingQueryForInstance('), - 'Open migration, recovery, or job work is not fenced before writes.'); +$replayPosition = strpos($reserve, 'return self::replayLocked('); +$admissionPosition = strpos( + $reserve, + 'CloudPublicNetworkMutationAdmission::assertCanBegin($instance)' +); +$activeTransitionPosition = strpos( + $reserve, + "->where('active_transition', 1)->lockForUpdate()->first()" +); +$assert($replayPosition !== false && $admissionPosition !== false + && $activeTransitionPosition !== false + && $replayPosition < $admissionPosition + && $admissionPosition < $activeTransitionPosition, + 'Exact replay, shared admission, or locked transition recheck drifted.'); $assert(str_contains($source, "(\$spec['managed'] ?? false) !== true") && str_contains($source, "(\$spec['os_source'] ?? null) !== 'cloud_template'"), diff --git a/web/tests/CloudPublicNetworkTransitionWorkflowTest.php b/web/tests/CloudPublicNetworkTransitionWorkflowTest.php index b2708c2f..ffce3bf0 100644 --- a/web/tests/CloudPublicNetworkTransitionWorkflowTest.php +++ b/web/tests/CloudPublicNetworkTransitionWorkflowTest.php @@ -789,6 +789,40 @@ namespace { Capsule::table('cloud_public_ip_pools')->where('id', 13) ->update(['state' => 'active']); + $beforeActiveTransition = [ + 'transitions' => Capsule::table( + 'cloud_public_network_transitions' + )->count(), + 'jobs' => Capsule::table('cloud_jobs')->count(), + 'claims' => Capsule::table( + 'cloud_resource_reservation_ips' + )->count(), + 'target_state' => Capsule::table('cloud_public_ips') + ->where('id', 201)->value('state'), + ]; + $rejects(static fn () => CloudPublicNetworkTransitionWorkflow::begin( + CloudPublicNetworkTransitionCommand::adminExact( + 1, + 99, + 'admin-network-active-transition-0001', + [4 => [ + 'pool_key' => str_repeat('c', 32), + 'address_cidr' => '8.8.9.20/24', + ]] + ) + ), 'A second request bypassed an unrelated active transition.'); + $assert($beforeActiveTransition === [ + 'transitions' => Capsule::table( + 'cloud_public_network_transitions' + )->count(), + 'jobs' => Capsule::table('cloud_jobs')->count(), + 'claims' => Capsule::table( + 'cloud_resource_reservation_ips' + )->count(), + 'target_state' => Capsule::table('cloud_public_ips') + ->where('id', 201)->value('state'), + ], 'Active-transition admission rejection left durable writes.'); + $rejects(static fn () => CloudPublicNetworkTransitionWorkflow::begin( CloudPublicNetworkTransitionCommand::adminExact( 1,