|
|
|
|
|
<?php
|
|
|
|
|
|
|
|
|
|
|
|
namespace App\Services\Cloud;
|
|
|
|
|
|
|
|
|
|
|
|
use App\Models\CloudHost;
|
|
|
|
|
|
use App\Models\CloudHostNetworkAttachment;
|
|
|
|
|
|
use App\Models\CloudHostResourcePoolMembership;
|
|
|
|
|
|
use App\Models\CloudInstance;
|
|
|
|
|
|
use App\Models\CloudNetworkLine;
|
|
|
|
|
|
use App\Models\CloudPublicIp;
|
|
|
|
|
|
use App\Models\CloudPublicIpPool;
|
|
|
|
|
|
use App\Models\CloudResourcePool;
|
|
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
|
* Administrator-owned desired state for one host's public IPv4 pools.
|
|
|
|
|
|
*
|
|
|
|
|
|
* PHP owns the commercial IP ledger and desired revisions. The private engine
|
|
|
|
|
|
* only applies the already validated host configuration. A write is recorded
|
|
|
|
|
|
* before dispatch and becomes applied only after the signed response matches
|
|
|
|
|
|
* the requested revision. Ambiguous transport failures remain uncertain.
|
|
|
|
|
|
*/
|
|
|
|
|
|
final class CloudAdminHostNetworkService
|
|
|
|
|
|
{
|
|
|
|
|
|
private const MAX_POOLS = 32;
|
|
|
|
|
|
private const STATES = ['active', 'draining', 'disabled'];
|
|
|
|
|
|
private const OCCUPIED_IP_STATES = [
|
|
|
|
|
|
'reserved', 'assigned', 'changing', 'uncertain',
|
|
|
|
|
|
];
|
|
|
|
|
|
|
|
|
|
|
|
public function __construct(private ?CloudEngineClient $engine = null)
|
|
|
|
|
|
{
|
|
|
|
|
|
$this->engine ??= new CloudEngineClient();
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
public function read(int $hostId, array $actor): array
|
|
|
|
|
|
{
|
|
|
|
|
|
$host = $this->host($hostId);
|
|
|
|
|
|
$catalog = $this->databaseCatalog($hostId);
|
|
|
|
|
|
return $this->response($host, $catalog);
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
public function create(int $hostId, array $actor, array $body): array
|
|
|
|
|
|
{
|
|
|
|
|
|
$requestKey = $body['request_key'] ?? null;
|
|
|
|
|
|
if (!is_string($requestKey)
|
|
|
|
|
|
|| preg_match('/^[0-9a-f]{32}$/D', $requestKey) !== 1) {
|
|
|
|
|
|
throw new CloudError('请提供有效的地址池创建请求标识', 422);
|
|
|
|
|
|
}
|
|
|
|
|
|
return $this->save(
|
|
|
|
|
|
$hostId,
|
|
|
|
|
|
$actor,
|
|
|
|
|
|
self::poolKeyForRequest($hostId, $requestKey),
|
|
|
|
|
|
$body,
|
|
|
|
|
|
true,
|
|
|
|
|
|
null
|
|
|
|
|
|
);
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
public function update(int $hostId, string $poolKey, array $actor,
|
|
|
|
|
|
array $body): array
|
|
|
|
|
|
{
|
|
|
|
|
|
if (preg_match('/^[0-9a-f]{32}$/D', $poolKey) !== 1) {
|
|
|
|
|
|
throw new CloudError('公网地址池标识无效', 404);
|
|
|
|
|
|
}
|
|
|
|
|
|
$expectedRevision = $body['expected_revision'] ?? null;
|
|
|
|
|
|
if (is_bool($expectedRevision) || !is_int($expectedRevision)
|
|
|
|
|
|
|| $expectedRevision < 1) {
|
|
|
|
|
|
throw new CloudError('公网地址池版本无效,请刷新后重试', 422);
|
|
|
|
|
|
}
|
|
|
|
|
|
return $this->save(
|
|
|
|
|
|
$hostId,
|
|
|
|
|
|
$actor,
|
|
|
|
|
|
$poolKey,
|
|
|
|
|
|
$body,
|
|
|
|
|
|
false,
|
|
|
|
|
|
$expectedRevision
|
|
|
|
|
|
);
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
private function save(int $hostId, array $actor, string $poolKey,
|
|
|
|
|
|
array $body, bool $creating, ?int $expectedRevision): array
|
|
|
|
|
|
{
|
|
|
|
|
|
$host = $this->host($hostId);
|
|
|
|
|
|
$catalog = $this->databaseCatalog($hostId);
|
|
|
|
|
|
if ($catalog['compatibility_mode']) {
|
|
|
|
|
|
throw new CloudError(
|
|
|
|
|
|
self::legacyRebuildMessage(),
|
|
|
|
|
|
409
|
|
|
|
|
|
);
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
$desired = self::normalizeInput($body);
|
|
|
|
|
|
$prepared = $this->prepareDesired(
|
|
|
|
|
|
$host,
|
|
|
|
|
|
$poolKey,
|
|
|
|
|
|
$desired,
|
|
|
|
|
|
$catalog['items'],
|
|
|
|
|
|
$creating,
|
|
|
|
|
|
$expectedRevision
|
|
|
|
|
|
);
|
|
|
|
|
|
if ($prepared['replay']) {
|
|
|
|
|
|
return $this->response($host, $catalog, $poolKey);
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
try {
|
|
|
|
|
|
$result = $this->engine->host(
|
|
|
|
|
|
$hostId,
|
|
|
|
|
|
'PUT',
|
|
|
|
|
|
'host/network-pools/' . rawurlencode($poolKey),
|
|
|
|
|
|
$actor,
|
|
|
|
|
|
$prepared['payload']
|
|
|
|
|
|
);
|
|
|
|
|
|
} catch (CloudError $error) {
|
|
|
|
|
|
$uncertain = !in_array($error->httpStatus, [400, 409, 422], true);
|
|
|
|
|
|
$this->markOutcome(
|
|
|
|
|
|
$hostId,
|
|
|
|
|
|
$poolKey,
|
|
|
|
|
|
$prepared['revision'],
|
|
|
|
|
|
$uncertain ? 'uncertain' : 'failed',
|
|
|
|
|
|
$uncertain
|
|
|
|
|
|
? 'host_public_pool_apply_uncertain'
|
|
|
|
|
|
: 'host_public_pool_apply_rejected'
|
|
|
|
|
|
);
|
|
|
|
|
|
throw self::safeWriteError($error, $uncertain);
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
if (!self::matchesAppliedPool(
|
|
|
|
|
|
$result['pool'] ?? null,
|
|
|
|
|
|
$poolKey,
|
|
|
|
|
|
$prepared['payload'])) {
|
|
|
|
|
|
$this->markOutcome(
|
|
|
|
|
|
$hostId,
|
|
|
|
|
|
$poolKey,
|
|
|
|
|
|
$prepared['revision'],
|
|
|
|
|
|
'uncertain',
|
|
|
|
|
|
'host_public_pool_response_mismatch'
|
|
|
|
|
|
);
|
|
|
|
|
|
throw new CloudError(
|
|
|
|
|
|
'公网地址池返回结果无法确认,请勿重复提交,稍后刷新核对',
|
|
|
|
|
|
502
|
|
|
|
|
|
);
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
if (!$this->markOutcome(
|
|
|
|
|
|
$hostId,
|
|
|
|
|
|
$poolKey,
|
|
|
|
|
|
$prepared['revision'],
|
|
|
|
|
|
'applied',
|
|
|
|
|
|
''
|
|
|
|
|
|
)) {
|
|
|
|
|
|
throw new CloudError(
|
|
|
|
|
|
'公网地址池已发送到引擎,但业务库状态无法确认,请勿重复提交',
|
|
|
|
|
|
503
|
|
|
|
|
|
);
|
|
|
|
|
|
}
|
|
|
|
|
|
$applied = [
|
|
|
|
|
|
'pool_key' => $poolKey,
|
|
|
|
|
|
...$prepared['payload'],
|
|
|
|
|
|
];
|
|
|
|
|
|
$replaced = false;
|
|
|
|
|
|
foreach ($catalog['items'] as $index => $item) {
|
|
|
|
|
|
if (hash_equals((string) $item['pool_key'], $poolKey)) {
|
|
|
|
|
|
$catalog['items'][$index] = $applied;
|
|
|
|
|
|
$replaced = true;
|
|
|
|
|
|
break;
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
if (!$replaced) {
|
|
|
|
|
|
$catalog['items'][] = $applied;
|
|
|
|
|
|
}
|
|
|
|
|
|
return $this->response(
|
|
|
|
|
|
$this->host($hostId),
|
|
|
|
|
|
$catalog,
|
|
|
|
|
|
$poolKey
|
|
|
|
|
|
);
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
/** Record one desired revision under the host-scoped lock order. */
|
|
|
|
|
|
private function prepareDesired(CloudHost $host, string $poolKey,
|
|
|
|
|
|
array $desired, array $catalogItems, bool $creating,
|
|
|
|
|
|
?int $expectedRevision): array
|
|
|
|
|
|
{
|
|
|
|
|
|
return $host->getConnection()->transaction(function () use (
|
|
|
|
|
|
$host, $poolKey, $desired, $catalogItems, $creating,
|
|
|
|
|
|
$expectedRevision
|
|
|
|
|
|
): array {
|
|
|
|
|
|
$lockedHost = CloudHost::where('id', (int) $host->id)
|
|
|
|
|
|
->lockForUpdate()->first();
|
|
|
|
|
|
if ($lockedHost === null) {
|
|
|
|
|
|
throw new CloudError('宿主机不存在', 404);
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
$memberships = CloudHostPoolMembershipService::lockedRows(
|
|
|
|
|
|
$lockedHost, time()
|
|
|
|
|
|
);
|
|
|
|
|
|
$membership = CloudHostPoolMembershipService::selected(
|
|
|
|
|
|
$lockedHost, $memberships
|
|
|
|
|
|
);
|
|
|
|
|
|
if ($membership === null) {
|
|
|
|
|
|
throw new CloudError(
|
|
|
|
|
|
'请先将宿主机加入资源池,再配置该线路的公网地址池',
|
|
|
|
|
|
409
|
|
|
|
|
|
);
|
|
|
|
|
|
}
|
|
|
|
|
|
$resourcePool = CloudResourcePool::where(
|
|
|
|
|
|
'id', (int) $membership->resource_pool_id
|
|
|
|
|
|
)->lockForUpdate()->first();
|
|
|
|
|
|
if ($resourcePool === null || (int) $resourcePool->line_id < 1) {
|
|
|
|
|
|
throw new CloudError('宿主机资源池或线路配置已失效', 409);
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
$selector = trim((string) $lockedHost->default_network);
|
|
|
|
|
|
if ($selector === '' || strlen($selector) > 128
|
|
|
|
|
|
|| preg_match('/[\x00-\x1f\x7f]/', $selector) === 1) {
|
|
|
|
|
|
throw new CloudError('宿主机默认网络配置无效', 409);
|
|
|
|
|
|
}
|
|
|
|
|
|
$attachmentKey = substr(hash('sha256', $selector), 0, 32);
|
|
|
|
|
|
$attachments = CloudHostNetworkAttachment::where(
|
|
|
|
|
|
'host_id', (int) $lockedHost->id
|
|
|
|
|
|
)->orderBy('id')->lockForUpdate()->get();
|
|
|
|
|
|
$attachment = $attachments->first(static fn ($row): bool =>
|
|
|
|
|
|
hash_equals((string) $row->attachment_key, $attachmentKey)
|
|
|
|
|
|
);
|
|
|
|
|
|
|
|
|
|
|
|
$poolRows = CloudPublicIpPool::where(
|
|
|
|
|
|
'host_id', (int) $lockedHost->id
|
|
|
|
|
|
)->orderBy('id')->lockForUpdate()->get();
|
|
|
|
|
|
foreach ($poolRows as $row) {
|
|
|
|
|
|
if ((string) $row->pool_key === 'legacy-default') {
|
|
|
|
|
|
throw new CloudError(self::legacyRebuildMessage(), 409);
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
$pool = $poolRows->first(static fn ($row): bool =>
|
|
|
|
|
|
hash_equals((string) $row->pool_key, $poolKey)
|
|
|
|
|
|
);
|
|
|
|
|
|
$originalAttachment = $pool === null ? null
|
|
|
|
|
|
: $attachments->first(static fn ($row): bool =>
|
|
|
|
|
|
(int) $row->id === (int) $pool->network_attachment_id
|
|
|
|
|
|
);
|
|
|
|
|
|
$live = null;
|
|
|
|
|
|
$liveKeys = [];
|
|
|
|
|
|
foreach ($catalogItems as $item) {
|
|
|
|
|
|
$liveKeys[(string) $item['pool_key']] = true;
|
|
|
|
|
|
if (hash_equals((string) $item['pool_key'], $poolKey)) {
|
|
|
|
|
|
$live = $item;
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
$liveRevision = $live === null ? 0 : (int) $live['revision'];
|
|
|
|
|
|
$revisionBase = max(
|
|
|
|
|
|
$liveRevision,
|
|
|
|
|
|
(int) ($pool?->desired_revision ?? 0),
|
|
|
|
|
|
(int) ($pool?->observed_revision ?? 0)
|
|
|
|
|
|
);
|
|
|
|
|
|
$retryExisting = false;
|
|
|
|
|
|
if ($creating) {
|
|
|
|
|
|
if ($pool !== null || $live !== null) {
|
|
|
|
|
|
if ($pool !== null && self::sameCreateRequest(
|
|
|
|
|
|
$pool,
|
|
|
|
|
|
$originalAttachment,
|
|
|
|
|
|
$desired,
|
|
|
|
|
|
$attachmentKey,
|
|
|
|
|
|
$selector,
|
|
|
|
|
|
(int) $resourcePool->line_id
|
|
|
|
|
|
)) {
|
|
|
|
|
|
if ((string) $pool->sync_state === 'applied') {
|
|
|
|
|
|
return [
|
|
|
|
|
|
'replay' => true,
|
|
|
|
|
|
'revision' => $revisionBase,
|
|
|
|
|
|
'payload' => [],
|
|
|
|
|
|
];
|
|
|
|
|
|
}
|
|
|
|
|
|
// The first response may have been lost after the
|
|
|
|
|
|
// engine accepted the write. Reusing this revision is
|
|
|
|
|
|
// safe because the engine's pool upsert is itself
|
|
|
|
|
|
// idempotent; it also repairs a rejected/uncertain
|
|
|
|
|
|
// panel record without creating a second pool.
|
|
|
|
|
|
$retryExisting = true;
|
|
|
|
|
|
}
|
|
|
|
|
|
if (!$retryExisting) {
|
|
|
|
|
|
throw new CloudError(
|
|
|
|
|
|
'同一创建请求标识不能用于不同的公网地址池配置',
|
|
|
|
|
|
409
|
|
|
|
|
|
);
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
foreach ($poolRows as $row) {
|
|
|
|
|
|
if (!isset($liveKeys[(string) $row->pool_key])) {
|
|
|
|
|
|
throw new CloudError(
|
|
|
|
|
|
'存在尚未核对的地址池创建记录,请刷新后编辑该记录,不能继续创建新地址池',
|
|
|
|
|
|
409
|
|
|
|
|
|
);
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
} else {
|
|
|
|
|
|
if ($pool === null && $live === null) {
|
|
|
|
|
|
throw new CloudError('公网地址池不存在', 404);
|
|
|
|
|
|
}
|
|
|
|
|
|
if ($expectedRevision !== $revisionBase) {
|
|
|
|
|
|
throw new CloudError(
|
|
|
|
|
|
'公网地址池版本已变化,请刷新后重试',
|
|
|
|
|
|
409
|
|
|
|
|
|
);
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
if ($pool === null && self::catalogCount($poolRows) >= self::MAX_POOLS) {
|
|
|
|
|
|
throw new CloudError(
|
|
|
|
|
|
'一台宿主机最多配置 32 个公网地址池',
|
|
|
|
|
|
409
|
|
|
|
|
|
);
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
self::assertNoOverlap(
|
|
|
|
|
|
$poolKey,
|
|
|
|
|
|
$desired['cidr'],
|
|
|
|
|
|
$poolRows
|
|
|
|
|
|
);
|
|
|
|
|
|
self::assertNoGlobalOverlap(
|
|
|
|
|
|
(int) $lockedHost->id,
|
|
|
|
|
|
$desired['cidr']
|
|
|
|
|
|
);
|
|
|
|
|
|
|
|
|
|
|
|
$occupiedByPool = [];
|
|
|
|
|
|
if (!$poolRows->isEmpty()) {
|
|
|
|
|
|
$occupiedRows = CloudPublicIp::where(
|
|
|
|
|
|
'host_id', (int) $lockedHost->id
|
|
|
|
|
|
)->whereIn('public_ip_pool_id', $poolRows->pluck('id')->all())
|
|
|
|
|
|
->where(function ($query): void {
|
|
|
|
|
|
$query->whereIn('state', self::OCCUPIED_IP_STATES)
|
|
|
|
|
|
->orWhereNotNull('reservation_id')
|
|
|
|
|
|
->orWhereNotNull('instance_id')
|
|
|
|
|
|
->orWhereNotNull('operation_id');
|
|
|
|
|
|
})->orderBy('id')->lockForUpdate()
|
|
|
|
|
|
->get(['id', 'public_ip_pool_id']);
|
|
|
|
|
|
foreach ($occupiedRows as $occupiedRow) {
|
|
|
|
|
|
$id = (int) $occupiedRow->public_ip_pool_id;
|
|
|
|
|
|
$occupiedByPool[$id] = ($occupiedByPool[$id] ?? 0) + 1;
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
$occupied = $pool === null ? 0
|
|
|
|
|
|
: ($occupiedByPool[(int) $pool->id] ?? 0);
|
|
|
|
|
|
$attachmentLineChanged = $attachment !== null
|
|
|
|
|
|
&& (int) $attachment->line_id
|
|
|
|
|
|
!== (int) $resourcePool->line_id;
|
|
|
|
|
|
if ($attachmentLineChanged) {
|
|
|
|
|
|
foreach ($poolRows as $row) {
|
|
|
|
|
|
if ((int) $row->network_attachment_id
|
|
|
|
|
|
=== (int) $attachment->id
|
|
|
|
|
|
&& ($occupiedByPool[(int) $row->id] ?? 0) > 0) {
|
|
|
|
|
|
throw new CloudError(
|
|
|
|
|
|
'公网接入仍有关联地址被占用,不能切换所属线路',
|
|
|
|
|
|
409
|
|
|
|
|
|
);
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
$current = $live ?? ($pool === null ? null : [
|
|
|
|
|
|
...self::poolConfig($pool),
|
|
|
|
|
|
'network_selector' => (string) (
|
|
|
|
|
|
$originalAttachment?->network_selector ?? ''
|
|
|
|
|
|
),
|
|
|
|
|
|
]);
|
|
|
|
|
|
$attachmentChanged = $pool !== null && (
|
|
|
|
|
|
$originalAttachment === null
|
|
|
|
|
|
|| (string) $originalAttachment->attachment_key
|
|
|
|
|
|
!== $attachmentKey
|
|
|
|
|
|
|| (string) $originalAttachment->network_selector
|
|
|
|
|
|
!== $selector
|
|
|
|
|
|
|| (int) $originalAttachment->line_id
|
|
|
|
|
|
!== (int) $resourcePool->line_id
|
|
|
|
|
|
);
|
|
|
|
|
|
if ($occupied > 0 && ($attachmentChanged
|
|
|
|
|
|
|| ($current !== null
|
|
|
|
|
|
&& self::runtimeChanged($current, $desired, $selector)))) {
|
|
|
|
|
|
throw new CloudError(
|
|
|
|
|
|
'该地址池仍有实例或任务占用,只能修改名称和运行状态',
|
|
|
|
|
|
409
|
|
|
|
|
|
);
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
if ($revisionBase >= PHP_INT_MAX) {
|
|
|
|
|
|
throw new CloudError('公网地址池版本已达到上限', 409);
|
|
|
|
|
|
}
|
|
|
|
|
|
$revision = $retryExisting ? max(1, $revisionBase)
|
|
|
|
|
|
: $revisionBase + 1;
|
|
|
|
|
|
|
|
|
|
|
|
$now = time();
|
|
|
|
|
|
if ($attachment === null) {
|
|
|
|
|
|
$attachment = new CloudHostNetworkAttachment();
|
|
|
|
|
|
$attachment->attachment_key = $attachmentKey;
|
|
|
|
|
|
$attachment->host_id = (int) $lockedHost->id;
|
|
|
|
|
|
$attachment->vlan_id = null;
|
|
|
|
|
|
$attachment->routing_table = null;
|
|
|
|
|
|
$attachment->state = 'disabled';
|
|
|
|
|
|
$attachment->priority = 0;
|
|
|
|
|
|
$attachment->desired_revision = 0;
|
|
|
|
|
|
$attachment->observed_revision = 0;
|
|
|
|
|
|
$attachment->sync_state = 'never';
|
|
|
|
|
|
$attachment->last_error_code = '';
|
|
|
|
|
|
$attachment->created_at = $now;
|
|
|
|
|
|
}
|
|
|
|
|
|
$attachment->line_id = (int) $resourcePool->line_id;
|
|
|
|
|
|
$attachment->name = mb_substr($selector, 0, 80);
|
|
|
|
|
|
$attachment->network_mode = str_starts_with($selector, 'bridge:')
|
|
|
|
|
|
? 'bridge' : 'network';
|
|
|
|
|
|
$attachment->network_selector = $selector;
|
|
|
|
|
|
|
|
|
|
|
|
if ($pool === null) {
|
|
|
|
|
|
$pool = new CloudPublicIpPool();
|
|
|
|
|
|
$pool->pool_key = $poolKey;
|
|
|
|
|
|
$pool->host_id = (int) $lockedHost->id;
|
|
|
|
|
|
$pool->network_attachment_id = 0;
|
|
|
|
|
|
$pool->priority = 0;
|
|
|
|
|
|
$pool->observed_revision = 0;
|
|
|
|
|
|
$pool->created_at = $now;
|
|
|
|
|
|
}
|
|
|
|
|
|
$pool->name = $desired['name'];
|
|
|
|
|
|
$pool->cidr = $desired['cidr'];
|
|
|
|
|
|
$pool->gateway = $desired['gateway'];
|
|
|
|
|
|
$pool->guest_prefix_length = $desired['guest_prefix_length'];
|
|
|
|
|
|
$pool->dns_servers_json = json_encode(
|
|
|
|
|
|
$desired['dns_servers'],
|
|
|
|
|
|
JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES
|
|
|
|
|
|
);
|
|
|
|
|
|
$pool->reserved_ips_json = json_encode(
|
|
|
|
|
|
$desired['reserved_ips'],
|
|
|
|
|
|
JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES
|
|
|
|
|
|
);
|
|
|
|
|
|
$pool->state = $desired['state'];
|
|
|
|
|
|
$pool->desired_revision = $revision;
|
|
|
|
|
|
$pool->sync_state = 'pending';
|
|
|
|
|
|
$pool->last_error_code = '';
|
|
|
|
|
|
$pool->updated_at = $now;
|
|
|
|
|
|
|
|
|
|
|
|
// The attachment row must exist before its identifier can be
|
|
|
|
|
|
// referenced by the pool row. Both remain inside this transaction.
|
|
|
|
|
|
// Pool revisions are independent. The parent revision represents
|
|
|
|
|
|
// evidence for the shared selector, so another child mutation
|
|
|
|
|
|
// must not invalidate an already confirmed egress attachment.
|
|
|
|
|
|
$attachment->desired_revision = max(
|
|
|
|
|
|
1, (int) $attachment->desired_revision
|
|
|
|
|
|
);
|
|
|
|
|
|
$attachment->updated_at = $now;
|
|
|
|
|
|
$attachment->save();
|
|
|
|
|
|
$pool->network_attachment_id = (int) $attachment->id;
|
|
|
|
|
|
$pool->save();
|
|
|
|
|
|
self::refreshAttachmentAggregate($attachment);
|
|
|
|
|
|
|
|
|
|
|
|
return [
|
|
|
|
|
|
'replay' => false,
|
|
|
|
|
|
'revision' => $revision,
|
|
|
|
|
|
'payload' => [
|
|
|
|
|
|
'revision' => $revision,
|
|
|
|
|
|
'state' => $desired['state'],
|
|
|
|
|
|
'network_selector' => $selector,
|
|
|
|
|
|
'cidr' => $desired['cidr'],
|
|
|
|
|
|
'gateway' => $desired['gateway'],
|
|
|
|
|
|
'dns_servers' => $desired['dns_servers'],
|
|
|
|
|
|
'reserved_ips' => $desired['reserved_ips'],
|
|
|
|
|
|
'guest_prefix_length' => $desired['guest_prefix_length'],
|
|
|
|
|
|
],
|
|
|
|
|
|
];
|
|
|
|
|
|
}, 3);
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
private function markOutcome(int $hostId, string $poolKey, int $revision,
|
|
|
|
|
|
string $syncState, string $errorCode): bool
|
|
|
|
|
|
{
|
|
|
|
|
|
try {
|
|
|
|
|
|
return (new CloudHost())->getConnection()->transaction(function () use (
|
|
|
|
|
|
$hostId, $poolKey, $revision, $syncState, $errorCode
|
|
|
|
|
|
): bool {
|
|
|
|
|
|
$host = CloudHost::where('id', $hostId)
|
|
|
|
|
|
->lockForUpdate()->first();
|
|
|
|
|
|
if ($host === null) {
|
|
|
|
|
|
return false;
|
|
|
|
|
|
}
|
|
|
|
|
|
$pool = CloudPublicIpPool::where('host_id', $hostId)
|
|
|
|
|
|
->where('pool_key', $poolKey)->lockForUpdate()->first();
|
|
|
|
|
|
if ($pool === null
|
|
|
|
|
|
|| (int) $pool->desired_revision !== $revision) {
|
|
|
|
|
|
return false;
|
|
|
|
|
|
}
|
|
|
|
|
|
$attachment = CloudHostNetworkAttachment::where(
|
|
|
|
|
|
'id', (int) $pool->network_attachment_id
|
|
|
|
|
|
)->lockForUpdate()->first();
|
|
|
|
|
|
$now = time();
|
|
|
|
|
|
if ($syncState === 'applied') {
|
|
|
|
|
|
$pool->observed_revision = $revision;
|
|
|
|
|
|
}
|
|
|
|
|
|
$pool->sync_state = $syncState;
|
|
|
|
|
|
$pool->last_error_code = $errorCode;
|
|
|
|
|
|
$pool->updated_at = $now;
|
|
|
|
|
|
$pool->save();
|
|
|
|
|
|
if ($syncState === 'applied') {
|
|
|
|
|
|
self::materializeIpLedger($pool);
|
|
|
|
|
|
}
|
|
|
|
|
|
if ($attachment !== null) {
|
|
|
|
|
|
$attachment->updated_at = $now;
|
|
|
|
|
|
self::refreshAttachmentAggregate($attachment);
|
|
|
|
|
|
}
|
|
|
|
|
|
return true;
|
|
|
|
|
|
}, 3);
|
|
|
|
|
|
} catch (\Throwable $error) {
|
|
|
|
|
|
error_log('Unable to record public pool sync outcome: '
|
|
|
|
|
|
. get_class($error));
|
|
|
|
|
|
return false;
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
private function response(CloudHost $host, array $catalog,
|
|
|
|
|
|
?string $savedPoolKey = null): array
|
|
|
|
|
|
{
|
|
|
|
|
|
$dbRows = CloudPublicIpPool::where('host_id', (int) $host->id)
|
|
|
|
|
|
->orderBy('priority')->orderBy('id')->get();
|
|
|
|
|
|
$byKey = [];
|
|
|
|
|
|
$hasLegacyDb = false;
|
|
|
|
|
|
foreach ($dbRows as $row) {
|
|
|
|
|
|
$byKey[(string) $row->pool_key] = $row;
|
|
|
|
|
|
$hasLegacyDb = $hasLegacyDb
|
|
|
|
|
|
|| (string) $row->pool_key === 'legacy-default';
|
|
|
|
|
|
}
|
|
|
|
|
|
$compatibilityMode = $catalog['compatibility_mode'] || $hasLegacyDb;
|
|
|
|
|
|
$occupied = [];
|
|
|
|
|
|
if (!$dbRows->isEmpty()) {
|
|
|
|
|
|
foreach (CloudPublicIp::whereIn(
|
|
|
|
|
|
'public_ip_pool_id', $dbRows->pluck('id')->all()
|
|
|
|
|
|
)->selectRaw('public_ip_pool_id, COUNT(*) AS aggregate')
|
|
|
|
|
|
->where(function ($query): void {
|
|
|
|
|
|
$query->whereIn('state', self::OCCUPIED_IP_STATES)
|
|
|
|
|
|
->orWhereNotNull('reservation_id')
|
|
|
|
|
|
->orWhereNotNull('instance_id')
|
|
|
|
|
|
->orWhereNotNull('operation_id');
|
|
|
|
|
|
})->groupBy('public_ip_pool_id')->get() as $count) {
|
|
|
|
|
|
$occupied[(int) $count->public_ip_pool_id] = (int) $count->aggregate;
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
$items = [];
|
|
|
|
|
|
$seen = [];
|
|
|
|
|
|
foreach ($catalog['items'] as $live) {
|
|
|
|
|
|
$key = (string) $live['pool_key'];
|
|
|
|
|
|
if ($key === 'legacy-default') {
|
|
|
|
|
|
$items[] = self::legacyDto($live);
|
|
|
|
|
|
$seen[$key] = true;
|
|
|
|
|
|
continue;
|
|
|
|
|
|
}
|
|
|
|
|
|
$row = $byKey[$key] ?? null;
|
|
|
|
|
|
$items[] = self::poolDto(
|
|
|
|
|
|
$row,
|
|
|
|
|
|
$live,
|
|
|
|
|
|
$row === null ? 0 : ($occupied[(int) $row->id] ?? 0),
|
|
|
|
|
|
$compatibilityMode
|
|
|
|
|
|
);
|
|
|
|
|
|
$seen[$key] = true;
|
|
|
|
|
|
}
|
|
|
|
|
|
foreach ($dbRows as $row) {
|
|
|
|
|
|
$key = (string) $row->pool_key;
|
|
|
|
|
|
if (isset($seen[$key])) {
|
|
|
|
|
|
continue;
|
|
|
|
|
|
}
|
|
|
|
|
|
if ($key === 'legacy-default') {
|
|
|
|
|
|
$items[] = self::legacyRowDto($row);
|
|
|
|
|
|
continue;
|
|
|
|
|
|
}
|
|
|
|
|
|
$items[] = self::poolDto(
|
|
|
|
|
|
$row,
|
|
|
|
|
|
null,
|
|
|
|
|
|
$occupied[(int) $row->id] ?? 0,
|
|
|
|
|
|
$compatibilityMode
|
|
|
|
|
|
);
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
$context = $this->lineContext((int) $host->id);
|
|
|
|
|
|
return [
|
|
|
|
|
|
'host' => [
|
|
|
|
|
|
'id' => (int) $host->id,
|
|
|
|
|
|
'name' => self::safeText((string) $host->name, 80),
|
|
|
|
|
|
'default_network' => self::safeText(
|
|
|
|
|
|
(string) $host->default_network, 128
|
|
|
|
|
|
),
|
|
|
|
|
|
],
|
|
|
|
|
|
'line' => $context,
|
|
|
|
|
|
'items' => $items,
|
|
|
|
|
|
'count' => count($items),
|
|
|
|
|
|
'maximum' => self::MAX_POOLS,
|
|
|
|
|
|
'compatibility_mode' => $compatibilityMode,
|
|
|
|
|
|
'saved_pool_key' => $savedPoolKey,
|
|
|
|
|
|
];
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
private function lineContext(int $hostId): ?array
|
|
|
|
|
|
{
|
|
|
|
|
|
$host = CloudHost::find($hostId);
|
|
|
|
|
|
$rows = CloudHostResourcePoolMembership::where('host_id', $hostId)
|
|
|
|
|
|
->orderBy('resource_pool_id')->orderBy('id')->get();
|
|
|
|
|
|
$membership = $host === null ? null
|
|
|
|
|
|
: CloudHostPoolMembershipService::selected($host, $rows);
|
|
|
|
|
|
$resourcePool = $membership === null ? null
|
|
|
|
|
|
: CloudResourcePool::find((int) $membership->resource_pool_id);
|
|
|
|
|
|
$lineId = $resourcePool === null ? null : (int) $resourcePool->line_id;
|
|
|
|
|
|
$line = $lineId === null ? null : CloudNetworkLine::find($lineId);
|
|
|
|
|
|
return $line === null ? null : [
|
|
|
|
|
|
'id' => (int) $line->id,
|
|
|
|
|
|
'name' => self::safeText((string) $line->name, 80),
|
|
|
|
|
|
'status' => self::safeText((string) $line->status, 16),
|
|
|
|
|
|
];
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
/** The panel database is the sole catalog for scheduling and IP capacity. */
|
|
|
|
|
|
private function databaseCatalog(int $hostId): array
|
|
|
|
|
|
{
|
|
|
|
|
|
$rows = CloudPublicIpPool::where('host_id', $hostId)
|
|
|
|
|
|
->orderBy('priority')->orderBy('id')->get();
|
|
|
|
|
|
$attachmentIds = $rows->pluck('network_attachment_id')->filter()
|
|
|
|
|
|
->map(static fn ($id): int => (int) $id)->unique()->values()->all();
|
|
|
|
|
|
$attachments = $attachmentIds === [] ? collect()
|
|
|
|
|
|
: CloudHostNetworkAttachment::whereIn('id', $attachmentIds)
|
|
|
|
|
|
->get(['id', 'network_selector'])->keyBy('id');
|
|
|
|
|
|
$items = [];
|
|
|
|
|
|
$compatibility = false;
|
|
|
|
|
|
foreach ($rows as $row) {
|
|
|
|
|
|
if ((string) $row->pool_key === 'legacy-default') {
|
|
|
|
|
|
$compatibility = true;
|
|
|
|
|
|
$items[] = self::legacyRowDto($row);
|
|
|
|
|
|
continue;
|
|
|
|
|
|
}
|
|
|
|
|
|
$items[] = [
|
|
|
|
|
|
'pool_key' => (string) $row->pool_key,
|
|
|
|
|
|
'revision' => max(0, (int) $row->desired_revision),
|
|
|
|
|
|
'network_selector' => (string) (
|
|
|
|
|
|
$attachments[(int) $row->network_attachment_id]->network_selector
|
|
|
|
|
|
?? ''
|
|
|
|
|
|
),
|
|
|
|
|
|
...self::poolConfig($row),
|
|
|
|
|
|
'state' => (string) $row->state,
|
|
|
|
|
|
];
|
|
|
|
|
|
}
|
|
|
|
|
|
return ['items' => $items, 'compatibility_mode' => $compatibility];
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
private static function normalizeInput(array $body, int $status = 422): array
|
|
|
|
|
|
{
|
|
|
|
|
|
$name = $body['name'] ?? '';
|
|
|
|
|
|
$state = $body['state'] ?? null;
|
|
|
|
|
|
$prefix = $body['guest_prefix_length'] ?? null;
|
|
|
|
|
|
$dns = $body['dns_servers'] ?? null;
|
|
|
|
|
|
$reserved = $body['reserved_ips'] ?? null;
|
|
|
|
|
|
if (!is_string($name) || trim($name) === ''
|
|
|
|
|
|
|| mb_strlen(trim($name)) > 80
|
|
|
|
|
|
|| preg_match('//u', $name) !== 1
|
|
|
|
|
|
|| preg_match('/[\x00-\x1f\x7f]/u', $name) === 1) {
|
|
|
|
|
|
throw new CloudError('地址池名称需为 1 至 80 个字符', $status);
|
|
|
|
|
|
}
|
|
|
|
|
|
if (!is_string($state) || !in_array($state, self::STATES, true)) {
|
|
|
|
|
|
throw new CloudError('公网地址池状态无效', $status);
|
|
|
|
|
|
}
|
|
|
|
|
|
if (is_bool($prefix) || !is_int($prefix)) {
|
|
|
|
|
|
throw new CloudError('客户机网络前缀无效', $status);
|
|
|
|
|
|
}
|
|
|
|
|
|
if (!is_array($dns) || !array_is_list($dns)
|
|
|
|
|
|
|| !is_array($reserved) || !array_is_list($reserved)) {
|
|
|
|
|
|
throw new CloudError('公网地址列表格式无效', $status);
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
$cidr = self::normalizeCidr($body['cidr'] ?? null, $status);
|
|
|
|
|
|
[, $networkPrefix] = explode('/', $cidr, 2);
|
|
|
|
|
|
$networkPrefix = (int) $networkPrefix;
|
|
|
|
|
|
if ($prefix < 24 || $prefix > $networkPrefix) {
|
|
|
|
|
|
throw new CloudError(
|
|
|
|
|
|
'客户机网络前缀必须介于 /24 和地址池前缀之间',
|
|
|
|
|
|
$status
|
|
|
|
|
|
);
|
|
|
|
|
|
}
|
|
|
|
|
|
$gateway = self::normalizeIpv4(
|
|
|
|
|
|
$body['gateway'] ?? null, '公网网关', false, $status
|
|
|
|
|
|
);
|
|
|
|
|
|
if (!self::containsUsable($cidr, $gateway)) {
|
|
|
|
|
|
throw new CloudError('公网网关必须位于地址池的可用范围内', $status);
|
|
|
|
|
|
}
|
|
|
|
|
|
if (count($dns) < 1 || count($dns) > 4) {
|
|
|
|
|
|
throw new CloudError('公网 DNS 必须填写 1 至 4 个地址', $status);
|
|
|
|
|
|
}
|
|
|
|
|
|
if (count($reserved) > 254) {
|
|
|
|
|
|
throw new CloudError('保留 IP 最多可填写 254 个地址', $status);
|
|
|
|
|
|
}
|
|
|
|
|
|
$dns = self::normalizeIpList(
|
|
|
|
|
|
$dns, '公网 DNS', false, null, $status
|
|
|
|
|
|
);
|
|
|
|
|
|
$reserved = self::normalizeIpList(
|
|
|
|
|
|
$reserved, '保留 IP', true, $cidr, $status
|
|
|
|
|
|
);
|
|
|
|
|
|
return [
|
|
|
|
|
|
'name' => trim($name),
|
|
|
|
|
|
'cidr' => $cidr,
|
|
|
|
|
|
'gateway' => $gateway,
|
|
|
|
|
|
'dns_servers' => $dns,
|
|
|
|
|
|
'reserved_ips' => $reserved,
|
|
|
|
|
|
'guest_prefix_length' => $prefix,
|
|
|
|
|
|
'state' => $state,
|
|
|
|
|
|
];
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
private static function normalizeCidr($value, int $status): string
|
|
|
|
|
|
{
|
|
|
|
|
|
if (!is_string($value)
|
|
|
|
|
|
|| preg_match('/^([^\/]+)\/([0-9]{1,2})$/D', trim($value), $match) !== 1) {
|
|
|
|
|
|
throw new CloudError('公网地址池必须填写有效的 IPv4 CIDR', $status);
|
|
|
|
|
|
}
|
|
|
|
|
|
$address = self::normalizeIpv4(
|
|
|
|
|
|
$match[1], '公网地址池', false, $status
|
|
|
|
|
|
);
|
|
|
|
|
|
$prefix = (int) $match[2];
|
|
|
|
|
|
if ($prefix < 24 || $prefix > 30) {
|
|
|
|
|
|
throw new CloudError('公网地址池前缀必须为 /24 至 /30', $status);
|
|
|
|
|
|
}
|
|
|
|
|
|
$network = self::uint($address) & self::mask($prefix);
|
|
|
|
|
|
$networkAddress = long2ip($network);
|
|
|
|
|
|
$broadcastAddress = long2ip($network | (~self::mask($prefix) & 0xffffffff));
|
|
|
|
|
|
if (!self::isPublicIpv4($networkAddress)
|
|
|
|
|
|
|| !self::isPublicIpv4($broadcastAddress)) {
|
|
|
|
|
|
throw new CloudError('公网地址池必须是可路由的 IPv4 网段', $status);
|
|
|
|
|
|
}
|
|
|
|
|
|
return $networkAddress . '/' . $prefix;
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
private static function normalizeIpList(array $values, string $label,
|
|
|
|
|
|
bool $allowReservedRange, ?string $cidr, int $status): array
|
|
|
|
|
|
{
|
|
|
|
|
|
$result = [];
|
|
|
|
|
|
foreach ($values as $value) {
|
|
|
|
|
|
$address = self::normalizeIpv4(
|
|
|
|
|
|
$value, $label, $allowReservedRange, $status
|
|
|
|
|
|
);
|
|
|
|
|
|
if ($cidr !== null && !self::containsUsable($cidr, $address)) {
|
|
|
|
|
|
throw new CloudError($label . '必须位于地址池的可用范围内', $status);
|
|
|
|
|
|
}
|
|
|
|
|
|
$result[$address] = true;
|
|
|
|
|
|
}
|
|
|
|
|
|
return array_keys($result);
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
private static function normalizeIpv4($value, string $label,
|
|
|
|
|
|
bool $allowReservedRange, int $status): string
|
|
|
|
|
|
{
|
|
|
|
|
|
if (!is_string($value) || trim($value) !== $value
|
|
|
|
|
|
|| filter_var($value, FILTER_VALIDATE_IP, FILTER_FLAG_IPV4) === false
|
|
|
|
|
|
|| (!$allowReservedRange && !self::isPublicIpv4($value))) {
|
|
|
|
|
|
throw new CloudError($label . '必须是有效的公网 IPv4 地址', $status);
|
|
|
|
|
|
}
|
|
|
|
|
|
return long2ip(self::uint($value));
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
private static function isPublicIpv4(string $address): bool
|
|
|
|
|
|
{
|
|
|
|
|
|
return filter_var(
|
|
|
|
|
|
$address,
|
|
|
|
|
|
FILTER_VALIDATE_IP,
|
|
|
|
|
|
FILTER_FLAG_IPV4 | FILTER_FLAG_NO_PRIV_RANGE | FILTER_FLAG_NO_RES_RANGE
|
|
|
|
|
|
) !== false;
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
private static function containsUsable(string $cidr, string $address): bool
|
|
|
|
|
|
{
|
|
|
|
|
|
[$networkAddress, $prefix] = explode('/', $cidr, 2);
|
|
|
|
|
|
$network = self::uint($networkAddress);
|
|
|
|
|
|
$broadcast = $network | (~self::mask((int) $prefix) & 0xffffffff);
|
|
|
|
|
|
$candidate = self::uint($address);
|
|
|
|
|
|
return $candidate > $network && $candidate < $broadcast;
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
private static function overlaps(string $left, string $right): bool
|
|
|
|
|
|
{
|
|
|
|
|
|
[$leftStart, $leftEnd] = self::bounds($left);
|
|
|
|
|
|
[$rightStart, $rightEnd] = self::bounds($right);
|
|
|
|
|
|
return $leftStart <= $rightEnd && $rightStart <= $leftEnd;
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
private static function bounds(string $cidr): array
|
|
|
|
|
|
{
|
|
|
|
|
|
[$address, $prefix] = explode('/', $cidr, 2);
|
|
|
|
|
|
$start = self::uint($address) & self::mask((int) $prefix);
|
|
|
|
|
|
return [$start, $start | (~self::mask((int) $prefix) & 0xffffffff)];
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
private static function uint(string $address): int
|
|
|
|
|
|
{
|
|
|
|
|
|
$packed = inet_pton($address);
|
|
|
|
|
|
if ($packed === false || strlen($packed) !== 4) {
|
|
|
|
|
|
throw new CloudError('IPv4 地址格式无效', 422);
|
|
|
|
|
|
}
|
|
|
|
|
|
return (int) unpack('Naddress', $packed)['address'];
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
private static function mask(int $prefix): int
|
|
|
|
|
|
{
|
|
|
|
|
|
return (0xffffffff << (32 - $prefix)) & 0xffffffff;
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
private static function assertNoOverlap(string $poolKey, string $cidr,
|
|
|
|
|
|
$poolRows): void
|
|
|
|
|
|
{
|
|
|
|
|
|
$checked = [];
|
|
|
|
|
|
foreach ($poolRows as $row) {
|
|
|
|
|
|
if ((string) $row->pool_key === $poolKey) {
|
|
|
|
|
|
continue;
|
|
|
|
|
|
}
|
|
|
|
|
|
$checked[(string) $row->pool_key] = (string) $row->cidr;
|
|
|
|
|
|
}
|
|
|
|
|
|
foreach ($checked as $other) {
|
|
|
|
|
|
if (self::overlaps($cidr, $other)) {
|
|
|
|
|
|
throw new CloudError('同一宿主机的公网地址池不能重叠', 409);
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
/** Public addresses are globally unique in cloud_public_ips. */
|
|
|
|
|
|
private static function assertNoGlobalOverlap(int $hostId, string $cidr): void
|
|
|
|
|
|
{
|
|
|
|
|
|
foreach (CloudPublicIpPool::where('host_id', '<>', $hostId)
|
|
|
|
|
|
->get(['pool_key', 'cidr']) as $row) {
|
|
|
|
|
|
if (self::overlaps($cidr, (string) $row->cidr)) {
|
|
|
|
|
|
throw new CloudError(
|
|
|
|
|
|
'公网地址池地址段已被其他宿主机占用,不能重复配置',
|
|
|
|
|
|
409
|
|
|
|
|
|
);
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
private static function catalogCount($poolRows): int
|
|
|
|
|
|
{
|
|
|
|
|
|
$keys = [];
|
|
|
|
|
|
foreach ($poolRows as $row) {
|
|
|
|
|
|
$keys[(string) $row->pool_key] = true;
|
|
|
|
|
|
}
|
|
|
|
|
|
return count($keys);
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
private static function poolKeyForRequest(int $hostId,
|
|
|
|
|
|
string $requestKey): string
|
|
|
|
|
|
{
|
|
|
|
|
|
return substr(hash_hmac(
|
|
|
|
|
|
'sha256',
|
|
|
|
|
|
'host-public-pool:' . $hostId . ':' . $requestKey,
|
|
|
|
|
|
CloudSecrets::key('payload_key')
|
|
|
|
|
|
), 0, 32);
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
private static function sameCreateRequest(CloudPublicIpPool $pool,
|
|
|
|
|
|
?CloudHostNetworkAttachment $attachment, array $desired,
|
|
|
|
|
|
string $attachmentKey, string $selector, int $lineId): bool
|
|
|
|
|
|
{
|
|
|
|
|
|
return (string) $pool->name === $desired['name']
|
|
|
|
|
|
&& (string) $pool->state === $desired['state']
|
|
|
|
|
|
&& !self::runtimeChanged([
|
|
|
|
|
|
...self::poolConfig($pool),
|
|
|
|
|
|
'network_selector' => (string) (
|
|
|
|
|
|
$attachment?->network_selector ?? ''
|
|
|
|
|
|
),
|
|
|
|
|
|
], $desired, $selector)
|
|
|
|
|
|
&& $attachment !== null
|
|
|
|
|
|
&& (string) $attachment->attachment_key === $attachmentKey
|
|
|
|
|
|
&& (int) $attachment->line_id === $lineId;
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
private static function legacyRebuildMessage(): string
|
|
|
|
|
|
{
|
|
|
|
|
|
return '检测到旧版 legacy-default 地址池;本项目不执行旧数据迁移,请先清空并按新版结构重建开发数据';
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
private static function poolConfig(CloudPublicIpPool $pool): array
|
|
|
|
|
|
{
|
|
|
|
|
|
return [
|
|
|
|
|
|
'cidr' => (string) $pool->cidr,
|
|
|
|
|
|
'gateway' => (string) $pool->gateway,
|
|
|
|
|
|
'dns_servers' => self::decodeIpList($pool->dns_servers_json),
|
|
|
|
|
|
'reserved_ips' => self::decodeIpList($pool->reserved_ips_json),
|
|
|
|
|
|
'guest_prefix_length' => (int) $pool->guest_prefix_length,
|
|
|
|
|
|
];
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
private static function runtimeChanged(array $current, array $desired,
|
|
|
|
|
|
string $selector): bool
|
|
|
|
|
|
{
|
|
|
|
|
|
return (string) ($current['network_selector'] ?? $selector) !== $selector
|
|
|
|
|
|
|| (string) ($current['cidr'] ?? '') !== $desired['cidr']
|
|
|
|
|
|
|| (string) ($current['gateway'] ?? '') !== $desired['gateway']
|
|
|
|
|
|
|| array_values($current['dns_servers'] ?? []) !== $desired['dns_servers']
|
|
|
|
|
|
|| array_values($current['reserved_ips'] ?? []) !== $desired['reserved_ips']
|
|
|
|
|
|
|| (int) ($current['guest_prefix_length'] ?? 0)
|
|
|
|
|
|
!== $desired['guest_prefix_length'];
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
private static function matchesAppliedPool($value, string $poolKey,
|
|
|
|
|
|
array $payload): bool
|
|
|
|
|
|
{
|
|
|
|
|
|
if (!is_array($value)
|
|
|
|
|
|
|| ($value['pool_key'] ?? null) !== $poolKey
|
|
|
|
|
|
|| ($value['revision'] ?? null) !== $payload['revision']) {
|
|
|
|
|
|
return false;
|
|
|
|
|
|
}
|
|
|
|
|
|
foreach (['state', 'network_selector', 'cidr', 'gateway',
|
|
|
|
|
|
'guest_prefix_length'] as $field) {
|
|
|
|
|
|
if (($value[$field] ?? null) !== $payload[$field]) {
|
|
|
|
|
|
return false;
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
return ($value['dns_servers'] ?? null) === $payload['dns_servers']
|
|
|
|
|
|
&& ($value['reserved_ips'] ?? null) === $payload['reserved_ips'];
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
/** Recompute selector availability without coupling independent pools. */
|
|
|
|
|
|
private static function refreshAttachmentAggregate(
|
|
|
|
|
|
CloudHostNetworkAttachment $attachment): void
|
|
|
|
|
|
{
|
|
|
|
|
|
$pools = CloudPublicIpPool::where(
|
|
|
|
|
|
'network_attachment_id', (int) $attachment->id
|
|
|
|
|
|
)->orderBy('id')->get();
|
|
|
|
|
|
$states = $pools->pluck('state')->all();
|
|
|
|
|
|
$attachment->state = in_array('active', $states, true)
|
|
|
|
|
|
? 'active'
|
|
|
|
|
|
: (in_array('draining', $states, true) ? 'draining' : 'disabled');
|
|
|
|
|
|
|
|
|
|
|
|
$rank = [
|
|
|
|
|
|
'applied' => 0,
|
|
|
|
|
|
'never' => 1,
|
|
|
|
|
|
'pending' => 2,
|
|
|
|
|
|
'failed' => 3,
|
|
|
|
|
|
'uncertain' => 4,
|
|
|
|
|
|
];
|
|
|
|
|
|
$aggregate = $pools->isEmpty() ? 'never' : 'applied';
|
|
|
|
|
|
$aggregateRank = $rank[$aggregate];
|
|
|
|
|
|
$errorCode = '';
|
|
|
|
|
|
$confirmedRevision = 0;
|
|
|
|
|
|
$desiredRevision = max(1, (int) $attachment->desired_revision);
|
|
|
|
|
|
$observedRevision = max(0, (int) $attachment->observed_revision);
|
|
|
|
|
|
foreach ($pools as $pool) {
|
|
|
|
|
|
$desiredRevision = max(
|
|
|
|
|
|
$desiredRevision, (int) $pool->desired_revision
|
|
|
|
|
|
);
|
|
|
|
|
|
$observedRevision = max(
|
|
|
|
|
|
$observedRevision, (int) $pool->observed_revision
|
|
|
|
|
|
);
|
|
|
|
|
|
$childState = (string) $pool->sync_state;
|
|
|
|
|
|
if ($childState === 'applied'
|
|
|
|
|
|
&& (int) $pool->observed_revision
|
|
|
|
|
|
< (int) $pool->desired_revision) {
|
|
|
|
|
|
$childState = 'pending';
|
|
|
|
|
|
}
|
|
|
|
|
|
if (!isset($rank[$childState])) {
|
|
|
|
|
|
$childState = 'uncertain';
|
|
|
|
|
|
}
|
|
|
|
|
|
if ($childState === 'applied') {
|
|
|
|
|
|
$confirmedRevision = max(
|
|
|
|
|
|
$confirmedRevision,
|
|
|
|
|
|
(int) $pool->observed_revision
|
|
|
|
|
|
);
|
|
|
|
|
|
}
|
|
|
|
|
|
if ($rank[$childState] > $aggregateRank) {
|
|
|
|
|
|
$aggregate = $childState;
|
|
|
|
|
|
$aggregateRank = $rank[$childState];
|
|
|
|
|
|
$candidate = (string) $pool->last_error_code;
|
|
|
|
|
|
$errorCode = preg_match('/^[a-z0-9_]{1,64}$/D', $candidate) === 1
|
|
|
|
|
|
? $candidate : 'host_public_pool_child_' . $childState;
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
if ($confirmedRevision > 0) {
|
|
|
|
|
|
// One applied child is sufficient to keep the shared selector
|
|
|
|
|
|
// usable; a failed sibling does not rewrite another pool's intent.
|
|
|
|
|
|
$evidenceRevision = max(
|
|
|
|
|
|
$desiredRevision, $observedRevision, $confirmedRevision
|
|
|
|
|
|
);
|
|
|
|
|
|
$attachment->desired_revision = $evidenceRevision;
|
|
|
|
|
|
$attachment->observed_revision = $evidenceRevision;
|
|
|
|
|
|
$attachment->sync_state = 'applied';
|
|
|
|
|
|
$attachment->last_error_code = '';
|
|
|
|
|
|
} else {
|
|
|
|
|
|
$attachment->desired_revision = $desiredRevision;
|
|
|
|
|
|
$attachment->observed_revision = min(
|
|
|
|
|
|
$observedRevision, max(0, $desiredRevision - 1)
|
|
|
|
|
|
);
|
|
|
|
|
|
$attachment->sync_state = $aggregate;
|
|
|
|
|
|
$attachment->last_error_code = $errorCode;
|
|
|
|
|
|
}
|
|
|
|
|
|
$attachment->save();
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
/** Materialize the configured CIDR into the panel-owned address ledger. */
|
|
|
|
|
|
private static function materializeIpLedger(CloudPublicIpPool $pool): void
|
|
|
|
|
|
{
|
|
|
|
|
|
$config = self::poolConfig($pool);
|
|
|
|
|
|
[$network, $prefix] = explode('/', $config['cidr'], 2);
|
|
|
|
|
|
$networkValue = self::uint($network);
|
|
|
|
|
|
$broadcast = $networkValue | (~self::mask((int) $prefix) & 0xffffffff);
|
|
|
|
|
|
$blocked = array_fill_keys(array_merge(
|
|
|
|
|
|
[$config['gateway']], $config['dns_servers'], $config['reserved_ips']
|
|
|
|
|
|
), true);
|
|
|
|
|
|
$managementAddress = CloudPanelUsageService::managementAddress(
|
|
|
|
|
|
CloudHost::find((int) $pool->host_id)
|
|
|
|
|
|
);
|
|
|
|
|
|
if ($managementAddress !== null) {
|
|
|
|
|
|
$blocked[$managementAddress] = true;
|
|
|
|
|
|
}
|
|
|
|
|
|
// A pre-ledger panel instance may already have a public address in
|
|
|
|
|
|
// its immutable spec without a reservation/IP row (for example, an
|
|
|
|
|
|
// instance created before the inventory subsystem existed). Preserve
|
|
|
|
|
|
// that address as assigned while materializing the new ledger. This
|
|
|
|
|
|
// is a database migration of panel-owned facts, never a host scan.
|
|
|
|
|
|
$legacyClaims = self::legacyInstanceIpClaims((int) $pool->host_id);
|
|
|
|
|
|
$wanted = [];
|
|
|
|
|
|
for ($value = $networkValue + 1; $value < $broadcast; $value++) {
|
|
|
|
|
|
$address = long2ip($value);
|
|
|
|
|
|
if ($address !== false) {
|
|
|
|
|
|
$wanted[$address] = isset($blocked[$address]) ? 'disabled' : 'available';
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
$rows = CloudPublicIp::where('public_ip_pool_id', (int) $pool->id)
|
|
|
|
|
|
->orderBy('id')->lockForUpdate()->get();
|
|
|
|
|
|
$byAddress = [];
|
|
|
|
|
|
foreach ($rows as $row) {
|
|
|
|
|
|
$byAddress[(string) $row->address] = $row;
|
|
|
|
|
|
}
|
|
|
|
|
|
$now = time();
|
|
|
|
|
|
foreach ($wanted as $address => $state) {
|
|
|
|
|
|
$row = $byAddress[$address] ?? null;
|
|
|
|
|
|
if ($row === null) {
|
|
|
|
|
|
$row = new CloudPublicIp();
|
|
|
|
|
|
$row->host_id = (int) $pool->host_id;
|
|
|
|
|
|
$row->public_ip_pool_id = (int) $pool->id;
|
|
|
|
|
|
$row->address = $address;
|
|
|
|
|
|
$row->created_at = $now;
|
|
|
|
|
|
}
|
|
|
|
|
|
$row->prefix_length = (int) $config['guest_prefix_length'];
|
|
|
|
|
|
if ($row->reservation_id === null && $row->instance_id === null
|
|
|
|
|
|
&& $row->operation_id === null) {
|
|
|
|
|
|
$legacyInstanceId = $legacyClaims[$address] ?? null;
|
|
|
|
|
|
$row->state = $legacyInstanceId === null ? $state
|
|
|
|
|
|
: ($legacyInstanceId > 0 ? 'assigned' : 'uncertain');
|
|
|
|
|
|
if ($legacyInstanceId > 0) {
|
|
|
|
|
|
$row->instance_id = $legacyInstanceId;
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
$row->updated_at = $now;
|
|
|
|
|
|
$row->save();
|
|
|
|
|
|
unset($byAddress[$address]);
|
|
|
|
|
|
}
|
|
|
|
|
|
foreach ($byAddress as $row) {
|
|
|
|
|
|
if ($row->reservation_id === null && $row->instance_id === null
|
|
|
|
|
|
&& $row->operation_id === null) {
|
|
|
|
|
|
$row->state = 'disabled';
|
|
|
|
|
|
$row->updated_at = $now;
|
|
|
|
|
|
$row->save();
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
/** @return array<string,int> address => existing panel instance id */
|
|
|
|
|
|
private static function legacyInstanceIpClaims(int $hostId): array
|
|
|
|
|
|
{
|
|
|
|
|
|
$claims = [];
|
|
|
|
|
|
foreach (CloudInstance::where('host_id', $hostId)
|
|
|
|
|
|
->where('status', '<>', 'deleted')->get(['id', 'spec_json']) as $instance) {
|
|
|
|
|
|
try {
|
|
|
|
|
|
$spec = json_decode((string) $instance->spec_json, true, 32,
|
|
|
|
|
|
JSON_THROW_ON_ERROR);
|
|
|
|
|
|
} catch (\Throwable $error) {
|
|
|
|
|
|
continue;
|
|
|
|
|
|
}
|
|
|
|
|
|
if (!is_array($spec) || array_is_list($spec)) {
|
|
|
|
|
|
continue;
|
|
|
|
|
|
}
|
|
|
|
|
|
$value = $spec['public_ip_cidr'] ?? $spec['public_ip'] ?? null;
|
|
|
|
|
|
if (!is_string($value) || trim($value) === '') {
|
|
|
|
|
|
continue;
|
|
|
|
|
|
}
|
|
|
|
|
|
$address = trim(explode('/', $value, 2)[0]);
|
|
|
|
|
|
if (filter_var($address, FILTER_VALIDATE_IP,
|
|
|
|
|
|
FILTER_FLAG_IPV4) === false) {
|
|
|
|
|
|
continue;
|
|
|
|
|
|
}
|
|
|
|
|
|
// Duplicate legacy metadata is an ambiguity. Keep the address
|
|
|
|
|
|
// unavailable, but do not attach it to either instance.
|
|
|
|
|
|
if (array_key_exists($address, $claims)) {
|
|
|
|
|
|
$claims[$address] = 0;
|
|
|
|
|
|
continue;
|
|
|
|
|
|
}
|
|
|
|
|
|
$claims[$address] = (int) $instance->id;
|
|
|
|
|
|
}
|
|
|
|
|
|
return $claims;
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
private static function poolDto(?CloudPublicIpPool $row, ?array $live,
|
|
|
|
|
|
int $occupied, bool $readOnly): array
|
|
|
|
|
|
{
|
|
|
|
|
|
$desired = $row === null ? null : self::poolConfig($row);
|
|
|
|
|
|
return [
|
|
|
|
|
|
'pool_key' => (string) ($row?->pool_key ?? $live['pool_key']),
|
|
|
|
|
|
'name' => self::safeText(
|
|
|
|
|
|
(string) ($row?->name ?? $live['cidr']), 80
|
|
|
|
|
|
),
|
|
|
|
|
|
'cidr' => (string) ($desired['cidr'] ?? $live['cidr']),
|
|
|
|
|
|
'gateway' => (string) ($desired['gateway'] ?? $live['gateway']),
|
|
|
|
|
|
'dns_servers' => $desired['dns_servers'] ?? $live['dns_servers'],
|
|
|
|
|
|
'reserved_ips' => $desired['reserved_ips'] ?? $live['reserved_ips'],
|
|
|
|
|
|
'guest_prefix_length' => (int) (
|
|
|
|
|
|
$desired['guest_prefix_length'] ?? $live['guest_prefix_length']
|
|
|
|
|
|
),
|
|
|
|
|
|
'state' => (string) ($row?->state ?? $live['state']),
|
|
|
|
|
|
'revision' => (int) ($row?->desired_revision ?? $live['revision']),
|
|
|
|
|
|
'expected_revision' => max(
|
|
|
|
|
|
(int) ($row?->desired_revision ?? 0),
|
|
|
|
|
|
(int) ($row?->observed_revision ?? 0),
|
|
|
|
|
|
(int) ($live['revision'] ?? 0)
|
|
|
|
|
|
),
|
|
|
|
|
|
'observed_revision' => (int) (
|
|
|
|
|
|
$row?->observed_revision ?? $live['revision']
|
|
|
|
|
|
),
|
|
|
|
|
|
'live_revision' => $row === null ? null : (int) $row->observed_revision,
|
|
|
|
|
|
'network_selector' => (string) ($live['network_selector'] ?? ''),
|
|
|
|
|
|
'sync_state' => (string) ($row?->sync_state ?? 'untracked'),
|
|
|
|
|
|
'sync_message' => self::syncMessage(
|
|
|
|
|
|
(string) ($row?->sync_state ?? 'untracked')
|
|
|
|
|
|
),
|
|
|
|
|
|
'occupied_count' => max(0, $occupied),
|
|
|
|
|
|
'read_only' => $readOnly,
|
|
|
|
|
|
];
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
private static function legacyDto(array $live): array
|
|
|
|
|
|
{
|
|
|
|
|
|
return [
|
|
|
|
|
|
'pool_key' => 'legacy-default',
|
|
|
|
|
|
'name' => '旧版默认地址池',
|
|
|
|
|
|
'cidr' => $live['cidr'],
|
|
|
|
|
|
'gateway' => $live['gateway'],
|
|
|
|
|
|
'dns_servers' => $live['dns_servers'],
|
|
|
|
|
|
'reserved_ips' => $live['reserved_ips'],
|
|
|
|
|
|
'guest_prefix_length' => $live['guest_prefix_length'],
|
|
|
|
|
|
'state' => $live['state'],
|
|
|
|
|
|
'revision' => $live['revision'],
|
|
|
|
|
|
'expected_revision' => null,
|
|
|
|
|
|
'observed_revision' => $live['revision'],
|
|
|
|
|
|
'live_revision' => $live['revision'],
|
|
|
|
|
|
'network_selector' => $live['network_selector'],
|
|
|
|
|
|
'sync_state' => 'legacy',
|
|
|
|
|
|
'sync_message' => '旧版兼容配置(只读)',
|
|
|
|
|
|
'occupied_count' => 0,
|
|
|
|
|
|
'read_only' => true,
|
|
|
|
|
|
];
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
private static function legacyRowDto(CloudPublicIpPool $row): array
|
|
|
|
|
|
{
|
|
|
|
|
|
$config = self::poolConfig($row);
|
|
|
|
|
|
return [
|
|
|
|
|
|
'pool_key' => 'legacy-default',
|
|
|
|
|
|
'name' => '旧版默认地址池',
|
|
|
|
|
|
'cidr' => $config['cidr'],
|
|
|
|
|
|
'gateway' => $config['gateway'],
|
|
|
|
|
|
'dns_servers' => $config['dns_servers'],
|
|
|
|
|
|
'reserved_ips' => $config['reserved_ips'],
|
|
|
|
|
|
'guest_prefix_length' => $config['guest_prefix_length'],
|
|
|
|
|
|
'state' => (string) $row->state,
|
|
|
|
|
|
'revision' => (int) $row->desired_revision,
|
|
|
|
|
|
'expected_revision' => null,
|
|
|
|
|
|
'observed_revision' => (int) $row->observed_revision,
|
|
|
|
|
|
'live_revision' => null,
|
|
|
|
|
|
'network_selector' => '',
|
|
|
|
|
|
'sync_state' => 'legacy',
|
|
|
|
|
|
'sync_message' => '旧版开发数据(只读)',
|
|
|
|
|
|
'occupied_count' => 0,
|
|
|
|
|
|
'read_only' => true,
|
|
|
|
|
|
];
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
private static function decodeIpList($value): array
|
|
|
|
|
|
{
|
|
|
|
|
|
try {
|
|
|
|
|
|
$decoded = json_decode((string) $value, true, 32, JSON_THROW_ON_ERROR);
|
|
|
|
|
|
} catch (\Throwable $error) {
|
|
|
|
|
|
return [];
|
|
|
|
|
|
}
|
|
|
|
|
|
return is_array($decoded) && array_is_list($decoded)
|
|
|
|
|
|
? array_values(array_filter($decoded, 'is_string')) : [];
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
private static function syncMessage(string $state): string
|
|
|
|
|
|
{
|
|
|
|
|
|
return [
|
|
|
|
|
|
'applied' => '已应用',
|
|
|
|
|
|
'pending' => '等待引擎应用',
|
|
|
|
|
|
'uncertain' => '应用结果待核对',
|
|
|
|
|
|
'failed' => '引擎未接受配置',
|
|
|
|
|
|
'never' => '尚未应用',
|
|
|
|
|
|
'untracked' => '面板配置',
|
|
|
|
|
|
][$state] ?? '应用状态未知';
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
private static function safeWriteError(CloudError $error,
|
|
|
|
|
|
bool $uncertain): CloudError
|
|
|
|
|
|
{
|
|
|
|
|
|
if ($uncertain) {
|
|
|
|
|
|
return new CloudError(
|
|
|
|
|
|
'公网地址池应用状态不确定,请勿重复提交,稍后刷新核对',
|
|
|
|
|
|
502
|
|
|
|
|
|
);
|
|
|
|
|
|
}
|
|
|
|
|
|
return new CloudError(
|
|
|
|
|
|
$error->httpStatus === 409
|
|
|
|
|
|
? '公网地址池配置冲突,请刷新后核对版本、网段和地址占用'
|
|
|
|
|
|
: '公网地址池配置未通过引擎校验,请检查网段、网关、DNS和保留地址',
|
|
|
|
|
|
$error->httpStatus === 409 ? 409 : 422
|
|
|
|
|
|
);
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
private function host(int $hostId): CloudHost
|
|
|
|
|
|
{
|
|
|
|
|
|
if ($hostId < 1 || ($host = CloudHost::find($hostId)) === null) {
|
|
|
|
|
|
throw new CloudError('宿主机不存在', 404);
|
|
|
|
|
|
}
|
|
|
|
|
|
return $host;
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
private static function safeText(string $value, int $maximum): string
|
|
|
|
|
|
{
|
|
|
|
|
|
$value = preg_replace('/[\x00-\x1f\x7f]/u', '', $value) ?? '';
|
|
|
|
|
|
return mb_substr(trim($value), 0, $maximum);
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|