You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
457 lines
18 KiB
PHP
457 lines
18 KiB
PHP
<?php
|
|
|
|
namespace App\Services\Cloud;
|
|
|
|
use App\Models\CloudTemplate;
|
|
use App\Models\CloudTemplateReplica;
|
|
|
|
/** Compare Agent catalogs with PHP intent, then queue only proven-safe repairs. */
|
|
final class CloudTemplateReconciliationService
|
|
{
|
|
private const TEMPLATE_STATES = ['active', 'degraded', 'error'];
|
|
private const MAX_REMOTE_TEMPLATES = 2048;
|
|
|
|
public function __construct(
|
|
private ?\Closure $clientFactory = null,
|
|
private ?\Closure $repairer = null
|
|
) {
|
|
}
|
|
|
|
/**
|
|
* @return array{hosts_checked:int,templates_checked:int,exact:int,
|
|
* missing:int,stale:int,divergent:int,orphans:int,scheduled:int,
|
|
* deferred:int,errors:int,last_host_id:int}
|
|
*/
|
|
public function reconcile(
|
|
array $actor,
|
|
int $hostLimit = 4,
|
|
int $repairLimit = 8,
|
|
int $afterHostId = 0
|
|
): array {
|
|
CloudAccess::manage($actor);
|
|
$hostLimit = max(1, min(32, $hostLimit));
|
|
$repairLimit = max(1, min(64, $repairLimit));
|
|
$eligibleHostIds = array_values(array_unique(array_map(
|
|
'intval',
|
|
CloudAgentSettingsService::enabledHostIds()
|
|
)));
|
|
sort($eligibleHostIds, SORT_NUMERIC);
|
|
$hostIds = self::rotatedHosts(
|
|
$eligibleHostIds,
|
|
$afterHostId,
|
|
$hostLimit
|
|
);
|
|
$report = [
|
|
'hosts_checked' => 0,
|
|
'templates_checked' => 0,
|
|
'exact' => 0,
|
|
'missing' => 0,
|
|
'stale' => 0,
|
|
'divergent' => 0,
|
|
'orphans' => 0,
|
|
'scheduled' => 0,
|
|
'deferred' => 0,
|
|
'errors' => 0,
|
|
'last_host_id' => $afterHostId,
|
|
];
|
|
$remainingRepairs = $repairLimit;
|
|
foreach ($hostIds as $hostId) {
|
|
$report['last_host_id'] = $hostId;
|
|
try {
|
|
$remote = $this->readHostCatalog($hostId, $actor);
|
|
$observation = $this->observeHost(
|
|
$hostId,
|
|
$remote,
|
|
$eligibleHostIds
|
|
);
|
|
$report['hosts_checked']++;
|
|
foreach ([
|
|
'templates_checked', 'exact', 'missing', 'stale',
|
|
'divergent', 'orphans', 'deferred',
|
|
] as $field) {
|
|
$report[$field] += $observation[$field];
|
|
}
|
|
foreach ($observation['repairs'] as $repair) {
|
|
if ($remainingRepairs < 1) {
|
|
$report['deferred']++;
|
|
continue;
|
|
}
|
|
try {
|
|
$result = $this->repair(
|
|
$repair['action'],
|
|
$repair['template_id'],
|
|
$hostId,
|
|
$actor
|
|
);
|
|
if (($result['scheduled'] ?? false) === true) {
|
|
$report['scheduled']++;
|
|
$remainingRepairs--;
|
|
}
|
|
} catch (\Throwable $error) {
|
|
$report['deferred']++;
|
|
error_log(sprintf(
|
|
'cloud template reconcile host=%d template=%s action=%s deferred: %s',
|
|
$hostId,
|
|
$repair['template_id'],
|
|
$repair['action'],
|
|
get_class($error)
|
|
));
|
|
}
|
|
}
|
|
} catch (\Throwable $error) {
|
|
$report['errors']++;
|
|
error_log(sprintf(
|
|
'cloud template reconcile host=%d unavailable: %s',
|
|
$hostId,
|
|
get_class($error)
|
|
));
|
|
}
|
|
}
|
|
return $report;
|
|
}
|
|
|
|
/** @param list<int> $hostIds @return list<int> */
|
|
private static function rotatedHosts(
|
|
array $hostIds,
|
|
int $afterHostId,
|
|
int $limit
|
|
): array {
|
|
if ($hostIds === []) {
|
|
return [];
|
|
}
|
|
$tail = array_values(array_filter(
|
|
$hostIds,
|
|
static fn (int $hostId): bool => $hostId > $afterHostId
|
|
));
|
|
$head = array_values(array_filter(
|
|
$hostIds,
|
|
static fn (int $hostId): bool => $hostId <= $afterHostId
|
|
));
|
|
return array_slice(array_merge($tail, $head), 0, $limit);
|
|
}
|
|
|
|
/** @return array<string,array<string,mixed>> */
|
|
private function readHostCatalog(int $hostId, array $actor): array
|
|
{
|
|
$client = $this->clientFactory === null
|
|
? CloudAgentSettingsService::clientForHost($hostId, 30.0)
|
|
: ($this->clientFactory)($hostId);
|
|
if (!is_object($client) || !method_exists($client, 'request')) {
|
|
throw new \RuntimeException('Invalid template catalog client');
|
|
}
|
|
$payload = $client->request('GET', '/v1/templates', $actor);
|
|
$rows = $payload['templates'] ?? null;
|
|
if (!is_array($rows) || !array_is_list($rows)
|
|
|| count($rows) > self::MAX_REMOTE_TEMPLATES) {
|
|
throw new CloudError('宿主机云镜像目录响应无效', 502);
|
|
}
|
|
$catalog = [];
|
|
foreach ($rows as $row) {
|
|
$normalized = self::remoteTemplate($row);
|
|
$id = $normalized['id'];
|
|
if (isset($catalog[$id])) {
|
|
throw new CloudError('宿主机云镜像目录包含重复标识', 502);
|
|
}
|
|
$catalog[$id] = $normalized;
|
|
}
|
|
ksort($catalog, SORT_STRING);
|
|
return $catalog;
|
|
}
|
|
|
|
/** @return array<string,mixed> */
|
|
private static function remoteTemplate($row): array
|
|
{
|
|
if (!is_array($row) || array_is_list($row)) {
|
|
throw new CloudError('宿主机云镜像目录条目无效', 502);
|
|
}
|
|
$id = $row['id'] ?? null;
|
|
$name = $row['name'] ?? null;
|
|
$family = $row['os_family'] ?? null;
|
|
$version = $row['version'] ?? null;
|
|
$architecture = $row['architecture'] ?? null;
|
|
$generation = $row['generation_token'] ?? null;
|
|
$sha256 = strtolower((string) ($row['sha256'] ?? ''));
|
|
$revision = filter_var($row['revision'] ?? null,
|
|
FILTER_VALIDATE_INT, ['options' => ['min_range' => 1]]);
|
|
$bytes = filter_var($row['size_bytes'] ?? null,
|
|
FILTER_VALIDATE_INT, ['options' => ['min_range' => 1]]);
|
|
$virtualBytes = filter_var($row['virtual_size_bytes'] ?? null,
|
|
FILTER_VALIDATE_INT, ['options' => ['min_range' => 1]]);
|
|
if (!is_string($id)
|
|
|| preg_match('/^[A-Za-z0-9][A-Za-z0-9_.-]{0,79}$/D', $id) !== 1
|
|
|| !self::plainLabel($name, 64)
|
|
|| !self::plainLabel($family, 64)
|
|
|| !self::plainLabel($version, 64)
|
|
|| $architecture !== 'x86_64'
|
|
|| !is_string($generation)
|
|
|| preg_match('/^[a-f0-9]{32}$/D', $generation) !== 1
|
|
|| preg_match('/^[a-f0-9]{64}$/D', $sha256) !== 1
|
|
|| $revision === false || $bytes === false
|
|
|| $virtualBytes === false) {
|
|
throw new CloudError('宿主机云镜像目录条目字段无效', 502);
|
|
}
|
|
return [
|
|
'id' => $id,
|
|
'name' => trim($name),
|
|
'os_family' => trim($family),
|
|
'version' => trim($version),
|
|
'architecture' => $architecture,
|
|
'revision' => (int) $revision,
|
|
'generation_token' => $generation,
|
|
'size_bytes' => (int) $bytes,
|
|
'virtual_size_bytes' => (int) $virtualBytes,
|
|
'sha256' => $sha256,
|
|
];
|
|
}
|
|
|
|
private static function plainLabel($value, int $maximum): bool
|
|
{
|
|
return is_string($value) && trim($value) !== ''
|
|
&& mb_strlen($value) <= $maximum
|
|
&& preg_match('/[\x00-\x1f\x7f]/u', $value) !== 1;
|
|
}
|
|
|
|
/**
|
|
* @param array<string,array<string,mixed>> $remote
|
|
* @param list<int> $eligibleHostIds
|
|
* @return array{templates_checked:int,exact:int,missing:int,stale:int,
|
|
* divergent:int,orphans:int,deferred:int,
|
|
* repairs:list<array{action:string,template_id:string}>}
|
|
*/
|
|
private function observeHost(
|
|
int $hostId,
|
|
array $remote,
|
|
array $eligibleHostIds
|
|
): array {
|
|
$templates = CloudTemplate::query()
|
|
->where('is_default', 1)
|
|
->whereIn('state', self::TEMPLATE_STATES)
|
|
->orderBy('catalog_key')->orderBy('version_label')
|
|
->orderBy('id')->get();
|
|
$knownIds = $templates->map(
|
|
static fn (CloudTemplate $template): string =>
|
|
(string) $template->id
|
|
)->all();
|
|
$report = [
|
|
'templates_checked' => 0,
|
|
'exact' => 0,
|
|
'missing' => 0,
|
|
'stale' => 0,
|
|
'divergent' => 0,
|
|
'orphans' => count(array_diff(array_keys($remote), $knownIds)),
|
|
'deferred' => 0,
|
|
'repairs' => [],
|
|
];
|
|
foreach ($templates as $snapshot) {
|
|
$templateId = (string) $snapshot->id;
|
|
$classification = self::classify(
|
|
$snapshot,
|
|
$remote[$templateId] ?? null
|
|
);
|
|
$report['templates_checked']++;
|
|
$report[$classification]++;
|
|
try {
|
|
$repair = $snapshot->getConnection()->transaction(
|
|
static function () use (
|
|
$templateId,
|
|
$hostId,
|
|
$classification,
|
|
$remote,
|
|
$eligibleHostIds
|
|
): ?array {
|
|
$template = CloudTemplateCatalogService::
|
|
lockForReplicaSync($templateId);
|
|
if ($template === null
|
|
|| !in_array((string) $template->state,
|
|
self::TEMPLATE_STATES, true)) {
|
|
return null;
|
|
}
|
|
$actual = $remote[$templateId] ?? null;
|
|
$currentClassification = self::classify(
|
|
$template,
|
|
$actual
|
|
);
|
|
if ($currentClassification !== $classification) {
|
|
return null;
|
|
}
|
|
$replica = CloudTemplateReplica::query()
|
|
->where('host_id', $hostId)
|
|
->where('template_id', $templateId)
|
|
->lockForUpdate()->first();
|
|
if ($replica !== null) {
|
|
self::recordObservation(
|
|
$replica,
|
|
$currentClassification,
|
|
$actual
|
|
);
|
|
}
|
|
$action = null;
|
|
if ($currentClassification === 'missing') {
|
|
if ($replica === null) {
|
|
$action = 'import';
|
|
} elseif (in_array((string) $replica->state, [
|
|
'active', 'stale',
|
|
], true)) {
|
|
CloudTemplateStateMachine::moveReplica(
|
|
$replica,
|
|
'failed'
|
|
);
|
|
$replica->error_code =
|
|
'template_observed_missing';
|
|
$replica->updated_at = time();
|
|
$replica->save();
|
|
$action = 'import';
|
|
} elseif ((string) $replica->state === 'failed') {
|
|
$action = 'import';
|
|
}
|
|
} elseif ($currentClassification === 'stale'
|
|
&& $replica !== null
|
|
&& in_array((string) $replica->state, [
|
|
'active', 'stale',
|
|
], true)) {
|
|
CloudTemplateStateMachine::moveReplica(
|
|
$replica,
|
|
'stale'
|
|
);
|
|
$replica->error_code =
|
|
'template_observed_stale';
|
|
$replica->updated_at = time();
|
|
$replica->save();
|
|
$action = 'metadata';
|
|
} elseif ($currentClassification === 'divergent'
|
|
&& $replica !== null
|
|
&& in_array((string) $replica->state, [
|
|
'active', 'stale',
|
|
], true)) {
|
|
CloudTemplateStateMachine::moveReplica(
|
|
$replica,
|
|
'failed'
|
|
);
|
|
$replica->error_code =
|
|
'template_observed_divergent';
|
|
$replica->updated_at = time();
|
|
$replica->save();
|
|
}
|
|
if ($currentClassification !== 'exact') {
|
|
$replicas = CloudTemplateReplica::query()
|
|
->where('template_id', $templateId)
|
|
->orderBy('host_id')->lockForUpdate()->get();
|
|
CloudTemplateStateMachine::moveTemplate(
|
|
$template,
|
|
CloudTemplateStateMachine::availability(
|
|
$template,
|
|
$replicas,
|
|
$eligibleHostIds
|
|
)
|
|
);
|
|
$template->updated_at = time();
|
|
$template->save();
|
|
}
|
|
return $action === null ? null : [
|
|
'action' => $action,
|
|
'template_id' => $templateId,
|
|
];
|
|
},
|
|
3
|
|
);
|
|
if ($repair !== null) {
|
|
$report['repairs'][] = $repair;
|
|
}
|
|
} catch (\Throwable $error) {
|
|
$report['deferred']++;
|
|
error_log(sprintf(
|
|
'cloud template observation host=%d template=%s deferred: %s',
|
|
$hostId,
|
|
$templateId,
|
|
get_class($error)
|
|
));
|
|
}
|
|
}
|
|
return $report;
|
|
}
|
|
|
|
private static function classify(
|
|
CloudTemplate $template,
|
|
?array $remote
|
|
): string {
|
|
if ($remote === null) {
|
|
return 'missing';
|
|
}
|
|
$sameContent = (int) $remote['size_bytes']
|
|
=== (int) $template->image_bytes
|
|
&& hash_equals(
|
|
(string) $template->sha256,
|
|
(string) $remote['sha256']
|
|
)
|
|
&& hash_equals(
|
|
(string) $template->generation_token,
|
|
(string) $remote['generation_token']
|
|
);
|
|
if ($sameContent
|
|
&& (int) $remote['revision'] === (int) $template->revision
|
|
&& (string) $remote['name'] === (string) $template->name
|
|
&& (string) $remote['os_family'] === (string) $template->os_family
|
|
&& (string) $remote['version'] === (string) $template->os_version) {
|
|
return 'exact';
|
|
}
|
|
if ($sameContent
|
|
&& (int) $remote['revision'] < (int) $template->revision
|
|
&& (string) $remote['os_family'] === (string) $template->os_family
|
|
&& (string) $remote['version'] === (string) $template->os_version) {
|
|
return 'stale';
|
|
}
|
|
return 'divergent';
|
|
}
|
|
|
|
private static function recordObservation(
|
|
CloudTemplateReplica $replica,
|
|
string $classification,
|
|
?array $remote
|
|
): void {
|
|
$replica->observed_state = $classification;
|
|
$replica->observed_revision = $remote === null
|
|
? 0 : (int) $remote['revision'];
|
|
$replica->observed_generation_token = $remote === null
|
|
? '' : (string) $remote['generation_token'];
|
|
$replica->observed_image_bytes = $remote === null
|
|
? 0 : (int) $remote['size_bytes'];
|
|
$replica->observed_sha256 = $remote === null
|
|
? '' : (string) $remote['sha256'];
|
|
$replica->last_verified_at = time();
|
|
$replica->save();
|
|
}
|
|
|
|
private function repair(
|
|
string $action,
|
|
string $templateId,
|
|
int $hostId,
|
|
array $actor
|
|
): array {
|
|
if ($this->repairer !== null) {
|
|
$result = ($this->repairer)(
|
|
$action,
|
|
$templateId,
|
|
$hostId,
|
|
$actor
|
|
);
|
|
if (!is_array($result)) {
|
|
throw new \RuntimeException('Invalid template repair result');
|
|
}
|
|
return $result;
|
|
}
|
|
return $action === 'metadata'
|
|
? CloudTemplateReplicaCoordinator::syncMetadataHost(
|
|
$templateId,
|
|
$hostId,
|
|
$actor,
|
|
'periodic_reconcile'
|
|
)
|
|
: CloudTemplateReplicaCoordinator::syncHost(
|
|
$templateId,
|
|
$hostId,
|
|
$actor,
|
|
'periodic_reconcile'
|
|
);
|
|
}
|
|
}
|