You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
virt/web/app/Services/Cloud/CloudTemplateReconciliation...

457 lines
18 KiB
PHP

<?php
namespace App\Services\Cloud;
use App\Models\CloudTemplate;
use App\Models\CloudTemplateReplica;
/** Compare Agent catalogs with PHP intent, then queue only proven-safe repairs. */
final class CloudTemplateReconciliationService
{
private const TEMPLATE_STATES = ['active', 'degraded', 'error'];
private const MAX_REMOTE_TEMPLATES = 2048;
public function __construct(
private ?\Closure $clientFactory = null,
private ?\Closure $repairer = null
) {
}
/**
* @return array{hosts_checked:int,templates_checked:int,exact:int,
* missing:int,stale:int,divergent:int,orphans:int,scheduled:int,
* deferred:int,errors:int,last_host_id:int}
*/
public function reconcile(
array $actor,
int $hostLimit = 4,
int $repairLimit = 8,
int $afterHostId = 0
): array {
CloudAccess::manage($actor);
$hostLimit = max(1, min(32, $hostLimit));
$repairLimit = max(1, min(64, $repairLimit));
$eligibleHostIds = array_values(array_unique(array_map(
'intval',
CloudAgentSettingsService::enabledHostIds()
)));
sort($eligibleHostIds, SORT_NUMERIC);
$hostIds = self::rotatedHosts(
$eligibleHostIds,
$afterHostId,
$hostLimit
);
$report = [
'hosts_checked' => 0,
'templates_checked' => 0,
'exact' => 0,
'missing' => 0,
'stale' => 0,
'divergent' => 0,
'orphans' => 0,
'scheduled' => 0,
'deferred' => 0,
'errors' => 0,
'last_host_id' => $afterHostId,
];
$remainingRepairs = $repairLimit;
foreach ($hostIds as $hostId) {
$report['last_host_id'] = $hostId;
try {
$remote = $this->readHostCatalog($hostId, $actor);
$observation = $this->observeHost(
$hostId,
$remote,
$eligibleHostIds
);
$report['hosts_checked']++;
foreach ([
'templates_checked', 'exact', 'missing', 'stale',
'divergent', 'orphans', 'deferred',
] as $field) {
$report[$field] += $observation[$field];
}
foreach ($observation['repairs'] as $repair) {
if ($remainingRepairs < 1) {
$report['deferred']++;
continue;
}
try {
$result = $this->repair(
$repair['action'],
$repair['template_id'],
$hostId,
$actor
);
if (($result['scheduled'] ?? false) === true) {
$report['scheduled']++;
$remainingRepairs--;
}
} catch (\Throwable $error) {
$report['deferred']++;
error_log(sprintf(
'cloud template reconcile host=%d template=%s action=%s deferred: %s',
$hostId,
$repair['template_id'],
$repair['action'],
get_class($error)
));
}
}
} catch (\Throwable $error) {
$report['errors']++;
error_log(sprintf(
'cloud template reconcile host=%d unavailable: %s',
$hostId,
get_class($error)
));
}
}
return $report;
}
/** @param list<int> $hostIds @return list<int> */
private static function rotatedHosts(
array $hostIds,
int $afterHostId,
int $limit
): array {
if ($hostIds === []) {
return [];
}
$tail = array_values(array_filter(
$hostIds,
static fn (int $hostId): bool => $hostId > $afterHostId
));
$head = array_values(array_filter(
$hostIds,
static fn (int $hostId): bool => $hostId <= $afterHostId
));
return array_slice(array_merge($tail, $head), 0, $limit);
}
/** @return array<string,array<string,mixed>> */
private function readHostCatalog(int $hostId, array $actor): array
{
$client = $this->clientFactory === null
? CloudAgentSettingsService::clientForHost($hostId, 30.0)
: ($this->clientFactory)($hostId);
if (!is_object($client) || !method_exists($client, 'request')) {
throw new \RuntimeException('Invalid template catalog client');
}
$payload = $client->request('GET', '/v1/templates', $actor);
$rows = $payload['templates'] ?? null;
if (!is_array($rows) || !array_is_list($rows)
|| count($rows) > self::MAX_REMOTE_TEMPLATES) {
throw new CloudError('宿主机云镜像目录响应无效', 502);
}
$catalog = [];
foreach ($rows as $row) {
$normalized = self::remoteTemplate($row);
$id = $normalized['id'];
if (isset($catalog[$id])) {
throw new CloudError('宿主机云镜像目录包含重复标识', 502);
}
$catalog[$id] = $normalized;
}
ksort($catalog, SORT_STRING);
return $catalog;
}
/** @return array<string,mixed> */
private static function remoteTemplate($row): array
{
if (!is_array($row) || array_is_list($row)) {
throw new CloudError('宿主机云镜像目录条目无效', 502);
}
$id = $row['id'] ?? null;
$name = $row['name'] ?? null;
$family = $row['os_family'] ?? null;
$version = $row['version'] ?? null;
$architecture = $row['architecture'] ?? null;
$generation = $row['generation_token'] ?? null;
$sha256 = strtolower((string) ($row['sha256'] ?? ''));
$revision = filter_var($row['revision'] ?? null,
FILTER_VALIDATE_INT, ['options' => ['min_range' => 1]]);
$bytes = filter_var($row['size_bytes'] ?? null,
FILTER_VALIDATE_INT, ['options' => ['min_range' => 1]]);
$virtualBytes = filter_var($row['virtual_size_bytes'] ?? null,
FILTER_VALIDATE_INT, ['options' => ['min_range' => 1]]);
if (!is_string($id)
|| preg_match('/^[A-Za-z0-9][A-Za-z0-9_.-]{0,79}$/D', $id) !== 1
|| !self::plainLabel($name, 64)
|| !self::plainLabel($family, 64)
|| !self::plainLabel($version, 64)
|| $architecture !== 'x86_64'
|| !is_string($generation)
|| preg_match('/^[a-f0-9]{32}$/D', $generation) !== 1
|| preg_match('/^[a-f0-9]{64}$/D', $sha256) !== 1
|| $revision === false || $bytes === false
|| $virtualBytes === false) {
throw new CloudError('宿主机云镜像目录条目字段无效', 502);
}
return [
'id' => $id,
'name' => trim($name),
'os_family' => trim($family),
'version' => trim($version),
'architecture' => $architecture,
'revision' => (int) $revision,
'generation_token' => $generation,
'size_bytes' => (int) $bytes,
'virtual_size_bytes' => (int) $virtualBytes,
'sha256' => $sha256,
];
}
private static function plainLabel($value, int $maximum): bool
{
return is_string($value) && trim($value) !== ''
&& mb_strlen($value) <= $maximum
&& preg_match('/[\x00-\x1f\x7f]/u', $value) !== 1;
}
/**
* @param array<string,array<string,mixed>> $remote
* @param list<int> $eligibleHostIds
* @return array{templates_checked:int,exact:int,missing:int,stale:int,
* divergent:int,orphans:int,deferred:int,
* repairs:list<array{action:string,template_id:string}>}
*/
private function observeHost(
int $hostId,
array $remote,
array $eligibleHostIds
): array {
$templates = CloudTemplate::query()
->where('is_default', 1)
->whereIn('state', self::TEMPLATE_STATES)
->orderBy('catalog_key')->orderBy('version_label')
->orderBy('id')->get();
$knownIds = $templates->map(
static fn (CloudTemplate $template): string =>
(string) $template->id
)->all();
$report = [
'templates_checked' => 0,
'exact' => 0,
'missing' => 0,
'stale' => 0,
'divergent' => 0,
'orphans' => count(array_diff(array_keys($remote), $knownIds)),
'deferred' => 0,
'repairs' => [],
];
foreach ($templates as $snapshot) {
$templateId = (string) $snapshot->id;
$classification = self::classify(
$snapshot,
$remote[$templateId] ?? null
);
$report['templates_checked']++;
$report[$classification]++;
try {
$repair = $snapshot->getConnection()->transaction(
static function () use (
$templateId,
$hostId,
$classification,
$remote,
$eligibleHostIds
): ?array {
$template = CloudTemplateCatalogService::
lockForReplicaSync($templateId);
if ($template === null
|| !in_array((string) $template->state,
self::TEMPLATE_STATES, true)) {
return null;
}
$actual = $remote[$templateId] ?? null;
$currentClassification = self::classify(
$template,
$actual
);
if ($currentClassification !== $classification) {
return null;
}
$replica = CloudTemplateReplica::query()
->where('host_id', $hostId)
->where('template_id', $templateId)
->lockForUpdate()->first();
if ($replica !== null) {
self::recordObservation(
$replica,
$currentClassification,
$actual
);
}
$action = null;
if ($currentClassification === 'missing') {
if ($replica === null) {
$action = 'import';
} elseif (in_array((string) $replica->state, [
'active', 'stale',
], true)) {
CloudTemplateStateMachine::moveReplica(
$replica,
'failed'
);
$replica->error_code =
'template_observed_missing';
$replica->updated_at = time();
$replica->save();
$action = 'import';
} elseif ((string) $replica->state === 'failed') {
$action = 'import';
}
} elseif ($currentClassification === 'stale'
&& $replica !== null
&& in_array((string) $replica->state, [
'active', 'stale',
], true)) {
CloudTemplateStateMachine::moveReplica(
$replica,
'stale'
);
$replica->error_code =
'template_observed_stale';
$replica->updated_at = time();
$replica->save();
$action = 'metadata';
} elseif ($currentClassification === 'divergent'
&& $replica !== null
&& in_array((string) $replica->state, [
'active', 'stale',
], true)) {
CloudTemplateStateMachine::moveReplica(
$replica,
'failed'
);
$replica->error_code =
'template_observed_divergent';
$replica->updated_at = time();
$replica->save();
}
if ($currentClassification !== 'exact') {
$replicas = CloudTemplateReplica::query()
->where('template_id', $templateId)
->orderBy('host_id')->lockForUpdate()->get();
CloudTemplateStateMachine::moveTemplate(
$template,
CloudTemplateStateMachine::availability(
$template,
$replicas,
$eligibleHostIds
)
);
$template->updated_at = time();
$template->save();
}
return $action === null ? null : [
'action' => $action,
'template_id' => $templateId,
];
},
3
);
if ($repair !== null) {
$report['repairs'][] = $repair;
}
} catch (\Throwable $error) {
$report['deferred']++;
error_log(sprintf(
'cloud template observation host=%d template=%s deferred: %s',
$hostId,
$templateId,
get_class($error)
));
}
}
return $report;
}
private static function classify(
CloudTemplate $template,
?array $remote
): string {
if ($remote === null) {
return 'missing';
}
$sameContent = (int) $remote['size_bytes']
=== (int) $template->image_bytes
&& hash_equals(
(string) $template->sha256,
(string) $remote['sha256']
)
&& hash_equals(
(string) $template->generation_token,
(string) $remote['generation_token']
);
if ($sameContent
&& (int) $remote['revision'] === (int) $template->revision
&& (string) $remote['name'] === (string) $template->name
&& (string) $remote['os_family'] === (string) $template->os_family
&& (string) $remote['version'] === (string) $template->os_version) {
return 'exact';
}
if ($sameContent
&& (int) $remote['revision'] < (int) $template->revision
&& (string) $remote['os_family'] === (string) $template->os_family
&& (string) $remote['version'] === (string) $template->os_version) {
return 'stale';
}
return 'divergent';
}
private static function recordObservation(
CloudTemplateReplica $replica,
string $classification,
?array $remote
): void {
$replica->observed_state = $classification;
$replica->observed_revision = $remote === null
? 0 : (int) $remote['revision'];
$replica->observed_generation_token = $remote === null
? '' : (string) $remote['generation_token'];
$replica->observed_image_bytes = $remote === null
? 0 : (int) $remote['size_bytes'];
$replica->observed_sha256 = $remote === null
? '' : (string) $remote['sha256'];
$replica->last_verified_at = time();
$replica->save();
}
private function repair(
string $action,
string $templateId,
int $hostId,
array $actor
): array {
if ($this->repairer !== null) {
$result = ($this->repairer)(
$action,
$templateId,
$hostId,
$actor
);
if (!is_array($result)) {
throw new \RuntimeException('Invalid template repair result');
}
return $result;
}
return $action === 'metadata'
? CloudTemplateReplicaCoordinator::syncMetadataHost(
$templateId,
$hostId,
$actor,
'periodic_reconcile'
)
: CloudTemplateReplicaCoordinator::syncHost(
$templateId,
$hostId,
$actor,
'periodic_reconcile'
);
}
}