You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
virt/web/app/Services/Cloud/CloudObjectStorageHealthPro...

210 lines
8.5 KiB
PHP

<?php
namespace App\Services\Cloud;
use App\Models\CloudStorageBackend;
/** Periodic read-only provider checks, isolated from long replica copies. */
final class CloudObjectStorageHealthProbeService
{
private const CLAIM_SECONDS = 45;
private const HEALTHY_INTERVAL_SECONDS = 300;
private \Closure $runtimeResolver;
private \Closure $headBucket;
private \Closure $clock;
/** @var list<string> */
private array $observationErrors = [];
public function __construct(
?callable $runtimeResolver = null,
?callable $headBucket = null,
?callable $clock = null
) {
$this->runtimeResolver = \Closure::fromCallable(
$runtimeResolver ?? static fn (int $backendId): array =>
CloudObjectStorageBackendRegistry::runtime($backendId, false)
);
$this->headBucket = \Closure::fromCallable(
$headBucket ?? static function (array $config, array $credentials): void {
(new CloudObjectStorageS3Client())->headBucket($config, $credentials);
}
);
$this->clock = \Closure::fromCallable($clock ?? 'time');
}
/** Claim and inspect at most one due backend. */
public function tick(): bool
{
$this->observationErrors = [];
$claim = $this->claimDueBackend();
if ($claim === null) {
return false;
}
try {
$runtime = ($this->runtimeResolver)($claim['backend_id']);
if ((int) ($runtime['backend_id'] ?? 0) !== $claim['backend_id']
|| (int) ($runtime['revision'] ?? 0) !== $claim['revision']) {
$this->releaseChangedRevision($claim);
return true;
}
// Deliberately outside every database transaction. Credentials
// exist only in this call frame and are never persisted or logged.
($this->headBucket)($runtime['config'], $runtime['credentials']);
$this->settleSuccess($claim);
} catch (CloudObjectStorageTestError $error) {
$this->settleFailure($claim, $error->publicCode);
$this->observationErrors[] = 'storage_backend_unhealthy';
} catch (\Throwable $error) {
// Runtime/decryption/client failures are bounded to a public code;
// exception messages can contain deployment details and stay local.
$this->settleFailure($claim, 'request_rejected');
$this->observationErrors[] = 'storage_health_failed';
}
return true;
}
/** @return list<string> */
public function observationErrors(): array
{
return $this->observationErrors;
}
/** @return array{backend_id:int,revision:int,claim_until:int}|null */
private function claimDueBackend(): ?array
{
$now = $this->now();
$connection = (new CloudStorageBackend())->getConnection();
return $connection->transaction(static function () use ($now): ?array {
$backend = CloudStorageBackend::query()
->where('admin_state', '<>', 'retired')
->where(static function ($query) use ($now): void {
$query->where(static function ($due) use ($now): void {
$due->where('next_probe_at', '>', 0)
->where('next_probe_at', '<=', $now);
})->orWhere(static function ($bootstrap): void {
// Earlier registry imports can carry valid current
// health evidence without a first periodic schedule.
$bootstrap->where('next_probe_at', 0)
->where('health_state', 'healthy')
->where('tested_revision', '>', 0)
->whereColumn('tested_revision', 'config_revision');
});
})
->orderBy('next_probe_at')->orderBy('id')
->lockForUpdate()->first();
if (!$backend instanceof CloudStorageBackend) {
return null;
}
$claimUntil = $now + self::CLAIM_SECONDS;
$backend->next_probe_at = $claimUntil;
$backend->updated_at = $now;
$backend->save();
return [
'backend_id' => (int) $backend->id,
'revision' => (int) $backend->config_revision,
'claim_until' => $claimUntil,
];
});
}
/** @param array{backend_id:int,revision:int,claim_until:int} $claim */
private function settleSuccess(array $claim): bool
{
$now = $this->now();
return $this->settle($claim, static function (
CloudStorageBackend $backend
) use ($now): void {
$backend->tested_revision = (int) $backend->config_revision;
$backend->health_state = 'healthy';
$backend->consecutive_failures = 0;
$backend->consecutive_successes =
(int) $backend->consecutive_successes + 1;
$backend->last_error_code = '';
$backend->last_tested_at = $now;
$backend->last_healthy_at = $now;
$backend->next_probe_at = $now + self::HEALTHY_INTERVAL_SECONDS;
$backend->updated_at = $now;
// admin_state is intentionally untouched. A successful probe can
// restore evidence, but cannot override an administrator decision.
});
}
/** @param array{backend_id:int,revision:int,claim_until:int} $claim */
private function settleFailure(array $claim, string $code): bool
{
$now = $this->now();
return $this->settle($claim, static function (
CloudStorageBackend $backend
) use ($now, $code): void {
$backend->tested_revision = 0;
$backend->health_state = in_array($code, [
'endpoint_unreachable', 'service_unavailable', 'rate_limited',
], true) ? 'unreachable' : 'degraded';
$failures = (int) $backend->consecutive_failures + 1;
$backend->consecutive_failures = $failures;
$backend->consecutive_successes = 0;
$backend->last_error_code = $code;
$backend->last_tested_at = $now;
$backend->next_probe_at = $now + min(
3600,
60 * (2 ** min(5, $failures - 1))
);
if ((string) $backend->admin_state === 'active') {
$backend->admin_state = 'read_only';
}
$backend->updated_at = $now;
});
}
/**
* @param array{backend_id:int,revision:int,claim_until:int} $claim
* @param callable(CloudStorageBackend):void $mutation
*/
private function settle(array $claim, callable $mutation): bool
{
$connection = (new CloudStorageBackend())->getConnection();
return $connection->transaction(static function () use (
$claim,
$mutation
): bool {
$backend = CloudStorageBackend::query()
->whereKey($claim['backend_id'])->lockForUpdate()->first();
if (!$backend instanceof CloudStorageBackend
|| (int) $backend->config_revision !== $claim['revision']
|| (int) $backend->next_probe_at !== $claim['claim_until']) {
return false;
}
$mutation($backend);
$backend->save();
return true;
});
}
/** @param array{backend_id:int,revision:int,claim_until:int} $claim */
private function releaseChangedRevision(array $claim): void
{
$now = $this->now();
$connection = (new CloudStorageBackend())->getConnection();
$connection->transaction(static function () use ($claim, $now): void {
$backend = CloudStorageBackend::query()
->whereKey($claim['backend_id'])->lockForUpdate()->first();
if ($backend instanceof CloudStorageBackend
&& (int) $backend->config_revision !== $claim['revision']
&& (int) $backend->next_probe_at === $claim['claim_until']) {
// Metadata-only transitions preserve the probe schedule. Let
// their new revision be inspected immediately. Runtime edits
// already reset next_probe_at to zero and do not match here.
$backend->next_probe_at = $now;
$backend->updated_at = $now;
$backend->save();
}
});
}
private function now(): int
{
return max(1, (int) ($this->clock)());
}
}