You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
210 lines
8.5 KiB
PHP
210 lines
8.5 KiB
PHP
<?php
|
|
|
|
namespace App\Services\Cloud;
|
|
|
|
use App\Models\CloudStorageBackend;
|
|
|
|
/** Periodic read-only provider checks, isolated from long replica copies. */
|
|
final class CloudObjectStorageHealthProbeService
|
|
{
|
|
private const CLAIM_SECONDS = 45;
|
|
private const HEALTHY_INTERVAL_SECONDS = 300;
|
|
private \Closure $runtimeResolver;
|
|
private \Closure $headBucket;
|
|
private \Closure $clock;
|
|
/** @var list<string> */
|
|
private array $observationErrors = [];
|
|
|
|
public function __construct(
|
|
?callable $runtimeResolver = null,
|
|
?callable $headBucket = null,
|
|
?callable $clock = null
|
|
) {
|
|
$this->runtimeResolver = \Closure::fromCallable(
|
|
$runtimeResolver ?? static fn (int $backendId): array =>
|
|
CloudObjectStorageBackendRegistry::runtime($backendId, false)
|
|
);
|
|
$this->headBucket = \Closure::fromCallable(
|
|
$headBucket ?? static function (array $config, array $credentials): void {
|
|
(new CloudObjectStorageS3Client())->headBucket($config, $credentials);
|
|
}
|
|
);
|
|
$this->clock = \Closure::fromCallable($clock ?? 'time');
|
|
}
|
|
|
|
/** Claim and inspect at most one due backend. */
|
|
public function tick(): bool
|
|
{
|
|
$this->observationErrors = [];
|
|
$claim = $this->claimDueBackend();
|
|
if ($claim === null) {
|
|
return false;
|
|
}
|
|
|
|
try {
|
|
$runtime = ($this->runtimeResolver)($claim['backend_id']);
|
|
if ((int) ($runtime['backend_id'] ?? 0) !== $claim['backend_id']
|
|
|| (int) ($runtime['revision'] ?? 0) !== $claim['revision']) {
|
|
$this->releaseChangedRevision($claim);
|
|
return true;
|
|
}
|
|
// Deliberately outside every database transaction. Credentials
|
|
// exist only in this call frame and are never persisted or logged.
|
|
($this->headBucket)($runtime['config'], $runtime['credentials']);
|
|
$this->settleSuccess($claim);
|
|
} catch (CloudObjectStorageTestError $error) {
|
|
$this->settleFailure($claim, $error->publicCode);
|
|
$this->observationErrors[] = 'storage_backend_unhealthy';
|
|
} catch (\Throwable $error) {
|
|
// Runtime/decryption/client failures are bounded to a public code;
|
|
// exception messages can contain deployment details and stay local.
|
|
$this->settleFailure($claim, 'request_rejected');
|
|
$this->observationErrors[] = 'storage_health_failed';
|
|
}
|
|
return true;
|
|
}
|
|
|
|
/** @return list<string> */
|
|
public function observationErrors(): array
|
|
{
|
|
return $this->observationErrors;
|
|
}
|
|
|
|
/** @return array{backend_id:int,revision:int,claim_until:int}|null */
|
|
private function claimDueBackend(): ?array
|
|
{
|
|
$now = $this->now();
|
|
$connection = (new CloudStorageBackend())->getConnection();
|
|
return $connection->transaction(static function () use ($now): ?array {
|
|
$backend = CloudStorageBackend::query()
|
|
->where('admin_state', '<>', 'retired')
|
|
->where(static function ($query) use ($now): void {
|
|
$query->where(static function ($due) use ($now): void {
|
|
$due->where('next_probe_at', '>', 0)
|
|
->where('next_probe_at', '<=', $now);
|
|
})->orWhere(static function ($bootstrap): void {
|
|
// Earlier registry imports can carry valid current
|
|
// health evidence without a first periodic schedule.
|
|
$bootstrap->where('next_probe_at', 0)
|
|
->where('health_state', 'healthy')
|
|
->where('tested_revision', '>', 0)
|
|
->whereColumn('tested_revision', 'config_revision');
|
|
});
|
|
})
|
|
->orderBy('next_probe_at')->orderBy('id')
|
|
->lockForUpdate()->first();
|
|
if (!$backend instanceof CloudStorageBackend) {
|
|
return null;
|
|
}
|
|
$claimUntil = $now + self::CLAIM_SECONDS;
|
|
$backend->next_probe_at = $claimUntil;
|
|
$backend->updated_at = $now;
|
|
$backend->save();
|
|
return [
|
|
'backend_id' => (int) $backend->id,
|
|
'revision' => (int) $backend->config_revision,
|
|
'claim_until' => $claimUntil,
|
|
];
|
|
});
|
|
}
|
|
|
|
/** @param array{backend_id:int,revision:int,claim_until:int} $claim */
|
|
private function settleSuccess(array $claim): bool
|
|
{
|
|
$now = $this->now();
|
|
return $this->settle($claim, static function (
|
|
CloudStorageBackend $backend
|
|
) use ($now): void {
|
|
$backend->tested_revision = (int) $backend->config_revision;
|
|
$backend->health_state = 'healthy';
|
|
$backend->consecutive_failures = 0;
|
|
$backend->consecutive_successes =
|
|
(int) $backend->consecutive_successes + 1;
|
|
$backend->last_error_code = '';
|
|
$backend->last_tested_at = $now;
|
|
$backend->last_healthy_at = $now;
|
|
$backend->next_probe_at = $now + self::HEALTHY_INTERVAL_SECONDS;
|
|
$backend->updated_at = $now;
|
|
// admin_state is intentionally untouched. A successful probe can
|
|
// restore evidence, but cannot override an administrator decision.
|
|
});
|
|
}
|
|
|
|
/** @param array{backend_id:int,revision:int,claim_until:int} $claim */
|
|
private function settleFailure(array $claim, string $code): bool
|
|
{
|
|
$now = $this->now();
|
|
return $this->settle($claim, static function (
|
|
CloudStorageBackend $backend
|
|
) use ($now, $code): void {
|
|
$backend->tested_revision = 0;
|
|
$backend->health_state = in_array($code, [
|
|
'endpoint_unreachable', 'service_unavailable', 'rate_limited',
|
|
], true) ? 'unreachable' : 'degraded';
|
|
$failures = (int) $backend->consecutive_failures + 1;
|
|
$backend->consecutive_failures = $failures;
|
|
$backend->consecutive_successes = 0;
|
|
$backend->last_error_code = $code;
|
|
$backend->last_tested_at = $now;
|
|
$backend->next_probe_at = $now + min(
|
|
3600,
|
|
60 * (2 ** min(5, $failures - 1))
|
|
);
|
|
if ((string) $backend->admin_state === 'active') {
|
|
$backend->admin_state = 'read_only';
|
|
}
|
|
$backend->updated_at = $now;
|
|
});
|
|
}
|
|
|
|
/**
|
|
* @param array{backend_id:int,revision:int,claim_until:int} $claim
|
|
* @param callable(CloudStorageBackend):void $mutation
|
|
*/
|
|
private function settle(array $claim, callable $mutation): bool
|
|
{
|
|
$connection = (new CloudStorageBackend())->getConnection();
|
|
return $connection->transaction(static function () use (
|
|
$claim,
|
|
$mutation
|
|
): bool {
|
|
$backend = CloudStorageBackend::query()
|
|
->whereKey($claim['backend_id'])->lockForUpdate()->first();
|
|
if (!$backend instanceof CloudStorageBackend
|
|
|| (int) $backend->config_revision !== $claim['revision']
|
|
|| (int) $backend->next_probe_at !== $claim['claim_until']) {
|
|
return false;
|
|
}
|
|
$mutation($backend);
|
|
$backend->save();
|
|
return true;
|
|
});
|
|
}
|
|
|
|
/** @param array{backend_id:int,revision:int,claim_until:int} $claim */
|
|
private function releaseChangedRevision(array $claim): void
|
|
{
|
|
$now = $this->now();
|
|
$connection = (new CloudStorageBackend())->getConnection();
|
|
$connection->transaction(static function () use ($claim, $now): void {
|
|
$backend = CloudStorageBackend::query()
|
|
->whereKey($claim['backend_id'])->lockForUpdate()->first();
|
|
if ($backend instanceof CloudStorageBackend
|
|
&& (int) $backend->config_revision !== $claim['revision']
|
|
&& (int) $backend->next_probe_at === $claim['claim_until']) {
|
|
// Metadata-only transitions preserve the probe schedule. Let
|
|
// their new revision be inspected immediately. Runtime edits
|
|
// already reset next_probe_at to zero and do not match here.
|
|
$backend->next_probe_at = $now;
|
|
$backend->updated_at = $now;
|
|
$backend->save();
|
|
}
|
|
});
|
|
}
|
|
|
|
private function now(): int
|
|
{
|
|
return max(1, (int) ($this->clock)());
|
|
}
|
|
}
|