*/ private array $observationErrors = []; public function __construct( ?callable $runtimeResolver = null, ?callable $headBucket = null, ?callable $clock = null ) { $this->runtimeResolver = \Closure::fromCallable( $runtimeResolver ?? static fn (int $backendId): array => CloudObjectStorageBackendRegistry::runtime($backendId, false) ); $this->headBucket = \Closure::fromCallable( $headBucket ?? static function (array $config, array $credentials): void { (new CloudObjectStorageS3Client())->headBucket($config, $credentials); } ); $this->clock = \Closure::fromCallable($clock ?? 'time'); } /** Claim and inspect at most one due backend. */ public function tick(): bool { $this->observationErrors = []; $claim = $this->claimDueBackend(); if ($claim === null) { return false; } try { $runtime = ($this->runtimeResolver)($claim['backend_id']); if ((int) ($runtime['backend_id'] ?? 0) !== $claim['backend_id'] || (int) ($runtime['revision'] ?? 0) !== $claim['revision']) { $this->releaseChangedRevision($claim); return true; } // Deliberately outside every database transaction. Credentials // exist only in this call frame and are never persisted or logged. ($this->headBucket)($runtime['config'], $runtime['credentials']); $this->settleSuccess($claim); } catch (CloudObjectStorageTestError $error) { $this->settleFailure($claim, $error->publicCode); $this->observationErrors[] = 'storage_backend_unhealthy'; } catch (\Throwable $error) { // Runtime/decryption/client failures are bounded to a public code; // exception messages can contain deployment details and stay local. $this->settleFailure($claim, 'request_rejected'); $this->observationErrors[] = 'storage_health_failed'; } return true; } /** @return list */ public function observationErrors(): array { return $this->observationErrors; } /** @return array{backend_id:int,revision:int,claim_until:int}|null */ private function claimDueBackend(): ?array { $now = $this->now(); $connection = (new CloudStorageBackend())->getConnection(); return $connection->transaction(static function () use ($now): ?array { $backend = CloudStorageBackend::query() ->where('admin_state', '<>', 'retired') ->where(static function ($query) use ($now): void { $query->where(static function ($due) use ($now): void { $due->where('next_probe_at', '>', 0) ->where('next_probe_at', '<=', $now); })->orWhere(static function ($bootstrap): void { // Earlier registry imports can carry valid current // health evidence without a first periodic schedule. $bootstrap->where('next_probe_at', 0) ->where('health_state', 'healthy') ->where('tested_revision', '>', 0) ->whereColumn('tested_revision', 'config_revision'); }); }) ->orderBy('next_probe_at')->orderBy('id') ->lockForUpdate()->first(); if (!$backend instanceof CloudStorageBackend) { return null; } $claimUntil = $now + self::CLAIM_SECONDS; $backend->next_probe_at = $claimUntil; $backend->updated_at = $now; $backend->save(); return [ 'backend_id' => (int) $backend->id, 'revision' => (int) $backend->config_revision, 'claim_until' => $claimUntil, ]; }); } /** @param array{backend_id:int,revision:int,claim_until:int} $claim */ private function settleSuccess(array $claim): bool { $now = $this->now(); return $this->settle($claim, static function ( CloudStorageBackend $backend ) use ($now): void { $backend->tested_revision = (int) $backend->config_revision; $backend->health_state = 'healthy'; $backend->consecutive_failures = 0; $backend->consecutive_successes = (int) $backend->consecutive_successes + 1; $backend->last_error_code = ''; $backend->last_tested_at = $now; $backend->last_healthy_at = $now; $backend->next_probe_at = $now + self::HEALTHY_INTERVAL_SECONDS; $backend->updated_at = $now; // admin_state is intentionally untouched. A successful probe can // restore evidence, but cannot override an administrator decision. }); } /** @param array{backend_id:int,revision:int,claim_until:int} $claim */ private function settleFailure(array $claim, string $code): bool { $now = $this->now(); return $this->settle($claim, static function ( CloudStorageBackend $backend ) use ($now, $code): void { $backend->tested_revision = 0; $backend->health_state = in_array($code, [ 'endpoint_unreachable', 'service_unavailable', 'rate_limited', ], true) ? 'unreachable' : 'degraded'; $failures = (int) $backend->consecutive_failures + 1; $backend->consecutive_failures = $failures; $backend->consecutive_successes = 0; $backend->last_error_code = $code; $backend->last_tested_at = $now; $backend->next_probe_at = $now + min( 3600, 60 * (2 ** min(5, $failures - 1)) ); if ((string) $backend->admin_state === 'active') { $backend->admin_state = 'read_only'; } $backend->updated_at = $now; }); } /** * @param array{backend_id:int,revision:int,claim_until:int} $claim * @param callable(CloudStorageBackend):void $mutation */ private function settle(array $claim, callable $mutation): bool { $connection = (new CloudStorageBackend())->getConnection(); return $connection->transaction(static function () use ( $claim, $mutation ): bool { $backend = CloudStorageBackend::query() ->whereKey($claim['backend_id'])->lockForUpdate()->first(); if (!$backend instanceof CloudStorageBackend || (int) $backend->config_revision !== $claim['revision'] || (int) $backend->next_probe_at !== $claim['claim_until']) { return false; } $mutation($backend); $backend->save(); return true; }); } /** @param array{backend_id:int,revision:int,claim_until:int} $claim */ private function releaseChangedRevision(array $claim): void { $now = $this->now(); $connection = (new CloudStorageBackend())->getConnection(); $connection->transaction(static function () use ($claim, $now): void { $backend = CloudStorageBackend::query() ->whereKey($claim['backend_id'])->lockForUpdate()->first(); if ($backend instanceof CloudStorageBackend && (int) $backend->config_revision !== $claim['revision'] && (int) $backend->next_probe_at === $claim['claim_until']) { // Metadata-only transitions preserve the probe schedule. Let // their new revision be inspected immediately. Runtime edits // already reset next_probe_at to zero and do not match here. $backend->next_probe_at = $now; $backend->updated_at = $now; $backend->save(); } }); } private function now(): int { return max(1, (int) ($this->clock)()); } }