addConnection(['driver' => 'sqlite', 'database' => ':memory:']); $capsule->setAsGlobal(); $capsule->bootEloquent(); $capsule->schema()->create('cloud_storage_backends', static function ( Blueprint $table ): void { $table->increments('id'); $table->string('backend_key')->unique(); $table->string('identity_sha256')->unique(); $table->string('name'); $table->string('driver'); $table->string('endpoint'); $table->string('region'); $table->string('bucket'); $table->string('prefix'); $table->boolean('path_style'); $table->string('failure_domain'); $table->text('sealed_credentials'); $table->unsignedBigInteger('config_revision'); $table->unsignedBigInteger('tested_revision'); $table->string('admin_state'); $table->string('health_state'); $table->unsignedInteger('write_priority'); $table->unsignedInteger('consecutive_failures'); $table->unsignedInteger('consecutive_successes'); $table->string('last_error_code'); $table->unsignedBigInteger('last_tested_at'); $table->unsignedBigInteger('last_healthy_at'); $table->unsignedBigInteger('next_probe_at'); $table->unsignedBigInteger('created_at'); $table->unsignedBigInteger('updated_at'); }); $now = 2000000000; $clock = static function () use (&$now): int { return $now; }; $insert = static function (array $values = []) use (&$now): int { $id = CloudStorageBackend::query()->max('id') + 1; return (int) Capsule::table('cloud_storage_backends')->insertGetId(array_replace([ 'backend_key' => str_pad(dechex($id), 32, '0', STR_PAD_LEFT), 'identity_sha256' => str_repeat(dechex(($id % 15) + 1), 64), 'name' => 'Backend ' . $id, 'driver' => 's3', 'endpoint' => 'https://storage-' . $id . '.example.test', 'region' => 'test-1', 'bucket' => 'bucket-' . $id, 'prefix' => 'virt', 'path_style' => 0, 'failure_domain' => 'provider-' . $id, 'sealed_credentials' => 'sealed-marker-' . $id, 'config_revision' => 3, 'tested_revision' => 3, 'admin_state' => 'active', 'health_state' => 'healthy', 'write_priority' => 10, 'consecutive_failures' => 0, 'consecutive_successes' => 1, 'last_error_code' => '', 'last_tested_at' => $now - 600, 'last_healthy_at' => $now - 600, 'next_probe_at' => $now - 1, 'created_at' => $now - 1000, 'updated_at' => $now - 600, ], $values)); }; $runtime = static function (int $backendId): array { $row = CloudStorageBackend::query()->findOrFail($backendId); return [ 'backend_id' => $backendId, 'revision' => (int) $row->config_revision, 'config' => [ 'endpoint' => (string) $row->endpoint, 'region' => (string) $row->region, 'bucket' => (string) $row->bucket, 'prefix' => (string) $row->prefix, 'path_style' => (bool) $row->path_style, ], 'credentials' => [ 'access_key_id' => 'runtime-access-key', 'secret_access_key' => 'runtime-secret-key', ], ]; }; $clear = static function (): void { Capsule::table('cloud_storage_backends')->delete(); }; // Successful probes restore current health evidence but preserve every manual // non-active state. Provider work must never execute inside a transaction. foreach (['read_only', 'draining', 'disabled'] as $manualState) { $clear(); $backendId = $insert([ 'admin_state' => $manualState, 'health_state' => 'unreachable', 'tested_revision' => 0, 'consecutive_failures' => 2, 'consecutive_successes' => 0, 'last_error_code' => 'service_unavailable', ]); $providerCalls = 0; $service = new CloudObjectStorageHealthProbeService( static function (int $id) use ($runtime, $assert, $backendId): array { $assert($id === $backendId && Capsule::connection()->transactionLevel() === 0, 'Runtime credentials were resolved inside a claim transaction.'); return $runtime($id); }, static function (array $config, array $credentials) use ( &$providerCalls, $assert ): void { $providerCalls++; $assert(Capsule::connection()->transactionLevel() === 0, 'HeadBucket ran inside a database transaction.'); $assert($config['bucket'] !== '' && $credentials['secret_access_key'] === 'runtime-secret-key', 'Probe did not receive the frozen backend runtime.'); }, $clock ); $assert($service->tick() && $providerCalls === 1 && $service->observationErrors() === [], 'A due healthy backend was not probed exactly once.'); $row = CloudStorageBackend::query()->findOrFail($backendId); $assert($row->admin_state === $manualState && $row->health_state === 'healthy' && $row->tested_revision === $row->config_revision && $row->consecutive_failures === 0 && $row->consecutive_successes === 1 && $row->last_error_code === '' && $row->last_tested_at === $now && $row->last_healthy_at === $now && $row->next_probe_at === $now + 300, 'Success changed manual state or failed to restore current health evidence.'); } // Provider failure removes the active backend from new-write placement and // applies bounded exponential backoff without persisting runtime credentials. $clear(); $backendId = $insert(['consecutive_failures' => 2]); $service = new CloudObjectStorageHealthProbeService( $runtime, static function (): void { if (Capsule::connection()->transactionLevel() !== 0) { throw new RuntimeException('provider_io_in_transaction'); } throw new CloudObjectStorageTestError( 'secret provider detail must not persist', 'endpoint_unreachable' ); }, $clock ); $assert($service->tick() && $service->observationErrors() === ['storage_backend_unhealthy'], 'Provider failure was not exposed as a bounded worker observation.'); $row = CloudStorageBackend::query()->findOrFail($backendId); $databaseJson = json_encode($row->toArray(), JSON_THROW_ON_ERROR); $assert($row->admin_state === 'read_only' && $row->health_state === 'unreachable' && $row->tested_revision === 0 && $row->consecutive_failures === 3 && $row->next_probe_at === $now + 240 && $row->last_error_code === 'endpoint_unreachable' && !str_contains($databaseJson, 'runtime-secret-key') && !str_contains($databaseJson, 'secret provider detail'), 'Failure did not stop writes, apply backoff, or leaked private details.'); // One tick claims only one due backend. A later tick handles the next row. $clear(); $firstId = $insert(['next_probe_at' => $now - 2]); $secondId = $insert(['next_probe_at' => $now - 1]); $calls = []; $service = new CloudObjectStorageHealthProbeService( $runtime, static function (array $config) use (&$calls): void { $calls[] = $config['bucket']; }, $clock ); $assert($service->tick() && count($calls) === 1 && CloudStorageBackend::query()->findOrFail($firstId)->last_tested_at === $now && CloudStorageBackend::query()->findOrFail($secondId)->last_tested_at === $now - 600, 'One health tick claimed more than one due backend.'); // A configuration revision changed during the remote read makes the result // stale. Settlement must leave the newer row untouched. $clear(); $backendId = $insert(); $service = new CloudObjectStorageHealthProbeService( $runtime, static function () use ($backendId, $now): void { Capsule::table('cloud_storage_backends')->where('id', $backendId)->update([ 'config_revision' => 4, 'tested_revision' => 0, 'health_state' => 'unknown', 'admin_state' => 'draft', 'last_error_code' => '', 'updated_at' => $now + 1, ]); }, $clock ); $assert($service->tick(), 'Revision-race probe did not complete its bounded attempt.'); $row = CloudStorageBackend::query()->findOrFail($backendId); $assert($row->config_revision === 4 && $row->tested_revision === 0 && $row->health_state === 'unknown' && $row->admin_state === 'draft' && $row->last_tested_at === $now - 600, 'A stale HeadBucket result overwrote a newer backend revision.'); // A previously imported healthy backend may predate the periodic schedule. // Current tested evidence is enough to bootstrap it once; an untested draft is // still excluded so automation cannot bypass first-time administrator review. $clear(); $importedId = $insert(['next_probe_at' => 0]); $draftId = $insert([ 'next_probe_at' => 0, 'tested_revision' => 0, 'health_state' => 'unknown', 'admin_state' => 'draft', ]); $providerCalls = 0; $service = new CloudObjectStorageHealthProbeService( $runtime, static function () use (&$providerCalls): void { $providerCalls++; }, $clock ); $assert($service->tick() && $providerCalls === 1 && CloudStorageBackend::query()->findOrFail($importedId)->next_probe_at === $now + 300 && CloudStorageBackend::query()->findOrFail($draftId)->next_probe_at === 0, 'Legacy current health was not scheduled or an untested draft was probed.'); echo "Object storage health probe service tests passed.\n";