You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
virt/web/tests/CloudObjectStorageHealthPro...

251 lines
9.7 KiB
PHTML

<?php
require dirname(__DIR__) . '/vendor/autoload.php';
use App\Models\CloudStorageBackend;
use App\Services\Cloud\CloudObjectStorageHealthProbeService;
use App\Services\Cloud\CloudObjectStorageTestError;
use Illuminate\Database\Capsule\Manager as Capsule;
use Illuminate\Database\Schema\Blueprint;
$assert = static function (bool $condition, string $message): void {
if (!$condition) {
throw new RuntimeException($message);
}
};
$capsule = new Capsule();
$capsule->addConnection(['driver' => 'sqlite', 'database' => ':memory:']);
$capsule->setAsGlobal();
$capsule->bootEloquent();
$capsule->schema()->create('cloud_storage_backends', static function (
Blueprint $table
): void {
$table->increments('id');
$table->string('backend_key')->unique();
$table->string('identity_sha256')->unique();
$table->string('name');
$table->string('driver');
$table->string('endpoint');
$table->string('region');
$table->string('bucket');
$table->string('prefix');
$table->boolean('path_style');
$table->string('failure_domain');
$table->text('sealed_credentials');
$table->unsignedBigInteger('config_revision');
$table->unsignedBigInteger('tested_revision');
$table->string('admin_state');
$table->string('health_state');
$table->unsignedInteger('write_priority');
$table->unsignedInteger('consecutive_failures');
$table->unsignedInteger('consecutive_successes');
$table->string('last_error_code');
$table->unsignedBigInteger('last_tested_at');
$table->unsignedBigInteger('last_healthy_at');
$table->unsignedBigInteger('next_probe_at');
$table->unsignedBigInteger('created_at');
$table->unsignedBigInteger('updated_at');
});
$now = 2000000000;
$clock = static function () use (&$now): int { return $now; };
$insert = static function (array $values = []) use (&$now): int {
$id = CloudStorageBackend::query()->max('id') + 1;
return (int) Capsule::table('cloud_storage_backends')->insertGetId(array_replace([
'backend_key' => str_pad(dechex($id), 32, '0', STR_PAD_LEFT),
'identity_sha256' => str_repeat(dechex(($id % 15) + 1), 64),
'name' => 'Backend ' . $id,
'driver' => 's3',
'endpoint' => 'https://storage-' . $id . '.example.test',
'region' => 'test-1',
'bucket' => 'bucket-' . $id,
'prefix' => 'virt',
'path_style' => 0,
'failure_domain' => 'provider-' . $id,
'sealed_credentials' => 'sealed-marker-' . $id,
'config_revision' => 3,
'tested_revision' => 3,
'admin_state' => 'active',
'health_state' => 'healthy',
'write_priority' => 10,
'consecutive_failures' => 0,
'consecutive_successes' => 1,
'last_error_code' => '',
'last_tested_at' => $now - 600,
'last_healthy_at' => $now - 600,
'next_probe_at' => $now - 1,
'created_at' => $now - 1000,
'updated_at' => $now - 600,
], $values));
};
$runtime = static function (int $backendId): array {
$row = CloudStorageBackend::query()->findOrFail($backendId);
return [
'backend_id' => $backendId,
'revision' => (int) $row->config_revision,
'config' => [
'endpoint' => (string) $row->endpoint,
'region' => (string) $row->region,
'bucket' => (string) $row->bucket,
'prefix' => (string) $row->prefix,
'path_style' => (bool) $row->path_style,
],
'credentials' => [
'access_key_id' => 'runtime-access-key',
'secret_access_key' => 'runtime-secret-key',
],
];
};
$clear = static function (): void {
Capsule::table('cloud_storage_backends')->delete();
};
// Successful probes restore current health evidence but preserve every manual
// non-active state. Provider work must never execute inside a transaction.
foreach (['read_only', 'draining', 'disabled'] as $manualState) {
$clear();
$backendId = $insert([
'admin_state' => $manualState,
'health_state' => 'unreachable',
'tested_revision' => 0,
'consecutive_failures' => 2,
'consecutive_successes' => 0,
'last_error_code' => 'service_unavailable',
]);
$providerCalls = 0;
$service = new CloudObjectStorageHealthProbeService(
static function (int $id) use ($runtime, $assert, $backendId): array {
$assert($id === $backendId
&& Capsule::connection()->transactionLevel() === 0,
'Runtime credentials were resolved inside a claim transaction.');
return $runtime($id);
},
static function (array $config, array $credentials) use (
&$providerCalls,
$assert
): void {
$providerCalls++;
$assert(Capsule::connection()->transactionLevel() === 0,
'HeadBucket ran inside a database transaction.');
$assert($config['bucket'] !== ''
&& $credentials['secret_access_key'] === 'runtime-secret-key',
'Probe did not receive the frozen backend runtime.');
},
$clock
);
$assert($service->tick() && $providerCalls === 1
&& $service->observationErrors() === [],
'A due healthy backend was not probed exactly once.');
$row = CloudStorageBackend::query()->findOrFail($backendId);
$assert($row->admin_state === $manualState
&& $row->health_state === 'healthy'
&& $row->tested_revision === $row->config_revision
&& $row->consecutive_failures === 0
&& $row->consecutive_successes === 1
&& $row->last_error_code === ''
&& $row->last_tested_at === $now
&& $row->last_healthy_at === $now
&& $row->next_probe_at === $now + 300,
'Success changed manual state or failed to restore current health evidence.');
}
// Provider failure removes the active backend from new-write placement and
// applies bounded exponential backoff without persisting runtime credentials.
$clear();
$backendId = $insert(['consecutive_failures' => 2]);
$service = new CloudObjectStorageHealthProbeService(
$runtime,
static function (): void {
if (Capsule::connection()->transactionLevel() !== 0) {
throw new RuntimeException('provider_io_in_transaction');
}
throw new CloudObjectStorageTestError(
'secret provider detail must not persist',
'endpoint_unreachable'
);
},
$clock
);
$assert($service->tick()
&& $service->observationErrors() === ['storage_backend_unhealthy'],
'Provider failure was not exposed as a bounded worker observation.');
$row = CloudStorageBackend::query()->findOrFail($backendId);
$databaseJson = json_encode($row->toArray(), JSON_THROW_ON_ERROR);
$assert($row->admin_state === 'read_only'
&& $row->health_state === 'unreachable'
&& $row->tested_revision === 0
&& $row->consecutive_failures === 3
&& $row->next_probe_at === $now + 240
&& $row->last_error_code === 'endpoint_unreachable'
&& !str_contains($databaseJson, 'runtime-secret-key')
&& !str_contains($databaseJson, 'secret provider detail'),
'Failure did not stop writes, apply backoff, or leaked private details.');
// One tick claims only one due backend. A later tick handles the next row.
$clear();
$firstId = $insert(['next_probe_at' => $now - 2]);
$secondId = $insert(['next_probe_at' => $now - 1]);
$calls = [];
$service = new CloudObjectStorageHealthProbeService(
$runtime,
static function (array $config) use (&$calls): void {
$calls[] = $config['bucket'];
},
$clock
);
$assert($service->tick() && count($calls) === 1
&& CloudStorageBackend::query()->findOrFail($firstId)->last_tested_at === $now
&& CloudStorageBackend::query()->findOrFail($secondId)->last_tested_at === $now - 600,
'One health tick claimed more than one due backend.');
// A configuration revision changed during the remote read makes the result
// stale. Settlement must leave the newer row untouched.
$clear();
$backendId = $insert();
$service = new CloudObjectStorageHealthProbeService(
$runtime,
static function () use ($backendId, $now): void {
Capsule::table('cloud_storage_backends')->where('id', $backendId)->update([
'config_revision' => 4,
'tested_revision' => 0,
'health_state' => 'unknown',
'admin_state' => 'draft',
'last_error_code' => '',
'updated_at' => $now + 1,
]);
},
$clock
);
$assert($service->tick(), 'Revision-race probe did not complete its bounded attempt.');
$row = CloudStorageBackend::query()->findOrFail($backendId);
$assert($row->config_revision === 4 && $row->tested_revision === 0
&& $row->health_state === 'unknown' && $row->admin_state === 'draft'
&& $row->last_tested_at === $now - 600,
'A stale HeadBucket result overwrote a newer backend revision.');
// A previously imported healthy backend may predate the periodic schedule.
// Current tested evidence is enough to bootstrap it once; an untested draft is
// still excluded so automation cannot bypass first-time administrator review.
$clear();
$importedId = $insert(['next_probe_at' => 0]);
$draftId = $insert([
'next_probe_at' => 0,
'tested_revision' => 0,
'health_state' => 'unknown',
'admin_state' => 'draft',
]);
$providerCalls = 0;
$service = new CloudObjectStorageHealthProbeService(
$runtime,
static function () use (&$providerCalls): void { $providerCalls++; },
$clock
);
$assert($service->tick() && $providerCalls === 1
&& CloudStorageBackend::query()->findOrFail($importedId)->next_probe_at
=== $now + 300
&& CloudStorageBackend::query()->findOrFail($draftId)->next_probe_at === 0,
'Legacy current health was not scheduled or an untested draft was probed.');
echo "Object storage health probe service tests passed.\n";