You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
251 lines
9.7 KiB
PHTML
251 lines
9.7 KiB
PHTML
|
2 weeks ago
|
<?php
|
||
|
|
|
||
|
|
require dirname(__DIR__) . '/vendor/autoload.php';
|
||
|
|
|
||
|
|
use App\Models\CloudStorageBackend;
|
||
|
|
use App\Services\Cloud\CloudObjectStorageHealthProbeService;
|
||
|
|
use App\Services\Cloud\CloudObjectStorageTestError;
|
||
|
|
use Illuminate\Database\Capsule\Manager as Capsule;
|
||
|
|
use Illuminate\Database\Schema\Blueprint;
|
||
|
|
|
||
|
|
$assert = static function (bool $condition, string $message): void {
|
||
|
|
if (!$condition) {
|
||
|
|
throw new RuntimeException($message);
|
||
|
|
}
|
||
|
|
};
|
||
|
|
|
||
|
|
$capsule = new Capsule();
|
||
|
|
$capsule->addConnection(['driver' => 'sqlite', 'database' => ':memory:']);
|
||
|
|
$capsule->setAsGlobal();
|
||
|
|
$capsule->bootEloquent();
|
||
|
|
$capsule->schema()->create('cloud_storage_backends', static function (
|
||
|
|
Blueprint $table
|
||
|
|
): void {
|
||
|
|
$table->increments('id');
|
||
|
|
$table->string('backend_key')->unique();
|
||
|
|
$table->string('identity_sha256')->unique();
|
||
|
|
$table->string('name');
|
||
|
|
$table->string('driver');
|
||
|
|
$table->string('endpoint');
|
||
|
|
$table->string('region');
|
||
|
|
$table->string('bucket');
|
||
|
|
$table->string('prefix');
|
||
|
|
$table->boolean('path_style');
|
||
|
|
$table->string('failure_domain');
|
||
|
|
$table->text('sealed_credentials');
|
||
|
|
$table->unsignedBigInteger('config_revision');
|
||
|
|
$table->unsignedBigInteger('tested_revision');
|
||
|
|
$table->string('admin_state');
|
||
|
|
$table->string('health_state');
|
||
|
|
$table->unsignedInteger('write_priority');
|
||
|
|
$table->unsignedInteger('consecutive_failures');
|
||
|
|
$table->unsignedInteger('consecutive_successes');
|
||
|
|
$table->string('last_error_code');
|
||
|
|
$table->unsignedBigInteger('last_tested_at');
|
||
|
|
$table->unsignedBigInteger('last_healthy_at');
|
||
|
|
$table->unsignedBigInteger('next_probe_at');
|
||
|
|
$table->unsignedBigInteger('created_at');
|
||
|
|
$table->unsignedBigInteger('updated_at');
|
||
|
|
});
|
||
|
|
|
||
|
|
$now = 2000000000;
|
||
|
|
$clock = static function () use (&$now): int { return $now; };
|
||
|
|
$insert = static function (array $values = []) use (&$now): int {
|
||
|
|
$id = CloudStorageBackend::query()->max('id') + 1;
|
||
|
|
return (int) Capsule::table('cloud_storage_backends')->insertGetId(array_replace([
|
||
|
|
'backend_key' => str_pad(dechex($id), 32, '0', STR_PAD_LEFT),
|
||
|
|
'identity_sha256' => str_repeat(dechex(($id % 15) + 1), 64),
|
||
|
|
'name' => 'Backend ' . $id,
|
||
|
|
'driver' => 's3',
|
||
|
|
'endpoint' => 'https://storage-' . $id . '.example.test',
|
||
|
|
'region' => 'test-1',
|
||
|
|
'bucket' => 'bucket-' . $id,
|
||
|
|
'prefix' => 'virt',
|
||
|
|
'path_style' => 0,
|
||
|
|
'failure_domain' => 'provider-' . $id,
|
||
|
|
'sealed_credentials' => 'sealed-marker-' . $id,
|
||
|
|
'config_revision' => 3,
|
||
|
|
'tested_revision' => 3,
|
||
|
|
'admin_state' => 'active',
|
||
|
|
'health_state' => 'healthy',
|
||
|
|
'write_priority' => 10,
|
||
|
|
'consecutive_failures' => 0,
|
||
|
|
'consecutive_successes' => 1,
|
||
|
|
'last_error_code' => '',
|
||
|
|
'last_tested_at' => $now - 600,
|
||
|
|
'last_healthy_at' => $now - 600,
|
||
|
|
'next_probe_at' => $now - 1,
|
||
|
|
'created_at' => $now - 1000,
|
||
|
|
'updated_at' => $now - 600,
|
||
|
|
], $values));
|
||
|
|
};
|
||
|
|
$runtime = static function (int $backendId): array {
|
||
|
|
$row = CloudStorageBackend::query()->findOrFail($backendId);
|
||
|
|
return [
|
||
|
|
'backend_id' => $backendId,
|
||
|
|
'revision' => (int) $row->config_revision,
|
||
|
|
'config' => [
|
||
|
|
'endpoint' => (string) $row->endpoint,
|
||
|
|
'region' => (string) $row->region,
|
||
|
|
'bucket' => (string) $row->bucket,
|
||
|
|
'prefix' => (string) $row->prefix,
|
||
|
|
'path_style' => (bool) $row->path_style,
|
||
|
|
],
|
||
|
|
'credentials' => [
|
||
|
|
'access_key_id' => 'runtime-access-key',
|
||
|
|
'secret_access_key' => 'runtime-secret-key',
|
||
|
|
],
|
||
|
|
];
|
||
|
|
};
|
||
|
|
$clear = static function (): void {
|
||
|
|
Capsule::table('cloud_storage_backends')->delete();
|
||
|
|
};
|
||
|
|
|
||
|
|
// Successful probes restore current health evidence but preserve every manual
|
||
|
|
// non-active state. Provider work must never execute inside a transaction.
|
||
|
|
foreach (['read_only', 'draining', 'disabled'] as $manualState) {
|
||
|
|
$clear();
|
||
|
|
$backendId = $insert([
|
||
|
|
'admin_state' => $manualState,
|
||
|
|
'health_state' => 'unreachable',
|
||
|
|
'tested_revision' => 0,
|
||
|
|
'consecutive_failures' => 2,
|
||
|
|
'consecutive_successes' => 0,
|
||
|
|
'last_error_code' => 'service_unavailable',
|
||
|
|
]);
|
||
|
|
$providerCalls = 0;
|
||
|
|
$service = new CloudObjectStorageHealthProbeService(
|
||
|
|
static function (int $id) use ($runtime, $assert, $backendId): array {
|
||
|
|
$assert($id === $backendId
|
||
|
|
&& Capsule::connection()->transactionLevel() === 0,
|
||
|
|
'Runtime credentials were resolved inside a claim transaction.');
|
||
|
|
return $runtime($id);
|
||
|
|
},
|
||
|
|
static function (array $config, array $credentials) use (
|
||
|
|
&$providerCalls,
|
||
|
|
$assert
|
||
|
|
): void {
|
||
|
|
$providerCalls++;
|
||
|
|
$assert(Capsule::connection()->transactionLevel() === 0,
|
||
|
|
'HeadBucket ran inside a database transaction.');
|
||
|
|
$assert($config['bucket'] !== ''
|
||
|
|
&& $credentials['secret_access_key'] === 'runtime-secret-key',
|
||
|
|
'Probe did not receive the frozen backend runtime.');
|
||
|
|
},
|
||
|
|
$clock
|
||
|
|
);
|
||
|
|
$assert($service->tick() && $providerCalls === 1
|
||
|
|
&& $service->observationErrors() === [],
|
||
|
|
'A due healthy backend was not probed exactly once.');
|
||
|
|
$row = CloudStorageBackend::query()->findOrFail($backendId);
|
||
|
|
$assert($row->admin_state === $manualState
|
||
|
|
&& $row->health_state === 'healthy'
|
||
|
|
&& $row->tested_revision === $row->config_revision
|
||
|
|
&& $row->consecutive_failures === 0
|
||
|
|
&& $row->consecutive_successes === 1
|
||
|
|
&& $row->last_error_code === ''
|
||
|
|
&& $row->last_tested_at === $now
|
||
|
|
&& $row->last_healthy_at === $now
|
||
|
|
&& $row->next_probe_at === $now + 300,
|
||
|
|
'Success changed manual state or failed to restore current health evidence.');
|
||
|
|
}
|
||
|
|
|
||
|
|
// Provider failure removes the active backend from new-write placement and
|
||
|
|
// applies bounded exponential backoff without persisting runtime credentials.
|
||
|
|
$clear();
|
||
|
|
$backendId = $insert(['consecutive_failures' => 2]);
|
||
|
|
$service = new CloudObjectStorageHealthProbeService(
|
||
|
|
$runtime,
|
||
|
|
static function (): void {
|
||
|
|
if (Capsule::connection()->transactionLevel() !== 0) {
|
||
|
|
throw new RuntimeException('provider_io_in_transaction');
|
||
|
|
}
|
||
|
|
throw new CloudObjectStorageTestError(
|
||
|
|
'secret provider detail must not persist',
|
||
|
|
'endpoint_unreachable'
|
||
|
|
);
|
||
|
|
},
|
||
|
|
$clock
|
||
|
|
);
|
||
|
|
$assert($service->tick()
|
||
|
|
&& $service->observationErrors() === ['storage_backend_unhealthy'],
|
||
|
|
'Provider failure was not exposed as a bounded worker observation.');
|
||
|
|
$row = CloudStorageBackend::query()->findOrFail($backendId);
|
||
|
|
$databaseJson = json_encode($row->toArray(), JSON_THROW_ON_ERROR);
|
||
|
|
$assert($row->admin_state === 'read_only'
|
||
|
|
&& $row->health_state === 'unreachable'
|
||
|
|
&& $row->tested_revision === 0
|
||
|
|
&& $row->consecutive_failures === 3
|
||
|
|
&& $row->next_probe_at === $now + 240
|
||
|
|
&& $row->last_error_code === 'endpoint_unreachable'
|
||
|
|
&& !str_contains($databaseJson, 'runtime-secret-key')
|
||
|
|
&& !str_contains($databaseJson, 'secret provider detail'),
|
||
|
|
'Failure did not stop writes, apply backoff, or leaked private details.');
|
||
|
|
|
||
|
|
// One tick claims only one due backend. A later tick handles the next row.
|
||
|
|
$clear();
|
||
|
|
$firstId = $insert(['next_probe_at' => $now - 2]);
|
||
|
|
$secondId = $insert(['next_probe_at' => $now - 1]);
|
||
|
|
$calls = [];
|
||
|
|
$service = new CloudObjectStorageHealthProbeService(
|
||
|
|
$runtime,
|
||
|
|
static function (array $config) use (&$calls): void {
|
||
|
|
$calls[] = $config['bucket'];
|
||
|
|
},
|
||
|
|
$clock
|
||
|
|
);
|
||
|
|
$assert($service->tick() && count($calls) === 1
|
||
|
|
&& CloudStorageBackend::query()->findOrFail($firstId)->last_tested_at === $now
|
||
|
|
&& CloudStorageBackend::query()->findOrFail($secondId)->last_tested_at === $now - 600,
|
||
|
|
'One health tick claimed more than one due backend.');
|
||
|
|
|
||
|
|
// A configuration revision changed during the remote read makes the result
|
||
|
|
// stale. Settlement must leave the newer row untouched.
|
||
|
|
$clear();
|
||
|
|
$backendId = $insert();
|
||
|
|
$service = new CloudObjectStorageHealthProbeService(
|
||
|
|
$runtime,
|
||
|
|
static function () use ($backendId, $now): void {
|
||
|
|
Capsule::table('cloud_storage_backends')->where('id', $backendId)->update([
|
||
|
|
'config_revision' => 4,
|
||
|
|
'tested_revision' => 0,
|
||
|
|
'health_state' => 'unknown',
|
||
|
|
'admin_state' => 'draft',
|
||
|
|
'last_error_code' => '',
|
||
|
|
'updated_at' => $now + 1,
|
||
|
|
]);
|
||
|
|
},
|
||
|
|
$clock
|
||
|
|
);
|
||
|
|
$assert($service->tick(), 'Revision-race probe did not complete its bounded attempt.');
|
||
|
|
$row = CloudStorageBackend::query()->findOrFail($backendId);
|
||
|
|
$assert($row->config_revision === 4 && $row->tested_revision === 0
|
||
|
|
&& $row->health_state === 'unknown' && $row->admin_state === 'draft'
|
||
|
|
&& $row->last_tested_at === $now - 600,
|
||
|
|
'A stale HeadBucket result overwrote a newer backend revision.');
|
||
|
|
|
||
|
|
// A previously imported healthy backend may predate the periodic schedule.
|
||
|
|
// Current tested evidence is enough to bootstrap it once; an untested draft is
|
||
|
|
// still excluded so automation cannot bypass first-time administrator review.
|
||
|
|
$clear();
|
||
|
|
$importedId = $insert(['next_probe_at' => 0]);
|
||
|
|
$draftId = $insert([
|
||
|
|
'next_probe_at' => 0,
|
||
|
|
'tested_revision' => 0,
|
||
|
|
'health_state' => 'unknown',
|
||
|
|
'admin_state' => 'draft',
|
||
|
|
]);
|
||
|
|
$providerCalls = 0;
|
||
|
|
$service = new CloudObjectStorageHealthProbeService(
|
||
|
|
$runtime,
|
||
|
|
static function () use (&$providerCalls): void { $providerCalls++; },
|
||
|
|
$clock
|
||
|
|
);
|
||
|
|
$assert($service->tick() && $providerCalls === 1
|
||
|
|
&& CloudStorageBackend::query()->findOrFail($importedId)->next_probe_at
|
||
|
|
=== $now + 300
|
||
|
|
&& CloudStorageBackend::query()->findOrFail($draftId)->next_probe_at === 0,
|
||
|
|
'Legacy current health was not scheduled or an untested draft was probed.');
|
||
|
|
|
||
|
|
echo "Object storage health probe service tests passed.\n";
|