You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
virt/web/tests/CloudBackupReadRoutingTest.php

285 lines
11 KiB
PHTML

<?php
namespace App\Services\Cloud {
/** Test-only credential envelope for local URL-signing behavior. */
final class CloudSecrets
{
public static function decrypt(string $value, string $purpose): array
{
if ($purpose === 'object-storage-backend-credentials') {
return [
'access_key_id' => 'TESTACCESSKEY',
'secret_access_key' => 'test-secret-key',
];
}
if ($purpose === 'object-storage-backup-runtime'
&& $value === 'legacy-runtime') {
return [
'config' => [
'endpoint' => 'https://legacy.example.test',
'region' => 'us-east-1',
'bucket' => 'legacy-bucket',
'prefix' => 'legacy',
'path_style' => true,
],
'credentials' => [
'access_key_id' => 'LEGACYACCESS',
'secret_access_key' => 'legacy-secret',
],
'revision' => 7,
];
}
throw new \RuntimeException('Unexpected test secret purpose.');
}
}
}
namespace {
require dirname(__DIR__) . '/vendor/autoload.php';
use App\Models\CloudInstanceBackup;
use App\Models\CloudStorageBackend;
use App\Models\CloudStorageReplica;
use App\Services\Cloud\CloudBackupService;
use App\Services\Cloud\CloudError;
use Illuminate\Database\Capsule\Manager as Capsule;
use Illuminate\Database\Schema\Blueprint;
$assert = static function (bool $condition, string $message): void {
if (!$condition) {
throw new RuntimeException($message);
}
};
$expect = static function (callable $callback, int $status) use ($assert): void {
try {
$callback();
} catch (CloudError $error) {
$assert($error->httpStatus === $status,
'Unexpected CloudError status: ' . $error->httpStatus);
return;
}
throw new RuntimeException('Expected CloudError was not raised.');
};
$db = new Capsule();
$db->addConnection([
'driver' => 'sqlite',
'database' => ':memory:',
], 'default');
$db->setAsGlobal();
$db->bootEloquent();
$schema = $db->schema('default');
$schema->create('cloud_storage_backends', static function (
Blueprint $table
): void {
$table->increments('id');
$table->string('backend_key');
$table->string('identity_sha256');
$table->string('name');
$table->string('driver');
$table->string('endpoint');
$table->string('region');
$table->string('bucket');
$table->string('prefix');
$table->boolean('path_style');
$table->string('failure_domain');
$table->text('sealed_credentials');
$table->unsignedBigInteger('config_revision');
$table->unsignedBigInteger('tested_revision');
$table->string('admin_state');
$table->string('health_state');
$table->unsignedInteger('write_priority');
$table->unsignedInteger('consecutive_failures')->default(0);
$table->unsignedInteger('consecutive_successes')->default(0);
$table->string('last_error_code')->default('');
$table->unsignedBigInteger('last_tested_at')->default(0);
$table->unsignedBigInteger('last_healthy_at')->default(0);
$table->unsignedBigInteger('next_probe_at')->default(0);
$table->unsignedBigInteger('created_at');
$table->unsignedBigInteger('updated_at');
});
$schema->create('cloud_storage_artifacts', static function (
Blueprint $table
): void {
$table->increments('id');
$table->string('artifact_key');
$table->string('kind');
$table->unsignedBigInteger('maximum_size_bytes');
$table->unsignedBigInteger('size_bytes');
$table->string('sha256');
$table->string('durability_policy');
$table->unsignedTinyInteger('desired_copies');
$table->unsignedTinyInteger('required_copies');
$table->string('state');
$table->string('last_operation_id')->nullable();
$table->unsignedBigInteger('verified_at');
$table->unsignedBigInteger('created_at');
$table->unsignedBigInteger('updated_at');
});
$schema->create('cloud_storage_replicas', static function (
Blueprint $table
): void {
$table->increments('id');
$table->string('replica_key');
$table->unsignedBigInteger('artifact_id');
$table->unsignedBigInteger('backend_id');
$table->string('role');
$table->string('object_key');
$table->string('object_version_id')->default('');
$table->string('object_etag')->default('');
$table->unsignedBigInteger('size_bytes')->default(0);
$table->string('sha256')->default('');
$table->string('state');
$table->string('publish_operation_id')->nullable();
$table->string('delete_operation_id')->nullable();
$table->unsignedInteger('delete_generation')->default(0);
$table->unsignedInteger('delete_cycle_attempts')->default(0);
$table->unsignedInteger('delete_presence_confirmations')->default(0);
$table->unsignedBigInteger('delete_last_checked_at')->default(0);
$table->unsignedBigInteger('delete_next_check_at')->default(0);
$table->unsignedBigInteger('delete_attention_at')->nullable();
$table->string('lease_token')->nullable();
$table->unsignedBigInteger('lease_until')->default(0);
$table->string('error_code')->default('');
$table->unsignedBigInteger('verified_at')->default(0);
$table->unsignedBigInteger('created_at');
$table->unsignedBigInteger('updated_at');
});
$now = time();
foreach ([
[1, 'primary', 'https://primary.example.test', 'unreachable', 10],
[2, 'secondary', 'https://secondary.example.test', 'healthy', 20],
] as [$id, $name, $endpoint, $health, $priority]) {
$backend = new CloudStorageBackend();
$backend->id = $id;
$backend->backend_key = str_repeat((string) $id, 32);
$backend->identity_sha256 = str_repeat((string) $id, 64);
$backend->name = $name;
$backend->driver = 's3';
$backend->endpoint = $endpoint;
$backend->region = 'us-east-1';
$backend->bucket = $name . '-bucket';
$backend->prefix = $name;
$backend->path_style = 1;
$backend->failure_domain = $name . '-domain';
$backend->sealed_credentials = 'backend-secret-' . $id;
$backend->config_revision = 1;
$backend->tested_revision = 1;
$backend->admin_state = 'active';
$backend->health_state = $health;
$backend->write_priority = $priority;
$backend->consecutive_failures = 0;
$backend->consecutive_successes = 1;
$backend->last_error_code = '';
$backend->last_tested_at = $now;
$backend->last_healthy_at = $now;
$backend->next_probe_at = $now + 300;
$backend->created_at = $now;
$backend->updated_at = $now;
$backend->save();
}
$sha256 = str_repeat('a', 64);
Capsule::table('cloud_storage_artifacts')->insert([
'id' => 11,
'artifact_key' => str_repeat('b', 32),
'kind' => 'backup',
'maximum_size_bytes' => 8192,
'size_bytes' => 4096,
'sha256' => $sha256,
'durability_policy' => 'mirror_async_2',
'desired_copies' => 2,
'required_copies' => 1,
'state' => 'protected',
'last_operation_id' => str_repeat('c', 32),
'verified_at' => $now,
'created_at' => $now,
'updated_at' => $now,
]);
foreach ([
[1, 1, 'primary', 'primary/backups/system.qcow2', 'primary-version'],
[2, 2, 'secondary', 'secondary/backups/system.qcow2',
'secondary-version'],
] as [$id, $backendId, $role, $key, $version]) {
$replica = new CloudStorageReplica();
$replica->id = $id;
$replica->replica_key = str_repeat((string) ($id + 2), 32);
$replica->artifact_id = 11;
$replica->backend_id = $backendId;
$replica->role = $role;
$replica->object_key = $key;
$replica->object_version_id = $version;
$replica->object_etag = '"etag-' . $id . '"';
$replica->size_bytes = 4096;
$replica->sha256 = $sha256;
$replica->state = 'ready';
$replica->publish_operation_id = str_repeat((string) ($id + 4), 32);
$replica->delete_operation_id = null;
$replica->delete_generation = 0;
$replica->delete_cycle_attempts = 0;
$replica->delete_presence_confirmations = 0;
$replica->delete_last_checked_at = 0;
$replica->delete_next_check_at = 0;
$replica->delete_attention_at = null;
$replica->lease_token = null;
$replica->lease_until = 0;
$replica->error_code = '';
$replica->verified_at = $now;
$replica->created_at = $now;
$replica->updated_at = $now;
$replica->save();
}
$method = (new ReflectionClass(CloudBackupService::class))
->getMethod('backupReadUrl');
$backup = new CloudInstanceBackup();
$backup->storage_artifact_id = 11;
$backup->object_size_bytes = 4096;
$backup->sha256 = $sha256;
$url = $method->invoke(null, $backup, false);
$urlParts = is_string($url) ? parse_url($url) : false;
$urlQuery = [];
if (is_array($urlParts)) {
parse_str((string) ($urlParts['query'] ?? ''), $urlQuery);
}
$assert(is_array($urlParts)
&& ($urlParts['host'] ?? '') === 'secondary.example.test'
&& ($urlParts['path'] ?? '')
=== '/secondary-bucket/secondary/backups/system.qcow2'
&& ($urlQuery['versionId'] ?? '') === 'secondary-version',
'Read routing did not fail over to the exact secondary version.');
$backup->object_size_bytes = 4097;
$expect(static fn () => $method->invoke(null, $backup, false), 409);
$backup->object_size_bytes = 4096;
$backup->sha256 = str_repeat('f', 64);
$expect(static fn () => $method->invoke(null, $backup, false), 409);
$backup->sha256 = $sha256;
CloudStorageReplica::query()->update([
'state' => 'degraded',
'verified_at' => 0,
]);
$expect(static fn () => $method->invoke(null, $backup, false), 503);
$legacy = new CloudInstanceBackup();
$legacy->storage_artifact_id = null;
$legacy->object_key = 'legacy/backups/system.qcow2';
$legacy->object_size_bytes = 4096;
$legacy->sha256 = $sha256;
$legacy->sealed_storage_runtime = 'legacy-runtime';
$legacy->storage_revision = 7;
$legacyUrl = $method->invoke(null, $legacy, false);
$legacyUrlParts = is_string($legacyUrl) ? parse_url($legacyUrl) : false;
$assert(is_array($legacyUrlParts)
&& ($legacyUrlParts['host'] ?? '') === 'legacy.example.test'
&& ($legacyUrlParts['path'] ?? '')
=== '/legacy-bucket/legacy/backups/system.qcow2',
'Legacy backup lost its frozen-runtime read path.');
echo "Cloud backup read routing behavior tests passed.\n";
}