You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
virt/web/tests/CloudStorageReplicaRepairSe...

289 lines
14 KiB
PHTML

<?php
require dirname(__DIR__) . '/vendor/autoload.php';
use App\Models\CloudStorageArtifact;
use App\Models\CloudStorageOperation;
use App\Models\CloudStoragePublication;
use App\Models\CloudStorageReplica;
use App\Services\Cloud\CloudStorageArtifactService;
use App\Services\Cloud\CloudStoragePublicationFinalizerService;
use App\Services\Cloud\CloudStorageReplicaRepairService;
use Illuminate\Database\Capsule\Manager as Capsule;
use Illuminate\Database\Schema\Blueprint;
$assert = static function (bool $condition, string $message): void {
if (!$condition) {
throw new RuntimeException($message);
}
};
$db = new Capsule();
$db->addConnection(['driver' => 'sqlite', 'database' => ':memory:']);
$db->setAsGlobal();
$db->bootEloquent();
$schema = $db->schema();
$schema->create('cloud_storage_backends', static function (Blueprint $t): void {
$t->increments('id');
$t->string('backend_key')->unique(); $t->string('identity_sha256')->unique();
$t->string('name'); $t->string('driver'); $t->string('endpoint');
$t->string('region'); $t->string('bucket'); $t->string('prefix');
$t->boolean('path_style'); $t->string('failure_domain');
$t->text('sealed_credentials'); $t->unsignedBigInteger('config_revision');
$t->unsignedBigInteger('tested_revision'); $t->string('admin_state');
$t->string('health_state'); $t->unsignedInteger('write_priority');
$t->unsignedInteger('consecutive_failures')->default(0);
$t->unsignedInteger('consecutive_successes')->default(0);
$t->string('last_error_code')->default('');
$t->unsignedBigInteger('last_tested_at')->default(0);
$t->unsignedBigInteger('last_healthy_at')->default(0);
$t->unsignedBigInteger('next_probe_at')->default(0);
$t->unsignedBigInteger('created_at'); $t->unsignedBigInteger('updated_at');
});
$schema->create('cloud_storage_artifacts', static function (Blueprint $t): void {
$t->increments('id'); $t->string('artifact_key')->unique();
$t->string('kind'); $t->unsignedBigInteger('maximum_size_bytes');
$t->unsignedBigInteger('size_bytes'); $t->string('sha256');
$t->string('durability_policy'); $t->unsignedTinyInteger('desired_copies');
$t->unsignedTinyInteger('required_copies'); $t->string('state');
$t->string('last_operation_id')->nullable();
$t->unsignedBigInteger('verified_at');
$t->unsignedBigInteger('created_at'); $t->unsignedBigInteger('updated_at');
});
$schema->create('cloud_storage_replicas', static function (Blueprint $t): void {
$t->increments('id'); $t->string('replica_key')->unique();
$t->unsignedBigInteger('artifact_id'); $t->unsignedBigInteger('backend_id');
$t->string('role'); $t->string('object_key');
$t->string('object_version_id')->default('');
$t->string('object_etag')->default('');
$t->unsignedBigInteger('size_bytes')->default(0);
$t->string('sha256')->default(''); $t->string('state');
$t->string('publish_operation_id')->nullable();
$t->string('delete_operation_id')->nullable();
$t->unsignedInteger('delete_generation')->default(0);
$t->unsignedInteger('delete_cycle_attempts')->default(0);
$t->unsignedInteger('delete_presence_confirmations')->default(0);
$t->unsignedBigInteger('delete_last_checked_at')->default(0);
$t->unsignedBigInteger('delete_next_check_at')->default(0);
$t->unsignedBigInteger('delete_attention_at')->nullable();
$t->string('lease_token')->nullable();
$t->unsignedBigInteger('lease_until')->default(0);
$t->string('error_code')->default('');
$t->unsignedBigInteger('verified_at')->default(0);
$t->unsignedBigInteger('created_at'); $t->unsignedBigInteger('updated_at');
$t->unique(['artifact_id', 'backend_id']);
$t->unique(['backend_id', 'object_key']);
});
$schema->create('cloud_storage_operations', static function (Blueprint $t): void {
$t->increments('id'); $t->string('operation_id')->unique();
$t->string('request_idempotency_key')->unique();
$t->string('request_fingerprint'); $t->unsignedBigInteger('artifact_id');
$t->unsignedBigInteger('replica_id')->nullable();
$t->unsignedBigInteger('source_backend_id')->nullable();
$t->unsignedBigInteger('target_backend_id')->nullable();
$t->string('kind'); $t->string('phase'); $t->string('outcome')->nullable();
$t->string('effect_resolution');
$t->boolean('effect_may_have_applied')->nullable();
$t->boolean('safe_to_resubmit')->nullable();
$t->unsignedInteger('attempts'); $t->string('lease_token')->nullable();
$t->unsignedBigInteger('lease_until'); $t->unsignedBigInteger('available_at');
$t->unsignedBigInteger('next_check_at'); $t->string('error_code');
$t->text('result_json')->nullable(); $t->unsignedBigInteger('terminal_at')->nullable();
$t->unsignedBigInteger('created_at'); $t->unsignedBigInteger('updated_at');
});
$schema->create('cloud_storage_publications', static function (Blueprint $t): void {
$t->increments('id'); $t->string('publication_key')->unique();
$t->unsignedBigInteger('artifact_id')->unique();
$t->unsignedBigInteger('backup_id')->nullable()->unique();
$t->string('template_import_intent_id')->nullable()->unique();
$t->string('rescue_template_id')->nullable()->unique();
$t->string('state'); $t->unsignedInteger('attempts')->default(0);
$t->unsignedBigInteger('available_at')->default(0);
$t->string('lease_token')->nullable();
$t->unsignedBigInteger('lease_until')->default(0);
$t->string('last_error_code')->default('');
$t->unsignedBigInteger('coverage_ready_at')->nullable();
$t->unsignedBigInteger('finalized_at')->nullable();
$t->unsignedBigInteger('created_at'); $t->unsignedBigInteger('updated_at');
});
$now = 1800000000;
$insertBackend = static function (
int $id,
string $state,
string $health,
int $tested,
string $domain,
int $priority,
string $prefix
) use ($now): void {
Capsule::table('cloud_storage_backends')->insert([
'id' => $id, 'backend_key' => str_pad((string) $id, 32, 'b'),
'identity_sha256' => str_pad((string) $id, 64, 'c'),
'name' => 'backend-' . $id, 'driver' => 's3',
'endpoint' => 'https://storage-' . $id . '.example.test',
'region' => 'us-east-1', 'bucket' => 'bucket-' . $id,
'prefix' => $prefix, 'path_style' => 1,
'failure_domain' => $domain, 'sealed_credentials' => 'sealed',
'config_revision' => 1, 'tested_revision' => $tested,
'admin_state' => $state, 'health_state' => $health,
'write_priority' => $priority, 'consecutive_failures' => 0,
'consecutive_successes' => $health === 'healthy' ? 1 : 0,
'last_error_code' => '', 'last_tested_at' => $tested > 0 ? $now : 0,
'last_healthy_at' => $health === 'healthy' ? $now : 0,
'next_probe_at' => $now + 300,
'created_at' => $now, 'updated_at' => $now,
]);
};
$insertBackend(1, 'active', 'healthy', 1, 'provider-a', 10, 'primary');
$insertBackend(2, 'draft', 'unknown', 0, 'provider-b', 20, 'secondary');
$view = CloudStorageArtifactService::beginPublication(
'backup', 'backups/late/system.qcow2', 4096, str_repeat('a', 64),
'repair-fixture-primary', hash('sha256', 'repair-fixture'),
['backup_id' => 101]
);
$assert(CloudStorageReplica::where('artifact_id', $view['artifact_id'])->count() === 1,
'Unavailable second backend was unexpectedly reserved at publication time.');
CloudStorageArtifactService::markPrimaryDispatching($view['operation_id']);
CloudStorageArtifactService::completePrimary(
$view['operation_id'], (int) $view['artifact_id'], [
'size_bytes' => 4096, 'sha256' => str_repeat('a', 64),
'etag' => 'primary-etag', 'version_id' => 'primary-version',
]
);
CloudStoragePublicationFinalizerService::complete(
'backup', 101, (int) $view['artifact_id'], $view['operation_id']
);
Capsule::table('cloud_storage_backends')->where('id', 2)->update([
'admin_state' => 'active', 'health_state' => 'healthy',
'tested_revision' => 1, 'last_tested_at' => $now,
'last_healthy_at' => $now, 'updated_at' => $now,
]);
$repair = new CloudStorageReplicaRepairService();
$assert($repair->tick($now + 1),
'Recovered second backend did not reserve a missing replica.');
$replicas = CloudStorageReplica::where('artifact_id', $view['artifact_id'])
->orderBy('id')->get();
$copy = CloudStorageOperation::where('artifact_id', $view['artifact_id'])
->where('kind', 'copy')->firstOrFail();
$assert($replicas->count() === 2
&& (int) $replicas->last()->backend_id === 2
&& (string) $replicas->last()->object_key
=== 'secondary/backups/late/system.qcow2'
&& (int) $copy->source_backend_id === 1
&& (int) $copy->target_backend_id === 2
&& (string) $copy->phase === 'accepted'
&& (int) $copy->available_at === $now + 1,
'Missing-replica repair did not freeze the expected copy intent.');
$assert(!$repair->tick($now + 2)
&& CloudStorageReplica::where('artifact_id', $view['artifact_id'])->count() === 2
&& CloudStorageOperation::where('artifact_id', $view['artifact_id'])
->where('kind', 'copy')->count() === 1,
'Repeated repair allocated duplicate replica work.');
CloudStorageArtifact::whereKey($view['artifact_id'])->update([
'desired_copies' => 1,
]);
// A healthy replica on a draining backend must not occupy forward durability
// coverage. This lets maintenance evacuate to a replacement before the old
// provider is taken offline. Required-two still needs a distinct domain.
$insertBackend(3, 'active', 'healthy', 1, 'provider-a', 1, 'same-domain');
$insertBackend(4, 'active', 'healthy', 1, 'provider-c', 50, 'replacement');
$artifactId = Capsule::table('cloud_storage_artifacts')->insertGetId([
'artifact_key' => str_repeat('d', 32), 'kind' => 'template',
'maximum_size_bytes' => 8192, 'size_bytes' => 8192,
'sha256' => str_repeat('e', 64),
'durability_policy' => 'mirror_required_2',
'desired_copies' => 2, 'required_copies' => 2,
'state' => 'available', 'last_operation_id' => str_repeat('1', 32),
'verified_at' => $now, 'created_at' => $now, 'updated_at' => $now,
]);
$primaryReplicaId = Capsule::table('cloud_storage_replicas')->insertGetId([
'replica_key' => str_repeat('4', 32), 'artifact_id' => $artifactId,
'backend_id' => 1, 'role' => 'primary',
'object_key' => 'primary/templates/image.qcow2',
'object_version_id' => 'v1', 'object_etag' => 'e1',
'size_bytes' => 8192, 'sha256' => str_repeat('e', 64),
'state' => 'ready', 'publish_operation_id' => str_repeat('1', 32),
'lease_until' => 0, 'error_code' => '', 'verified_at' => $now,
'created_at' => $now, 'updated_at' => $now,
]);
Capsule::table('cloud_storage_replicas')->insert([
'replica_key' => str_repeat('5', 32), 'artifact_id' => $artifactId,
'backend_id' => 2, 'role' => 'secondary',
'object_key' => 'secondary/templates/image.qcow2',
'object_version_id' => 'v2', 'object_etag' => 'e2',
'size_bytes' => 8192, 'sha256' => str_repeat('e', 64),
'state' => 'ready', 'publish_operation_id' => str_repeat('2', 32),
'lease_until' => 0, 'error_code' => '', 'verified_at' => $now,
'created_at' => $now, 'updated_at' => $now,
]);
Capsule::table('cloud_storage_operations')->insert([
'operation_id' => str_repeat('1', 32),
'request_idempotency_key' => 'template-primary-fixture',
'request_fingerprint' => str_repeat('6', 64),
'artifact_id' => $artifactId, 'replica_id' => $primaryReplicaId,
'target_backend_id' => 1, 'kind' => 'publish', 'phase' => 'terminal',
'outcome' => 'succeeded', 'effect_resolution' => 'applied',
'effect_may_have_applied' => 0, 'safe_to_resubmit' => 0,
'attempts' => 1, 'lease_until' => 0, 'available_at' => 0,
'next_check_at' => 0, 'error_code' => '',
'terminal_at' => $now, 'created_at' => $now, 'updated_at' => $now,
]);
Capsule::table('cloud_storage_operations')->insert([
'operation_id' => str_repeat('2', 32),
'request_idempotency_key' => 'template-old-copy-fixture',
'request_fingerprint' => str_repeat('7', 64),
'artifact_id' => $artifactId,
'replica_id' => CloudStorageReplica::where('artifact_id', $artifactId)
->where('backend_id', 2)->value('id'),
'source_backend_id' => 1, 'target_backend_id' => 2,
'kind' => 'copy', 'phase' => 'terminal', 'outcome' => 'failed',
'effect_resolution' => 'not_applied', 'effect_may_have_applied' => 0,
'safe_to_resubmit' => 0, 'attempts' => 1, 'lease_until' => 0,
'available_at' => 0, 'next_check_at' => 0,
'error_code' => 'provider_lost', 'terminal_at' => $now,
'created_at' => $now, 'updated_at' => $now,
]);
Capsule::table('cloud_storage_publications')->insert([
'publication_key' => str_repeat('8', 32), 'artifact_id' => $artifactId,
'template_import_intent_id' => str_repeat('9', 32),
'state' => 'waiting_protection', 'attempts' => 1, 'available_at' => 0,
'lease_until' => 0, 'last_error_code' => '',
'coverage_ready_at' => null, 'finalized_at' => null,
'created_at' => $now, 'updated_at' => $now,
]);
Capsule::table('cloud_storage_backends')->where('id', 2)->update([
'admin_state' => 'draining', 'health_state' => 'healthy',
'tested_revision' => 1, 'last_error_code' => '',
'updated_at' => $now,
]);
$assert($repair->tick($now + 3),
'Replacement backend did not repair required-two coverage while the '
. 'owner publication was still waiting for protection.');
$replacement = CloudStorageReplica::where('artifact_id', $artifactId)
->where('state', 'reserved')->firstOrFail();
$assert((int) $replacement->backend_id === 4
&& (string) $replacement->object_key
=== 'replacement/templates/image.qcow2'
&& CloudStorageReplica::where('artifact_id', $artifactId)->count() === 3,
'Repair counted draining evidence or selected an existing failure domain.');
$workerSource = file_get_contents(dirname(__DIR__)
. '/app/Services/Cloud/CloudStorageReplicaWorkerService.php');
$repairPosition = is_string($workerSource)
? strpos($workerSource, 'CloudStorageReplicaRepairService') : false;
$claimPosition = is_string($workerSource)
? strpos($workerSource, 'claimReplicaWork') : false;
$assert(is_string($workerSource)
&& str_contains($workerSource, 'CloudStorageReplicaRepairService')
&& str_contains($workerSource, 'storage_replica_repair_tick_failed')
&& is_int($repairPosition) && is_int($claimPosition)
&& $repairPosition < $claimPosition,
'Replica worker does not consume the bounded repair queue.');
echo "Storage replica repair behavior passed.\n";