feat(storage): repair missing artifact replicas

go
codex 2 weeks ago
parent 9eb795364d
commit 434ff60561

@ -0,0 +1,368 @@
<?php
namespace App\Services\Cloud;
use App\Models\CloudStorageArtifact;
use App\Models\CloudStorageBackend;
use App\Models\CloudStorageOperation;
use App\Models\CloudStoragePublication;
use App\Models\CloudStorageReplica;
/**
* Reserves one missing immutable replica without contacting a provider.
*
* A later replica worker executes the frozen copy intent. Existing degraded
* or deleted replica rows do not count as durability coverage, which allows a
* replacement backend to repair an artifact without rewriting old evidence.
*/
final class CloudStorageReplicaRepairService
{
/** Reserve at most one repair so each worker cycle remains bounded. */
public function tick(int $now): bool
{
if ($now < 1) {
throw new \InvalidArgumentException('Invalid repair clock');
}
$candidates = CloudStorageArtifact::query()
->whereNotIn('state', [
'retiring', 'delete_attention', 'deleted',
])
->whereIn('id', static function ($query): void {
$query->select('artifact_id')
->from('cloud_storage_publications')
->where('state', 'completed');
})
->whereRaw(
"(SELECT COUNT(*) FROM cloud_storage_replicas repair_coverage "
. "INNER JOIN cloud_storage_backends repair_backend "
. "ON repair_backend.id = repair_coverage.backend_id "
. "WHERE repair_coverage.artifact_id = cloud_storage_artifacts.id "
. "AND ((repair_coverage.state = 'ready' "
. "AND repair_coverage.size_bytes = cloud_storage_artifacts.size_bytes "
. "AND repair_coverage.sha256 = cloud_storage_artifacts.sha256 "
. "AND repair_coverage.verified_at > 0 "
. "AND repair_backend.admin_state <> 'retired' "
. "AND repair_backend.health_state IN ('healthy','degraded','unknown')) "
. "OR (repair_coverage.state IN ('reserved','publishing','copying') "
. "AND repair_backend.admin_state = 'active' "
. "AND repair_backend.health_state = 'healthy' "
. "AND repair_backend.tested_revision = repair_backend.config_revision))) "
. '< cloud_storage_artifacts.desired_copies'
)
->orderBy('id');
// Page through every SQL-level coverage gap. A hard first-page limit
// would let an unrecoverable early artifact starve all later repairs.
foreach ($candidates->lazyById(100, 'id') as $artifact) {
if ($this->reserveForArtifact((int) $artifact->id, $now)) {
return true;
}
}
return false;
}
private function reserveForArtifact(int $artifactId, int $now): bool
{
$connection = (new CloudStorageArtifact())->getConnection();
return $connection->transaction(static function () use (
$artifactId,
$now
): bool {
// Match primary settlement's lock order.
$primaryOperation = CloudStorageOperation::query()
->where('artifact_id', $artifactId)
->where('kind', 'publish')->orderBy('id')
->lockForUpdate()->first();
if (!$primaryOperation instanceof CloudStorageOperation
|| (string) $primaryOperation->phase !== 'terminal'
|| (string) $primaryOperation->outcome !== 'succeeded') {
return false;
}
CloudStorageOperation::query()
->where('artifact_id', $artifactId)
->where('kind', 'copy')->orderBy('id')
->lockForUpdate()->get();
$artifact = CloudStorageArtifact::query()->whereKey($artifactId)
->lockForUpdate()->first();
if (!$artifact instanceof CloudStorageArtifact
|| in_array((string) $artifact->state,
['retiring', 'delete_attention', 'deleted'], true)) {
return false;
}
$replicas = CloudStorageReplica::query()
->where('artifact_id', $artifactId)->orderBy('id')
->lockForUpdate()->get()->all();
$publication = CloudStoragePublication::query()
->where('artifact_id', $artifactId)
->lockForUpdate()->first();
if (!$publication instanceof CloudStoragePublication
|| (string) $publication->state !== 'completed') {
return false;
}
$backends = CloudStorageBackend::query()->orderBy('id')
->lockForUpdate()->get()->keyBy('id');
$ready = [];
$pending = 0;
$represented = [];
foreach ($replicas as $replica) {
$backendId = (int) $replica->backend_id;
$represented[$backendId] = true;
if (self::isExactReady($artifact, $replica)) {
$backend = $backends->get($backendId);
if ($backend instanceof CloudStorageBackend
&& (string) $backend->admin_state !== 'retired'
&& in_array((string) $backend->health_state,
['healthy', 'degraded', 'unknown'], true)) {
$ready[] = [$replica, $backend];
}
} elseif (in_array((string) $replica->state,
['reserved', 'publishing', 'copying'], true)) {
$backend = $backends->get($backendId);
if ($backend instanceof CloudStorageBackend
&& (string) $backend->admin_state === 'active'
&& (string) $backend->health_state === 'healthy'
&& (int) $backend->config_revision > 0
&& (int) $backend->tested_revision
=== (int) $backend->config_revision) {
$pending++;
}
}
}
$desired = (int) $artifact->desired_copies;
if ($desired < 1 || $desired > 2
|| count($ready) >= $desired
|| count($ready) + $pending >= $desired
|| $ready === []) {
return false;
}
usort($ready, static function (array $left, array $right): int {
$leftBackend = $left[1];
$rightBackend = $right[1];
return [
self::healthRank((string) $leftBackend->health_state),
self::adminRank((string) $leftBackend->admin_state),
(int) $leftBackend->write_priority,
(int) $leftBackend->id,
] <=> [
self::healthRank((string) $rightBackend->health_state),
self::adminRank((string) $rightBackend->admin_state),
(int) $rightBackend->write_priority,
(int) $rightBackend->id,
];
});
$source = $ready[0][0];
$sourceBackend = $ready[0][1];
$suffix = self::objectSuffixFromSource(
(string) $sourceBackend->prefix,
(string) $source->object_key
);
if ($suffix === null) {
return false;
}
$readyDomains = [];
foreach ($ready as [, $backend]) {
$readyDomains[self::domain($backend)] = true;
}
$candidates = [];
foreach ($backends as $backend) {
if (!$backend instanceof CloudStorageBackend
|| isset($represented[(int) $backend->id])
|| (string) $backend->admin_state !== 'active'
|| (string) $backend->health_state !== 'healthy'
|| (int) $backend->config_revision < 1
|| (int) $backend->tested_revision
!== (int) $backend->config_revision) {
continue;
}
$isNewDomain = !isset($readyDomains[self::domain($backend)]);
if ((int) $artifact->required_copies > 1 && !$isNewDomain) {
continue;
}
$candidates[] = [$backend, $isNewDomain ? 0 : 1];
}
usort($candidates, static fn (array $left, array $right): int => [
$left[1],
(int) $left[0]->write_priority,
(int) $left[0]->id,
] <=> [
$right[1],
(int) $right[0]->write_priority,
(int) $right[0]->id,
]);
if ($candidates === []) {
return false;
}
$target = $candidates[0][0];
$targetKey = self::objectKey((string) $target->prefix, $suffix);
$collision = CloudStorageReplica::query()
->where('backend_id', (int) $target->id)
->where('object_key', $targetKey)
->lockForUpdate()->first();
if ($collision instanceof CloudStorageReplica) {
return false;
}
self::createRepair(
$artifact,
$source,
$sourceBackend,
$target,
$targetKey,
$now
);
return true;
});
}
private static function createRepair(
CloudStorageArtifact $artifact,
CloudStorageReplica $source,
CloudStorageBackend $sourceBackend,
CloudStorageBackend $target,
string $targetKey,
int $now
): void {
$operationId = bin2hex(random_bytes(16));
$replica = new CloudStorageReplica();
$replica->replica_key = bin2hex(random_bytes(16));
$replica->artifact_id = (int) $artifact->id;
$replica->backend_id = (int) $target->id;
$replica->role = 'secondary';
$replica->object_key = $targetKey;
$replica->object_version_id = '';
$replica->object_etag = '';
$replica->size_bytes = 0;
$replica->sha256 = '';
$replica->state = 'reserved';
$replica->publish_operation_id = $operationId;
$replica->delete_operation_id = null;
$replica->delete_generation = 0;
$replica->delete_cycle_attempts = 0;
$replica->delete_presence_confirmations = 0;
$replica->delete_last_checked_at = 0;
$replica->delete_next_check_at = 0;
$replica->delete_attention_at = null;
$replica->lease_token = null;
$replica->lease_until = 0;
$replica->error_code = '';
$replica->verified_at = 0;
$replica->created_at = $now;
$replica->updated_at = $now;
$replica->save();
$operation = new CloudStorageOperation();
$operation->operation_id = $operationId;
$operation->request_idempotency_key =
'storage-repair-copy-' . substr(hash('sha256',
(string) $artifact->artifact_key . '|'
. (string) $target->id), 0, 48);
$operation->request_fingerprint = hash('sha256', implode('|', [
'repair-copy-v1',
(string) $artifact->artifact_key,
(string) $sourceBackend->id,
(string) $source->object_key,
(string) $source->object_version_id,
(string) $target->id,
$targetKey,
(string) $artifact->size_bytes,
(string) $artifact->sha256,
]));
$operation->artifact_id = (int) $artifact->id;
$operation->replica_id = (int) $replica->id;
$operation->source_backend_id = (int) $sourceBackend->id;
$operation->target_backend_id = (int) $target->id;
$operation->kind = 'copy';
$operation->phase = 'accepted';
$operation->outcome = null;
$operation->effect_resolution = 'not_applicable';
$operation->effect_may_have_applied = null;
$operation->safe_to_resubmit = null;
$operation->attempts = 0;
$operation->lease_token = null;
$operation->lease_until = 0;
$operation->available_at = $now;
$operation->next_check_at = 0;
$operation->error_code = '';
$operation->result_json = null;
$operation->terminal_at = null;
$operation->created_at = $now;
$operation->updated_at = $now;
$operation->save();
}
private static function isExactReady(
CloudStorageArtifact $artifact,
CloudStorageReplica $replica
): bool {
return (string) $replica->state === 'ready'
&& (int) $replica->size_bytes === (int) $artifact->size_bytes
&& hash_equals((string) $artifact->sha256,
(string) $replica->sha256)
&& (int) $replica->verified_at > 0;
}
private static function objectSuffixFromSource(
string $prefix,
string $objectKey
): ?string {
$prefix = trim($prefix, '/');
if ($prefix !== '') {
$start = $prefix . '/';
if (!str_starts_with($objectKey, $start)) {
return null;
}
$objectKey = substr($objectKey, strlen($start));
}
if ($objectKey === '' || strlen($objectKey) > 512
|| preg_match('/^[A-Za-z0-9][A-Za-z0-9._\/-]*$/D',
$objectKey) !== 1
|| str_contains($objectKey, '//')) {
return null;
}
foreach (explode('/', $objectKey) as $segment) {
if ($segment === '.' || $segment === '..') {
return null;
}
}
return $objectKey;
}
private static function objectKey(string $prefix, string $suffix): string
{
$prefix = trim($prefix, '/');
$key = ($prefix === '' ? '' : $prefix . '/') . $suffix;
if (strlen($key) > 512) {
throw new CloudError('对象存储制品路径过长', 422);
}
return $key;
}
private static function domain(CloudStorageBackend $backend): string
{
$domain = trim((string) $backend->failure_domain);
return $domain !== '' ? $domain : 'backend:' . $backend->id;
}
private static function healthRank(string $state): int
{
return match ($state) {
'healthy' => 0,
'degraded' => 1,
'unknown' => 2,
default => 3,
};
}
private static function adminRank(string $state): int
{
return match ($state) {
'active' => 0,
'read_only' => 1,
'draining' => 2,
default => 3,
};
}
}

@ -32,7 +32,9 @@ final class CloudStorageReplicaWorkerService
?callable $runtimeResolver = null,
?callable $clock = null,
private readonly ?CloudStoragePublicationOutboxService
$publicationOutbox = null
$publicationOutbox = null,
private readonly ?CloudStorageReplicaRepairService
$replicaRepair = null
) {
$this->runtimeResolver = $runtimeResolver === null
? static fn (int $backendId, bool $healthy): array =>
@ -66,6 +68,18 @@ final class CloudStorageReplicaWorkerService
$this->errors[] = 'storage_publication_outbox_tick_failed';
}
// Repair only reserves one local, durable copy intent. It performs no
// provider I/O, so a backend that became writable can start healing
// old single-copy artifacts without delaying existing remote work.
$repairWorked = false;
try {
$repair = $this->replicaRepair
?? new CloudStorageReplicaRepairService();
$repairWorked = $repair->tick($now);
} catch (\Throwable) {
$this->errors[] = 'storage_replica_repair_tick_failed';
}
$work = CloudStorageArtifactService::claimReplicaWork($now);
$replicaWorked = $work !== null;
if ($work !== null) {
@ -92,7 +106,7 @@ final class CloudStorageReplicaWorkerService
}
}
$this->errors = array_values(array_unique($this->errors));
return $replicaWorked || $publicationWorked;
return $replicaWorked || $publicationWorked || $repairWorked;
}
/** @return list<string> */

@ -133,6 +133,13 @@ $db->schema()->create('cloud_storage_backends', static function (
): void {
$table->increments('id');
$table->string('name');
$table->string('prefix')->default('');
$table->string('failure_domain')->default('');
$table->unsignedBigInteger('config_revision')->default(1);
$table->unsignedBigInteger('tested_revision')->default(0);
$table->string('admin_state')->default('draft');
$table->string('health_state')->default('unknown');
$table->unsignedInteger('write_priority')->default(0);
});
$clock = 2000000000;

@ -0,0 +1,281 @@
<?php
require dirname(__DIR__) . '/vendor/autoload.php';
use App\Models\CloudStorageArtifact;
use App\Models\CloudStorageOperation;
use App\Models\CloudStoragePublication;
use App\Models\CloudStorageReplica;
use App\Services\Cloud\CloudStorageArtifactService;
use App\Services\Cloud\CloudStoragePublicationFinalizerService;
use App\Services\Cloud\CloudStorageReplicaRepairService;
use Illuminate\Database\Capsule\Manager as Capsule;
use Illuminate\Database\Schema\Blueprint;
$assert = static function (bool $condition, string $message): void {
if (!$condition) {
throw new RuntimeException($message);
}
};
$db = new Capsule();
$db->addConnection(['driver' => 'sqlite', 'database' => ':memory:']);
$db->setAsGlobal();
$db->bootEloquent();
$schema = $db->schema();
$schema->create('cloud_storage_backends', static function (Blueprint $t): void {
$t->increments('id');
$t->string('backend_key')->unique(); $t->string('identity_sha256')->unique();
$t->string('name'); $t->string('driver'); $t->string('endpoint');
$t->string('region'); $t->string('bucket'); $t->string('prefix');
$t->boolean('path_style'); $t->string('failure_domain');
$t->text('sealed_credentials'); $t->unsignedBigInteger('config_revision');
$t->unsignedBigInteger('tested_revision'); $t->string('admin_state');
$t->string('health_state'); $t->unsignedInteger('write_priority');
$t->unsignedInteger('consecutive_failures')->default(0);
$t->unsignedInteger('consecutive_successes')->default(0);
$t->string('last_error_code')->default('');
$t->unsignedBigInteger('last_tested_at')->default(0);
$t->unsignedBigInteger('last_healthy_at')->default(0);
$t->unsignedBigInteger('next_probe_at')->default(0);
$t->unsignedBigInteger('created_at'); $t->unsignedBigInteger('updated_at');
});
$schema->create('cloud_storage_artifacts', static function (Blueprint $t): void {
$t->increments('id'); $t->string('artifact_key')->unique();
$t->string('kind'); $t->unsignedBigInteger('maximum_size_bytes');
$t->unsignedBigInteger('size_bytes'); $t->string('sha256');
$t->string('durability_policy'); $t->unsignedTinyInteger('desired_copies');
$t->unsignedTinyInteger('required_copies'); $t->string('state');
$t->string('last_operation_id')->nullable();
$t->unsignedBigInteger('verified_at');
$t->unsignedBigInteger('created_at'); $t->unsignedBigInteger('updated_at');
});
$schema->create('cloud_storage_replicas', static function (Blueprint $t): void {
$t->increments('id'); $t->string('replica_key')->unique();
$t->unsignedBigInteger('artifact_id'); $t->unsignedBigInteger('backend_id');
$t->string('role'); $t->string('object_key');
$t->string('object_version_id')->default('');
$t->string('object_etag')->default('');
$t->unsignedBigInteger('size_bytes')->default(0);
$t->string('sha256')->default(''); $t->string('state');
$t->string('publish_operation_id')->nullable();
$t->string('delete_operation_id')->nullable();
$t->unsignedInteger('delete_generation')->default(0);
$t->unsignedInteger('delete_cycle_attempts')->default(0);
$t->unsignedInteger('delete_presence_confirmations')->default(0);
$t->unsignedBigInteger('delete_last_checked_at')->default(0);
$t->unsignedBigInteger('delete_next_check_at')->default(0);
$t->unsignedBigInteger('delete_attention_at')->nullable();
$t->string('lease_token')->nullable();
$t->unsignedBigInteger('lease_until')->default(0);
$t->string('error_code')->default('');
$t->unsignedBigInteger('verified_at')->default(0);
$t->unsignedBigInteger('created_at'); $t->unsignedBigInteger('updated_at');
$t->unique(['artifact_id', 'backend_id']);
$t->unique(['backend_id', 'object_key']);
});
$schema->create('cloud_storage_operations', static function (Blueprint $t): void {
$t->increments('id'); $t->string('operation_id')->unique();
$t->string('request_idempotency_key')->unique();
$t->string('request_fingerprint'); $t->unsignedBigInteger('artifact_id');
$t->unsignedBigInteger('replica_id')->nullable();
$t->unsignedBigInteger('source_backend_id')->nullable();
$t->unsignedBigInteger('target_backend_id')->nullable();
$t->string('kind'); $t->string('phase'); $t->string('outcome')->nullable();
$t->string('effect_resolution');
$t->boolean('effect_may_have_applied')->nullable();
$t->boolean('safe_to_resubmit')->nullable();
$t->unsignedInteger('attempts'); $t->string('lease_token')->nullable();
$t->unsignedBigInteger('lease_until'); $t->unsignedBigInteger('available_at');
$t->unsignedBigInteger('next_check_at'); $t->string('error_code');
$t->text('result_json')->nullable(); $t->unsignedBigInteger('terminal_at')->nullable();
$t->unsignedBigInteger('created_at'); $t->unsignedBigInteger('updated_at');
});
$schema->create('cloud_storage_publications', static function (Blueprint $t): void {
$t->increments('id'); $t->string('publication_key')->unique();
$t->unsignedBigInteger('artifact_id')->unique();
$t->unsignedBigInteger('backup_id')->nullable()->unique();
$t->string('template_import_intent_id')->nullable()->unique();
$t->string('rescue_template_id')->nullable()->unique();
$t->string('state'); $t->unsignedInteger('attempts')->default(0);
$t->unsignedBigInteger('available_at')->default(0);
$t->string('lease_token')->nullable();
$t->unsignedBigInteger('lease_until')->default(0);
$t->string('last_error_code')->default('');
$t->unsignedBigInteger('coverage_ready_at')->nullable();
$t->unsignedBigInteger('finalized_at')->nullable();
$t->unsignedBigInteger('created_at'); $t->unsignedBigInteger('updated_at');
});
$now = 1800000000;
$insertBackend = static function (
int $id,
string $state,
string $health,
int $tested,
string $domain,
int $priority,
string $prefix
) use ($now): void {
Capsule::table('cloud_storage_backends')->insert([
'id' => $id, 'backend_key' => str_pad((string) $id, 32, 'b'),
'identity_sha256' => str_pad((string) $id, 64, 'c'),
'name' => 'backend-' . $id, 'driver' => 's3',
'endpoint' => 'https://storage-' . $id . '.example.test',
'region' => 'us-east-1', 'bucket' => 'bucket-' . $id,
'prefix' => $prefix, 'path_style' => 1,
'failure_domain' => $domain, 'sealed_credentials' => 'sealed',
'config_revision' => 1, 'tested_revision' => $tested,
'admin_state' => $state, 'health_state' => $health,
'write_priority' => $priority, 'consecutive_failures' => 0,
'consecutive_successes' => $health === 'healthy' ? 1 : 0,
'last_error_code' => '', 'last_tested_at' => $tested > 0 ? $now : 0,
'last_healthy_at' => $health === 'healthy' ? $now : 0,
'next_probe_at' => $now + 300,
'created_at' => $now, 'updated_at' => $now,
]);
};
$insertBackend(1, 'active', 'healthy', 1, 'provider-a', 10, 'primary');
$insertBackend(2, 'draft', 'unknown', 0, 'provider-b', 20, 'secondary');
$view = CloudStorageArtifactService::beginPublication(
'backup', 'backups/late/system.qcow2', 4096, str_repeat('a', 64),
'repair-fixture-primary', hash('sha256', 'repair-fixture'),
['backup_id' => 101]
);
$assert(CloudStorageReplica::where('artifact_id', $view['artifact_id'])->count() === 1,
'Unavailable second backend was unexpectedly reserved at publication time.');
CloudStorageArtifactService::markPrimaryDispatching($view['operation_id']);
CloudStorageArtifactService::completePrimary(
$view['operation_id'], (int) $view['artifact_id'], [
'size_bytes' => 4096, 'sha256' => str_repeat('a', 64),
'etag' => 'primary-etag', 'version_id' => 'primary-version',
]
);
CloudStoragePublicationFinalizerService::complete(
'backup', 101, (int) $view['artifact_id'], $view['operation_id']
);
Capsule::table('cloud_storage_backends')->where('id', 2)->update([
'admin_state' => 'active', 'health_state' => 'healthy',
'tested_revision' => 1, 'last_tested_at' => $now,
'last_healthy_at' => $now, 'updated_at' => $now,
]);
$repair = new CloudStorageReplicaRepairService();
$assert($repair->tick($now + 1),
'Recovered second backend did not reserve a missing replica.');
$replicas = CloudStorageReplica::where('artifact_id', $view['artifact_id'])
->orderBy('id')->get();
$copy = CloudStorageOperation::where('artifact_id', $view['artifact_id'])
->where('kind', 'copy')->firstOrFail();
$assert($replicas->count() === 2
&& (int) $replicas->last()->backend_id === 2
&& (string) $replicas->last()->object_key
=== 'secondary/backups/late/system.qcow2'
&& (int) $copy->source_backend_id === 1
&& (int) $copy->target_backend_id === 2
&& (string) $copy->phase === 'accepted'
&& (int) $copy->available_at === $now + 1,
'Missing-replica repair did not freeze the expected copy intent.');
$assert(!$repair->tick($now + 2)
&& CloudStorageReplica::where('artifact_id', $view['artifact_id'])->count() === 2
&& CloudStorageOperation::where('artifact_id', $view['artifact_id'])
->where('kind', 'copy')->count() === 1,
'Repeated repair allocated duplicate replica work.');
CloudStorageArtifact::whereKey($view['artifact_id'])->update([
'desired_copies' => 1,
]);
// A failed old replica must not occupy durability coverage. For a required-two
// artifact, replacement also must use a failure domain distinct from the only
// exact ready source.
$insertBackend(3, 'active', 'healthy', 1, 'provider-a', 1, 'same-domain');
$insertBackend(4, 'active', 'healthy', 1, 'provider-c', 50, 'replacement');
$artifactId = Capsule::table('cloud_storage_artifacts')->insertGetId([
'artifact_key' => str_repeat('d', 32), 'kind' => 'template',
'maximum_size_bytes' => 8192, 'size_bytes' => 8192,
'sha256' => str_repeat('e', 64),
'durability_policy' => 'mirror_required_2',
'desired_copies' => 2, 'required_copies' => 2,
'state' => 'protected', 'last_operation_id' => str_repeat('1', 32),
'verified_at' => $now, 'created_at' => $now, 'updated_at' => $now,
]);
$primaryReplicaId = Capsule::table('cloud_storage_replicas')->insertGetId([
'replica_key' => str_repeat('4', 32), 'artifact_id' => $artifactId,
'backend_id' => 1, 'role' => 'primary',
'object_key' => 'primary/templates/image.qcow2',
'object_version_id' => 'v1', 'object_etag' => 'e1',
'size_bytes' => 8192, 'sha256' => str_repeat('e', 64),
'state' => 'ready', 'publish_operation_id' => str_repeat('1', 32),
'lease_until' => 0, 'error_code' => '', 'verified_at' => $now,
'created_at' => $now, 'updated_at' => $now,
]);
Capsule::table('cloud_storage_replicas')->insert([
'replica_key' => str_repeat('5', 32), 'artifact_id' => $artifactId,
'backend_id' => 2, 'role' => 'secondary',
'object_key' => 'secondary/templates/image.qcow2',
'object_version_id' => 'v2', 'object_etag' => 'e2',
'size_bytes' => 8192, 'sha256' => str_repeat('e', 64),
'state' => 'ready', 'publish_operation_id' => str_repeat('2', 32),
'lease_until' => 0, 'error_code' => '', 'verified_at' => $now,
'created_at' => $now, 'updated_at' => $now,
]);
Capsule::table('cloud_storage_operations')->insert([
'operation_id' => str_repeat('1', 32),
'request_idempotency_key' => 'template-primary-fixture',
'request_fingerprint' => str_repeat('6', 64),
'artifact_id' => $artifactId, 'replica_id' => $primaryReplicaId,
'target_backend_id' => 1, 'kind' => 'publish', 'phase' => 'terminal',
'outcome' => 'succeeded', 'effect_resolution' => 'applied',
'effect_may_have_applied' => 0, 'safe_to_resubmit' => 0,
'attempts' => 1, 'lease_until' => 0, 'available_at' => 0,
'next_check_at' => 0, 'error_code' => '',
'terminal_at' => $now, 'created_at' => $now, 'updated_at' => $now,
]);
Capsule::table('cloud_storage_operations')->insert([
'operation_id' => str_repeat('2', 32),
'request_idempotency_key' => 'template-old-copy-fixture',
'request_fingerprint' => str_repeat('7', 64),
'artifact_id' => $artifactId,
'replica_id' => CloudStorageReplica::where('artifact_id', $artifactId)
->where('backend_id', 2)->value('id'),
'source_backend_id' => 1, 'target_backend_id' => 2,
'kind' => 'copy', 'phase' => 'terminal', 'outcome' => 'failed',
'effect_resolution' => 'not_applied', 'effect_may_have_applied' => 0,
'safe_to_resubmit' => 0, 'attempts' => 1, 'lease_until' => 0,
'available_at' => 0, 'next_check_at' => 0,
'error_code' => 'provider_lost', 'terminal_at' => $now,
'created_at' => $now, 'updated_at' => $now,
]);
Capsule::table('cloud_storage_publications')->insert([
'publication_key' => str_repeat('8', 32), 'artifact_id' => $artifactId,
'template_import_intent_id' => str_repeat('9', 32),
'state' => 'completed', 'attempts' => 1, 'available_at' => 0,
'lease_until' => 0, 'last_error_code' => '',
'coverage_ready_at' => $now, 'finalized_at' => $now,
'created_at' => $now, 'updated_at' => $now,
]);
Capsule::table('cloud_storage_backends')->where('id', 2)->update([
'admin_state' => 'read_only', 'health_state' => 'unreachable',
'tested_revision' => 0, 'last_error_code' => 'endpoint_unreachable',
'updated_at' => $now,
]);
$assert($repair->tick($now + 3),
'Replacement backend did not repair unreachable required-two coverage.');
$replacement = CloudStorageReplica::where('artifact_id', $artifactId)
->where('state', 'reserved')->firstOrFail();
$assert((int) $replacement->backend_id === 4
&& (string) $replacement->object_key
=== 'replacement/templates/image.qcow2'
&& CloudStorageReplica::where('artifact_id', $artifactId)->count() === 3,
'Repair counted degraded evidence or selected the source failure domain.');
$workerSource = file_get_contents(dirname(__DIR__)
. '/app/Services/Cloud/CloudStorageReplicaWorkerService.php');
$assert(is_string($workerSource)
&& str_contains($workerSource, 'CloudStorageReplicaRepairService')
&& str_contains($workerSource, 'storage_replica_repair_tick_failed'),
'Replica worker does not consume the bounded repair queue.');
echo "Storage replica repair behavior passed.\n";
Loading…
Cancel
Save