feat(storage): repair missing artifact replicas
parent
9eb795364d
commit
434ff60561
@ -0,0 +1,368 @@
|
||||
<?php
|
||||
|
||||
namespace App\Services\Cloud;
|
||||
|
||||
use App\Models\CloudStorageArtifact;
|
||||
use App\Models\CloudStorageBackend;
|
||||
use App\Models\CloudStorageOperation;
|
||||
use App\Models\CloudStoragePublication;
|
||||
use App\Models\CloudStorageReplica;
|
||||
|
||||
/**
|
||||
* Reserves one missing immutable replica without contacting a provider.
|
||||
*
|
||||
* A later replica worker executes the frozen copy intent. Existing degraded
|
||||
* or deleted replica rows do not count as durability coverage, which allows a
|
||||
* replacement backend to repair an artifact without rewriting old evidence.
|
||||
*/
|
||||
final class CloudStorageReplicaRepairService
|
||||
{
|
||||
/** Reserve at most one repair so each worker cycle remains bounded. */
|
||||
public function tick(int $now): bool
|
||||
{
|
||||
if ($now < 1) {
|
||||
throw new \InvalidArgumentException('Invalid repair clock');
|
||||
}
|
||||
$candidates = CloudStorageArtifact::query()
|
||||
->whereNotIn('state', [
|
||||
'retiring', 'delete_attention', 'deleted',
|
||||
])
|
||||
->whereIn('id', static function ($query): void {
|
||||
$query->select('artifact_id')
|
||||
->from('cloud_storage_publications')
|
||||
->where('state', 'completed');
|
||||
})
|
||||
->whereRaw(
|
||||
"(SELECT COUNT(*) FROM cloud_storage_replicas repair_coverage "
|
||||
. "INNER JOIN cloud_storage_backends repair_backend "
|
||||
. "ON repair_backend.id = repair_coverage.backend_id "
|
||||
. "WHERE repair_coverage.artifact_id = cloud_storage_artifacts.id "
|
||||
. "AND ((repair_coverage.state = 'ready' "
|
||||
. "AND repair_coverage.size_bytes = cloud_storage_artifacts.size_bytes "
|
||||
. "AND repair_coverage.sha256 = cloud_storage_artifacts.sha256 "
|
||||
. "AND repair_coverage.verified_at > 0 "
|
||||
. "AND repair_backend.admin_state <> 'retired' "
|
||||
. "AND repair_backend.health_state IN ('healthy','degraded','unknown')) "
|
||||
. "OR (repair_coverage.state IN ('reserved','publishing','copying') "
|
||||
. "AND repair_backend.admin_state = 'active' "
|
||||
. "AND repair_backend.health_state = 'healthy' "
|
||||
. "AND repair_backend.tested_revision = repair_backend.config_revision))) "
|
||||
. '< cloud_storage_artifacts.desired_copies'
|
||||
)
|
||||
->orderBy('id');
|
||||
|
||||
// Page through every SQL-level coverage gap. A hard first-page limit
|
||||
// would let an unrecoverable early artifact starve all later repairs.
|
||||
foreach ($candidates->lazyById(100, 'id') as $artifact) {
|
||||
if ($this->reserveForArtifact((int) $artifact->id, $now)) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
private function reserveForArtifact(int $artifactId, int $now): bool
|
||||
{
|
||||
$connection = (new CloudStorageArtifact())->getConnection();
|
||||
return $connection->transaction(static function () use (
|
||||
$artifactId,
|
||||
$now
|
||||
): bool {
|
||||
// Match primary settlement's lock order.
|
||||
$primaryOperation = CloudStorageOperation::query()
|
||||
->where('artifact_id', $artifactId)
|
||||
->where('kind', 'publish')->orderBy('id')
|
||||
->lockForUpdate()->first();
|
||||
if (!$primaryOperation instanceof CloudStorageOperation
|
||||
|| (string) $primaryOperation->phase !== 'terminal'
|
||||
|| (string) $primaryOperation->outcome !== 'succeeded') {
|
||||
return false;
|
||||
}
|
||||
CloudStorageOperation::query()
|
||||
->where('artifact_id', $artifactId)
|
||||
->where('kind', 'copy')->orderBy('id')
|
||||
->lockForUpdate()->get();
|
||||
$artifact = CloudStorageArtifact::query()->whereKey($artifactId)
|
||||
->lockForUpdate()->first();
|
||||
if (!$artifact instanceof CloudStorageArtifact
|
||||
|| in_array((string) $artifact->state,
|
||||
['retiring', 'delete_attention', 'deleted'], true)) {
|
||||
return false;
|
||||
}
|
||||
$replicas = CloudStorageReplica::query()
|
||||
->where('artifact_id', $artifactId)->orderBy('id')
|
||||
->lockForUpdate()->get()->all();
|
||||
$publication = CloudStoragePublication::query()
|
||||
->where('artifact_id', $artifactId)
|
||||
->lockForUpdate()->first();
|
||||
if (!$publication instanceof CloudStoragePublication
|
||||
|| (string) $publication->state !== 'completed') {
|
||||
return false;
|
||||
}
|
||||
$backends = CloudStorageBackend::query()->orderBy('id')
|
||||
->lockForUpdate()->get()->keyBy('id');
|
||||
|
||||
$ready = [];
|
||||
$pending = 0;
|
||||
$represented = [];
|
||||
foreach ($replicas as $replica) {
|
||||
$backendId = (int) $replica->backend_id;
|
||||
$represented[$backendId] = true;
|
||||
if (self::isExactReady($artifact, $replica)) {
|
||||
$backend = $backends->get($backendId);
|
||||
if ($backend instanceof CloudStorageBackend
|
||||
&& (string) $backend->admin_state !== 'retired'
|
||||
&& in_array((string) $backend->health_state,
|
||||
['healthy', 'degraded', 'unknown'], true)) {
|
||||
$ready[] = [$replica, $backend];
|
||||
}
|
||||
} elseif (in_array((string) $replica->state,
|
||||
['reserved', 'publishing', 'copying'], true)) {
|
||||
$backend = $backends->get($backendId);
|
||||
if ($backend instanceof CloudStorageBackend
|
||||
&& (string) $backend->admin_state === 'active'
|
||||
&& (string) $backend->health_state === 'healthy'
|
||||
&& (int) $backend->config_revision > 0
|
||||
&& (int) $backend->tested_revision
|
||||
=== (int) $backend->config_revision) {
|
||||
$pending++;
|
||||
}
|
||||
}
|
||||
}
|
||||
$desired = (int) $artifact->desired_copies;
|
||||
if ($desired < 1 || $desired > 2
|
||||
|| count($ready) >= $desired
|
||||
|| count($ready) + $pending >= $desired
|
||||
|| $ready === []) {
|
||||
return false;
|
||||
}
|
||||
|
||||
usort($ready, static function (array $left, array $right): int {
|
||||
$leftBackend = $left[1];
|
||||
$rightBackend = $right[1];
|
||||
return [
|
||||
self::healthRank((string) $leftBackend->health_state),
|
||||
self::adminRank((string) $leftBackend->admin_state),
|
||||
(int) $leftBackend->write_priority,
|
||||
(int) $leftBackend->id,
|
||||
] <=> [
|
||||
self::healthRank((string) $rightBackend->health_state),
|
||||
self::adminRank((string) $rightBackend->admin_state),
|
||||
(int) $rightBackend->write_priority,
|
||||
(int) $rightBackend->id,
|
||||
];
|
||||
});
|
||||
$source = $ready[0][0];
|
||||
$sourceBackend = $ready[0][1];
|
||||
$suffix = self::objectSuffixFromSource(
|
||||
(string) $sourceBackend->prefix,
|
||||
(string) $source->object_key
|
||||
);
|
||||
if ($suffix === null) {
|
||||
return false;
|
||||
}
|
||||
|
||||
$readyDomains = [];
|
||||
foreach ($ready as [, $backend]) {
|
||||
$readyDomains[self::domain($backend)] = true;
|
||||
}
|
||||
$candidates = [];
|
||||
foreach ($backends as $backend) {
|
||||
if (!$backend instanceof CloudStorageBackend
|
||||
|| isset($represented[(int) $backend->id])
|
||||
|| (string) $backend->admin_state !== 'active'
|
||||
|| (string) $backend->health_state !== 'healthy'
|
||||
|| (int) $backend->config_revision < 1
|
||||
|| (int) $backend->tested_revision
|
||||
!== (int) $backend->config_revision) {
|
||||
continue;
|
||||
}
|
||||
$isNewDomain = !isset($readyDomains[self::domain($backend)]);
|
||||
if ((int) $artifact->required_copies > 1 && !$isNewDomain) {
|
||||
continue;
|
||||
}
|
||||
$candidates[] = [$backend, $isNewDomain ? 0 : 1];
|
||||
}
|
||||
usort($candidates, static fn (array $left, array $right): int => [
|
||||
$left[1],
|
||||
(int) $left[0]->write_priority,
|
||||
(int) $left[0]->id,
|
||||
] <=> [
|
||||
$right[1],
|
||||
(int) $right[0]->write_priority,
|
||||
(int) $right[0]->id,
|
||||
]);
|
||||
if ($candidates === []) {
|
||||
return false;
|
||||
}
|
||||
$target = $candidates[0][0];
|
||||
$targetKey = self::objectKey((string) $target->prefix, $suffix);
|
||||
$collision = CloudStorageReplica::query()
|
||||
->where('backend_id', (int) $target->id)
|
||||
->where('object_key', $targetKey)
|
||||
->lockForUpdate()->first();
|
||||
if ($collision instanceof CloudStorageReplica) {
|
||||
return false;
|
||||
}
|
||||
|
||||
self::createRepair(
|
||||
$artifact,
|
||||
$source,
|
||||
$sourceBackend,
|
||||
$target,
|
||||
$targetKey,
|
||||
$now
|
||||
);
|
||||
return true;
|
||||
});
|
||||
}
|
||||
|
||||
private static function createRepair(
|
||||
CloudStorageArtifact $artifact,
|
||||
CloudStorageReplica $source,
|
||||
CloudStorageBackend $sourceBackend,
|
||||
CloudStorageBackend $target,
|
||||
string $targetKey,
|
||||
int $now
|
||||
): void {
|
||||
$operationId = bin2hex(random_bytes(16));
|
||||
$replica = new CloudStorageReplica();
|
||||
$replica->replica_key = bin2hex(random_bytes(16));
|
||||
$replica->artifact_id = (int) $artifact->id;
|
||||
$replica->backend_id = (int) $target->id;
|
||||
$replica->role = 'secondary';
|
||||
$replica->object_key = $targetKey;
|
||||
$replica->object_version_id = '';
|
||||
$replica->object_etag = '';
|
||||
$replica->size_bytes = 0;
|
||||
$replica->sha256 = '';
|
||||
$replica->state = 'reserved';
|
||||
$replica->publish_operation_id = $operationId;
|
||||
$replica->delete_operation_id = null;
|
||||
$replica->delete_generation = 0;
|
||||
$replica->delete_cycle_attempts = 0;
|
||||
$replica->delete_presence_confirmations = 0;
|
||||
$replica->delete_last_checked_at = 0;
|
||||
$replica->delete_next_check_at = 0;
|
||||
$replica->delete_attention_at = null;
|
||||
$replica->lease_token = null;
|
||||
$replica->lease_until = 0;
|
||||
$replica->error_code = '';
|
||||
$replica->verified_at = 0;
|
||||
$replica->created_at = $now;
|
||||
$replica->updated_at = $now;
|
||||
$replica->save();
|
||||
|
||||
$operation = new CloudStorageOperation();
|
||||
$operation->operation_id = $operationId;
|
||||
$operation->request_idempotency_key =
|
||||
'storage-repair-copy-' . substr(hash('sha256',
|
||||
(string) $artifact->artifact_key . '|'
|
||||
. (string) $target->id), 0, 48);
|
||||
$operation->request_fingerprint = hash('sha256', implode('|', [
|
||||
'repair-copy-v1',
|
||||
(string) $artifact->artifact_key,
|
||||
(string) $sourceBackend->id,
|
||||
(string) $source->object_key,
|
||||
(string) $source->object_version_id,
|
||||
(string) $target->id,
|
||||
$targetKey,
|
||||
(string) $artifact->size_bytes,
|
||||
(string) $artifact->sha256,
|
||||
]));
|
||||
$operation->artifact_id = (int) $artifact->id;
|
||||
$operation->replica_id = (int) $replica->id;
|
||||
$operation->source_backend_id = (int) $sourceBackend->id;
|
||||
$operation->target_backend_id = (int) $target->id;
|
||||
$operation->kind = 'copy';
|
||||
$operation->phase = 'accepted';
|
||||
$operation->outcome = null;
|
||||
$operation->effect_resolution = 'not_applicable';
|
||||
$operation->effect_may_have_applied = null;
|
||||
$operation->safe_to_resubmit = null;
|
||||
$operation->attempts = 0;
|
||||
$operation->lease_token = null;
|
||||
$operation->lease_until = 0;
|
||||
$operation->available_at = $now;
|
||||
$operation->next_check_at = 0;
|
||||
$operation->error_code = '';
|
||||
$operation->result_json = null;
|
||||
$operation->terminal_at = null;
|
||||
$operation->created_at = $now;
|
||||
$operation->updated_at = $now;
|
||||
$operation->save();
|
||||
}
|
||||
|
||||
private static function isExactReady(
|
||||
CloudStorageArtifact $artifact,
|
||||
CloudStorageReplica $replica
|
||||
): bool {
|
||||
return (string) $replica->state === 'ready'
|
||||
&& (int) $replica->size_bytes === (int) $artifact->size_bytes
|
||||
&& hash_equals((string) $artifact->sha256,
|
||||
(string) $replica->sha256)
|
||||
&& (int) $replica->verified_at > 0;
|
||||
}
|
||||
|
||||
private static function objectSuffixFromSource(
|
||||
string $prefix,
|
||||
string $objectKey
|
||||
): ?string {
|
||||
$prefix = trim($prefix, '/');
|
||||
if ($prefix !== '') {
|
||||
$start = $prefix . '/';
|
||||
if (!str_starts_with($objectKey, $start)) {
|
||||
return null;
|
||||
}
|
||||
$objectKey = substr($objectKey, strlen($start));
|
||||
}
|
||||
if ($objectKey === '' || strlen($objectKey) > 512
|
||||
|| preg_match('/^[A-Za-z0-9][A-Za-z0-9._\/-]*$/D',
|
||||
$objectKey) !== 1
|
||||
|| str_contains($objectKey, '//')) {
|
||||
return null;
|
||||
}
|
||||
foreach (explode('/', $objectKey) as $segment) {
|
||||
if ($segment === '.' || $segment === '..') {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
return $objectKey;
|
||||
}
|
||||
|
||||
private static function objectKey(string $prefix, string $suffix): string
|
||||
{
|
||||
$prefix = trim($prefix, '/');
|
||||
$key = ($prefix === '' ? '' : $prefix . '/') . $suffix;
|
||||
if (strlen($key) > 512) {
|
||||
throw new CloudError('对象存储制品路径过长', 422);
|
||||
}
|
||||
return $key;
|
||||
}
|
||||
|
||||
private static function domain(CloudStorageBackend $backend): string
|
||||
{
|
||||
$domain = trim((string) $backend->failure_domain);
|
||||
return $domain !== '' ? $domain : 'backend:' . $backend->id;
|
||||
}
|
||||
|
||||
private static function healthRank(string $state): int
|
||||
{
|
||||
return match ($state) {
|
||||
'healthy' => 0,
|
||||
'degraded' => 1,
|
||||
'unknown' => 2,
|
||||
default => 3,
|
||||
};
|
||||
}
|
||||
|
||||
private static function adminRank(string $state): int
|
||||
{
|
||||
return match ($state) {
|
||||
'active' => 0,
|
||||
'read_only' => 1,
|
||||
'draining' => 2,
|
||||
default => 3,
|
||||
};
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,281 @@
|
||||
<?php
|
||||
|
||||
require dirname(__DIR__) . '/vendor/autoload.php';
|
||||
|
||||
use App\Models\CloudStorageArtifact;
|
||||
use App\Models\CloudStorageOperation;
|
||||
use App\Models\CloudStoragePublication;
|
||||
use App\Models\CloudStorageReplica;
|
||||
use App\Services\Cloud\CloudStorageArtifactService;
|
||||
use App\Services\Cloud\CloudStoragePublicationFinalizerService;
|
||||
use App\Services\Cloud\CloudStorageReplicaRepairService;
|
||||
use Illuminate\Database\Capsule\Manager as Capsule;
|
||||
use Illuminate\Database\Schema\Blueprint;
|
||||
|
||||
$assert = static function (bool $condition, string $message): void {
|
||||
if (!$condition) {
|
||||
throw new RuntimeException($message);
|
||||
}
|
||||
};
|
||||
|
||||
$db = new Capsule();
|
||||
$db->addConnection(['driver' => 'sqlite', 'database' => ':memory:']);
|
||||
$db->setAsGlobal();
|
||||
$db->bootEloquent();
|
||||
$schema = $db->schema();
|
||||
$schema->create('cloud_storage_backends', static function (Blueprint $t): void {
|
||||
$t->increments('id');
|
||||
$t->string('backend_key')->unique(); $t->string('identity_sha256')->unique();
|
||||
$t->string('name'); $t->string('driver'); $t->string('endpoint');
|
||||
$t->string('region'); $t->string('bucket'); $t->string('prefix');
|
||||
$t->boolean('path_style'); $t->string('failure_domain');
|
||||
$t->text('sealed_credentials'); $t->unsignedBigInteger('config_revision');
|
||||
$t->unsignedBigInteger('tested_revision'); $t->string('admin_state');
|
||||
$t->string('health_state'); $t->unsignedInteger('write_priority');
|
||||
$t->unsignedInteger('consecutive_failures')->default(0);
|
||||
$t->unsignedInteger('consecutive_successes')->default(0);
|
||||
$t->string('last_error_code')->default('');
|
||||
$t->unsignedBigInteger('last_tested_at')->default(0);
|
||||
$t->unsignedBigInteger('last_healthy_at')->default(0);
|
||||
$t->unsignedBigInteger('next_probe_at')->default(0);
|
||||
$t->unsignedBigInteger('created_at'); $t->unsignedBigInteger('updated_at');
|
||||
});
|
||||
$schema->create('cloud_storage_artifacts', static function (Blueprint $t): void {
|
||||
$t->increments('id'); $t->string('artifact_key')->unique();
|
||||
$t->string('kind'); $t->unsignedBigInteger('maximum_size_bytes');
|
||||
$t->unsignedBigInteger('size_bytes'); $t->string('sha256');
|
||||
$t->string('durability_policy'); $t->unsignedTinyInteger('desired_copies');
|
||||
$t->unsignedTinyInteger('required_copies'); $t->string('state');
|
||||
$t->string('last_operation_id')->nullable();
|
||||
$t->unsignedBigInteger('verified_at');
|
||||
$t->unsignedBigInteger('created_at'); $t->unsignedBigInteger('updated_at');
|
||||
});
|
||||
$schema->create('cloud_storage_replicas', static function (Blueprint $t): void {
|
||||
$t->increments('id'); $t->string('replica_key')->unique();
|
||||
$t->unsignedBigInteger('artifact_id'); $t->unsignedBigInteger('backend_id');
|
||||
$t->string('role'); $t->string('object_key');
|
||||
$t->string('object_version_id')->default('');
|
||||
$t->string('object_etag')->default('');
|
||||
$t->unsignedBigInteger('size_bytes')->default(0);
|
||||
$t->string('sha256')->default(''); $t->string('state');
|
||||
$t->string('publish_operation_id')->nullable();
|
||||
$t->string('delete_operation_id')->nullable();
|
||||
$t->unsignedInteger('delete_generation')->default(0);
|
||||
$t->unsignedInteger('delete_cycle_attempts')->default(0);
|
||||
$t->unsignedInteger('delete_presence_confirmations')->default(0);
|
||||
$t->unsignedBigInteger('delete_last_checked_at')->default(0);
|
||||
$t->unsignedBigInteger('delete_next_check_at')->default(0);
|
||||
$t->unsignedBigInteger('delete_attention_at')->nullable();
|
||||
$t->string('lease_token')->nullable();
|
||||
$t->unsignedBigInteger('lease_until')->default(0);
|
||||
$t->string('error_code')->default('');
|
||||
$t->unsignedBigInteger('verified_at')->default(0);
|
||||
$t->unsignedBigInteger('created_at'); $t->unsignedBigInteger('updated_at');
|
||||
$t->unique(['artifact_id', 'backend_id']);
|
||||
$t->unique(['backend_id', 'object_key']);
|
||||
});
|
||||
$schema->create('cloud_storage_operations', static function (Blueprint $t): void {
|
||||
$t->increments('id'); $t->string('operation_id')->unique();
|
||||
$t->string('request_idempotency_key')->unique();
|
||||
$t->string('request_fingerprint'); $t->unsignedBigInteger('artifact_id');
|
||||
$t->unsignedBigInteger('replica_id')->nullable();
|
||||
$t->unsignedBigInteger('source_backend_id')->nullable();
|
||||
$t->unsignedBigInteger('target_backend_id')->nullable();
|
||||
$t->string('kind'); $t->string('phase'); $t->string('outcome')->nullable();
|
||||
$t->string('effect_resolution');
|
||||
$t->boolean('effect_may_have_applied')->nullable();
|
||||
$t->boolean('safe_to_resubmit')->nullable();
|
||||
$t->unsignedInteger('attempts'); $t->string('lease_token')->nullable();
|
||||
$t->unsignedBigInteger('lease_until'); $t->unsignedBigInteger('available_at');
|
||||
$t->unsignedBigInteger('next_check_at'); $t->string('error_code');
|
||||
$t->text('result_json')->nullable(); $t->unsignedBigInteger('terminal_at')->nullable();
|
||||
$t->unsignedBigInteger('created_at'); $t->unsignedBigInteger('updated_at');
|
||||
});
|
||||
$schema->create('cloud_storage_publications', static function (Blueprint $t): void {
|
||||
$t->increments('id'); $t->string('publication_key')->unique();
|
||||
$t->unsignedBigInteger('artifact_id')->unique();
|
||||
$t->unsignedBigInteger('backup_id')->nullable()->unique();
|
||||
$t->string('template_import_intent_id')->nullable()->unique();
|
||||
$t->string('rescue_template_id')->nullable()->unique();
|
||||
$t->string('state'); $t->unsignedInteger('attempts')->default(0);
|
||||
$t->unsignedBigInteger('available_at')->default(0);
|
||||
$t->string('lease_token')->nullable();
|
||||
$t->unsignedBigInteger('lease_until')->default(0);
|
||||
$t->string('last_error_code')->default('');
|
||||
$t->unsignedBigInteger('coverage_ready_at')->nullable();
|
||||
$t->unsignedBigInteger('finalized_at')->nullable();
|
||||
$t->unsignedBigInteger('created_at'); $t->unsignedBigInteger('updated_at');
|
||||
});
|
||||
|
||||
$now = 1800000000;
|
||||
$insertBackend = static function (
|
||||
int $id,
|
||||
string $state,
|
||||
string $health,
|
||||
int $tested,
|
||||
string $domain,
|
||||
int $priority,
|
||||
string $prefix
|
||||
) use ($now): void {
|
||||
Capsule::table('cloud_storage_backends')->insert([
|
||||
'id' => $id, 'backend_key' => str_pad((string) $id, 32, 'b'),
|
||||
'identity_sha256' => str_pad((string) $id, 64, 'c'),
|
||||
'name' => 'backend-' . $id, 'driver' => 's3',
|
||||
'endpoint' => 'https://storage-' . $id . '.example.test',
|
||||
'region' => 'us-east-1', 'bucket' => 'bucket-' . $id,
|
||||
'prefix' => $prefix, 'path_style' => 1,
|
||||
'failure_domain' => $domain, 'sealed_credentials' => 'sealed',
|
||||
'config_revision' => 1, 'tested_revision' => $tested,
|
||||
'admin_state' => $state, 'health_state' => $health,
|
||||
'write_priority' => $priority, 'consecutive_failures' => 0,
|
||||
'consecutive_successes' => $health === 'healthy' ? 1 : 0,
|
||||
'last_error_code' => '', 'last_tested_at' => $tested > 0 ? $now : 0,
|
||||
'last_healthy_at' => $health === 'healthy' ? $now : 0,
|
||||
'next_probe_at' => $now + 300,
|
||||
'created_at' => $now, 'updated_at' => $now,
|
||||
]);
|
||||
};
|
||||
$insertBackend(1, 'active', 'healthy', 1, 'provider-a', 10, 'primary');
|
||||
$insertBackend(2, 'draft', 'unknown', 0, 'provider-b', 20, 'secondary');
|
||||
|
||||
$view = CloudStorageArtifactService::beginPublication(
|
||||
'backup', 'backups/late/system.qcow2', 4096, str_repeat('a', 64),
|
||||
'repair-fixture-primary', hash('sha256', 'repair-fixture'),
|
||||
['backup_id' => 101]
|
||||
);
|
||||
$assert(CloudStorageReplica::where('artifact_id', $view['artifact_id'])->count() === 1,
|
||||
'Unavailable second backend was unexpectedly reserved at publication time.');
|
||||
CloudStorageArtifactService::markPrimaryDispatching($view['operation_id']);
|
||||
CloudStorageArtifactService::completePrimary(
|
||||
$view['operation_id'], (int) $view['artifact_id'], [
|
||||
'size_bytes' => 4096, 'sha256' => str_repeat('a', 64),
|
||||
'etag' => 'primary-etag', 'version_id' => 'primary-version',
|
||||
]
|
||||
);
|
||||
CloudStoragePublicationFinalizerService::complete(
|
||||
'backup', 101, (int) $view['artifact_id'], $view['operation_id']
|
||||
);
|
||||
Capsule::table('cloud_storage_backends')->where('id', 2)->update([
|
||||
'admin_state' => 'active', 'health_state' => 'healthy',
|
||||
'tested_revision' => 1, 'last_tested_at' => $now,
|
||||
'last_healthy_at' => $now, 'updated_at' => $now,
|
||||
]);
|
||||
|
||||
$repair = new CloudStorageReplicaRepairService();
|
||||
$assert($repair->tick($now + 1),
|
||||
'Recovered second backend did not reserve a missing replica.');
|
||||
$replicas = CloudStorageReplica::where('artifact_id', $view['artifact_id'])
|
||||
->orderBy('id')->get();
|
||||
$copy = CloudStorageOperation::where('artifact_id', $view['artifact_id'])
|
||||
->where('kind', 'copy')->firstOrFail();
|
||||
$assert($replicas->count() === 2
|
||||
&& (int) $replicas->last()->backend_id === 2
|
||||
&& (string) $replicas->last()->object_key
|
||||
=== 'secondary/backups/late/system.qcow2'
|
||||
&& (int) $copy->source_backend_id === 1
|
||||
&& (int) $copy->target_backend_id === 2
|
||||
&& (string) $copy->phase === 'accepted'
|
||||
&& (int) $copy->available_at === $now + 1,
|
||||
'Missing-replica repair did not freeze the expected copy intent.');
|
||||
$assert(!$repair->tick($now + 2)
|
||||
&& CloudStorageReplica::where('artifact_id', $view['artifact_id'])->count() === 2
|
||||
&& CloudStorageOperation::where('artifact_id', $view['artifact_id'])
|
||||
->where('kind', 'copy')->count() === 1,
|
||||
'Repeated repair allocated duplicate replica work.');
|
||||
CloudStorageArtifact::whereKey($view['artifact_id'])->update([
|
||||
'desired_copies' => 1,
|
||||
]);
|
||||
|
||||
// A failed old replica must not occupy durability coverage. For a required-two
|
||||
// artifact, replacement also must use a failure domain distinct from the only
|
||||
// exact ready source.
|
||||
$insertBackend(3, 'active', 'healthy', 1, 'provider-a', 1, 'same-domain');
|
||||
$insertBackend(4, 'active', 'healthy', 1, 'provider-c', 50, 'replacement');
|
||||
$artifactId = Capsule::table('cloud_storage_artifacts')->insertGetId([
|
||||
'artifact_key' => str_repeat('d', 32), 'kind' => 'template',
|
||||
'maximum_size_bytes' => 8192, 'size_bytes' => 8192,
|
||||
'sha256' => str_repeat('e', 64),
|
||||
'durability_policy' => 'mirror_required_2',
|
||||
'desired_copies' => 2, 'required_copies' => 2,
|
||||
'state' => 'protected', 'last_operation_id' => str_repeat('1', 32),
|
||||
'verified_at' => $now, 'created_at' => $now, 'updated_at' => $now,
|
||||
]);
|
||||
$primaryReplicaId = Capsule::table('cloud_storage_replicas')->insertGetId([
|
||||
'replica_key' => str_repeat('4', 32), 'artifact_id' => $artifactId,
|
||||
'backend_id' => 1, 'role' => 'primary',
|
||||
'object_key' => 'primary/templates/image.qcow2',
|
||||
'object_version_id' => 'v1', 'object_etag' => 'e1',
|
||||
'size_bytes' => 8192, 'sha256' => str_repeat('e', 64),
|
||||
'state' => 'ready', 'publish_operation_id' => str_repeat('1', 32),
|
||||
'lease_until' => 0, 'error_code' => '', 'verified_at' => $now,
|
||||
'created_at' => $now, 'updated_at' => $now,
|
||||
]);
|
||||
Capsule::table('cloud_storage_replicas')->insert([
|
||||
'replica_key' => str_repeat('5', 32), 'artifact_id' => $artifactId,
|
||||
'backend_id' => 2, 'role' => 'secondary',
|
||||
'object_key' => 'secondary/templates/image.qcow2',
|
||||
'object_version_id' => 'v2', 'object_etag' => 'e2',
|
||||
'size_bytes' => 8192, 'sha256' => str_repeat('e', 64),
|
||||
'state' => 'ready', 'publish_operation_id' => str_repeat('2', 32),
|
||||
'lease_until' => 0, 'error_code' => '', 'verified_at' => $now,
|
||||
'created_at' => $now, 'updated_at' => $now,
|
||||
]);
|
||||
Capsule::table('cloud_storage_operations')->insert([
|
||||
'operation_id' => str_repeat('1', 32),
|
||||
'request_idempotency_key' => 'template-primary-fixture',
|
||||
'request_fingerprint' => str_repeat('6', 64),
|
||||
'artifact_id' => $artifactId, 'replica_id' => $primaryReplicaId,
|
||||
'target_backend_id' => 1, 'kind' => 'publish', 'phase' => 'terminal',
|
||||
'outcome' => 'succeeded', 'effect_resolution' => 'applied',
|
||||
'effect_may_have_applied' => 0, 'safe_to_resubmit' => 0,
|
||||
'attempts' => 1, 'lease_until' => 0, 'available_at' => 0,
|
||||
'next_check_at' => 0, 'error_code' => '',
|
||||
'terminal_at' => $now, 'created_at' => $now, 'updated_at' => $now,
|
||||
]);
|
||||
Capsule::table('cloud_storage_operations')->insert([
|
||||
'operation_id' => str_repeat('2', 32),
|
||||
'request_idempotency_key' => 'template-old-copy-fixture',
|
||||
'request_fingerprint' => str_repeat('7', 64),
|
||||
'artifact_id' => $artifactId,
|
||||
'replica_id' => CloudStorageReplica::where('artifact_id', $artifactId)
|
||||
->where('backend_id', 2)->value('id'),
|
||||
'source_backend_id' => 1, 'target_backend_id' => 2,
|
||||
'kind' => 'copy', 'phase' => 'terminal', 'outcome' => 'failed',
|
||||
'effect_resolution' => 'not_applied', 'effect_may_have_applied' => 0,
|
||||
'safe_to_resubmit' => 0, 'attempts' => 1, 'lease_until' => 0,
|
||||
'available_at' => 0, 'next_check_at' => 0,
|
||||
'error_code' => 'provider_lost', 'terminal_at' => $now,
|
||||
'created_at' => $now, 'updated_at' => $now,
|
||||
]);
|
||||
Capsule::table('cloud_storage_publications')->insert([
|
||||
'publication_key' => str_repeat('8', 32), 'artifact_id' => $artifactId,
|
||||
'template_import_intent_id' => str_repeat('9', 32),
|
||||
'state' => 'completed', 'attempts' => 1, 'available_at' => 0,
|
||||
'lease_until' => 0, 'last_error_code' => '',
|
||||
'coverage_ready_at' => $now, 'finalized_at' => $now,
|
||||
'created_at' => $now, 'updated_at' => $now,
|
||||
]);
|
||||
Capsule::table('cloud_storage_backends')->where('id', 2)->update([
|
||||
'admin_state' => 'read_only', 'health_state' => 'unreachable',
|
||||
'tested_revision' => 0, 'last_error_code' => 'endpoint_unreachable',
|
||||
'updated_at' => $now,
|
||||
]);
|
||||
|
||||
$assert($repair->tick($now + 3),
|
||||
'Replacement backend did not repair unreachable required-two coverage.');
|
||||
$replacement = CloudStorageReplica::where('artifact_id', $artifactId)
|
||||
->where('state', 'reserved')->firstOrFail();
|
||||
$assert((int) $replacement->backend_id === 4
|
||||
&& (string) $replacement->object_key
|
||||
=== 'replacement/templates/image.qcow2'
|
||||
&& CloudStorageReplica::where('artifact_id', $artifactId)->count() === 3,
|
||||
'Repair counted degraded evidence or selected the source failure domain.');
|
||||
|
||||
$workerSource = file_get_contents(dirname(__DIR__)
|
||||
. '/app/Services/Cloud/CloudStorageReplicaWorkerService.php');
|
||||
$assert(is_string($workerSource)
|
||||
&& str_contains($workerSource, 'CloudStorageReplicaRepairService')
|
||||
&& str_contains($workerSource, 'storage_replica_repair_tick_failed'),
|
||||
'Replica worker does not consume the bounded repair queue.');
|
||||
|
||||
echo "Storage replica repair behavior passed.\n";
|
||||
Loading…
Reference in New Issue