You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
virt/web/app/Services/Cloud/CloudStorageReplicaRepairSe...

414 lines
16 KiB
PHP

<?php
namespace App\Services\Cloud;
use App\Models\CloudStorageArtifact;
use App\Models\CloudStorageBackend;
use App\Models\CloudStorageOperation;
use App\Models\CloudStoragePublication;
use App\Models\CloudStorageReplica;
/**
* Reserves one missing immutable replica without contacting a provider.
*
* A later replica worker executes the frozen copy intent. Existing degraded
* or deleted replica rows do not count as durability coverage, which allows a
* replacement backend to repair an artifact without rewriting old evidence.
*/
final class CloudStorageReplicaRepairService
{
private const REPAIRABLE_PUBLICATION_STATES = [
'waiting_primary',
'waiting_protection',
'ready_to_finalize',
'finalizing',
'completed',
];
/** Reserve at most one repair so each worker cycle remains bounded. */
public function tick(int $now): bool
{
if ($now < 1) {
throw new \InvalidArgumentException('Invalid repair clock');
}
$candidates = CloudStorageArtifact::query()
->whereNotIn('state', [
'retiring', 'delete_attention', 'deleted',
])
->whereIn('id', static function ($query): void {
$query->select('artifact_id')
->from('cloud_storage_publications')
->whereIn(
'state',
self::REPAIRABLE_PUBLICATION_STATES
);
})
->whereRaw(
"(SELECT COUNT(DISTINCT CASE "
. "WHEN cloud_storage_artifacts.required_copies > 1 "
. "THEN LOWER(TRIM(repair_backend.failure_domain)) "
. "ELSE repair_coverage.id END) "
. "FROM cloud_storage_replicas repair_coverage "
. "INNER JOIN cloud_storage_backends repair_backend "
. "ON repair_backend.id = repair_coverage.backend_id "
. "WHERE repair_coverage.artifact_id = cloud_storage_artifacts.id "
. "AND ((repair_coverage.state = 'ready' "
. "AND repair_coverage.size_bytes = cloud_storage_artifacts.size_bytes "
. "AND repair_coverage.sha256 = cloud_storage_artifacts.sha256 "
. "AND repair_coverage.verified_at > 0 "
. "AND repair_backend.admin_state IN ('active','read_only') "
. "AND repair_backend.health_state IN ('healthy','degraded','unknown')) "
. "OR (repair_coverage.state IN ('reserved','publishing','copying') "
. "AND repair_backend.admin_state = 'active' "
. "AND repair_backend.health_state = 'healthy' "
. "AND repair_backend.tested_revision = repair_backend.config_revision))) "
. '< cloud_storage_artifacts.desired_copies'
)
->orderBy('id');
// Page through every SQL-level coverage gap. A hard first-page limit
// would let an unrecoverable early artifact starve all later repairs.
foreach ($candidates->lazyById(100, 'id') as $artifact) {
if ($this->reserveForArtifact((int) $artifact->id, $now)) {
return true;
}
}
return false;
}
private function reserveForArtifact(int $artifactId, int $now): bool
{
$connection = (new CloudStorageArtifact())->getConnection();
return $connection->transaction(static function () use (
$artifactId,
$now
): bool {
// Match primary settlement's lock order.
$primaryOperation = CloudStorageOperation::query()
->where('artifact_id', $artifactId)
->where('kind', 'publish')->orderBy('id')
->lockForUpdate()->first();
if (!$primaryOperation instanceof CloudStorageOperation
|| (string) $primaryOperation->phase !== 'terminal'
|| (string) $primaryOperation->outcome !== 'succeeded') {
return false;
}
CloudStorageOperation::query()
->where('artifact_id', $artifactId)
->where('kind', 'copy')->orderBy('id')
->lockForUpdate()->get();
$artifact = CloudStorageArtifact::query()->whereKey($artifactId)
->lockForUpdate()->first();
if (!$artifact instanceof CloudStorageArtifact
|| in_array((string) $artifact->state,
['retiring', 'delete_attention', 'deleted'], true)) {
return false;
}
$replicas = CloudStorageReplica::query()
->where('artifact_id', $artifactId)->orderBy('id')
->lockForUpdate()->get()->all();
$publication = CloudStoragePublication::query()
->where('artifact_id', $artifactId)
->lockForUpdate()->first();
if (!$publication instanceof CloudStoragePublication
|| !in_array(
(string) $publication->state,
self::REPAIRABLE_PUBLICATION_STATES,
true
)) {
return false;
}
$backends = CloudStorageBackend::query()->orderBy('id')
->lockForUpdate()->get()->keyBy('id');
$ready = [];
$required = (int) $artifact->required_copies;
$coveredReady = 0;
$pending = 0;
$coveredReadyDomains = [];
$pendingDomains = [];
$represented = [];
foreach ($replicas as $replica) {
$backendId = (int) $replica->backend_id;
$represented[$backendId] = true;
if (self::isExactReady($artifact, $replica)) {
$backend = $backends->get($backendId);
if ($backend instanceof CloudStorageBackend
&& (string) $backend->admin_state !== 'retired'
&& in_array((string) $backend->health_state,
['healthy', 'degraded', 'unknown'], true)) {
$ready[] = [$replica, $backend];
if (in_array((string) $backend->admin_state,
['active', 'read_only'], true)) {
if ($required > 1) {
$coveredReadyDomains[
self::domain($backend)
] = true;
} else {
$coveredReady++;
}
}
}
} elseif (in_array((string) $replica->state,
['reserved', 'publishing', 'copying'], true)) {
$backend = $backends->get($backendId);
if ($backend instanceof CloudStorageBackend
&& (string) $backend->admin_state === 'active'
&& (string) $backend->health_state === 'healthy'
&& (int) $backend->config_revision > 0
&& (int) $backend->tested_revision
=== (int) $backend->config_revision) {
if ($required > 1) {
$pendingDomains[self::domain($backend)] = true;
} else {
$pending++;
}
}
}
}
if ($required > 1) {
$coveredReady = count($coveredReadyDomains);
$pending = count(array_diff_key(
$pendingDomains,
$coveredReadyDomains
));
}
$desired = (int) $artifact->desired_copies;
if ($desired < 1 || $desired > 2
|| $coveredReady >= $desired
|| $coveredReady + $pending >= $desired
|| $ready === []) {
return false;
}
usort($ready, static function (array $left, array $right): int {
$leftBackend = $left[1];
$rightBackend = $right[1];
return [
self::healthRank((string) $leftBackend->health_state),
self::adminRank((string) $leftBackend->admin_state),
(int) $leftBackend->write_priority,
(int) $leftBackend->id,
] <=> [
self::healthRank((string) $rightBackend->health_state),
self::adminRank((string) $rightBackend->admin_state),
(int) $rightBackend->write_priority,
(int) $rightBackend->id,
];
});
$source = $ready[0][0];
$sourceBackend = $ready[0][1];
$suffix = self::objectSuffixFromSource(
(string) $sourceBackend->prefix,
(string) $source->object_key
);
if ($suffix === null) {
return false;
}
$readyDomains = [];
foreach ($ready as [, $backend]) {
$readyDomains[self::domain($backend)] = true;
}
$candidates = [];
foreach ($backends as $backend) {
if (!$backend instanceof CloudStorageBackend
|| isset($represented[(int) $backend->id])
|| (string) $backend->admin_state !== 'active'
|| (string) $backend->health_state !== 'healthy'
|| (int) $backend->config_revision < 1
|| (int) $backend->tested_revision
!== (int) $backend->config_revision) {
continue;
}
$isNewDomain = !isset($readyDomains[self::domain($backend)]);
if ((int) $artifact->required_copies > 1 && !$isNewDomain) {
continue;
}
$candidates[] = [$backend, $isNewDomain ? 0 : 1];
}
usort($candidates, static fn (array $left, array $right): int => [
$left[1],
(int) $left[0]->write_priority,
(int) $left[0]->id,
] <=> [
$right[1],
(int) $right[0]->write_priority,
(int) $right[0]->id,
]);
if ($candidates === []) {
return false;
}
$target = $candidates[0][0];
$targetKey = self::objectKey((string) $target->prefix, $suffix);
$collision = CloudStorageReplica::query()
->where('backend_id', (int) $target->id)
->where('object_key', $targetKey)
->lockForUpdate()->first();
if ($collision instanceof CloudStorageReplica) {
return false;
}
self::createRepair(
$artifact,
$source,
$sourceBackend,
$target,
$targetKey,
$now
);
return true;
});
}
private static function createRepair(
CloudStorageArtifact $artifact,
CloudStorageReplica $source,
CloudStorageBackend $sourceBackend,
CloudStorageBackend $target,
string $targetKey,
int $now
): void {
$operationId = bin2hex(random_bytes(16));
$replica = new CloudStorageReplica();
$replica->replica_key = bin2hex(random_bytes(16));
$replica->artifact_id = (int) $artifact->id;
$replica->backend_id = (int) $target->id;
$replica->role = 'secondary';
$replica->object_key = $targetKey;
$replica->object_version_id = '';
$replica->object_etag = '';
$replica->size_bytes = 0;
$replica->sha256 = '';
$replica->state = 'reserved';
$replica->publish_operation_id = $operationId;
$replica->delete_operation_id = null;
$replica->delete_generation = 0;
$replica->delete_cycle_attempts = 0;
$replica->delete_presence_confirmations = 0;
$replica->delete_last_checked_at = 0;
$replica->delete_next_check_at = 0;
$replica->delete_attention_at = null;
$replica->lease_token = null;
$replica->lease_until = 0;
$replica->error_code = '';
$replica->verified_at = 0;
$replica->created_at = $now;
$replica->updated_at = $now;
$replica->save();
$operation = new CloudStorageOperation();
$operation->operation_id = $operationId;
$operation->request_idempotency_key =
'storage-repair-copy-' . substr(hash('sha256',
(string) $artifact->artifact_key . '|'
. (string) $target->id), 0, 48);
$operation->request_fingerprint = hash('sha256', implode('|', [
'repair-copy-v1',
(string) $artifact->artifact_key,
(string) $sourceBackend->id,
(string) $source->object_key,
(string) $source->object_version_id,
(string) $target->id,
$targetKey,
(string) $artifact->size_bytes,
(string) $artifact->sha256,
]));
$operation->artifact_id = (int) $artifact->id;
$operation->replica_id = (int) $replica->id;
$operation->source_backend_id = (int) $sourceBackend->id;
$operation->target_backend_id = (int) $target->id;
$operation->kind = 'copy';
$operation->phase = 'accepted';
$operation->outcome = null;
$operation->effect_resolution = 'not_applicable';
$operation->effect_may_have_applied = null;
$operation->safe_to_resubmit = null;
$operation->attempts = 0;
$operation->lease_token = null;
$operation->lease_until = 0;
$operation->available_at = $now;
$operation->next_check_at = 0;
$operation->error_code = '';
$operation->result_json = null;
$operation->terminal_at = null;
$operation->created_at = $now;
$operation->updated_at = $now;
$operation->save();
}
private static function isExactReady(
CloudStorageArtifact $artifact,
CloudStorageReplica $replica
): bool {
return (string) $replica->state === 'ready'
&& (int) $replica->size_bytes === (int) $artifact->size_bytes
&& hash_equals((string) $artifact->sha256,
(string) $replica->sha256)
&& (int) $replica->verified_at > 0;
}
private static function objectSuffixFromSource(
string $prefix,
string $objectKey
): ?string {
$prefix = trim($prefix, '/');
if ($prefix !== '') {
$start = $prefix . '/';
if (!str_starts_with($objectKey, $start)) {
return null;
}
$objectKey = substr($objectKey, strlen($start));
}
if ($objectKey === '' || strlen($objectKey) > 512
|| preg_match('/^[A-Za-z0-9][A-Za-z0-9._\/-]*$/D',
$objectKey) !== 1
|| str_contains($objectKey, '//')) {
return null;
}
foreach (explode('/', $objectKey) as $segment) {
if ($segment === '.' || $segment === '..') {
return null;
}
}
return $objectKey;
}
private static function objectKey(string $prefix, string $suffix): string
{
$prefix = trim($prefix, '/');
$key = ($prefix === '' ? '' : $prefix . '/') . $suffix;
if (strlen($key) > 512) {
throw new CloudError('对象存储制品路径过长', 422);
}
return $key;
}
private static function domain(CloudStorageBackend $backend): string
{
// Never manufacture independence from a backend id. Empty metadata is
// one shared unknown domain and cannot satisfy required-two coverage.
return strtolower(trim((string) $backend->failure_domain));
}
private static function healthRank(string $state): int
{
return match ($state) {
'healthy' => 0,
'degraded' => 1,
'unknown' => 2,
default => 3,
};
}
private static function adminRank(string $state): int
{
return match ($state) {
'active' => 0,
'read_only' => 1,
'draining' => 2,
default => 3,
};
}
}