You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
414 lines
16 KiB
PHP
414 lines
16 KiB
PHP
<?php
|
|
|
|
namespace App\Services\Cloud;
|
|
|
|
use App\Models\CloudStorageArtifact;
|
|
use App\Models\CloudStorageBackend;
|
|
use App\Models\CloudStorageOperation;
|
|
use App\Models\CloudStoragePublication;
|
|
use App\Models\CloudStorageReplica;
|
|
|
|
/**
|
|
* Reserves one missing immutable replica without contacting a provider.
|
|
*
|
|
* A later replica worker executes the frozen copy intent. Existing degraded
|
|
* or deleted replica rows do not count as durability coverage, which allows a
|
|
* replacement backend to repair an artifact without rewriting old evidence.
|
|
*/
|
|
final class CloudStorageReplicaRepairService
|
|
{
|
|
private const REPAIRABLE_PUBLICATION_STATES = [
|
|
'waiting_primary',
|
|
'waiting_protection',
|
|
'ready_to_finalize',
|
|
'finalizing',
|
|
'completed',
|
|
];
|
|
|
|
/** Reserve at most one repair so each worker cycle remains bounded. */
|
|
public function tick(int $now): bool
|
|
{
|
|
if ($now < 1) {
|
|
throw new \InvalidArgumentException('Invalid repair clock');
|
|
}
|
|
$candidates = CloudStorageArtifact::query()
|
|
->whereNotIn('state', [
|
|
'retiring', 'delete_attention', 'deleted',
|
|
])
|
|
->whereIn('id', static function ($query): void {
|
|
$query->select('artifact_id')
|
|
->from('cloud_storage_publications')
|
|
->whereIn(
|
|
'state',
|
|
self::REPAIRABLE_PUBLICATION_STATES
|
|
);
|
|
})
|
|
->whereRaw(
|
|
"(SELECT COUNT(DISTINCT CASE "
|
|
. "WHEN cloud_storage_artifacts.required_copies > 1 "
|
|
. "THEN LOWER(TRIM(repair_backend.failure_domain)) "
|
|
. "ELSE repair_coverage.id END) "
|
|
. "FROM cloud_storage_replicas repair_coverage "
|
|
. "INNER JOIN cloud_storage_backends repair_backend "
|
|
. "ON repair_backend.id = repair_coverage.backend_id "
|
|
. "WHERE repair_coverage.artifact_id = cloud_storage_artifacts.id "
|
|
. "AND ((repair_coverage.state = 'ready' "
|
|
. "AND repair_coverage.size_bytes = cloud_storage_artifacts.size_bytes "
|
|
. "AND repair_coverage.sha256 = cloud_storage_artifacts.sha256 "
|
|
. "AND repair_coverage.verified_at > 0 "
|
|
. "AND repair_backend.admin_state IN ('active','read_only') "
|
|
. "AND repair_backend.health_state IN ('healthy','degraded','unknown')) "
|
|
. "OR (repair_coverage.state IN ('reserved','publishing','copying') "
|
|
. "AND repair_backend.admin_state = 'active' "
|
|
. "AND repair_backend.health_state = 'healthy' "
|
|
. "AND repair_backend.tested_revision = repair_backend.config_revision))) "
|
|
. '< cloud_storage_artifacts.desired_copies'
|
|
)
|
|
->orderBy('id');
|
|
|
|
// Page through every SQL-level coverage gap. A hard first-page limit
|
|
// would let an unrecoverable early artifact starve all later repairs.
|
|
foreach ($candidates->lazyById(100, 'id') as $artifact) {
|
|
if ($this->reserveForArtifact((int) $artifact->id, $now)) {
|
|
return true;
|
|
}
|
|
}
|
|
return false;
|
|
}
|
|
|
|
private function reserveForArtifact(int $artifactId, int $now): bool
|
|
{
|
|
$connection = (new CloudStorageArtifact())->getConnection();
|
|
return $connection->transaction(static function () use (
|
|
$artifactId,
|
|
$now
|
|
): bool {
|
|
// Match primary settlement's lock order.
|
|
$primaryOperation = CloudStorageOperation::query()
|
|
->where('artifact_id', $artifactId)
|
|
->where('kind', 'publish')->orderBy('id')
|
|
->lockForUpdate()->first();
|
|
if (!$primaryOperation instanceof CloudStorageOperation
|
|
|| (string) $primaryOperation->phase !== 'terminal'
|
|
|| (string) $primaryOperation->outcome !== 'succeeded') {
|
|
return false;
|
|
}
|
|
CloudStorageOperation::query()
|
|
->where('artifact_id', $artifactId)
|
|
->where('kind', 'copy')->orderBy('id')
|
|
->lockForUpdate()->get();
|
|
$artifact = CloudStorageArtifact::query()->whereKey($artifactId)
|
|
->lockForUpdate()->first();
|
|
if (!$artifact instanceof CloudStorageArtifact
|
|
|| in_array((string) $artifact->state,
|
|
['retiring', 'delete_attention', 'deleted'], true)) {
|
|
return false;
|
|
}
|
|
$replicas = CloudStorageReplica::query()
|
|
->where('artifact_id', $artifactId)->orderBy('id')
|
|
->lockForUpdate()->get()->all();
|
|
$publication = CloudStoragePublication::query()
|
|
->where('artifact_id', $artifactId)
|
|
->lockForUpdate()->first();
|
|
if (!$publication instanceof CloudStoragePublication
|
|
|| !in_array(
|
|
(string) $publication->state,
|
|
self::REPAIRABLE_PUBLICATION_STATES,
|
|
true
|
|
)) {
|
|
return false;
|
|
}
|
|
$backends = CloudStorageBackend::query()->orderBy('id')
|
|
->lockForUpdate()->get()->keyBy('id');
|
|
|
|
$ready = [];
|
|
$required = (int) $artifact->required_copies;
|
|
$coveredReady = 0;
|
|
$pending = 0;
|
|
$coveredReadyDomains = [];
|
|
$pendingDomains = [];
|
|
$represented = [];
|
|
foreach ($replicas as $replica) {
|
|
$backendId = (int) $replica->backend_id;
|
|
$represented[$backendId] = true;
|
|
if (self::isExactReady($artifact, $replica)) {
|
|
$backend = $backends->get($backendId);
|
|
if ($backend instanceof CloudStorageBackend
|
|
&& (string) $backend->admin_state !== 'retired'
|
|
&& in_array((string) $backend->health_state,
|
|
['healthy', 'degraded', 'unknown'], true)) {
|
|
$ready[] = [$replica, $backend];
|
|
if (in_array((string) $backend->admin_state,
|
|
['active', 'read_only'], true)) {
|
|
if ($required > 1) {
|
|
$coveredReadyDomains[
|
|
self::domain($backend)
|
|
] = true;
|
|
} else {
|
|
$coveredReady++;
|
|
}
|
|
}
|
|
}
|
|
} elseif (in_array((string) $replica->state,
|
|
['reserved', 'publishing', 'copying'], true)) {
|
|
$backend = $backends->get($backendId);
|
|
if ($backend instanceof CloudStorageBackend
|
|
&& (string) $backend->admin_state === 'active'
|
|
&& (string) $backend->health_state === 'healthy'
|
|
&& (int) $backend->config_revision > 0
|
|
&& (int) $backend->tested_revision
|
|
=== (int) $backend->config_revision) {
|
|
if ($required > 1) {
|
|
$pendingDomains[self::domain($backend)] = true;
|
|
} else {
|
|
$pending++;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
if ($required > 1) {
|
|
$coveredReady = count($coveredReadyDomains);
|
|
$pending = count(array_diff_key(
|
|
$pendingDomains,
|
|
$coveredReadyDomains
|
|
));
|
|
}
|
|
$desired = (int) $artifact->desired_copies;
|
|
if ($desired < 1 || $desired > 2
|
|
|| $coveredReady >= $desired
|
|
|| $coveredReady + $pending >= $desired
|
|
|| $ready === []) {
|
|
return false;
|
|
}
|
|
|
|
usort($ready, static function (array $left, array $right): int {
|
|
$leftBackend = $left[1];
|
|
$rightBackend = $right[1];
|
|
return [
|
|
self::healthRank((string) $leftBackend->health_state),
|
|
self::adminRank((string) $leftBackend->admin_state),
|
|
(int) $leftBackend->write_priority,
|
|
(int) $leftBackend->id,
|
|
] <=> [
|
|
self::healthRank((string) $rightBackend->health_state),
|
|
self::adminRank((string) $rightBackend->admin_state),
|
|
(int) $rightBackend->write_priority,
|
|
(int) $rightBackend->id,
|
|
];
|
|
});
|
|
$source = $ready[0][0];
|
|
$sourceBackend = $ready[0][1];
|
|
$suffix = self::objectSuffixFromSource(
|
|
(string) $sourceBackend->prefix,
|
|
(string) $source->object_key
|
|
);
|
|
if ($suffix === null) {
|
|
return false;
|
|
}
|
|
|
|
$readyDomains = [];
|
|
foreach ($ready as [, $backend]) {
|
|
$readyDomains[self::domain($backend)] = true;
|
|
}
|
|
$candidates = [];
|
|
foreach ($backends as $backend) {
|
|
if (!$backend instanceof CloudStorageBackend
|
|
|| isset($represented[(int) $backend->id])
|
|
|| (string) $backend->admin_state !== 'active'
|
|
|| (string) $backend->health_state !== 'healthy'
|
|
|| (int) $backend->config_revision < 1
|
|
|| (int) $backend->tested_revision
|
|
!== (int) $backend->config_revision) {
|
|
continue;
|
|
}
|
|
$isNewDomain = !isset($readyDomains[self::domain($backend)]);
|
|
if ((int) $artifact->required_copies > 1 && !$isNewDomain) {
|
|
continue;
|
|
}
|
|
$candidates[] = [$backend, $isNewDomain ? 0 : 1];
|
|
}
|
|
usort($candidates, static fn (array $left, array $right): int => [
|
|
$left[1],
|
|
(int) $left[0]->write_priority,
|
|
(int) $left[0]->id,
|
|
] <=> [
|
|
$right[1],
|
|
(int) $right[0]->write_priority,
|
|
(int) $right[0]->id,
|
|
]);
|
|
if ($candidates === []) {
|
|
return false;
|
|
}
|
|
$target = $candidates[0][0];
|
|
$targetKey = self::objectKey((string) $target->prefix, $suffix);
|
|
$collision = CloudStorageReplica::query()
|
|
->where('backend_id', (int) $target->id)
|
|
->where('object_key', $targetKey)
|
|
->lockForUpdate()->first();
|
|
if ($collision instanceof CloudStorageReplica) {
|
|
return false;
|
|
}
|
|
|
|
self::createRepair(
|
|
$artifact,
|
|
$source,
|
|
$sourceBackend,
|
|
$target,
|
|
$targetKey,
|
|
$now
|
|
);
|
|
return true;
|
|
});
|
|
}
|
|
|
|
private static function createRepair(
|
|
CloudStorageArtifact $artifact,
|
|
CloudStorageReplica $source,
|
|
CloudStorageBackend $sourceBackend,
|
|
CloudStorageBackend $target,
|
|
string $targetKey,
|
|
int $now
|
|
): void {
|
|
$operationId = bin2hex(random_bytes(16));
|
|
$replica = new CloudStorageReplica();
|
|
$replica->replica_key = bin2hex(random_bytes(16));
|
|
$replica->artifact_id = (int) $artifact->id;
|
|
$replica->backend_id = (int) $target->id;
|
|
$replica->role = 'secondary';
|
|
$replica->object_key = $targetKey;
|
|
$replica->object_version_id = '';
|
|
$replica->object_etag = '';
|
|
$replica->size_bytes = 0;
|
|
$replica->sha256 = '';
|
|
$replica->state = 'reserved';
|
|
$replica->publish_operation_id = $operationId;
|
|
$replica->delete_operation_id = null;
|
|
$replica->delete_generation = 0;
|
|
$replica->delete_cycle_attempts = 0;
|
|
$replica->delete_presence_confirmations = 0;
|
|
$replica->delete_last_checked_at = 0;
|
|
$replica->delete_next_check_at = 0;
|
|
$replica->delete_attention_at = null;
|
|
$replica->lease_token = null;
|
|
$replica->lease_until = 0;
|
|
$replica->error_code = '';
|
|
$replica->verified_at = 0;
|
|
$replica->created_at = $now;
|
|
$replica->updated_at = $now;
|
|
$replica->save();
|
|
|
|
$operation = new CloudStorageOperation();
|
|
$operation->operation_id = $operationId;
|
|
$operation->request_idempotency_key =
|
|
'storage-repair-copy-' . substr(hash('sha256',
|
|
(string) $artifact->artifact_key . '|'
|
|
. (string) $target->id), 0, 48);
|
|
$operation->request_fingerprint = hash('sha256', implode('|', [
|
|
'repair-copy-v1',
|
|
(string) $artifact->artifact_key,
|
|
(string) $sourceBackend->id,
|
|
(string) $source->object_key,
|
|
(string) $source->object_version_id,
|
|
(string) $target->id,
|
|
$targetKey,
|
|
(string) $artifact->size_bytes,
|
|
(string) $artifact->sha256,
|
|
]));
|
|
$operation->artifact_id = (int) $artifact->id;
|
|
$operation->replica_id = (int) $replica->id;
|
|
$operation->source_backend_id = (int) $sourceBackend->id;
|
|
$operation->target_backend_id = (int) $target->id;
|
|
$operation->kind = 'copy';
|
|
$operation->phase = 'accepted';
|
|
$operation->outcome = null;
|
|
$operation->effect_resolution = 'not_applicable';
|
|
$operation->effect_may_have_applied = null;
|
|
$operation->safe_to_resubmit = null;
|
|
$operation->attempts = 0;
|
|
$operation->lease_token = null;
|
|
$operation->lease_until = 0;
|
|
$operation->available_at = $now;
|
|
$operation->next_check_at = 0;
|
|
$operation->error_code = '';
|
|
$operation->result_json = null;
|
|
$operation->terminal_at = null;
|
|
$operation->created_at = $now;
|
|
$operation->updated_at = $now;
|
|
$operation->save();
|
|
}
|
|
|
|
private static function isExactReady(
|
|
CloudStorageArtifact $artifact,
|
|
CloudStorageReplica $replica
|
|
): bool {
|
|
return (string) $replica->state === 'ready'
|
|
&& (int) $replica->size_bytes === (int) $artifact->size_bytes
|
|
&& hash_equals((string) $artifact->sha256,
|
|
(string) $replica->sha256)
|
|
&& (int) $replica->verified_at > 0;
|
|
}
|
|
|
|
private static function objectSuffixFromSource(
|
|
string $prefix,
|
|
string $objectKey
|
|
): ?string {
|
|
$prefix = trim($prefix, '/');
|
|
if ($prefix !== '') {
|
|
$start = $prefix . '/';
|
|
if (!str_starts_with($objectKey, $start)) {
|
|
return null;
|
|
}
|
|
$objectKey = substr($objectKey, strlen($start));
|
|
}
|
|
if ($objectKey === '' || strlen($objectKey) > 512
|
|
|| preg_match('/^[A-Za-z0-9][A-Za-z0-9._\/-]*$/D',
|
|
$objectKey) !== 1
|
|
|| str_contains($objectKey, '//')) {
|
|
return null;
|
|
}
|
|
foreach (explode('/', $objectKey) as $segment) {
|
|
if ($segment === '.' || $segment === '..') {
|
|
return null;
|
|
}
|
|
}
|
|
return $objectKey;
|
|
}
|
|
|
|
private static function objectKey(string $prefix, string $suffix): string
|
|
{
|
|
$prefix = trim($prefix, '/');
|
|
$key = ($prefix === '' ? '' : $prefix . '/') . $suffix;
|
|
if (strlen($key) > 512) {
|
|
throw new CloudError('对象存储制品路径过长', 422);
|
|
}
|
|
return $key;
|
|
}
|
|
|
|
private static function domain(CloudStorageBackend $backend): string
|
|
{
|
|
// Never manufacture independence from a backend id. Empty metadata is
|
|
// one shared unknown domain and cannot satisfy required-two coverage.
|
|
return strtolower(trim((string) $backend->failure_domain));
|
|
}
|
|
|
|
private static function healthRank(string $state): int
|
|
{
|
|
return match ($state) {
|
|
'healthy' => 0,
|
|
'degraded' => 1,
|
|
'unknown' => 2,
|
|
default => 3,
|
|
};
|
|
}
|
|
|
|
private static function adminRank(string $state): int
|
|
{
|
|
return match ($state) {
|
|
'active' => 0,
|
|
'read_only' => 1,
|
|
'draining' => 2,
|
|
default => 3,
|
|
};
|
|
}
|
|
}
|