You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
virt/web/app/Services/Cloud/CloudStorageReplicaWorkerSe...

677 lines
23 KiB
PHP

<?php
namespace App\Services\Cloud;
use App\Models\CloudStorageBackend;
/**
* Streams a pinned immutable replica and reconciles ambiguous completion from
* read-only evidence. Capability URLs exist only inside this process.
*/
final class CloudStorageReplicaWorkerService
{
private const CAPABILITY_SECONDS = 21600;
private const RECONCILE_DELAY = 60;
/** @var \Closure(int,bool):array<string,mixed> */
private readonly \Closure $runtimeResolver;
/** @var \Closure():int */
private readonly \Closure $clock;
/** @var list<string> */
private array $errors = [];
/**
* @param null|callable(int,bool):array<string,mixed> $runtimeResolver
* @param null|callable():int $clock
*/
public function __construct(
private readonly ?CloudStorageReplicaHelper $helper = null,
private readonly ?CloudObjectStorageS3Client $client = null,
?callable $runtimeResolver = null,
?callable $clock = null,
private readonly ?CloudStorageReplicaRepairService
$replicaRepair = null
) {
$this->runtimeResolver = $runtimeResolver === null
? static fn (int $backendId, bool $healthy): array =>
CloudObjectStorageBackendRegistry::runtime($backendId, $healthy)
: \Closure::fromCallable($runtimeResolver);
$this->clock = $clock === null
? static fn (): int => time()
: \Closure::fromCallable($clock);
}
public function tick(): bool
{
$this->errors = [];
$now = ($this->clock)();
// Repair only reserves one local, durable copy intent. It performs no
// provider I/O, so a backend that became writable can start healing
// old single-copy artifacts without delaying existing remote work.
$repairWorked = false;
try {
$repair = $this->replicaRepair
?? new CloudStorageReplicaRepairService();
$repairWorked = $repair->tick($now);
} catch (\Throwable) {
$this->errors[] = 'storage_replica_repair_tick_failed';
}
$work = CloudStorageArtifactService::claimReplicaWork($now);
$replicaWorked = $work !== null;
if ($work !== null) {
try {
if (($work['mode'] ?? null) === 'dispatch') {
$this->dispatch($work, $now);
} else {
$this->reconcile($work, $now);
}
} catch (\Throwable) {
$this->errors[] = 'storage_replica_tick_failed';
try {
$this->markUnknown(
$work,
'storage_replica_tick_failed',
is_array($work['metadata'] ?? null)
? $work['metadata'] : []
);
} catch (\Throwable) {
// The durable lease remains the recovery fence. Never
// create a replacement operation after an uncertain
// settlement.
}
}
}
$this->errors = array_values(array_unique($this->errors));
return $replicaWorked || $repairWorked;
}
/** @return list<string> */
public function observationErrors(): array
{
return $this->errors;
}
/** @param array<string,mixed> $work */
private function dispatch(array $work, int $startedAt): void
{
if (!$this->validSource($work)) {
$this->defer($work, 'replica_source_not_ready', 60);
return;
}
$target = CloudStorageBackend::query()
->whereKey((int) $work['target_backend_id'])->first();
if (!$target instanceof CloudStorageBackend
|| (string) $target->admin_state !== 'active'
|| (string) $target->health_state !== 'healthy'
|| (int) $target->tested_revision !== (int) $target->config_revision) {
$this->defer($work, 'replica_target_not_writable', 300);
return;
}
try {
$sourceRuntime = ($this->runtimeResolver)(
(int) $work['source_backend_id'],
false
);
$targetRuntime = ($this->runtimeResolver)(
(int) $work['target_backend_id'],
true
);
} catch (\Throwable) {
$this->defer($work, 'replica_runtime_unavailable', 300);
return;
}
try {
$plan = $this->s3()->beginMultipartPlan(
$targetRuntime['config'],
$targetRuntime['credentials'],
(string) $work['target_object_key'],
(int) $work['size_bytes'],
self::CAPABILITY_SECONDS
);
} catch (CloudObjectStorageTransferError $error) {
if (!$error->effectMayHaveApplied) {
$this->rescheduleNotApplied($work, $error->publicCode);
return;
}
$this->markUnknown($work, $error->publicCode, []);
return;
} catch (\Throwable) {
$this->markUnknown($work, 'replica_multipart_start_unknown', []);
return;
}
$metadata = [
'upload_id' => (string) $plan['upload_id'],
'part_size_bytes' => (int) $plan['part_size_bytes'],
'part_count' => count($plan['part_urls']),
'maximum_size_bytes' => (int) $plan['maximum_size_bytes'],
'multipart_absence_confirmations' => 0,
'multipart_presence_confirmations' => 0,
];
CloudStorageArtifactService::markReplicaAwaitingReceipt(
(string) $work['operation_id'],
(string) $work['lease_token'],
$metadata,
($this->clock)()
);
$this->invokeTransfer(
$work,
$startedAt,
$sourceRuntime,
$targetRuntime,
$plan,
$metadata
);
}
/** @param array<string,mixed> $work */
private function reconcile(array $work, int $startedAt): void
{
$metadata = is_array($work['metadata'] ?? null)
? $work['metadata'] : [];
try {
$runtime = ($this->runtimeResolver)(
(int) $work['target_backend_id'],
false
);
$identity = $this->s3()->inspectObject(
$runtime['config'],
$runtime['credentials'],
(string) $work['target_object_key']
);
} catch (\Throwable) {
$this->markUnknown(
$work,
'replica_inspection_unavailable',
$metadata
);
return;
}
if ($identity === null) {
$this->reconcileAbsence(
$work,
$runtime,
$metadata,
$startedAt
);
return;
}
if ((int) $identity['size_bytes'] !== (int) $work['size_bytes']) {
$this->errors[] = 'replica_identity_mismatch';
CloudStorageArtifactService::failReplicaAppliedMismatch(
(string) $work['operation_id'],
(string) $work['lease_token'],
'replica_identity_mismatch',
($this->clock)()
);
return;
}
try {
$url = $this->s3()->presignedVersionedGet(
$runtime['config'],
$runtime['credentials'],
(string) $work['target_object_key'],
(string) $identity['version_id'],
self::CAPABILITY_SECONDS
);
$receipt = $this->mover()->invoke([
'protocol' => 'object-replica-verify-v1',
'operation_id' => (string) $work['operation_id'],
'deadline_at' => $startedAt + self::CAPABILITY_SECONDS,
'get_url' => $url,
'size_bytes' => (int) $work['size_bytes'],
'sha256' => (string) $work['sha256'],
]);
} catch (\Throwable) {
$this->markUnknown(
$work,
'replica_verify_unavailable',
$metadata
);
return;
}
$this->settleReceipt($work, $receipt, $metadata, $identity);
}
/**
* @param array<string,mixed> $work
* @param array<string,mixed> $runtime
* @param array<string,mixed> $metadata
*/
private function reconcileAbsence(
array $work,
array $runtime,
array $metadata,
int $startedAt
): void {
try {
$matches = $this->s3()->findMultipartUploadsByExactObjectKey(
$runtime['config'],
$runtime['credentials'],
(string) $work['target_object_key']
);
} catch (CloudObjectStorageTransferError $error) {
$this->markUnknown(
$work,
$error->publicCode,
$metadata
);
return;
} catch (\Throwable) {
$this->markUnknown(
$work,
'replica_multipart_lookup_unknown',
$metadata
);
return;
}
if (count($matches) > 1) {
$this->markMultipartAttention(
$work,
'replica_multipart_identity_ambiguous',
$metadata
);
return;
}
if (count($matches) === 1) {
$match = $matches[0];
if (isset($metadata['upload_id'])) {
if (!hash_equals(
(string) $metadata['upload_id'],
(string) $match['upload_id']
)) {
$this->markMultipartAttention(
$work,
'replica_multipart_identity_mismatch',
$metadata
);
return;
}
$metadata['multipart_presence_confirmations'] = min(
2,
(int) ($metadata['multipart_presence_confirmations'] ?? 0)
+ 1
);
if ($metadata['multipart_presence_confirmations'] < 2) {
// The exact session and absent object must remain stable
// across two leases before replaying idempotent UploadPart
// calls against that same upload_id.
$this->markUnknown(
$work,
'replica_multipart_present_unconfirmed',
$metadata
);
return;
}
}
if (!$this->adoptableMultipart($work, $match, $startedAt)
|| !$this->validSource($work)) {
$this->markMultipartAttention(
$work,
'replica_multipart_identity_untrusted',
$metadata
);
return;
}
try {
$sourceRuntime = ($this->runtimeResolver)(
(int) $work['source_backend_id'],
false
);
$plan = $this->s3()->resumeMultipartUpload(
$runtime['config'],
$runtime['credentials'],
(string) $work['target_object_key'],
(string) $match['upload_id'],
(int) $work['size_bytes'],
($this->clock)(),
self::CAPABILITY_SECONDS
);
} catch (\Throwable) {
$this->markUnknown(
$work,
'replica_multipart_resume_unknown',
$metadata
);
return;
}
$adopted = [
'upload_id' => (string) $plan['upload_id'],
'part_size_bytes' => (int) $plan['part_size_bytes'],
'part_count' => count($plan['part_urls']),
'maximum_size_bytes' => (int) $plan['maximum_size_bytes'],
'multipart_absence_confirmations' => 0,
'multipart_presence_confirmations' => isset(
$metadata['upload_id']
) ? 2 : 0,
'initiated_at' => (int) $match['initiated_at'],
];
CloudStorageArtifactService::markReplicaAwaitingReceipt(
(string) $work['operation_id'],
(string) $work['lease_token'],
$adopted,
($this->clock)()
);
$this->invokeTransfer(
$work,
$startedAt,
$sourceRuntime,
$runtime,
$plan,
$adopted
);
return;
}
$metadata['multipart_absence_confirmations'] = min(
2,
(int) ($metadata['multipart_absence_confirmations'] ?? 0) + 1
);
if ($metadata['multipart_absence_confirmations'] < 2) {
$this->markUnknown(
$work,
'replica_multipart_absent_unconfirmed',
$metadata
);
return;
}
CloudStorageArtifactService::rescheduleReplicaAfterProof(
(string) $work['operation_id'],
(string) $work['lease_token'],
$metadata,
($this->clock)(),
self::RECONCILE_DELAY
);
$this->errors[] = 'replica_retry_scheduled';
}
/**
* @param array<string,mixed> $work
* @param array<string,mixed> $receipt
* @param array<string,mixed> $metadata
* @param array<string,mixed>|null $observedIdentity
*/
private function settleReceipt(
array $work,
array $receipt,
array $metadata,
?array $observedIdentity
): void {
$now = ($this->clock)();
if (($receipt['outcome'] ?? null) === 'succeeded'
&& ($receipt['effect_resolution'] ?? null) === 'applied') {
$observedVersion = (string) (
$observedIdentity['version_id'] ?? ''
);
$receiptVersion = (string) ($receipt['version_id'] ?? '');
$mixedVersionIdentity = ($observedVersion === '')
!== ($receiptVersion === '');
$identityChanged = $observedIdentity !== null
&& ((int) ($observedIdentity['size_bytes'] ?? 0)
!== (int) ($receipt['size_bytes'] ?? 0)
|| (int) ($observedIdentity['size_bytes'] ?? 0)
!== (int) $work['size_bytes']
|| $mixedVersionIdentity
|| !hash_equals(
(string) ($observedIdentity['etag'] ?? ''),
(string) ($receipt['etag'] ?? '')
)
|| ($observedVersion !== ''
&& !hash_equals(
$observedVersion,
$receiptVersion
)));
if ($identityChanged) {
$this->errors[] = 'replica_identity_changed';
CloudStorageArtifactService::failReplicaAppliedMismatch(
(string) $work['operation_id'],
(string) $work['lease_token'],
'replica_identity_changed',
$now
);
return;
}
CloudStorageArtifactService::completeReplica(
(string) $work['operation_id'],
(string) $work['lease_token'],
$receipt,
$now
);
return;
}
$code = is_string($receipt['error_code'] ?? null)
? $receipt['error_code'] : 'replicator_invalid_receipt';
if (($receipt['effect_resolution'] ?? null) === 'not_applied') {
if ($observedIdentity !== null) {
$this->errors[] = $code;
CloudStorageArtifactService::failReplicaAppliedMismatch(
(string) $work['operation_id'],
(string) $work['lease_token'],
$code,
$now
);
} else {
$this->rescheduleNotApplied($work, $code);
}
return;
}
$this->markUnknown($work, $code, $metadata);
}
/** @param array<string,mixed> $work @param array<string,mixed> $metadata */
private function markUnknown(
array $work,
string $errorCode,
array $metadata
): void {
$this->errors[] = $errorCode;
CloudStorageArtifactService::markReplicaUnknown(
(string) $work['operation_id'],
(string) $work['lease_token'],
$errorCode,
($this->clock)(),
self::RECONCILE_DELAY,
$metadata
);
}
/**
* Bound stable multipart identity conflicts. Automatic reconciliation
* stops after two observations; an administrator may then re-open this
* same operation after correcting the provider-side sessions.
*
* @param array<string,mixed> $work
* @param array<string,mixed> $metadata
*/
private function markMultipartAttention(
array $work,
string $errorCode,
array $metadata
): void {
$previous = $metadata['multipart_attention_error_code'] ?? '';
$confirmations = $previous === $errorCode
? (int) ($metadata['multipart_attention_confirmations'] ?? 0) + 1
: 1;
$metadata['multipart_attention_error_code'] = $errorCode;
$metadata['multipart_attention_confirmations'] = min(
2,
$confirmations
);
if ($metadata['multipart_attention_confirmations'] < 2) {
$this->markUnknown($work, $errorCode, $metadata);
return;
}
$this->errors[] = $errorCode;
CloudStorageArtifactService::markReplicaAttention(
(string) $work['operation_id'],
(string) $work['lease_token'],
$errorCode,
($this->clock)(),
$metadata
);
}
/** @param array<string,mixed> $work */
private function defer(array $work, string $code, int $delay): void
{
$this->errors[] = $code;
CloudStorageArtifactService::deferReplica(
(string) $work['operation_id'],
(string) $work['lease_token'],
$code,
($this->clock)(),
$delay
);
}
/**
* A definite no-effect result means either multipart creation never
* happened or the helper proved no object was committed and ended its
* session. Re-open the same durable copy intent; never allocate a
* replacement operation.
*
* @param array<string,mixed> $work
*/
private function rescheduleNotApplied(array $work, string $code): void
{
$this->errors[] = $code;
CloudStorageArtifactService::rescheduleReplicaNotApplied(
(string) $work['operation_id'],
(string) $work['lease_token'],
$code,
($this->clock)(),
self::RECONCILE_DELAY
);
}
/**
* @param array<string,mixed> $work
* @param array{upload_id:string,initiated_at:int} $match
*/
private function adoptableMultipart(
array $work,
array $match,
int $now
): bool {
$createdAt = $work['operation_created_at'] ?? null;
$availableAt = $work['operation_available_at'] ?? null;
$initiatedAt = $match['initiated_at'] ?? null;
return is_int($createdAt)
&& is_int($availableAt)
&& is_int($initiatedAt)
&& $initiatedAt >= max(1, $createdAt, $availableAt) - 300
&& $initiatedAt <= $now + 60;
}
/**
* @param array<string,mixed> $work
* @param array<string,mixed> $sourceRuntime
* @param array<string,mixed> $targetRuntime
* @param array<string,mixed> $plan
* @param array<string,mixed> $metadata
*/
private function invokeTransfer(
array $work,
int $startedAt,
array $sourceRuntime,
array $targetRuntime,
array $plan,
array $metadata
): void {
try {
$sourceURL = $this->s3()->presignedVersionedGet(
$sourceRuntime['config'],
$sourceRuntime['credentials'],
(string) $work['source_object_key'],
(string) $work['source_version_id'],
self::CAPABILITY_SECONDS
);
$receipt = $this->mover()->invoke([
'protocol' => 'object-replica-v1',
'operation_id' => (string) $work['operation_id'],
'deadline_at' => $startedAt + self::CAPABILITY_SECONDS,
'source' => [
'get_url' => $sourceURL,
'size_bytes' => (int) $work['size_bytes'],
'sha256' => (string) $work['sha256'],
],
'destination' => self::helperPlan($plan),
]);
} catch (\Throwable) {
$this->markUnknown(
$work,
'replicator_invalid_receipt',
$metadata
);
return;
}
$identity = null;
if (($receipt['outcome'] ?? null) === 'succeeded'
&& ($receipt['effect_resolution'] ?? null) === 'applied') {
try {
$identity = $this->s3()->inspectObject(
$targetRuntime['config'],
$targetRuntime['credentials'],
(string) $work['target_object_key']
);
} catch (\Throwable) {
$this->markUnknown(
$work,
'replica_completion_inspection_unavailable',
$metadata
);
return;
}
if ($identity === null) {
$this->markUnknown(
$work,
'replica_completion_object_absent',
$metadata
);
return;
}
}
$this->settleReceipt($work, $receipt, $metadata, $identity);
}
/** @param array<string,mixed> $work */
private function validSource(array $work): bool
{
return ($work['source_state'] ?? null) === 'ready'
&& is_int($work['size_bytes'] ?? null)
&& $work['size_bytes'] > 0
&& is_string($work['sha256'] ?? null)
&& preg_match('/^[a-f0-9]{64}$/D', $work['sha256']) === 1
&& is_string($work['source_version_id'] ?? null);
}
private function s3(): CloudObjectStorageS3Client
{
return $this->client ?? new CloudObjectStorageS3Client();
}
private function mover(): CloudStorageReplicaHelper
{
return $this->helper ?? new CloudStorageReplicaHelper();
}
/** @param array<string,mixed> $plan @return array<string,mixed> */
private static function helperPlan(array $plan): array
{
return [
'part_size_bytes' => (int) $plan['part_size_bytes'],
'part_urls' => array_values($plan['part_urls']),
'complete_url' => (string) $plan['complete_url'],
'abort_url' => (string) $plan['abort_url'],
'verify_url' => (string) $plan['verify_url'],
'maximum_size_bytes' => (int) $plan['maximum_size_bytes'],
];
}
}