You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
677 lines
23 KiB
PHP
677 lines
23 KiB
PHP
<?php
|
|
|
|
namespace App\Services\Cloud;
|
|
|
|
use App\Models\CloudStorageBackend;
|
|
|
|
/**
|
|
* Streams a pinned immutable replica and reconciles ambiguous completion from
|
|
* read-only evidence. Capability URLs exist only inside this process.
|
|
*/
|
|
final class CloudStorageReplicaWorkerService
|
|
{
|
|
private const CAPABILITY_SECONDS = 21600;
|
|
private const RECONCILE_DELAY = 60;
|
|
|
|
/** @var \Closure(int,bool):array<string,mixed> */
|
|
private readonly \Closure $runtimeResolver;
|
|
|
|
/** @var \Closure():int */
|
|
private readonly \Closure $clock;
|
|
|
|
/** @var list<string> */
|
|
private array $errors = [];
|
|
|
|
/**
|
|
* @param null|callable(int,bool):array<string,mixed> $runtimeResolver
|
|
* @param null|callable():int $clock
|
|
*/
|
|
public function __construct(
|
|
private readonly ?CloudStorageReplicaHelper $helper = null,
|
|
private readonly ?CloudObjectStorageS3Client $client = null,
|
|
?callable $runtimeResolver = null,
|
|
?callable $clock = null,
|
|
private readonly ?CloudStorageReplicaRepairService
|
|
$replicaRepair = null
|
|
) {
|
|
$this->runtimeResolver = $runtimeResolver === null
|
|
? static fn (int $backendId, bool $healthy): array =>
|
|
CloudObjectStorageBackendRegistry::runtime($backendId, $healthy)
|
|
: \Closure::fromCallable($runtimeResolver);
|
|
$this->clock = $clock === null
|
|
? static fn (): int => time()
|
|
: \Closure::fromCallable($clock);
|
|
}
|
|
|
|
public function tick(): bool
|
|
{
|
|
$this->errors = [];
|
|
$now = ($this->clock)();
|
|
// Repair only reserves one local, durable copy intent. It performs no
|
|
// provider I/O, so a backend that became writable can start healing
|
|
// old single-copy artifacts without delaying existing remote work.
|
|
$repairWorked = false;
|
|
try {
|
|
$repair = $this->replicaRepair
|
|
?? new CloudStorageReplicaRepairService();
|
|
$repairWorked = $repair->tick($now);
|
|
} catch (\Throwable) {
|
|
$this->errors[] = 'storage_replica_repair_tick_failed';
|
|
}
|
|
|
|
$work = CloudStorageArtifactService::claimReplicaWork($now);
|
|
$replicaWorked = $work !== null;
|
|
if ($work !== null) {
|
|
try {
|
|
if (($work['mode'] ?? null) === 'dispatch') {
|
|
$this->dispatch($work, $now);
|
|
} else {
|
|
$this->reconcile($work, $now);
|
|
}
|
|
} catch (\Throwable) {
|
|
$this->errors[] = 'storage_replica_tick_failed';
|
|
try {
|
|
$this->markUnknown(
|
|
$work,
|
|
'storage_replica_tick_failed',
|
|
is_array($work['metadata'] ?? null)
|
|
? $work['metadata'] : []
|
|
);
|
|
} catch (\Throwable) {
|
|
// The durable lease remains the recovery fence. Never
|
|
// create a replacement operation after an uncertain
|
|
// settlement.
|
|
}
|
|
}
|
|
}
|
|
$this->errors = array_values(array_unique($this->errors));
|
|
return $replicaWorked || $repairWorked;
|
|
}
|
|
|
|
/** @return list<string> */
|
|
public function observationErrors(): array
|
|
{
|
|
return $this->errors;
|
|
}
|
|
|
|
/** @param array<string,mixed> $work */
|
|
private function dispatch(array $work, int $startedAt): void
|
|
{
|
|
if (!$this->validSource($work)) {
|
|
$this->defer($work, 'replica_source_not_ready', 60);
|
|
return;
|
|
}
|
|
$target = CloudStorageBackend::query()
|
|
->whereKey((int) $work['target_backend_id'])->first();
|
|
if (!$target instanceof CloudStorageBackend
|
|
|| (string) $target->admin_state !== 'active'
|
|
|| (string) $target->health_state !== 'healthy'
|
|
|| (int) $target->tested_revision !== (int) $target->config_revision) {
|
|
$this->defer($work, 'replica_target_not_writable', 300);
|
|
return;
|
|
}
|
|
|
|
try {
|
|
$sourceRuntime = ($this->runtimeResolver)(
|
|
(int) $work['source_backend_id'],
|
|
false
|
|
);
|
|
$targetRuntime = ($this->runtimeResolver)(
|
|
(int) $work['target_backend_id'],
|
|
true
|
|
);
|
|
} catch (\Throwable) {
|
|
$this->defer($work, 'replica_runtime_unavailable', 300);
|
|
return;
|
|
}
|
|
|
|
try {
|
|
$plan = $this->s3()->beginMultipartPlan(
|
|
$targetRuntime['config'],
|
|
$targetRuntime['credentials'],
|
|
(string) $work['target_object_key'],
|
|
(int) $work['size_bytes'],
|
|
self::CAPABILITY_SECONDS
|
|
);
|
|
} catch (CloudObjectStorageTransferError $error) {
|
|
if (!$error->effectMayHaveApplied) {
|
|
$this->rescheduleNotApplied($work, $error->publicCode);
|
|
return;
|
|
}
|
|
$this->markUnknown($work, $error->publicCode, []);
|
|
return;
|
|
} catch (\Throwable) {
|
|
$this->markUnknown($work, 'replica_multipart_start_unknown', []);
|
|
return;
|
|
}
|
|
|
|
$metadata = [
|
|
'upload_id' => (string) $plan['upload_id'],
|
|
'part_size_bytes' => (int) $plan['part_size_bytes'],
|
|
'part_count' => count($plan['part_urls']),
|
|
'maximum_size_bytes' => (int) $plan['maximum_size_bytes'],
|
|
'multipart_absence_confirmations' => 0,
|
|
'multipart_presence_confirmations' => 0,
|
|
];
|
|
CloudStorageArtifactService::markReplicaAwaitingReceipt(
|
|
(string) $work['operation_id'],
|
|
(string) $work['lease_token'],
|
|
$metadata,
|
|
($this->clock)()
|
|
);
|
|
$this->invokeTransfer(
|
|
$work,
|
|
$startedAt,
|
|
$sourceRuntime,
|
|
$targetRuntime,
|
|
$plan,
|
|
$metadata
|
|
);
|
|
}
|
|
|
|
/** @param array<string,mixed> $work */
|
|
private function reconcile(array $work, int $startedAt): void
|
|
{
|
|
$metadata = is_array($work['metadata'] ?? null)
|
|
? $work['metadata'] : [];
|
|
try {
|
|
$runtime = ($this->runtimeResolver)(
|
|
(int) $work['target_backend_id'],
|
|
false
|
|
);
|
|
$identity = $this->s3()->inspectObject(
|
|
$runtime['config'],
|
|
$runtime['credentials'],
|
|
(string) $work['target_object_key']
|
|
);
|
|
} catch (\Throwable) {
|
|
$this->markUnknown(
|
|
$work,
|
|
'replica_inspection_unavailable',
|
|
$metadata
|
|
);
|
|
return;
|
|
}
|
|
if ($identity === null) {
|
|
$this->reconcileAbsence(
|
|
$work,
|
|
$runtime,
|
|
$metadata,
|
|
$startedAt
|
|
);
|
|
return;
|
|
}
|
|
if ((int) $identity['size_bytes'] !== (int) $work['size_bytes']) {
|
|
$this->errors[] = 'replica_identity_mismatch';
|
|
CloudStorageArtifactService::failReplicaAppliedMismatch(
|
|
(string) $work['operation_id'],
|
|
(string) $work['lease_token'],
|
|
'replica_identity_mismatch',
|
|
($this->clock)()
|
|
);
|
|
return;
|
|
}
|
|
try {
|
|
$url = $this->s3()->presignedVersionedGet(
|
|
$runtime['config'],
|
|
$runtime['credentials'],
|
|
(string) $work['target_object_key'],
|
|
(string) $identity['version_id'],
|
|
self::CAPABILITY_SECONDS
|
|
);
|
|
$receipt = $this->mover()->invoke([
|
|
'protocol' => 'object-replica-verify-v1',
|
|
'operation_id' => (string) $work['operation_id'],
|
|
'deadline_at' => $startedAt + self::CAPABILITY_SECONDS,
|
|
'get_url' => $url,
|
|
'size_bytes' => (int) $work['size_bytes'],
|
|
'sha256' => (string) $work['sha256'],
|
|
]);
|
|
} catch (\Throwable) {
|
|
$this->markUnknown(
|
|
$work,
|
|
'replica_verify_unavailable',
|
|
$metadata
|
|
);
|
|
return;
|
|
}
|
|
$this->settleReceipt($work, $receipt, $metadata, $identity);
|
|
}
|
|
|
|
/**
|
|
* @param array<string,mixed> $work
|
|
* @param array<string,mixed> $runtime
|
|
* @param array<string,mixed> $metadata
|
|
*/
|
|
private function reconcileAbsence(
|
|
array $work,
|
|
array $runtime,
|
|
array $metadata,
|
|
int $startedAt
|
|
): void {
|
|
try {
|
|
$matches = $this->s3()->findMultipartUploadsByExactObjectKey(
|
|
$runtime['config'],
|
|
$runtime['credentials'],
|
|
(string) $work['target_object_key']
|
|
);
|
|
} catch (CloudObjectStorageTransferError $error) {
|
|
$this->markUnknown(
|
|
$work,
|
|
$error->publicCode,
|
|
$metadata
|
|
);
|
|
return;
|
|
} catch (\Throwable) {
|
|
$this->markUnknown(
|
|
$work,
|
|
'replica_multipart_lookup_unknown',
|
|
$metadata
|
|
);
|
|
return;
|
|
}
|
|
|
|
if (count($matches) > 1) {
|
|
$this->markMultipartAttention(
|
|
$work,
|
|
'replica_multipart_identity_ambiguous',
|
|
$metadata
|
|
);
|
|
return;
|
|
}
|
|
if (count($matches) === 1) {
|
|
$match = $matches[0];
|
|
if (isset($metadata['upload_id'])) {
|
|
if (!hash_equals(
|
|
(string) $metadata['upload_id'],
|
|
(string) $match['upload_id']
|
|
)) {
|
|
$this->markMultipartAttention(
|
|
$work,
|
|
'replica_multipart_identity_mismatch',
|
|
$metadata
|
|
);
|
|
return;
|
|
}
|
|
$metadata['multipart_presence_confirmations'] = min(
|
|
2,
|
|
(int) ($metadata['multipart_presence_confirmations'] ?? 0)
|
|
+ 1
|
|
);
|
|
if ($metadata['multipart_presence_confirmations'] < 2) {
|
|
// The exact session and absent object must remain stable
|
|
// across two leases before replaying idempotent UploadPart
|
|
// calls against that same upload_id.
|
|
$this->markUnknown(
|
|
$work,
|
|
'replica_multipart_present_unconfirmed',
|
|
$metadata
|
|
);
|
|
return;
|
|
}
|
|
}
|
|
if (!$this->adoptableMultipart($work, $match, $startedAt)
|
|
|| !$this->validSource($work)) {
|
|
$this->markMultipartAttention(
|
|
$work,
|
|
'replica_multipart_identity_untrusted',
|
|
$metadata
|
|
);
|
|
return;
|
|
}
|
|
try {
|
|
$sourceRuntime = ($this->runtimeResolver)(
|
|
(int) $work['source_backend_id'],
|
|
false
|
|
);
|
|
$plan = $this->s3()->resumeMultipartUpload(
|
|
$runtime['config'],
|
|
$runtime['credentials'],
|
|
(string) $work['target_object_key'],
|
|
(string) $match['upload_id'],
|
|
(int) $work['size_bytes'],
|
|
($this->clock)(),
|
|
self::CAPABILITY_SECONDS
|
|
);
|
|
} catch (\Throwable) {
|
|
$this->markUnknown(
|
|
$work,
|
|
'replica_multipart_resume_unknown',
|
|
$metadata
|
|
);
|
|
return;
|
|
}
|
|
$adopted = [
|
|
'upload_id' => (string) $plan['upload_id'],
|
|
'part_size_bytes' => (int) $plan['part_size_bytes'],
|
|
'part_count' => count($plan['part_urls']),
|
|
'maximum_size_bytes' => (int) $plan['maximum_size_bytes'],
|
|
'multipart_absence_confirmations' => 0,
|
|
'multipart_presence_confirmations' => isset(
|
|
$metadata['upload_id']
|
|
) ? 2 : 0,
|
|
'initiated_at' => (int) $match['initiated_at'],
|
|
];
|
|
CloudStorageArtifactService::markReplicaAwaitingReceipt(
|
|
(string) $work['operation_id'],
|
|
(string) $work['lease_token'],
|
|
$adopted,
|
|
($this->clock)()
|
|
);
|
|
$this->invokeTransfer(
|
|
$work,
|
|
$startedAt,
|
|
$sourceRuntime,
|
|
$runtime,
|
|
$plan,
|
|
$adopted
|
|
);
|
|
return;
|
|
}
|
|
|
|
$metadata['multipart_absence_confirmations'] = min(
|
|
2,
|
|
(int) ($metadata['multipart_absence_confirmations'] ?? 0) + 1
|
|
);
|
|
if ($metadata['multipart_absence_confirmations'] < 2) {
|
|
$this->markUnknown(
|
|
$work,
|
|
'replica_multipart_absent_unconfirmed',
|
|
$metadata
|
|
);
|
|
return;
|
|
}
|
|
CloudStorageArtifactService::rescheduleReplicaAfterProof(
|
|
(string) $work['operation_id'],
|
|
(string) $work['lease_token'],
|
|
$metadata,
|
|
($this->clock)(),
|
|
self::RECONCILE_DELAY
|
|
);
|
|
$this->errors[] = 'replica_retry_scheduled';
|
|
}
|
|
|
|
/**
|
|
* @param array<string,mixed> $work
|
|
* @param array<string,mixed> $receipt
|
|
* @param array<string,mixed> $metadata
|
|
* @param array<string,mixed>|null $observedIdentity
|
|
*/
|
|
private function settleReceipt(
|
|
array $work,
|
|
array $receipt,
|
|
array $metadata,
|
|
?array $observedIdentity
|
|
): void {
|
|
$now = ($this->clock)();
|
|
if (($receipt['outcome'] ?? null) === 'succeeded'
|
|
&& ($receipt['effect_resolution'] ?? null) === 'applied') {
|
|
$observedVersion = (string) (
|
|
$observedIdentity['version_id'] ?? ''
|
|
);
|
|
$receiptVersion = (string) ($receipt['version_id'] ?? '');
|
|
$mixedVersionIdentity = ($observedVersion === '')
|
|
!== ($receiptVersion === '');
|
|
$identityChanged = $observedIdentity !== null
|
|
&& ((int) ($observedIdentity['size_bytes'] ?? 0)
|
|
!== (int) ($receipt['size_bytes'] ?? 0)
|
|
|| (int) ($observedIdentity['size_bytes'] ?? 0)
|
|
!== (int) $work['size_bytes']
|
|
|| $mixedVersionIdentity
|
|
|| !hash_equals(
|
|
(string) ($observedIdentity['etag'] ?? ''),
|
|
(string) ($receipt['etag'] ?? '')
|
|
)
|
|
|| ($observedVersion !== ''
|
|
&& !hash_equals(
|
|
$observedVersion,
|
|
$receiptVersion
|
|
)));
|
|
if ($identityChanged) {
|
|
$this->errors[] = 'replica_identity_changed';
|
|
CloudStorageArtifactService::failReplicaAppliedMismatch(
|
|
(string) $work['operation_id'],
|
|
(string) $work['lease_token'],
|
|
'replica_identity_changed',
|
|
$now
|
|
);
|
|
return;
|
|
}
|
|
CloudStorageArtifactService::completeReplica(
|
|
(string) $work['operation_id'],
|
|
(string) $work['lease_token'],
|
|
$receipt,
|
|
$now
|
|
);
|
|
return;
|
|
}
|
|
$code = is_string($receipt['error_code'] ?? null)
|
|
? $receipt['error_code'] : 'replicator_invalid_receipt';
|
|
if (($receipt['effect_resolution'] ?? null) === 'not_applied') {
|
|
if ($observedIdentity !== null) {
|
|
$this->errors[] = $code;
|
|
CloudStorageArtifactService::failReplicaAppliedMismatch(
|
|
(string) $work['operation_id'],
|
|
(string) $work['lease_token'],
|
|
$code,
|
|
$now
|
|
);
|
|
} else {
|
|
$this->rescheduleNotApplied($work, $code);
|
|
}
|
|
return;
|
|
}
|
|
$this->markUnknown($work, $code, $metadata);
|
|
}
|
|
|
|
/** @param array<string,mixed> $work @param array<string,mixed> $metadata */
|
|
private function markUnknown(
|
|
array $work,
|
|
string $errorCode,
|
|
array $metadata
|
|
): void {
|
|
$this->errors[] = $errorCode;
|
|
CloudStorageArtifactService::markReplicaUnknown(
|
|
(string) $work['operation_id'],
|
|
(string) $work['lease_token'],
|
|
$errorCode,
|
|
($this->clock)(),
|
|
self::RECONCILE_DELAY,
|
|
$metadata
|
|
);
|
|
}
|
|
|
|
/**
|
|
* Bound stable multipart identity conflicts. Automatic reconciliation
|
|
* stops after two observations; an administrator may then re-open this
|
|
* same operation after correcting the provider-side sessions.
|
|
*
|
|
* @param array<string,mixed> $work
|
|
* @param array<string,mixed> $metadata
|
|
*/
|
|
private function markMultipartAttention(
|
|
array $work,
|
|
string $errorCode,
|
|
array $metadata
|
|
): void {
|
|
$previous = $metadata['multipart_attention_error_code'] ?? '';
|
|
$confirmations = $previous === $errorCode
|
|
? (int) ($metadata['multipart_attention_confirmations'] ?? 0) + 1
|
|
: 1;
|
|
$metadata['multipart_attention_error_code'] = $errorCode;
|
|
$metadata['multipart_attention_confirmations'] = min(
|
|
2,
|
|
$confirmations
|
|
);
|
|
if ($metadata['multipart_attention_confirmations'] < 2) {
|
|
$this->markUnknown($work, $errorCode, $metadata);
|
|
return;
|
|
}
|
|
$this->errors[] = $errorCode;
|
|
CloudStorageArtifactService::markReplicaAttention(
|
|
(string) $work['operation_id'],
|
|
(string) $work['lease_token'],
|
|
$errorCode,
|
|
($this->clock)(),
|
|
$metadata
|
|
);
|
|
}
|
|
|
|
/** @param array<string,mixed> $work */
|
|
private function defer(array $work, string $code, int $delay): void
|
|
{
|
|
$this->errors[] = $code;
|
|
CloudStorageArtifactService::deferReplica(
|
|
(string) $work['operation_id'],
|
|
(string) $work['lease_token'],
|
|
$code,
|
|
($this->clock)(),
|
|
$delay
|
|
);
|
|
}
|
|
|
|
/**
|
|
* A definite no-effect result means either multipart creation never
|
|
* happened or the helper proved no object was committed and ended its
|
|
* session. Re-open the same durable copy intent; never allocate a
|
|
* replacement operation.
|
|
*
|
|
* @param array<string,mixed> $work
|
|
*/
|
|
private function rescheduleNotApplied(array $work, string $code): void
|
|
{
|
|
$this->errors[] = $code;
|
|
CloudStorageArtifactService::rescheduleReplicaNotApplied(
|
|
(string) $work['operation_id'],
|
|
(string) $work['lease_token'],
|
|
$code,
|
|
($this->clock)(),
|
|
self::RECONCILE_DELAY
|
|
);
|
|
}
|
|
|
|
/**
|
|
* @param array<string,mixed> $work
|
|
* @param array{upload_id:string,initiated_at:int} $match
|
|
*/
|
|
private function adoptableMultipart(
|
|
array $work,
|
|
array $match,
|
|
int $now
|
|
): bool {
|
|
$createdAt = $work['operation_created_at'] ?? null;
|
|
$availableAt = $work['operation_available_at'] ?? null;
|
|
$initiatedAt = $match['initiated_at'] ?? null;
|
|
return is_int($createdAt)
|
|
&& is_int($availableAt)
|
|
&& is_int($initiatedAt)
|
|
&& $initiatedAt >= max(1, $createdAt, $availableAt) - 300
|
|
&& $initiatedAt <= $now + 60;
|
|
}
|
|
|
|
/**
|
|
* @param array<string,mixed> $work
|
|
* @param array<string,mixed> $sourceRuntime
|
|
* @param array<string,mixed> $targetRuntime
|
|
* @param array<string,mixed> $plan
|
|
* @param array<string,mixed> $metadata
|
|
*/
|
|
private function invokeTransfer(
|
|
array $work,
|
|
int $startedAt,
|
|
array $sourceRuntime,
|
|
array $targetRuntime,
|
|
array $plan,
|
|
array $metadata
|
|
): void {
|
|
try {
|
|
$sourceURL = $this->s3()->presignedVersionedGet(
|
|
$sourceRuntime['config'],
|
|
$sourceRuntime['credentials'],
|
|
(string) $work['source_object_key'],
|
|
(string) $work['source_version_id'],
|
|
self::CAPABILITY_SECONDS
|
|
);
|
|
$receipt = $this->mover()->invoke([
|
|
'protocol' => 'object-replica-v1',
|
|
'operation_id' => (string) $work['operation_id'],
|
|
'deadline_at' => $startedAt + self::CAPABILITY_SECONDS,
|
|
'source' => [
|
|
'get_url' => $sourceURL,
|
|
'size_bytes' => (int) $work['size_bytes'],
|
|
'sha256' => (string) $work['sha256'],
|
|
],
|
|
'destination' => self::helperPlan($plan),
|
|
]);
|
|
} catch (\Throwable) {
|
|
$this->markUnknown(
|
|
$work,
|
|
'replicator_invalid_receipt',
|
|
$metadata
|
|
);
|
|
return;
|
|
}
|
|
$identity = null;
|
|
if (($receipt['outcome'] ?? null) === 'succeeded'
|
|
&& ($receipt['effect_resolution'] ?? null) === 'applied') {
|
|
try {
|
|
$identity = $this->s3()->inspectObject(
|
|
$targetRuntime['config'],
|
|
$targetRuntime['credentials'],
|
|
(string) $work['target_object_key']
|
|
);
|
|
} catch (\Throwable) {
|
|
$this->markUnknown(
|
|
$work,
|
|
'replica_completion_inspection_unavailable',
|
|
$metadata
|
|
);
|
|
return;
|
|
}
|
|
if ($identity === null) {
|
|
$this->markUnknown(
|
|
$work,
|
|
'replica_completion_object_absent',
|
|
$metadata
|
|
);
|
|
return;
|
|
}
|
|
}
|
|
$this->settleReceipt($work, $receipt, $metadata, $identity);
|
|
}
|
|
|
|
/** @param array<string,mixed> $work */
|
|
private function validSource(array $work): bool
|
|
{
|
|
return ($work['source_state'] ?? null) === 'ready'
|
|
&& is_int($work['size_bytes'] ?? null)
|
|
&& $work['size_bytes'] > 0
|
|
&& is_string($work['sha256'] ?? null)
|
|
&& preg_match('/^[a-f0-9]{64}$/D', $work['sha256']) === 1
|
|
&& is_string($work['source_version_id'] ?? null);
|
|
}
|
|
|
|
private function s3(): CloudObjectStorageS3Client
|
|
{
|
|
return $this->client ?? new CloudObjectStorageS3Client();
|
|
}
|
|
|
|
private function mover(): CloudStorageReplicaHelper
|
|
{
|
|
return $this->helper ?? new CloudStorageReplicaHelper();
|
|
}
|
|
|
|
/** @param array<string,mixed> $plan @return array<string,mixed> */
|
|
private static function helperPlan(array $plan): array
|
|
{
|
|
return [
|
|
'part_size_bytes' => (int) $plan['part_size_bytes'],
|
|
'part_urls' => array_values($plan['part_urls']),
|
|
'complete_url' => (string) $plan['complete_url'],
|
|
'abort_url' => (string) $plan['abort_url'],
|
|
'verify_url' => (string) $plan['verify_url'],
|
|
'maximum_size_bytes' => (int) $plan['maximum_size_bytes'],
|
|
];
|
|
}
|
|
}
|