You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
virt/web/app/Services/Cloud/CloudStorageImmutablePrimar...

1109 lines
43 KiB
PHP

<?php
namespace App\Services\Cloud;
use App\Models\CloudStorageArtifact;
use App\Models\CloudStorageOperation;
use App\Models\CloudStoragePublication;
use App\Models\CloudStorageReplica;
/**
* Publishes one known local file through an already-reserved primary intent.
*
* The business owner must first call CloudStorageArtifactService::
* beginPublication(). This module never allocates a replacement artifact or
* object key. A lost PUT receipt is reconciled only with exact-key HEAD
* evidence; the same operation becomes dispatchable again only after two
* spaced, stable absence observations.
*/
final class CloudStorageImmutablePrimaryPublication
{
/** Secondary work remains dormant until the primary is verified again. */
private const COPY_BLOCKED_UNTIL_PRIMARY = 253402300799;
private const LEASE_SECONDS = 300;
private const RECONCILE_DELAY = 60;
private const RETRY_DELAY = 300;
private const MAX_DISPATCH_ATTEMPTS = 3;
private const MAX_RECONCILE_FAILURES = 6;
/** @var list<string> */
private const UNKNOWN_ATTENTION_ERRORS = [
'immutable_primary_identity_conflict',
'immutable_primary_reconcile_exhausted',
];
/** @var list<string> */
private const NOT_APPLIED_ATTENTION_ERRORS = [
'immutable_primary_publish_retry_exhausted',
'immutable_primary_absence_retry_exhausted',
];
/** @var \Closure(int,bool):array<string,mixed> */
private readonly \Closure $runtimeResolver;
/** @var \Closure():int */
private readonly \Closure $clock;
/**
* @param null|callable(int,bool):array<string,mixed> $runtimeResolver
* @param null|callable():int $clock
*/
public function __construct(
private readonly ?CloudObjectStorageS3Client $client = null,
?callable $runtimeResolver = null,
?callable $clock = null
) {
$this->runtimeResolver = $runtimeResolver === null
? static fn (int $backendId, bool $healthy): array =>
CloudObjectStorageBackendRegistry::runtime(
$backendId,
$healthy
)
: \Closure::fromCallable($runtimeResolver);
$this->clock = $clock === null
? static fn (): int => time()
: \Closure::fromCallable($clock);
}
/** @return list<string> */
public static function attentionErrors(): array
{
return array_merge(
self::UNKNOWN_ATTENTION_ERRORS,
self::NOT_APPLIED_ATTENTION_ERRORS
);
}
/** @return list<string> */
public static function unknownAttentionErrors(): array
{
return self::UNKNOWN_ATTENTION_ERRORS;
}
/** @return list<string> */
public static function notAppliedAttentionErrors(): array
{
return self::NOT_APPLIED_ATTENTION_ERRORS;
}
/**
* Re-open only the original frozen publish operation.
*
* Unknown effects resume with an exact-key HEAD. A provider write becomes
* dispatchable again only after this class observes two spaced absences.
* Proven non-application may dispatch the same operation again. Secondary
* copies stopped solely by the primary attention state are restored behind
* the primary fence; no new artifact, key or operation is allocated.
*/
public static function reopenAttention(
string $operationId,
int $now
): void {
self::assertOperationId($operationId);
if ($now < 1) {
throw new \InvalidArgumentException('Invalid attention time');
}
$connection = (new CloudStorageOperation())->getConnection();
$connection->transaction(static function () use (
$operationId,
$now
): void {
$operation = CloudStorageOperation::query()
->where('operation_id', $operationId)
->lockForUpdate()->first();
if (!$operation instanceof CloudStorageOperation
|| (string) $operation->kind !== 'publish') {
throw new CloudError('对象存储发布核对操作不存在', 404);
}
$copyOperations = CloudStorageOperation::query()
->where('artifact_id', (int) $operation->artifact_id)
->where('kind', 'copy')->orderBy('id')
->lockForUpdate()->get()->all();
$artifact = CloudStorageArtifact::query()
->whereKey((int) $operation->artifact_id)
->lockForUpdate()->first();
$replicas = CloudStorageReplica::query()
->where('artifact_id', (int) $operation->artifact_id)
->orderBy('id')->lockForUpdate()->get()->all();
$publication = CloudStoragePublication::query()
->where('artifact_id', (int) $operation->artifact_id)
->lockForUpdate()->first();
if (!$artifact instanceof CloudStorageArtifact
|| !$publication instanceof CloudStoragePublication
|| $publication->backup_id !== null
|| (($publication->template_import_intent_id === null)
=== ($publication->rescue_template_id === null))) {
throw new CloudError('对象存储发布核对归属不一致', 409);
}
$primary = null;
$replicasById = [];
foreach ($replicas as $replica) {
$replicasById[(int) $replica->id] = $replica;
if ((int) $replica->id === (int) $operation->replica_id) {
$primary = $replica;
}
}
$errorCode = (string) $operation->error_code;
$unknown = in_array(
$errorCode,
self::UNKNOWN_ATTENTION_ERRORS,
true
);
$notApplied = in_array(
$errorCode,
self::NOT_APPLIED_ATTENTION_ERRORS,
true
);
if (!$primary instanceof CloudStorageReplica
|| (string) $primary->role !== 'primary'
|| (!$unknown && !$notApplied)
|| (string) $operation->phase !== 'terminal'
|| (string) $operation->outcome !== 'failed'
|| (bool) $operation->safe_to_resubmit
|| (bool) $operation->effect_may_have_applied !== $unknown
|| (string) $operation->effect_resolution
!== ($unknown ? 'unknown' : 'not_applied')
|| (string) $artifact->state !== 'degraded'
|| (string) $primary->state !== 'degraded'
|| !hash_equals($errorCode, (string) $primary->error_code)
|| (string) $publication->state !== 'failed'
|| !hash_equals(
$errorCode,
(string) $publication->last_error_code
)) {
throw new CloudError(
'对象存储主副本不在可重新核对的人工处理终态',
409
);
}
foreach ($copyOperations as $copy) {
$secondary = $replicasById[(int) $copy->replica_id] ?? null;
if (!$secondary instanceof CloudStorageReplica
|| (string) $secondary->role !== 'secondary'
|| (string) $copy->phase !== 'terminal'
|| (string) $copy->outcome !== 'failed'
|| (string) $copy->effect_resolution !== 'not_applied'
|| (bool) $copy->effect_may_have_applied
|| (bool) $copy->safe_to_resubmit
|| (int) $copy->attempts !== 0
|| (string) $copy->error_code
!== 'primary_publish_attention'
|| (string) $secondary->state !== 'degraded'
|| (string) $secondary->error_code
!== 'primary_publish_attention') {
throw new CloudError(
'对象存储次副本核对账本不一致',
409
);
}
}
$metadata = self::resetEvidence($operation->result_json);
$operation->phase = $unknown ? 'reconciling' : 'accepted';
$operation->outcome = null;
$operation->effect_resolution = $unknown
? 'unknown' : 'not_applicable';
$operation->effect_may_have_applied = null;
$operation->safe_to_resubmit = null;
$operation->lease_token = null;
$operation->lease_until = 0;
$operation->available_at = $unknown ? 0 : $now;
$operation->next_check_at = $unknown ? $now : 0;
$operation->error_code = $unknown
? 'admin_reconcile_requested'
: 'admin_retry_requested';
$operation->result_json = self::encodedMetadata($metadata);
$operation->terminal_at = null;
$operation->updated_at = $now;
$operation->save();
$primary->state = $unknown ? 'publishing' : 'reserved';
$primary->lease_token = null;
$primary->lease_until = 0;
$primary->error_code = $operation->error_code;
$primary->updated_at = $now;
$primary->save();
foreach ($copyOperations as $copy) {
/** @var CloudStorageReplica $secondary */
$secondary = $replicasById[(int) $copy->replica_id];
$copy->phase = 'accepted';
$copy->outcome = null;
$copy->effect_resolution = 'not_applicable';
$copy->effect_may_have_applied = null;
$copy->safe_to_resubmit = null;
$copy->lease_token = null;
$copy->lease_until = 0;
$copy->available_at = self::COPY_BLOCKED_UNTIL_PRIMARY;
$copy->next_check_at = 0;
$copy->error_code = '';
$copy->terminal_at = null;
$copy->updated_at = $now;
$copy->save();
$secondary->state = 'reserved';
$secondary->lease_token = null;
$secondary->lease_until = 0;
$secondary->error_code = '';
$secondary->updated_at = $now;
$secondary->save();
}
$artifact->state = 'publishing';
$artifact->updated_at = $now;
$artifact->save();
$publication->state = 'waiting_primary';
$publication->attempts = 0;
$publication->available_at = 0;
$publication->lease_token = null;
$publication->lease_until = 0;
$publication->last_error_code = '';
$publication->coverage_ready_at = null;
$publication->finalized_at = null;
$publication->updated_at = $now;
$publication->save();
}, 3);
}
/**
* Advance one immutable primary publication by at most one remote action.
*
* @param array{path:string,size_bytes:int,sha256:string} $file
* @return array<string,mixed>
*/
public function publish(
string $operationId,
int $expectedArtifactId,
array $file
): array {
self::assertOperationId($operationId);
if ($expectedArtifactId < 1) {
throw new CloudError('对象存储制品身份无效', 422);
}
$snapshot = $this->snapshot($operationId, $expectedArtifactId);
if (($snapshot['terminal'] ?? false) === true) {
return $snapshot;
}
// Reconciliation is intentionally independent of the temporary local
// file. Once a PUT may have happened, only the frozen ledger identity
// and an exact-key HEAD are authoritative.
if ((string) $snapshot['phase'] === 'accepted') {
self::assertFile($file, $snapshot);
}
$now = ($this->clock)();
$work = $this->claim(
$operationId,
$expectedArtifactId,
$now
);
if ($work === null) {
return $this->snapshot($operationId, $expectedArtifactId);
}
if ((string) $work['mode'] === 'dispatch') {
try {
// Recheck after the dispatch lease is owned so a changed
// source can never reach the provider under this intent.
self::assertFile($file, $work);
} catch (\Throwable $error) {
$this->safeFailure(
$work,
'immutable_primary_source_changed',
$now
);
throw $error;
}
}
try {
$runtime = ($this->runtimeResolver)(
(int) $work['backend_id'],
(string) $work['mode'] === 'dispatch'
);
self::assertRuntime($runtime);
} catch (\Throwable) {
if ((string) $work['mode'] === 'dispatch') {
$this->safeFailure(
$work,
'immutable_primary_runtime_unavailable',
$now
);
} else {
$this->reconcileFailure(
$work,
'immutable_primary_runtime_unavailable',
$now
);
}
return $this->snapshot($operationId, $expectedArtifactId);
}
if ((string) $work['mode'] === 'dispatch') {
$this->dispatch($work, $runtime, $file);
} else {
$this->reconcile($work, $runtime);
}
return $this->snapshot($operationId, $expectedArtifactId);
}
/** @return array<string,mixed> */
private function snapshot(
string $operationId,
int $expectedArtifactId
): array {
$operation = CloudStorageOperation::query()
->where('operation_id', $operationId)->first();
if (!$operation instanceof CloudStorageOperation
|| (string) $operation->kind !== 'publish'
|| (int) $operation->artifact_id !== $expectedArtifactId) {
throw new CloudError('对象存储发布操作与制品身份不一致', 409);
}
$artifact = CloudStorageArtifact::query()
->whereKey($expectedArtifactId)->first();
$replica = CloudStorageReplica::query()
->whereKey((int) $operation->replica_id)->first();
$publication = CloudStoragePublication::query()
->where('artifact_id', $expectedArtifactId)->first();
if (!$artifact instanceof CloudStorageArtifact
|| !$replica instanceof CloudStorageReplica
|| !$publication instanceof CloudStoragePublication
|| (int) $replica->artifact_id !== $expectedArtifactId
|| (string) $replica->role !== 'primary') {
throw new CloudError('对象存储主副本账本不一致', 409);
}
$terminal = (string) $operation->phase === 'terminal';
return [
'operation_id' => (string) $operation->operation_id,
'artifact_id' => (int) $artifact->id,
'phase' => (string) $operation->phase,
'outcome' => $operation->outcome === null
? null : (string) $operation->outcome,
'effect_resolution' => (string) $operation->effect_resolution,
'attempts' => (int) $operation->attempts,
'next_check_at' => (int) $operation->next_check_at,
'error_code' => (string) $operation->error_code,
'terminal' => $terminal,
'primary_completed' => $terminal
&& (string) $operation->outcome === 'succeeded',
'publication_state' => (string) $publication->state,
'ready_to_finalize' => in_array(
(string) $publication->state,
['ready_to_finalize', 'finalizing', 'completed'],
true
),
'requires_attention' => ($terminal
&& (string) $operation->outcome === 'failed')
|| (string) $publication->state === 'failed',
'object_key' => (string) $replica->object_key,
'size_bytes' => (int) $artifact->size_bytes,
'sha256' => (string) $artifact->sha256,
];
}
/** @return array<string,mixed>|null */
private function claim(
string $operationId,
int $expectedArtifactId,
int $now
): ?array {
$connection = (new CloudStorageOperation())->getConnection();
return $connection->transaction(function () use (
$operationId,
$expectedArtifactId,
$now
): ?array {
$operation = CloudStorageOperation::query()
->where('operation_id', $operationId)
->lockForUpdate()->first();
if (!$operation instanceof CloudStorageOperation
|| (string) $operation->kind !== 'publish'
|| (int) $operation->artifact_id !== $expectedArtifactId) {
throw new CloudError(
'对象存储发布操作与制品身份不一致',
409
);
}
[$artifact, $replica, $publication] =
$this->lockedContext($operation);
if ((string) $operation->phase === 'terminal'
|| in_array((string) $publication->state,
['completed', 'failed', 'retired'], true)) {
return null;
}
if ((string) $artifact->state !== 'publishing'
|| !in_array((string) $replica->state,
['reserved', 'publishing'], true)) {
throw new CloudError('对象存储主副本发布状态不一致', 409);
}
if (is_string($operation->lease_token)
&& $operation->lease_token !== ''
&& (int) $operation->lease_until > $now) {
return null;
}
$phase = (string) $operation->phase;
$mode = null;
if ($phase === 'accepted'
&& (int) $operation->available_at <= $now) {
$mode = 'dispatch';
$operation->phase = 'dispatching';
$operation->effect_resolution = 'not_applicable';
$operation->attempts = (int) $operation->attempts + 1;
$replica->state = 'publishing';
} elseif ($phase === 'reconciling'
&& (int) $operation->next_check_at <= $now) {
$mode = 'reconcile';
} elseif (in_array($phase,
['dispatching', 'awaiting_receipt'], true)
&& (int) $operation->lease_until <= $now) {
$mode = 'reconcile';
$operation->phase = 'reconciling';
$operation->effect_resolution = 'unknown';
$operation->error_code =
'immutable_primary_lease_expired';
$operation->next_check_at = $now;
}
if ($mode === null) {
return null;
}
$lease = bin2hex(random_bytes(16));
$operation->lease_token = $lease;
$operation->lease_until = $now + self::LEASE_SECONDS;
$operation->updated_at = $now;
$operation->save();
$replica->lease_token = $lease;
$replica->lease_until = $now + self::LEASE_SECONDS;
$replica->updated_at = $now;
$replica->save();
return [
'mode' => $mode,
'operation_id' => (string) $operation->operation_id,
'artifact_id' => (int) $artifact->id,
'replica_id' => (int) $replica->id,
'backend_id' => (int) $replica->backend_id,
'object_key' => (string) $replica->object_key,
'size_bytes' => (int) $artifact->size_bytes,
'sha256' => (string) $artifact->sha256,
'lease_token' => $lease,
'attempts' => (int) $operation->attempts,
];
}, 3);
}
/**
* @param array<string,mixed> $work
* @param array<string,mixed> $runtime
* @param array{path:string,size_bytes:int,sha256:string} $file
*/
private function dispatch(array $work, array $runtime, array $file): void
{
try {
$identity = $this->s3()->putImmutableFile(
$runtime['config'],
$runtime['credentials'],
(string) $work['object_key'],
(string) $file['path'],
(int) $file['size_bytes'],
(string) $file['sha256']
);
} catch (CloudObjectStorageTransferError $error) {
if ($error->effectMayHaveApplied
|| $error->publicCode === 'object_already_exists') {
$this->markUnknown($work, $error->publicCode);
} else {
$this->safeFailure(
$work,
$error->publicCode,
($this->clock)()
);
}
return;
} catch (\Throwable) {
$this->markUnknown($work, 'immutable_primary_put_unknown');
return;
}
if ((int) ($identity['size_bytes'] ?? 0)
!== (int) $work['size_bytes']
|| !is_string($identity['etag'] ?? null)
|| !is_string($identity['version_id'] ?? null)) {
$this->markUnknown(
$work,
'immutable_primary_put_receipt_invalid'
);
return;
}
CloudStorageArtifactService::completePrimary(
(string) $work['operation_id'],
(int) $work['artifact_id'],
[
'size_bytes' => (int) $work['size_bytes'],
'sha256' => (string) $work['sha256'],
'etag' => (string) $identity['etag'],
'version_id' => (string) $identity['version_id'],
]
);
}
/** @param array<string,mixed> $work @param array<string,mixed> $runtime */
private function reconcile(array $work, array $runtime): void
{
try {
$identity = $this->s3()->inspectObject(
$runtime['config'],
$runtime['credentials'],
(string) $work['object_key']
);
} catch (\Throwable) {
$this->reconcileFailure(
$work,
'immutable_primary_head_unavailable',
($this->clock)()
);
return;
}
if ($identity === null) {
$this->recordAbsence($work, ($this->clock)());
return;
}
if ((int) ($identity['size_bytes'] ?? 0)
!== (int) $work['size_bytes']
|| !is_string($identity['sha256'] ?? null)
|| !hash_equals(
(string) $work['sha256'],
(string) $identity['sha256']
)) {
$this->attention(
$work,
'immutable_primary_identity_conflict',
($this->clock)()
);
return;
}
CloudStorageArtifactService::completePrimary(
(string) $work['operation_id'],
(int) $work['artifact_id'],
[
'size_bytes' => (int) $work['size_bytes'],
'sha256' => (string) $work['sha256'],
'etag' => (string) ($identity['etag'] ?? ''),
'version_id' => (string) ($identity['version_id'] ?? ''),
]
);
}
/** @param array<string,mixed> $work */
private function markUnknown(array $work, string $code): void
{
$now = ($this->clock)();
$this->mutateLeased($work, function (
CloudStorageOperation $operation,
CloudStorageArtifact $artifact,
CloudStorageReplica $replica,
CloudStoragePublication $publication,
array $copyOperations,
array $replicas
) use ($code, $now): void {
$metadata = self::metadata($operation->result_json);
$metadata['absence_confirmations'] = 0;
$operation->phase = 'reconciling';
$operation->effect_resolution = 'unknown';
$operation->lease_token = null;
$operation->lease_until = 0;
$operation->next_check_at = $now + self::RECONCILE_DELAY;
$operation->error_code = self::errorCode($code);
$operation->result_json = self::encodedMetadata($metadata);
$operation->updated_at = $now;
$operation->save();
$replica->state = 'publishing';
$replica->lease_token = null;
$replica->lease_until = 0;
$replica->error_code = self::errorCode($code);
$replica->updated_at = $now;
$replica->save();
$artifact->state = 'publishing';
$artifact->updated_at = $now;
$artifact->save();
});
}
/** @param array<string,mixed> $work */
private function safeFailure(array $work, string $code, int $now): void
{
$this->mutateLeased($work, function (
CloudStorageOperation $operation,
CloudStorageArtifact $artifact,
CloudStorageReplica $replica,
CloudStoragePublication $publication,
array $copyOperations,
array $replicas
) use ($code, $now): void {
$code = self::errorCode($code);
if ((int) $operation->attempts >= self::MAX_DISPATCH_ATTEMPTS) {
$this->settleAttention(
$operation,
$artifact,
$replica,
$publication,
$copyOperations,
$replicas,
false,
'immutable_primary_publish_retry_exhausted',
$now
);
return;
}
$operation->phase = 'accepted';
$operation->effect_resolution = 'not_applicable';
$operation->lease_token = null;
$operation->lease_until = 0;
$operation->available_at = $now + self::RETRY_DELAY;
$operation->next_check_at = 0;
$operation->error_code = $code;
$operation->result_json = self::encodedMetadata(
self::resetEvidence($operation->result_json)
);
$operation->updated_at = $now;
$operation->save();
$replica->state = 'reserved';
$replica->lease_token = null;
$replica->lease_until = 0;
$replica->error_code = $code;
$replica->updated_at = $now;
$replica->save();
});
}
/** @param array<string,mixed> $work */
private function reconcileFailure(
array $work,
string $code,
int $now
): void {
$this->mutateLeased($work, function (
CloudStorageOperation $operation,
CloudStorageArtifact $artifact,
CloudStorageReplica $replica,
CloudStoragePublication $publication,
array $copyOperations,
array $replicas
) use ($code, $now): void {
$metadata = self::metadata($operation->result_json);
$failures = (int) ($metadata['reconcile_failures'] ?? 0) + 1;
if ($failures >= self::MAX_RECONCILE_FAILURES) {
$this->settleAttention(
$operation,
$artifact,
$replica,
$publication,
$copyOperations,
$replicas,
true,
'immutable_primary_reconcile_exhausted',
$now
);
return;
}
$metadata['reconcile_failures'] = $failures;
$operation->phase = 'reconciling';
$operation->effect_resolution = 'unknown';
$operation->lease_token = null;
$operation->lease_until = 0;
$operation->next_check_at = $now + self::RETRY_DELAY;
$operation->error_code = self::errorCode($code);
$operation->result_json = self::encodedMetadata($metadata);
$operation->updated_at = $now;
$operation->save();
$replica->state = 'publishing';
$replica->lease_token = null;
$replica->lease_until = 0;
$replica->error_code = self::errorCode($code);
$replica->updated_at = $now;
$replica->save();
});
}
/** @param array<string,mixed> $work */
private function recordAbsence(array $work, int $now): void
{
$this->mutateLeased($work, function (
CloudStorageOperation $operation,
CloudStorageArtifact $artifact,
CloudStorageReplica $replica,
CloudStoragePublication $publication,
array $copyOperations,
array $replicas
) use ($now): void {
$metadata = self::metadata($operation->result_json);
$confirmations = min(
2,
(int) ($metadata['absence_confirmations'] ?? 0) + 1
);
$metadata['absence_confirmations'] = $confirmations;
$metadata['reconcile_failures'] = 0;
if ($confirmations < 2) {
$operation->phase = 'reconciling';
$operation->effect_resolution = 'unknown';
$operation->lease_token = null;
$operation->lease_until = 0;
$operation->next_check_at = $now + self::RECONCILE_DELAY;
$operation->error_code = 'immutable_primary_absent_once';
$operation->result_json = self::encodedMetadata($metadata);
$operation->updated_at = $now;
$operation->save();
$replica->state = 'publishing';
$replica->lease_token = null;
$replica->lease_until = 0;
$replica->error_code = 'immutable_primary_absent_once';
$replica->updated_at = $now;
$replica->save();
return;
}
if ((int) $operation->attempts >= self::MAX_DISPATCH_ATTEMPTS) {
$this->settleAttention(
$operation,
$artifact,
$replica,
$publication,
$copyOperations,
$replicas,
false,
'immutable_primary_absence_retry_exhausted',
$now
);
return;
}
$operation->phase = 'accepted';
$operation->effect_resolution = 'not_applicable';
$operation->lease_token = null;
$operation->lease_until = 0;
$operation->available_at = $now + self::RECONCILE_DELAY;
$operation->next_check_at = 0;
$operation->error_code = '';
$operation->result_json = self::encodedMetadata(
self::resetEvidence($operation->result_json)
);
$operation->updated_at = $now;
$operation->save();
$replica->state = 'reserved';
$replica->lease_token = null;
$replica->lease_until = 0;
$replica->error_code = '';
$replica->updated_at = $now;
$replica->save();
});
}
/** @param array<string,mixed> $work */
private function attention(array $work, string $code, int $now): void
{
$this->mutateLeased($work, function (
CloudStorageOperation $operation,
CloudStorageArtifact $artifact,
CloudStorageReplica $replica,
CloudStoragePublication $publication,
array $copyOperations,
array $replicas
) use ($code, $now): void {
$this->settleAttention(
$operation,
$artifact,
$replica,
$publication,
$copyOperations,
$replicas,
true,
self::errorCode($code),
$now
);
});
}
/**
* @param array<string,mixed> $work
* @param callable(CloudStorageOperation,CloudStorageArtifact,
* CloudStorageReplica,CloudStoragePublication,
* array<int,CloudStorageOperation>,array<int,CloudStorageReplica>):void $callback
*/
private function mutateLeased(array $work, callable $callback): void
{
$connection = (new CloudStorageOperation())->getConnection();
$connection->transaction(function () use ($work, $callback): void {
$operation = CloudStorageOperation::query()
->where('operation_id', (string) $work['operation_id'])
->lockForUpdate()->firstOrFail();
if ((int) $operation->artifact_id !== (int) $work['artifact_id']) {
throw new CloudError('对象存储发布制品身份已变化', 409);
}
$copyOperations = CloudStorageOperation::query()
->where('artifact_id', (int) $operation->artifact_id)
->where('kind', 'copy')->orderBy('id')
->lockForUpdate()->get()->all();
[$artifact, $replica, $publication, $replicas] =
$this->lockedContext($operation, true);
if (!hash_equals(
(string) $operation->lease_token,
(string) $work['lease_token']
)
|| !hash_equals(
(string) $replica->lease_token,
(string) $work['lease_token']
)) {
throw new CloudError('对象存储主副本发布租约已失效', 409);
}
$callback(
$operation,
$artifact,
$replica,
$publication,
$copyOperations,
$replicas
);
}, 3);
}
/**
* @return array{0:CloudStorageArtifact,1:CloudStorageReplica,
* 2:CloudStoragePublication,3?:array<int,CloudStorageReplica>}
*/
private function lockedContext(
CloudStorageOperation $operation,
bool $includeReplicas = false
): array {
$artifact = CloudStorageArtifact::query()
->whereKey((int) $operation->artifact_id)
->lockForUpdate()->firstOrFail();
$replicas = CloudStorageReplica::query()
->where('artifact_id', (int) $artifact->id)->orderBy('id')
->lockForUpdate()->get()->all();
$replica = null;
foreach ($replicas as $candidate) {
if ((int) $candidate->id === (int) $operation->replica_id) {
$replica = $candidate;
break;
}
}
$publication = CloudStoragePublication::query()
->where('artifact_id', (int) $artifact->id)
->lockForUpdate()->firstOrFail();
if (!$replica instanceof CloudStorageReplica
|| (string) $replica->role !== 'primary') {
throw new CloudError('对象存储主副本账本不一致', 409);
}
return $includeReplicas
? [$artifact, $replica, $publication, $replicas]
: [$artifact, $replica, $publication];
}
/**
* @param array<int,CloudStorageOperation> $copyOperations
* @param array<int,CloudStorageReplica> $replicas
*/
private function settleAttention(
CloudStorageOperation $operation,
CloudStorageArtifact $artifact,
CloudStorageReplica $replica,
CloudStoragePublication $publication,
array $copyOperations,
array $replicas,
bool $effectUnknown,
string $code,
int $now
): void {
$operation->phase = 'terminal';
$operation->outcome = 'failed';
$operation->effect_resolution = $effectUnknown
? 'unknown' : 'not_applied';
$operation->effect_may_have_applied = $effectUnknown ? 1 : 0;
$operation->safe_to_resubmit = 0;
$operation->lease_token = null;
$operation->lease_until = 0;
$operation->available_at = 0;
$operation->next_check_at = 0;
$operation->error_code = self::errorCode($code);
$operation->terminal_at = $now;
$operation->updated_at = $now;
$operation->save();
$replica->state = 'degraded';
$replica->lease_token = null;
$replica->lease_until = 0;
$replica->error_code = self::errorCode($code);
$replica->updated_at = $now;
$replica->save();
foreach ($copyOperations as $copy) {
if ((string) $copy->phase === 'terminal') {
continue;
}
// Secondary work is blocked until primary success. Never invent a
// safe non-dispatch result if an inconsistent ledger says a copy
// has already crossed its own dispatch fence.
if ((string) $copy->phase !== 'accepted') {
continue;
}
$copy->phase = 'terminal';
$copy->outcome = 'failed';
$copy->effect_resolution = 'not_applied';
$copy->effect_may_have_applied = 0;
$copy->safe_to_resubmit = 0;
$copy->lease_token = null;
$copy->lease_until = 0;
$copy->available_at = 0;
$copy->next_check_at = 0;
$copy->error_code = 'primary_publish_attention';
$copy->terminal_at = $now;
$copy->updated_at = $now;
$copy->save();
foreach ($replicas as $secondary) {
if ((int) $secondary->id !== (int) $copy->replica_id) {
continue;
}
$secondary->state = 'degraded';
$secondary->lease_token = null;
$secondary->lease_until = 0;
$secondary->error_code = 'primary_publish_attention';
$secondary->updated_at = $now;
$secondary->save();
break;
}
}
$artifact->state = 'degraded';
$artifact->updated_at = $now;
$artifact->save();
$publication->state = 'failed';
$publication->available_at = 0;
$publication->lease_token = null;
$publication->lease_until = 0;
$publication->last_error_code = self::errorCode($code);
$publication->updated_at = $now;
$publication->save();
}
/**
* @return array{absence_confirmations:int,reconcile_failures:int,
* maximum_size_bytes?:int,object_suffix?:string}
*/
private static function metadata(?string $json): array
{
if ($json === null || trim($json) === '') {
return [
'absence_confirmations' => 0,
'reconcile_failures' => 0,
];
}
try {
$value = json_decode($json, true, 16, JSON_THROW_ON_ERROR);
} catch (\Throwable) {
throw new CloudError('对象存储发布核对证据无效', 409);
}
if (!is_array($value) || array_is_list($value)
|| array_diff(array_keys($value), [
'absence_confirmations', 'reconcile_failures',
'maximum_size_bytes', 'object_suffix',
]) !== []) {
throw new CloudError('对象存储发布核对证据无效', 409);
}
$absence = $value['absence_confirmations'] ?? 0;
$failures = $value['reconcile_failures'] ?? 0;
if (!is_int($absence) || $absence < 0 || $absence > 2
|| !is_int($failures) || $failures < 0
|| $failures > self::MAX_RECONCILE_FAILURES) {
throw new CloudError('对象存储发布核对证据无效', 409);
}
$metadata = [
'absence_confirmations' => $absence,
'reconcile_failures' => $failures,
];
if (array_key_exists('maximum_size_bytes', $value)) {
$maximum = $value['maximum_size_bytes'];
if (!is_int($maximum) || $maximum < 1) {
throw new CloudError('对象存储发布核对证据无效', 409);
}
$metadata['maximum_size_bytes'] = $maximum;
}
if (array_key_exists('object_suffix', $value)) {
$suffix = $value['object_suffix'];
if (!is_string($suffix) || $suffix === ''
|| strlen($suffix) > 512
|| preg_match('/[\x00-\x1f\x7f]/', $suffix) === 1) {
throw new CloudError('对象存储发布核对证据无效', 409);
}
$metadata['object_suffix'] = $suffix;
}
return $metadata;
}
private static function encodedMetadata(array $metadata): string
{
$normalized = self::metadata(json_encode(
$metadata,
JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES
));
return json_encode(
$normalized,
JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES
);
}
/**
* Preserve beginPublication()'s immutable context while clearing only
* this module's transient reconciliation evidence.
*
* @return array{absence_confirmations:int,reconcile_failures:int,
* maximum_size_bytes?:int,object_suffix?:string}
*/
private static function resetEvidence(?string $json): array
{
$metadata = self::metadata($json);
$metadata['absence_confirmations'] = 0;
$metadata['reconcile_failures'] = 0;
return $metadata;
}
/** @param array<string,mixed> $file @param array<string,mixed> $snapshot */
private static function assertFile(array $file, array $snapshot): void
{
$path = $file['path'] ?? null;
$size = $file['size_bytes'] ?? null;
$sha = $file['sha256'] ?? null;
$actual = is_string($path) && is_file($path) && is_readable($path)
? hash_file('sha256', $path) : false;
if (!is_string($path) || !is_int($size) || $size < 1
|| !is_string($sha)
|| preg_match('/^[a-f0-9]{64}$/D', $sha) !== 1
|| @filesize($path) !== $size
|| !is_string($actual) || !hash_equals($sha, $actual)
|| $size !== (int) $snapshot['size_bytes']
|| !hash_equals($sha, (string) $snapshot['sha256'])) {
throw new CloudError('本地不可变制品文件身份无效', 422);
}
}
/** @param array<string,mixed> $runtime */
private static function assertRuntime(array $runtime): void
{
if (!is_array($runtime['config'] ?? null)
|| array_is_list($runtime['config'])
|| !is_array($runtime['credentials'] ?? null)
|| array_is_list($runtime['credentials'])
|| !is_int($runtime['revision'] ?? null)
|| (int) $runtime['revision'] < 1) {
throw new CloudError('对象存储发布运行时无效', 503);
}
}
private static function assertOperationId(string $operationId): void
{
if (preg_match('/^[a-f0-9]{32}$/D', $operationId) !== 1) {
throw new CloudError('对象存储发布操作标识无效', 422);
}
}
private static function errorCode(string $code): string
{
$code = strtolower(trim($code));
return preg_match('/^[a-z][a-z0-9_]{0,95}$/D', $code) === 1
? $code : 'immutable_primary_publication_failed';
}
private function s3(): CloudObjectStorageS3Client
{
return $this->client ?? new CloudObjectStorageS3Client();
}
}