You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
1109 lines
43 KiB
PHP
1109 lines
43 KiB
PHP
<?php
|
|
|
|
namespace App\Services\Cloud;
|
|
|
|
use App\Models\CloudStorageArtifact;
|
|
use App\Models\CloudStorageOperation;
|
|
use App\Models\CloudStoragePublication;
|
|
use App\Models\CloudStorageReplica;
|
|
|
|
/**
|
|
* Publishes one known local file through an already-reserved primary intent.
|
|
*
|
|
* The business owner must first call CloudStorageArtifactService::
|
|
* beginPublication(). This module never allocates a replacement artifact or
|
|
* object key. A lost PUT receipt is reconciled only with exact-key HEAD
|
|
* evidence; the same operation becomes dispatchable again only after two
|
|
* spaced, stable absence observations.
|
|
*/
|
|
final class CloudStorageImmutablePrimaryPublication
|
|
{
|
|
/** Secondary work remains dormant until the primary is verified again. */
|
|
private const COPY_BLOCKED_UNTIL_PRIMARY = 253402300799;
|
|
private const LEASE_SECONDS = 300;
|
|
private const RECONCILE_DELAY = 60;
|
|
private const RETRY_DELAY = 300;
|
|
private const MAX_DISPATCH_ATTEMPTS = 3;
|
|
private const MAX_RECONCILE_FAILURES = 6;
|
|
|
|
/** @var list<string> */
|
|
private const UNKNOWN_ATTENTION_ERRORS = [
|
|
'immutable_primary_identity_conflict',
|
|
'immutable_primary_reconcile_exhausted',
|
|
];
|
|
|
|
/** @var list<string> */
|
|
private const NOT_APPLIED_ATTENTION_ERRORS = [
|
|
'immutable_primary_publish_retry_exhausted',
|
|
'immutable_primary_absence_retry_exhausted',
|
|
];
|
|
|
|
/** @var \Closure(int,bool):array<string,mixed> */
|
|
private readonly \Closure $runtimeResolver;
|
|
|
|
/** @var \Closure():int */
|
|
private readonly \Closure $clock;
|
|
|
|
/**
|
|
* @param null|callable(int,bool):array<string,mixed> $runtimeResolver
|
|
* @param null|callable():int $clock
|
|
*/
|
|
public function __construct(
|
|
private readonly ?CloudObjectStorageS3Client $client = null,
|
|
?callable $runtimeResolver = null,
|
|
?callable $clock = null
|
|
) {
|
|
$this->runtimeResolver = $runtimeResolver === null
|
|
? static fn (int $backendId, bool $healthy): array =>
|
|
CloudObjectStorageBackendRegistry::runtime(
|
|
$backendId,
|
|
$healthy
|
|
)
|
|
: \Closure::fromCallable($runtimeResolver);
|
|
$this->clock = $clock === null
|
|
? static fn (): int => time()
|
|
: \Closure::fromCallable($clock);
|
|
}
|
|
|
|
/** @return list<string> */
|
|
public static function attentionErrors(): array
|
|
{
|
|
return array_merge(
|
|
self::UNKNOWN_ATTENTION_ERRORS,
|
|
self::NOT_APPLIED_ATTENTION_ERRORS
|
|
);
|
|
}
|
|
|
|
/** @return list<string> */
|
|
public static function unknownAttentionErrors(): array
|
|
{
|
|
return self::UNKNOWN_ATTENTION_ERRORS;
|
|
}
|
|
|
|
/** @return list<string> */
|
|
public static function notAppliedAttentionErrors(): array
|
|
{
|
|
return self::NOT_APPLIED_ATTENTION_ERRORS;
|
|
}
|
|
|
|
/**
|
|
* Re-open only the original frozen publish operation.
|
|
*
|
|
* Unknown effects resume with an exact-key HEAD. A provider write becomes
|
|
* dispatchable again only after this class observes two spaced absences.
|
|
* Proven non-application may dispatch the same operation again. Secondary
|
|
* copies stopped solely by the primary attention state are restored behind
|
|
* the primary fence; no new artifact, key or operation is allocated.
|
|
*/
|
|
public static function reopenAttention(
|
|
string $operationId,
|
|
int $now
|
|
): void {
|
|
self::assertOperationId($operationId);
|
|
if ($now < 1) {
|
|
throw new \InvalidArgumentException('Invalid attention time');
|
|
}
|
|
$connection = (new CloudStorageOperation())->getConnection();
|
|
$connection->transaction(static function () use (
|
|
$operationId,
|
|
$now
|
|
): void {
|
|
$operation = CloudStorageOperation::query()
|
|
->where('operation_id', $operationId)
|
|
->lockForUpdate()->first();
|
|
if (!$operation instanceof CloudStorageOperation
|
|
|| (string) $operation->kind !== 'publish') {
|
|
throw new CloudError('对象存储发布核对操作不存在', 404);
|
|
}
|
|
$copyOperations = CloudStorageOperation::query()
|
|
->where('artifact_id', (int) $operation->artifact_id)
|
|
->where('kind', 'copy')->orderBy('id')
|
|
->lockForUpdate()->get()->all();
|
|
$artifact = CloudStorageArtifact::query()
|
|
->whereKey((int) $operation->artifact_id)
|
|
->lockForUpdate()->first();
|
|
$replicas = CloudStorageReplica::query()
|
|
->where('artifact_id', (int) $operation->artifact_id)
|
|
->orderBy('id')->lockForUpdate()->get()->all();
|
|
$publication = CloudStoragePublication::query()
|
|
->where('artifact_id', (int) $operation->artifact_id)
|
|
->lockForUpdate()->first();
|
|
if (!$artifact instanceof CloudStorageArtifact
|
|
|| !$publication instanceof CloudStoragePublication
|
|
|| $publication->backup_id !== null
|
|
|| (($publication->template_import_intent_id === null)
|
|
=== ($publication->rescue_template_id === null))) {
|
|
throw new CloudError('对象存储发布核对归属不一致', 409);
|
|
}
|
|
$primary = null;
|
|
$replicasById = [];
|
|
foreach ($replicas as $replica) {
|
|
$replicasById[(int) $replica->id] = $replica;
|
|
if ((int) $replica->id === (int) $operation->replica_id) {
|
|
$primary = $replica;
|
|
}
|
|
}
|
|
$errorCode = (string) $operation->error_code;
|
|
$unknown = in_array(
|
|
$errorCode,
|
|
self::UNKNOWN_ATTENTION_ERRORS,
|
|
true
|
|
);
|
|
$notApplied = in_array(
|
|
$errorCode,
|
|
self::NOT_APPLIED_ATTENTION_ERRORS,
|
|
true
|
|
);
|
|
if (!$primary instanceof CloudStorageReplica
|
|
|| (string) $primary->role !== 'primary'
|
|
|| (!$unknown && !$notApplied)
|
|
|| (string) $operation->phase !== 'terminal'
|
|
|| (string) $operation->outcome !== 'failed'
|
|
|| (bool) $operation->safe_to_resubmit
|
|
|| (bool) $operation->effect_may_have_applied !== $unknown
|
|
|| (string) $operation->effect_resolution
|
|
!== ($unknown ? 'unknown' : 'not_applied')
|
|
|| (string) $artifact->state !== 'degraded'
|
|
|| (string) $primary->state !== 'degraded'
|
|
|| !hash_equals($errorCode, (string) $primary->error_code)
|
|
|| (string) $publication->state !== 'failed'
|
|
|| !hash_equals(
|
|
$errorCode,
|
|
(string) $publication->last_error_code
|
|
)) {
|
|
throw new CloudError(
|
|
'对象存储主副本不在可重新核对的人工处理终态',
|
|
409
|
|
);
|
|
}
|
|
foreach ($copyOperations as $copy) {
|
|
$secondary = $replicasById[(int) $copy->replica_id] ?? null;
|
|
if (!$secondary instanceof CloudStorageReplica
|
|
|| (string) $secondary->role !== 'secondary'
|
|
|| (string) $copy->phase !== 'terminal'
|
|
|| (string) $copy->outcome !== 'failed'
|
|
|| (string) $copy->effect_resolution !== 'not_applied'
|
|
|| (bool) $copy->effect_may_have_applied
|
|
|| (bool) $copy->safe_to_resubmit
|
|
|| (int) $copy->attempts !== 0
|
|
|| (string) $copy->error_code
|
|
!== 'primary_publish_attention'
|
|
|| (string) $secondary->state !== 'degraded'
|
|
|| (string) $secondary->error_code
|
|
!== 'primary_publish_attention') {
|
|
throw new CloudError(
|
|
'对象存储次副本核对账本不一致',
|
|
409
|
|
);
|
|
}
|
|
}
|
|
|
|
$metadata = self::resetEvidence($operation->result_json);
|
|
$operation->phase = $unknown ? 'reconciling' : 'accepted';
|
|
$operation->outcome = null;
|
|
$operation->effect_resolution = $unknown
|
|
? 'unknown' : 'not_applicable';
|
|
$operation->effect_may_have_applied = null;
|
|
$operation->safe_to_resubmit = null;
|
|
$operation->lease_token = null;
|
|
$operation->lease_until = 0;
|
|
$operation->available_at = $unknown ? 0 : $now;
|
|
$operation->next_check_at = $unknown ? $now : 0;
|
|
$operation->error_code = $unknown
|
|
? 'admin_reconcile_requested'
|
|
: 'admin_retry_requested';
|
|
$operation->result_json = self::encodedMetadata($metadata);
|
|
$operation->terminal_at = null;
|
|
$operation->updated_at = $now;
|
|
$operation->save();
|
|
|
|
$primary->state = $unknown ? 'publishing' : 'reserved';
|
|
$primary->lease_token = null;
|
|
$primary->lease_until = 0;
|
|
$primary->error_code = $operation->error_code;
|
|
$primary->updated_at = $now;
|
|
$primary->save();
|
|
|
|
foreach ($copyOperations as $copy) {
|
|
/** @var CloudStorageReplica $secondary */
|
|
$secondary = $replicasById[(int) $copy->replica_id];
|
|
$copy->phase = 'accepted';
|
|
$copy->outcome = null;
|
|
$copy->effect_resolution = 'not_applicable';
|
|
$copy->effect_may_have_applied = null;
|
|
$copy->safe_to_resubmit = null;
|
|
$copy->lease_token = null;
|
|
$copy->lease_until = 0;
|
|
$copy->available_at = self::COPY_BLOCKED_UNTIL_PRIMARY;
|
|
$copy->next_check_at = 0;
|
|
$copy->error_code = '';
|
|
$copy->terminal_at = null;
|
|
$copy->updated_at = $now;
|
|
$copy->save();
|
|
$secondary->state = 'reserved';
|
|
$secondary->lease_token = null;
|
|
$secondary->lease_until = 0;
|
|
$secondary->error_code = '';
|
|
$secondary->updated_at = $now;
|
|
$secondary->save();
|
|
}
|
|
|
|
$artifact->state = 'publishing';
|
|
$artifact->updated_at = $now;
|
|
$artifact->save();
|
|
$publication->state = 'waiting_primary';
|
|
$publication->attempts = 0;
|
|
$publication->available_at = 0;
|
|
$publication->lease_token = null;
|
|
$publication->lease_until = 0;
|
|
$publication->last_error_code = '';
|
|
$publication->coverage_ready_at = null;
|
|
$publication->finalized_at = null;
|
|
$publication->updated_at = $now;
|
|
$publication->save();
|
|
}, 3);
|
|
}
|
|
|
|
/**
|
|
* Advance one immutable primary publication by at most one remote action.
|
|
*
|
|
* @param array{path:string,size_bytes:int,sha256:string} $file
|
|
* @return array<string,mixed>
|
|
*/
|
|
public function publish(
|
|
string $operationId,
|
|
int $expectedArtifactId,
|
|
array $file
|
|
): array {
|
|
self::assertOperationId($operationId);
|
|
if ($expectedArtifactId < 1) {
|
|
throw new CloudError('对象存储制品身份无效', 422);
|
|
}
|
|
$snapshot = $this->snapshot($operationId, $expectedArtifactId);
|
|
if (($snapshot['terminal'] ?? false) === true) {
|
|
return $snapshot;
|
|
}
|
|
// Reconciliation is intentionally independent of the temporary local
|
|
// file. Once a PUT may have happened, only the frozen ledger identity
|
|
// and an exact-key HEAD are authoritative.
|
|
if ((string) $snapshot['phase'] === 'accepted') {
|
|
self::assertFile($file, $snapshot);
|
|
}
|
|
$now = ($this->clock)();
|
|
$work = $this->claim(
|
|
$operationId,
|
|
$expectedArtifactId,
|
|
$now
|
|
);
|
|
if ($work === null) {
|
|
return $this->snapshot($operationId, $expectedArtifactId);
|
|
}
|
|
if ((string) $work['mode'] === 'dispatch') {
|
|
try {
|
|
// Recheck after the dispatch lease is owned so a changed
|
|
// source can never reach the provider under this intent.
|
|
self::assertFile($file, $work);
|
|
} catch (\Throwable $error) {
|
|
$this->safeFailure(
|
|
$work,
|
|
'immutable_primary_source_changed',
|
|
$now
|
|
);
|
|
throw $error;
|
|
}
|
|
}
|
|
try {
|
|
$runtime = ($this->runtimeResolver)(
|
|
(int) $work['backend_id'],
|
|
(string) $work['mode'] === 'dispatch'
|
|
);
|
|
self::assertRuntime($runtime);
|
|
} catch (\Throwable) {
|
|
if ((string) $work['mode'] === 'dispatch') {
|
|
$this->safeFailure(
|
|
$work,
|
|
'immutable_primary_runtime_unavailable',
|
|
$now
|
|
);
|
|
} else {
|
|
$this->reconcileFailure(
|
|
$work,
|
|
'immutable_primary_runtime_unavailable',
|
|
$now
|
|
);
|
|
}
|
|
return $this->snapshot($operationId, $expectedArtifactId);
|
|
}
|
|
|
|
if ((string) $work['mode'] === 'dispatch') {
|
|
$this->dispatch($work, $runtime, $file);
|
|
} else {
|
|
$this->reconcile($work, $runtime);
|
|
}
|
|
return $this->snapshot($operationId, $expectedArtifactId);
|
|
}
|
|
|
|
/** @return array<string,mixed> */
|
|
private function snapshot(
|
|
string $operationId,
|
|
int $expectedArtifactId
|
|
): array {
|
|
$operation = CloudStorageOperation::query()
|
|
->where('operation_id', $operationId)->first();
|
|
if (!$operation instanceof CloudStorageOperation
|
|
|| (string) $operation->kind !== 'publish'
|
|
|| (int) $operation->artifact_id !== $expectedArtifactId) {
|
|
throw new CloudError('对象存储发布操作与制品身份不一致', 409);
|
|
}
|
|
$artifact = CloudStorageArtifact::query()
|
|
->whereKey($expectedArtifactId)->first();
|
|
$replica = CloudStorageReplica::query()
|
|
->whereKey((int) $operation->replica_id)->first();
|
|
$publication = CloudStoragePublication::query()
|
|
->where('artifact_id', $expectedArtifactId)->first();
|
|
if (!$artifact instanceof CloudStorageArtifact
|
|
|| !$replica instanceof CloudStorageReplica
|
|
|| !$publication instanceof CloudStoragePublication
|
|
|| (int) $replica->artifact_id !== $expectedArtifactId
|
|
|| (string) $replica->role !== 'primary') {
|
|
throw new CloudError('对象存储主副本账本不一致', 409);
|
|
}
|
|
$terminal = (string) $operation->phase === 'terminal';
|
|
return [
|
|
'operation_id' => (string) $operation->operation_id,
|
|
'artifact_id' => (int) $artifact->id,
|
|
'phase' => (string) $operation->phase,
|
|
'outcome' => $operation->outcome === null
|
|
? null : (string) $operation->outcome,
|
|
'effect_resolution' => (string) $operation->effect_resolution,
|
|
'attempts' => (int) $operation->attempts,
|
|
'next_check_at' => (int) $operation->next_check_at,
|
|
'error_code' => (string) $operation->error_code,
|
|
'terminal' => $terminal,
|
|
'primary_completed' => $terminal
|
|
&& (string) $operation->outcome === 'succeeded',
|
|
'publication_state' => (string) $publication->state,
|
|
'ready_to_finalize' => in_array(
|
|
(string) $publication->state,
|
|
['ready_to_finalize', 'finalizing', 'completed'],
|
|
true
|
|
),
|
|
'requires_attention' => ($terminal
|
|
&& (string) $operation->outcome === 'failed')
|
|
|| (string) $publication->state === 'failed',
|
|
'object_key' => (string) $replica->object_key,
|
|
'size_bytes' => (int) $artifact->size_bytes,
|
|
'sha256' => (string) $artifact->sha256,
|
|
];
|
|
}
|
|
|
|
/** @return array<string,mixed>|null */
|
|
private function claim(
|
|
string $operationId,
|
|
int $expectedArtifactId,
|
|
int $now
|
|
): ?array {
|
|
$connection = (new CloudStorageOperation())->getConnection();
|
|
return $connection->transaction(function () use (
|
|
$operationId,
|
|
$expectedArtifactId,
|
|
$now
|
|
): ?array {
|
|
$operation = CloudStorageOperation::query()
|
|
->where('operation_id', $operationId)
|
|
->lockForUpdate()->first();
|
|
if (!$operation instanceof CloudStorageOperation
|
|
|| (string) $operation->kind !== 'publish'
|
|
|| (int) $operation->artifact_id !== $expectedArtifactId) {
|
|
throw new CloudError(
|
|
'对象存储发布操作与制品身份不一致',
|
|
409
|
|
);
|
|
}
|
|
[$artifact, $replica, $publication] =
|
|
$this->lockedContext($operation);
|
|
if ((string) $operation->phase === 'terminal'
|
|
|| in_array((string) $publication->state,
|
|
['completed', 'failed', 'retired'], true)) {
|
|
return null;
|
|
}
|
|
if ((string) $artifact->state !== 'publishing'
|
|
|| !in_array((string) $replica->state,
|
|
['reserved', 'publishing'], true)) {
|
|
throw new CloudError('对象存储主副本发布状态不一致', 409);
|
|
}
|
|
if (is_string($operation->lease_token)
|
|
&& $operation->lease_token !== ''
|
|
&& (int) $operation->lease_until > $now) {
|
|
return null;
|
|
}
|
|
|
|
$phase = (string) $operation->phase;
|
|
$mode = null;
|
|
if ($phase === 'accepted'
|
|
&& (int) $operation->available_at <= $now) {
|
|
$mode = 'dispatch';
|
|
$operation->phase = 'dispatching';
|
|
$operation->effect_resolution = 'not_applicable';
|
|
$operation->attempts = (int) $operation->attempts + 1;
|
|
$replica->state = 'publishing';
|
|
} elseif ($phase === 'reconciling'
|
|
&& (int) $operation->next_check_at <= $now) {
|
|
$mode = 'reconcile';
|
|
} elseif (in_array($phase,
|
|
['dispatching', 'awaiting_receipt'], true)
|
|
&& (int) $operation->lease_until <= $now) {
|
|
$mode = 'reconcile';
|
|
$operation->phase = 'reconciling';
|
|
$operation->effect_resolution = 'unknown';
|
|
$operation->error_code =
|
|
'immutable_primary_lease_expired';
|
|
$operation->next_check_at = $now;
|
|
}
|
|
if ($mode === null) {
|
|
return null;
|
|
}
|
|
$lease = bin2hex(random_bytes(16));
|
|
$operation->lease_token = $lease;
|
|
$operation->lease_until = $now + self::LEASE_SECONDS;
|
|
$operation->updated_at = $now;
|
|
$operation->save();
|
|
$replica->lease_token = $lease;
|
|
$replica->lease_until = $now + self::LEASE_SECONDS;
|
|
$replica->updated_at = $now;
|
|
$replica->save();
|
|
return [
|
|
'mode' => $mode,
|
|
'operation_id' => (string) $operation->operation_id,
|
|
'artifact_id' => (int) $artifact->id,
|
|
'replica_id' => (int) $replica->id,
|
|
'backend_id' => (int) $replica->backend_id,
|
|
'object_key' => (string) $replica->object_key,
|
|
'size_bytes' => (int) $artifact->size_bytes,
|
|
'sha256' => (string) $artifact->sha256,
|
|
'lease_token' => $lease,
|
|
'attempts' => (int) $operation->attempts,
|
|
];
|
|
}, 3);
|
|
}
|
|
|
|
/**
|
|
* @param array<string,mixed> $work
|
|
* @param array<string,mixed> $runtime
|
|
* @param array{path:string,size_bytes:int,sha256:string} $file
|
|
*/
|
|
private function dispatch(array $work, array $runtime, array $file): void
|
|
{
|
|
try {
|
|
$identity = $this->s3()->putImmutableFile(
|
|
$runtime['config'],
|
|
$runtime['credentials'],
|
|
(string) $work['object_key'],
|
|
(string) $file['path'],
|
|
(int) $file['size_bytes'],
|
|
(string) $file['sha256']
|
|
);
|
|
} catch (CloudObjectStorageTransferError $error) {
|
|
if ($error->effectMayHaveApplied
|
|
|| $error->publicCode === 'object_already_exists') {
|
|
$this->markUnknown($work, $error->publicCode);
|
|
} else {
|
|
$this->safeFailure(
|
|
$work,
|
|
$error->publicCode,
|
|
($this->clock)()
|
|
);
|
|
}
|
|
return;
|
|
} catch (\Throwable) {
|
|
$this->markUnknown($work, 'immutable_primary_put_unknown');
|
|
return;
|
|
}
|
|
if ((int) ($identity['size_bytes'] ?? 0)
|
|
!== (int) $work['size_bytes']
|
|
|| !is_string($identity['etag'] ?? null)
|
|
|| !is_string($identity['version_id'] ?? null)) {
|
|
$this->markUnknown(
|
|
$work,
|
|
'immutable_primary_put_receipt_invalid'
|
|
);
|
|
return;
|
|
}
|
|
CloudStorageArtifactService::completePrimary(
|
|
(string) $work['operation_id'],
|
|
(int) $work['artifact_id'],
|
|
[
|
|
'size_bytes' => (int) $work['size_bytes'],
|
|
'sha256' => (string) $work['sha256'],
|
|
'etag' => (string) $identity['etag'],
|
|
'version_id' => (string) $identity['version_id'],
|
|
]
|
|
);
|
|
}
|
|
|
|
/** @param array<string,mixed> $work @param array<string,mixed> $runtime */
|
|
private function reconcile(array $work, array $runtime): void
|
|
{
|
|
try {
|
|
$identity = $this->s3()->inspectObject(
|
|
$runtime['config'],
|
|
$runtime['credentials'],
|
|
(string) $work['object_key']
|
|
);
|
|
} catch (\Throwable) {
|
|
$this->reconcileFailure(
|
|
$work,
|
|
'immutable_primary_head_unavailable',
|
|
($this->clock)()
|
|
);
|
|
return;
|
|
}
|
|
if ($identity === null) {
|
|
$this->recordAbsence($work, ($this->clock)());
|
|
return;
|
|
}
|
|
if ((int) ($identity['size_bytes'] ?? 0)
|
|
!== (int) $work['size_bytes']
|
|
|| !is_string($identity['sha256'] ?? null)
|
|
|| !hash_equals(
|
|
(string) $work['sha256'],
|
|
(string) $identity['sha256']
|
|
)) {
|
|
$this->attention(
|
|
$work,
|
|
'immutable_primary_identity_conflict',
|
|
($this->clock)()
|
|
);
|
|
return;
|
|
}
|
|
CloudStorageArtifactService::completePrimary(
|
|
(string) $work['operation_id'],
|
|
(int) $work['artifact_id'],
|
|
[
|
|
'size_bytes' => (int) $work['size_bytes'],
|
|
'sha256' => (string) $work['sha256'],
|
|
'etag' => (string) ($identity['etag'] ?? ''),
|
|
'version_id' => (string) ($identity['version_id'] ?? ''),
|
|
]
|
|
);
|
|
}
|
|
|
|
/** @param array<string,mixed> $work */
|
|
private function markUnknown(array $work, string $code): void
|
|
{
|
|
$now = ($this->clock)();
|
|
$this->mutateLeased($work, function (
|
|
CloudStorageOperation $operation,
|
|
CloudStorageArtifact $artifact,
|
|
CloudStorageReplica $replica,
|
|
CloudStoragePublication $publication,
|
|
array $copyOperations,
|
|
array $replicas
|
|
) use ($code, $now): void {
|
|
$metadata = self::metadata($operation->result_json);
|
|
$metadata['absence_confirmations'] = 0;
|
|
$operation->phase = 'reconciling';
|
|
$operation->effect_resolution = 'unknown';
|
|
$operation->lease_token = null;
|
|
$operation->lease_until = 0;
|
|
$operation->next_check_at = $now + self::RECONCILE_DELAY;
|
|
$operation->error_code = self::errorCode($code);
|
|
$operation->result_json = self::encodedMetadata($metadata);
|
|
$operation->updated_at = $now;
|
|
$operation->save();
|
|
$replica->state = 'publishing';
|
|
$replica->lease_token = null;
|
|
$replica->lease_until = 0;
|
|
$replica->error_code = self::errorCode($code);
|
|
$replica->updated_at = $now;
|
|
$replica->save();
|
|
$artifact->state = 'publishing';
|
|
$artifact->updated_at = $now;
|
|
$artifact->save();
|
|
});
|
|
}
|
|
|
|
/** @param array<string,mixed> $work */
|
|
private function safeFailure(array $work, string $code, int $now): void
|
|
{
|
|
$this->mutateLeased($work, function (
|
|
CloudStorageOperation $operation,
|
|
CloudStorageArtifact $artifact,
|
|
CloudStorageReplica $replica,
|
|
CloudStoragePublication $publication,
|
|
array $copyOperations,
|
|
array $replicas
|
|
) use ($code, $now): void {
|
|
$code = self::errorCode($code);
|
|
if ((int) $operation->attempts >= self::MAX_DISPATCH_ATTEMPTS) {
|
|
$this->settleAttention(
|
|
$operation,
|
|
$artifact,
|
|
$replica,
|
|
$publication,
|
|
$copyOperations,
|
|
$replicas,
|
|
false,
|
|
'immutable_primary_publish_retry_exhausted',
|
|
$now
|
|
);
|
|
return;
|
|
}
|
|
$operation->phase = 'accepted';
|
|
$operation->effect_resolution = 'not_applicable';
|
|
$operation->lease_token = null;
|
|
$operation->lease_until = 0;
|
|
$operation->available_at = $now + self::RETRY_DELAY;
|
|
$operation->next_check_at = 0;
|
|
$operation->error_code = $code;
|
|
$operation->result_json = self::encodedMetadata(
|
|
self::resetEvidence($operation->result_json)
|
|
);
|
|
$operation->updated_at = $now;
|
|
$operation->save();
|
|
$replica->state = 'reserved';
|
|
$replica->lease_token = null;
|
|
$replica->lease_until = 0;
|
|
$replica->error_code = $code;
|
|
$replica->updated_at = $now;
|
|
$replica->save();
|
|
});
|
|
}
|
|
|
|
/** @param array<string,mixed> $work */
|
|
private function reconcileFailure(
|
|
array $work,
|
|
string $code,
|
|
int $now
|
|
): void {
|
|
$this->mutateLeased($work, function (
|
|
CloudStorageOperation $operation,
|
|
CloudStorageArtifact $artifact,
|
|
CloudStorageReplica $replica,
|
|
CloudStoragePublication $publication,
|
|
array $copyOperations,
|
|
array $replicas
|
|
) use ($code, $now): void {
|
|
$metadata = self::metadata($operation->result_json);
|
|
$failures = (int) ($metadata['reconcile_failures'] ?? 0) + 1;
|
|
if ($failures >= self::MAX_RECONCILE_FAILURES) {
|
|
$this->settleAttention(
|
|
$operation,
|
|
$artifact,
|
|
$replica,
|
|
$publication,
|
|
$copyOperations,
|
|
$replicas,
|
|
true,
|
|
'immutable_primary_reconcile_exhausted',
|
|
$now
|
|
);
|
|
return;
|
|
}
|
|
$metadata['reconcile_failures'] = $failures;
|
|
$operation->phase = 'reconciling';
|
|
$operation->effect_resolution = 'unknown';
|
|
$operation->lease_token = null;
|
|
$operation->lease_until = 0;
|
|
$operation->next_check_at = $now + self::RETRY_DELAY;
|
|
$operation->error_code = self::errorCode($code);
|
|
$operation->result_json = self::encodedMetadata($metadata);
|
|
$operation->updated_at = $now;
|
|
$operation->save();
|
|
$replica->state = 'publishing';
|
|
$replica->lease_token = null;
|
|
$replica->lease_until = 0;
|
|
$replica->error_code = self::errorCode($code);
|
|
$replica->updated_at = $now;
|
|
$replica->save();
|
|
});
|
|
}
|
|
|
|
/** @param array<string,mixed> $work */
|
|
private function recordAbsence(array $work, int $now): void
|
|
{
|
|
$this->mutateLeased($work, function (
|
|
CloudStorageOperation $operation,
|
|
CloudStorageArtifact $artifact,
|
|
CloudStorageReplica $replica,
|
|
CloudStoragePublication $publication,
|
|
array $copyOperations,
|
|
array $replicas
|
|
) use ($now): void {
|
|
$metadata = self::metadata($operation->result_json);
|
|
$confirmations = min(
|
|
2,
|
|
(int) ($metadata['absence_confirmations'] ?? 0) + 1
|
|
);
|
|
$metadata['absence_confirmations'] = $confirmations;
|
|
$metadata['reconcile_failures'] = 0;
|
|
if ($confirmations < 2) {
|
|
$operation->phase = 'reconciling';
|
|
$operation->effect_resolution = 'unknown';
|
|
$operation->lease_token = null;
|
|
$operation->lease_until = 0;
|
|
$operation->next_check_at = $now + self::RECONCILE_DELAY;
|
|
$operation->error_code = 'immutable_primary_absent_once';
|
|
$operation->result_json = self::encodedMetadata($metadata);
|
|
$operation->updated_at = $now;
|
|
$operation->save();
|
|
$replica->state = 'publishing';
|
|
$replica->lease_token = null;
|
|
$replica->lease_until = 0;
|
|
$replica->error_code = 'immutable_primary_absent_once';
|
|
$replica->updated_at = $now;
|
|
$replica->save();
|
|
return;
|
|
}
|
|
if ((int) $operation->attempts >= self::MAX_DISPATCH_ATTEMPTS) {
|
|
$this->settleAttention(
|
|
$operation,
|
|
$artifact,
|
|
$replica,
|
|
$publication,
|
|
$copyOperations,
|
|
$replicas,
|
|
false,
|
|
'immutable_primary_absence_retry_exhausted',
|
|
$now
|
|
);
|
|
return;
|
|
}
|
|
$operation->phase = 'accepted';
|
|
$operation->effect_resolution = 'not_applicable';
|
|
$operation->lease_token = null;
|
|
$operation->lease_until = 0;
|
|
$operation->available_at = $now + self::RECONCILE_DELAY;
|
|
$operation->next_check_at = 0;
|
|
$operation->error_code = '';
|
|
$operation->result_json = self::encodedMetadata(
|
|
self::resetEvidence($operation->result_json)
|
|
);
|
|
$operation->updated_at = $now;
|
|
$operation->save();
|
|
$replica->state = 'reserved';
|
|
$replica->lease_token = null;
|
|
$replica->lease_until = 0;
|
|
$replica->error_code = '';
|
|
$replica->updated_at = $now;
|
|
$replica->save();
|
|
});
|
|
}
|
|
|
|
/** @param array<string,mixed> $work */
|
|
private function attention(array $work, string $code, int $now): void
|
|
{
|
|
$this->mutateLeased($work, function (
|
|
CloudStorageOperation $operation,
|
|
CloudStorageArtifact $artifact,
|
|
CloudStorageReplica $replica,
|
|
CloudStoragePublication $publication,
|
|
array $copyOperations,
|
|
array $replicas
|
|
) use ($code, $now): void {
|
|
$this->settleAttention(
|
|
$operation,
|
|
$artifact,
|
|
$replica,
|
|
$publication,
|
|
$copyOperations,
|
|
$replicas,
|
|
true,
|
|
self::errorCode($code),
|
|
$now
|
|
);
|
|
});
|
|
}
|
|
|
|
/**
|
|
* @param array<string,mixed> $work
|
|
* @param callable(CloudStorageOperation,CloudStorageArtifact,
|
|
* CloudStorageReplica,CloudStoragePublication,
|
|
* array<int,CloudStorageOperation>,array<int,CloudStorageReplica>):void $callback
|
|
*/
|
|
private function mutateLeased(array $work, callable $callback): void
|
|
{
|
|
$connection = (new CloudStorageOperation())->getConnection();
|
|
$connection->transaction(function () use ($work, $callback): void {
|
|
$operation = CloudStorageOperation::query()
|
|
->where('operation_id', (string) $work['operation_id'])
|
|
->lockForUpdate()->firstOrFail();
|
|
if ((int) $operation->artifact_id !== (int) $work['artifact_id']) {
|
|
throw new CloudError('对象存储发布制品身份已变化', 409);
|
|
}
|
|
$copyOperations = CloudStorageOperation::query()
|
|
->where('artifact_id', (int) $operation->artifact_id)
|
|
->where('kind', 'copy')->orderBy('id')
|
|
->lockForUpdate()->get()->all();
|
|
[$artifact, $replica, $publication, $replicas] =
|
|
$this->lockedContext($operation, true);
|
|
if (!hash_equals(
|
|
(string) $operation->lease_token,
|
|
(string) $work['lease_token']
|
|
)
|
|
|| !hash_equals(
|
|
(string) $replica->lease_token,
|
|
(string) $work['lease_token']
|
|
)) {
|
|
throw new CloudError('对象存储主副本发布租约已失效', 409);
|
|
}
|
|
$callback(
|
|
$operation,
|
|
$artifact,
|
|
$replica,
|
|
$publication,
|
|
$copyOperations,
|
|
$replicas
|
|
);
|
|
}, 3);
|
|
}
|
|
|
|
/**
|
|
* @return array{0:CloudStorageArtifact,1:CloudStorageReplica,
|
|
* 2:CloudStoragePublication,3?:array<int,CloudStorageReplica>}
|
|
*/
|
|
private function lockedContext(
|
|
CloudStorageOperation $operation,
|
|
bool $includeReplicas = false
|
|
): array {
|
|
$artifact = CloudStorageArtifact::query()
|
|
->whereKey((int) $operation->artifact_id)
|
|
->lockForUpdate()->firstOrFail();
|
|
$replicas = CloudStorageReplica::query()
|
|
->where('artifact_id', (int) $artifact->id)->orderBy('id')
|
|
->lockForUpdate()->get()->all();
|
|
$replica = null;
|
|
foreach ($replicas as $candidate) {
|
|
if ((int) $candidate->id === (int) $operation->replica_id) {
|
|
$replica = $candidate;
|
|
break;
|
|
}
|
|
}
|
|
$publication = CloudStoragePublication::query()
|
|
->where('artifact_id', (int) $artifact->id)
|
|
->lockForUpdate()->firstOrFail();
|
|
if (!$replica instanceof CloudStorageReplica
|
|
|| (string) $replica->role !== 'primary') {
|
|
throw new CloudError('对象存储主副本账本不一致', 409);
|
|
}
|
|
return $includeReplicas
|
|
? [$artifact, $replica, $publication, $replicas]
|
|
: [$artifact, $replica, $publication];
|
|
}
|
|
|
|
/**
|
|
* @param array<int,CloudStorageOperation> $copyOperations
|
|
* @param array<int,CloudStorageReplica> $replicas
|
|
*/
|
|
private function settleAttention(
|
|
CloudStorageOperation $operation,
|
|
CloudStorageArtifact $artifact,
|
|
CloudStorageReplica $replica,
|
|
CloudStoragePublication $publication,
|
|
array $copyOperations,
|
|
array $replicas,
|
|
bool $effectUnknown,
|
|
string $code,
|
|
int $now
|
|
): void {
|
|
$operation->phase = 'terminal';
|
|
$operation->outcome = 'failed';
|
|
$operation->effect_resolution = $effectUnknown
|
|
? 'unknown' : 'not_applied';
|
|
$operation->effect_may_have_applied = $effectUnknown ? 1 : 0;
|
|
$operation->safe_to_resubmit = 0;
|
|
$operation->lease_token = null;
|
|
$operation->lease_until = 0;
|
|
$operation->available_at = 0;
|
|
$operation->next_check_at = 0;
|
|
$operation->error_code = self::errorCode($code);
|
|
$operation->terminal_at = $now;
|
|
$operation->updated_at = $now;
|
|
$operation->save();
|
|
$replica->state = 'degraded';
|
|
$replica->lease_token = null;
|
|
$replica->lease_until = 0;
|
|
$replica->error_code = self::errorCode($code);
|
|
$replica->updated_at = $now;
|
|
$replica->save();
|
|
foreach ($copyOperations as $copy) {
|
|
if ((string) $copy->phase === 'terminal') {
|
|
continue;
|
|
}
|
|
// Secondary work is blocked until primary success. Never invent a
|
|
// safe non-dispatch result if an inconsistent ledger says a copy
|
|
// has already crossed its own dispatch fence.
|
|
if ((string) $copy->phase !== 'accepted') {
|
|
continue;
|
|
}
|
|
$copy->phase = 'terminal';
|
|
$copy->outcome = 'failed';
|
|
$copy->effect_resolution = 'not_applied';
|
|
$copy->effect_may_have_applied = 0;
|
|
$copy->safe_to_resubmit = 0;
|
|
$copy->lease_token = null;
|
|
$copy->lease_until = 0;
|
|
$copy->available_at = 0;
|
|
$copy->next_check_at = 0;
|
|
$copy->error_code = 'primary_publish_attention';
|
|
$copy->terminal_at = $now;
|
|
$copy->updated_at = $now;
|
|
$copy->save();
|
|
foreach ($replicas as $secondary) {
|
|
if ((int) $secondary->id !== (int) $copy->replica_id) {
|
|
continue;
|
|
}
|
|
$secondary->state = 'degraded';
|
|
$secondary->lease_token = null;
|
|
$secondary->lease_until = 0;
|
|
$secondary->error_code = 'primary_publish_attention';
|
|
$secondary->updated_at = $now;
|
|
$secondary->save();
|
|
break;
|
|
}
|
|
}
|
|
$artifact->state = 'degraded';
|
|
$artifact->updated_at = $now;
|
|
$artifact->save();
|
|
$publication->state = 'failed';
|
|
$publication->available_at = 0;
|
|
$publication->lease_token = null;
|
|
$publication->lease_until = 0;
|
|
$publication->last_error_code = self::errorCode($code);
|
|
$publication->updated_at = $now;
|
|
$publication->save();
|
|
}
|
|
|
|
/**
|
|
* @return array{absence_confirmations:int,reconcile_failures:int,
|
|
* maximum_size_bytes?:int,object_suffix?:string}
|
|
*/
|
|
private static function metadata(?string $json): array
|
|
{
|
|
if ($json === null || trim($json) === '') {
|
|
return [
|
|
'absence_confirmations' => 0,
|
|
'reconcile_failures' => 0,
|
|
];
|
|
}
|
|
try {
|
|
$value = json_decode($json, true, 16, JSON_THROW_ON_ERROR);
|
|
} catch (\Throwable) {
|
|
throw new CloudError('对象存储发布核对证据无效', 409);
|
|
}
|
|
if (!is_array($value) || array_is_list($value)
|
|
|| array_diff(array_keys($value), [
|
|
'absence_confirmations', 'reconcile_failures',
|
|
'maximum_size_bytes', 'object_suffix',
|
|
]) !== []) {
|
|
throw new CloudError('对象存储发布核对证据无效', 409);
|
|
}
|
|
$absence = $value['absence_confirmations'] ?? 0;
|
|
$failures = $value['reconcile_failures'] ?? 0;
|
|
if (!is_int($absence) || $absence < 0 || $absence > 2
|
|
|| !is_int($failures) || $failures < 0
|
|
|| $failures > self::MAX_RECONCILE_FAILURES) {
|
|
throw new CloudError('对象存储发布核对证据无效', 409);
|
|
}
|
|
$metadata = [
|
|
'absence_confirmations' => $absence,
|
|
'reconcile_failures' => $failures,
|
|
];
|
|
if (array_key_exists('maximum_size_bytes', $value)) {
|
|
$maximum = $value['maximum_size_bytes'];
|
|
if (!is_int($maximum) || $maximum < 1) {
|
|
throw new CloudError('对象存储发布核对证据无效', 409);
|
|
}
|
|
$metadata['maximum_size_bytes'] = $maximum;
|
|
}
|
|
if (array_key_exists('object_suffix', $value)) {
|
|
$suffix = $value['object_suffix'];
|
|
if (!is_string($suffix) || $suffix === ''
|
|
|| strlen($suffix) > 512
|
|
|| preg_match('/[\x00-\x1f\x7f]/', $suffix) === 1) {
|
|
throw new CloudError('对象存储发布核对证据无效', 409);
|
|
}
|
|
$metadata['object_suffix'] = $suffix;
|
|
}
|
|
return $metadata;
|
|
}
|
|
|
|
private static function encodedMetadata(array $metadata): string
|
|
{
|
|
$normalized = self::metadata(json_encode(
|
|
$metadata,
|
|
JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES
|
|
));
|
|
return json_encode(
|
|
$normalized,
|
|
JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES
|
|
);
|
|
}
|
|
|
|
/**
|
|
* Preserve beginPublication()'s immutable context while clearing only
|
|
* this module's transient reconciliation evidence.
|
|
*
|
|
* @return array{absence_confirmations:int,reconcile_failures:int,
|
|
* maximum_size_bytes?:int,object_suffix?:string}
|
|
*/
|
|
private static function resetEvidence(?string $json): array
|
|
{
|
|
$metadata = self::metadata($json);
|
|
$metadata['absence_confirmations'] = 0;
|
|
$metadata['reconcile_failures'] = 0;
|
|
return $metadata;
|
|
}
|
|
|
|
/** @param array<string,mixed> $file @param array<string,mixed> $snapshot */
|
|
private static function assertFile(array $file, array $snapshot): void
|
|
{
|
|
$path = $file['path'] ?? null;
|
|
$size = $file['size_bytes'] ?? null;
|
|
$sha = $file['sha256'] ?? null;
|
|
$actual = is_string($path) && is_file($path) && is_readable($path)
|
|
? hash_file('sha256', $path) : false;
|
|
if (!is_string($path) || !is_int($size) || $size < 1
|
|
|| !is_string($sha)
|
|
|| preg_match('/^[a-f0-9]{64}$/D', $sha) !== 1
|
|
|| @filesize($path) !== $size
|
|
|| !is_string($actual) || !hash_equals($sha, $actual)
|
|
|| $size !== (int) $snapshot['size_bytes']
|
|
|| !hash_equals($sha, (string) $snapshot['sha256'])) {
|
|
throw new CloudError('本地不可变制品文件身份无效', 422);
|
|
}
|
|
}
|
|
|
|
/** @param array<string,mixed> $runtime */
|
|
private static function assertRuntime(array $runtime): void
|
|
{
|
|
if (!is_array($runtime['config'] ?? null)
|
|
|| array_is_list($runtime['config'])
|
|
|| !is_array($runtime['credentials'] ?? null)
|
|
|| array_is_list($runtime['credentials'])
|
|
|| !is_int($runtime['revision'] ?? null)
|
|
|| (int) $runtime['revision'] < 1) {
|
|
throw new CloudError('对象存储发布运行时无效', 503);
|
|
}
|
|
}
|
|
|
|
private static function assertOperationId(string $operationId): void
|
|
{
|
|
if (preg_match('/^[a-f0-9]{32}$/D', $operationId) !== 1) {
|
|
throw new CloudError('对象存储发布操作标识无效', 422);
|
|
}
|
|
}
|
|
|
|
private static function errorCode(string $code): string
|
|
{
|
|
$code = strtolower(trim($code));
|
|
return preg_match('/^[a-z][a-z0-9_]{0,95}$/D', $code) === 1
|
|
? $code : 'immutable_primary_publication_failed';
|
|
}
|
|
|
|
private function s3(): CloudObjectStorageS3Client
|
|
{
|
|
return $this->client ?? new CloudObjectStorageS3Client();
|
|
}
|
|
}
|