feat(storage): add durable immutable primary publisher
parent
c11bfb65d2
commit
72a0358903
@ -0,0 +1,895 @@
|
||||
<?php
|
||||
|
||||
namespace App\Services\Cloud;
|
||||
|
||||
use App\Models\CloudStorageArtifact;
|
||||
use App\Models\CloudStorageOperation;
|
||||
use App\Models\CloudStoragePublication;
|
||||
use App\Models\CloudStorageReplica;
|
||||
|
||||
/**
|
||||
* Publishes one known local file through an already-reserved primary intent.
|
||||
*
|
||||
* The business owner must first call CloudStorageArtifactService::
|
||||
* beginPublication(). This module never allocates a replacement artifact or
|
||||
* object key. A lost PUT receipt is reconciled only with exact-key HEAD
|
||||
* evidence; the same operation becomes dispatchable again only after two
|
||||
* spaced, stable absence observations.
|
||||
*/
|
||||
final class CloudStorageImmutablePrimaryPublication
|
||||
{
|
||||
private const LEASE_SECONDS = 300;
|
||||
private const RECONCILE_DELAY = 60;
|
||||
private const RETRY_DELAY = 300;
|
||||
private const MAX_DISPATCH_ATTEMPTS = 3;
|
||||
private const MAX_RECONCILE_FAILURES = 6;
|
||||
|
||||
/** @var \Closure(int,bool):array<string,mixed> */
|
||||
private readonly \Closure $runtimeResolver;
|
||||
|
||||
/** @var \Closure():int */
|
||||
private readonly \Closure $clock;
|
||||
|
||||
/**
|
||||
* @param null|callable(int,bool):array<string,mixed> $runtimeResolver
|
||||
* @param null|callable():int $clock
|
||||
*/
|
||||
public function __construct(
|
||||
private readonly ?CloudObjectStorageS3Client $client = null,
|
||||
?callable $runtimeResolver = null,
|
||||
?callable $clock = null
|
||||
) {
|
||||
$this->runtimeResolver = $runtimeResolver === null
|
||||
? static fn (int $backendId, bool $healthy): array =>
|
||||
CloudObjectStorageBackendRegistry::runtime(
|
||||
$backendId,
|
||||
$healthy
|
||||
)
|
||||
: \Closure::fromCallable($runtimeResolver);
|
||||
$this->clock = $clock === null
|
||||
? static fn (): int => time()
|
||||
: \Closure::fromCallable($clock);
|
||||
}
|
||||
|
||||
/**
|
||||
* Advance one immutable primary publication by at most one remote action.
|
||||
*
|
||||
* @param array{path:string,size_bytes:int,sha256:string} $file
|
||||
* @return array<string,mixed>
|
||||
*/
|
||||
public function publish(
|
||||
string $operationId,
|
||||
int $expectedArtifactId,
|
||||
array $file
|
||||
): array {
|
||||
self::assertOperationId($operationId);
|
||||
if ($expectedArtifactId < 1) {
|
||||
throw new CloudError('对象存储制品身份无效', 422);
|
||||
}
|
||||
$snapshot = $this->snapshot($operationId, $expectedArtifactId);
|
||||
if (($snapshot['terminal'] ?? false) === true) {
|
||||
return $snapshot;
|
||||
}
|
||||
// Reconciliation is intentionally independent of the temporary local
|
||||
// file. Once a PUT may have happened, only the frozen ledger identity
|
||||
// and an exact-key HEAD are authoritative.
|
||||
if ((string) $snapshot['phase'] === 'accepted') {
|
||||
self::assertFile($file, $snapshot);
|
||||
}
|
||||
$now = ($this->clock)();
|
||||
$work = $this->claim(
|
||||
$operationId,
|
||||
$expectedArtifactId,
|
||||
$now
|
||||
);
|
||||
if ($work === null) {
|
||||
return $this->snapshot($operationId, $expectedArtifactId);
|
||||
}
|
||||
if ((string) $work['mode'] === 'dispatch') {
|
||||
try {
|
||||
// Recheck after the dispatch lease is owned so a changed
|
||||
// source can never reach the provider under this intent.
|
||||
self::assertFile($file, $work);
|
||||
} catch (\Throwable $error) {
|
||||
$this->safeFailure(
|
||||
$work,
|
||||
'immutable_primary_source_changed',
|
||||
$now
|
||||
);
|
||||
throw $error;
|
||||
}
|
||||
}
|
||||
try {
|
||||
$runtime = ($this->runtimeResolver)(
|
||||
(int) $work['backend_id'],
|
||||
(string) $work['mode'] === 'dispatch'
|
||||
);
|
||||
self::assertRuntime($runtime);
|
||||
} catch (\Throwable) {
|
||||
if ((string) $work['mode'] === 'dispatch') {
|
||||
$this->safeFailure(
|
||||
$work,
|
||||
'immutable_primary_runtime_unavailable',
|
||||
$now
|
||||
);
|
||||
} else {
|
||||
$this->reconcileFailure(
|
||||
$work,
|
||||
'immutable_primary_runtime_unavailable',
|
||||
$now
|
||||
);
|
||||
}
|
||||
return $this->snapshot($operationId, $expectedArtifactId);
|
||||
}
|
||||
|
||||
if ((string) $work['mode'] === 'dispatch') {
|
||||
$this->dispatch($work, $runtime, $file);
|
||||
} else {
|
||||
$this->reconcile($work, $runtime);
|
||||
}
|
||||
return $this->snapshot($operationId, $expectedArtifactId);
|
||||
}
|
||||
|
||||
/** @return array<string,mixed> */
|
||||
private function snapshot(
|
||||
string $operationId,
|
||||
int $expectedArtifactId
|
||||
): array {
|
||||
$operation = CloudStorageOperation::query()
|
||||
->where('operation_id', $operationId)->first();
|
||||
if (!$operation instanceof CloudStorageOperation
|
||||
|| (string) $operation->kind !== 'publish'
|
||||
|| (int) $operation->artifact_id !== $expectedArtifactId) {
|
||||
throw new CloudError('对象存储发布操作与制品身份不一致', 409);
|
||||
}
|
||||
$artifact = CloudStorageArtifact::query()
|
||||
->whereKey($expectedArtifactId)->first();
|
||||
$replica = CloudStorageReplica::query()
|
||||
->whereKey((int) $operation->replica_id)->first();
|
||||
$publication = CloudStoragePublication::query()
|
||||
->where('artifact_id', $expectedArtifactId)->first();
|
||||
if (!$artifact instanceof CloudStorageArtifact
|
||||
|| !$replica instanceof CloudStorageReplica
|
||||
|| !$publication instanceof CloudStoragePublication
|
||||
|| (int) $replica->artifact_id !== $expectedArtifactId
|
||||
|| (string) $replica->role !== 'primary') {
|
||||
throw new CloudError('对象存储主副本账本不一致', 409);
|
||||
}
|
||||
$terminal = (string) $operation->phase === 'terminal';
|
||||
return [
|
||||
'operation_id' => (string) $operation->operation_id,
|
||||
'artifact_id' => (int) $artifact->id,
|
||||
'phase' => (string) $operation->phase,
|
||||
'outcome' => $operation->outcome === null
|
||||
? null : (string) $operation->outcome,
|
||||
'effect_resolution' => (string) $operation->effect_resolution,
|
||||
'attempts' => (int) $operation->attempts,
|
||||
'next_check_at' => (int) $operation->next_check_at,
|
||||
'error_code' => (string) $operation->error_code,
|
||||
'terminal' => $terminal,
|
||||
'primary_completed' => $terminal
|
||||
&& (string) $operation->outcome === 'succeeded',
|
||||
'publication_state' => (string) $publication->state,
|
||||
'ready_to_finalize' => in_array(
|
||||
(string) $publication->state,
|
||||
['ready_to_finalize', 'finalizing', 'completed'],
|
||||
true
|
||||
),
|
||||
'requires_attention' => ($terminal
|
||||
&& (string) $operation->outcome === 'failed')
|
||||
|| (string) $publication->state === 'failed',
|
||||
'object_key' => (string) $replica->object_key,
|
||||
'size_bytes' => (int) $artifact->size_bytes,
|
||||
'sha256' => (string) $artifact->sha256,
|
||||
];
|
||||
}
|
||||
|
||||
/** @return array<string,mixed>|null */
|
||||
private function claim(
|
||||
string $operationId,
|
||||
int $expectedArtifactId,
|
||||
int $now
|
||||
): ?array {
|
||||
$connection = (new CloudStorageOperation())->getConnection();
|
||||
return $connection->transaction(function () use (
|
||||
$operationId,
|
||||
$expectedArtifactId,
|
||||
$now
|
||||
): ?array {
|
||||
$operation = CloudStorageOperation::query()
|
||||
->where('operation_id', $operationId)
|
||||
->lockForUpdate()->first();
|
||||
if (!$operation instanceof CloudStorageOperation
|
||||
|| (string) $operation->kind !== 'publish'
|
||||
|| (int) $operation->artifact_id !== $expectedArtifactId) {
|
||||
throw new CloudError(
|
||||
'对象存储发布操作与制品身份不一致',
|
||||
409
|
||||
);
|
||||
}
|
||||
[$artifact, $replica, $publication] =
|
||||
$this->lockedContext($operation);
|
||||
if ((string) $operation->phase === 'terminal'
|
||||
|| in_array((string) $publication->state,
|
||||
['completed', 'failed', 'retired'], true)) {
|
||||
return null;
|
||||
}
|
||||
if ((string) $artifact->state !== 'publishing'
|
||||
|| !in_array((string) $replica->state,
|
||||
['reserved', 'publishing'], true)) {
|
||||
throw new CloudError('对象存储主副本发布状态不一致', 409);
|
||||
}
|
||||
if (is_string($operation->lease_token)
|
||||
&& $operation->lease_token !== ''
|
||||
&& (int) $operation->lease_until > $now) {
|
||||
return null;
|
||||
}
|
||||
|
||||
$phase = (string) $operation->phase;
|
||||
$mode = null;
|
||||
if ($phase === 'accepted'
|
||||
&& (int) $operation->available_at <= $now) {
|
||||
$mode = 'dispatch';
|
||||
$operation->phase = 'dispatching';
|
||||
$operation->effect_resolution = 'not_applicable';
|
||||
$operation->attempts = (int) $operation->attempts + 1;
|
||||
$replica->state = 'publishing';
|
||||
} elseif ($phase === 'reconciling'
|
||||
&& (int) $operation->next_check_at <= $now) {
|
||||
$mode = 'reconcile';
|
||||
} elseif (in_array($phase,
|
||||
['dispatching', 'awaiting_receipt'], true)
|
||||
&& (int) $operation->lease_until <= $now) {
|
||||
$mode = 'reconcile';
|
||||
$operation->phase = 'reconciling';
|
||||
$operation->effect_resolution = 'unknown';
|
||||
$operation->error_code =
|
||||
'immutable_primary_lease_expired';
|
||||
$operation->next_check_at = $now;
|
||||
}
|
||||
if ($mode === null) {
|
||||
return null;
|
||||
}
|
||||
$lease = bin2hex(random_bytes(16));
|
||||
$operation->lease_token = $lease;
|
||||
$operation->lease_until = $now + self::LEASE_SECONDS;
|
||||
$operation->updated_at = $now;
|
||||
$operation->save();
|
||||
$replica->lease_token = $lease;
|
||||
$replica->lease_until = $now + self::LEASE_SECONDS;
|
||||
$replica->updated_at = $now;
|
||||
$replica->save();
|
||||
return [
|
||||
'mode' => $mode,
|
||||
'operation_id' => (string) $operation->operation_id,
|
||||
'artifact_id' => (int) $artifact->id,
|
||||
'replica_id' => (int) $replica->id,
|
||||
'backend_id' => (int) $replica->backend_id,
|
||||
'object_key' => (string) $replica->object_key,
|
||||
'size_bytes' => (int) $artifact->size_bytes,
|
||||
'sha256' => (string) $artifact->sha256,
|
||||
'lease_token' => $lease,
|
||||
'attempts' => (int) $operation->attempts,
|
||||
];
|
||||
}, 3);
|
||||
}
|
||||
|
||||
/**
|
||||
* @param array<string,mixed> $work
|
||||
* @param array<string,mixed> $runtime
|
||||
* @param array{path:string,size_bytes:int,sha256:string} $file
|
||||
*/
|
||||
private function dispatch(array $work, array $runtime, array $file): void
|
||||
{
|
||||
try {
|
||||
$identity = $this->s3()->putImmutableFile(
|
||||
$runtime['config'],
|
||||
$runtime['credentials'],
|
||||
(string) $work['object_key'],
|
||||
(string) $file['path'],
|
||||
(int) $file['size_bytes'],
|
||||
(string) $file['sha256']
|
||||
);
|
||||
} catch (CloudObjectStorageTransferError $error) {
|
||||
if ($error->effectMayHaveApplied
|
||||
|| $error->publicCode === 'object_already_exists') {
|
||||
$this->markUnknown($work, $error->publicCode);
|
||||
} else {
|
||||
$this->safeFailure(
|
||||
$work,
|
||||
$error->publicCode,
|
||||
($this->clock)()
|
||||
);
|
||||
}
|
||||
return;
|
||||
} catch (\Throwable) {
|
||||
$this->markUnknown($work, 'immutable_primary_put_unknown');
|
||||
return;
|
||||
}
|
||||
if ((int) ($identity['size_bytes'] ?? 0)
|
||||
!== (int) $work['size_bytes']
|
||||
|| !is_string($identity['etag'] ?? null)
|
||||
|| !is_string($identity['version_id'] ?? null)) {
|
||||
$this->markUnknown(
|
||||
$work,
|
||||
'immutable_primary_put_receipt_invalid'
|
||||
);
|
||||
return;
|
||||
}
|
||||
CloudStorageArtifactService::completePrimary(
|
||||
(string) $work['operation_id'],
|
||||
(int) $work['artifact_id'],
|
||||
[
|
||||
'size_bytes' => (int) $work['size_bytes'],
|
||||
'sha256' => (string) $work['sha256'],
|
||||
'etag' => (string) $identity['etag'],
|
||||
'version_id' => (string) $identity['version_id'],
|
||||
]
|
||||
);
|
||||
}
|
||||
|
||||
/** @param array<string,mixed> $work @param array<string,mixed> $runtime */
|
||||
private function reconcile(array $work, array $runtime): void
|
||||
{
|
||||
try {
|
||||
$identity = $this->s3()->inspectObject(
|
||||
$runtime['config'],
|
||||
$runtime['credentials'],
|
||||
(string) $work['object_key']
|
||||
);
|
||||
} catch (\Throwable) {
|
||||
$this->reconcileFailure(
|
||||
$work,
|
||||
'immutable_primary_head_unavailable',
|
||||
($this->clock)()
|
||||
);
|
||||
return;
|
||||
}
|
||||
if ($identity === null) {
|
||||
$this->recordAbsence($work, ($this->clock)());
|
||||
return;
|
||||
}
|
||||
if ((int) ($identity['size_bytes'] ?? 0)
|
||||
!== (int) $work['size_bytes']
|
||||
|| !is_string($identity['sha256'] ?? null)
|
||||
|| !hash_equals(
|
||||
(string) $work['sha256'],
|
||||
(string) $identity['sha256']
|
||||
)) {
|
||||
$this->attention(
|
||||
$work,
|
||||
'immutable_primary_identity_conflict',
|
||||
($this->clock)()
|
||||
);
|
||||
return;
|
||||
}
|
||||
CloudStorageArtifactService::completePrimary(
|
||||
(string) $work['operation_id'],
|
||||
(int) $work['artifact_id'],
|
||||
[
|
||||
'size_bytes' => (int) $work['size_bytes'],
|
||||
'sha256' => (string) $work['sha256'],
|
||||
'etag' => (string) ($identity['etag'] ?? ''),
|
||||
'version_id' => (string) ($identity['version_id'] ?? ''),
|
||||
]
|
||||
);
|
||||
}
|
||||
|
||||
/** @param array<string,mixed> $work */
|
||||
private function markUnknown(array $work, string $code): void
|
||||
{
|
||||
$now = ($this->clock)();
|
||||
$this->mutateLeased($work, function (
|
||||
CloudStorageOperation $operation,
|
||||
CloudStorageArtifact $artifact,
|
||||
CloudStorageReplica $replica,
|
||||
CloudStoragePublication $publication,
|
||||
array $copyOperations,
|
||||
array $replicas
|
||||
) use ($code, $now): void {
|
||||
$metadata = self::metadata($operation->result_json);
|
||||
$metadata['absence_confirmations'] = 0;
|
||||
$operation->phase = 'reconciling';
|
||||
$operation->effect_resolution = 'unknown';
|
||||
$operation->lease_token = null;
|
||||
$operation->lease_until = 0;
|
||||
$operation->next_check_at = $now + self::RECONCILE_DELAY;
|
||||
$operation->error_code = self::errorCode($code);
|
||||
$operation->result_json = self::encodedMetadata($metadata);
|
||||
$operation->updated_at = $now;
|
||||
$operation->save();
|
||||
$replica->state = 'publishing';
|
||||
$replica->lease_token = null;
|
||||
$replica->lease_until = 0;
|
||||
$replica->error_code = self::errorCode($code);
|
||||
$replica->updated_at = $now;
|
||||
$replica->save();
|
||||
$artifact->state = 'publishing';
|
||||
$artifact->updated_at = $now;
|
||||
$artifact->save();
|
||||
});
|
||||
}
|
||||
|
||||
/** @param array<string,mixed> $work */
|
||||
private function safeFailure(array $work, string $code, int $now): void
|
||||
{
|
||||
$this->mutateLeased($work, function (
|
||||
CloudStorageOperation $operation,
|
||||
CloudStorageArtifact $artifact,
|
||||
CloudStorageReplica $replica,
|
||||
CloudStoragePublication $publication,
|
||||
array $copyOperations,
|
||||
array $replicas
|
||||
) use ($code, $now): void {
|
||||
$code = self::errorCode($code);
|
||||
if ((int) $operation->attempts >= self::MAX_DISPATCH_ATTEMPTS) {
|
||||
$this->settleAttention(
|
||||
$operation,
|
||||
$artifact,
|
||||
$replica,
|
||||
$publication,
|
||||
$copyOperations,
|
||||
$replicas,
|
||||
false,
|
||||
'immutable_primary_publish_retry_exhausted',
|
||||
$now
|
||||
);
|
||||
return;
|
||||
}
|
||||
$operation->phase = 'accepted';
|
||||
$operation->effect_resolution = 'not_applicable';
|
||||
$operation->lease_token = null;
|
||||
$operation->lease_until = 0;
|
||||
$operation->available_at = $now + self::RETRY_DELAY;
|
||||
$operation->next_check_at = 0;
|
||||
$operation->error_code = $code;
|
||||
$operation->result_json = self::encodedMetadata(
|
||||
self::resetEvidence($operation->result_json)
|
||||
);
|
||||
$operation->updated_at = $now;
|
||||
$operation->save();
|
||||
$replica->state = 'reserved';
|
||||
$replica->lease_token = null;
|
||||
$replica->lease_until = 0;
|
||||
$replica->error_code = $code;
|
||||
$replica->updated_at = $now;
|
||||
$replica->save();
|
||||
});
|
||||
}
|
||||
|
||||
/** @param array<string,mixed> $work */
|
||||
private function reconcileFailure(
|
||||
array $work,
|
||||
string $code,
|
||||
int $now
|
||||
): void {
|
||||
$this->mutateLeased($work, function (
|
||||
CloudStorageOperation $operation,
|
||||
CloudStorageArtifact $artifact,
|
||||
CloudStorageReplica $replica,
|
||||
CloudStoragePublication $publication,
|
||||
array $copyOperations,
|
||||
array $replicas
|
||||
) use ($code, $now): void {
|
||||
$metadata = self::metadata($operation->result_json);
|
||||
$failures = (int) ($metadata['reconcile_failures'] ?? 0) + 1;
|
||||
if ($failures >= self::MAX_RECONCILE_FAILURES) {
|
||||
$this->settleAttention(
|
||||
$operation,
|
||||
$artifact,
|
||||
$replica,
|
||||
$publication,
|
||||
$copyOperations,
|
||||
$replicas,
|
||||
true,
|
||||
'immutable_primary_reconcile_exhausted',
|
||||
$now
|
||||
);
|
||||
return;
|
||||
}
|
||||
$metadata['reconcile_failures'] = $failures;
|
||||
$operation->phase = 'reconciling';
|
||||
$operation->effect_resolution = 'unknown';
|
||||
$operation->lease_token = null;
|
||||
$operation->lease_until = 0;
|
||||
$operation->next_check_at = $now + self::RETRY_DELAY;
|
||||
$operation->error_code = self::errorCode($code);
|
||||
$operation->result_json = self::encodedMetadata($metadata);
|
||||
$operation->updated_at = $now;
|
||||
$operation->save();
|
||||
$replica->state = 'publishing';
|
||||
$replica->lease_token = null;
|
||||
$replica->lease_until = 0;
|
||||
$replica->error_code = self::errorCode($code);
|
||||
$replica->updated_at = $now;
|
||||
$replica->save();
|
||||
});
|
||||
}
|
||||
|
||||
/** @param array<string,mixed> $work */
|
||||
private function recordAbsence(array $work, int $now): void
|
||||
{
|
||||
$this->mutateLeased($work, function (
|
||||
CloudStorageOperation $operation,
|
||||
CloudStorageArtifact $artifact,
|
||||
CloudStorageReplica $replica,
|
||||
CloudStoragePublication $publication,
|
||||
array $copyOperations,
|
||||
array $replicas
|
||||
) use ($now): void {
|
||||
$metadata = self::metadata($operation->result_json);
|
||||
$confirmations = min(
|
||||
2,
|
||||
(int) ($metadata['absence_confirmations'] ?? 0) + 1
|
||||
);
|
||||
$metadata['absence_confirmations'] = $confirmations;
|
||||
$metadata['reconcile_failures'] = 0;
|
||||
if ($confirmations < 2) {
|
||||
$operation->phase = 'reconciling';
|
||||
$operation->effect_resolution = 'unknown';
|
||||
$operation->lease_token = null;
|
||||
$operation->lease_until = 0;
|
||||
$operation->next_check_at = $now + self::RECONCILE_DELAY;
|
||||
$operation->error_code = 'immutable_primary_absent_once';
|
||||
$operation->result_json = self::encodedMetadata($metadata);
|
||||
$operation->updated_at = $now;
|
||||
$operation->save();
|
||||
$replica->state = 'publishing';
|
||||
$replica->lease_token = null;
|
||||
$replica->lease_until = 0;
|
||||
$replica->error_code = 'immutable_primary_absent_once';
|
||||
$replica->updated_at = $now;
|
||||
$replica->save();
|
||||
return;
|
||||
}
|
||||
if ((int) $operation->attempts >= self::MAX_DISPATCH_ATTEMPTS) {
|
||||
$this->settleAttention(
|
||||
$operation,
|
||||
$artifact,
|
||||
$replica,
|
||||
$publication,
|
||||
$copyOperations,
|
||||
$replicas,
|
||||
false,
|
||||
'immutable_primary_absence_retry_exhausted',
|
||||
$now
|
||||
);
|
||||
return;
|
||||
}
|
||||
$operation->phase = 'accepted';
|
||||
$operation->effect_resolution = 'not_applicable';
|
||||
$operation->lease_token = null;
|
||||
$operation->lease_until = 0;
|
||||
$operation->available_at = $now + self::RECONCILE_DELAY;
|
||||
$operation->next_check_at = 0;
|
||||
$operation->error_code = '';
|
||||
$operation->result_json = self::encodedMetadata(
|
||||
self::resetEvidence($operation->result_json)
|
||||
);
|
||||
$operation->updated_at = $now;
|
||||
$operation->save();
|
||||
$replica->state = 'reserved';
|
||||
$replica->lease_token = null;
|
||||
$replica->lease_until = 0;
|
||||
$replica->error_code = '';
|
||||
$replica->updated_at = $now;
|
||||
$replica->save();
|
||||
});
|
||||
}
|
||||
|
||||
/** @param array<string,mixed> $work */
|
||||
private function attention(array $work, string $code, int $now): void
|
||||
{
|
||||
$this->mutateLeased($work, function (
|
||||
CloudStorageOperation $operation,
|
||||
CloudStorageArtifact $artifact,
|
||||
CloudStorageReplica $replica,
|
||||
CloudStoragePublication $publication,
|
||||
array $copyOperations,
|
||||
array $replicas
|
||||
) use ($code, $now): void {
|
||||
$this->settleAttention(
|
||||
$operation,
|
||||
$artifact,
|
||||
$replica,
|
||||
$publication,
|
||||
$copyOperations,
|
||||
$replicas,
|
||||
true,
|
||||
self::errorCode($code),
|
||||
$now
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* @param array<string,mixed> $work
|
||||
* @param callable(CloudStorageOperation,CloudStorageArtifact,
|
||||
* CloudStorageReplica,CloudStoragePublication,
|
||||
* array<int,CloudStorageOperation>,array<int,CloudStorageReplica>):void $callback
|
||||
*/
|
||||
private function mutateLeased(array $work, callable $callback): void
|
||||
{
|
||||
$connection = (new CloudStorageOperation())->getConnection();
|
||||
$connection->transaction(function () use ($work, $callback): void {
|
||||
$operation = CloudStorageOperation::query()
|
||||
->where('operation_id', (string) $work['operation_id'])
|
||||
->lockForUpdate()->firstOrFail();
|
||||
if ((int) $operation->artifact_id !== (int) $work['artifact_id']) {
|
||||
throw new CloudError('对象存储发布制品身份已变化', 409);
|
||||
}
|
||||
$copyOperations = CloudStorageOperation::query()
|
||||
->where('artifact_id', (int) $operation->artifact_id)
|
||||
->where('kind', 'copy')->orderBy('id')
|
||||
->lockForUpdate()->get()->all();
|
||||
[$artifact, $replica, $publication, $replicas] =
|
||||
$this->lockedContext($operation, true);
|
||||
if (!hash_equals(
|
||||
(string) $operation->lease_token,
|
||||
(string) $work['lease_token']
|
||||
)
|
||||
|| !hash_equals(
|
||||
(string) $replica->lease_token,
|
||||
(string) $work['lease_token']
|
||||
)) {
|
||||
throw new CloudError('对象存储主副本发布租约已失效', 409);
|
||||
}
|
||||
$callback(
|
||||
$operation,
|
||||
$artifact,
|
||||
$replica,
|
||||
$publication,
|
||||
$copyOperations,
|
||||
$replicas
|
||||
);
|
||||
}, 3);
|
||||
}
|
||||
|
||||
/**
|
||||
* @return array{0:CloudStorageArtifact,1:CloudStorageReplica,
|
||||
* 2:CloudStoragePublication,3?:array<int,CloudStorageReplica>}
|
||||
*/
|
||||
private function lockedContext(
|
||||
CloudStorageOperation $operation,
|
||||
bool $includeReplicas = false
|
||||
): array {
|
||||
$artifact = CloudStorageArtifact::query()
|
||||
->whereKey((int) $operation->artifact_id)
|
||||
->lockForUpdate()->firstOrFail();
|
||||
$replicas = CloudStorageReplica::query()
|
||||
->where('artifact_id', (int) $artifact->id)->orderBy('id')
|
||||
->lockForUpdate()->get()->all();
|
||||
$replica = null;
|
||||
foreach ($replicas as $candidate) {
|
||||
if ((int) $candidate->id === (int) $operation->replica_id) {
|
||||
$replica = $candidate;
|
||||
break;
|
||||
}
|
||||
}
|
||||
$publication = CloudStoragePublication::query()
|
||||
->where('artifact_id', (int) $artifact->id)
|
||||
->lockForUpdate()->firstOrFail();
|
||||
if (!$replica instanceof CloudStorageReplica
|
||||
|| (string) $replica->role !== 'primary') {
|
||||
throw new CloudError('对象存储主副本账本不一致', 409);
|
||||
}
|
||||
return $includeReplicas
|
||||
? [$artifact, $replica, $publication, $replicas]
|
||||
: [$artifact, $replica, $publication];
|
||||
}
|
||||
|
||||
/**
|
||||
* @param array<int,CloudStorageOperation> $copyOperations
|
||||
* @param array<int,CloudStorageReplica> $replicas
|
||||
*/
|
||||
private function settleAttention(
|
||||
CloudStorageOperation $operation,
|
||||
CloudStorageArtifact $artifact,
|
||||
CloudStorageReplica $replica,
|
||||
CloudStoragePublication $publication,
|
||||
array $copyOperations,
|
||||
array $replicas,
|
||||
bool $effectUnknown,
|
||||
string $code,
|
||||
int $now
|
||||
): void {
|
||||
$operation->phase = 'terminal';
|
||||
$operation->outcome = 'failed';
|
||||
$operation->effect_resolution = $effectUnknown
|
||||
? 'unknown' : 'not_applied';
|
||||
$operation->effect_may_have_applied = $effectUnknown ? 1 : 0;
|
||||
$operation->safe_to_resubmit = 0;
|
||||
$operation->lease_token = null;
|
||||
$operation->lease_until = 0;
|
||||
$operation->available_at = 0;
|
||||
$operation->next_check_at = 0;
|
||||
$operation->error_code = self::errorCode($code);
|
||||
$operation->terminal_at = $now;
|
||||
$operation->updated_at = $now;
|
||||
$operation->save();
|
||||
$replica->state = 'degraded';
|
||||
$replica->lease_token = null;
|
||||
$replica->lease_until = 0;
|
||||
$replica->error_code = self::errorCode($code);
|
||||
$replica->updated_at = $now;
|
||||
$replica->save();
|
||||
foreach ($copyOperations as $copy) {
|
||||
if ((string) $copy->phase === 'terminal') {
|
||||
continue;
|
||||
}
|
||||
// Secondary work is blocked until primary success. Never invent a
|
||||
// safe non-dispatch result if an inconsistent ledger says a copy
|
||||
// has already crossed its own dispatch fence.
|
||||
if ((string) $copy->phase !== 'accepted') {
|
||||
continue;
|
||||
}
|
||||
$copy->phase = 'terminal';
|
||||
$copy->outcome = 'failed';
|
||||
$copy->effect_resolution = 'not_applied';
|
||||
$copy->effect_may_have_applied = 0;
|
||||
$copy->safe_to_resubmit = 0;
|
||||
$copy->lease_token = null;
|
||||
$copy->lease_until = 0;
|
||||
$copy->available_at = 0;
|
||||
$copy->next_check_at = 0;
|
||||
$copy->error_code = 'primary_publish_attention';
|
||||
$copy->terminal_at = $now;
|
||||
$copy->updated_at = $now;
|
||||
$copy->save();
|
||||
foreach ($replicas as $secondary) {
|
||||
if ((int) $secondary->id !== (int) $copy->replica_id) {
|
||||
continue;
|
||||
}
|
||||
$secondary->state = 'degraded';
|
||||
$secondary->lease_token = null;
|
||||
$secondary->lease_until = 0;
|
||||
$secondary->error_code = 'primary_publish_attention';
|
||||
$secondary->updated_at = $now;
|
||||
$secondary->save();
|
||||
break;
|
||||
}
|
||||
}
|
||||
$artifact->state = 'degraded';
|
||||
$artifact->updated_at = $now;
|
||||
$artifact->save();
|
||||
$publication->state = 'failed';
|
||||
$publication->available_at = 0;
|
||||
$publication->lease_token = null;
|
||||
$publication->lease_until = 0;
|
||||
$publication->last_error_code = self::errorCode($code);
|
||||
$publication->updated_at = $now;
|
||||
$publication->save();
|
||||
}
|
||||
|
||||
/**
|
||||
* @return array{absence_confirmations:int,reconcile_failures:int,
|
||||
* maximum_size_bytes?:int,object_suffix?:string}
|
||||
*/
|
||||
private static function metadata(?string $json): array
|
||||
{
|
||||
if ($json === null || trim($json) === '') {
|
||||
return [
|
||||
'absence_confirmations' => 0,
|
||||
'reconcile_failures' => 0,
|
||||
];
|
||||
}
|
||||
try {
|
||||
$value = json_decode($json, true, 16, JSON_THROW_ON_ERROR);
|
||||
} catch (\Throwable) {
|
||||
throw new CloudError('对象存储发布核对证据无效', 409);
|
||||
}
|
||||
if (!is_array($value) || array_is_list($value)
|
||||
|| array_diff(array_keys($value), [
|
||||
'absence_confirmations', 'reconcile_failures',
|
||||
'maximum_size_bytes', 'object_suffix',
|
||||
]) !== []) {
|
||||
throw new CloudError('对象存储发布核对证据无效', 409);
|
||||
}
|
||||
$absence = $value['absence_confirmations'] ?? 0;
|
||||
$failures = $value['reconcile_failures'] ?? 0;
|
||||
if (!is_int($absence) || $absence < 0 || $absence > 2
|
||||
|| !is_int($failures) || $failures < 0
|
||||
|| $failures > self::MAX_RECONCILE_FAILURES) {
|
||||
throw new CloudError('对象存储发布核对证据无效', 409);
|
||||
}
|
||||
$metadata = [
|
||||
'absence_confirmations' => $absence,
|
||||
'reconcile_failures' => $failures,
|
||||
];
|
||||
if (array_key_exists('maximum_size_bytes', $value)) {
|
||||
$maximum = $value['maximum_size_bytes'];
|
||||
if (!is_int($maximum) || $maximum < 1) {
|
||||
throw new CloudError('对象存储发布核对证据无效', 409);
|
||||
}
|
||||
$metadata['maximum_size_bytes'] = $maximum;
|
||||
}
|
||||
if (array_key_exists('object_suffix', $value)) {
|
||||
$suffix = $value['object_suffix'];
|
||||
if (!is_string($suffix) || $suffix === ''
|
||||
|| strlen($suffix) > 512
|
||||
|| preg_match('/[\x00-\x1f\x7f]/', $suffix) === 1) {
|
||||
throw new CloudError('对象存储发布核对证据无效', 409);
|
||||
}
|
||||
$metadata['object_suffix'] = $suffix;
|
||||
}
|
||||
return $metadata;
|
||||
}
|
||||
|
||||
private static function encodedMetadata(array $metadata): string
|
||||
{
|
||||
$normalized = self::metadata(json_encode(
|
||||
$metadata,
|
||||
JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES
|
||||
));
|
||||
return json_encode(
|
||||
$normalized,
|
||||
JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Preserve beginPublication()'s immutable context while clearing only
|
||||
* this module's transient reconciliation evidence.
|
||||
*
|
||||
* @return array{absence_confirmations:int,reconcile_failures:int,
|
||||
* maximum_size_bytes?:int,object_suffix?:string}
|
||||
*/
|
||||
private static function resetEvidence(?string $json): array
|
||||
{
|
||||
$metadata = self::metadata($json);
|
||||
$metadata['absence_confirmations'] = 0;
|
||||
$metadata['reconcile_failures'] = 0;
|
||||
return $metadata;
|
||||
}
|
||||
|
||||
/** @param array<string,mixed> $file @param array<string,mixed> $snapshot */
|
||||
private static function assertFile(array $file, array $snapshot): void
|
||||
{
|
||||
$path = $file['path'] ?? null;
|
||||
$size = $file['size_bytes'] ?? null;
|
||||
$sha = $file['sha256'] ?? null;
|
||||
$actual = is_string($path) && is_file($path) && is_readable($path)
|
||||
? hash_file('sha256', $path) : false;
|
||||
if (!is_string($path) || !is_int($size) || $size < 1
|
||||
|| !is_string($sha)
|
||||
|| preg_match('/^[a-f0-9]{64}$/D', $sha) !== 1
|
||||
|| @filesize($path) !== $size
|
||||
|| !is_string($actual) || !hash_equals($sha, $actual)
|
||||
|| $size !== (int) $snapshot['size_bytes']
|
||||
|| !hash_equals($sha, (string) $snapshot['sha256'])) {
|
||||
throw new CloudError('本地不可变制品文件身份无效', 422);
|
||||
}
|
||||
}
|
||||
|
||||
/** @param array<string,mixed> $runtime */
|
||||
private static function assertRuntime(array $runtime): void
|
||||
{
|
||||
if (!is_array($runtime['config'] ?? null)
|
||||
|| array_is_list($runtime['config'])
|
||||
|| !is_array($runtime['credentials'] ?? null)
|
||||
|| array_is_list($runtime['credentials'])
|
||||
|| !is_int($runtime['revision'] ?? null)
|
||||
|| (int) $runtime['revision'] < 1) {
|
||||
throw new CloudError('对象存储发布运行时无效', 503);
|
||||
}
|
||||
}
|
||||
|
||||
private static function assertOperationId(string $operationId): void
|
||||
{
|
||||
if (preg_match('/^[a-f0-9]{32}$/D', $operationId) !== 1) {
|
||||
throw new CloudError('对象存储发布操作标识无效', 422);
|
||||
}
|
||||
}
|
||||
|
||||
private static function errorCode(string $code): string
|
||||
{
|
||||
$code = strtolower(trim($code));
|
||||
return preg_match('/^[a-z][a-z0-9_]{0,95}$/D', $code) === 1
|
||||
? $code : 'immutable_primary_publication_failed';
|
||||
}
|
||||
|
||||
private function s3(): CloudObjectStorageS3Client
|
||||
{
|
||||
return $this->client ?? new CloudObjectStorageS3Client();
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,544 @@
|
||||
<?php
|
||||
|
||||
require dirname(__DIR__) . '/vendor/autoload.php';
|
||||
|
||||
use App\Models\CloudStorageArtifact;
|
||||
use App\Models\CloudStorageOperation;
|
||||
use App\Models\CloudStoragePublication;
|
||||
use App\Models\CloudStorageReplica;
|
||||
use App\Services\Cloud\CloudError;
|
||||
use App\Services\Cloud\CloudObjectStorageS3Client;
|
||||
use App\Services\Cloud\CloudStorageArtifactService;
|
||||
use App\Services\Cloud\CloudStorageImmutablePrimaryPublication;
|
||||
use GuzzleHttp\Client;
|
||||
use GuzzleHttp\HandlerStack;
|
||||
use GuzzleHttp\Promise\Create;
|
||||
use GuzzleHttp\Psr7\Response;
|
||||
use Illuminate\Database\Capsule\Manager as Capsule;
|
||||
use Illuminate\Database\Schema\Blueprint;
|
||||
use Psr\Http\Message\RequestInterface;
|
||||
|
||||
$assert = static function (bool $condition, string $message): void {
|
||||
if (!$condition) {
|
||||
throw new RuntimeException($message);
|
||||
}
|
||||
};
|
||||
$expectCloudError = static function (
|
||||
callable $callback,
|
||||
int $status
|
||||
) use ($assert): void {
|
||||
try {
|
||||
$callback();
|
||||
} catch (CloudError $error) {
|
||||
$assert($error->httpStatus === $status,
|
||||
'Unexpected CloudError status: ' . $error->httpStatus);
|
||||
return;
|
||||
}
|
||||
throw new RuntimeException('Expected CloudError was not raised.');
|
||||
};
|
||||
|
||||
$db = new Capsule();
|
||||
$db->addConnection(['driver' => 'sqlite', 'database' => ':memory:']);
|
||||
$db->setAsGlobal();
|
||||
$db->bootEloquent();
|
||||
$db->schema()->create('cloud_storage_backends', static function (
|
||||
Blueprint $table
|
||||
): void {
|
||||
$table->increments('id');
|
||||
$table->string('backend_key');
|
||||
$table->string('identity_sha256');
|
||||
$table->string('name');
|
||||
$table->string('driver');
|
||||
$table->string('endpoint');
|
||||
$table->string('region');
|
||||
$table->string('bucket');
|
||||
$table->string('prefix');
|
||||
$table->boolean('path_style');
|
||||
$table->string('failure_domain');
|
||||
$table->text('sealed_credentials');
|
||||
$table->unsignedBigInteger('config_revision');
|
||||
$table->unsignedBigInteger('tested_revision');
|
||||
$table->string('admin_state');
|
||||
$table->string('health_state');
|
||||
$table->unsignedInteger('write_priority');
|
||||
$table->unsignedInteger('consecutive_failures')->default(0);
|
||||
$table->unsignedInteger('consecutive_successes')->default(0);
|
||||
$table->string('last_error_code')->default('');
|
||||
$table->unsignedBigInteger('last_tested_at')->default(0);
|
||||
$table->unsignedBigInteger('last_healthy_at')->default(0);
|
||||
$table->unsignedBigInteger('next_probe_at')->default(0);
|
||||
$table->unsignedBigInteger('created_at');
|
||||
$table->unsignedBigInteger('updated_at');
|
||||
});
|
||||
$db->schema()->create('cloud_storage_artifacts', static function (
|
||||
Blueprint $table
|
||||
): void {
|
||||
$table->increments('id');
|
||||
$table->string('artifact_key')->unique();
|
||||
$table->string('kind');
|
||||
$table->unsignedBigInteger('maximum_size_bytes');
|
||||
$table->unsignedBigInteger('size_bytes');
|
||||
$table->string('sha256');
|
||||
$table->string('durability_policy');
|
||||
$table->unsignedTinyInteger('desired_copies');
|
||||
$table->unsignedTinyInteger('required_copies');
|
||||
$table->string('state');
|
||||
$table->string('last_operation_id')->nullable();
|
||||
$table->unsignedBigInteger('verified_at');
|
||||
$table->unsignedBigInteger('created_at');
|
||||
$table->unsignedBigInteger('updated_at');
|
||||
});
|
||||
$db->schema()->create('cloud_storage_replicas', static function (
|
||||
Blueprint $table
|
||||
): void {
|
||||
$table->increments('id');
|
||||
$table->string('replica_key')->unique();
|
||||
$table->unsignedBigInteger('artifact_id');
|
||||
$table->unsignedBigInteger('backend_id');
|
||||
$table->string('role');
|
||||
$table->string('object_key');
|
||||
$table->string('object_version_id')->default('');
|
||||
$table->string('object_etag')->default('');
|
||||
$table->unsignedBigInteger('size_bytes')->default(0);
|
||||
$table->string('sha256')->default('');
|
||||
$table->string('state');
|
||||
$table->string('publish_operation_id')->nullable();
|
||||
$table->string('delete_operation_id')->nullable();
|
||||
$table->unsignedInteger('delete_generation')->default(0);
|
||||
$table->unsignedInteger('delete_cycle_attempts')->default(0);
|
||||
$table->unsignedInteger('delete_presence_confirmations')->default(0);
|
||||
$table->unsignedBigInteger('delete_last_checked_at')->default(0);
|
||||
$table->unsignedBigInteger('delete_next_check_at')->default(0);
|
||||
$table->unsignedBigInteger('delete_attention_at')->nullable();
|
||||
$table->string('lease_token')->nullable();
|
||||
$table->unsignedBigInteger('lease_until')->default(0);
|
||||
$table->string('error_code')->default('');
|
||||
$table->unsignedBigInteger('verified_at')->default(0);
|
||||
$table->unsignedBigInteger('created_at');
|
||||
$table->unsignedBigInteger('updated_at');
|
||||
});
|
||||
$db->schema()->create('cloud_storage_operations', static function (
|
||||
Blueprint $table
|
||||
): void {
|
||||
$table->increments('id');
|
||||
$table->string('operation_id')->unique();
|
||||
$table->string('request_idempotency_key')->unique();
|
||||
$table->string('request_fingerprint');
|
||||
$table->unsignedBigInteger('artifact_id');
|
||||
$table->unsignedBigInteger('replica_id')->nullable();
|
||||
$table->unsignedBigInteger('source_backend_id')->nullable();
|
||||
$table->unsignedBigInteger('target_backend_id')->nullable();
|
||||
$table->string('kind');
|
||||
$table->string('phase');
|
||||
$table->string('outcome')->nullable();
|
||||
$table->string('effect_resolution');
|
||||
$table->boolean('effect_may_have_applied')->nullable();
|
||||
$table->boolean('safe_to_resubmit')->nullable();
|
||||
$table->unsignedInteger('attempts');
|
||||
$table->string('lease_token')->nullable();
|
||||
$table->unsignedBigInteger('lease_until');
|
||||
$table->unsignedBigInteger('available_at');
|
||||
$table->unsignedBigInteger('next_check_at');
|
||||
$table->string('error_code');
|
||||
$table->text('result_json')->nullable();
|
||||
$table->unsignedBigInteger('terminal_at')->nullable();
|
||||
$table->unsignedBigInteger('created_at');
|
||||
$table->unsignedBigInteger('updated_at');
|
||||
});
|
||||
$db->schema()->create('cloud_storage_publications', static function (
|
||||
Blueprint $table
|
||||
): void {
|
||||
$table->increments('id');
|
||||
$table->string('publication_key')->unique();
|
||||
$table->unsignedBigInteger('artifact_id')->unique();
|
||||
$table->unsignedBigInteger('backup_id')->nullable()->unique();
|
||||
$table->string('template_import_intent_id')->nullable()->unique();
|
||||
$table->string('rescue_template_id')->nullable()->unique();
|
||||
$table->string('state');
|
||||
$table->unsignedInteger('attempts')->default(0);
|
||||
$table->unsignedBigInteger('available_at')->default(0);
|
||||
$table->string('lease_token')->nullable();
|
||||
$table->unsignedBigInteger('lease_until')->default(0);
|
||||
$table->string('last_error_code')->default('');
|
||||
$table->unsignedBigInteger('coverage_ready_at')->nullable();
|
||||
$table->unsignedBigInteger('finalized_at')->nullable();
|
||||
$table->unsignedBigInteger('created_at');
|
||||
$table->unsignedBigInteger('updated_at');
|
||||
});
|
||||
|
||||
$databaseNow = time();
|
||||
foreach ([
|
||||
[1, 'primary', 'provider-a', 10],
|
||||
[2, 'secondary', 'provider-b', 20],
|
||||
] as [$id, $name, $domain, $priority]) {
|
||||
Capsule::table('cloud_storage_backends')->insert([
|
||||
'id' => $id,
|
||||
'backend_key' => str_repeat((string) $id, 32),
|
||||
'identity_sha256' => str_repeat((string) $id, 64),
|
||||
'name' => $name,
|
||||
'driver' => 's3',
|
||||
'endpoint' => 'https://' . $name . '.example.test',
|
||||
'region' => 'us-east-1',
|
||||
'bucket' => $name . '-bucket',
|
||||
'prefix' => $name,
|
||||
'path_style' => 1,
|
||||
'failure_domain' => $domain,
|
||||
'sealed_credentials' => 'sealed',
|
||||
'config_revision' => 1,
|
||||
'tested_revision' => 1,
|
||||
'admin_state' => 'active',
|
||||
'health_state' => 'healthy',
|
||||
'write_priority' => $priority,
|
||||
'consecutive_failures' => 0,
|
||||
'consecutive_successes' => 1,
|
||||
'last_error_code' => '',
|
||||
'last_tested_at' => $databaseNow,
|
||||
'last_healthy_at' => $databaseNow,
|
||||
'next_probe_at' => $databaseNow + 300,
|
||||
'created_at' => $databaseNow,
|
||||
'updated_at' => $databaseNow,
|
||||
]);
|
||||
}
|
||||
|
||||
$temporaryPaths = [];
|
||||
register_shutdown_function(static function () use (&$temporaryPaths): void {
|
||||
foreach ($temporaryPaths as $path) {
|
||||
@unlink($path);
|
||||
}
|
||||
});
|
||||
$fixtureSequence = 0;
|
||||
$fixture = static function (
|
||||
string $kind = 'template'
|
||||
) use (&$fixtureSequence, &$temporaryPaths): array {
|
||||
$fixtureSequence++;
|
||||
$path = tempnam(sys_get_temp_dir(), 'virt-primary-publication-');
|
||||
if ($path === false) {
|
||||
throw new RuntimeException('Cannot create publication fixture.');
|
||||
}
|
||||
file_put_contents(
|
||||
$path,
|
||||
str_repeat('immutable-primary-' . $fixtureSequence . '-', 64)
|
||||
);
|
||||
$temporaryPaths[] = $path;
|
||||
$size = filesize($path);
|
||||
$sha256 = hash_file('sha256', $path);
|
||||
if (!is_int($size) || !is_string($sha256)) {
|
||||
throw new RuntimeException('Cannot identify publication fixture.');
|
||||
}
|
||||
$ownerId = str_pad(dechex($fixtureSequence), 32, '0', STR_PAD_LEFT);
|
||||
$owner = $kind === 'rescue'
|
||||
? ['rescue_template_id' => $ownerId]
|
||||
: ['template_import_intent_id' => $ownerId];
|
||||
$view = CloudStorageArtifactService::beginPublication(
|
||||
$kind,
|
||||
$kind . '/fixture-' . $fixtureSequence . '/system.qcow2',
|
||||
$size,
|
||||
$sha256,
|
||||
'immutable-primary-' . str_pad(
|
||||
(string) $fixtureSequence,
|
||||
4,
|
||||
'0',
|
||||
STR_PAD_LEFT
|
||||
),
|
||||
hash('sha256', $kind . '|' . $fixtureSequence),
|
||||
$owner
|
||||
);
|
||||
return [
|
||||
'path' => $path,
|
||||
'size_bytes' => $size,
|
||||
'sha256' => $sha256,
|
||||
'operation_id' => (string) $view['operation_id'],
|
||||
'artifact_id' => (int) $view['artifact_id'],
|
||||
'replica_id' => (int) $view['replica_id'],
|
||||
'object_key' => (string) $view['object_key'],
|
||||
];
|
||||
};
|
||||
|
||||
$publisher = static function (
|
||||
array &$responses,
|
||||
array &$requests,
|
||||
array &$transactionLevels,
|
||||
int &$clock
|
||||
): CloudStorageImmutablePrimaryPublication {
|
||||
$handler = static function (
|
||||
RequestInterface $request,
|
||||
array $options
|
||||
) use (&$responses, &$requests, &$transactionLevels) {
|
||||
$transactionLevels[] = Capsule::connection()->transactionLevel();
|
||||
$requests[] = $request;
|
||||
if ($responses === []) {
|
||||
throw new RuntimeException('Unexpected object-storage request.');
|
||||
}
|
||||
$response = array_shift($responses);
|
||||
if (!$response instanceof Response) {
|
||||
throw new RuntimeException('Invalid object-storage response fixture.');
|
||||
}
|
||||
return Create::promiseFor($response);
|
||||
};
|
||||
$client = new CloudObjectStorageS3Client(new Client([
|
||||
'handler' => HandlerStack::create($handler),
|
||||
]));
|
||||
return new CloudStorageImmutablePrimaryPublication(
|
||||
$client,
|
||||
static fn (int $backendId, bool $healthy): array => [
|
||||
'backend_id' => $backendId,
|
||||
'revision' => 1,
|
||||
'config' => [
|
||||
'endpoint' => 'https://objects.example.test',
|
||||
'region' => 'us-east-1',
|
||||
'bucket' => 'immutable-fixtures',
|
||||
'prefix' => 'unused-runtime-prefix',
|
||||
'path_style' => true,
|
||||
],
|
||||
'credentials' => [
|
||||
'access_key_id' => 'IMMUTABLEACCESSKEY',
|
||||
'secret_access_key' => 'immutable-secret-must-not-persist',
|
||||
],
|
||||
],
|
||||
static function () use (&$clock): int {
|
||||
return $clock;
|
||||
}
|
||||
);
|
||||
};
|
||||
|
||||
// A successful immutable PUT settles only the primary. Template publication
|
||||
// remains behind the required-two protection gate until its copy completes.
|
||||
$applied = $fixture('template');
|
||||
$clock = time() + 10;
|
||||
$responses = [new Response(200, [
|
||||
'ETag' => '"applied-etag"',
|
||||
'x-amz-version-id' => 'applied-version',
|
||||
])];
|
||||
$requests = [];
|
||||
$transactionLevels = [];
|
||||
$module = $publisher($responses, $requests, $transactionLevels, $clock);
|
||||
$appliedResult = $module->publish(
|
||||
$applied['operation_id'],
|
||||
$applied['artifact_id'],
|
||||
$applied
|
||||
);
|
||||
$assert($appliedResult['primary_completed'] === true
|
||||
&& $appliedResult['publication_state'] === 'waiting_protection'
|
||||
&& $appliedResult['ready_to_finalize'] === false,
|
||||
'Primary success bypassed the template publication quorum.');
|
||||
$assert(count($requests) === 1 && $requests[0]->getMethod() === 'PUT'
|
||||
&& $requests[0]->getHeaderLine('If-None-Match') === '*'
|
||||
&& $requests[0]->getHeaderLine('x-amz-meta-virt-sha256')
|
||||
=== $applied['sha256']
|
||||
&& str_contains(
|
||||
$requests[0]->getHeaderLine('Authorization'),
|
||||
'x-amz-meta-virt-sha256'
|
||||
), 'Immutable primary PUT lost its conditional signed identity.');
|
||||
$assert($transactionLevels === [0],
|
||||
'Immutable primary PUT ran inside a database transaction.');
|
||||
|
||||
// A provider rejection proves non-application and schedules a bounded retry;
|
||||
// it never recursively replays inside the same call.
|
||||
$rejected = $fixture('rescue');
|
||||
$clock += 1000;
|
||||
$responses = [new Response(403)];
|
||||
$requests = [];
|
||||
$transactionLevels = [];
|
||||
$module = $publisher($responses, $requests, $transactionLevels, $clock);
|
||||
$rejectedResult = $module->publish(
|
||||
$rejected['operation_id'],
|
||||
$rejected['artifact_id'],
|
||||
$rejected
|
||||
);
|
||||
$rejectedOperation = CloudStorageOperation::where(
|
||||
'operation_id',
|
||||
$rejected['operation_id']
|
||||
)->firstOrFail();
|
||||
$rejectedReplica = CloudStorageReplica::whereKey(
|
||||
$rejected['replica_id']
|
||||
)->firstOrFail();
|
||||
$assert($rejectedResult['phase'] === 'accepted'
|
||||
&& (int) $rejectedOperation->attempts === 1
|
||||
&& $rejectedOperation->outcome === null
|
||||
&& (int) $rejectedOperation->available_at > $clock
|
||||
&& (string) $rejectedReplica->state === 'reserved'
|
||||
&& count($requests) === 1,
|
||||
'Proven non-application was not delayed without replay.');
|
||||
|
||||
// A lost PUT receipt is settled only by exact-key size plus signed SHA
|
||||
// metadata. The temporary local file is deliberately removed before HEAD.
|
||||
$unknownApplied = $fixture('template');
|
||||
$clock += 1000;
|
||||
$responses = [
|
||||
new Response(503),
|
||||
new Response(200, [
|
||||
'Content-Length' => (string) $unknownApplied['size_bytes'],
|
||||
'ETag' => '"unknown-applied-etag"',
|
||||
'x-amz-version-id' => 'unknown-applied-version',
|
||||
'x-amz-meta-virt-sha256' => $unknownApplied['sha256'],
|
||||
]),
|
||||
];
|
||||
$requests = [];
|
||||
$transactionLevels = [];
|
||||
$module = $publisher($responses, $requests, $transactionLevels, $clock);
|
||||
$unknownResult = $module->publish(
|
||||
$unknownApplied['operation_id'],
|
||||
$unknownApplied['artifact_id'],
|
||||
$unknownApplied
|
||||
);
|
||||
$assert($unknownResult['phase'] === 'reconciling',
|
||||
'Unknown PUT did not enter read-only reconciliation.');
|
||||
@unlink($unknownApplied['path']);
|
||||
$clock += 60;
|
||||
$unknownResult = $module->publish(
|
||||
$unknownApplied['operation_id'],
|
||||
$unknownApplied['artifact_id'],
|
||||
$unknownApplied
|
||||
);
|
||||
$assert($unknownResult['primary_completed'] === true
|
||||
&& array_map(
|
||||
static fn (RequestInterface $request): string => $request->getMethod(),
|
||||
$requests
|
||||
) === ['PUT', 'HEAD']
|
||||
&& (string) $requests[0]->getUri() === (string) $requests[1]->getUri(),
|
||||
'Unknown PUT was replayed or reconciled against a different key.');
|
||||
$assert($transactionLevels === [0, 0],
|
||||
'Immutable PUT reconciliation ran remote I/O inside a transaction.');
|
||||
|
||||
// Two spaced exact-key absences are required before the original operation,
|
||||
// artifact and key may become dispatchable again.
|
||||
$absent = $fixture('template');
|
||||
$clock += 1000;
|
||||
$responses = [
|
||||
new Response(503),
|
||||
new Response(404),
|
||||
new Response(404),
|
||||
new Response(200, [
|
||||
'ETag' => '"retried-etag"',
|
||||
'x-amz-version-id' => 'retried-version',
|
||||
]),
|
||||
];
|
||||
$requests = [];
|
||||
$transactionLevels = [];
|
||||
$module = $publisher($responses, $requests, $transactionLevels, $clock);
|
||||
$artifactCount = CloudStorageArtifact::count();
|
||||
$operationCount = CloudStorageOperation::count();
|
||||
$module->publish($absent['operation_id'], $absent['artifact_id'], $absent);
|
||||
$clock += 60;
|
||||
$firstAbsent = $module->publish(
|
||||
$absent['operation_id'], $absent['artifact_id'], $absent
|
||||
);
|
||||
$assert($firstAbsent['phase'] === 'reconciling',
|
||||
'One absence observation incorrectly authorized a retry.');
|
||||
$clock += 60;
|
||||
$secondAbsent = $module->publish(
|
||||
$absent['operation_id'], $absent['artifact_id'], $absent
|
||||
);
|
||||
$assert($secondAbsent['phase'] === 'accepted',
|
||||
'Two stable absence observations did not release the original intent.');
|
||||
$clock += 60;
|
||||
$retried = $module->publish(
|
||||
$absent['operation_id'], $absent['artifact_id'], $absent
|
||||
);
|
||||
$methods = array_map(
|
||||
static fn (RequestInterface $request): string => $request->getMethod(),
|
||||
$requests
|
||||
);
|
||||
$assert($retried['primary_completed'] === true
|
||||
&& $methods === ['PUT', 'HEAD', 'HEAD', 'PUT']
|
||||
&& (string) $requests[0]->getUri() === (string) $requests[1]->getUri()
|
||||
&& (string) $requests[0]->getUri() === (string) $requests[2]->getUri()
|
||||
&& (string) $requests[0]->getUri() === (string) $requests[3]->getUri()
|
||||
&& (int) CloudStorageOperation::where(
|
||||
'operation_id', $absent['operation_id']
|
||||
)->value('attempts') === 2
|
||||
&& CloudStorageArtifact::count() === $artifactCount
|
||||
&& CloudStorageOperation::count() === $operationCount,
|
||||
'Stable absence did not retry the same frozen publication intent.');
|
||||
|
||||
// Existing content with a different signed identity is an attention state,
|
||||
// never evidence for success and never permission to overwrite the key.
|
||||
$conflict = $fixture('rescue');
|
||||
$clock += 1000;
|
||||
$responses = [
|
||||
new Response(503),
|
||||
new Response(200, [
|
||||
'Content-Length' => (string) $conflict['size_bytes'],
|
||||
'ETag' => '"conflicting-etag"',
|
||||
'x-amz-version-id' => 'conflicting-version',
|
||||
'x-amz-meta-virt-sha256' => str_repeat('f', 64),
|
||||
]),
|
||||
];
|
||||
$requests = [];
|
||||
$transactionLevels = [];
|
||||
$module = $publisher($responses, $requests, $transactionLevels, $clock);
|
||||
$module->publish($conflict['operation_id'], $conflict['artifact_id'], $conflict);
|
||||
$clock += 60;
|
||||
$conflictResult = $module->publish(
|
||||
$conflict['operation_id'], $conflict['artifact_id'], $conflict
|
||||
);
|
||||
$conflictOperation = CloudStorageOperation::where(
|
||||
'operation_id', $conflict['operation_id']
|
||||
)->firstOrFail();
|
||||
$assert($conflictResult['requires_attention'] === true
|
||||
&& (string) $conflictOperation->phase === 'terminal'
|
||||
&& (string) $conflictOperation->outcome === 'failed'
|
||||
&& (string) $conflictOperation->effect_resolution === 'unknown'
|
||||
&& (bool) $conflictOperation->safe_to_resubmit === false
|
||||
&& (string) CloudStorageArtifact::whereKey(
|
||||
$conflict['artifact_id']
|
||||
)->value('state') === 'degraded'
|
||||
&& (string) CloudStoragePublication::where(
|
||||
'artifact_id', $conflict['artifact_id']
|
||||
)->value('state') === 'failed'
|
||||
&& count($requests) === 2,
|
||||
'Conflicting immutable identity did not stop in operator attention.');
|
||||
|
||||
// The expected artifact fence runs before source validation or provider I/O.
|
||||
$fencedA = $fixture('template');
|
||||
$fencedB = $fixture('rescue');
|
||||
$clock += 1000;
|
||||
$responses = [new Response(200)];
|
||||
$requests = [];
|
||||
$transactionLevels = [];
|
||||
$module = $publisher($responses, $requests, $transactionLevels, $clock);
|
||||
$beforeA = CloudStorageOperation::where(
|
||||
'operation_id', $fencedA['operation_id']
|
||||
)->firstOrFail()->toArray();
|
||||
$expectCloudError(static fn () => $module->publish(
|
||||
$fencedA['operation_id'], $fencedB['artifact_id'], $fencedA
|
||||
), 409);
|
||||
$afterA = CloudStorageOperation::where(
|
||||
'operation_id', $fencedA['operation_id']
|
||||
)->firstOrFail()->toArray();
|
||||
$assert($requests === [] && $beforeA === $afterA,
|
||||
'Wrong expected artifact crossed the durable or remote mutation fence.');
|
||||
|
||||
// A settled publication is idempotent even after its local source is gone.
|
||||
@unlink($applied['path']);
|
||||
$requestsBefore = count($requests);
|
||||
$terminalAgain = new CloudStorageImmutablePrimaryPublication(
|
||||
new CloudObjectStorageS3Client(new Client([
|
||||
'handler' => HandlerStack::create(static function () {
|
||||
throw new RuntimeException('Terminal publication used HTTP.');
|
||||
}),
|
||||
])),
|
||||
static function (): array {
|
||||
throw new RuntimeException('Terminal publication resolved runtime.');
|
||||
},
|
||||
static fn (): int => time()
|
||||
);
|
||||
$idempotent = $terminalAgain->publish(
|
||||
$applied['operation_id'], $applied['artifact_id'], $applied
|
||||
);
|
||||
$assert($idempotent['primary_completed'] === true
|
||||
&& (int) CloudStorageOperation::where(
|
||||
'operation_id', $applied['operation_id']
|
||||
)->value('attempts') === 1
|
||||
&& count($requests) === $requestsBefore,
|
||||
'Terminal publication was not idempotent without its local file.');
|
||||
|
||||
foreach (CloudStorageOperation::query()->get() as $operation) {
|
||||
$stored = (string) $operation->result_json;
|
||||
$assert(!str_contains($stored, 'https://')
|
||||
&& !str_contains($stored, 'IMMUTABLEACCESSKEY')
|
||||
&& !str_contains($stored, 'immutable-secret-must-not-persist'),
|
||||
'Publication ledger persisted a credential or capability URL.');
|
||||
}
|
||||
|
||||
echo "Immutable primary publication behavior passed.\n";
|
||||
Loading…
Reference in New Issue