feat(storage): add durable immutable primary publisher

go
codex 2 weeks ago
parent c11bfb65d2
commit 72a0358903

@ -0,0 +1,895 @@
<?php
namespace App\Services\Cloud;
use App\Models\CloudStorageArtifact;
use App\Models\CloudStorageOperation;
use App\Models\CloudStoragePublication;
use App\Models\CloudStorageReplica;
/**
* Publishes one known local file through an already-reserved primary intent.
*
* The business owner must first call CloudStorageArtifactService::
* beginPublication(). This module never allocates a replacement artifact or
* object key. A lost PUT receipt is reconciled only with exact-key HEAD
* evidence; the same operation becomes dispatchable again only after two
* spaced, stable absence observations.
*/
final class CloudStorageImmutablePrimaryPublication
{
private const LEASE_SECONDS = 300;
private const RECONCILE_DELAY = 60;
private const RETRY_DELAY = 300;
private const MAX_DISPATCH_ATTEMPTS = 3;
private const MAX_RECONCILE_FAILURES = 6;
/** @var \Closure(int,bool):array<string,mixed> */
private readonly \Closure $runtimeResolver;
/** @var \Closure():int */
private readonly \Closure $clock;
/**
* @param null|callable(int,bool):array<string,mixed> $runtimeResolver
* @param null|callable():int $clock
*/
public function __construct(
private readonly ?CloudObjectStorageS3Client $client = null,
?callable $runtimeResolver = null,
?callable $clock = null
) {
$this->runtimeResolver = $runtimeResolver === null
? static fn (int $backendId, bool $healthy): array =>
CloudObjectStorageBackendRegistry::runtime(
$backendId,
$healthy
)
: \Closure::fromCallable($runtimeResolver);
$this->clock = $clock === null
? static fn (): int => time()
: \Closure::fromCallable($clock);
}
/**
* Advance one immutable primary publication by at most one remote action.
*
* @param array{path:string,size_bytes:int,sha256:string} $file
* @return array<string,mixed>
*/
public function publish(
string $operationId,
int $expectedArtifactId,
array $file
): array {
self::assertOperationId($operationId);
if ($expectedArtifactId < 1) {
throw new CloudError('对象存储制品身份无效', 422);
}
$snapshot = $this->snapshot($operationId, $expectedArtifactId);
if (($snapshot['terminal'] ?? false) === true) {
return $snapshot;
}
// Reconciliation is intentionally independent of the temporary local
// file. Once a PUT may have happened, only the frozen ledger identity
// and an exact-key HEAD are authoritative.
if ((string) $snapshot['phase'] === 'accepted') {
self::assertFile($file, $snapshot);
}
$now = ($this->clock)();
$work = $this->claim(
$operationId,
$expectedArtifactId,
$now
);
if ($work === null) {
return $this->snapshot($operationId, $expectedArtifactId);
}
if ((string) $work['mode'] === 'dispatch') {
try {
// Recheck after the dispatch lease is owned so a changed
// source can never reach the provider under this intent.
self::assertFile($file, $work);
} catch (\Throwable $error) {
$this->safeFailure(
$work,
'immutable_primary_source_changed',
$now
);
throw $error;
}
}
try {
$runtime = ($this->runtimeResolver)(
(int) $work['backend_id'],
(string) $work['mode'] === 'dispatch'
);
self::assertRuntime($runtime);
} catch (\Throwable) {
if ((string) $work['mode'] === 'dispatch') {
$this->safeFailure(
$work,
'immutable_primary_runtime_unavailable',
$now
);
} else {
$this->reconcileFailure(
$work,
'immutable_primary_runtime_unavailable',
$now
);
}
return $this->snapshot($operationId, $expectedArtifactId);
}
if ((string) $work['mode'] === 'dispatch') {
$this->dispatch($work, $runtime, $file);
} else {
$this->reconcile($work, $runtime);
}
return $this->snapshot($operationId, $expectedArtifactId);
}
/** @return array<string,mixed> */
private function snapshot(
string $operationId,
int $expectedArtifactId
): array {
$operation = CloudStorageOperation::query()
->where('operation_id', $operationId)->first();
if (!$operation instanceof CloudStorageOperation
|| (string) $operation->kind !== 'publish'
|| (int) $operation->artifact_id !== $expectedArtifactId) {
throw new CloudError('对象存储发布操作与制品身份不一致', 409);
}
$artifact = CloudStorageArtifact::query()
->whereKey($expectedArtifactId)->first();
$replica = CloudStorageReplica::query()
->whereKey((int) $operation->replica_id)->first();
$publication = CloudStoragePublication::query()
->where('artifact_id', $expectedArtifactId)->first();
if (!$artifact instanceof CloudStorageArtifact
|| !$replica instanceof CloudStorageReplica
|| !$publication instanceof CloudStoragePublication
|| (int) $replica->artifact_id !== $expectedArtifactId
|| (string) $replica->role !== 'primary') {
throw new CloudError('对象存储主副本账本不一致', 409);
}
$terminal = (string) $operation->phase === 'terminal';
return [
'operation_id' => (string) $operation->operation_id,
'artifact_id' => (int) $artifact->id,
'phase' => (string) $operation->phase,
'outcome' => $operation->outcome === null
? null : (string) $operation->outcome,
'effect_resolution' => (string) $operation->effect_resolution,
'attempts' => (int) $operation->attempts,
'next_check_at' => (int) $operation->next_check_at,
'error_code' => (string) $operation->error_code,
'terminal' => $terminal,
'primary_completed' => $terminal
&& (string) $operation->outcome === 'succeeded',
'publication_state' => (string) $publication->state,
'ready_to_finalize' => in_array(
(string) $publication->state,
['ready_to_finalize', 'finalizing', 'completed'],
true
),
'requires_attention' => ($terminal
&& (string) $operation->outcome === 'failed')
|| (string) $publication->state === 'failed',
'object_key' => (string) $replica->object_key,
'size_bytes' => (int) $artifact->size_bytes,
'sha256' => (string) $artifact->sha256,
];
}
/** @return array<string,mixed>|null */
private function claim(
string $operationId,
int $expectedArtifactId,
int $now
): ?array {
$connection = (new CloudStorageOperation())->getConnection();
return $connection->transaction(function () use (
$operationId,
$expectedArtifactId,
$now
): ?array {
$operation = CloudStorageOperation::query()
->where('operation_id', $operationId)
->lockForUpdate()->first();
if (!$operation instanceof CloudStorageOperation
|| (string) $operation->kind !== 'publish'
|| (int) $operation->artifact_id !== $expectedArtifactId) {
throw new CloudError(
'对象存储发布操作与制品身份不一致',
409
);
}
[$artifact, $replica, $publication] =
$this->lockedContext($operation);
if ((string) $operation->phase === 'terminal'
|| in_array((string) $publication->state,
['completed', 'failed', 'retired'], true)) {
return null;
}
if ((string) $artifact->state !== 'publishing'
|| !in_array((string) $replica->state,
['reserved', 'publishing'], true)) {
throw new CloudError('对象存储主副本发布状态不一致', 409);
}
if (is_string($operation->lease_token)
&& $operation->lease_token !== ''
&& (int) $operation->lease_until > $now) {
return null;
}
$phase = (string) $operation->phase;
$mode = null;
if ($phase === 'accepted'
&& (int) $operation->available_at <= $now) {
$mode = 'dispatch';
$operation->phase = 'dispatching';
$operation->effect_resolution = 'not_applicable';
$operation->attempts = (int) $operation->attempts + 1;
$replica->state = 'publishing';
} elseif ($phase === 'reconciling'
&& (int) $operation->next_check_at <= $now) {
$mode = 'reconcile';
} elseif (in_array($phase,
['dispatching', 'awaiting_receipt'], true)
&& (int) $operation->lease_until <= $now) {
$mode = 'reconcile';
$operation->phase = 'reconciling';
$operation->effect_resolution = 'unknown';
$operation->error_code =
'immutable_primary_lease_expired';
$operation->next_check_at = $now;
}
if ($mode === null) {
return null;
}
$lease = bin2hex(random_bytes(16));
$operation->lease_token = $lease;
$operation->lease_until = $now + self::LEASE_SECONDS;
$operation->updated_at = $now;
$operation->save();
$replica->lease_token = $lease;
$replica->lease_until = $now + self::LEASE_SECONDS;
$replica->updated_at = $now;
$replica->save();
return [
'mode' => $mode,
'operation_id' => (string) $operation->operation_id,
'artifact_id' => (int) $artifact->id,
'replica_id' => (int) $replica->id,
'backend_id' => (int) $replica->backend_id,
'object_key' => (string) $replica->object_key,
'size_bytes' => (int) $artifact->size_bytes,
'sha256' => (string) $artifact->sha256,
'lease_token' => $lease,
'attempts' => (int) $operation->attempts,
];
}, 3);
}
/**
* @param array<string,mixed> $work
* @param array<string,mixed> $runtime
* @param array{path:string,size_bytes:int,sha256:string} $file
*/
private function dispatch(array $work, array $runtime, array $file): void
{
try {
$identity = $this->s3()->putImmutableFile(
$runtime['config'],
$runtime['credentials'],
(string) $work['object_key'],
(string) $file['path'],
(int) $file['size_bytes'],
(string) $file['sha256']
);
} catch (CloudObjectStorageTransferError $error) {
if ($error->effectMayHaveApplied
|| $error->publicCode === 'object_already_exists') {
$this->markUnknown($work, $error->publicCode);
} else {
$this->safeFailure(
$work,
$error->publicCode,
($this->clock)()
);
}
return;
} catch (\Throwable) {
$this->markUnknown($work, 'immutable_primary_put_unknown');
return;
}
if ((int) ($identity['size_bytes'] ?? 0)
!== (int) $work['size_bytes']
|| !is_string($identity['etag'] ?? null)
|| !is_string($identity['version_id'] ?? null)) {
$this->markUnknown(
$work,
'immutable_primary_put_receipt_invalid'
);
return;
}
CloudStorageArtifactService::completePrimary(
(string) $work['operation_id'],
(int) $work['artifact_id'],
[
'size_bytes' => (int) $work['size_bytes'],
'sha256' => (string) $work['sha256'],
'etag' => (string) $identity['etag'],
'version_id' => (string) $identity['version_id'],
]
);
}
/** @param array<string,mixed> $work @param array<string,mixed> $runtime */
private function reconcile(array $work, array $runtime): void
{
try {
$identity = $this->s3()->inspectObject(
$runtime['config'],
$runtime['credentials'],
(string) $work['object_key']
);
} catch (\Throwable) {
$this->reconcileFailure(
$work,
'immutable_primary_head_unavailable',
($this->clock)()
);
return;
}
if ($identity === null) {
$this->recordAbsence($work, ($this->clock)());
return;
}
if ((int) ($identity['size_bytes'] ?? 0)
!== (int) $work['size_bytes']
|| !is_string($identity['sha256'] ?? null)
|| !hash_equals(
(string) $work['sha256'],
(string) $identity['sha256']
)) {
$this->attention(
$work,
'immutable_primary_identity_conflict',
($this->clock)()
);
return;
}
CloudStorageArtifactService::completePrimary(
(string) $work['operation_id'],
(int) $work['artifact_id'],
[
'size_bytes' => (int) $work['size_bytes'],
'sha256' => (string) $work['sha256'],
'etag' => (string) ($identity['etag'] ?? ''),
'version_id' => (string) ($identity['version_id'] ?? ''),
]
);
}
/** @param array<string,mixed> $work */
private function markUnknown(array $work, string $code): void
{
$now = ($this->clock)();
$this->mutateLeased($work, function (
CloudStorageOperation $operation,
CloudStorageArtifact $artifact,
CloudStorageReplica $replica,
CloudStoragePublication $publication,
array $copyOperations,
array $replicas
) use ($code, $now): void {
$metadata = self::metadata($operation->result_json);
$metadata['absence_confirmations'] = 0;
$operation->phase = 'reconciling';
$operation->effect_resolution = 'unknown';
$operation->lease_token = null;
$operation->lease_until = 0;
$operation->next_check_at = $now + self::RECONCILE_DELAY;
$operation->error_code = self::errorCode($code);
$operation->result_json = self::encodedMetadata($metadata);
$operation->updated_at = $now;
$operation->save();
$replica->state = 'publishing';
$replica->lease_token = null;
$replica->lease_until = 0;
$replica->error_code = self::errorCode($code);
$replica->updated_at = $now;
$replica->save();
$artifact->state = 'publishing';
$artifact->updated_at = $now;
$artifact->save();
});
}
/** @param array<string,mixed> $work */
private function safeFailure(array $work, string $code, int $now): void
{
$this->mutateLeased($work, function (
CloudStorageOperation $operation,
CloudStorageArtifact $artifact,
CloudStorageReplica $replica,
CloudStoragePublication $publication,
array $copyOperations,
array $replicas
) use ($code, $now): void {
$code = self::errorCode($code);
if ((int) $operation->attempts >= self::MAX_DISPATCH_ATTEMPTS) {
$this->settleAttention(
$operation,
$artifact,
$replica,
$publication,
$copyOperations,
$replicas,
false,
'immutable_primary_publish_retry_exhausted',
$now
);
return;
}
$operation->phase = 'accepted';
$operation->effect_resolution = 'not_applicable';
$operation->lease_token = null;
$operation->lease_until = 0;
$operation->available_at = $now + self::RETRY_DELAY;
$operation->next_check_at = 0;
$operation->error_code = $code;
$operation->result_json = self::encodedMetadata(
self::resetEvidence($operation->result_json)
);
$operation->updated_at = $now;
$operation->save();
$replica->state = 'reserved';
$replica->lease_token = null;
$replica->lease_until = 0;
$replica->error_code = $code;
$replica->updated_at = $now;
$replica->save();
});
}
/** @param array<string,mixed> $work */
private function reconcileFailure(
array $work,
string $code,
int $now
): void {
$this->mutateLeased($work, function (
CloudStorageOperation $operation,
CloudStorageArtifact $artifact,
CloudStorageReplica $replica,
CloudStoragePublication $publication,
array $copyOperations,
array $replicas
) use ($code, $now): void {
$metadata = self::metadata($operation->result_json);
$failures = (int) ($metadata['reconcile_failures'] ?? 0) + 1;
if ($failures >= self::MAX_RECONCILE_FAILURES) {
$this->settleAttention(
$operation,
$artifact,
$replica,
$publication,
$copyOperations,
$replicas,
true,
'immutable_primary_reconcile_exhausted',
$now
);
return;
}
$metadata['reconcile_failures'] = $failures;
$operation->phase = 'reconciling';
$operation->effect_resolution = 'unknown';
$operation->lease_token = null;
$operation->lease_until = 0;
$operation->next_check_at = $now + self::RETRY_DELAY;
$operation->error_code = self::errorCode($code);
$operation->result_json = self::encodedMetadata($metadata);
$operation->updated_at = $now;
$operation->save();
$replica->state = 'publishing';
$replica->lease_token = null;
$replica->lease_until = 0;
$replica->error_code = self::errorCode($code);
$replica->updated_at = $now;
$replica->save();
});
}
/** @param array<string,mixed> $work */
private function recordAbsence(array $work, int $now): void
{
$this->mutateLeased($work, function (
CloudStorageOperation $operation,
CloudStorageArtifact $artifact,
CloudStorageReplica $replica,
CloudStoragePublication $publication,
array $copyOperations,
array $replicas
) use ($now): void {
$metadata = self::metadata($operation->result_json);
$confirmations = min(
2,
(int) ($metadata['absence_confirmations'] ?? 0) + 1
);
$metadata['absence_confirmations'] = $confirmations;
$metadata['reconcile_failures'] = 0;
if ($confirmations < 2) {
$operation->phase = 'reconciling';
$operation->effect_resolution = 'unknown';
$operation->lease_token = null;
$operation->lease_until = 0;
$operation->next_check_at = $now + self::RECONCILE_DELAY;
$operation->error_code = 'immutable_primary_absent_once';
$operation->result_json = self::encodedMetadata($metadata);
$operation->updated_at = $now;
$operation->save();
$replica->state = 'publishing';
$replica->lease_token = null;
$replica->lease_until = 0;
$replica->error_code = 'immutable_primary_absent_once';
$replica->updated_at = $now;
$replica->save();
return;
}
if ((int) $operation->attempts >= self::MAX_DISPATCH_ATTEMPTS) {
$this->settleAttention(
$operation,
$artifact,
$replica,
$publication,
$copyOperations,
$replicas,
false,
'immutable_primary_absence_retry_exhausted',
$now
);
return;
}
$operation->phase = 'accepted';
$operation->effect_resolution = 'not_applicable';
$operation->lease_token = null;
$operation->lease_until = 0;
$operation->available_at = $now + self::RECONCILE_DELAY;
$operation->next_check_at = 0;
$operation->error_code = '';
$operation->result_json = self::encodedMetadata(
self::resetEvidence($operation->result_json)
);
$operation->updated_at = $now;
$operation->save();
$replica->state = 'reserved';
$replica->lease_token = null;
$replica->lease_until = 0;
$replica->error_code = '';
$replica->updated_at = $now;
$replica->save();
});
}
/** @param array<string,mixed> $work */
private function attention(array $work, string $code, int $now): void
{
$this->mutateLeased($work, function (
CloudStorageOperation $operation,
CloudStorageArtifact $artifact,
CloudStorageReplica $replica,
CloudStoragePublication $publication,
array $copyOperations,
array $replicas
) use ($code, $now): void {
$this->settleAttention(
$operation,
$artifact,
$replica,
$publication,
$copyOperations,
$replicas,
true,
self::errorCode($code),
$now
);
});
}
/**
* @param array<string,mixed> $work
* @param callable(CloudStorageOperation,CloudStorageArtifact,
* CloudStorageReplica,CloudStoragePublication,
* array<int,CloudStorageOperation>,array<int,CloudStorageReplica>):void $callback
*/
private function mutateLeased(array $work, callable $callback): void
{
$connection = (new CloudStorageOperation())->getConnection();
$connection->transaction(function () use ($work, $callback): void {
$operation = CloudStorageOperation::query()
->where('operation_id', (string) $work['operation_id'])
->lockForUpdate()->firstOrFail();
if ((int) $operation->artifact_id !== (int) $work['artifact_id']) {
throw new CloudError('对象存储发布制品身份已变化', 409);
}
$copyOperations = CloudStorageOperation::query()
->where('artifact_id', (int) $operation->artifact_id)
->where('kind', 'copy')->orderBy('id')
->lockForUpdate()->get()->all();
[$artifact, $replica, $publication, $replicas] =
$this->lockedContext($operation, true);
if (!hash_equals(
(string) $operation->lease_token,
(string) $work['lease_token']
)
|| !hash_equals(
(string) $replica->lease_token,
(string) $work['lease_token']
)) {
throw new CloudError('对象存储主副本发布租约已失效', 409);
}
$callback(
$operation,
$artifact,
$replica,
$publication,
$copyOperations,
$replicas
);
}, 3);
}
/**
* @return array{0:CloudStorageArtifact,1:CloudStorageReplica,
* 2:CloudStoragePublication,3?:array<int,CloudStorageReplica>}
*/
private function lockedContext(
CloudStorageOperation $operation,
bool $includeReplicas = false
): array {
$artifact = CloudStorageArtifact::query()
->whereKey((int) $operation->artifact_id)
->lockForUpdate()->firstOrFail();
$replicas = CloudStorageReplica::query()
->where('artifact_id', (int) $artifact->id)->orderBy('id')
->lockForUpdate()->get()->all();
$replica = null;
foreach ($replicas as $candidate) {
if ((int) $candidate->id === (int) $operation->replica_id) {
$replica = $candidate;
break;
}
}
$publication = CloudStoragePublication::query()
->where('artifact_id', (int) $artifact->id)
->lockForUpdate()->firstOrFail();
if (!$replica instanceof CloudStorageReplica
|| (string) $replica->role !== 'primary') {
throw new CloudError('对象存储主副本账本不一致', 409);
}
return $includeReplicas
? [$artifact, $replica, $publication, $replicas]
: [$artifact, $replica, $publication];
}
/**
* @param array<int,CloudStorageOperation> $copyOperations
* @param array<int,CloudStorageReplica> $replicas
*/
private function settleAttention(
CloudStorageOperation $operation,
CloudStorageArtifact $artifact,
CloudStorageReplica $replica,
CloudStoragePublication $publication,
array $copyOperations,
array $replicas,
bool $effectUnknown,
string $code,
int $now
): void {
$operation->phase = 'terminal';
$operation->outcome = 'failed';
$operation->effect_resolution = $effectUnknown
? 'unknown' : 'not_applied';
$operation->effect_may_have_applied = $effectUnknown ? 1 : 0;
$operation->safe_to_resubmit = 0;
$operation->lease_token = null;
$operation->lease_until = 0;
$operation->available_at = 0;
$operation->next_check_at = 0;
$operation->error_code = self::errorCode($code);
$operation->terminal_at = $now;
$operation->updated_at = $now;
$operation->save();
$replica->state = 'degraded';
$replica->lease_token = null;
$replica->lease_until = 0;
$replica->error_code = self::errorCode($code);
$replica->updated_at = $now;
$replica->save();
foreach ($copyOperations as $copy) {
if ((string) $copy->phase === 'terminal') {
continue;
}
// Secondary work is blocked until primary success. Never invent a
// safe non-dispatch result if an inconsistent ledger says a copy
// has already crossed its own dispatch fence.
if ((string) $copy->phase !== 'accepted') {
continue;
}
$copy->phase = 'terminal';
$copy->outcome = 'failed';
$copy->effect_resolution = 'not_applied';
$copy->effect_may_have_applied = 0;
$copy->safe_to_resubmit = 0;
$copy->lease_token = null;
$copy->lease_until = 0;
$copy->available_at = 0;
$copy->next_check_at = 0;
$copy->error_code = 'primary_publish_attention';
$copy->terminal_at = $now;
$copy->updated_at = $now;
$copy->save();
foreach ($replicas as $secondary) {
if ((int) $secondary->id !== (int) $copy->replica_id) {
continue;
}
$secondary->state = 'degraded';
$secondary->lease_token = null;
$secondary->lease_until = 0;
$secondary->error_code = 'primary_publish_attention';
$secondary->updated_at = $now;
$secondary->save();
break;
}
}
$artifact->state = 'degraded';
$artifact->updated_at = $now;
$artifact->save();
$publication->state = 'failed';
$publication->available_at = 0;
$publication->lease_token = null;
$publication->lease_until = 0;
$publication->last_error_code = self::errorCode($code);
$publication->updated_at = $now;
$publication->save();
}
/**
* @return array{absence_confirmations:int,reconcile_failures:int,
* maximum_size_bytes?:int,object_suffix?:string}
*/
private static function metadata(?string $json): array
{
if ($json === null || trim($json) === '') {
return [
'absence_confirmations' => 0,
'reconcile_failures' => 0,
];
}
try {
$value = json_decode($json, true, 16, JSON_THROW_ON_ERROR);
} catch (\Throwable) {
throw new CloudError('对象存储发布核对证据无效', 409);
}
if (!is_array($value) || array_is_list($value)
|| array_diff(array_keys($value), [
'absence_confirmations', 'reconcile_failures',
'maximum_size_bytes', 'object_suffix',
]) !== []) {
throw new CloudError('对象存储发布核对证据无效', 409);
}
$absence = $value['absence_confirmations'] ?? 0;
$failures = $value['reconcile_failures'] ?? 0;
if (!is_int($absence) || $absence < 0 || $absence > 2
|| !is_int($failures) || $failures < 0
|| $failures > self::MAX_RECONCILE_FAILURES) {
throw new CloudError('对象存储发布核对证据无效', 409);
}
$metadata = [
'absence_confirmations' => $absence,
'reconcile_failures' => $failures,
];
if (array_key_exists('maximum_size_bytes', $value)) {
$maximum = $value['maximum_size_bytes'];
if (!is_int($maximum) || $maximum < 1) {
throw new CloudError('对象存储发布核对证据无效', 409);
}
$metadata['maximum_size_bytes'] = $maximum;
}
if (array_key_exists('object_suffix', $value)) {
$suffix = $value['object_suffix'];
if (!is_string($suffix) || $suffix === ''
|| strlen($suffix) > 512
|| preg_match('/[\x00-\x1f\x7f]/', $suffix) === 1) {
throw new CloudError('对象存储发布核对证据无效', 409);
}
$metadata['object_suffix'] = $suffix;
}
return $metadata;
}
private static function encodedMetadata(array $metadata): string
{
$normalized = self::metadata(json_encode(
$metadata,
JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES
));
return json_encode(
$normalized,
JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES
);
}
/**
* Preserve beginPublication()'s immutable context while clearing only
* this module's transient reconciliation evidence.
*
* @return array{absence_confirmations:int,reconcile_failures:int,
* maximum_size_bytes?:int,object_suffix?:string}
*/
private static function resetEvidence(?string $json): array
{
$metadata = self::metadata($json);
$metadata['absence_confirmations'] = 0;
$metadata['reconcile_failures'] = 0;
return $metadata;
}
/** @param array<string,mixed> $file @param array<string,mixed> $snapshot */
private static function assertFile(array $file, array $snapshot): void
{
$path = $file['path'] ?? null;
$size = $file['size_bytes'] ?? null;
$sha = $file['sha256'] ?? null;
$actual = is_string($path) && is_file($path) && is_readable($path)
? hash_file('sha256', $path) : false;
if (!is_string($path) || !is_int($size) || $size < 1
|| !is_string($sha)
|| preg_match('/^[a-f0-9]{64}$/D', $sha) !== 1
|| @filesize($path) !== $size
|| !is_string($actual) || !hash_equals($sha, $actual)
|| $size !== (int) $snapshot['size_bytes']
|| !hash_equals($sha, (string) $snapshot['sha256'])) {
throw new CloudError('本地不可变制品文件身份无效', 422);
}
}
/** @param array<string,mixed> $runtime */
private static function assertRuntime(array $runtime): void
{
if (!is_array($runtime['config'] ?? null)
|| array_is_list($runtime['config'])
|| !is_array($runtime['credentials'] ?? null)
|| array_is_list($runtime['credentials'])
|| !is_int($runtime['revision'] ?? null)
|| (int) $runtime['revision'] < 1) {
throw new CloudError('对象存储发布运行时无效', 503);
}
}
private static function assertOperationId(string $operationId): void
{
if (preg_match('/^[a-f0-9]{32}$/D', $operationId) !== 1) {
throw new CloudError('对象存储发布操作标识无效', 422);
}
}
private static function errorCode(string $code): string
{
$code = strtolower(trim($code));
return preg_match('/^[a-z][a-z0-9_]{0,95}$/D', $code) === 1
? $code : 'immutable_primary_publication_failed';
}
private function s3(): CloudObjectStorageS3Client
{
return $this->client ?? new CloudObjectStorageS3Client();
}
}

@ -0,0 +1,544 @@
<?php
require dirname(__DIR__) . '/vendor/autoload.php';
use App\Models\CloudStorageArtifact;
use App\Models\CloudStorageOperation;
use App\Models\CloudStoragePublication;
use App\Models\CloudStorageReplica;
use App\Services\Cloud\CloudError;
use App\Services\Cloud\CloudObjectStorageS3Client;
use App\Services\Cloud\CloudStorageArtifactService;
use App\Services\Cloud\CloudStorageImmutablePrimaryPublication;
use GuzzleHttp\Client;
use GuzzleHttp\HandlerStack;
use GuzzleHttp\Promise\Create;
use GuzzleHttp\Psr7\Response;
use Illuminate\Database\Capsule\Manager as Capsule;
use Illuminate\Database\Schema\Blueprint;
use Psr\Http\Message\RequestInterface;
$assert = static function (bool $condition, string $message): void {
if (!$condition) {
throw new RuntimeException($message);
}
};
$expectCloudError = static function (
callable $callback,
int $status
) use ($assert): void {
try {
$callback();
} catch (CloudError $error) {
$assert($error->httpStatus === $status,
'Unexpected CloudError status: ' . $error->httpStatus);
return;
}
throw new RuntimeException('Expected CloudError was not raised.');
};
$db = new Capsule();
$db->addConnection(['driver' => 'sqlite', 'database' => ':memory:']);
$db->setAsGlobal();
$db->bootEloquent();
$db->schema()->create('cloud_storage_backends', static function (
Blueprint $table
): void {
$table->increments('id');
$table->string('backend_key');
$table->string('identity_sha256');
$table->string('name');
$table->string('driver');
$table->string('endpoint');
$table->string('region');
$table->string('bucket');
$table->string('prefix');
$table->boolean('path_style');
$table->string('failure_domain');
$table->text('sealed_credentials');
$table->unsignedBigInteger('config_revision');
$table->unsignedBigInteger('tested_revision');
$table->string('admin_state');
$table->string('health_state');
$table->unsignedInteger('write_priority');
$table->unsignedInteger('consecutive_failures')->default(0);
$table->unsignedInteger('consecutive_successes')->default(0);
$table->string('last_error_code')->default('');
$table->unsignedBigInteger('last_tested_at')->default(0);
$table->unsignedBigInteger('last_healthy_at')->default(0);
$table->unsignedBigInteger('next_probe_at')->default(0);
$table->unsignedBigInteger('created_at');
$table->unsignedBigInteger('updated_at');
});
$db->schema()->create('cloud_storage_artifacts', static function (
Blueprint $table
): void {
$table->increments('id');
$table->string('artifact_key')->unique();
$table->string('kind');
$table->unsignedBigInteger('maximum_size_bytes');
$table->unsignedBigInteger('size_bytes');
$table->string('sha256');
$table->string('durability_policy');
$table->unsignedTinyInteger('desired_copies');
$table->unsignedTinyInteger('required_copies');
$table->string('state');
$table->string('last_operation_id')->nullable();
$table->unsignedBigInteger('verified_at');
$table->unsignedBigInteger('created_at');
$table->unsignedBigInteger('updated_at');
});
$db->schema()->create('cloud_storage_replicas', static function (
Blueprint $table
): void {
$table->increments('id');
$table->string('replica_key')->unique();
$table->unsignedBigInteger('artifact_id');
$table->unsignedBigInteger('backend_id');
$table->string('role');
$table->string('object_key');
$table->string('object_version_id')->default('');
$table->string('object_etag')->default('');
$table->unsignedBigInteger('size_bytes')->default(0);
$table->string('sha256')->default('');
$table->string('state');
$table->string('publish_operation_id')->nullable();
$table->string('delete_operation_id')->nullable();
$table->unsignedInteger('delete_generation')->default(0);
$table->unsignedInteger('delete_cycle_attempts')->default(0);
$table->unsignedInteger('delete_presence_confirmations')->default(0);
$table->unsignedBigInteger('delete_last_checked_at')->default(0);
$table->unsignedBigInteger('delete_next_check_at')->default(0);
$table->unsignedBigInteger('delete_attention_at')->nullable();
$table->string('lease_token')->nullable();
$table->unsignedBigInteger('lease_until')->default(0);
$table->string('error_code')->default('');
$table->unsignedBigInteger('verified_at')->default(0);
$table->unsignedBigInteger('created_at');
$table->unsignedBigInteger('updated_at');
});
$db->schema()->create('cloud_storage_operations', static function (
Blueprint $table
): void {
$table->increments('id');
$table->string('operation_id')->unique();
$table->string('request_idempotency_key')->unique();
$table->string('request_fingerprint');
$table->unsignedBigInteger('artifact_id');
$table->unsignedBigInteger('replica_id')->nullable();
$table->unsignedBigInteger('source_backend_id')->nullable();
$table->unsignedBigInteger('target_backend_id')->nullable();
$table->string('kind');
$table->string('phase');
$table->string('outcome')->nullable();
$table->string('effect_resolution');
$table->boolean('effect_may_have_applied')->nullable();
$table->boolean('safe_to_resubmit')->nullable();
$table->unsignedInteger('attempts');
$table->string('lease_token')->nullable();
$table->unsignedBigInteger('lease_until');
$table->unsignedBigInteger('available_at');
$table->unsignedBigInteger('next_check_at');
$table->string('error_code');
$table->text('result_json')->nullable();
$table->unsignedBigInteger('terminal_at')->nullable();
$table->unsignedBigInteger('created_at');
$table->unsignedBigInteger('updated_at');
});
$db->schema()->create('cloud_storage_publications', static function (
Blueprint $table
): void {
$table->increments('id');
$table->string('publication_key')->unique();
$table->unsignedBigInteger('artifact_id')->unique();
$table->unsignedBigInteger('backup_id')->nullable()->unique();
$table->string('template_import_intent_id')->nullable()->unique();
$table->string('rescue_template_id')->nullable()->unique();
$table->string('state');
$table->unsignedInteger('attempts')->default(0);
$table->unsignedBigInteger('available_at')->default(0);
$table->string('lease_token')->nullable();
$table->unsignedBigInteger('lease_until')->default(0);
$table->string('last_error_code')->default('');
$table->unsignedBigInteger('coverage_ready_at')->nullable();
$table->unsignedBigInteger('finalized_at')->nullable();
$table->unsignedBigInteger('created_at');
$table->unsignedBigInteger('updated_at');
});
$databaseNow = time();
foreach ([
[1, 'primary', 'provider-a', 10],
[2, 'secondary', 'provider-b', 20],
] as [$id, $name, $domain, $priority]) {
Capsule::table('cloud_storage_backends')->insert([
'id' => $id,
'backend_key' => str_repeat((string) $id, 32),
'identity_sha256' => str_repeat((string) $id, 64),
'name' => $name,
'driver' => 's3',
'endpoint' => 'https://' . $name . '.example.test',
'region' => 'us-east-1',
'bucket' => $name . '-bucket',
'prefix' => $name,
'path_style' => 1,
'failure_domain' => $domain,
'sealed_credentials' => 'sealed',
'config_revision' => 1,
'tested_revision' => 1,
'admin_state' => 'active',
'health_state' => 'healthy',
'write_priority' => $priority,
'consecutive_failures' => 0,
'consecutive_successes' => 1,
'last_error_code' => '',
'last_tested_at' => $databaseNow,
'last_healthy_at' => $databaseNow,
'next_probe_at' => $databaseNow + 300,
'created_at' => $databaseNow,
'updated_at' => $databaseNow,
]);
}
$temporaryPaths = [];
register_shutdown_function(static function () use (&$temporaryPaths): void {
foreach ($temporaryPaths as $path) {
@unlink($path);
}
});
$fixtureSequence = 0;
$fixture = static function (
string $kind = 'template'
) use (&$fixtureSequence, &$temporaryPaths): array {
$fixtureSequence++;
$path = tempnam(sys_get_temp_dir(), 'virt-primary-publication-');
if ($path === false) {
throw new RuntimeException('Cannot create publication fixture.');
}
file_put_contents(
$path,
str_repeat('immutable-primary-' . $fixtureSequence . '-', 64)
);
$temporaryPaths[] = $path;
$size = filesize($path);
$sha256 = hash_file('sha256', $path);
if (!is_int($size) || !is_string($sha256)) {
throw new RuntimeException('Cannot identify publication fixture.');
}
$ownerId = str_pad(dechex($fixtureSequence), 32, '0', STR_PAD_LEFT);
$owner = $kind === 'rescue'
? ['rescue_template_id' => $ownerId]
: ['template_import_intent_id' => $ownerId];
$view = CloudStorageArtifactService::beginPublication(
$kind,
$kind . '/fixture-' . $fixtureSequence . '/system.qcow2',
$size,
$sha256,
'immutable-primary-' . str_pad(
(string) $fixtureSequence,
4,
'0',
STR_PAD_LEFT
),
hash('sha256', $kind . '|' . $fixtureSequence),
$owner
);
return [
'path' => $path,
'size_bytes' => $size,
'sha256' => $sha256,
'operation_id' => (string) $view['operation_id'],
'artifact_id' => (int) $view['artifact_id'],
'replica_id' => (int) $view['replica_id'],
'object_key' => (string) $view['object_key'],
];
};
$publisher = static function (
array &$responses,
array &$requests,
array &$transactionLevels,
int &$clock
): CloudStorageImmutablePrimaryPublication {
$handler = static function (
RequestInterface $request,
array $options
) use (&$responses, &$requests, &$transactionLevels) {
$transactionLevels[] = Capsule::connection()->transactionLevel();
$requests[] = $request;
if ($responses === []) {
throw new RuntimeException('Unexpected object-storage request.');
}
$response = array_shift($responses);
if (!$response instanceof Response) {
throw new RuntimeException('Invalid object-storage response fixture.');
}
return Create::promiseFor($response);
};
$client = new CloudObjectStorageS3Client(new Client([
'handler' => HandlerStack::create($handler),
]));
return new CloudStorageImmutablePrimaryPublication(
$client,
static fn (int $backendId, bool $healthy): array => [
'backend_id' => $backendId,
'revision' => 1,
'config' => [
'endpoint' => 'https://objects.example.test',
'region' => 'us-east-1',
'bucket' => 'immutable-fixtures',
'prefix' => 'unused-runtime-prefix',
'path_style' => true,
],
'credentials' => [
'access_key_id' => 'IMMUTABLEACCESSKEY',
'secret_access_key' => 'immutable-secret-must-not-persist',
],
],
static function () use (&$clock): int {
return $clock;
}
);
};
// A successful immutable PUT settles only the primary. Template publication
// remains behind the required-two protection gate until its copy completes.
$applied = $fixture('template');
$clock = time() + 10;
$responses = [new Response(200, [
'ETag' => '"applied-etag"',
'x-amz-version-id' => 'applied-version',
])];
$requests = [];
$transactionLevels = [];
$module = $publisher($responses, $requests, $transactionLevels, $clock);
$appliedResult = $module->publish(
$applied['operation_id'],
$applied['artifact_id'],
$applied
);
$assert($appliedResult['primary_completed'] === true
&& $appliedResult['publication_state'] === 'waiting_protection'
&& $appliedResult['ready_to_finalize'] === false,
'Primary success bypassed the template publication quorum.');
$assert(count($requests) === 1 && $requests[0]->getMethod() === 'PUT'
&& $requests[0]->getHeaderLine('If-None-Match') === '*'
&& $requests[0]->getHeaderLine('x-amz-meta-virt-sha256')
=== $applied['sha256']
&& str_contains(
$requests[0]->getHeaderLine('Authorization'),
'x-amz-meta-virt-sha256'
), 'Immutable primary PUT lost its conditional signed identity.');
$assert($transactionLevels === [0],
'Immutable primary PUT ran inside a database transaction.');
// A provider rejection proves non-application and schedules a bounded retry;
// it never recursively replays inside the same call.
$rejected = $fixture('rescue');
$clock += 1000;
$responses = [new Response(403)];
$requests = [];
$transactionLevels = [];
$module = $publisher($responses, $requests, $transactionLevels, $clock);
$rejectedResult = $module->publish(
$rejected['operation_id'],
$rejected['artifact_id'],
$rejected
);
$rejectedOperation = CloudStorageOperation::where(
'operation_id',
$rejected['operation_id']
)->firstOrFail();
$rejectedReplica = CloudStorageReplica::whereKey(
$rejected['replica_id']
)->firstOrFail();
$assert($rejectedResult['phase'] === 'accepted'
&& (int) $rejectedOperation->attempts === 1
&& $rejectedOperation->outcome === null
&& (int) $rejectedOperation->available_at > $clock
&& (string) $rejectedReplica->state === 'reserved'
&& count($requests) === 1,
'Proven non-application was not delayed without replay.');
// A lost PUT receipt is settled only by exact-key size plus signed SHA
// metadata. The temporary local file is deliberately removed before HEAD.
$unknownApplied = $fixture('template');
$clock += 1000;
$responses = [
new Response(503),
new Response(200, [
'Content-Length' => (string) $unknownApplied['size_bytes'],
'ETag' => '"unknown-applied-etag"',
'x-amz-version-id' => 'unknown-applied-version',
'x-amz-meta-virt-sha256' => $unknownApplied['sha256'],
]),
];
$requests = [];
$transactionLevels = [];
$module = $publisher($responses, $requests, $transactionLevels, $clock);
$unknownResult = $module->publish(
$unknownApplied['operation_id'],
$unknownApplied['artifact_id'],
$unknownApplied
);
$assert($unknownResult['phase'] === 'reconciling',
'Unknown PUT did not enter read-only reconciliation.');
@unlink($unknownApplied['path']);
$clock += 60;
$unknownResult = $module->publish(
$unknownApplied['operation_id'],
$unknownApplied['artifact_id'],
$unknownApplied
);
$assert($unknownResult['primary_completed'] === true
&& array_map(
static fn (RequestInterface $request): string => $request->getMethod(),
$requests
) === ['PUT', 'HEAD']
&& (string) $requests[0]->getUri() === (string) $requests[1]->getUri(),
'Unknown PUT was replayed or reconciled against a different key.');
$assert($transactionLevels === [0, 0],
'Immutable PUT reconciliation ran remote I/O inside a transaction.');
// Two spaced exact-key absences are required before the original operation,
// artifact and key may become dispatchable again.
$absent = $fixture('template');
$clock += 1000;
$responses = [
new Response(503),
new Response(404),
new Response(404),
new Response(200, [
'ETag' => '"retried-etag"',
'x-amz-version-id' => 'retried-version',
]),
];
$requests = [];
$transactionLevels = [];
$module = $publisher($responses, $requests, $transactionLevels, $clock);
$artifactCount = CloudStorageArtifact::count();
$operationCount = CloudStorageOperation::count();
$module->publish($absent['operation_id'], $absent['artifact_id'], $absent);
$clock += 60;
$firstAbsent = $module->publish(
$absent['operation_id'], $absent['artifact_id'], $absent
);
$assert($firstAbsent['phase'] === 'reconciling',
'One absence observation incorrectly authorized a retry.');
$clock += 60;
$secondAbsent = $module->publish(
$absent['operation_id'], $absent['artifact_id'], $absent
);
$assert($secondAbsent['phase'] === 'accepted',
'Two stable absence observations did not release the original intent.');
$clock += 60;
$retried = $module->publish(
$absent['operation_id'], $absent['artifact_id'], $absent
);
$methods = array_map(
static fn (RequestInterface $request): string => $request->getMethod(),
$requests
);
$assert($retried['primary_completed'] === true
&& $methods === ['PUT', 'HEAD', 'HEAD', 'PUT']
&& (string) $requests[0]->getUri() === (string) $requests[1]->getUri()
&& (string) $requests[0]->getUri() === (string) $requests[2]->getUri()
&& (string) $requests[0]->getUri() === (string) $requests[3]->getUri()
&& (int) CloudStorageOperation::where(
'operation_id', $absent['operation_id']
)->value('attempts') === 2
&& CloudStorageArtifact::count() === $artifactCount
&& CloudStorageOperation::count() === $operationCount,
'Stable absence did not retry the same frozen publication intent.');
// Existing content with a different signed identity is an attention state,
// never evidence for success and never permission to overwrite the key.
$conflict = $fixture('rescue');
$clock += 1000;
$responses = [
new Response(503),
new Response(200, [
'Content-Length' => (string) $conflict['size_bytes'],
'ETag' => '"conflicting-etag"',
'x-amz-version-id' => 'conflicting-version',
'x-amz-meta-virt-sha256' => str_repeat('f', 64),
]),
];
$requests = [];
$transactionLevels = [];
$module = $publisher($responses, $requests, $transactionLevels, $clock);
$module->publish($conflict['operation_id'], $conflict['artifact_id'], $conflict);
$clock += 60;
$conflictResult = $module->publish(
$conflict['operation_id'], $conflict['artifact_id'], $conflict
);
$conflictOperation = CloudStorageOperation::where(
'operation_id', $conflict['operation_id']
)->firstOrFail();
$assert($conflictResult['requires_attention'] === true
&& (string) $conflictOperation->phase === 'terminal'
&& (string) $conflictOperation->outcome === 'failed'
&& (string) $conflictOperation->effect_resolution === 'unknown'
&& (bool) $conflictOperation->safe_to_resubmit === false
&& (string) CloudStorageArtifact::whereKey(
$conflict['artifact_id']
)->value('state') === 'degraded'
&& (string) CloudStoragePublication::where(
'artifact_id', $conflict['artifact_id']
)->value('state') === 'failed'
&& count($requests) === 2,
'Conflicting immutable identity did not stop in operator attention.');
// The expected artifact fence runs before source validation or provider I/O.
$fencedA = $fixture('template');
$fencedB = $fixture('rescue');
$clock += 1000;
$responses = [new Response(200)];
$requests = [];
$transactionLevels = [];
$module = $publisher($responses, $requests, $transactionLevels, $clock);
$beforeA = CloudStorageOperation::where(
'operation_id', $fencedA['operation_id']
)->firstOrFail()->toArray();
$expectCloudError(static fn () => $module->publish(
$fencedA['operation_id'], $fencedB['artifact_id'], $fencedA
), 409);
$afterA = CloudStorageOperation::where(
'operation_id', $fencedA['operation_id']
)->firstOrFail()->toArray();
$assert($requests === [] && $beforeA === $afterA,
'Wrong expected artifact crossed the durable or remote mutation fence.');
// A settled publication is idempotent even after its local source is gone.
@unlink($applied['path']);
$requestsBefore = count($requests);
$terminalAgain = new CloudStorageImmutablePrimaryPublication(
new CloudObjectStorageS3Client(new Client([
'handler' => HandlerStack::create(static function () {
throw new RuntimeException('Terminal publication used HTTP.');
}),
])),
static function (): array {
throw new RuntimeException('Terminal publication resolved runtime.');
},
static fn (): int => time()
);
$idempotent = $terminalAgain->publish(
$applied['operation_id'], $applied['artifact_id'], $applied
);
$assert($idempotent['primary_completed'] === true
&& (int) CloudStorageOperation::where(
'operation_id', $applied['operation_id']
)->value('attempts') === 1
&& count($requests) === $requestsBefore,
'Terminal publication was not idempotent without its local file.');
foreach (CloudStorageOperation::query()->get() as $operation) {
$stored = (string) $operation->result_json;
$assert(!str_contains($stored, 'https://')
&& !str_contains($stored, 'IMMUTABLEACCESSKEY')
&& !str_contains($stored, 'immutable-secret-must-not-persist'),
'Publication ledger persisted a credential or capability URL.');
}
echo "Immutable primary publication behavior passed.\n";
Loading…
Cancel
Save