|
|
|
|
<?php
|
|
|
|
|
|
|
|
|
|
require dirname(__DIR__) . '/vendor/autoload.php';
|
|
|
|
|
|
|
|
|
|
use App\Models\CloudStorageBackend;
|
|
|
|
|
use App\Models\CloudInstanceBackup;
|
|
|
|
|
use App\Models\CloudStorageOperation;
|
|
|
|
|
use App\Models\CloudStorageReplica;
|
|
|
|
|
use App\Services\Cloud\CloudError;
|
|
|
|
|
use App\Services\Cloud\CloudStorageArtifactService;
|
|
|
|
|
use App\Services\Cloud\CloudStorageReplicaAttentionService;
|
|
|
|
|
use Illuminate\Database\Capsule\Manager as Capsule;
|
|
|
|
|
use Illuminate\Database\Schema\Blueprint;
|
|
|
|
|
|
|
|
|
|
$assert = static function (bool $condition, string $message): void {
|
|
|
|
|
if (!$condition) {
|
|
|
|
|
throw new RuntimeException($message);
|
|
|
|
|
}
|
|
|
|
|
};
|
|
|
|
|
$expectConflict = static function (callable $callback) use ($assert): void {
|
|
|
|
|
try {
|
|
|
|
|
$callback();
|
|
|
|
|
} catch (CloudError $error) {
|
|
|
|
|
$assert($error->httpStatus === 409,
|
|
|
|
|
'Unexpected CloudError status: ' . $error->httpStatus);
|
|
|
|
|
return;
|
|
|
|
|
}
|
|
|
|
|
throw new RuntimeException('Expected CloudError was not raised.');
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
$db = new Capsule();
|
|
|
|
|
$db->addConnection(['driver' => 'sqlite', 'database' => ':memory:']);
|
|
|
|
|
$db->setAsGlobal();
|
|
|
|
|
$db->bootEloquent();
|
|
|
|
|
$db->schema()->create('cloud_storage_backends', static function (Blueprint $t): void {
|
|
|
|
|
$t->increments('id');
|
|
|
|
|
$t->string('backend_key'); $t->string('identity_sha256');
|
|
|
|
|
$t->string('name'); $t->string('driver'); $t->string('endpoint');
|
|
|
|
|
$t->string('region'); $t->string('bucket'); $t->string('prefix');
|
|
|
|
|
$t->boolean('path_style'); $t->string('failure_domain');
|
|
|
|
|
$t->text('sealed_credentials'); $t->unsignedBigInteger('config_revision');
|
|
|
|
|
$t->unsignedBigInteger('tested_revision'); $t->string('admin_state');
|
|
|
|
|
$t->string('health_state'); $t->unsignedInteger('write_priority');
|
|
|
|
|
$t->unsignedInteger('consecutive_failures')->default(0);
|
|
|
|
|
$t->unsignedInteger('consecutive_successes')->default(0);
|
|
|
|
|
$t->string('last_error_code')->default('');
|
|
|
|
|
$t->unsignedBigInteger('last_tested_at')->default(0);
|
|
|
|
|
$t->unsignedBigInteger('last_healthy_at')->default(0);
|
|
|
|
|
$t->unsignedBigInteger('next_probe_at')->default(0);
|
|
|
|
|
$t->unsignedBigInteger('created_at'); $t->unsignedBigInteger('updated_at');
|
|
|
|
|
});
|
|
|
|
|
$db->schema()->create('cloud_storage_artifacts', static function (Blueprint $t): void {
|
|
|
|
|
$t->increments('id'); $t->string('artifact_key')->unique();
|
|
|
|
|
$t->string('kind'); $t->unsignedBigInteger('maximum_size_bytes');
|
|
|
|
|
$t->unsignedBigInteger('size_bytes'); $t->string('sha256');
|
|
|
|
|
$t->string('durability_policy'); $t->unsignedTinyInteger('desired_copies');
|
|
|
|
|
$t->unsignedTinyInteger('required_copies'); $t->string('state');
|
|
|
|
|
$t->string('last_operation_id')->nullable();
|
|
|
|
|
$t->unsignedBigInteger('verified_at');
|
|
|
|
|
$t->unsignedBigInteger('created_at'); $t->unsignedBigInteger('updated_at');
|
|
|
|
|
});
|
|
|
|
|
$db->schema()->create('cloud_storage_replicas', static function (Blueprint $t): void {
|
|
|
|
|
$t->increments('id'); $t->string('replica_key')->unique();
|
|
|
|
|
$t->unsignedBigInteger('artifact_id'); $t->unsignedBigInteger('backend_id');
|
|
|
|
|
$t->string('role'); $t->string('object_key');
|
|
|
|
|
$t->string('object_version_id')->default('');
|
|
|
|
|
$t->string('object_etag')->default('');
|
|
|
|
|
$t->unsignedBigInteger('size_bytes')->default(0);
|
|
|
|
|
$t->string('sha256')->default(''); $t->string('state');
|
|
|
|
|
$t->string('publish_operation_id')->nullable();
|
|
|
|
|
$t->string('delete_operation_id')->nullable();
|
|
|
|
|
$t->unsignedInteger('delete_generation')->default(0);
|
|
|
|
|
$t->unsignedInteger('delete_cycle_attempts')->default(0);
|
|
|
|
|
$t->unsignedInteger('delete_presence_confirmations')->default(0);
|
|
|
|
|
$t->unsignedBigInteger('delete_last_checked_at')->default(0);
|
|
|
|
|
$t->unsignedBigInteger('delete_next_check_at')->default(0);
|
|
|
|
|
$t->unsignedBigInteger('delete_attention_at')->nullable();
|
|
|
|
|
$t->string('lease_token')->nullable();
|
|
|
|
|
$t->unsignedBigInteger('lease_until')->default(0);
|
|
|
|
|
$t->string('error_code')->default('');
|
|
|
|
|
$t->unsignedBigInteger('verified_at')->default(0);
|
|
|
|
|
$t->unsignedBigInteger('created_at'); $t->unsignedBigInteger('updated_at');
|
|
|
|
|
});
|
|
|
|
|
$db->schema()->create('cloud_storage_operations', static function (Blueprint $t): void {
|
|
|
|
|
$t->increments('id'); $t->string('operation_id')->unique();
|
|
|
|
|
$t->string('request_idempotency_key')->unique();
|
|
|
|
|
$t->string('request_fingerprint'); $t->unsignedBigInteger('artifact_id');
|
|
|
|
|
$t->unsignedBigInteger('replica_id')->nullable();
|
|
|
|
|
$t->unsignedBigInteger('source_backend_id')->nullable();
|
|
|
|
|
$t->unsignedBigInteger('target_backend_id')->nullable();
|
|
|
|
|
$t->string('kind'); $t->string('phase'); $t->string('outcome')->nullable();
|
|
|
|
|
$t->string('effect_resolution');
|
|
|
|
|
$t->boolean('effect_may_have_applied')->nullable();
|
|
|
|
|
$t->boolean('safe_to_resubmit')->nullable();
|
|
|
|
|
$t->unsignedInteger('attempts'); $t->string('lease_token')->nullable();
|
|
|
|
|
$t->unsignedBigInteger('lease_until'); $t->unsignedBigInteger('available_at');
|
|
|
|
|
$t->unsignedBigInteger('next_check_at'); $t->string('error_code');
|
|
|
|
|
$t->text('result_json')->nullable(); $t->unsignedBigInteger('terminal_at')->nullable();
|
|
|
|
|
$t->unsignedBigInteger('created_at'); $t->unsignedBigInteger('updated_at');
|
|
|
|
|
});
|
|
|
|
|
$db->schema()->create('cloud_storage_publications', static function (Blueprint $t): void {
|
|
|
|
|
$t->increments('id'); $t->string('publication_key')->unique();
|
|
|
|
|
$t->unsignedBigInteger('artifact_id')->unique();
|
|
|
|
|
$t->unsignedBigInteger('backup_id')->nullable()->unique();
|
|
|
|
|
$t->string('template_import_intent_id')->nullable()->unique();
|
|
|
|
|
$t->string('rescue_template_id')->nullable()->unique();
|
|
|
|
|
$t->string('state'); $t->unsignedInteger('attempts')->default(0);
|
|
|
|
|
$t->unsignedBigInteger('available_at')->default(0);
|
|
|
|
|
$t->string('lease_token')->nullable(); $t->unsignedBigInteger('lease_until')->default(0);
|
|
|
|
|
$t->string('last_error_code')->default('');
|
|
|
|
|
$t->unsignedBigInteger('coverage_ready_at')->nullable();
|
|
|
|
|
$t->unsignedBigInteger('finalized_at')->nullable();
|
|
|
|
|
$t->unsignedBigInteger('created_at'); $t->unsignedBigInteger('updated_at');
|
|
|
|
|
});
|
|
|
|
|
$db->schema()->create('cloud_instance_backups', static function (Blueprint $t): void {
|
|
|
|
|
$t->increments('id');
|
|
|
|
|
$t->unsignedBigInteger('storage_artifact_id')->nullable();
|
|
|
|
|
$t->unsignedBigInteger('active_job_id')->nullable();
|
|
|
|
|
$t->string('state');
|
|
|
|
|
$t->string('error_code')->default('');
|
|
|
|
|
$t->unsignedBigInteger('updated_at')->default(0);
|
|
|
|
|
});
|
|
|
|
|
|
|
|
|
|
$now = 2000000000;
|
|
|
|
|
foreach ([[1, 'a', 10], [2, 'b', 20]] as [$id, $domain, $priority]) {
|
|
|
|
|
$backend = new CloudStorageBackend();
|
|
|
|
|
$backend->id = $id;
|
|
|
|
|
$backend->backend_key = str_repeat((string) $id, 32);
|
|
|
|
|
$backend->identity_sha256 = str_repeat((string) $id, 64);
|
|
|
|
|
$backend->name = 'provider-' . $domain;
|
|
|
|
|
$backend->driver = 's3';
|
|
|
|
|
$backend->endpoint = 'https://' . $domain . '.example.test';
|
|
|
|
|
$backend->region = 'us-east-1';
|
|
|
|
|
$backend->bucket = 'bucket-' . $domain;
|
|
|
|
|
$backend->prefix = $domain;
|
|
|
|
|
$backend->path_style = true;
|
|
|
|
|
$backend->failure_domain = $domain;
|
|
|
|
|
$backend->sealed_credentials = 'sealed';
|
|
|
|
|
$backend->config_revision = $backend->tested_revision = 1;
|
|
|
|
|
$backend->admin_state = 'active';
|
|
|
|
|
$backend->health_state = 'healthy';
|
|
|
|
|
$backend->write_priority = $priority;
|
|
|
|
|
$backend->consecutive_failures = 0;
|
|
|
|
|
$backend->consecutive_successes = 1;
|
|
|
|
|
$backend->last_error_code = '';
|
|
|
|
|
$backend->last_tested_at = $backend->last_healthy_at = $now;
|
|
|
|
|
$backend->next_probe_at = $now + 300;
|
|
|
|
|
$backend->created_at = $backend->updated_at = $now;
|
|
|
|
|
$backend->save();
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
$newPublication = static function (int $backupId, string $suffix) use ($now): array {
|
|
|
|
|
return CloudStorageArtifactService::beginAgentPublication(
|
|
|
|
|
'backup', $suffix, 20 * 1073741824,
|
|
|
|
|
'backup-prepare-' . $backupId,
|
|
|
|
|
hash('sha256', 'backup-' . $backupId),
|
|
|
|
|
['backup_id' => $backupId]
|
|
|
|
|
);
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
$view = $newPublication(41, 'vm-backups/a/system.qcow2');
|
|
|
|
|
$operationId = (string) $view['operation_id'];
|
|
|
|
|
$assert(CloudStorageArtifactService::primaryOperationForBackup(41)
|
|
|
|
|
=== $operationId,
|
|
|
|
|
'Backup owner did not resolve to its original primary operation.');
|
|
|
|
|
$work = CloudStorageArtifactService::claimPrimaryPreparation(
|
|
|
|
|
$operationId, $now, 120
|
|
|
|
|
);
|
|
|
|
|
$assert(is_array($work) && $work['mode'] === 'dispatch'
|
|
|
|
|
&& $work['backup_id'] === 41,
|
|
|
|
|
'Accepted primary preparation was not leased for dispatch.');
|
|
|
|
|
$assert(CloudStorageArtifactService::claimPrimaryPreparation(
|
|
|
|
|
$operationId, $now + 1, 120
|
|
|
|
|
) === null, 'A second worker crossed the active preparation lease.');
|
|
|
|
|
|
|
|
|
|
CloudStorageArtifactService::markPrimaryPreparationUnknown(
|
|
|
|
|
$operationId, (string) $work['lease_token'],
|
|
|
|
|
'multipart_create_unknown', $now + 2, 5,
|
|
|
|
|
[
|
|
|
|
|
'absence_confirmations' => 1,
|
|
|
|
|
'multipart_absence_confirmations' => 1,
|
|
|
|
|
]
|
|
|
|
|
);
|
|
|
|
|
$reconcile = CloudStorageArtifactService::claimPrimaryPreparation(
|
|
|
|
|
$operationId, $now + 7, 120
|
|
|
|
|
);
|
|
|
|
|
$assert(is_array($reconcile) && $reconcile['mode'] === 'reconcile',
|
|
|
|
|
'Unknown CreateMultipart effect did not enter read-only reconciliation.');
|
|
|
|
|
$expectConflict(static fn () =>
|
|
|
|
|
CloudStorageArtifactService::reschedulePrimaryAfterStableAbsence(
|
|
|
|
|
$operationId, (string) $reconcile['lease_token'],
|
|
|
|
|
[
|
|
|
|
|
'absence_confirmations' => 1,
|
|
|
|
|
'multipart_absence_confirmations' => 1,
|
|
|
|
|
], $now + 8, 5
|
|
|
|
|
)
|
|
|
|
|
);
|
|
|
|
|
CloudStorageArtifactService::reschedulePrimaryAfterStableAbsence(
|
|
|
|
|
$operationId, (string) $reconcile['lease_token'],
|
|
|
|
|
[
|
|
|
|
|
'absence_confirmations' => 2,
|
|
|
|
|
'multipart_absence_confirmations' => 2,
|
|
|
|
|
], $now + 8, 5
|
|
|
|
|
);
|
|
|
|
|
$rescheduled = CloudStorageOperation::where(
|
|
|
|
|
'operation_id', $operationId
|
|
|
|
|
)->firstOrFail();
|
|
|
|
|
$assert((string) $rescheduled->phase === 'accepted'
|
|
|
|
|
&& (int) $rescheduled->attempts === 1,
|
|
|
|
|
'Two stable absence proofs did not reopen the same operation.');
|
|
|
|
|
|
|
|
|
|
$dispatch = CloudStorageArtifactService::claimPrimaryPreparation(
|
|
|
|
|
$operationId, $now + 13, 120
|
|
|
|
|
);
|
|
|
|
|
$metadata = [
|
|
|
|
|
'upload_id' => 'upload-a',
|
|
|
|
|
'part_size_bytes' => 67108864,
|
|
|
|
|
'part_count' => 320,
|
|
|
|
|
'maximum_size_bytes' => 20 * 1073741824,
|
|
|
|
|
'multipart_presence_confirmations' => 0,
|
|
|
|
|
];
|
|
|
|
|
CloudStorageArtifactService::markPrimaryAwaitingReceipt(
|
|
|
|
|
$operationId, (string) $dispatch['lease_token'], $metadata, $now + 14
|
|
|
|
|
);
|
|
|
|
|
$stored = (string) CloudStorageOperation::where(
|
|
|
|
|
'operation_id', $operationId
|
|
|
|
|
)->value('result_json');
|
|
|
|
|
$assert(str_contains($stored, 'upload-a') && !str_contains($stored, 'http'),
|
|
|
|
|
'Primary preparation persisted a capability URL or lost upload identity.');
|
|
|
|
|
$expectConflict(static fn () =>
|
|
|
|
|
CloudStorageArtifactService::markPrimaryAwaitingReceipt(
|
|
|
|
|
$operationId, (string) $dispatch['lease_token'],
|
|
|
|
|
array_merge($metadata, ['upload_id' => 'replacement']), $now + 15
|
|
|
|
|
)
|
|
|
|
|
);
|
|
|
|
|
$expectConflict(static fn () =>
|
|
|
|
|
CloudStorageArtifactService::markPrimaryAwaitingReceipt(
|
|
|
|
|
$operationId, (string) $dispatch['lease_token'],
|
|
|
|
|
array_merge($metadata, ['part_count' => 319]), $now + 15
|
|
|
|
|
)
|
|
|
|
|
);
|
|
|
|
|
$expectConflict(static fn () =>
|
|
|
|
|
CloudStorageArtifactService::markPrimaryPreparationUnknown(
|
|
|
|
|
$operationId,
|
|
|
|
|
(string) $dispatch['lease_token'],
|
|
|
|
|
'multipart_create_unknown',
|
|
|
|
|
$now + 15,
|
|
|
|
|
5,
|
|
|
|
|
array_merge($metadata, ['part_count' => 319])
|
|
|
|
|
)
|
|
|
|
|
);
|
|
|
|
|
CloudStorageArtifactService::releasePrimaryPreparationLease(
|
|
|
|
|
$operationId, (string) $dispatch['lease_token'], $now + 15
|
|
|
|
|
);
|
|
|
|
|
$sign = CloudStorageArtifactService::claimPrimaryPreparation(
|
|
|
|
|
$operationId, $now + 16, 120
|
|
|
|
|
);
|
|
|
|
|
$assert(is_array($sign) && $sign['mode'] === 'sign'
|
|
|
|
|
&& ($sign['metadata']['upload_id'] ?? '') === 'upload-a',
|
|
|
|
|
'Persisted multipart identity was not recoverable for local re-signing.');
|
|
|
|
|
CloudStorageArtifactService::releasePrimaryPreparationLease(
|
|
|
|
|
$operationId, (string) $sign['lease_token'], $now + 17
|
|
|
|
|
);
|
|
|
|
|
|
|
|
|
|
$expiredView = $newPublication(43, 'vm-backups/expired/system.qcow2');
|
|
|
|
|
$expiredId = (string) $expiredView['operation_id'];
|
|
|
|
|
$expiredWork = CloudStorageArtifactService::claimPrimaryPreparation(
|
|
|
|
|
$expiredId, $now + 18, 120
|
|
|
|
|
);
|
|
|
|
|
$expiredMetadata = [
|
|
|
|
|
'upload_id' => 'expired-upload',
|
|
|
|
|
'part_size_bytes' => 67108864,
|
|
|
|
|
'part_count' => 320,
|
|
|
|
|
'maximum_size_bytes' => 20 * 1073741824,
|
|
|
|
|
];
|
|
|
|
|
CloudStorageArtifactService::markPrimaryAwaitingReceipt(
|
|
|
|
|
$expiredId,
|
|
|
|
|
(string) $expiredWork['lease_token'],
|
|
|
|
|
$expiredMetadata,
|
|
|
|
|
$now + 19
|
|
|
|
|
);
|
|
|
|
|
CloudStorageArtifactService::markPrimaryPreparationUnknown(
|
|
|
|
|
$expiredId,
|
|
|
|
|
(string) $expiredWork['lease_token'],
|
|
|
|
|
'primary_multipart_absence_unconfirmed',
|
|
|
|
|
$now + 20,
|
|
|
|
|
5,
|
|
|
|
|
$expiredMetadata + [
|
|
|
|
|
'absence_confirmations' => 1,
|
|
|
|
|
'multipart_absence_confirmations' => 1,
|
|
|
|
|
]
|
|
|
|
|
);
|
|
|
|
|
$expiredReconcile = CloudStorageArtifactService::claimPrimaryPreparation(
|
|
|
|
|
$expiredId, $now + 25, 120
|
|
|
|
|
);
|
|
|
|
|
CloudStorageArtifactService::reschedulePrimaryAfterStableAbsence(
|
|
|
|
|
$expiredId,
|
|
|
|
|
(string) $expiredReconcile['lease_token'],
|
|
|
|
|
$expiredMetadata + [
|
|
|
|
|
'absence_confirmations' => 2,
|
|
|
|
|
'multipart_absence_confirmations' => 2,
|
|
|
|
|
],
|
|
|
|
|
$now + 26,
|
|
|
|
|
5
|
|
|
|
|
);
|
|
|
|
|
$expiredOperation = CloudStorageOperation::where(
|
|
|
|
|
'operation_id', $expiredId
|
|
|
|
|
)->firstOrFail();
|
|
|
|
|
$assert((string) $expiredOperation->phase === 'accepted'
|
|
|
|
|
&& !str_contains((string) $expiredOperation->result_json, 'expired-upload'),
|
|
|
|
|
'Stable object and multipart absence did not retire the old upload identity.');
|
|
|
|
|
|
|
|
|
|
$attentionView = $newPublication(42, 'vm-backups/b/system.qcow2');
|
|
|
|
|
$attentionId = (string) $attentionView['operation_id'];
|
|
|
|
|
Capsule::table('cloud_instance_backups')->insert([
|
|
|
|
|
'id' => 42,
|
|
|
|
|
'storage_artifact_id' => (int) $attentionView['artifact_id'],
|
|
|
|
|
'active_job_id' => null,
|
|
|
|
|
'state' => 'creating',
|
|
|
|
|
]);
|
|
|
|
|
$attentionWork = CloudStorageArtifactService::claimPrimaryPreparation(
|
|
|
|
|
$attentionId, $now + 20, 120
|
|
|
|
|
);
|
|
|
|
|
CloudStorageArtifactService::markPrimaryPreparationUnknown(
|
|
|
|
|
$attentionId, (string) $attentionWork['lease_token'],
|
|
|
|
|
'multipart_create_unknown', $now + 21, 5, [
|
|
|
|
|
'multipart_attention_confirmations' => 1,
|
|
|
|
|
'multipart_attention_error_code' =>
|
|
|
|
|
'primary_multipart_identity_ambiguous',
|
|
|
|
|
]
|
|
|
|
|
);
|
|
|
|
|
$attentionReconcile = CloudStorageArtifactService::claimPrimaryPreparation(
|
|
|
|
|
$attentionId, $now + 26, 120
|
|
|
|
|
);
|
|
|
|
|
$expectConflict(static fn () =>
|
|
|
|
|
CloudStorageArtifactService::markPrimaryPreparationAttention(
|
|
|
|
|
$attentionId, (string) $attentionReconcile['lease_token'],
|
|
|
|
|
'primary_multipart_identity_ambiguous', $now + 27, [
|
|
|
|
|
'multipart_attention_confirmations' => 1,
|
|
|
|
|
'multipart_attention_error_code' =>
|
|
|
|
|
'primary_multipart_identity_ambiguous',
|
|
|
|
|
]
|
|
|
|
|
)
|
|
|
|
|
);
|
|
|
|
|
CloudStorageArtifactService::markPrimaryPreparationAttention(
|
|
|
|
|
$attentionId, (string) $attentionReconcile['lease_token'],
|
|
|
|
|
'primary_multipart_identity_ambiguous', $now + 27, [
|
|
|
|
|
'multipart_attention_confirmations' => 2,
|
|
|
|
|
'multipart_attention_error_code' =>
|
|
|
|
|
'primary_multipart_identity_ambiguous',
|
|
|
|
|
]
|
|
|
|
|
);
|
|
|
|
|
$attention = CloudStorageOperation::where(
|
|
|
|
|
'operation_id', $attentionId
|
|
|
|
|
)->firstOrFail();
|
|
|
|
|
$attentionReplica = CloudStorageReplica::where(
|
|
|
|
|
'publish_operation_id', $attentionId
|
|
|
|
|
)->firstOrFail();
|
|
|
|
|
$attentionBackup = CloudInstanceBackup::findOrFail(42);
|
|
|
|
|
$assert((string) $attention->phase === 'terminal'
|
|
|
|
|
&& (string) $attention->effect_resolution === 'unknown'
|
|
|
|
|
&& !(bool) $attention->safe_to_resubmit
|
|
|
|
|
&& (string) $attentionReplica->state === 'degraded'
|
|
|
|
|
&& (string) $attentionBackup->state === 'uncertain',
|
|
|
|
|
'Stable multipart conflict did not stop automatic replay in attention.');
|
|
|
|
|
$attentionList = CloudStorageReplicaAttentionService::listView();
|
|
|
|
|
$attentionItem = $attentionList['items'][0] ?? [];
|
|
|
|
|
$encodedAttention = json_encode($attentionList, JSON_THROW_ON_ERROR);
|
|
|
|
|
$assert(($attentionList['summary']['count'] ?? 0) === 1
|
|
|
|
|
&& ($attentionItem['operation_kind'] ?? '') === 'publish'
|
|
|
|
|
&& ($attentionItem['owner_type'] ?? '') === 'backup'
|
|
|
|
|
&& ($attentionItem['owner_id'] ?? 0) === 42
|
|
|
|
|
&& ($attentionItem['can_retry'] ?? false) === true
|
|
|
|
|
&& !str_contains($encodedAttention, 'object_key')
|
|
|
|
|
&& !str_contains($encodedAttention, 'upload_id')
|
|
|
|
|
&& !str_contains($encodedAttention, 'part_urls'),
|
|
|
|
|
'Primary preparation attention was not safely exposed to administrators.');
|
|
|
|
|
CloudStorageReplicaAttentionService::retry($attentionId);
|
|
|
|
|
$attention->refresh();
|
|
|
|
|
$attentionReplica->refresh();
|
|
|
|
|
$attentionBackup->refresh();
|
|
|
|
|
$assert((string) $attention->phase === 'reconciling'
|
|
|
|
|
&& (string) $attention->error_code === 'admin_reconcile_requested'
|
|
|
|
|
&& $attention->terminal_at === null
|
|
|
|
|
&& (string) $attentionReplica->state === 'publishing'
|
|
|
|
|
&& (string) $attentionBackup->state === 'creating',
|
|
|
|
|
'Administrator reconciliation did not reopen the original operation.');
|
|
|
|
|
$attentionBackup->state = 'uncertain';
|
|
|
|
|
$attentionBackup->error_code = 'stale_projection';
|
|
|
|
|
$attentionBackup->save();
|
|
|
|
|
CloudStorageArtifactService::reopenPrimaryPreparationAttention(
|
|
|
|
|
$attentionId,
|
|
|
|
|
$now + 28
|
|
|
|
|
);
|
|
|
|
|
$attentionBackup->refresh();
|
|
|
|
|
$assert((string) $attentionBackup->state === 'creating'
|
|
|
|
|
&& (string) $attentionBackup->error_code
|
|
|
|
|
=== 'admin_reconcile_requested',
|
|
|
|
|
'Idempotent administrator retry did not normalize the backup projection.');
|
|
|
|
|
$reopened = CloudStorageArtifactService::claimPrimaryPreparation(
|
|
|
|
|
$attentionId,
|
|
|
|
|
$now + 28,
|
|
|
|
|
120
|
|
|
|
|
);
|
|
|
|
|
$assert(is_array($reopened) && $reopened['mode'] === 'reconcile',
|
|
|
|
|
'Reopened primary attention did not return to read-only reconciliation.');
|
|
|
|
|
|
|
|
|
|
echo "Storage primary preparation tests passed.\n";
|