You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
virt/web/tests/CloudStoragePrimaryPreparat...

409 lines
17 KiB
PHTML

<?php
require dirname(__DIR__) . '/vendor/autoload.php';
use App\Models\CloudStorageBackend;
use App\Models\CloudInstanceBackup;
use App\Models\CloudStorageOperation;
use App\Models\CloudStorageReplica;
use App\Services\Cloud\CloudError;
use App\Services\Cloud\CloudStorageArtifactService;
use App\Services\Cloud\CloudStorageReplicaAttentionService;
use Illuminate\Database\Capsule\Manager as Capsule;
use Illuminate\Database\Schema\Blueprint;
$assert = static function (bool $condition, string $message): void {
if (!$condition) {
throw new RuntimeException($message);
}
};
$expectConflict = static function (callable $callback) use ($assert): void {
try {
$callback();
} catch (CloudError $error) {
$assert($error->httpStatus === 409,
'Unexpected CloudError status: ' . $error->httpStatus);
return;
}
throw new RuntimeException('Expected CloudError was not raised.');
};
$db = new Capsule();
$db->addConnection(['driver' => 'sqlite', 'database' => ':memory:']);
$db->setAsGlobal();
$db->bootEloquent();
$db->schema()->create('cloud_storage_backends', static function (Blueprint $t): void {
$t->increments('id');
$t->string('backend_key'); $t->string('identity_sha256');
$t->string('name'); $t->string('driver'); $t->string('endpoint');
$t->string('region'); $t->string('bucket'); $t->string('prefix');
$t->boolean('path_style'); $t->string('failure_domain');
$t->text('sealed_credentials'); $t->unsignedBigInteger('config_revision');
$t->unsignedBigInteger('tested_revision'); $t->string('admin_state');
$t->string('health_state'); $t->unsignedInteger('write_priority');
$t->unsignedInteger('consecutive_failures')->default(0);
$t->unsignedInteger('consecutive_successes')->default(0);
$t->string('last_error_code')->default('');
$t->unsignedBigInteger('last_tested_at')->default(0);
$t->unsignedBigInteger('last_healthy_at')->default(0);
$t->unsignedBigInteger('next_probe_at')->default(0);
$t->unsignedBigInteger('created_at'); $t->unsignedBigInteger('updated_at');
});
$db->schema()->create('cloud_storage_artifacts', static function (Blueprint $t): void {
$t->increments('id'); $t->string('artifact_key')->unique();
$t->string('kind'); $t->unsignedBigInteger('maximum_size_bytes');
$t->unsignedBigInteger('size_bytes'); $t->string('sha256');
$t->string('durability_policy'); $t->unsignedTinyInteger('desired_copies');
$t->unsignedTinyInteger('required_copies'); $t->string('state');
$t->string('last_operation_id')->nullable();
$t->unsignedBigInteger('verified_at');
$t->unsignedBigInteger('created_at'); $t->unsignedBigInteger('updated_at');
});
$db->schema()->create('cloud_storage_replicas', static function (Blueprint $t): void {
$t->increments('id'); $t->string('replica_key')->unique();
$t->unsignedBigInteger('artifact_id'); $t->unsignedBigInteger('backend_id');
$t->string('role'); $t->string('object_key');
$t->string('object_version_id')->default('');
$t->string('object_etag')->default('');
$t->unsignedBigInteger('size_bytes')->default(0);
$t->string('sha256')->default(''); $t->string('state');
$t->string('publish_operation_id')->nullable();
$t->string('delete_operation_id')->nullable();
$t->unsignedInteger('delete_generation')->default(0);
$t->unsignedInteger('delete_cycle_attempts')->default(0);
$t->unsignedInteger('delete_presence_confirmations')->default(0);
$t->unsignedBigInteger('delete_last_checked_at')->default(0);
$t->unsignedBigInteger('delete_next_check_at')->default(0);
$t->unsignedBigInteger('delete_attention_at')->nullable();
$t->string('lease_token')->nullable();
$t->unsignedBigInteger('lease_until')->default(0);
$t->string('error_code')->default('');
$t->unsignedBigInteger('verified_at')->default(0);
$t->unsignedBigInteger('created_at'); $t->unsignedBigInteger('updated_at');
});
$db->schema()->create('cloud_storage_operations', static function (Blueprint $t): void {
$t->increments('id'); $t->string('operation_id')->unique();
$t->string('request_idempotency_key')->unique();
$t->string('request_fingerprint'); $t->unsignedBigInteger('artifact_id');
$t->unsignedBigInteger('replica_id')->nullable();
$t->unsignedBigInteger('source_backend_id')->nullable();
$t->unsignedBigInteger('target_backend_id')->nullable();
$t->string('kind'); $t->string('phase'); $t->string('outcome')->nullable();
$t->string('effect_resolution');
$t->boolean('effect_may_have_applied')->nullable();
$t->boolean('safe_to_resubmit')->nullable();
$t->unsignedInteger('attempts'); $t->string('lease_token')->nullable();
$t->unsignedBigInteger('lease_until'); $t->unsignedBigInteger('available_at');
$t->unsignedBigInteger('next_check_at'); $t->string('error_code');
$t->text('result_json')->nullable(); $t->unsignedBigInteger('terminal_at')->nullable();
$t->unsignedBigInteger('created_at'); $t->unsignedBigInteger('updated_at');
});
$db->schema()->create('cloud_storage_publications', static function (Blueprint $t): void {
$t->increments('id'); $t->string('publication_key')->unique();
$t->unsignedBigInteger('artifact_id')->unique();
$t->unsignedBigInteger('backup_id')->nullable()->unique();
$t->string('template_import_intent_id')->nullable()->unique();
$t->string('rescue_template_id')->nullable()->unique();
$t->string('state'); $t->unsignedInteger('attempts')->default(0);
$t->unsignedBigInteger('available_at')->default(0);
$t->string('lease_token')->nullable(); $t->unsignedBigInteger('lease_until')->default(0);
$t->string('last_error_code')->default('');
$t->unsignedBigInteger('coverage_ready_at')->nullable();
$t->unsignedBigInteger('finalized_at')->nullable();
$t->unsignedBigInteger('created_at'); $t->unsignedBigInteger('updated_at');
});
$db->schema()->create('cloud_instance_backups', static function (Blueprint $t): void {
$t->increments('id');
$t->unsignedBigInteger('storage_artifact_id')->nullable();
$t->unsignedBigInteger('active_job_id')->nullable();
$t->string('state');
$t->string('error_code')->default('');
$t->unsignedBigInteger('updated_at')->default(0);
});
$now = 2000000000;
foreach ([[1, 'a', 10], [2, 'b', 20]] as [$id, $domain, $priority]) {
$backend = new CloudStorageBackend();
$backend->id = $id;
$backend->backend_key = str_repeat((string) $id, 32);
$backend->identity_sha256 = str_repeat((string) $id, 64);
$backend->name = 'provider-' . $domain;
$backend->driver = 's3';
$backend->endpoint = 'https://' . $domain . '.example.test';
$backend->region = 'us-east-1';
$backend->bucket = 'bucket-' . $domain;
$backend->prefix = $domain;
$backend->path_style = true;
$backend->failure_domain = $domain;
$backend->sealed_credentials = 'sealed';
$backend->config_revision = $backend->tested_revision = 1;
$backend->admin_state = 'active';
$backend->health_state = 'healthy';
$backend->write_priority = $priority;
$backend->consecutive_failures = 0;
$backend->consecutive_successes = 1;
$backend->last_error_code = '';
$backend->last_tested_at = $backend->last_healthy_at = $now;
$backend->next_probe_at = $now + 300;
$backend->created_at = $backend->updated_at = $now;
$backend->save();
}
$newPublication = static function (int $backupId, string $suffix) use ($now): array {
return CloudStorageArtifactService::beginAgentPublication(
'backup', $suffix, 20 * 1073741824,
'backup-prepare-' . $backupId,
hash('sha256', 'backup-' . $backupId),
['backup_id' => $backupId]
);
};
$view = $newPublication(41, 'vm-backups/a/system.qcow2');
$operationId = (string) $view['operation_id'];
$assert(CloudStorageArtifactService::primaryOperationForBackup(41)
=== $operationId,
'Backup owner did not resolve to its original primary operation.');
$work = CloudStorageArtifactService::claimPrimaryPreparation(
$operationId, $now, 120
);
$assert(is_array($work) && $work['mode'] === 'dispatch'
&& $work['backup_id'] === 41,
'Accepted primary preparation was not leased for dispatch.');
$assert(CloudStorageArtifactService::claimPrimaryPreparation(
$operationId, $now + 1, 120
) === null, 'A second worker crossed the active preparation lease.');
CloudStorageArtifactService::markPrimaryPreparationUnknown(
$operationId, (string) $work['lease_token'],
'multipart_create_unknown', $now + 2, 5,
[
'absence_confirmations' => 1,
'multipart_absence_confirmations' => 1,
]
);
$reconcile = CloudStorageArtifactService::claimPrimaryPreparation(
$operationId, $now + 7, 120
);
$assert(is_array($reconcile) && $reconcile['mode'] === 'reconcile',
'Unknown CreateMultipart effect did not enter read-only reconciliation.');
$expectConflict(static fn () =>
CloudStorageArtifactService::reschedulePrimaryAfterStableAbsence(
$operationId, (string) $reconcile['lease_token'],
[
'absence_confirmations' => 1,
'multipart_absence_confirmations' => 1,
], $now + 8, 5
)
);
CloudStorageArtifactService::reschedulePrimaryAfterStableAbsence(
$operationId, (string) $reconcile['lease_token'],
[
'absence_confirmations' => 2,
'multipart_absence_confirmations' => 2,
], $now + 8, 5
);
$rescheduled = CloudStorageOperation::where(
'operation_id', $operationId
)->firstOrFail();
$assert((string) $rescheduled->phase === 'accepted'
&& (int) $rescheduled->attempts === 1,
'Two stable absence proofs did not reopen the same operation.');
$dispatch = CloudStorageArtifactService::claimPrimaryPreparation(
$operationId, $now + 13, 120
);
$metadata = [
'upload_id' => 'upload-a',
'part_size_bytes' => 67108864,
'part_count' => 320,
'maximum_size_bytes' => 20 * 1073741824,
'multipart_presence_confirmations' => 0,
];
CloudStorageArtifactService::markPrimaryAwaitingReceipt(
$operationId, (string) $dispatch['lease_token'], $metadata, $now + 14
);
$stored = (string) CloudStorageOperation::where(
'operation_id', $operationId
)->value('result_json');
$assert(str_contains($stored, 'upload-a') && !str_contains($stored, 'http'),
'Primary preparation persisted a capability URL or lost upload identity.');
$expectConflict(static fn () =>
CloudStorageArtifactService::markPrimaryAwaitingReceipt(
$operationId, (string) $dispatch['lease_token'],
array_merge($metadata, ['upload_id' => 'replacement']), $now + 15
)
);
$expectConflict(static fn () =>
CloudStorageArtifactService::markPrimaryAwaitingReceipt(
$operationId, (string) $dispatch['lease_token'],
array_merge($metadata, ['part_count' => 319]), $now + 15
)
);
$expectConflict(static fn () =>
CloudStorageArtifactService::markPrimaryPreparationUnknown(
$operationId,
(string) $dispatch['lease_token'],
'multipart_create_unknown',
$now + 15,
5,
array_merge($metadata, ['part_count' => 319])
)
);
CloudStorageArtifactService::releasePrimaryPreparationLease(
$operationId, (string) $dispatch['lease_token'], $now + 15
);
$sign = CloudStorageArtifactService::claimPrimaryPreparation(
$operationId, $now + 16, 120
);
$assert(is_array($sign) && $sign['mode'] === 'sign'
&& ($sign['metadata']['upload_id'] ?? '') === 'upload-a',
'Persisted multipart identity was not recoverable for local re-signing.');
CloudStorageArtifactService::releasePrimaryPreparationLease(
$operationId, (string) $sign['lease_token'], $now + 17
);
$expiredView = $newPublication(43, 'vm-backups/expired/system.qcow2');
$expiredId = (string) $expiredView['operation_id'];
$expiredWork = CloudStorageArtifactService::claimPrimaryPreparation(
$expiredId, $now + 18, 120
);
$expiredMetadata = [
'upload_id' => 'expired-upload',
'part_size_bytes' => 67108864,
'part_count' => 320,
'maximum_size_bytes' => 20 * 1073741824,
];
CloudStorageArtifactService::markPrimaryAwaitingReceipt(
$expiredId,
(string) $expiredWork['lease_token'],
$expiredMetadata,
$now + 19
);
CloudStorageArtifactService::markPrimaryPreparationUnknown(
$expiredId,
(string) $expiredWork['lease_token'],
'primary_multipart_absence_unconfirmed',
$now + 20,
5,
$expiredMetadata + [
'absence_confirmations' => 1,
'multipart_absence_confirmations' => 1,
]
);
$expiredReconcile = CloudStorageArtifactService::claimPrimaryPreparation(
$expiredId, $now + 25, 120
);
CloudStorageArtifactService::reschedulePrimaryAfterStableAbsence(
$expiredId,
(string) $expiredReconcile['lease_token'],
$expiredMetadata + [
'absence_confirmations' => 2,
'multipart_absence_confirmations' => 2,
],
$now + 26,
5
);
$expiredOperation = CloudStorageOperation::where(
'operation_id', $expiredId
)->firstOrFail();
$assert((string) $expiredOperation->phase === 'accepted'
&& !str_contains((string) $expiredOperation->result_json, 'expired-upload'),
'Stable object and multipart absence did not retire the old upload identity.');
$attentionView = $newPublication(42, 'vm-backups/b/system.qcow2');
$attentionId = (string) $attentionView['operation_id'];
Capsule::table('cloud_instance_backups')->insert([
'id' => 42,
'storage_artifact_id' => (int) $attentionView['artifact_id'],
'active_job_id' => null,
'state' => 'creating',
]);
$attentionWork = CloudStorageArtifactService::claimPrimaryPreparation(
$attentionId, $now + 20, 120
);
CloudStorageArtifactService::markPrimaryPreparationUnknown(
$attentionId, (string) $attentionWork['lease_token'],
'multipart_create_unknown', $now + 21, 5, [
'multipart_attention_confirmations' => 1,
'multipart_attention_error_code' =>
'primary_multipart_identity_ambiguous',
]
);
$attentionReconcile = CloudStorageArtifactService::claimPrimaryPreparation(
$attentionId, $now + 26, 120
);
$expectConflict(static fn () =>
CloudStorageArtifactService::markPrimaryPreparationAttention(
$attentionId, (string) $attentionReconcile['lease_token'],
'primary_multipart_identity_ambiguous', $now + 27, [
'multipart_attention_confirmations' => 1,
'multipart_attention_error_code' =>
'primary_multipart_identity_ambiguous',
]
)
);
CloudStorageArtifactService::markPrimaryPreparationAttention(
$attentionId, (string) $attentionReconcile['lease_token'],
'primary_multipart_identity_ambiguous', $now + 27, [
'multipart_attention_confirmations' => 2,
'multipart_attention_error_code' =>
'primary_multipart_identity_ambiguous',
]
);
$attention = CloudStorageOperation::where(
'operation_id', $attentionId
)->firstOrFail();
$attentionReplica = CloudStorageReplica::where(
'publish_operation_id', $attentionId
)->firstOrFail();
$attentionBackup = CloudInstanceBackup::findOrFail(42);
$assert((string) $attention->phase === 'terminal'
&& (string) $attention->effect_resolution === 'unknown'
&& !(bool) $attention->safe_to_resubmit
&& (string) $attentionReplica->state === 'degraded'
&& (string) $attentionBackup->state === 'uncertain',
'Stable multipart conflict did not stop automatic replay in attention.');
$attentionList = CloudStorageReplicaAttentionService::listView();
$attentionItem = $attentionList['items'][0] ?? [];
$encodedAttention = json_encode($attentionList, JSON_THROW_ON_ERROR);
$assert(($attentionList['summary']['count'] ?? 0) === 1
&& ($attentionItem['operation_kind'] ?? '') === 'publish'
&& ($attentionItem['owner_type'] ?? '') === 'backup'
&& ($attentionItem['owner_id'] ?? 0) === 42
&& ($attentionItem['can_retry'] ?? false) === true
&& !str_contains($encodedAttention, 'object_key')
&& !str_contains($encodedAttention, 'upload_id')
&& !str_contains($encodedAttention, 'part_urls'),
'Primary preparation attention was not safely exposed to administrators.');
CloudStorageReplicaAttentionService::retry($attentionId);
$attention->refresh();
$attentionReplica->refresh();
$attentionBackup->refresh();
$assert((string) $attention->phase === 'reconciling'
&& (string) $attention->error_code === 'admin_reconcile_requested'
&& $attention->terminal_at === null
&& (string) $attentionReplica->state === 'publishing'
&& (string) $attentionBackup->state === 'creating',
'Administrator reconciliation did not reopen the original operation.');
$attentionBackup->state = 'uncertain';
$attentionBackup->error_code = 'stale_projection';
$attentionBackup->save();
CloudStorageArtifactService::reopenPrimaryPreparationAttention(
$attentionId,
$now + 28
);
$attentionBackup->refresh();
$assert((string) $attentionBackup->state === 'creating'
&& (string) $attentionBackup->error_code
=== 'admin_reconcile_requested',
'Idempotent administrator retry did not normalize the backup projection.');
$reopened = CloudStorageArtifactService::claimPrimaryPreparation(
$attentionId,
$now + 28,
120
);
$assert(is_array($reopened) && $reopened['mode'] === 'reconcile',
'Reopened primary attention did not return to read-only reconciliation.');
echo "Storage primary preparation tests passed.\n";