You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
virt/web/tests/CloudBackupStorageFinalizer...

386 lines
18 KiB
PHP

<?php
require dirname(__DIR__) . '/vendor/autoload.php';
use App\Models\CloudInstanceBackup;
use App\Models\CloudJob;
use App\Models\CloudStorageArtifact;
use App\Models\CloudStorageOperation;
use App\Models\CloudStoragePublication;
use App\Models\CloudStorageReplica;
use App\Services\Cloud\CloudBackupService;
use App\Services\Cloud\CloudError;
use App\Services\Cloud\CloudPortableRestoreCompatibility;
use App\Services\Cloud\CloudStorageArtifactService;
use Illuminate\Database\Capsule\Manager as Capsule;
use Illuminate\Database\Schema\Blueprint;
$assert = static function (bool $condition, string $message): void {
if (!$condition) {
throw new RuntimeException($message);
}
};
$expectConflict = static function (callable $callback) use ($assert): void {
try {
$callback();
} catch (CloudError $error) {
$assert($error->httpStatus === 409,
'Unexpected conflict status: ' . $error->httpStatus);
return;
}
throw new RuntimeException('Expected storage identity conflict.');
};
$db = new Capsule();
$db->addConnection(['driver' => 'sqlite', 'database' => ':memory:'], 'default');
$db->setAsGlobal();
$db->bootEloquent();
$schema = $db->schema('default');
$schema->create('cloud_storage_backends', static function (Blueprint $t): void {
$t->increments('id');
$t->string('backend_key'); $t->string('identity_sha256');
$t->string('name'); $t->string('driver'); $t->string('endpoint');
$t->string('region'); $t->string('bucket'); $t->string('prefix');
$t->boolean('path_style'); $t->string('failure_domain');
$t->text('sealed_credentials'); $t->unsignedBigInteger('config_revision');
$t->unsignedBigInteger('tested_revision'); $t->string('admin_state');
$t->string('health_state'); $t->unsignedInteger('write_priority');
$t->unsignedInteger('consecutive_failures')->default(0);
$t->unsignedInteger('consecutive_successes')->default(0);
$t->string('last_error_code')->default('');
$t->unsignedBigInteger('last_tested_at')->default(0);
$t->unsignedBigInteger('last_healthy_at')->default(0);
$t->unsignedBigInteger('next_probe_at')->default(0);
$t->unsignedBigInteger('created_at'); $t->unsignedBigInteger('updated_at');
});
$schema->create('cloud_storage_artifacts', static function (Blueprint $t): void {
$t->increments('id'); $t->string('artifact_key')->unique();
$t->string('kind'); $t->unsignedBigInteger('maximum_size_bytes');
$t->unsignedBigInteger('size_bytes'); $t->string('sha256');
$t->string('durability_policy'); $t->unsignedTinyInteger('desired_copies');
$t->unsignedTinyInteger('required_copies'); $t->string('state');
$t->string('last_operation_id')->nullable();
$t->unsignedBigInteger('verified_at');
$t->unsignedBigInteger('created_at'); $t->unsignedBigInteger('updated_at');
});
$schema->create('cloud_storage_replicas', static function (Blueprint $t): void {
$t->increments('id'); $t->string('replica_key')->unique();
$t->unsignedBigInteger('artifact_id'); $t->unsignedBigInteger('backend_id');
$t->string('role'); $t->string('object_key');
$t->string('object_version_id')->default('');
$t->string('object_etag')->default('');
$t->unsignedBigInteger('size_bytes')->default(0);
$t->string('sha256')->default(''); $t->string('state');
$t->string('publish_operation_id')->nullable();
$t->string('delete_operation_id')->nullable();
$t->unsignedInteger('delete_generation')->default(0);
$t->unsignedInteger('delete_cycle_attempts')->default(0);
$t->unsignedInteger('delete_presence_confirmations')->default(0);
$t->unsignedBigInteger('delete_last_checked_at')->default(0);
$t->unsignedBigInteger('delete_next_check_at')->default(0);
$t->unsignedBigInteger('delete_attention_at')->nullable();
$t->string('lease_token')->nullable();
$t->unsignedBigInteger('lease_until')->default(0);
$t->string('error_code')->default('');
$t->unsignedBigInteger('verified_at')->default(0);
$t->unsignedBigInteger('created_at'); $t->unsignedBigInteger('updated_at');
});
$schema->create('cloud_storage_operations', static function (Blueprint $t): void {
$t->increments('id'); $t->string('operation_id')->unique();
$t->string('request_idempotency_key')->unique();
$t->string('request_fingerprint'); $t->unsignedBigInteger('artifact_id');
$t->unsignedBigInteger('replica_id')->nullable();
$t->unsignedBigInteger('source_backend_id')->nullable();
$t->unsignedBigInteger('target_backend_id')->nullable();
$t->string('kind'); $t->string('phase'); $t->string('outcome')->nullable();
$t->string('effect_resolution');
$t->boolean('effect_may_have_applied')->nullable();
$t->boolean('safe_to_resubmit')->nullable();
$t->unsignedInteger('attempts'); $t->string('lease_token')->nullable();
$t->unsignedBigInteger('lease_until');
$t->unsignedBigInteger('available_at'); $t->unsignedBigInteger('next_check_at');
$t->string('error_code'); $t->text('result_json')->nullable();
$t->unsignedBigInteger('terminal_at')->nullable();
$t->unsignedBigInteger('created_at'); $t->unsignedBigInteger('updated_at');
});
$schema->create('cloud_storage_publications', static function (Blueprint $t): void {
$t->increments('id'); $t->string('publication_key')->unique();
$t->unsignedBigInteger('artifact_id')->unique();
$t->unsignedBigInteger('backup_id')->nullable()->unique();
$t->string('template_import_intent_id')->nullable()->unique();
$t->string('rescue_template_id')->nullable()->unique();
$t->string('state'); $t->unsignedInteger('attempts')->default(0);
$t->unsignedBigInteger('available_at')->default(0);
$t->string('lease_token')->nullable();
$t->unsignedBigInteger('lease_until')->default(0);
$t->string('last_error_code')->default('');
$t->unsignedBigInteger('coverage_ready_at')->nullable();
$t->unsignedBigInteger('finalized_at')->nullable();
$t->unsignedBigInteger('created_at'); $t->unsignedBigInteger('updated_at');
});
$schema->create('cloud_instance_backups', static function (Blueprint $t): void {
$t->increments('id'); $t->string('backup_key')->unique();
$t->unsignedBigInteger('storage_artifact_id')->nullable();
$t->unsignedBigInteger('source_instance_id');
$t->unsignedBigInteger('source_host_id'); $t->unsignedBigInteger('user_id');
$t->string('source_uuid'); $t->string('name');
$t->string('purpose')->default('portable'); $t->string('source')->default('manual');
$t->unsignedBigInteger('subscription_id')->nullable();
$t->unsignedBigInteger('scheduled_for')->nullable();
$t->string('state'); $t->string('consistency'); $t->string('object_key');
$t->unsignedBigInteger('object_size_bytes')->default(0);
$t->unsignedBigInteger('virtual_size_bytes')->default(0);
$t->string('sha256')->default(''); $t->string('object_etag')->default('');
$t->string('object_version_id')->default('');
$t->text('source_portable_restore_contract_json')->default('{}');
$t->string('source_portable_restore_contract_sha256')->default('');
$t->text('spec_json'); $t->text('sealed_storage_runtime');
$t->text('sealed_credential')->nullable();
$t->unsignedBigInteger('storage_revision')->default(0);
$t->unsignedBigInteger('active_job_id')->nullable();
$t->string('last_operation_id')->nullable();
$t->string('error_code')->default(''); $t->unsignedBigInteger('ready_at')->nullable();
$t->unsignedBigInteger('created_at'); $t->unsignedBigInteger('updated_at');
});
$schema->create('cloud_jobs', static function (Blueprint $t): void {
$t->increments('id'); $t->string('kind'); $t->unsignedBigInteger('host_id');
$t->unsignedBigInteger('instance_id')->nullable();
$t->unsignedBigInteger('backup_id')->nullable();
$t->unsignedBigInteger('user_id')->nullable();
$t->string('operation_id')->unique();
$t->string('effect_resolution')->default('not_applicable');
});
$now = time();
foreach ([[1, 'provider-a', 10], [2, 'provider-b', 20]] as [$id, $name, $priority]) {
Capsule::table('cloud_storage_backends')->insert([
'id' => $id, 'backend_key' => str_repeat((string) $id, 32),
'identity_sha256' => str_repeat((string) $id, 64),
'name' => $name, 'driver' => 's3',
'endpoint' => 'https://' . $name . '.example.test',
'region' => 'us-east-1', 'bucket' => $name . '-bucket',
'prefix' => $name, 'path_style' => 1,
'failure_domain' => $name, 'sealed_credentials' => 'sealed',
'config_revision' => 1, 'tested_revision' => 1,
'admin_state' => 'active', 'health_state' => 'healthy',
'write_priority' => $priority, 'consecutive_failures' => 0,
'consecutive_successes' => 1, 'last_error_code' => '',
'last_tested_at' => $now, 'last_healthy_at' => $now,
'next_probe_at' => $now + 300, 'created_at' => $now,
'updated_at' => $now,
]);
}
$portableRestore = [
'contract' => 1, 'architecture' => 'x86_64', 'machine' => 'q35',
'firmware' => 'uefi', 'secure_boot' => false, 'disk_format' => 'qcow2',
'system_disk_only' => true, 'public_network_contract' => 1,
'vpc_inherited' => false,
];
$compatibility = CloudPortableRestoreCompatibility::fromPayload(
$portableRestore
);
$sourceSpec = json_encode([
'managed' => true, 'vcpu' => 2, 'memory_mib' => 2048,
'disk_gib' => 20, 'bandwidth_mbps' => 1000,
], JSON_THROW_ON_ERROR);
$fixture = static function (int $id) use (
$now,
$sourceSpec,
$compatibility
): array {
$backupKey = substr(hash('sha256', 'storage-finalizer-' . $id), 0, 32);
$agentOperationId = substr(hash('sha256', 'agent-' . $id), 0, 32);
Capsule::table('cloud_instance_backups')->insert([
'id' => $id, 'backup_key' => $backupKey,
'storage_artifact_id' => null,
'source_instance_id' => 100 + $id, 'source_host_id' => 3,
'user_id' => 9,
'source_uuid' => sprintf('00000000-0000-4000-8000-%012d', $id),
'name' => 'storage-finalizer-' . $id, 'state' => 'creating',
'consistency' => 'filesystem',
'object_key' => 'vm-backups/' . $backupKey . '/system.qcow2',
'source_portable_restore_contract_json' => $compatibility->json(),
'source_portable_restore_contract_sha256' => $compatibility->sha256(),
'spec_json' => $sourceSpec, 'sealed_storage_runtime' => '',
'sealed_credential' => 'sealed-guest', 'storage_revision' => 0,
'active_job_id' => $id, 'last_operation_id' => $agentOperationId,
'created_at' => $now, 'updated_at' => $now,
]);
$publication = CloudStorageArtifactService::beginAgentPublication(
'backup', 'vm-backups/' . $backupKey . '/system.qcow2',
21 * 1073741824, 'storage-finalizer-' . str_pad((string) $id, 4, '0', STR_PAD_LEFT),
hash('sha256', 'storage-finalizer-fingerprint-' . $id),
['backup_id' => $id]
);
Capsule::table('cloud_instance_backups')->where('id', $id)->update([
'storage_artifact_id' => (int) $publication['artifact_id'],
'object_key' => (string) $publication['object_key'],
]);
$work = CloudStorageArtifactService::claimPrimaryPreparation(
(string) $publication['operation_id'], $now + $id, 300
);
CloudStorageArtifactService::markPrimaryAwaitingReceipt(
(string) $publication['operation_id'],
(string) $work['lease_token'],
[
'upload_id' => 'upload-' . $id,
'part_size_bytes' => 67108864,
'part_count' => 336,
'maximum_size_bytes' => 21 * 1073741824,
],
$now + $id
);
Capsule::table('cloud_jobs')->insert([
'id' => $id, 'kind' => 'backup_create', 'host_id' => 3,
'instance_id' => 100 + $id, 'backup_id' => $id,
'user_id' => 9, 'operation_id' => $agentOperationId,
'effect_resolution' => 'not_applicable',
]);
return [
'backup' => CloudInstanceBackup::findOrFail($id),
'job' => CloudJob::findOrFail($id),
'storage_operation_id' => (string) $publication['operation_id'],
'artifact_id' => (int) $publication['artifact_id'],
];
};
$payload = static fn (array $item): array => [
'owner_id' => 9,
'backup_key' => (string) $item['backup']->backup_key,
'maximum_size_bytes' => 21 * 1073741824,
'storage_artifact_id' => (int) $item['artifact_id'],
'storage_publication_operation_id' =>
(string) $item['storage_operation_id'],
'source_portable_restore_contract_sha256' => $compatibility->sha256(),
];
$receipt = static fn (array $item): array => [
'backup_key' => (string) $item['backup']->backup_key,
'size_bytes' => 4096,
'virtual_size_bytes' => 20 * 1073741824,
'sha256' => str_repeat(dechex(((int) $item['backup']->id % 14) + 1), 64),
'object_etag' => '"etag-' . (int) $item['backup']->id . '"',
'object_version_id' => 'version-' . (int) $item['backup']->id,
'consistency' => 'filesystem', 'cleanup_pending' => false,
'source_contract' => $portableRestore,
];
$success = $fixture(1);
$successPayload = $payload($success);
$successReceipt = $receipt($success);
CloudBackupService::finishJobLocked(
$success['job'], 'succeeded', $successReceipt, $successPayload
);
$ready = CloudInstanceBackup::findOrFail(1);
$artifact = CloudStorageArtifact::findOrFail($success['artifact_id']);
$primary = CloudStorageReplica::where(
'publish_operation_id', $success['storage_operation_id']
)->firstOrFail();
$operation = CloudStorageOperation::where(
'operation_id', $success['storage_operation_id']
)->firstOrFail();
$successPublication = CloudStoragePublication::where(
'artifact_id', $success['artifact_id']
)->firstOrFail();
$successFinalizedAt = (int) $successPublication->finalized_at;
$assert((string) $ready->state === 'ready'
&& $ready->active_job_id === null
&& (string) $operation->outcome === 'succeeded'
&& (string) $artifact->sha256 === (string) $ready->sha256
&& (int) $artifact->size_bytes === (int) $ready->object_size_bytes
&& (string) $primary->state === 'ready'
&& (string) $primary->object_version_id
=== (string) $ready->object_version_id
&& (string) $successPublication->state === 'completed'
&& (int) $successPublication->attempts === 1
&& $successFinalizedAt > 0,
'Agent success did not settle primary storage before exposing the backup.');
CloudBackupService::finishJobLocked(
$success['job'], 'succeeded', $successReceipt, $successPayload
);
$assert(CloudStorageArtifact::count() === 1
&& CloudStorageOperation::where('kind', 'publish')->count() === 1
&& (int) CloudStoragePublication::where(
'artifact_id', $success['artifact_id']
)->value('attempts') === 1
&& (int) CloudStoragePublication::where(
'artifact_id', $success['artifact_id']
)->value('finalized_at') === $successFinalizedAt,
'Identical finalizer replay allocated or replaced storage identity.');
$mismatch = $fixture(2);
$badPayload = $payload($mismatch);
$badPayload['storage_artifact_id'] = $success['artifact_id'];
$expectConflict(static fn () => CloudBackupService::finishJobLocked(
$mismatch['job'], 'succeeded', $receipt($mismatch), $badPayload
));
$assert((string) CloudInstanceBackup::findOrFail(2)->state === 'creating'
&& (string) CloudStorageOperation::where(
'operation_id', $mismatch['storage_operation_id']
)->value('phase') === 'awaiting_receipt',
'Mismatched artifact payload mutated backup or storage state.');
$uncertain = $fixture(3);
CloudBackupService::finishJobLocked(
$uncertain['job'], 'uncertain', null, $payload($uncertain)
);
$uncertainOperation = CloudStorageOperation::where(
'operation_id', $uncertain['storage_operation_id']
)->firstOrFail();
$assert((string) CloudInstanceBackup::findOrFail(3)->state === 'uncertain'
&& (string) $uncertainOperation->phase === 'reconciling'
&& (string) $uncertainOperation->effect_resolution === 'unknown'
&& (string) $uncertainOperation->error_code
=== 'backup_operation_effect_unknown',
'Unknown Agent effect did not preserve original primary reconciliation.');
$failed = $fixture(4);
CloudBackupService::finishJobLocked(
$failed['job'], 'failed', null, $payload($failed)
);
$failedOperation = CloudStorageOperation::where(
'operation_id', $failed['storage_operation_id']
)->firstOrFail();
$failedPublication = CloudStoragePublication::where(
'artifact_id', $failed['artifact_id']
)->firstOrFail();
$assert((string) CloudInstanceBackup::findOrFail(4)->state === 'failed'
&& (string) $failedOperation->phase === 'terminal'
&& (string) $failedOperation->effect_resolution === 'not_applied'
&& (string) $failedPublication->state === 'failed',
'A failed Agent result did not close the primary publication as definitely not applied.');
$bound = $fixture(5);
$foreign = $fixture(6);
$wrongOperationPayload = $payload($bound);
$wrongOperationPayload['storage_publication_operation_id'] =
$foreign['storage_operation_id'];
$expectConflict(static fn () => CloudBackupService::finishJobLocked(
$bound['job'],
'succeeded',
$receipt($bound),
$wrongOperationPayload
));
$assert((string) CloudInstanceBackup::findOrFail(5)->state === 'creating'
&& (string) CloudStorageOperation::where(
'operation_id', $bound['storage_operation_id']
)->value('phase') === 'awaiting_receipt'
&& (string) CloudStorageOperation::where(
'operation_id', $foreign['storage_operation_id']
)->value('phase') === 'awaiting_receipt',
'A foreign successful receipt mutated either artifact ledger.');
$expectConflict(static fn () => CloudBackupService::finishJobLocked(
$bound['job'], 'failed', null, $wrongOperationPayload
));
$assert((string) CloudInstanceBackup::findOrFail(5)->state === 'creating'
&& (string) CloudStorageOperation::where(
'operation_id', $bound['storage_operation_id']
)->value('phase') === 'awaiting_receipt'
&& (string) CloudStorageOperation::where(
'operation_id', $foreign['storage_operation_id']
)->value('phase') === 'awaiting_receipt',
'A foreign storage operation mutated either artifact ledger.');
echo "Cloud backup storage finalizer tests passed.\n";