You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
virt/web/tests/CloudPortableBackupDeletion...

396 lines
16 KiB
PHP

<?php
namespace App\Services {
final class Config
{
public static string $dataKey = '';
public static function get($name, $default = null)
{
return $name === 'cloud_data_key' ? self::$dataKey : $default;
}
}
}
namespace {
require dirname(__DIR__) . '/vendor/autoload.php';
use App\Models\CloudBackupRestore;
use App\Models\CloudInstanceBackup;
use App\Services\Config;
use App\Services\Cloud\CloudBackupService;
use App\Services\Cloud\CloudError;
use App\Services\Cloud\CloudObjectStorageS3Client;
use App\Services\Cloud\CloudSecrets;
use GuzzleHttp\Client;
use GuzzleHttp\Exception\ConnectException;
use GuzzleHttp\Handler\MockHandler;
use GuzzleHttp\HandlerStack;
use GuzzleHttp\Middleware;
use GuzzleHttp\Psr7\Request;
use GuzzleHttp\Psr7\Response;
use Illuminate\Database\Capsule\Manager as Capsule;
use Illuminate\Database\Schema\Blueprint;
if (!function_exists('sodium_crypto_aead_xchacha20poly1305_ietf_encrypt')) {
echo "SKIPPED CloudPortableBackupDeletionTest: runtime lacks sodium.\n";
exit(0);
}
$assert = static function (bool $condition, string $message): void {
if (!$condition) {
throw new RuntimeException($message);
}
};
$expectConflict = static function (callable $callback) use ($assert): void {
try {
$callback();
} catch (CloudError $error) {
$assert($error->httpStatus === 409,
'Deletion blocker returned an unexpected status.');
return;
}
throw new RuntimeException('Expected deletion blocker was not raised.');
};
$capsule = new Capsule();
$capsule->addConnection([
'driver' => 'sqlite', 'database' => ':memory:',
], 'default');
$capsule->setAsGlobal();
$capsule->bootEloquent();
$schema = $capsule->schema('default');
$schema->create('cloud_instance_backups',
static function (Blueprint $table): void {
$table->increments('id');
$table->string('backup_key')->unique();
$table->integer('source_instance_id');
$table->integer('source_host_id');
$table->integer('user_id');
$table->string('source_uuid');
$table->string('name');
$table->string('purpose')->default('portable');
$table->string('state');
$table->string('consistency');
$table->string('object_key')->unique();
$table->integer('object_size_bytes');
$table->integer('virtual_size_bytes');
$table->string('sha256');
$table->string('object_etag')->default('');
$table->string('object_version_id')->default('');
$table->text('spec_json');
$table->text('sealed_storage_runtime');
$table->text('sealed_credential')->nullable();
$table->integer('storage_revision');
$table->integer('active_job_id')->nullable();
$table->string('last_operation_id')->nullable();
$table->string('delete_operation_id')->nullable();
$table->integer('delete_generation')->default(0);
$table->integer('delete_cycle_attempts')->default(0);
$table->integer('delete_presence_confirmations')->default(0);
$table->integer('delete_requested_at')->nullable();
$table->integer('delete_last_checked_at')->nullable();
$table->integer('delete_next_check_at')->default(0);
$table->integer('delete_attention_at')->nullable();
$table->string('error_code');
$table->integer('ready_at')->nullable();
$table->integer('created_at');
$table->integer('updated_at');
});
$schema->create('cloud_deployment_snapshots',
static function (Blueprint $table): void {
$table->increments('id');
$table->integer('backup_id')->unique();
$table->string('state');
$table->string('error_code')->default('');
$table->integer('deleted_at')->nullable();
$table->integer('updated_at')->default(0);
});
$schema->create('cloud_backup_restores',
static function (Blueprint $table): void {
$table->increments('id');
$table->integer('backup_id');
$table->string('state');
});
$schema->create('cloud_backup_recoveries',
static function (Blueprint $table): void {
$table->increments('id');
$table->integer('backup_id');
$table->string('state');
});
$schema->create('cloud_jobs', static function (Blueprint $table): void {
$table->increments('id');
$table->integer('backup_id')->nullable();
$table->string('phase');
$table->string('effect_resolution');
$table->integer('finalization_incident')->default(0);
});
Config::$dataKey = base64_encode(str_repeat('b', 32));
$runtime = [
'config' => [
'endpoint' => 'https://objects.example.com',
'region' => 'us-east-1',
'bucket' => 'backup-bucket',
'prefix' => 'virt',
'path_style' => true,
],
'credentials' => [
'access_key_id' => 'BACKUPACCESS',
'secret_access_key' => 'backup-secret-value',
],
'revision' => 7,
];
$sealedRuntime = CloudSecrets::encrypt(
$runtime,
'object-storage-backup-runtime'
);
$actor = [
'user_id' => 1,
'is_admin' => true,
'is_super_admin' => true,
'can_manage' => true,
];
$newBackup = static function (int $sourceInstanceId, string $suffix,
string $state = 'ready') use ($sealedRuntime): CloudInstanceBackup {
$backup = new CloudInstanceBackup();
$backup->backup_key = str_pad($suffix, 32, '0');
$backup->source_instance_id = $sourceInstanceId;
$backup->source_host_id = 3;
$backup->user_id = 9;
$backup->source_uuid = '11111111-1111-4111-8111-111111111111';
$backup->name = 'backup-' . $suffix;
$backup->state = $state;
$backup->consistency = 'crash';
$backup->object_key = 'virt/vm-backups/' . $suffix . '/system.qcow2';
$backup->object_size_bytes = 1024;
$backup->virtual_size_bytes = 2048;
$backup->sha256 = str_repeat('a', 64);
$backup->object_etag = '';
$backup->object_version_id = '';
$backup->spec_json = '{}';
$backup->sealed_storage_runtime = $sealedRuntime;
$backup->sealed_credential = 'sealed-guest-credential';
$backup->storage_revision = 7;
$backup->active_job_id = null;
$backup->last_operation_id = null;
$backup->delete_operation_id = null;
$backup->delete_generation = 0;
$backup->delete_cycle_attempts = 0;
$backup->delete_presence_confirmations = 0;
$backup->delete_requested_at = null;
$backup->delete_last_checked_at = null;
$backup->delete_next_check_at = 0;
$backup->delete_attention_at = null;
$backup->error_code = '';
$backup->ready_at = time();
$backup->created_at = $backup->updated_at = time();
$backup->save();
return $backup;
};
$client = static function (array $responses, array &$history): CloudObjectStorageS3Client {
$stack = HandlerStack::create(new MockHandler($responses));
$stack->push(Middleware::history($history));
return new CloudObjectStorageS3Client(new Client(['handler' => $stack]));
};
$identity = static fn (): Response => new Response(200, [
'Content-Length' => '1024',
'ETag' => '"catalog-etag"',
'x-amz-version-id' => 'catalog-version',
]);
$history = [];
$backup = $newBackup(10, 'one');
$deleted = CloudBackupService::delete(
$backup, $actor, $client([$identity(), new Response(204)], $history)
);
$backup->refresh();
$assert($deleted['deleted'] === true
&& (string) $backup->state === 'deleted'
&& (string) $backup->sealed_storage_runtime === ''
&& $backup->sealed_credential === null
&& count($history) === 2
&& $history[0]['request']->getMethod() === 'HEAD'
&& $history[1]['request']->getMethod() === 'DELETE'
&& str_contains((string) $history[1]['request']->getUri(),
'versionId=catalog-version'),
'Explicit DeleteObject success did not tombstone the catalog and remove frozen credentials.');
$unknown = $newBackup(11, 'two');
$unknownHistory = [];
$unknownResult = CloudBackupService::delete($unknown, $actor, $client([
$identity(),
new ConnectException('transport lost',
new Request('DELETE', 'https://objects.example.com')),
], $unknownHistory));
$unknown->refresh();
$assert($unknownResult['requires_reconciliation'] === true
&& (string) $unknown->state === 'delete_uncertain'
&& (string) $unknown->sealed_storage_runtime !== '',
'Ambiguous DeleteObject did not retain the frozen runtime and unknown-effect fence.');
$unknown->delete_next_check_at = 0;
$unknown->save();
$probeHistory = [];
$reconciled = CloudBackupService::delete(
$unknown, $actor, $client([new Response(404)], $probeHistory)
);
$assert($reconciled['deleted'] === true
&& count($probeHistory) === 1
&& $probeHistory[0]['request']->getMethod() === 'HEAD',
'A later explicit 404 did not converge deletion without replaying DELETE.');
$present = $newBackup(12, 'three');
$firstHistory = [];
CloudBackupService::delete($present, $actor, $client([
$identity(),
new ConnectException('transport lost',
new Request('DELETE', 'https://objects.example.com')),
], $firstHistory));
$present->refresh();
$present->delete_next_check_at = 0;
$present->save();
$presentHistory = [];
$stillPresent = CloudBackupService::delete(
$present, $actor, $client([$identity()], $presentHistory)
);
$present->refresh();
$assert($stillPresent['requires_reconciliation'] === true
&& (string) $present->state === 'delete_uncertain'
&& count($presentHistory) === 1
&& $presentHistory[0]['request']->getMethod() === 'HEAD',
'An existing object after ambiguous deletion was replayed or lost its fence.');
$blocked = $newBackup(13, 'four');
$restore = new CloudBackupRestore();
$restore->backup_id = (int) $blocked->id;
$restore->state = 'restoring';
$restore->save();
$expectConflict(static function () use ($blocked, $actor): void {
CloudBackupService::delete($blocked, $actor);
});
$blocked->refresh();
$assert((string) $blocked->state === 'ready',
'An active restore did not preserve its source backup.');
$readOnlyBackup = $newBackup(14, 'read-only');
$displayMethod = (new ReflectionClass(CloudBackupService::class))
->getMethod('displayBackup');
$readOnlyDisplay = $displayMethod->invoke(
null, $readOnlyBackup, null, false
);
$assert($readOnlyDisplay['can_delete'] === false,
'Read-only administrators were offered a backup delete action.');
$quotaBackups = [];
for ($index = 0; $index < CloudBackupService::MAX_PER_INSTANCE; $index++) {
$quotaBackups[] = $newBackup(99, 'quota-' . $index);
}
$reflection = new ReflectionClass(CloudBackupService::class);
$activeStates = $reflection->getReflectionConstant(
'ACTIVE_BACKUP_STATES'
)->getValue();
$before = CloudInstanceBackup::where('source_instance_id', 99)
->whereIn('state', $activeStates)->count();
$quotaHistory = [];
CloudBackupService::delete(
$quotaBackups[0], $actor,
$client([$identity(), new Response(204)], $quotaHistory)
);
$after = CloudInstanceBackup::where('source_instance_id', 99)
->whereIn('state', $activeStates)->count();
$assert($before === CloudBackupService::MAX_PER_INSTANCE
&& $after === CloudBackupService::MAX_PER_INSTANCE - 1,
'Confirmed deletion did not release one retained-backup quota slot.');
// One ambiguous generation may be retried only after two stable, spaced
// identity probes. The retry is a new deletion generation bound to the
// exact S3 version instead of a replay of the ambiguous request.
$retry = $newBackup(100, 'stable-retry', 'delete_uncertain');
$retry->object_etag = '"stable-etag"';
$retry->object_version_id = 'version/7+opaque';
$retry->delete_operation_id = str_repeat('1', 32);
$retry->delete_generation = 1;
$retry->delete_cycle_attempts = 1;
$retry->delete_presence_confirmations = 1;
$retry->delete_requested_at = time() - 300;
$retry->delete_last_checked_at = time() - 120;
$retry->delete_next_check_at = 0;
$retry->save();
$retryHistory = [];
$retryResult = CloudBackupService::delete($retry, $actor, $client([
new Response(200, [
'Content-Length' => '1024',
'ETag' => '"stable-etag"',
'x-amz-version-id' => 'version/7+opaque',
]),
new ConnectException('retry response lost',
new Request('DELETE', 'https://objects.example.com')),
], $retryHistory));
$retry->refresh();
$retryUris = array_map(static fn ($entry): string =>
(string) $entry['request']->getUri(), $retryHistory);
$assert($retryResult['requires_reconciliation'] === true
&& (string) $retry->state === 'delete_uncertain'
&& (int) $retry->delete_generation === 2
&& (int) $retry->delete_cycle_attempts === 2
&& count($retryHistory) === 2
&& $retryHistory[0]['request']->getMethod() === 'HEAD'
&& $retryHistory[1]['request']->getMethod() === 'DELETE'
&& str_contains($retryUris[0], 'versionId=version%2F7%2Bopaque')
&& str_contains($retryUris[1], 'versionId=version%2F7%2Bopaque'),
'Stable object identity did not create a fresh, exact-version deletion generation.');
$assert(!in_array('delete_uncertain', $activeStates, true),
'Object cleanup debt still consumes the user retained-backup quota.');
$retry->delete_cycle_attempts = 3;
$retry->delete_generation = 3;
$retry->delete_presence_confirmations = 1;
$retry->delete_last_checked_at = time() - 120;
$retry->delete_next_check_at = 0;
$retry->save();
$exhaustedHistory = [];
$exhausted = CloudBackupService::delete($retry, $actor, $client([
new Response(200, [
'Content-Length' => '1024',
'ETag' => '"stable-etag"',
'x-amz-version-id' => 'version/7+opaque',
]),
], $exhaustedHistory));
$retry->refresh();
$assert(($exhausted['requires_attention'] ?? false) === true
&& (string) $retry->state === 'delete_attention'
&& count($exhaustedHistory) === 1
&& $exhaustedHistory[0]['request']->getMethod() === 'HEAD',
'Three automatic deletion attempts did not stop at administrator attention.');
$debt = CloudBackupService::cleanupDebt($actor);
$debtItem = array_values(array_filter($debt['items'],
static fn (array $item): bool => $item['id'] === (string) $retry->id))[0] ?? null;
$assert(($debt['summary']['attention_count'] ?? 0) >= 1
&& is_array($debtItem)
&& ($debtItem['can_retry'] ?? false) === true
&& !array_key_exists('object_key', $debtItem)
&& !array_key_exists('object_version_id', $debtItem),
'Cleanup debt was not listed safely for administrators.');
$manualHistory = [];
$manual = CloudBackupService::retryCleanup($retry, $actor, $client([
new Response(200, [
'Content-Length' => '1024',
'ETag' => '"stable-etag"',
'x-amz-version-id' => 'version/7+opaque',
]),
], $manualHistory));
$retry->refresh();
$assert($manual['requires_reconciliation'] === true
&& (string) $retry->state === 'delete_uncertain'
&& (int) $retry->delete_cycle_attempts === 0
&& (int) $retry->delete_presence_confirmations === 1
&& (int) $retry->delete_next_check_at > time()
&& count($manualHistory) === 1,
'Manual cleanup retry did not begin a new bounded stable-evidence cycle.');
echo "Portable backup deletion, reconciliation, blocker and quota tests passed.\n";
}