You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
396 lines
16 KiB
PHP
396 lines
16 KiB
PHP
<?php
|
|
|
|
namespace App\Services {
|
|
final class Config
|
|
{
|
|
public static string $dataKey = '';
|
|
|
|
public static function get($name, $default = null)
|
|
{
|
|
return $name === 'cloud_data_key' ? self::$dataKey : $default;
|
|
}
|
|
}
|
|
}
|
|
|
|
namespace {
|
|
require dirname(__DIR__) . '/vendor/autoload.php';
|
|
|
|
use App\Models\CloudBackupRestore;
|
|
use App\Models\CloudInstanceBackup;
|
|
use App\Services\Config;
|
|
use App\Services\Cloud\CloudBackupService;
|
|
use App\Services\Cloud\CloudError;
|
|
use App\Services\Cloud\CloudObjectStorageS3Client;
|
|
use App\Services\Cloud\CloudSecrets;
|
|
use GuzzleHttp\Client;
|
|
use GuzzleHttp\Exception\ConnectException;
|
|
use GuzzleHttp\Handler\MockHandler;
|
|
use GuzzleHttp\HandlerStack;
|
|
use GuzzleHttp\Middleware;
|
|
use GuzzleHttp\Psr7\Request;
|
|
use GuzzleHttp\Psr7\Response;
|
|
use Illuminate\Database\Capsule\Manager as Capsule;
|
|
use Illuminate\Database\Schema\Blueprint;
|
|
|
|
if (!function_exists('sodium_crypto_aead_xchacha20poly1305_ietf_encrypt')) {
|
|
echo "SKIPPED CloudPortableBackupDeletionTest: runtime lacks sodium.\n";
|
|
exit(0);
|
|
}
|
|
|
|
$assert = static function (bool $condition, string $message): void {
|
|
if (!$condition) {
|
|
throw new RuntimeException($message);
|
|
}
|
|
};
|
|
$expectConflict = static function (callable $callback) use ($assert): void {
|
|
try {
|
|
$callback();
|
|
} catch (CloudError $error) {
|
|
$assert($error->httpStatus === 409,
|
|
'Deletion blocker returned an unexpected status.');
|
|
return;
|
|
}
|
|
throw new RuntimeException('Expected deletion blocker was not raised.');
|
|
};
|
|
|
|
$capsule = new Capsule();
|
|
$capsule->addConnection([
|
|
'driver' => 'sqlite', 'database' => ':memory:',
|
|
], 'default');
|
|
$capsule->setAsGlobal();
|
|
$capsule->bootEloquent();
|
|
$schema = $capsule->schema('default');
|
|
$schema->create('cloud_instance_backups',
|
|
static function (Blueprint $table): void {
|
|
$table->increments('id');
|
|
$table->string('backup_key')->unique();
|
|
$table->integer('source_instance_id');
|
|
$table->integer('source_host_id');
|
|
$table->integer('user_id');
|
|
$table->string('source_uuid');
|
|
$table->string('name');
|
|
$table->string('purpose')->default('portable');
|
|
$table->string('state');
|
|
$table->string('consistency');
|
|
$table->string('object_key')->unique();
|
|
$table->integer('object_size_bytes');
|
|
$table->integer('virtual_size_bytes');
|
|
$table->string('sha256');
|
|
$table->string('object_etag')->default('');
|
|
$table->string('object_version_id')->default('');
|
|
$table->text('spec_json');
|
|
$table->text('sealed_storage_runtime');
|
|
$table->text('sealed_credential')->nullable();
|
|
$table->integer('storage_revision');
|
|
$table->integer('active_job_id')->nullable();
|
|
$table->string('last_operation_id')->nullable();
|
|
$table->string('delete_operation_id')->nullable();
|
|
$table->integer('delete_generation')->default(0);
|
|
$table->integer('delete_cycle_attempts')->default(0);
|
|
$table->integer('delete_presence_confirmations')->default(0);
|
|
$table->integer('delete_requested_at')->nullable();
|
|
$table->integer('delete_last_checked_at')->nullable();
|
|
$table->integer('delete_next_check_at')->default(0);
|
|
$table->integer('delete_attention_at')->nullable();
|
|
$table->string('error_code');
|
|
$table->integer('ready_at')->nullable();
|
|
$table->integer('created_at');
|
|
$table->integer('updated_at');
|
|
});
|
|
$schema->create('cloud_deployment_snapshots',
|
|
static function (Blueprint $table): void {
|
|
$table->increments('id');
|
|
$table->integer('backup_id')->unique();
|
|
$table->string('state');
|
|
$table->string('error_code')->default('');
|
|
$table->integer('deleted_at')->nullable();
|
|
$table->integer('updated_at')->default(0);
|
|
});
|
|
$schema->create('cloud_backup_restores',
|
|
static function (Blueprint $table): void {
|
|
$table->increments('id');
|
|
$table->integer('backup_id');
|
|
$table->string('state');
|
|
});
|
|
$schema->create('cloud_backup_recoveries',
|
|
static function (Blueprint $table): void {
|
|
$table->increments('id');
|
|
$table->integer('backup_id');
|
|
$table->string('state');
|
|
});
|
|
$schema->create('cloud_jobs', static function (Blueprint $table): void {
|
|
$table->increments('id');
|
|
$table->integer('backup_id')->nullable();
|
|
$table->string('phase');
|
|
$table->string('effect_resolution');
|
|
$table->integer('finalization_incident')->default(0);
|
|
});
|
|
|
|
Config::$dataKey = base64_encode(str_repeat('b', 32));
|
|
$runtime = [
|
|
'config' => [
|
|
'endpoint' => 'https://objects.example.com',
|
|
'region' => 'us-east-1',
|
|
'bucket' => 'backup-bucket',
|
|
'prefix' => 'virt',
|
|
'path_style' => true,
|
|
],
|
|
'credentials' => [
|
|
'access_key_id' => 'BACKUPACCESS',
|
|
'secret_access_key' => 'backup-secret-value',
|
|
],
|
|
'revision' => 7,
|
|
];
|
|
$sealedRuntime = CloudSecrets::encrypt(
|
|
$runtime,
|
|
'object-storage-backup-runtime'
|
|
);
|
|
$actor = [
|
|
'user_id' => 1,
|
|
'is_admin' => true,
|
|
'is_super_admin' => true,
|
|
'can_manage' => true,
|
|
];
|
|
$newBackup = static function (int $sourceInstanceId, string $suffix,
|
|
string $state = 'ready') use ($sealedRuntime): CloudInstanceBackup {
|
|
$backup = new CloudInstanceBackup();
|
|
$backup->backup_key = str_pad($suffix, 32, '0');
|
|
$backup->source_instance_id = $sourceInstanceId;
|
|
$backup->source_host_id = 3;
|
|
$backup->user_id = 9;
|
|
$backup->source_uuid = '11111111-1111-4111-8111-111111111111';
|
|
$backup->name = 'backup-' . $suffix;
|
|
$backup->state = $state;
|
|
$backup->consistency = 'crash';
|
|
$backup->object_key = 'virt/vm-backups/' . $suffix . '/system.qcow2';
|
|
$backup->object_size_bytes = 1024;
|
|
$backup->virtual_size_bytes = 2048;
|
|
$backup->sha256 = str_repeat('a', 64);
|
|
$backup->object_etag = '';
|
|
$backup->object_version_id = '';
|
|
$backup->spec_json = '{}';
|
|
$backup->sealed_storage_runtime = $sealedRuntime;
|
|
$backup->sealed_credential = 'sealed-guest-credential';
|
|
$backup->storage_revision = 7;
|
|
$backup->active_job_id = null;
|
|
$backup->last_operation_id = null;
|
|
$backup->delete_operation_id = null;
|
|
$backup->delete_generation = 0;
|
|
$backup->delete_cycle_attempts = 0;
|
|
$backup->delete_presence_confirmations = 0;
|
|
$backup->delete_requested_at = null;
|
|
$backup->delete_last_checked_at = null;
|
|
$backup->delete_next_check_at = 0;
|
|
$backup->delete_attention_at = null;
|
|
$backup->error_code = '';
|
|
$backup->ready_at = time();
|
|
$backup->created_at = $backup->updated_at = time();
|
|
$backup->save();
|
|
return $backup;
|
|
};
|
|
$client = static function (array $responses, array &$history): CloudObjectStorageS3Client {
|
|
$stack = HandlerStack::create(new MockHandler($responses));
|
|
$stack->push(Middleware::history($history));
|
|
return new CloudObjectStorageS3Client(new Client(['handler' => $stack]));
|
|
};
|
|
$identity = static fn (): Response => new Response(200, [
|
|
'Content-Length' => '1024',
|
|
'ETag' => '"catalog-etag"',
|
|
'x-amz-version-id' => 'catalog-version',
|
|
]);
|
|
|
|
$history = [];
|
|
$backup = $newBackup(10, 'one');
|
|
$deleted = CloudBackupService::delete(
|
|
$backup, $actor, $client([$identity(), new Response(204)], $history)
|
|
);
|
|
$backup->refresh();
|
|
$assert($deleted['deleted'] === true
|
|
&& (string) $backup->state === 'deleted'
|
|
&& (string) $backup->sealed_storage_runtime === ''
|
|
&& $backup->sealed_credential === null
|
|
&& count($history) === 2
|
|
&& $history[0]['request']->getMethod() === 'HEAD'
|
|
&& $history[1]['request']->getMethod() === 'DELETE'
|
|
&& str_contains((string) $history[1]['request']->getUri(),
|
|
'versionId=catalog-version'),
|
|
'Explicit DeleteObject success did not tombstone the catalog and remove frozen credentials.');
|
|
|
|
$unknown = $newBackup(11, 'two');
|
|
$unknownHistory = [];
|
|
$unknownResult = CloudBackupService::delete($unknown, $actor, $client([
|
|
$identity(),
|
|
new ConnectException('transport lost',
|
|
new Request('DELETE', 'https://objects.example.com')),
|
|
], $unknownHistory));
|
|
$unknown->refresh();
|
|
$assert($unknownResult['requires_reconciliation'] === true
|
|
&& (string) $unknown->state === 'delete_uncertain'
|
|
&& (string) $unknown->sealed_storage_runtime !== '',
|
|
'Ambiguous DeleteObject did not retain the frozen runtime and unknown-effect fence.');
|
|
$unknown->delete_next_check_at = 0;
|
|
$unknown->save();
|
|
$probeHistory = [];
|
|
$reconciled = CloudBackupService::delete(
|
|
$unknown, $actor, $client([new Response(404)], $probeHistory)
|
|
);
|
|
$assert($reconciled['deleted'] === true
|
|
&& count($probeHistory) === 1
|
|
&& $probeHistory[0]['request']->getMethod() === 'HEAD',
|
|
'A later explicit 404 did not converge deletion without replaying DELETE.');
|
|
|
|
$present = $newBackup(12, 'three');
|
|
$firstHistory = [];
|
|
CloudBackupService::delete($present, $actor, $client([
|
|
$identity(),
|
|
new ConnectException('transport lost',
|
|
new Request('DELETE', 'https://objects.example.com')),
|
|
], $firstHistory));
|
|
$present->refresh();
|
|
$present->delete_next_check_at = 0;
|
|
$present->save();
|
|
$presentHistory = [];
|
|
$stillPresent = CloudBackupService::delete(
|
|
$present, $actor, $client([$identity()], $presentHistory)
|
|
);
|
|
$present->refresh();
|
|
$assert($stillPresent['requires_reconciliation'] === true
|
|
&& (string) $present->state === 'delete_uncertain'
|
|
&& count($presentHistory) === 1
|
|
&& $presentHistory[0]['request']->getMethod() === 'HEAD',
|
|
'An existing object after ambiguous deletion was replayed or lost its fence.');
|
|
|
|
$blocked = $newBackup(13, 'four');
|
|
$restore = new CloudBackupRestore();
|
|
$restore->backup_id = (int) $blocked->id;
|
|
$restore->state = 'restoring';
|
|
$restore->save();
|
|
$expectConflict(static function () use ($blocked, $actor): void {
|
|
CloudBackupService::delete($blocked, $actor);
|
|
});
|
|
$blocked->refresh();
|
|
$assert((string) $blocked->state === 'ready',
|
|
'An active restore did not preserve its source backup.');
|
|
|
|
$readOnlyBackup = $newBackup(14, 'read-only');
|
|
$displayMethod = (new ReflectionClass(CloudBackupService::class))
|
|
->getMethod('displayBackup');
|
|
$readOnlyDisplay = $displayMethod->invoke(
|
|
null, $readOnlyBackup, null, false
|
|
);
|
|
$assert($readOnlyDisplay['can_delete'] === false,
|
|
'Read-only administrators were offered a backup delete action.');
|
|
|
|
$quotaBackups = [];
|
|
for ($index = 0; $index < CloudBackupService::MAX_PER_INSTANCE; $index++) {
|
|
$quotaBackups[] = $newBackup(99, 'quota-' . $index);
|
|
}
|
|
$reflection = new ReflectionClass(CloudBackupService::class);
|
|
$activeStates = $reflection->getReflectionConstant(
|
|
'ACTIVE_BACKUP_STATES'
|
|
)->getValue();
|
|
$before = CloudInstanceBackup::where('source_instance_id', 99)
|
|
->whereIn('state', $activeStates)->count();
|
|
$quotaHistory = [];
|
|
CloudBackupService::delete(
|
|
$quotaBackups[0], $actor,
|
|
$client([$identity(), new Response(204)], $quotaHistory)
|
|
);
|
|
$after = CloudInstanceBackup::where('source_instance_id', 99)
|
|
->whereIn('state', $activeStates)->count();
|
|
$assert($before === CloudBackupService::MAX_PER_INSTANCE
|
|
&& $after === CloudBackupService::MAX_PER_INSTANCE - 1,
|
|
'Confirmed deletion did not release one retained-backup quota slot.');
|
|
|
|
// One ambiguous generation may be retried only after two stable, spaced
|
|
// identity probes. The retry is a new deletion generation bound to the
|
|
// exact S3 version instead of a replay of the ambiguous request.
|
|
$retry = $newBackup(100, 'stable-retry', 'delete_uncertain');
|
|
$retry->object_etag = '"stable-etag"';
|
|
$retry->object_version_id = 'version/7+opaque';
|
|
$retry->delete_operation_id = str_repeat('1', 32);
|
|
$retry->delete_generation = 1;
|
|
$retry->delete_cycle_attempts = 1;
|
|
$retry->delete_presence_confirmations = 1;
|
|
$retry->delete_requested_at = time() - 300;
|
|
$retry->delete_last_checked_at = time() - 120;
|
|
$retry->delete_next_check_at = 0;
|
|
$retry->save();
|
|
$retryHistory = [];
|
|
$retryResult = CloudBackupService::delete($retry, $actor, $client([
|
|
new Response(200, [
|
|
'Content-Length' => '1024',
|
|
'ETag' => '"stable-etag"',
|
|
'x-amz-version-id' => 'version/7+opaque',
|
|
]),
|
|
new ConnectException('retry response lost',
|
|
new Request('DELETE', 'https://objects.example.com')),
|
|
], $retryHistory));
|
|
$retry->refresh();
|
|
$retryUris = array_map(static fn ($entry): string =>
|
|
(string) $entry['request']->getUri(), $retryHistory);
|
|
$assert($retryResult['requires_reconciliation'] === true
|
|
&& (string) $retry->state === 'delete_uncertain'
|
|
&& (int) $retry->delete_generation === 2
|
|
&& (int) $retry->delete_cycle_attempts === 2
|
|
&& count($retryHistory) === 2
|
|
&& $retryHistory[0]['request']->getMethod() === 'HEAD'
|
|
&& $retryHistory[1]['request']->getMethod() === 'DELETE'
|
|
&& str_contains($retryUris[0], 'versionId=version%2F7%2Bopaque')
|
|
&& str_contains($retryUris[1], 'versionId=version%2F7%2Bopaque'),
|
|
'Stable object identity did not create a fresh, exact-version deletion generation.');
|
|
|
|
$assert(!in_array('delete_uncertain', $activeStates, true),
|
|
'Object cleanup debt still consumes the user retained-backup quota.');
|
|
|
|
$retry->delete_cycle_attempts = 3;
|
|
$retry->delete_generation = 3;
|
|
$retry->delete_presence_confirmations = 1;
|
|
$retry->delete_last_checked_at = time() - 120;
|
|
$retry->delete_next_check_at = 0;
|
|
$retry->save();
|
|
$exhaustedHistory = [];
|
|
$exhausted = CloudBackupService::delete($retry, $actor, $client([
|
|
new Response(200, [
|
|
'Content-Length' => '1024',
|
|
'ETag' => '"stable-etag"',
|
|
'x-amz-version-id' => 'version/7+opaque',
|
|
]),
|
|
], $exhaustedHistory));
|
|
$retry->refresh();
|
|
$assert(($exhausted['requires_attention'] ?? false) === true
|
|
&& (string) $retry->state === 'delete_attention'
|
|
&& count($exhaustedHistory) === 1
|
|
&& $exhaustedHistory[0]['request']->getMethod() === 'HEAD',
|
|
'Three automatic deletion attempts did not stop at administrator attention.');
|
|
|
|
$debt = CloudBackupService::cleanupDebt($actor);
|
|
$debtItem = array_values(array_filter($debt['items'],
|
|
static fn (array $item): bool => $item['id'] === (string) $retry->id))[0] ?? null;
|
|
$assert(($debt['summary']['attention_count'] ?? 0) >= 1
|
|
&& is_array($debtItem)
|
|
&& ($debtItem['can_retry'] ?? false) === true
|
|
&& !array_key_exists('object_key', $debtItem)
|
|
&& !array_key_exists('object_version_id', $debtItem),
|
|
'Cleanup debt was not listed safely for administrators.');
|
|
|
|
$manualHistory = [];
|
|
$manual = CloudBackupService::retryCleanup($retry, $actor, $client([
|
|
new Response(200, [
|
|
'Content-Length' => '1024',
|
|
'ETag' => '"stable-etag"',
|
|
'x-amz-version-id' => 'version/7+opaque',
|
|
]),
|
|
], $manualHistory));
|
|
$retry->refresh();
|
|
$assert($manual['requires_reconciliation'] === true
|
|
&& (string) $retry->state === 'delete_uncertain'
|
|
&& (int) $retry->delete_cycle_attempts === 0
|
|
&& (int) $retry->delete_presence_confirmations === 1
|
|
&& (int) $retry->delete_next_check_at > time()
|
|
&& count($manualHistory) === 1,
|
|
'Manual cleanup retry did not begin a new bounded stable-evidence cycle.');
|
|
|
|
echo "Portable backup deletion, reconciliation, blocker and quota tests passed.\n";
|
|
}
|