You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
virt/web/tests/CloudPortableBackupDeletion...

362 lines
17 KiB
PHP

<?php
require dirname(__DIR__) . '/vendor/autoload.php';
use App\Models\CloudBackupRestore;
use App\Models\CloudInstanceBackup;
use App\Models\CloudStorageArtifact;
use App\Models\CloudStoragePublication;
use App\Models\CloudStorageReplica;
use App\Services\Cloud\CloudBackupService;
use App\Services\Cloud\CloudError;
use App\Services\Cloud\CloudObjectStorageS3Client;
use App\Services\Cloud\CloudStorageArtifactDeletionService;
use GuzzleHttp\Client;
use GuzzleHttp\Exception\ConnectException;
use GuzzleHttp\Handler\MockHandler;
use GuzzleHttp\HandlerStack;
use GuzzleHttp\Middleware;
use GuzzleHttp\Psr7\Response;
use GuzzleHttp\Psr7\Request;
use Illuminate\Database\Capsule\Manager as Capsule;
use Illuminate\Database\Schema\Blueprint;
$assert = static function (bool $condition, string $message): void {
if (!$condition) {
throw new RuntimeException($message);
}
};
$expectConflict = static function (callable $callback) use ($assert): void {
try {
$callback();
} catch (CloudError $error) {
$assert($error->httpStatus === 409,
'Deletion blocker returned an unexpected status.');
return;
}
throw new RuntimeException('Expected deletion blocker was not raised.');
};
$db = new Capsule();
$db->addConnection(['driver' => 'sqlite', 'database' => ':memory:']);
$db->setAsGlobal();
$db->bootEloquent();
$schema = $db->schema();
$schema->create('cloud_instance_backups', static function (Blueprint $t): void {
$t->increments('id'); $t->string('backup_key')->unique();
$t->integer('source_instance_id'); $t->integer('source_host_id');
$t->integer('user_id'); $t->string('source_uuid'); $t->string('name');
$t->string('purpose')->default('portable'); $t->string('source')->default('manual');
$t->integer('subscription_id')->nullable(); $t->string('state');
$t->string('consistency'); $t->string('object_key')->unique();
$t->integer('object_size_bytes'); $t->integer('virtual_size_bytes');
$t->string('sha256'); $t->string('object_etag')->default('');
$t->string('object_version_id')->default(''); $t->text('spec_json');
$t->text('sealed_credential')->nullable();
$t->integer('storage_revision'); $t->integer('storage_artifact_id')->nullable();
$t->integer('active_job_id')->nullable(); $t->string('last_operation_id')->nullable();
$t->string('delete_operation_id')->nullable();
$t->integer('delete_generation')->default(0);
$t->integer('delete_cycle_attempts')->default(0);
$t->integer('delete_presence_confirmations')->default(0);
$t->integer('delete_requested_at')->nullable();
$t->integer('delete_last_checked_at')->nullable();
$t->integer('delete_next_check_at')->default(0);
$t->integer('delete_attention_at')->nullable();
$t->string('error_code')->default(''); $t->integer('ready_at')->nullable();
$t->integer('created_at'); $t->integer('updated_at');
});
$schema->create('cloud_deployment_snapshots', static function (Blueprint $t): void {
$t->increments('id'); $t->integer('backup_id')->unique();
$t->string('state'); $t->string('error_code')->default('');
$t->integer('deleted_at')->nullable(); $t->integer('updated_at')->default(0);
});
$schema->create('cloud_backup_restores', static function (Blueprint $t): void {
$t->increments('id'); $t->integer('backup_id'); $t->string('state');
});
$schema->create('cloud_backup_restore_attempts', static function (Blueprint $t): void {
$t->increments('id'); $t->integer('backup_restore_id'); $t->string('state');
$t->integer('reservation_id')->nullable();
});
$schema->create('cloud_backup_recoveries', static function (Blueprint $t): void {
$t->increments('id'); $t->integer('backup_id'); $t->string('state');
});
$schema->create('cloud_jobs', static function (Blueprint $t): void {
$t->increments('id'); $t->integer('backup_id')->nullable();
$t->string('phase'); $t->string('effect_resolution');
$t->integer('finalization_incident')->default(0);
});
$schema->create('cloud_resource_reservations', static function (Blueprint $t): void {
$t->increments('id'); $t->string('state'); $t->integer('released_at')->nullable();
});
$schema->create('cloud_resource_reservation_ips', static function (Blueprint $t): void {
$t->increments('id'); $t->integer('reservation_id');
});
$schema->create('cloud_storage_artifacts', static function (Blueprint $t): void {
$t->increments('id'); $t->string('artifact_key')->unique(); $t->string('kind');
$t->integer('maximum_size_bytes'); $t->integer('size_bytes'); $t->string('sha256');
$t->string('durability_policy'); $t->integer('desired_copies');
$t->integer('required_copies'); $t->string('state');
$t->string('last_operation_id')->nullable(); $t->integer('verified_at');
$t->integer('created_at'); $t->integer('updated_at');
});
$schema->create('cloud_storage_replicas', static function (Blueprint $t): void {
$t->increments('id'); $t->string('replica_key')->unique();
$t->integer('artifact_id'); $t->integer('backend_id'); $t->string('role');
$t->string('object_key'); $t->string('object_version_id')->default('');
$t->string('object_etag')->default(''); $t->integer('size_bytes');
$t->string('sha256'); $t->string('state');
$t->string('publish_operation_id')->nullable();
$t->string('delete_operation_id')->nullable();
$t->integer('delete_generation')->default(0);
$t->integer('delete_cycle_attempts')->default(0);
$t->integer('delete_presence_confirmations')->default(0);
$t->integer('delete_last_checked_at')->default(0);
$t->integer('delete_next_check_at')->default(0);
$t->integer('delete_attention_at')->nullable();
$t->string('lease_token')->nullable(); $t->integer('lease_until')->default(0);
$t->string('error_code')->default(''); $t->integer('verified_at')->default(0);
$t->integer('created_at'); $t->integer('updated_at');
});
$schema->create('cloud_storage_operations', static function (Blueprint $t): void {
$t->increments('id'); $t->string('operation_id')->unique();
$t->string('request_idempotency_key')->unique(); $t->string('request_fingerprint');
$t->integer('artifact_id'); $t->integer('replica_id')->nullable();
$t->integer('source_backend_id')->nullable(); $t->integer('target_backend_id')->nullable();
$t->string('kind'); $t->string('phase'); $t->string('outcome')->nullable();
$t->string('effect_resolution'); $t->integer('effect_may_have_applied')->nullable();
$t->integer('safe_to_resubmit')->nullable(); $t->integer('attempts');
$t->string('lease_token')->nullable(); $t->integer('lease_until');
$t->integer('available_at'); $t->integer('next_check_at'); $t->string('error_code');
$t->text('result_json')->nullable(); $t->integer('terminal_at')->nullable();
$t->integer('created_at'); $t->integer('updated_at');
});
$schema->create('cloud_storage_publications', static function (Blueprint $t): void {
$t->increments('id'); $t->string('publication_key')->unique();
$t->integer('artifact_id')->unique(); $t->integer('backup_id')->nullable();
$t->string('template_import_intent_id')->nullable();
$t->string('rescue_template_id')->nullable(); $t->string('state');
$t->integer('attempts'); $t->integer('available_at');
$t->string('lease_token')->nullable(); $t->integer('lease_until');
$t->string('last_error_code'); $t->integer('coverage_ready_at')->nullable();
$t->integer('finalized_at')->nullable(); $t->integer('created_at');
$t->integer('updated_at');
});
$now = time();
$actor = ['user_id' => 1, 'is_admin' => true, 'is_super_admin' => true,
'can_manage' => true];
$newBackup = static function (int $sourceId, string $suffix) use ($now): CloudInstanceBackup {
$row = new CloudInstanceBackup();
$row->backup_key = str_pad($suffix, 32, '0');
$row->source_instance_id = $sourceId; $row->source_host_id = 3;
$row->user_id = 9; $row->source_uuid = '11111111-1111-4111-8111-111111111111';
$row->name = 'backup-' . $suffix; $row->purpose = 'portable';
$row->source = 'manual'; $row->subscription_id = null; $row->state = 'ready';
$row->consistency = 'crash';
$row->object_key = 'placeholder/' . $suffix; $row->object_size_bytes = 1024;
$row->virtual_size_bytes = 2048; $row->sha256 = str_repeat('a', 64);
$row->object_etag = ''; $row->object_version_id = ''; $row->spec_json = '{}';
$row->sealed_credential = 'guest-secret';
$row->storage_revision = 0; $row->storage_artifact_id = null;
$row->active_job_id = null; $row->last_operation_id = null;
$row->delete_operation_id = null; $row->delete_generation = 0;
$row->delete_cycle_attempts = 0; $row->delete_presence_confirmations = 0;
$row->delete_requested_at = null; $row->delete_last_checked_at = null;
$row->delete_next_check_at = 0; $row->delete_attention_at = null;
$row->error_code = ''; $row->ready_at = $now;
$row->created_at = $row->updated_at = $now; $row->save();
return $row;
};
$bindArtifact = static function (
CloudInstanceBackup $owner,
string $seed,
int $copies = 2
) use ($now): CloudStorageArtifact {
$artifact = new CloudStorageArtifact();
$artifact->artifact_key = str_repeat($seed, 32); $artifact->kind = 'backup';
$artifact->maximum_size_bytes = 1024; $artifact->size_bytes = 1024;
$artifact->sha256 = str_repeat('a', 64); $artifact->durability_policy = 'mirror_async_2';
$artifact->desired_copies = $copies; $artifact->required_copies = 1;
$artifact->state = 'protected'; $artifact->last_operation_id = null;
$artifact->verified_at = $now; $artifact->created_at = $artifact->updated_at = $now;
$artifact->save();
$publication = new CloudStoragePublication();
$publication->publication_key = str_repeat(chr(ord($seed) + 1), 32);
$publication->artifact_id = (int) $artifact->id; $publication->backup_id = (int) $owner->id;
$publication->template_import_intent_id = null; $publication->rescue_template_id = null;
$publication->state = 'completed'; $publication->attempts = 0;
$publication->available_at = 0; $publication->lease_token = null;
$publication->lease_until = 0; $publication->last_error_code = '';
$publication->coverage_ready_at = $now; $publication->finalized_at = $now;
$publication->created_at = $publication->updated_at = $now; $publication->save();
$roles = [1 => 'primary', 2 => 'secondary'];
if ($copies === 1) {
unset($roles[2]);
}
foreach ($roles as $backendId => $role) {
$replica = new CloudStorageReplica();
$replica->replica_key = str_repeat($seed . $backendId, 16);
$replica->artifact_id = (int) $artifact->id; $replica->backend_id = $backendId;
$replica->role = $role; $replica->object_key = 'dual/' . $seed . '/' . $backendId;
$replica->object_version_id = 'version-' . $backendId;
$replica->object_etag = '"etag-' . $backendId . '"';
$replica->size_bytes = 1024; $replica->sha256 = str_repeat('a', 64);
$replica->state = 'ready'; $replica->publish_operation_id = null;
$replica->delete_operation_id = null; $replica->delete_generation = 0;
$replica->delete_cycle_attempts = 0; $replica->delete_presence_confirmations = 0;
$replica->delete_last_checked_at = 0; $replica->delete_next_check_at = 0;
$replica->delete_attention_at = null; $replica->lease_token = null;
$replica->lease_until = 0; $replica->error_code = ''; $replica->verified_at = $now;
$replica->created_at = $replica->updated_at = $now; $replica->save();
}
$owner->storage_artifact_id = (int) $artifact->id; $owner->save();
return $artifact;
};
$client = static function (array $responses, array &$history): CloudObjectStorageS3Client {
$stack = HandlerStack::create(new MockHandler($responses));
$stack->push(Middleware::history($history));
return new CloudObjectStorageS3Client(new Client(['handler' => $stack]));
};
$runtime = ['config' => ['endpoint' => 'https://objects.example.test',
'region' => 'us-east-1', 'bucket' => 'bucket', 'prefix' => '', 'path_style' => true],
'credentials' => ['access_key_id' => 'KEY', 'secret_access_key' => 'secret'],
'revision' => 1, 'backend_id' => 1];
$legacy = $newBackup(10, 'legacy');
$legacyHistory = [];
$expectConflict(static fn () => CloudBackupService::delete(
$legacy, $actor, $client([], $legacyHistory)
));
$assert($legacyHistory === [] && (string) $legacy->fresh()->state === 'ready',
'A backup without a replica ledger attempted provider deletion.');
$backup = $newBackup(11, 'artifact');
$artifact = $bindArtifact($backup, 'a');
$history = [];
$artifactClient = $client([
new Response(200, ['Content-Length' => '1024', 'ETag' => '"etag-1"',
'x-amz-version-id' => 'version-1']),
new Response(204),
new Response(200, ['Content-Length' => '1024', 'ETag' => '"etag-2"',
'x-amz-version-id' => 'version-2']),
new Response(204),
], $history);
$deletion = new CloudStorageArtifactDeletionService(
$artifactClient,
static fn (int $backendId, bool $healthy): array => $runtime
);
$result = CloudBackupService::delete($backup, $actor, $artifactClient, null, $deletion);
$backup->refresh(); $artifact->refresh();
$assert(($result['deleted'] ?? false) === true
&& (string) $backup->state === 'deleted'
&& (string) $artifact->state === 'deleted'
&& $backup->sealed_credential === null
&& count($history) === 4
&& CloudStorageReplica::where('artifact_id', $artifact->id)
->where('state', 'deleted')->count() === 2,
'Artifact-backed backup did not delete every exact replica and tombstone its owner.');
$blocked = $newBackup(12, 'blocked');
$blockedArtifact = $bindArtifact($blocked, 'c');
$restore = new CloudBackupRestore(); $restore->backup_id = (int) $blocked->id;
$restore->state = 'restoring'; $restore->save();
$blockedHistory = [];
$expectConflict(static fn () => CloudBackupService::delete(
$blocked, $actor, $client([], $blockedHistory)
));
$blocked->refresh(); $blockedArtifact->refresh();
$assert($blockedHistory === [] && (string) $blocked->state === 'ready'
&& (string) $blockedArtifact->state === 'protected',
'An active restore did not preserve its replicated source backup.');
$unknown = $newBackup(13, 'unknown');
$unknownArtifact = $bindArtifact($unknown, 'e', 1);
$unknownHistory = [];
$unknownClient = $client([
new Response(200, ['Content-Length' => '1024', 'ETag' => '"etag-1"',
'x-amz-version-id' => 'version-1']),
new ConnectException(
'delete response lost',
new Request('DELETE', 'https://objects.example.test')
),
new Response(404),
], $unknownHistory);
$clock = 1_000;
$unknownDeletion = new CloudStorageArtifactDeletionService(
$unknownClient,
static fn (int $backendId, bool $healthy): array => $runtime,
static function () use (&$clock): int {
return $clock;
}
);
$unknownResult = CloudBackupService::delete(
$unknown,
$actor,
$unknownClient,
null,
$unknownDeletion
);
$unknownOperation = Capsule::table('cloud_storage_operations')
->where('artifact_id', (int) $unknownArtifact->id)
->where('kind', 'delete')->first();
$unknownReplica = CloudStorageReplica::where(
'artifact_id',
(int) $unknownArtifact->id
)->firstOrFail();
$assert(($unknownResult['requires_reconciliation'] ?? false) === true
&& $unknownOperation !== null
&& $unknownOperation->effect_resolution === 'unknown'
&& $unknownOperation->phase === 'reconciling'
&& $unknownOperation->outcome === null,
'Ambiguous replica deletion lost its original durable operation.');
// Model an administrator-attention projection after the same unknown
// operation exhausted its bounded automatic observations. Reopening must
// continue that operation with read-only evidence, not create another DELETE.
$unknownArtifact->state = 'delete_attention'; $unknownArtifact->save();
$operationId = (string) $unknownOperation->operation_id;
Capsule::table('cloud_storage_operations')
->where('operation_id', $operationId)->update([
'phase' => 'terminal',
'outcome' => 'failed',
'effect_resolution' => 'unknown',
'effect_may_have_applied' => 1,
'safe_to_resubmit' => 0,
'next_check_at' => 0,
'terminal_at' => $clock,
]);
$unknownReplica->state = 'delete_attention';
$unknownReplica->delete_attention_at = $clock;
$unknownReplica->delete_presence_confirmations = 0;
$unknownReplica->save();
$unknown->state = 'delete_attention'; $unknown->delete_attention_at = $clock;
$unknown->save();
$operationCount = Capsule::table('cloud_storage_operations')
->where('artifact_id', (int) $unknownArtifact->id)
->where('kind', 'delete')->count();
$clock = 1_061;
$recovered = CloudBackupService::retryCleanup(
$unknown,
$actor,
$unknownClient,
$unknownDeletion
);
$unknown->refresh(); $unknownArtifact->refresh(); $unknownReplica->refresh();
$assert(($recovered['deleted'] ?? false) === true
&& (string) $unknown->state === 'deleted'
&& (string) $unknownArtifact->state === 'deleted'
&& (string) $unknownReplica->state === 'deleted'
&& Capsule::table('cloud_storage_operations')
->where('artifact_id', (int) $unknownArtifact->id)
->where('kind', 'delete')->count() === $operationCount
&& Capsule::table('cloud_storage_operations')
->where('operation_id', $operationId)->exists()
&& array_map(static fn (array $entry): string =>
$entry['request']->getMethod(), $unknownHistory) === [
'HEAD', 'DELETE', 'HEAD',
],
'Manual recovery did not reconcile the original ambiguous deletion.');
echo "Portable backup replicated deletion and clean-break tests passed.\n";