You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
virt/web/tests/CloudPortableBackupDeletion...

968 lines
41 KiB
PHP

<?php
namespace App\Services {
final class Config
{
public static string $dataKey = '';
public static function get($name, $default = null)
{
return $name === 'cloud_data_key' ? self::$dataKey : $default;
}
}
}
namespace {
require dirname(__DIR__) . '/vendor/autoload.php';
use App\Models\CloudBackupRestore;
use App\Models\CloudInstanceBackup;
use App\Models\CloudStorageArtifact;
use App\Models\CloudStorageOperation;
use App\Models\CloudStoragePublication;
use App\Models\CloudStorageReplica;
use App\Services\Config;
use App\Services\Cloud\CloudBackupService;
use App\Services\Cloud\CloudError;
use App\Services\Cloud\CloudObjectStorageS3Client;
use App\Services\Cloud\CloudStorageArtifactDeletionService;
use App\Services\Cloud\CloudSecrets;
use GuzzleHttp\Client;
use GuzzleHttp\Exception\ConnectException;
use GuzzleHttp\Handler\MockHandler;
use GuzzleHttp\HandlerStack;
use GuzzleHttp\Middleware;
use GuzzleHttp\Psr7\Request;
use GuzzleHttp\Psr7\Response;
use Illuminate\Database\Capsule\Manager as Capsule;
use Illuminate\Database\Schema\Blueprint;
if (!function_exists('sodium_crypto_aead_xchacha20poly1305_ietf_encrypt')) {
echo "SKIPPED CloudPortableBackupDeletionTest: runtime lacks sodium.\n";
exit(0);
}
$assert = static function (bool $condition, string $message): void {
if (!$condition) {
throw new RuntimeException($message);
}
};
$expectConflict = static function (callable $callback) use ($assert): void {
try {
$callback();
} catch (CloudError $error) {
$assert($error->httpStatus === 409,
'Deletion blocker returned an unexpected status.');
return;
}
throw new RuntimeException('Expected deletion blocker was not raised.');
};
$capsule = new Capsule();
$capsule->addConnection([
'driver' => 'sqlite', 'database' => ':memory:',
], 'default');
$capsule->setAsGlobal();
$capsule->bootEloquent();
$schema = $capsule->schema('default');
$schema->create('cloud_instance_backups',
static function (Blueprint $table): void {
$table->increments('id');
$table->string('backup_key')->unique();
$table->integer('source_instance_id');
$table->integer('source_host_id');
$table->integer('user_id');
$table->string('source_uuid');
$table->string('name');
$table->string('purpose')->default('portable');
$table->string('state');
$table->string('consistency');
$table->string('object_key')->unique();
$table->integer('object_size_bytes');
$table->integer('virtual_size_bytes');
$table->string('sha256');
$table->string('object_etag')->default('');
$table->string('object_version_id')->default('');
$table->text('spec_json');
$table->text('sealed_storage_runtime');
$table->text('sealed_credential')->nullable();
$table->integer('storage_revision');
$table->integer('storage_artifact_id')->nullable();
$table->integer('active_job_id')->nullable();
$table->string('last_operation_id')->nullable();
$table->string('delete_operation_id')->nullable();
$table->integer('delete_generation')->default(0);
$table->integer('delete_cycle_attempts')->default(0);
$table->integer('delete_presence_confirmations')->default(0);
$table->integer('delete_requested_at')->nullable();
$table->integer('delete_last_checked_at')->nullable();
$table->integer('delete_next_check_at')->default(0);
$table->integer('delete_attention_at')->nullable();
$table->string('error_code');
$table->integer('ready_at')->nullable();
$table->integer('created_at');
$table->integer('updated_at');
});
$schema->create('cloud_deployment_snapshots',
static function (Blueprint $table): void {
$table->increments('id');
$table->integer('backup_id')->unique();
$table->string('state');
$table->string('error_code')->default('');
$table->integer('deleted_at')->nullable();
$table->integer('updated_at')->default(0);
});
$schema->create('cloud_backup_restores',
static function (Blueprint $table): void {
$table->increments('id');
$table->integer('backup_id');
$table->string('state');
});
$schema->create('cloud_backup_recoveries',
static function (Blueprint $table): void {
$table->increments('id');
$table->integer('backup_id');
$table->string('state');
});
$schema->create('cloud_jobs', static function (Blueprint $table): void {
$table->increments('id');
$table->integer('backup_id')->nullable();
$table->string('phase');
$table->string('effect_resolution');
$table->integer('finalization_incident')->default(0);
});
$schema->create('cloud_storage_artifacts',
static function (Blueprint $table): void {
$table->increments('id');
$table->string('artifact_key')->unique();
$table->string('kind');
$table->integer('maximum_size_bytes');
$table->integer('size_bytes');
$table->string('sha256');
$table->string('durability_policy');
$table->integer('desired_copies');
$table->integer('required_copies');
$table->string('state');
$table->string('last_operation_id')->nullable();
$table->integer('verified_at');
$table->integer('created_at');
$table->integer('updated_at');
});
$schema->create('cloud_storage_replicas',
static function (Blueprint $table): void {
$table->increments('id');
$table->string('replica_key')->unique();
$table->integer('artifact_id');
$table->integer('backend_id');
$table->string('role');
$table->string('object_key');
$table->string('object_version_id')->default('');
$table->string('object_etag')->default('');
$table->integer('size_bytes');
$table->string('sha256');
$table->string('state');
$table->string('publish_operation_id')->nullable();
$table->string('delete_operation_id')->nullable();
$table->integer('delete_generation')->default(0);
$table->integer('delete_cycle_attempts')->default(0);
$table->integer('delete_presence_confirmations')->default(0);
$table->integer('delete_last_checked_at')->default(0);
$table->integer('delete_next_check_at')->default(0);
$table->integer('delete_attention_at')->nullable();
$table->string('lease_token')->nullable();
$table->integer('lease_until')->default(0);
$table->string('error_code')->default('');
$table->integer('verified_at')->default(0);
$table->integer('created_at');
$table->integer('updated_at');
});
$schema->create('cloud_storage_operations',
static function (Blueprint $table): void {
$table->increments('id');
$table->string('operation_id')->unique();
$table->string('request_idempotency_key')->unique();
$table->string('request_fingerprint');
$table->integer('artifact_id');
$table->integer('replica_id')->nullable();
$table->integer('source_backend_id')->nullable();
$table->integer('target_backend_id')->nullable();
$table->string('kind');
$table->string('phase');
$table->string('outcome')->nullable();
$table->string('effect_resolution');
$table->integer('effect_may_have_applied')->nullable();
$table->integer('safe_to_resubmit')->nullable();
$table->integer('attempts');
$table->string('lease_token')->nullable();
$table->integer('lease_until');
$table->integer('available_at');
$table->integer('next_check_at');
$table->string('error_code');
$table->text('result_json')->nullable();
$table->integer('terminal_at')->nullable();
$table->integer('created_at');
$table->integer('updated_at');
});
$schema->create('cloud_storage_publications',
static function (Blueprint $table): void {
$table->increments('id');
$table->string('publication_key')->unique();
$table->integer('artifact_id')->unique();
$table->integer('backup_id')->nullable();
$table->string('template_import_intent_id')->nullable();
$table->string('rescue_template_id')->nullable();
$table->string('state');
$table->integer('attempts');
$table->integer('available_at');
$table->string('lease_token')->nullable();
$table->integer('lease_until');
$table->string('last_error_code');
$table->integer('coverage_ready_at')->nullable();
$table->integer('finalized_at')->nullable();
$table->integer('created_at');
$table->integer('updated_at');
});
Config::$dataKey = base64_encode(str_repeat('b', 32));
$runtime = [
'config' => [
'endpoint' => 'https://objects.example.com',
'region' => 'us-east-1',
'bucket' => 'backup-bucket',
'prefix' => 'virt',
'path_style' => true,
],
'credentials' => [
'access_key_id' => 'BACKUPACCESS',
'secret_access_key' => 'backup-secret-value',
],
'revision' => 7,
];
$sealedRuntime = CloudSecrets::encrypt(
$runtime,
'object-storage-backup-runtime'
);
$actor = [
'user_id' => 1,
'is_admin' => true,
'is_super_admin' => true,
'can_manage' => true,
];
$newBackup = static function (int $sourceInstanceId, string $suffix,
string $state = 'ready') use ($sealedRuntime): CloudInstanceBackup {
$backup = new CloudInstanceBackup();
$backup->backup_key = str_pad($suffix, 32, '0');
$backup->source_instance_id = $sourceInstanceId;
$backup->source_host_id = 3;
$backup->user_id = 9;
$backup->source_uuid = '11111111-1111-4111-8111-111111111111';
$backup->name = 'backup-' . $suffix;
$backup->state = $state;
$backup->consistency = 'crash';
$backup->object_key = 'virt/vm-backups/' . $suffix . '/system.qcow2';
$backup->object_size_bytes = 1024;
$backup->virtual_size_bytes = 2048;
$backup->sha256 = str_repeat('a', 64);
$backup->object_etag = '';
$backup->object_version_id = '';
$backup->spec_json = '{}';
$backup->sealed_storage_runtime = $sealedRuntime;
$backup->sealed_credential = 'sealed-guest-credential';
$backup->storage_revision = 7;
$backup->storage_artifact_id = null;
$backup->active_job_id = null;
$backup->last_operation_id = null;
$backup->delete_operation_id = null;
$backup->delete_generation = 0;
$backup->delete_cycle_attempts = 0;
$backup->delete_presence_confirmations = 0;
$backup->delete_requested_at = null;
$backup->delete_last_checked_at = null;
$backup->delete_next_check_at = 0;
$backup->delete_attention_at = null;
$backup->error_code = '';
$backup->ready_at = time();
$backup->created_at = $backup->updated_at = time();
$backup->save();
return $backup;
};
$client = static function (array $responses, array &$history): CloudObjectStorageS3Client {
$stack = HandlerStack::create(new MockHandler($responses));
$stack->push(Middleware::history($history));
return new CloudObjectStorageS3Client(new Client(['handler' => $stack]));
};
$identity = static fn (): Response => new Response(200, [
'Content-Length' => '1024',
'ETag' => '"catalog-etag"',
'x-amz-version-id' => 'catalog-version',
]);
$history = [];
$backup = $newBackup(10, 'one');
$deleted = CloudBackupService::delete(
$backup, $actor, $client([$identity(), new Response(204)], $history)
);
$backup->refresh();
$assert($deleted['deleted'] === true
&& (string) $backup->state === 'deleted'
&& (string) $backup->sealed_storage_runtime === ''
&& $backup->sealed_credential === null
&& count($history) === 2
&& $history[0]['request']->getMethod() === 'HEAD'
&& $history[1]['request']->getMethod() === 'DELETE'
&& str_contains((string) $history[1]['request']->getUri(),
'versionId=catalog-version'),
'Explicit DeleteObject success did not tombstone the catalog and remove frozen credentials.');
$unknown = $newBackup(11, 'two');
$unknownHistory = [];
$unknownResult = CloudBackupService::delete($unknown, $actor, $client([
$identity(),
new ConnectException('transport lost',
new Request('DELETE', 'https://objects.example.com')),
], $unknownHistory));
$unknown->refresh();
$assert($unknownResult['requires_reconciliation'] === true
&& (string) $unknown->state === 'delete_uncertain'
&& (string) $unknown->sealed_storage_runtime !== '',
'Ambiguous DeleteObject did not retain the frozen runtime and unknown-effect fence.');
$unknown->delete_next_check_at = 0;
$unknown->save();
$probeHistory = [];
$reconciled = CloudBackupService::delete(
$unknown, $actor, $client([new Response(404)], $probeHistory)
);
$assert($reconciled['deleted'] === true
&& count($probeHistory) === 1
&& $probeHistory[0]['request']->getMethod() === 'HEAD',
'A later explicit 404 did not converge deletion without replaying DELETE.');
$present = $newBackup(12, 'three');
$firstHistory = [];
CloudBackupService::delete($present, $actor, $client([
$identity(),
new ConnectException('transport lost',
new Request('DELETE', 'https://objects.example.com')),
], $firstHistory));
$present->refresh();
$present->delete_next_check_at = 0;
$present->save();
$presentHistory = [];
$stillPresent = CloudBackupService::delete(
$present, $actor, $client([$identity()], $presentHistory)
);
$present->refresh();
$assert($stillPresent['requires_reconciliation'] === true
&& (string) $present->state === 'delete_uncertain'
&& count($presentHistory) === 1
&& $presentHistory[0]['request']->getMethod() === 'HEAD',
'An existing object after ambiguous deletion was replayed or lost its fence.');
$blocked = $newBackup(13, 'four');
$restore = new CloudBackupRestore();
$restore->backup_id = (int) $blocked->id;
$restore->state = 'restoring';
$restore->save();
$expectConflict(static function () use ($blocked, $actor): void {
CloudBackupService::delete($blocked, $actor);
});
$blocked->refresh();
$assert((string) $blocked->state === 'ready',
'An active restore did not preserve its source backup.');
$artifactBacked = $newBackup(15, 'artifact-backed');
$now = time();
$artifact = new CloudStorageArtifact();
$artifact->artifact_key = str_repeat('9', 32);
$artifact->kind = 'backup';
$artifact->maximum_size_bytes = 1024;
$artifact->size_bytes = 1024;
$artifact->sha256 = str_repeat('a', 64);
$artifact->durability_policy = 'mirror_async_2';
$artifact->desired_copies = 2;
$artifact->required_copies = 1;
$artifact->state = 'available';
$artifact->last_operation_id = null;
$artifact->verified_at = $now;
$artifact->created_at = $artifact->updated_at = $now;
$artifact->save();
$publication = new CloudStoragePublication();
$publication->publication_key = str_repeat('8', 32);
$publication->artifact_id = (int) $artifact->id;
$publication->backup_id = (int) $artifactBacked->id;
$publication->template_import_intent_id = null;
$publication->rescue_template_id = null;
$publication->state = 'completed';
$publication->attempts = 1;
$publication->available_at = 0;
$publication->lease_token = null;
$publication->lease_until = 0;
$publication->last_error_code = '';
$publication->coverage_ready_at = $now;
$publication->finalized_at = $now;
$publication->created_at = $publication->updated_at = $now;
$publication->save();
foreach ([1 => 'primary', 2 => 'secondary'] as $backendId => $role) {
$replica = new CloudStorageReplica();
$replica->replica_key = str_repeat((string) $backendId, 32);
$replica->artifact_id = (int) $artifact->id;
$replica->backend_id = $backendId;
$replica->role = $role;
$replica->object_key = 'dual/artifact-backed/system.qcow2';
$replica->object_version_id = 'version-' . $backendId;
$replica->object_etag = '"etag-' . $backendId . '"';
$replica->size_bytes = 1024;
$replica->sha256 = str_repeat('a', 64);
$replica->state = 'ready';
$replica->publish_operation_id = null;
$replica->delete_operation_id = null;
$replica->delete_generation = 0;
$replica->delete_cycle_attempts = 0;
$replica->delete_presence_confirmations = 0;
$replica->delete_last_checked_at = 0;
$replica->delete_next_check_at = 0;
$replica->delete_attention_at = null;
$replica->lease_token = null;
$replica->lease_until = 0;
$replica->error_code = '';
$replica->verified_at = $now;
$replica->created_at = $replica->updated_at = $now;
$replica->save();
}
$artifactBacked->storage_artifact_id = (int) $artifact->id;
$artifactBacked->save();
$artifactHistory = [];
$artifactClient = $client([
new Response(200, [
'Content-Length' => '1024', 'ETag' => '"etag-1"',
'x-amz-version-id' => 'version-1',
]),
new Response(204),
new Response(200, [
'Content-Length' => '1024', 'ETag' => '"etag-2"',
'x-amz-version-id' => 'version-2',
]),
new Response(204),
], $artifactHistory);
$artifactDeletion = new CloudStorageArtifactDeletionService(
$artifactClient,
static fn (int $backendId, bool $healthy): array => $runtime
);
$artifactUpdatedBeforeWrongOwner = (int) $artifact->updated_at;
$publicationUpdatedBeforeWrongOwner = (int) $publication->updated_at;
$expectConflict(static fn () => $artifactDeletion->process(
(int) $artifact->id,
['backup_id' => (int) $artifactBacked->id + 1000]
));
$expectConflict(static fn () => $artifactDeletion->reopenAttention(
(int) $artifact->id,
['backup_id' => (int) $artifactBacked->id + 1000]
));
$artifact->refresh();
$publication->refresh();
$assert((string) $artifact->state === 'available'
&& (int) $artifact->updated_at === $artifactUpdatedBeforeWrongOwner
&& (string) $publication->state === 'completed'
&& (int) $publication->updated_at
=== $publicationUpdatedBeforeWrongOwner
&& CloudStorageReplica::where('artifact_id', $artifact->id)
->where('state', 'ready')->count() === 2
&& CloudStorageReplica::where('artifact_id', $artifact->id)
->whereNotNull('delete_operation_id')->count() === 0
&& count($artifactHistory) === 0
&& \App\Models\CloudStorageOperation::where(
'artifact_id', $artifact->id
)->count() === 0,
'Wrong backup owner changed artifact state or issued remote I/O.');
$artifactResult = CloudBackupService::delete(
$artifactBacked,
$actor,
$artifactClient,
null,
$artifactDeletion
);
$artifactBacked->refresh();
$artifact->refresh();
$assert(($artifactResult['deleted'] ?? false) === true
&& (string) $artifactBacked->state === 'deleted'
&& (string) $artifact->state === 'deleted'
&& CloudStorageReplica::where('artifact_id', $artifact->id)
->where('state', 'deleted')->count() === 2
&& count($artifactHistory) === 4
&& $artifactHistory[0]['request']->getMethod() === 'HEAD'
&& $artifactHistory[1]['request']->getMethod() === 'DELETE'
&& $artifactHistory[2]['request']->getMethod() === 'HEAD'
&& $artifactHistory[3]['request']->getMethod() === 'DELETE',
'Artifact-backed deletion did not clear every exact replica before tombstoning the backup.');
// A lost DELETE response is reconciled with two spaced HEAD probes. The
// original generation is never replayed; only a newly persisted
// generation may issue the second DELETE.
$unknownOwner = $newBackup(16, 'artifact-unknown');
$unknownArtifact = new CloudStorageArtifact();
$unknownArtifact->artifact_key = str_repeat('7', 32);
$unknownArtifact->kind = 'backup';
$unknownArtifact->maximum_size_bytes = 1024;
$unknownArtifact->size_bytes = 1024;
$unknownArtifact->sha256 = str_repeat('a', 64);
$unknownArtifact->durability_policy = 'mirror_async_2';
$unknownArtifact->desired_copies = 1;
$unknownArtifact->required_copies = 1;
$unknownArtifact->state = 'available';
$unknownArtifact->last_operation_id = null;
$unknownArtifact->verified_at = $now;
$unknownArtifact->created_at = $unknownArtifact->updated_at = $now;
$unknownArtifact->save();
$unknownPublication = new CloudStoragePublication();
$unknownPublication->publication_key = str_repeat('6', 32);
$unknownPublication->artifact_id = (int) $unknownArtifact->id;
$unknownPublication->backup_id = (int) $unknownOwner->id;
$unknownPublication->template_import_intent_id = null;
$unknownPublication->rescue_template_id = null;
$unknownPublication->state = 'completed';
$unknownPublication->attempts = 1;
$unknownPublication->available_at = 0;
$unknownPublication->lease_token = null;
$unknownPublication->lease_until = 0;
$unknownPublication->last_error_code = '';
$unknownPublication->coverage_ready_at = $now;
$unknownPublication->finalized_at = $now;
$unknownPublication->created_at =
$unknownPublication->updated_at = $now;
$unknownPublication->save();
$unknownReplica = new CloudStorageReplica();
$unknownReplica->replica_key = str_repeat('5', 32);
$unknownReplica->artifact_id = (int) $unknownArtifact->id;
$unknownReplica->backend_id = 1;
$unknownReplica->role = 'primary';
$unknownReplica->object_key = 'dual/unknown/system.qcow2';
$unknownReplica->object_version_id = 'unknown-version';
$unknownReplica->object_etag = '"unknown-etag"';
$unknownReplica->size_bytes = 1024;
$unknownReplica->sha256 = str_repeat('a', 64);
$unknownReplica->state = 'ready';
$unknownReplica->publish_operation_id = null;
$unknownReplica->delete_operation_id = null;
$unknownReplica->delete_generation = 0;
$unknownReplica->delete_cycle_attempts = 0;
$unknownReplica->delete_presence_confirmations = 0;
$unknownReplica->delete_last_checked_at = 0;
$unknownReplica->delete_next_check_at = 0;
$unknownReplica->delete_attention_at = null;
$unknownReplica->lease_token = null;
$unknownReplica->lease_until = 0;
$unknownReplica->error_code = '';
$unknownReplica->verified_at = $now;
$unknownReplica->created_at = $unknownReplica->updated_at = $now;
$unknownReplica->save();
$presentUnknown = static fn (): Response => new Response(200, [
'Content-Length' => '1024', 'ETag' => '"unknown-etag"',
'x-amz-version-id' => 'unknown-version',
]);
$unknownHistory = [];
$unknownClient = $client([
$presentUnknown(),
new ConnectException(
'delete response lost',
new Request('DELETE', 'https://objects.example.com')
),
$presentUnknown(),
$presentUnknown(),
$presentUnknown(),
new Response(204),
], $unknownHistory);
$clock = 1000;
$unknownDeletion = new CloudStorageArtifactDeletionService(
$unknownClient,
static fn (int $backendId, bool $healthy): array => $runtime,
static function () use (&$clock): int {
return $clock;
}
);
$unknownExpectedOwner = ['backup_id' => (int) $unknownOwner->id];
$firstUnknown = $unknownDeletion->process(
(int) $unknownArtifact->id,
$unknownExpectedOwner
);
$clock = 1061;
$firstProbe = $unknownDeletion->process(
(int) $unknownArtifact->id,
$unknownExpectedOwner
);
$clock = 1122;
$recoveredDelete = $unknownDeletion->process(
(int) $unknownArtifact->id,
$unknownExpectedOwner
);
$unknownReplica->refresh();
$unknownDeleteMethods = array_values(array_filter(array_map(
static fn ($entry): string => $entry['request']->getMethod(),
$unknownHistory
), static fn (string $method): bool => $method === 'DELETE'));
$unknownOperations = \App\Models\CloudStorageOperation::where(
'artifact_id',
(int) $unknownArtifact->id
)->where('kind', 'delete')->orderBy('id')->get();
$assert(($firstUnknown['requires_reconciliation'] ?? false) === true
&& ($firstProbe['requires_reconciliation'] ?? false) === true
&& ($recoveredDelete['deleted'] ?? false) === true
&& count($unknownDeleteMethods) === 2
&& $unknownOperations->count() === 2
&& (string) $unknownOperations[0]->phase === 'terminal'
&& (string) $unknownOperations[0]->outcome === 'failed'
&& (string) $unknownOperations[0]->effect_resolution === 'unknown'
&& (string) $unknownOperations[1]->outcome === 'succeeded'
&& (int) $unknownReplica->delete_generation === 2,
'Ambiguous replica deletion replayed an old generation or failed to converge from stable evidence.');
// A pre-Agent intent whose provider runtime never became available can
// be cancelled without inventing an Agent job. Both never-dispatched
// source operations are fenced as definite no-effect before exact-key
// absence checks retire the replicas.
$newPendingArtifact = static function (
CloudInstanceBackup $owner,
string $seed
) use ($now): array {
$artifact = new CloudStorageArtifact();
$artifact->artifact_key = str_repeat($seed, 32);
$artifact->kind = 'backup';
$artifact->maximum_size_bytes = 4096;
$artifact->size_bytes = 0;
$artifact->sha256 = '';
$artifact->durability_policy = 'mirror_async_2';
$artifact->desired_copies = 2;
$artifact->required_copies = 1;
$artifact->state = 'publishing';
$artifact->last_operation_id = null;
$artifact->verified_at = 0;
$artifact->created_at = $artifact->updated_at = $now;
$artifact->save();
$publication = new CloudStoragePublication();
$publication->publication_key = str_repeat($seed, 32);
$publication->artifact_id = (int) $artifact->id;
$publication->backup_id = (int) $owner->id;
$publication->template_import_intent_id = null;
$publication->rescue_template_id = null;
$publication->state = 'waiting_primary';
$publication->attempts = 0;
$publication->available_at = 0;
$publication->lease_token = null;
$publication->lease_until = 0;
$publication->last_error_code = '';
$publication->coverage_ready_at = null;
$publication->finalized_at = null;
$publication->created_at = $publication->updated_at = $now;
$publication->save();
$operations = [];
$replicas = [];
foreach ([1 => 'primary', 2 => 'secondary'] as $backendId => $role) {
$replica = new CloudStorageReplica();
$replica->replica_key = str_repeat(
$seed . (string) $backendId,
16
);
$replica->artifact_id = (int) $artifact->id;
$replica->backend_id = $backendId;
$replica->role = $role;
$replica->object_key = 'dual/pending-' . $seed . '-'
. $backendId . '/system.qcow2';
$replica->object_version_id = '';
$replica->object_etag = '';
$replica->size_bytes = 0;
$replica->sha256 = '';
$replica->state = 'reserved';
$replica->publish_operation_id = null;
$replica->delete_operation_id = null;
$replica->delete_generation = 0;
$replica->delete_cycle_attempts = 0;
$replica->delete_presence_confirmations = 0;
$replica->delete_last_checked_at = 0;
$replica->delete_next_check_at = 0;
$replica->delete_attention_at = null;
$replica->lease_token = null;
$replica->lease_until = 0;
$replica->error_code = '';
$replica->verified_at = 0;
$replica->created_at = $replica->updated_at = $now;
$replica->save();
$operation = new CloudStorageOperation();
$operation->operation_id = str_repeat(
$seed . (string) ($backendId + 2),
16
);
$operation->request_idempotency_key = 'pending-' . $seed
. '-' . $backendId;
$operation->request_fingerprint = hash(
'sha256',
'pending|' . $seed . '|' . $backendId
);
$operation->artifact_id = (int) $artifact->id;
$operation->replica_id = (int) $replica->id;
$operation->source_backend_id = $backendId === 1 ? null : 1;
$operation->target_backend_id = $backendId;
$operation->kind = $backendId === 1 ? 'publish' : 'copy';
$operation->phase = 'accepted';
$operation->outcome = null;
$operation->effect_resolution = 'not_applicable';
$operation->effect_may_have_applied = null;
$operation->safe_to_resubmit = null;
$operation->attempts = 0;
$operation->lease_token = null;
$operation->lease_until = 0;
$operation->available_at = $backendId === 1
? $now : 4294967295;
$operation->next_check_at = 0;
$operation->error_code = '';
$operation->result_json = null;
$operation->terminal_at = null;
$operation->created_at = $operation->updated_at = $now;
$operation->save();
$replica->publish_operation_id =
(string) $operation->operation_id;
$replica->save();
$operations[] = $operation;
$replicas[] = $replica;
}
$artifact->last_operation_id = (string) $operations[0]->operation_id;
$artifact->save();
$owner->storage_artifact_id = (int) $artifact->id;
$owner->sealed_storage_runtime = '';
$owner->storage_revision = 0;
$owner->object_size_bytes = 0;
$owner->sha256 = '';
$owner->ready_at = null;
$owner->save();
return [$artifact, $publication, $operations, $replicas];
};
$pendingOwner = $newBackup(17, 'pending-cancel', 'creating');
[$pendingArtifact, $pendingPublication, $pendingOperations,
$pendingReplicas] = $newPendingArtifact($pendingOwner, 'a');
$pendingHistory = [];
$pendingClient = $client([
new Response(404),
new Response(404),
], $pendingHistory);
$pendingDeletion = new CloudStorageArtifactDeletionService(
$pendingClient,
static fn (int $backendId, bool $healthy): array => $runtime
);
$pendingOwner->refresh();
try {
(new ReflectionClass(CloudBackupService::class))
->getMethod('assertBackupDeletionAvailable')
->invoke(null, $pendingOwner);
} catch (CloudError $error) {
throw new RuntimeException(
'Cancelable pending backup was blocked: ' . $error->getMessage()
);
}
$pendingDisplay = (new ReflectionClass(CloudBackupService::class))
->getMethod('displayBackup')->invoke(
null,
$pendingOwner,
null,
true
);
$assert(($pendingDisplay['can_delete'] ?? false) === true,
'Never-dispatched artifact backup was not exposed as cancelable.');
$pendingResult = CloudBackupService::delete(
$pendingOwner,
$actor,
$pendingClient,
null,
$pendingDeletion
);
$pendingOwner->refresh();
$pendingArtifact->refresh();
foreach ($pendingOperations as $operation) {
$operation->refresh();
}
$assert(($pendingResult['deleted'] ?? false) === true
&& (string) $pendingOwner->state === 'deleted'
&& (string) $pendingArtifact->state === 'deleted'
&& count($pendingHistory) === 2
&& $pendingHistory[0]['request']->getMethod() === 'HEAD'
&& $pendingHistory[1]['request']->getMethod() === 'HEAD'
&& CloudStorageOperation::where(
'artifact_id', (int) $pendingArtifact->id
)->whereIn('kind', ['publish', 'copy'])
->where('phase', 'terminal')
->where('outcome', 'failed')
->where('effect_resolution', 'not_applied')->count() === 2
&& CloudStorageReplica::where(
'artifact_id', (int) $pendingArtifact->id
)->where('state', 'deleted')->count() === 2,
'Never-dispatched backup intent was not cancelled and tombstoned safely.');
// Once provider dispatch may have occurred, cancellation must leave the
// original operation, artifact and backup untouched. No HEAD or DELETE is
// authorized until that same operation completes its normal reconcile.
$inFlightOwner = $newBackup(18, 'pending-in-flight', 'creating');
[$inFlightArtifact, $inFlightPublication, $inFlightOperations,
$inFlightReplicas] = $newPendingArtifact($inFlightOwner, 'b');
$inFlight = $inFlightOperations[0];
$inFlight->phase = 'dispatching';
$inFlight->effect_resolution = 'unknown';
$inFlight->lease_token = str_repeat('c', 32);
$inFlight->lease_until = $now + 300;
$inFlight->save();
$inFlightReplicas[0]->state = 'publishing';
$inFlightReplicas[0]->lease_token = str_repeat('c', 32);
$inFlightReplicas[0]->lease_until = $now + 300;
$inFlightReplicas[0]->save();
$inFlightHistory = [];
$inFlightClient = $client([], $inFlightHistory);
$inFlightDeletion = new CloudStorageArtifactDeletionService(
$inFlightClient,
static fn (int $backendId, bool $healthy): array => $runtime
);
$expectConflict(static function () use (
$inFlightOwner,
$actor,
$inFlightClient,
$inFlightDeletion
): void {
CloudBackupService::delete(
$inFlightOwner,
$actor,
$inFlightClient,
null,
$inFlightDeletion
);
});
$inFlightOwner->refresh();
$inFlightArtifact->refresh();
$inFlight->refresh();
$assert((string) $inFlightOwner->state === 'creating'
&& (string) $inFlightArtifact->state === 'publishing'
&& (string) $inFlight->phase === 'dispatching'
&& (string) $inFlight->lease_token === str_repeat('c', 32)
&& CloudStorageOperation::where(
'artifact_id', (int) $inFlightArtifact->id
)->where('kind', 'delete')->count() === 0
&& $inFlightHistory === [],
'In-flight publication cancellation mutated or replayed an unknown-effect operation.');
$readOnlyBackup = $newBackup(14, 'read-only');
$displayMethod = (new ReflectionClass(CloudBackupService::class))
->getMethod('displayBackup');
$readOnlyDisplay = $displayMethod->invoke(
null, $readOnlyBackup, null, false
);
$assert($readOnlyDisplay['can_delete'] === false,
'Read-only administrators were offered a backup delete action.');
$quotaBackups = [];
for ($index = 0; $index < CloudBackupService::MAX_PER_INSTANCE; $index++) {
$quotaBackups[] = $newBackup(99, 'quota-' . $index);
}
$reflection = new ReflectionClass(CloudBackupService::class);
$activeStates = $reflection->getReflectionConstant(
'ACTIVE_BACKUP_STATES'
)->getValue();
$before = CloudInstanceBackup::where('source_instance_id', 99)
->whereIn('state', $activeStates)->count();
$quotaHistory = [];
CloudBackupService::delete(
$quotaBackups[0], $actor,
$client([$identity(), new Response(204)], $quotaHistory)
);
$after = CloudInstanceBackup::where('source_instance_id', 99)
->whereIn('state', $activeStates)->count();
$assert($before === CloudBackupService::MAX_PER_INSTANCE
&& $after === CloudBackupService::MAX_PER_INSTANCE - 1,
'Confirmed deletion did not release one retained-backup quota slot.');
// One ambiguous generation may be retried only after two stable, spaced
// identity probes. The retry is a new deletion generation bound to the
// exact S3 version instead of a replay of the ambiguous request.
$retry = $newBackup(100, 'stable-retry', 'delete_uncertain');
$retry->object_etag = '"stable-etag"';
$retry->object_version_id = 'version/7+opaque';
$retry->delete_operation_id = str_repeat('1', 32);
$retry->delete_generation = 1;
$retry->delete_cycle_attempts = 1;
$retry->delete_presence_confirmations = 1;
$retry->delete_requested_at = time() - 300;
$retry->delete_last_checked_at = time() - 120;
$retry->delete_next_check_at = 0;
$retry->save();
$retryHistory = [];
$retryResult = CloudBackupService::delete($retry, $actor, $client([
new Response(200, [
'Content-Length' => '1024',
'ETag' => '"stable-etag"',
'x-amz-version-id' => 'version/7+opaque',
]),
new ConnectException('retry response lost',
new Request('DELETE', 'https://objects.example.com')),
], $retryHistory));
$retry->refresh();
$retryUris = array_map(static fn ($entry): string =>
(string) $entry['request']->getUri(), $retryHistory);
$assert($retryResult['requires_reconciliation'] === true
&& (string) $retry->state === 'delete_uncertain'
&& (int) $retry->delete_generation === 2
&& (int) $retry->delete_cycle_attempts === 2
&& count($retryHistory) === 2
&& $retryHistory[0]['request']->getMethod() === 'HEAD'
&& $retryHistory[1]['request']->getMethod() === 'DELETE'
&& str_contains($retryUris[0], 'versionId=version%2F7%2Bopaque')
&& str_contains($retryUris[1], 'versionId=version%2F7%2Bopaque'),
'Stable object identity did not create a fresh, exact-version deletion generation.');
$assert(!in_array('delete_uncertain', $activeStates, true),
'Object cleanup debt still consumes the user retained-backup quota.');
$retry->delete_cycle_attempts = 3;
$retry->delete_generation = 3;
$retry->delete_presence_confirmations = 1;
$retry->delete_last_checked_at = time() - 120;
$retry->delete_next_check_at = 0;
$retry->save();
$exhaustedHistory = [];
$exhausted = CloudBackupService::delete($retry, $actor, $client([
new Response(200, [
'Content-Length' => '1024',
'ETag' => '"stable-etag"',
'x-amz-version-id' => 'version/7+opaque',
]),
], $exhaustedHistory));
$retry->refresh();
$assert(($exhausted['requires_attention'] ?? false) === true
&& (string) $retry->state === 'delete_attention'
&& count($exhaustedHistory) === 1
&& $exhaustedHistory[0]['request']->getMethod() === 'HEAD',
'Three automatic deletion attempts did not stop at administrator attention.');
$debt = CloudBackupService::cleanupDebt($actor);
$debtItem = array_values(array_filter($debt['items'],
static fn (array $item): bool => $item['id'] === (string) $retry->id))[0] ?? null;
$assert(($debt['summary']['attention_count'] ?? 0) >= 1
&& is_array($debtItem)
&& ($debtItem['can_retry'] ?? false) === true
&& !array_key_exists('object_key', $debtItem)
&& !array_key_exists('object_version_id', $debtItem),
'Cleanup debt was not listed safely for administrators.');
$manualHistory = [];
$manual = CloudBackupService::retryCleanup($retry, $actor, $client([
new Response(200, [
'Content-Length' => '1024',
'ETag' => '"stable-etag"',
'x-amz-version-id' => 'version/7+opaque',
]),
], $manualHistory));
$retry->refresh();
$assert($manual['requires_reconciliation'] === true
&& (string) $retry->state === 'delete_uncertain'
&& (int) $retry->delete_cycle_attempts === 0
&& (int) $retry->delete_presence_confirmations === 1
&& (int) $retry->delete_next_check_at > time()
&& count($manualHistory) === 1,
'Manual cleanup retry did not begin a new bounded stable-evidence cycle.');
echo "Portable backup deletion, reconciliation, blocker and quota tests passed.\n";
}