You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
968 lines
41 KiB
PHP
968 lines
41 KiB
PHP
<?php
|
|
|
|
namespace App\Services {
|
|
final class Config
|
|
{
|
|
public static string $dataKey = '';
|
|
|
|
public static function get($name, $default = null)
|
|
{
|
|
return $name === 'cloud_data_key' ? self::$dataKey : $default;
|
|
}
|
|
}
|
|
}
|
|
|
|
namespace {
|
|
require dirname(__DIR__) . '/vendor/autoload.php';
|
|
|
|
use App\Models\CloudBackupRestore;
|
|
use App\Models\CloudInstanceBackup;
|
|
use App\Models\CloudStorageArtifact;
|
|
use App\Models\CloudStorageOperation;
|
|
use App\Models\CloudStoragePublication;
|
|
use App\Models\CloudStorageReplica;
|
|
use App\Services\Config;
|
|
use App\Services\Cloud\CloudBackupService;
|
|
use App\Services\Cloud\CloudError;
|
|
use App\Services\Cloud\CloudObjectStorageS3Client;
|
|
use App\Services\Cloud\CloudStorageArtifactDeletionService;
|
|
use App\Services\Cloud\CloudSecrets;
|
|
use GuzzleHttp\Client;
|
|
use GuzzleHttp\Exception\ConnectException;
|
|
use GuzzleHttp\Handler\MockHandler;
|
|
use GuzzleHttp\HandlerStack;
|
|
use GuzzleHttp\Middleware;
|
|
use GuzzleHttp\Psr7\Request;
|
|
use GuzzleHttp\Psr7\Response;
|
|
use Illuminate\Database\Capsule\Manager as Capsule;
|
|
use Illuminate\Database\Schema\Blueprint;
|
|
|
|
if (!function_exists('sodium_crypto_aead_xchacha20poly1305_ietf_encrypt')) {
|
|
echo "SKIPPED CloudPortableBackupDeletionTest: runtime lacks sodium.\n";
|
|
exit(0);
|
|
}
|
|
|
|
$assert = static function (bool $condition, string $message): void {
|
|
if (!$condition) {
|
|
throw new RuntimeException($message);
|
|
}
|
|
};
|
|
$expectConflict = static function (callable $callback) use ($assert): void {
|
|
try {
|
|
$callback();
|
|
} catch (CloudError $error) {
|
|
$assert($error->httpStatus === 409,
|
|
'Deletion blocker returned an unexpected status.');
|
|
return;
|
|
}
|
|
throw new RuntimeException('Expected deletion blocker was not raised.');
|
|
};
|
|
|
|
$capsule = new Capsule();
|
|
$capsule->addConnection([
|
|
'driver' => 'sqlite', 'database' => ':memory:',
|
|
], 'default');
|
|
$capsule->setAsGlobal();
|
|
$capsule->bootEloquent();
|
|
$schema = $capsule->schema('default');
|
|
$schema->create('cloud_instance_backups',
|
|
static function (Blueprint $table): void {
|
|
$table->increments('id');
|
|
$table->string('backup_key')->unique();
|
|
$table->integer('source_instance_id');
|
|
$table->integer('source_host_id');
|
|
$table->integer('user_id');
|
|
$table->string('source_uuid');
|
|
$table->string('name');
|
|
$table->string('purpose')->default('portable');
|
|
$table->string('state');
|
|
$table->string('consistency');
|
|
$table->string('object_key')->unique();
|
|
$table->integer('object_size_bytes');
|
|
$table->integer('virtual_size_bytes');
|
|
$table->string('sha256');
|
|
$table->string('object_etag')->default('');
|
|
$table->string('object_version_id')->default('');
|
|
$table->text('spec_json');
|
|
$table->text('sealed_storage_runtime');
|
|
$table->text('sealed_credential')->nullable();
|
|
$table->integer('storage_revision');
|
|
$table->integer('storage_artifact_id')->nullable();
|
|
$table->integer('active_job_id')->nullable();
|
|
$table->string('last_operation_id')->nullable();
|
|
$table->string('delete_operation_id')->nullable();
|
|
$table->integer('delete_generation')->default(0);
|
|
$table->integer('delete_cycle_attempts')->default(0);
|
|
$table->integer('delete_presence_confirmations')->default(0);
|
|
$table->integer('delete_requested_at')->nullable();
|
|
$table->integer('delete_last_checked_at')->nullable();
|
|
$table->integer('delete_next_check_at')->default(0);
|
|
$table->integer('delete_attention_at')->nullable();
|
|
$table->string('error_code');
|
|
$table->integer('ready_at')->nullable();
|
|
$table->integer('created_at');
|
|
$table->integer('updated_at');
|
|
});
|
|
$schema->create('cloud_deployment_snapshots',
|
|
static function (Blueprint $table): void {
|
|
$table->increments('id');
|
|
$table->integer('backup_id')->unique();
|
|
$table->string('state');
|
|
$table->string('error_code')->default('');
|
|
$table->integer('deleted_at')->nullable();
|
|
$table->integer('updated_at')->default(0);
|
|
});
|
|
$schema->create('cloud_backup_restores',
|
|
static function (Blueprint $table): void {
|
|
$table->increments('id');
|
|
$table->integer('backup_id');
|
|
$table->string('state');
|
|
});
|
|
$schema->create('cloud_backup_recoveries',
|
|
static function (Blueprint $table): void {
|
|
$table->increments('id');
|
|
$table->integer('backup_id');
|
|
$table->string('state');
|
|
});
|
|
$schema->create('cloud_jobs', static function (Blueprint $table): void {
|
|
$table->increments('id');
|
|
$table->integer('backup_id')->nullable();
|
|
$table->string('phase');
|
|
$table->string('effect_resolution');
|
|
$table->integer('finalization_incident')->default(0);
|
|
});
|
|
$schema->create('cloud_storage_artifacts',
|
|
static function (Blueprint $table): void {
|
|
$table->increments('id');
|
|
$table->string('artifact_key')->unique();
|
|
$table->string('kind');
|
|
$table->integer('maximum_size_bytes');
|
|
$table->integer('size_bytes');
|
|
$table->string('sha256');
|
|
$table->string('durability_policy');
|
|
$table->integer('desired_copies');
|
|
$table->integer('required_copies');
|
|
$table->string('state');
|
|
$table->string('last_operation_id')->nullable();
|
|
$table->integer('verified_at');
|
|
$table->integer('created_at');
|
|
$table->integer('updated_at');
|
|
});
|
|
$schema->create('cloud_storage_replicas',
|
|
static function (Blueprint $table): void {
|
|
$table->increments('id');
|
|
$table->string('replica_key')->unique();
|
|
$table->integer('artifact_id');
|
|
$table->integer('backend_id');
|
|
$table->string('role');
|
|
$table->string('object_key');
|
|
$table->string('object_version_id')->default('');
|
|
$table->string('object_etag')->default('');
|
|
$table->integer('size_bytes');
|
|
$table->string('sha256');
|
|
$table->string('state');
|
|
$table->string('publish_operation_id')->nullable();
|
|
$table->string('delete_operation_id')->nullable();
|
|
$table->integer('delete_generation')->default(0);
|
|
$table->integer('delete_cycle_attempts')->default(0);
|
|
$table->integer('delete_presence_confirmations')->default(0);
|
|
$table->integer('delete_last_checked_at')->default(0);
|
|
$table->integer('delete_next_check_at')->default(0);
|
|
$table->integer('delete_attention_at')->nullable();
|
|
$table->string('lease_token')->nullable();
|
|
$table->integer('lease_until')->default(0);
|
|
$table->string('error_code')->default('');
|
|
$table->integer('verified_at')->default(0);
|
|
$table->integer('created_at');
|
|
$table->integer('updated_at');
|
|
});
|
|
$schema->create('cloud_storage_operations',
|
|
static function (Blueprint $table): void {
|
|
$table->increments('id');
|
|
$table->string('operation_id')->unique();
|
|
$table->string('request_idempotency_key')->unique();
|
|
$table->string('request_fingerprint');
|
|
$table->integer('artifact_id');
|
|
$table->integer('replica_id')->nullable();
|
|
$table->integer('source_backend_id')->nullable();
|
|
$table->integer('target_backend_id')->nullable();
|
|
$table->string('kind');
|
|
$table->string('phase');
|
|
$table->string('outcome')->nullable();
|
|
$table->string('effect_resolution');
|
|
$table->integer('effect_may_have_applied')->nullable();
|
|
$table->integer('safe_to_resubmit')->nullable();
|
|
$table->integer('attempts');
|
|
$table->string('lease_token')->nullable();
|
|
$table->integer('lease_until');
|
|
$table->integer('available_at');
|
|
$table->integer('next_check_at');
|
|
$table->string('error_code');
|
|
$table->text('result_json')->nullable();
|
|
$table->integer('terminal_at')->nullable();
|
|
$table->integer('created_at');
|
|
$table->integer('updated_at');
|
|
});
|
|
$schema->create('cloud_storage_publications',
|
|
static function (Blueprint $table): void {
|
|
$table->increments('id');
|
|
$table->string('publication_key')->unique();
|
|
$table->integer('artifact_id')->unique();
|
|
$table->integer('backup_id')->nullable();
|
|
$table->string('template_import_intent_id')->nullable();
|
|
$table->string('rescue_template_id')->nullable();
|
|
$table->string('state');
|
|
$table->integer('attempts');
|
|
$table->integer('available_at');
|
|
$table->string('lease_token')->nullable();
|
|
$table->integer('lease_until');
|
|
$table->string('last_error_code');
|
|
$table->integer('coverage_ready_at')->nullable();
|
|
$table->integer('finalized_at')->nullable();
|
|
$table->integer('created_at');
|
|
$table->integer('updated_at');
|
|
});
|
|
|
|
Config::$dataKey = base64_encode(str_repeat('b', 32));
|
|
$runtime = [
|
|
'config' => [
|
|
'endpoint' => 'https://objects.example.com',
|
|
'region' => 'us-east-1',
|
|
'bucket' => 'backup-bucket',
|
|
'prefix' => 'virt',
|
|
'path_style' => true,
|
|
],
|
|
'credentials' => [
|
|
'access_key_id' => 'BACKUPACCESS',
|
|
'secret_access_key' => 'backup-secret-value',
|
|
],
|
|
'revision' => 7,
|
|
];
|
|
$sealedRuntime = CloudSecrets::encrypt(
|
|
$runtime,
|
|
'object-storage-backup-runtime'
|
|
);
|
|
$actor = [
|
|
'user_id' => 1,
|
|
'is_admin' => true,
|
|
'is_super_admin' => true,
|
|
'can_manage' => true,
|
|
];
|
|
$newBackup = static function (int $sourceInstanceId, string $suffix,
|
|
string $state = 'ready') use ($sealedRuntime): CloudInstanceBackup {
|
|
$backup = new CloudInstanceBackup();
|
|
$backup->backup_key = str_pad($suffix, 32, '0');
|
|
$backup->source_instance_id = $sourceInstanceId;
|
|
$backup->source_host_id = 3;
|
|
$backup->user_id = 9;
|
|
$backup->source_uuid = '11111111-1111-4111-8111-111111111111';
|
|
$backup->name = 'backup-' . $suffix;
|
|
$backup->state = $state;
|
|
$backup->consistency = 'crash';
|
|
$backup->object_key = 'virt/vm-backups/' . $suffix . '/system.qcow2';
|
|
$backup->object_size_bytes = 1024;
|
|
$backup->virtual_size_bytes = 2048;
|
|
$backup->sha256 = str_repeat('a', 64);
|
|
$backup->object_etag = '';
|
|
$backup->object_version_id = '';
|
|
$backup->spec_json = '{}';
|
|
$backup->sealed_storage_runtime = $sealedRuntime;
|
|
$backup->sealed_credential = 'sealed-guest-credential';
|
|
$backup->storage_revision = 7;
|
|
$backup->storage_artifact_id = null;
|
|
$backup->active_job_id = null;
|
|
$backup->last_operation_id = null;
|
|
$backup->delete_operation_id = null;
|
|
$backup->delete_generation = 0;
|
|
$backup->delete_cycle_attempts = 0;
|
|
$backup->delete_presence_confirmations = 0;
|
|
$backup->delete_requested_at = null;
|
|
$backup->delete_last_checked_at = null;
|
|
$backup->delete_next_check_at = 0;
|
|
$backup->delete_attention_at = null;
|
|
$backup->error_code = '';
|
|
$backup->ready_at = time();
|
|
$backup->created_at = $backup->updated_at = time();
|
|
$backup->save();
|
|
return $backup;
|
|
};
|
|
$client = static function (array $responses, array &$history): CloudObjectStorageS3Client {
|
|
$stack = HandlerStack::create(new MockHandler($responses));
|
|
$stack->push(Middleware::history($history));
|
|
return new CloudObjectStorageS3Client(new Client(['handler' => $stack]));
|
|
};
|
|
$identity = static fn (): Response => new Response(200, [
|
|
'Content-Length' => '1024',
|
|
'ETag' => '"catalog-etag"',
|
|
'x-amz-version-id' => 'catalog-version',
|
|
]);
|
|
|
|
$history = [];
|
|
$backup = $newBackup(10, 'one');
|
|
$deleted = CloudBackupService::delete(
|
|
$backup, $actor, $client([$identity(), new Response(204)], $history)
|
|
);
|
|
$backup->refresh();
|
|
$assert($deleted['deleted'] === true
|
|
&& (string) $backup->state === 'deleted'
|
|
&& (string) $backup->sealed_storage_runtime === ''
|
|
&& $backup->sealed_credential === null
|
|
&& count($history) === 2
|
|
&& $history[0]['request']->getMethod() === 'HEAD'
|
|
&& $history[1]['request']->getMethod() === 'DELETE'
|
|
&& str_contains((string) $history[1]['request']->getUri(),
|
|
'versionId=catalog-version'),
|
|
'Explicit DeleteObject success did not tombstone the catalog and remove frozen credentials.');
|
|
|
|
$unknown = $newBackup(11, 'two');
|
|
$unknownHistory = [];
|
|
$unknownResult = CloudBackupService::delete($unknown, $actor, $client([
|
|
$identity(),
|
|
new ConnectException('transport lost',
|
|
new Request('DELETE', 'https://objects.example.com')),
|
|
], $unknownHistory));
|
|
$unknown->refresh();
|
|
$assert($unknownResult['requires_reconciliation'] === true
|
|
&& (string) $unknown->state === 'delete_uncertain'
|
|
&& (string) $unknown->sealed_storage_runtime !== '',
|
|
'Ambiguous DeleteObject did not retain the frozen runtime and unknown-effect fence.');
|
|
$unknown->delete_next_check_at = 0;
|
|
$unknown->save();
|
|
$probeHistory = [];
|
|
$reconciled = CloudBackupService::delete(
|
|
$unknown, $actor, $client([new Response(404)], $probeHistory)
|
|
);
|
|
$assert($reconciled['deleted'] === true
|
|
&& count($probeHistory) === 1
|
|
&& $probeHistory[0]['request']->getMethod() === 'HEAD',
|
|
'A later explicit 404 did not converge deletion without replaying DELETE.');
|
|
|
|
$present = $newBackup(12, 'three');
|
|
$firstHistory = [];
|
|
CloudBackupService::delete($present, $actor, $client([
|
|
$identity(),
|
|
new ConnectException('transport lost',
|
|
new Request('DELETE', 'https://objects.example.com')),
|
|
], $firstHistory));
|
|
$present->refresh();
|
|
$present->delete_next_check_at = 0;
|
|
$present->save();
|
|
$presentHistory = [];
|
|
$stillPresent = CloudBackupService::delete(
|
|
$present, $actor, $client([$identity()], $presentHistory)
|
|
);
|
|
$present->refresh();
|
|
$assert($stillPresent['requires_reconciliation'] === true
|
|
&& (string) $present->state === 'delete_uncertain'
|
|
&& count($presentHistory) === 1
|
|
&& $presentHistory[0]['request']->getMethod() === 'HEAD',
|
|
'An existing object after ambiguous deletion was replayed or lost its fence.');
|
|
|
|
$blocked = $newBackup(13, 'four');
|
|
$restore = new CloudBackupRestore();
|
|
$restore->backup_id = (int) $blocked->id;
|
|
$restore->state = 'restoring';
|
|
$restore->save();
|
|
$expectConflict(static function () use ($blocked, $actor): void {
|
|
CloudBackupService::delete($blocked, $actor);
|
|
});
|
|
$blocked->refresh();
|
|
$assert((string) $blocked->state === 'ready',
|
|
'An active restore did not preserve its source backup.');
|
|
|
|
$artifactBacked = $newBackup(15, 'artifact-backed');
|
|
$now = time();
|
|
$artifact = new CloudStorageArtifact();
|
|
$artifact->artifact_key = str_repeat('9', 32);
|
|
$artifact->kind = 'backup';
|
|
$artifact->maximum_size_bytes = 1024;
|
|
$artifact->size_bytes = 1024;
|
|
$artifact->sha256 = str_repeat('a', 64);
|
|
$artifact->durability_policy = 'mirror_async_2';
|
|
$artifact->desired_copies = 2;
|
|
$artifact->required_copies = 1;
|
|
$artifact->state = 'available';
|
|
$artifact->last_operation_id = null;
|
|
$artifact->verified_at = $now;
|
|
$artifact->created_at = $artifact->updated_at = $now;
|
|
$artifact->save();
|
|
$publication = new CloudStoragePublication();
|
|
$publication->publication_key = str_repeat('8', 32);
|
|
$publication->artifact_id = (int) $artifact->id;
|
|
$publication->backup_id = (int) $artifactBacked->id;
|
|
$publication->template_import_intent_id = null;
|
|
$publication->rescue_template_id = null;
|
|
$publication->state = 'completed';
|
|
$publication->attempts = 1;
|
|
$publication->available_at = 0;
|
|
$publication->lease_token = null;
|
|
$publication->lease_until = 0;
|
|
$publication->last_error_code = '';
|
|
$publication->coverage_ready_at = $now;
|
|
$publication->finalized_at = $now;
|
|
$publication->created_at = $publication->updated_at = $now;
|
|
$publication->save();
|
|
foreach ([1 => 'primary', 2 => 'secondary'] as $backendId => $role) {
|
|
$replica = new CloudStorageReplica();
|
|
$replica->replica_key = str_repeat((string) $backendId, 32);
|
|
$replica->artifact_id = (int) $artifact->id;
|
|
$replica->backend_id = $backendId;
|
|
$replica->role = $role;
|
|
$replica->object_key = 'dual/artifact-backed/system.qcow2';
|
|
$replica->object_version_id = 'version-' . $backendId;
|
|
$replica->object_etag = '"etag-' . $backendId . '"';
|
|
$replica->size_bytes = 1024;
|
|
$replica->sha256 = str_repeat('a', 64);
|
|
$replica->state = 'ready';
|
|
$replica->publish_operation_id = null;
|
|
$replica->delete_operation_id = null;
|
|
$replica->delete_generation = 0;
|
|
$replica->delete_cycle_attempts = 0;
|
|
$replica->delete_presence_confirmations = 0;
|
|
$replica->delete_last_checked_at = 0;
|
|
$replica->delete_next_check_at = 0;
|
|
$replica->delete_attention_at = null;
|
|
$replica->lease_token = null;
|
|
$replica->lease_until = 0;
|
|
$replica->error_code = '';
|
|
$replica->verified_at = $now;
|
|
$replica->created_at = $replica->updated_at = $now;
|
|
$replica->save();
|
|
}
|
|
$artifactBacked->storage_artifact_id = (int) $artifact->id;
|
|
$artifactBacked->save();
|
|
$artifactHistory = [];
|
|
$artifactClient = $client([
|
|
new Response(200, [
|
|
'Content-Length' => '1024', 'ETag' => '"etag-1"',
|
|
'x-amz-version-id' => 'version-1',
|
|
]),
|
|
new Response(204),
|
|
new Response(200, [
|
|
'Content-Length' => '1024', 'ETag' => '"etag-2"',
|
|
'x-amz-version-id' => 'version-2',
|
|
]),
|
|
new Response(204),
|
|
], $artifactHistory);
|
|
$artifactDeletion = new CloudStorageArtifactDeletionService(
|
|
$artifactClient,
|
|
static fn (int $backendId, bool $healthy): array => $runtime
|
|
);
|
|
$artifactUpdatedBeforeWrongOwner = (int) $artifact->updated_at;
|
|
$publicationUpdatedBeforeWrongOwner = (int) $publication->updated_at;
|
|
$expectConflict(static fn () => $artifactDeletion->process(
|
|
(int) $artifact->id,
|
|
['backup_id' => (int) $artifactBacked->id + 1000]
|
|
));
|
|
$expectConflict(static fn () => $artifactDeletion->reopenAttention(
|
|
(int) $artifact->id,
|
|
['backup_id' => (int) $artifactBacked->id + 1000]
|
|
));
|
|
$artifact->refresh();
|
|
$publication->refresh();
|
|
$assert((string) $artifact->state === 'available'
|
|
&& (int) $artifact->updated_at === $artifactUpdatedBeforeWrongOwner
|
|
&& (string) $publication->state === 'completed'
|
|
&& (int) $publication->updated_at
|
|
=== $publicationUpdatedBeforeWrongOwner
|
|
&& CloudStorageReplica::where('artifact_id', $artifact->id)
|
|
->where('state', 'ready')->count() === 2
|
|
&& CloudStorageReplica::where('artifact_id', $artifact->id)
|
|
->whereNotNull('delete_operation_id')->count() === 0
|
|
&& count($artifactHistory) === 0
|
|
&& \App\Models\CloudStorageOperation::where(
|
|
'artifact_id', $artifact->id
|
|
)->count() === 0,
|
|
'Wrong backup owner changed artifact state or issued remote I/O.');
|
|
$artifactResult = CloudBackupService::delete(
|
|
$artifactBacked,
|
|
$actor,
|
|
$artifactClient,
|
|
null,
|
|
$artifactDeletion
|
|
);
|
|
$artifactBacked->refresh();
|
|
$artifact->refresh();
|
|
$assert(($artifactResult['deleted'] ?? false) === true
|
|
&& (string) $artifactBacked->state === 'deleted'
|
|
&& (string) $artifact->state === 'deleted'
|
|
&& CloudStorageReplica::where('artifact_id', $artifact->id)
|
|
->where('state', 'deleted')->count() === 2
|
|
&& count($artifactHistory) === 4
|
|
&& $artifactHistory[0]['request']->getMethod() === 'HEAD'
|
|
&& $artifactHistory[1]['request']->getMethod() === 'DELETE'
|
|
&& $artifactHistory[2]['request']->getMethod() === 'HEAD'
|
|
&& $artifactHistory[3]['request']->getMethod() === 'DELETE',
|
|
'Artifact-backed deletion did not clear every exact replica before tombstoning the backup.');
|
|
|
|
// A lost DELETE response is reconciled with two spaced HEAD probes. The
|
|
// original generation is never replayed; only a newly persisted
|
|
// generation may issue the second DELETE.
|
|
$unknownOwner = $newBackup(16, 'artifact-unknown');
|
|
$unknownArtifact = new CloudStorageArtifact();
|
|
$unknownArtifact->artifact_key = str_repeat('7', 32);
|
|
$unknownArtifact->kind = 'backup';
|
|
$unknownArtifact->maximum_size_bytes = 1024;
|
|
$unknownArtifact->size_bytes = 1024;
|
|
$unknownArtifact->sha256 = str_repeat('a', 64);
|
|
$unknownArtifact->durability_policy = 'mirror_async_2';
|
|
$unknownArtifact->desired_copies = 1;
|
|
$unknownArtifact->required_copies = 1;
|
|
$unknownArtifact->state = 'available';
|
|
$unknownArtifact->last_operation_id = null;
|
|
$unknownArtifact->verified_at = $now;
|
|
$unknownArtifact->created_at = $unknownArtifact->updated_at = $now;
|
|
$unknownArtifact->save();
|
|
$unknownPublication = new CloudStoragePublication();
|
|
$unknownPublication->publication_key = str_repeat('6', 32);
|
|
$unknownPublication->artifact_id = (int) $unknownArtifact->id;
|
|
$unknownPublication->backup_id = (int) $unknownOwner->id;
|
|
$unknownPublication->template_import_intent_id = null;
|
|
$unknownPublication->rescue_template_id = null;
|
|
$unknownPublication->state = 'completed';
|
|
$unknownPublication->attempts = 1;
|
|
$unknownPublication->available_at = 0;
|
|
$unknownPublication->lease_token = null;
|
|
$unknownPublication->lease_until = 0;
|
|
$unknownPublication->last_error_code = '';
|
|
$unknownPublication->coverage_ready_at = $now;
|
|
$unknownPublication->finalized_at = $now;
|
|
$unknownPublication->created_at =
|
|
$unknownPublication->updated_at = $now;
|
|
$unknownPublication->save();
|
|
$unknownReplica = new CloudStorageReplica();
|
|
$unknownReplica->replica_key = str_repeat('5', 32);
|
|
$unknownReplica->artifact_id = (int) $unknownArtifact->id;
|
|
$unknownReplica->backend_id = 1;
|
|
$unknownReplica->role = 'primary';
|
|
$unknownReplica->object_key = 'dual/unknown/system.qcow2';
|
|
$unknownReplica->object_version_id = 'unknown-version';
|
|
$unknownReplica->object_etag = '"unknown-etag"';
|
|
$unknownReplica->size_bytes = 1024;
|
|
$unknownReplica->sha256 = str_repeat('a', 64);
|
|
$unknownReplica->state = 'ready';
|
|
$unknownReplica->publish_operation_id = null;
|
|
$unknownReplica->delete_operation_id = null;
|
|
$unknownReplica->delete_generation = 0;
|
|
$unknownReplica->delete_cycle_attempts = 0;
|
|
$unknownReplica->delete_presence_confirmations = 0;
|
|
$unknownReplica->delete_last_checked_at = 0;
|
|
$unknownReplica->delete_next_check_at = 0;
|
|
$unknownReplica->delete_attention_at = null;
|
|
$unknownReplica->lease_token = null;
|
|
$unknownReplica->lease_until = 0;
|
|
$unknownReplica->error_code = '';
|
|
$unknownReplica->verified_at = $now;
|
|
$unknownReplica->created_at = $unknownReplica->updated_at = $now;
|
|
$unknownReplica->save();
|
|
$presentUnknown = static fn (): Response => new Response(200, [
|
|
'Content-Length' => '1024', 'ETag' => '"unknown-etag"',
|
|
'x-amz-version-id' => 'unknown-version',
|
|
]);
|
|
$unknownHistory = [];
|
|
$unknownClient = $client([
|
|
$presentUnknown(),
|
|
new ConnectException(
|
|
'delete response lost',
|
|
new Request('DELETE', 'https://objects.example.com')
|
|
),
|
|
$presentUnknown(),
|
|
$presentUnknown(),
|
|
$presentUnknown(),
|
|
new Response(204),
|
|
], $unknownHistory);
|
|
$clock = 1000;
|
|
$unknownDeletion = new CloudStorageArtifactDeletionService(
|
|
$unknownClient,
|
|
static fn (int $backendId, bool $healthy): array => $runtime,
|
|
static function () use (&$clock): int {
|
|
return $clock;
|
|
}
|
|
);
|
|
$unknownExpectedOwner = ['backup_id' => (int) $unknownOwner->id];
|
|
$firstUnknown = $unknownDeletion->process(
|
|
(int) $unknownArtifact->id,
|
|
$unknownExpectedOwner
|
|
);
|
|
$clock = 1061;
|
|
$firstProbe = $unknownDeletion->process(
|
|
(int) $unknownArtifact->id,
|
|
$unknownExpectedOwner
|
|
);
|
|
$clock = 1122;
|
|
$recoveredDelete = $unknownDeletion->process(
|
|
(int) $unknownArtifact->id,
|
|
$unknownExpectedOwner
|
|
);
|
|
$unknownReplica->refresh();
|
|
$unknownDeleteMethods = array_values(array_filter(array_map(
|
|
static fn ($entry): string => $entry['request']->getMethod(),
|
|
$unknownHistory
|
|
), static fn (string $method): bool => $method === 'DELETE'));
|
|
$unknownOperations = \App\Models\CloudStorageOperation::where(
|
|
'artifact_id',
|
|
(int) $unknownArtifact->id
|
|
)->where('kind', 'delete')->orderBy('id')->get();
|
|
$assert(($firstUnknown['requires_reconciliation'] ?? false) === true
|
|
&& ($firstProbe['requires_reconciliation'] ?? false) === true
|
|
&& ($recoveredDelete['deleted'] ?? false) === true
|
|
&& count($unknownDeleteMethods) === 2
|
|
&& $unknownOperations->count() === 2
|
|
&& (string) $unknownOperations[0]->phase === 'terminal'
|
|
&& (string) $unknownOperations[0]->outcome === 'failed'
|
|
&& (string) $unknownOperations[0]->effect_resolution === 'unknown'
|
|
&& (string) $unknownOperations[1]->outcome === 'succeeded'
|
|
&& (int) $unknownReplica->delete_generation === 2,
|
|
'Ambiguous replica deletion replayed an old generation or failed to converge from stable evidence.');
|
|
|
|
// A pre-Agent intent whose provider runtime never became available can
|
|
// be cancelled without inventing an Agent job. Both never-dispatched
|
|
// source operations are fenced as definite no-effect before exact-key
|
|
// absence checks retire the replicas.
|
|
$newPendingArtifact = static function (
|
|
CloudInstanceBackup $owner,
|
|
string $seed
|
|
) use ($now): array {
|
|
$artifact = new CloudStorageArtifact();
|
|
$artifact->artifact_key = str_repeat($seed, 32);
|
|
$artifact->kind = 'backup';
|
|
$artifact->maximum_size_bytes = 4096;
|
|
$artifact->size_bytes = 0;
|
|
$artifact->sha256 = '';
|
|
$artifact->durability_policy = 'mirror_async_2';
|
|
$artifact->desired_copies = 2;
|
|
$artifact->required_copies = 1;
|
|
$artifact->state = 'publishing';
|
|
$artifact->last_operation_id = null;
|
|
$artifact->verified_at = 0;
|
|
$artifact->created_at = $artifact->updated_at = $now;
|
|
$artifact->save();
|
|
$publication = new CloudStoragePublication();
|
|
$publication->publication_key = str_repeat($seed, 32);
|
|
$publication->artifact_id = (int) $artifact->id;
|
|
$publication->backup_id = (int) $owner->id;
|
|
$publication->template_import_intent_id = null;
|
|
$publication->rescue_template_id = null;
|
|
$publication->state = 'waiting_primary';
|
|
$publication->attempts = 0;
|
|
$publication->available_at = 0;
|
|
$publication->lease_token = null;
|
|
$publication->lease_until = 0;
|
|
$publication->last_error_code = '';
|
|
$publication->coverage_ready_at = null;
|
|
$publication->finalized_at = null;
|
|
$publication->created_at = $publication->updated_at = $now;
|
|
$publication->save();
|
|
$operations = [];
|
|
$replicas = [];
|
|
foreach ([1 => 'primary', 2 => 'secondary'] as $backendId => $role) {
|
|
$replica = new CloudStorageReplica();
|
|
$replica->replica_key = str_repeat(
|
|
$seed . (string) $backendId,
|
|
16
|
|
);
|
|
$replica->artifact_id = (int) $artifact->id;
|
|
$replica->backend_id = $backendId;
|
|
$replica->role = $role;
|
|
$replica->object_key = 'dual/pending-' . $seed . '-'
|
|
. $backendId . '/system.qcow2';
|
|
$replica->object_version_id = '';
|
|
$replica->object_etag = '';
|
|
$replica->size_bytes = 0;
|
|
$replica->sha256 = '';
|
|
$replica->state = 'reserved';
|
|
$replica->publish_operation_id = null;
|
|
$replica->delete_operation_id = null;
|
|
$replica->delete_generation = 0;
|
|
$replica->delete_cycle_attempts = 0;
|
|
$replica->delete_presence_confirmations = 0;
|
|
$replica->delete_last_checked_at = 0;
|
|
$replica->delete_next_check_at = 0;
|
|
$replica->delete_attention_at = null;
|
|
$replica->lease_token = null;
|
|
$replica->lease_until = 0;
|
|
$replica->error_code = '';
|
|
$replica->verified_at = 0;
|
|
$replica->created_at = $replica->updated_at = $now;
|
|
$replica->save();
|
|
$operation = new CloudStorageOperation();
|
|
$operation->operation_id = str_repeat(
|
|
$seed . (string) ($backendId + 2),
|
|
16
|
|
);
|
|
$operation->request_idempotency_key = 'pending-' . $seed
|
|
. '-' . $backendId;
|
|
$operation->request_fingerprint = hash(
|
|
'sha256',
|
|
'pending|' . $seed . '|' . $backendId
|
|
);
|
|
$operation->artifact_id = (int) $artifact->id;
|
|
$operation->replica_id = (int) $replica->id;
|
|
$operation->source_backend_id = $backendId === 1 ? null : 1;
|
|
$operation->target_backend_id = $backendId;
|
|
$operation->kind = $backendId === 1 ? 'publish' : 'copy';
|
|
$operation->phase = 'accepted';
|
|
$operation->outcome = null;
|
|
$operation->effect_resolution = 'not_applicable';
|
|
$operation->effect_may_have_applied = null;
|
|
$operation->safe_to_resubmit = null;
|
|
$operation->attempts = 0;
|
|
$operation->lease_token = null;
|
|
$operation->lease_until = 0;
|
|
$operation->available_at = $backendId === 1
|
|
? $now : 4294967295;
|
|
$operation->next_check_at = 0;
|
|
$operation->error_code = '';
|
|
$operation->result_json = null;
|
|
$operation->terminal_at = null;
|
|
$operation->created_at = $operation->updated_at = $now;
|
|
$operation->save();
|
|
$replica->publish_operation_id =
|
|
(string) $operation->operation_id;
|
|
$replica->save();
|
|
$operations[] = $operation;
|
|
$replicas[] = $replica;
|
|
}
|
|
$artifact->last_operation_id = (string) $operations[0]->operation_id;
|
|
$artifact->save();
|
|
$owner->storage_artifact_id = (int) $artifact->id;
|
|
$owner->sealed_storage_runtime = '';
|
|
$owner->storage_revision = 0;
|
|
$owner->object_size_bytes = 0;
|
|
$owner->sha256 = '';
|
|
$owner->ready_at = null;
|
|
$owner->save();
|
|
return [$artifact, $publication, $operations, $replicas];
|
|
};
|
|
|
|
$pendingOwner = $newBackup(17, 'pending-cancel', 'creating');
|
|
[$pendingArtifact, $pendingPublication, $pendingOperations,
|
|
$pendingReplicas] = $newPendingArtifact($pendingOwner, 'a');
|
|
$pendingHistory = [];
|
|
$pendingClient = $client([
|
|
new Response(404),
|
|
new Response(404),
|
|
], $pendingHistory);
|
|
$pendingDeletion = new CloudStorageArtifactDeletionService(
|
|
$pendingClient,
|
|
static fn (int $backendId, bool $healthy): array => $runtime
|
|
);
|
|
$pendingOwner->refresh();
|
|
try {
|
|
(new ReflectionClass(CloudBackupService::class))
|
|
->getMethod('assertBackupDeletionAvailable')
|
|
->invoke(null, $pendingOwner);
|
|
} catch (CloudError $error) {
|
|
throw new RuntimeException(
|
|
'Cancelable pending backup was blocked: ' . $error->getMessage()
|
|
);
|
|
}
|
|
$pendingDisplay = (new ReflectionClass(CloudBackupService::class))
|
|
->getMethod('displayBackup')->invoke(
|
|
null,
|
|
$pendingOwner,
|
|
null,
|
|
true
|
|
);
|
|
$assert(($pendingDisplay['can_delete'] ?? false) === true,
|
|
'Never-dispatched artifact backup was not exposed as cancelable.');
|
|
$pendingResult = CloudBackupService::delete(
|
|
$pendingOwner,
|
|
$actor,
|
|
$pendingClient,
|
|
null,
|
|
$pendingDeletion
|
|
);
|
|
$pendingOwner->refresh();
|
|
$pendingArtifact->refresh();
|
|
foreach ($pendingOperations as $operation) {
|
|
$operation->refresh();
|
|
}
|
|
$assert(($pendingResult['deleted'] ?? false) === true
|
|
&& (string) $pendingOwner->state === 'deleted'
|
|
&& (string) $pendingArtifact->state === 'deleted'
|
|
&& count($pendingHistory) === 2
|
|
&& $pendingHistory[0]['request']->getMethod() === 'HEAD'
|
|
&& $pendingHistory[1]['request']->getMethod() === 'HEAD'
|
|
&& CloudStorageOperation::where(
|
|
'artifact_id', (int) $pendingArtifact->id
|
|
)->whereIn('kind', ['publish', 'copy'])
|
|
->where('phase', 'terminal')
|
|
->where('outcome', 'failed')
|
|
->where('effect_resolution', 'not_applied')->count() === 2
|
|
&& CloudStorageReplica::where(
|
|
'artifact_id', (int) $pendingArtifact->id
|
|
)->where('state', 'deleted')->count() === 2,
|
|
'Never-dispatched backup intent was not cancelled and tombstoned safely.');
|
|
|
|
// Once provider dispatch may have occurred, cancellation must leave the
|
|
// original operation, artifact and backup untouched. No HEAD or DELETE is
|
|
// authorized until that same operation completes its normal reconcile.
|
|
$inFlightOwner = $newBackup(18, 'pending-in-flight', 'creating');
|
|
[$inFlightArtifact, $inFlightPublication, $inFlightOperations,
|
|
$inFlightReplicas] = $newPendingArtifact($inFlightOwner, 'b');
|
|
$inFlight = $inFlightOperations[0];
|
|
$inFlight->phase = 'dispatching';
|
|
$inFlight->effect_resolution = 'unknown';
|
|
$inFlight->lease_token = str_repeat('c', 32);
|
|
$inFlight->lease_until = $now + 300;
|
|
$inFlight->save();
|
|
$inFlightReplicas[0]->state = 'publishing';
|
|
$inFlightReplicas[0]->lease_token = str_repeat('c', 32);
|
|
$inFlightReplicas[0]->lease_until = $now + 300;
|
|
$inFlightReplicas[0]->save();
|
|
$inFlightHistory = [];
|
|
$inFlightClient = $client([], $inFlightHistory);
|
|
$inFlightDeletion = new CloudStorageArtifactDeletionService(
|
|
$inFlightClient,
|
|
static fn (int $backendId, bool $healthy): array => $runtime
|
|
);
|
|
$expectConflict(static function () use (
|
|
$inFlightOwner,
|
|
$actor,
|
|
$inFlightClient,
|
|
$inFlightDeletion
|
|
): void {
|
|
CloudBackupService::delete(
|
|
$inFlightOwner,
|
|
$actor,
|
|
$inFlightClient,
|
|
null,
|
|
$inFlightDeletion
|
|
);
|
|
});
|
|
$inFlightOwner->refresh();
|
|
$inFlightArtifact->refresh();
|
|
$inFlight->refresh();
|
|
$assert((string) $inFlightOwner->state === 'creating'
|
|
&& (string) $inFlightArtifact->state === 'publishing'
|
|
&& (string) $inFlight->phase === 'dispatching'
|
|
&& (string) $inFlight->lease_token === str_repeat('c', 32)
|
|
&& CloudStorageOperation::where(
|
|
'artifact_id', (int) $inFlightArtifact->id
|
|
)->where('kind', 'delete')->count() === 0
|
|
&& $inFlightHistory === [],
|
|
'In-flight publication cancellation mutated or replayed an unknown-effect operation.');
|
|
|
|
$readOnlyBackup = $newBackup(14, 'read-only');
|
|
$displayMethod = (new ReflectionClass(CloudBackupService::class))
|
|
->getMethod('displayBackup');
|
|
$readOnlyDisplay = $displayMethod->invoke(
|
|
null, $readOnlyBackup, null, false
|
|
);
|
|
$assert($readOnlyDisplay['can_delete'] === false,
|
|
'Read-only administrators were offered a backup delete action.');
|
|
|
|
$quotaBackups = [];
|
|
for ($index = 0; $index < CloudBackupService::MAX_PER_INSTANCE; $index++) {
|
|
$quotaBackups[] = $newBackup(99, 'quota-' . $index);
|
|
}
|
|
$reflection = new ReflectionClass(CloudBackupService::class);
|
|
$activeStates = $reflection->getReflectionConstant(
|
|
'ACTIVE_BACKUP_STATES'
|
|
)->getValue();
|
|
$before = CloudInstanceBackup::where('source_instance_id', 99)
|
|
->whereIn('state', $activeStates)->count();
|
|
$quotaHistory = [];
|
|
CloudBackupService::delete(
|
|
$quotaBackups[0], $actor,
|
|
$client([$identity(), new Response(204)], $quotaHistory)
|
|
);
|
|
$after = CloudInstanceBackup::where('source_instance_id', 99)
|
|
->whereIn('state', $activeStates)->count();
|
|
$assert($before === CloudBackupService::MAX_PER_INSTANCE
|
|
&& $after === CloudBackupService::MAX_PER_INSTANCE - 1,
|
|
'Confirmed deletion did not release one retained-backup quota slot.');
|
|
|
|
// One ambiguous generation may be retried only after two stable, spaced
|
|
// identity probes. The retry is a new deletion generation bound to the
|
|
// exact S3 version instead of a replay of the ambiguous request.
|
|
$retry = $newBackup(100, 'stable-retry', 'delete_uncertain');
|
|
$retry->object_etag = '"stable-etag"';
|
|
$retry->object_version_id = 'version/7+opaque';
|
|
$retry->delete_operation_id = str_repeat('1', 32);
|
|
$retry->delete_generation = 1;
|
|
$retry->delete_cycle_attempts = 1;
|
|
$retry->delete_presence_confirmations = 1;
|
|
$retry->delete_requested_at = time() - 300;
|
|
$retry->delete_last_checked_at = time() - 120;
|
|
$retry->delete_next_check_at = 0;
|
|
$retry->save();
|
|
$retryHistory = [];
|
|
$retryResult = CloudBackupService::delete($retry, $actor, $client([
|
|
new Response(200, [
|
|
'Content-Length' => '1024',
|
|
'ETag' => '"stable-etag"',
|
|
'x-amz-version-id' => 'version/7+opaque',
|
|
]),
|
|
new ConnectException('retry response lost',
|
|
new Request('DELETE', 'https://objects.example.com')),
|
|
], $retryHistory));
|
|
$retry->refresh();
|
|
$retryUris = array_map(static fn ($entry): string =>
|
|
(string) $entry['request']->getUri(), $retryHistory);
|
|
$assert($retryResult['requires_reconciliation'] === true
|
|
&& (string) $retry->state === 'delete_uncertain'
|
|
&& (int) $retry->delete_generation === 2
|
|
&& (int) $retry->delete_cycle_attempts === 2
|
|
&& count($retryHistory) === 2
|
|
&& $retryHistory[0]['request']->getMethod() === 'HEAD'
|
|
&& $retryHistory[1]['request']->getMethod() === 'DELETE'
|
|
&& str_contains($retryUris[0], 'versionId=version%2F7%2Bopaque')
|
|
&& str_contains($retryUris[1], 'versionId=version%2F7%2Bopaque'),
|
|
'Stable object identity did not create a fresh, exact-version deletion generation.');
|
|
|
|
$assert(!in_array('delete_uncertain', $activeStates, true),
|
|
'Object cleanup debt still consumes the user retained-backup quota.');
|
|
|
|
$retry->delete_cycle_attempts = 3;
|
|
$retry->delete_generation = 3;
|
|
$retry->delete_presence_confirmations = 1;
|
|
$retry->delete_last_checked_at = time() - 120;
|
|
$retry->delete_next_check_at = 0;
|
|
$retry->save();
|
|
$exhaustedHistory = [];
|
|
$exhausted = CloudBackupService::delete($retry, $actor, $client([
|
|
new Response(200, [
|
|
'Content-Length' => '1024',
|
|
'ETag' => '"stable-etag"',
|
|
'x-amz-version-id' => 'version/7+opaque',
|
|
]),
|
|
], $exhaustedHistory));
|
|
$retry->refresh();
|
|
$assert(($exhausted['requires_attention'] ?? false) === true
|
|
&& (string) $retry->state === 'delete_attention'
|
|
&& count($exhaustedHistory) === 1
|
|
&& $exhaustedHistory[0]['request']->getMethod() === 'HEAD',
|
|
'Three automatic deletion attempts did not stop at administrator attention.');
|
|
|
|
$debt = CloudBackupService::cleanupDebt($actor);
|
|
$debtItem = array_values(array_filter($debt['items'],
|
|
static fn (array $item): bool => $item['id'] === (string) $retry->id))[0] ?? null;
|
|
$assert(($debt['summary']['attention_count'] ?? 0) >= 1
|
|
&& is_array($debtItem)
|
|
&& ($debtItem['can_retry'] ?? false) === true
|
|
&& !array_key_exists('object_key', $debtItem)
|
|
&& !array_key_exists('object_version_id', $debtItem),
|
|
'Cleanup debt was not listed safely for administrators.');
|
|
|
|
$manualHistory = [];
|
|
$manual = CloudBackupService::retryCleanup($retry, $actor, $client([
|
|
new Response(200, [
|
|
'Content-Length' => '1024',
|
|
'ETag' => '"stable-etag"',
|
|
'x-amz-version-id' => 'version/7+opaque',
|
|
]),
|
|
], $manualHistory));
|
|
$retry->refresh();
|
|
$assert($manual['requires_reconciliation'] === true
|
|
&& (string) $retry->state === 'delete_uncertain'
|
|
&& (int) $retry->delete_cycle_attempts === 0
|
|
&& (int) $retry->delete_presence_confirmations === 1
|
|
&& (int) $retry->delete_next_check_at > time()
|
|
&& count($manualHistory) === 1,
|
|
'Manual cleanup retry did not begin a new bounded stable-evidence cycle.');
|
|
|
|
echo "Portable backup deletion, reconciliation, blocker and quota tests passed.\n";
|
|
}
|