You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
362 lines
17 KiB
PHP
362 lines
17 KiB
PHP
<?php
|
|
|
|
require dirname(__DIR__) . '/vendor/autoload.php';
|
|
|
|
use App\Models\CloudBackupRestore;
|
|
use App\Models\CloudInstanceBackup;
|
|
use App\Models\CloudStorageArtifact;
|
|
use App\Models\CloudStoragePublication;
|
|
use App\Models\CloudStorageReplica;
|
|
use App\Services\Cloud\CloudBackupService;
|
|
use App\Services\Cloud\CloudError;
|
|
use App\Services\Cloud\CloudObjectStorageS3Client;
|
|
use App\Services\Cloud\CloudStorageArtifactDeletionService;
|
|
use GuzzleHttp\Client;
|
|
use GuzzleHttp\Exception\ConnectException;
|
|
use GuzzleHttp\Handler\MockHandler;
|
|
use GuzzleHttp\HandlerStack;
|
|
use GuzzleHttp\Middleware;
|
|
use GuzzleHttp\Psr7\Response;
|
|
use GuzzleHttp\Psr7\Request;
|
|
use Illuminate\Database\Capsule\Manager as Capsule;
|
|
use Illuminate\Database\Schema\Blueprint;
|
|
|
|
$assert = static function (bool $condition, string $message): void {
|
|
if (!$condition) {
|
|
throw new RuntimeException($message);
|
|
}
|
|
};
|
|
$expectConflict = static function (callable $callback) use ($assert): void {
|
|
try {
|
|
$callback();
|
|
} catch (CloudError $error) {
|
|
$assert($error->httpStatus === 409,
|
|
'Deletion blocker returned an unexpected status.');
|
|
return;
|
|
}
|
|
throw new RuntimeException('Expected deletion blocker was not raised.');
|
|
};
|
|
|
|
$db = new Capsule();
|
|
$db->addConnection(['driver' => 'sqlite', 'database' => ':memory:']);
|
|
$db->setAsGlobal();
|
|
$db->bootEloquent();
|
|
$schema = $db->schema();
|
|
$schema->create('cloud_instance_backups', static function (Blueprint $t): void {
|
|
$t->increments('id'); $t->string('backup_key')->unique();
|
|
$t->integer('source_instance_id'); $t->integer('source_host_id');
|
|
$t->integer('user_id'); $t->string('source_uuid'); $t->string('name');
|
|
$t->string('purpose')->default('portable'); $t->string('source')->default('manual');
|
|
$t->integer('subscription_id')->nullable(); $t->string('state');
|
|
$t->string('consistency'); $t->string('object_key')->unique();
|
|
$t->integer('object_size_bytes'); $t->integer('virtual_size_bytes');
|
|
$t->string('sha256'); $t->string('object_etag')->default('');
|
|
$t->string('object_version_id')->default(''); $t->text('spec_json');
|
|
$t->text('sealed_credential')->nullable();
|
|
$t->integer('storage_revision'); $t->integer('storage_artifact_id')->nullable();
|
|
$t->integer('active_job_id')->nullable(); $t->string('last_operation_id')->nullable();
|
|
$t->string('delete_operation_id')->nullable();
|
|
$t->integer('delete_generation')->default(0);
|
|
$t->integer('delete_cycle_attempts')->default(0);
|
|
$t->integer('delete_presence_confirmations')->default(0);
|
|
$t->integer('delete_requested_at')->nullable();
|
|
$t->integer('delete_last_checked_at')->nullable();
|
|
$t->integer('delete_next_check_at')->default(0);
|
|
$t->integer('delete_attention_at')->nullable();
|
|
$t->string('error_code')->default(''); $t->integer('ready_at')->nullable();
|
|
$t->integer('created_at'); $t->integer('updated_at');
|
|
});
|
|
$schema->create('cloud_deployment_snapshots', static function (Blueprint $t): void {
|
|
$t->increments('id'); $t->integer('backup_id')->unique();
|
|
$t->string('state'); $t->string('error_code')->default('');
|
|
$t->integer('deleted_at')->nullable(); $t->integer('updated_at')->default(0);
|
|
});
|
|
$schema->create('cloud_backup_restores', static function (Blueprint $t): void {
|
|
$t->increments('id'); $t->integer('backup_id'); $t->string('state');
|
|
});
|
|
$schema->create('cloud_backup_restore_attempts', static function (Blueprint $t): void {
|
|
$t->increments('id'); $t->integer('backup_restore_id'); $t->string('state');
|
|
$t->integer('reservation_id')->nullable();
|
|
});
|
|
$schema->create('cloud_backup_recoveries', static function (Blueprint $t): void {
|
|
$t->increments('id'); $t->integer('backup_id'); $t->string('state');
|
|
});
|
|
$schema->create('cloud_jobs', static function (Blueprint $t): void {
|
|
$t->increments('id'); $t->integer('backup_id')->nullable();
|
|
$t->string('phase'); $t->string('effect_resolution');
|
|
$t->integer('finalization_incident')->default(0);
|
|
});
|
|
$schema->create('cloud_resource_reservations', static function (Blueprint $t): void {
|
|
$t->increments('id'); $t->string('state'); $t->integer('released_at')->nullable();
|
|
});
|
|
$schema->create('cloud_resource_reservation_ips', static function (Blueprint $t): void {
|
|
$t->increments('id'); $t->integer('reservation_id');
|
|
});
|
|
$schema->create('cloud_storage_artifacts', static function (Blueprint $t): void {
|
|
$t->increments('id'); $t->string('artifact_key')->unique(); $t->string('kind');
|
|
$t->integer('maximum_size_bytes'); $t->integer('size_bytes'); $t->string('sha256');
|
|
$t->string('durability_policy'); $t->integer('desired_copies');
|
|
$t->integer('required_copies'); $t->string('state');
|
|
$t->string('last_operation_id')->nullable(); $t->integer('verified_at');
|
|
$t->integer('created_at'); $t->integer('updated_at');
|
|
});
|
|
$schema->create('cloud_storage_replicas', static function (Blueprint $t): void {
|
|
$t->increments('id'); $t->string('replica_key')->unique();
|
|
$t->integer('artifact_id'); $t->integer('backend_id'); $t->string('role');
|
|
$t->string('object_key'); $t->string('object_version_id')->default('');
|
|
$t->string('object_etag')->default(''); $t->integer('size_bytes');
|
|
$t->string('sha256'); $t->string('state');
|
|
$t->string('publish_operation_id')->nullable();
|
|
$t->string('delete_operation_id')->nullable();
|
|
$t->integer('delete_generation')->default(0);
|
|
$t->integer('delete_cycle_attempts')->default(0);
|
|
$t->integer('delete_presence_confirmations')->default(0);
|
|
$t->integer('delete_last_checked_at')->default(0);
|
|
$t->integer('delete_next_check_at')->default(0);
|
|
$t->integer('delete_attention_at')->nullable();
|
|
$t->string('lease_token')->nullable(); $t->integer('lease_until')->default(0);
|
|
$t->string('error_code')->default(''); $t->integer('verified_at')->default(0);
|
|
$t->integer('created_at'); $t->integer('updated_at');
|
|
});
|
|
$schema->create('cloud_storage_operations', static function (Blueprint $t): void {
|
|
$t->increments('id'); $t->string('operation_id')->unique();
|
|
$t->string('request_idempotency_key')->unique(); $t->string('request_fingerprint');
|
|
$t->integer('artifact_id'); $t->integer('replica_id')->nullable();
|
|
$t->integer('source_backend_id')->nullable(); $t->integer('target_backend_id')->nullable();
|
|
$t->string('kind'); $t->string('phase'); $t->string('outcome')->nullable();
|
|
$t->string('effect_resolution'); $t->integer('effect_may_have_applied')->nullable();
|
|
$t->integer('safe_to_resubmit')->nullable(); $t->integer('attempts');
|
|
$t->string('lease_token')->nullable(); $t->integer('lease_until');
|
|
$t->integer('available_at'); $t->integer('next_check_at'); $t->string('error_code');
|
|
$t->text('result_json')->nullable(); $t->integer('terminal_at')->nullable();
|
|
$t->integer('created_at'); $t->integer('updated_at');
|
|
});
|
|
$schema->create('cloud_storage_publications', static function (Blueprint $t): void {
|
|
$t->increments('id'); $t->string('publication_key')->unique();
|
|
$t->integer('artifact_id')->unique(); $t->integer('backup_id')->nullable();
|
|
$t->string('template_import_intent_id')->nullable();
|
|
$t->string('rescue_template_id')->nullable(); $t->string('state');
|
|
$t->integer('attempts'); $t->integer('available_at');
|
|
$t->string('lease_token')->nullable(); $t->integer('lease_until');
|
|
$t->string('last_error_code'); $t->integer('coverage_ready_at')->nullable();
|
|
$t->integer('finalized_at')->nullable(); $t->integer('created_at');
|
|
$t->integer('updated_at');
|
|
});
|
|
|
|
$now = time();
|
|
$actor = ['user_id' => 1, 'is_admin' => true, 'is_super_admin' => true,
|
|
'can_manage' => true];
|
|
$newBackup = static function (int $sourceId, string $suffix) use ($now): CloudInstanceBackup {
|
|
$row = new CloudInstanceBackup();
|
|
$row->backup_key = str_pad($suffix, 32, '0');
|
|
$row->source_instance_id = $sourceId; $row->source_host_id = 3;
|
|
$row->user_id = 9; $row->source_uuid = '11111111-1111-4111-8111-111111111111';
|
|
$row->name = 'backup-' . $suffix; $row->purpose = 'portable';
|
|
$row->source = 'manual'; $row->subscription_id = null; $row->state = 'ready';
|
|
$row->consistency = 'crash';
|
|
$row->object_key = 'placeholder/' . $suffix; $row->object_size_bytes = 1024;
|
|
$row->virtual_size_bytes = 2048; $row->sha256 = str_repeat('a', 64);
|
|
$row->object_etag = ''; $row->object_version_id = ''; $row->spec_json = '{}';
|
|
$row->sealed_credential = 'guest-secret';
|
|
$row->storage_revision = 0; $row->storage_artifact_id = null;
|
|
$row->active_job_id = null; $row->last_operation_id = null;
|
|
$row->delete_operation_id = null; $row->delete_generation = 0;
|
|
$row->delete_cycle_attempts = 0; $row->delete_presence_confirmations = 0;
|
|
$row->delete_requested_at = null; $row->delete_last_checked_at = null;
|
|
$row->delete_next_check_at = 0; $row->delete_attention_at = null;
|
|
$row->error_code = ''; $row->ready_at = $now;
|
|
$row->created_at = $row->updated_at = $now; $row->save();
|
|
return $row;
|
|
};
|
|
$bindArtifact = static function (
|
|
CloudInstanceBackup $owner,
|
|
string $seed,
|
|
int $copies = 2
|
|
) use ($now): CloudStorageArtifact {
|
|
$artifact = new CloudStorageArtifact();
|
|
$artifact->artifact_key = str_repeat($seed, 32); $artifact->kind = 'backup';
|
|
$artifact->maximum_size_bytes = 1024; $artifact->size_bytes = 1024;
|
|
$artifact->sha256 = str_repeat('a', 64); $artifact->durability_policy = 'mirror_async_2';
|
|
$artifact->desired_copies = $copies; $artifact->required_copies = 1;
|
|
$artifact->state = 'protected'; $artifact->last_operation_id = null;
|
|
$artifact->verified_at = $now; $artifact->created_at = $artifact->updated_at = $now;
|
|
$artifact->save();
|
|
$publication = new CloudStoragePublication();
|
|
$publication->publication_key = str_repeat(chr(ord($seed) + 1), 32);
|
|
$publication->artifact_id = (int) $artifact->id; $publication->backup_id = (int) $owner->id;
|
|
$publication->template_import_intent_id = null; $publication->rescue_template_id = null;
|
|
$publication->state = 'completed'; $publication->attempts = 0;
|
|
$publication->available_at = 0; $publication->lease_token = null;
|
|
$publication->lease_until = 0; $publication->last_error_code = '';
|
|
$publication->coverage_ready_at = $now; $publication->finalized_at = $now;
|
|
$publication->created_at = $publication->updated_at = $now; $publication->save();
|
|
$roles = [1 => 'primary', 2 => 'secondary'];
|
|
if ($copies === 1) {
|
|
unset($roles[2]);
|
|
}
|
|
foreach ($roles as $backendId => $role) {
|
|
$replica = new CloudStorageReplica();
|
|
$replica->replica_key = str_repeat($seed . $backendId, 16);
|
|
$replica->artifact_id = (int) $artifact->id; $replica->backend_id = $backendId;
|
|
$replica->role = $role; $replica->object_key = 'dual/' . $seed . '/' . $backendId;
|
|
$replica->object_version_id = 'version-' . $backendId;
|
|
$replica->object_etag = '"etag-' . $backendId . '"';
|
|
$replica->size_bytes = 1024; $replica->sha256 = str_repeat('a', 64);
|
|
$replica->state = 'ready'; $replica->publish_operation_id = null;
|
|
$replica->delete_operation_id = null; $replica->delete_generation = 0;
|
|
$replica->delete_cycle_attempts = 0; $replica->delete_presence_confirmations = 0;
|
|
$replica->delete_last_checked_at = 0; $replica->delete_next_check_at = 0;
|
|
$replica->delete_attention_at = null; $replica->lease_token = null;
|
|
$replica->lease_until = 0; $replica->error_code = ''; $replica->verified_at = $now;
|
|
$replica->created_at = $replica->updated_at = $now; $replica->save();
|
|
}
|
|
$owner->storage_artifact_id = (int) $artifact->id; $owner->save();
|
|
return $artifact;
|
|
};
|
|
$client = static function (array $responses, array &$history): CloudObjectStorageS3Client {
|
|
$stack = HandlerStack::create(new MockHandler($responses));
|
|
$stack->push(Middleware::history($history));
|
|
return new CloudObjectStorageS3Client(new Client(['handler' => $stack]));
|
|
};
|
|
$runtime = ['config' => ['endpoint' => 'https://objects.example.test',
|
|
'region' => 'us-east-1', 'bucket' => 'bucket', 'prefix' => '', 'path_style' => true],
|
|
'credentials' => ['access_key_id' => 'KEY', 'secret_access_key' => 'secret'],
|
|
'revision' => 1, 'backend_id' => 1];
|
|
|
|
$legacy = $newBackup(10, 'legacy');
|
|
$legacyHistory = [];
|
|
$expectConflict(static fn () => CloudBackupService::delete(
|
|
$legacy, $actor, $client([], $legacyHistory)
|
|
));
|
|
$assert($legacyHistory === [] && (string) $legacy->fresh()->state === 'ready',
|
|
'A backup without a replica ledger attempted provider deletion.');
|
|
|
|
$backup = $newBackup(11, 'artifact');
|
|
$artifact = $bindArtifact($backup, 'a');
|
|
$history = [];
|
|
$artifactClient = $client([
|
|
new Response(200, ['Content-Length' => '1024', 'ETag' => '"etag-1"',
|
|
'x-amz-version-id' => 'version-1']),
|
|
new Response(204),
|
|
new Response(200, ['Content-Length' => '1024', 'ETag' => '"etag-2"',
|
|
'x-amz-version-id' => 'version-2']),
|
|
new Response(204),
|
|
], $history);
|
|
$deletion = new CloudStorageArtifactDeletionService(
|
|
$artifactClient,
|
|
static fn (int $backendId, bool $healthy): array => $runtime
|
|
);
|
|
$result = CloudBackupService::delete($backup, $actor, $artifactClient, null, $deletion);
|
|
$backup->refresh(); $artifact->refresh();
|
|
$assert(($result['deleted'] ?? false) === true
|
|
&& (string) $backup->state === 'deleted'
|
|
&& (string) $artifact->state === 'deleted'
|
|
&& $backup->sealed_credential === null
|
|
&& count($history) === 4
|
|
&& CloudStorageReplica::where('artifact_id', $artifact->id)
|
|
->where('state', 'deleted')->count() === 2,
|
|
'Artifact-backed backup did not delete every exact replica and tombstone its owner.');
|
|
|
|
$blocked = $newBackup(12, 'blocked');
|
|
$blockedArtifact = $bindArtifact($blocked, 'c');
|
|
$restore = new CloudBackupRestore(); $restore->backup_id = (int) $blocked->id;
|
|
$restore->state = 'restoring'; $restore->save();
|
|
$blockedHistory = [];
|
|
$expectConflict(static fn () => CloudBackupService::delete(
|
|
$blocked, $actor, $client([], $blockedHistory)
|
|
));
|
|
$blocked->refresh(); $blockedArtifact->refresh();
|
|
$assert($blockedHistory === [] && (string) $blocked->state === 'ready'
|
|
&& (string) $blockedArtifact->state === 'protected',
|
|
'An active restore did not preserve its replicated source backup.');
|
|
|
|
$unknown = $newBackup(13, 'unknown');
|
|
$unknownArtifact = $bindArtifact($unknown, 'e', 1);
|
|
$unknownHistory = [];
|
|
$unknownClient = $client([
|
|
new Response(200, ['Content-Length' => '1024', 'ETag' => '"etag-1"',
|
|
'x-amz-version-id' => 'version-1']),
|
|
new ConnectException(
|
|
'delete response lost',
|
|
new Request('DELETE', 'https://objects.example.test')
|
|
),
|
|
new Response(404),
|
|
], $unknownHistory);
|
|
$clock = 1_000;
|
|
$unknownDeletion = new CloudStorageArtifactDeletionService(
|
|
$unknownClient,
|
|
static fn (int $backendId, bool $healthy): array => $runtime,
|
|
static function () use (&$clock): int {
|
|
return $clock;
|
|
}
|
|
);
|
|
$unknownResult = CloudBackupService::delete(
|
|
$unknown,
|
|
$actor,
|
|
$unknownClient,
|
|
null,
|
|
$unknownDeletion
|
|
);
|
|
$unknownOperation = Capsule::table('cloud_storage_operations')
|
|
->where('artifact_id', (int) $unknownArtifact->id)
|
|
->where('kind', 'delete')->first();
|
|
$unknownReplica = CloudStorageReplica::where(
|
|
'artifact_id',
|
|
(int) $unknownArtifact->id
|
|
)->firstOrFail();
|
|
$assert(($unknownResult['requires_reconciliation'] ?? false) === true
|
|
&& $unknownOperation !== null
|
|
&& $unknownOperation->effect_resolution === 'unknown'
|
|
&& $unknownOperation->phase === 'reconciling'
|
|
&& $unknownOperation->outcome === null,
|
|
'Ambiguous replica deletion lost its original durable operation.');
|
|
|
|
// Model an administrator-attention projection after the same unknown
|
|
// operation exhausted its bounded automatic observations. Reopening must
|
|
// continue that operation with read-only evidence, not create another DELETE.
|
|
$unknownArtifact->state = 'delete_attention'; $unknownArtifact->save();
|
|
$operationId = (string) $unknownOperation->operation_id;
|
|
Capsule::table('cloud_storage_operations')
|
|
->where('operation_id', $operationId)->update([
|
|
'phase' => 'terminal',
|
|
'outcome' => 'failed',
|
|
'effect_resolution' => 'unknown',
|
|
'effect_may_have_applied' => 1,
|
|
'safe_to_resubmit' => 0,
|
|
'next_check_at' => 0,
|
|
'terminal_at' => $clock,
|
|
]);
|
|
$unknownReplica->state = 'delete_attention';
|
|
$unknownReplica->delete_attention_at = $clock;
|
|
$unknownReplica->delete_presence_confirmations = 0;
|
|
$unknownReplica->save();
|
|
$unknown->state = 'delete_attention'; $unknown->delete_attention_at = $clock;
|
|
$unknown->save();
|
|
$operationCount = Capsule::table('cloud_storage_operations')
|
|
->where('artifact_id', (int) $unknownArtifact->id)
|
|
->where('kind', 'delete')->count();
|
|
$clock = 1_061;
|
|
$recovered = CloudBackupService::retryCleanup(
|
|
$unknown,
|
|
$actor,
|
|
$unknownClient,
|
|
$unknownDeletion
|
|
);
|
|
$unknown->refresh(); $unknownArtifact->refresh(); $unknownReplica->refresh();
|
|
$assert(($recovered['deleted'] ?? false) === true
|
|
&& (string) $unknown->state === 'deleted'
|
|
&& (string) $unknownArtifact->state === 'deleted'
|
|
&& (string) $unknownReplica->state === 'deleted'
|
|
&& Capsule::table('cloud_storage_operations')
|
|
->where('artifact_id', (int) $unknownArtifact->id)
|
|
->where('kind', 'delete')->count() === $operationCount
|
|
&& Capsule::table('cloud_storage_operations')
|
|
->where('operation_id', $operationId)->exists()
|
|
&& array_map(static fn (array $entry): string =>
|
|
$entry['request']->getMethod(), $unknownHistory) === [
|
|
'HEAD', 'DELETE', 'HEAD',
|
|
],
|
|
'Manual recovery did not reconcile the original ambiguous deletion.');
|
|
|
|
echo "Portable backup replicated deletion and clean-break tests passed.\n";
|