You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
virt/web/tests/CloudMigrationBundleCleanup...

464 lines
19 KiB
PHP

<?php
namespace App\Services {
final class Config
{
public static string $dataKey = '';
public static function get($name, $default = null)
{
return $name === 'cloud_data_key' ? self::$dataKey : $default;
}
}
}
namespace {
require dirname(__DIR__) . '/vendor/autoload.php';
use App\Models\CloudInstanceBackup;
use App\Models\CloudInstanceMigration;
use App\Models\CloudMigrationBundleArtifact;
use App\Services\Config;
use App\Services\Cloud\CloudError;
use App\Services\Cloud\CloudMigrationBundleService;
use App\Services\Cloud\CloudObjectStorageS3Client;
use App\Services\Cloud\CloudObjectStorageTransferError;
use App\Services\Cloud\CloudSecrets;
use GuzzleHttp\Client;
use GuzzleHttp\Exception\ConnectException;
use GuzzleHttp\Handler\MockHandler;
use GuzzleHttp\HandlerStack;
use GuzzleHttp\Middleware;
use GuzzleHttp\Psr7\Request;
use GuzzleHttp\Psr7\Response;
use Illuminate\Database\Capsule\Manager as Capsule;
use Illuminate\Database\Schema\Blueprint;
if (!function_exists('sodium_crypto_aead_xchacha20poly1305_ietf_encrypt')) {
echo "SKIPPED CloudMigrationBundleCleanupTest: runtime lacks sodium.\n";
exit(0);
}
$assert = static function (bool $condition, string $message): void {
if (!$condition) {
throw new RuntimeException($message);
}
};
$expectCleanupFailure = static function (callable $callback,
string $message): void {
try {
$callback();
} catch (CloudError|CloudObjectStorageTransferError $error) {
return;
}
throw new RuntimeException($message);
};
$capsule = new Capsule();
$capsule->addConnection([
'driver' => 'sqlite', 'database' => ':memory:',
], 'default');
$capsule->setAsGlobal();
$capsule->bootEloquent();
$schema = $capsule->schema('default');
$schema->create('cloud_instance_backups',
static function (Blueprint $table): void {
$table->increments('id');
$table->string('purpose');
$table->string('state');
$table->string('object_key');
$table->text('sealed_storage_runtime');
$table->text('sealed_credential')->nullable();
$table->integer('storage_revision');
$table->integer('active_job_id')->nullable();
$table->string('error_code')->default('');
$table->integer('updated_at');
});
$schema->create('cloud_instance_migrations',
static function (Blueprint $table): void {
$table->increments('id');
$table->integer('backup_id');
$table->string('state');
$table->string('error_code')->default('');
$table->integer('updated_at');
});
$schema->create('cloud_migration_bundle_artifacts',
static function (Blueprint $table): void {
$table->increments('id');
$table->integer('migration_id');
$table->string('object_key');
$table->string('state');
$table->integer('updated_at');
});
Config::$dataKey = base64_encode(str_repeat('m', 32));
$runtime = [
'config' => [
'endpoint' => 'https://objects.example.com',
'region' => 'us-east-1',
'bucket' => 'backup-bucket',
'prefix' => 'virt',
'path_style' => true,
],
'credentials' => [
'access_key_id' => 'MIGRATIONACCESS',
'secret_access_key' => 'migration-secret-value',
],
'revision' => 9,
];
$sealedRuntime = CloudSecrets::encrypt(
$runtime,
'object-storage-backup-runtime'
);
$client = static function (array $responses, array &$history):
CloudObjectStorageS3Client {
$stack = HandlerStack::create(new MockHandler($responses));
$stack->push(Middleware::history($history));
return new CloudObjectStorageS3Client(new Client(['handler' => $stack]));
};
$newMigration = static function (int $id, string $state,
int $artifactCount = 2) use ($sealedRuntime, $capsule): void {
$now = time() - 600;
$capsule->table('cloud_instance_backups')->insert([
'id' => 100 + $id,
'purpose' => 'migration_bundle',
'state' => 'ready',
'object_key' => 'virt/vm-migrations/bundle-' . $id
. '/system.qcow2',
'sealed_storage_runtime' => $sealedRuntime,
'sealed_credential' => 'sealed-guest-credential',
'storage_revision' => 9,
'active_job_id' => null,
'error_code' => '',
'updated_at' => $now,
]);
$capsule->table('cloud_instance_migrations')->insert([
'id' => $id,
'backup_id' => 100 + $id,
'state' => $state,
'error_code' => '',
'updated_at' => $now,
]);
for ($index = 1; $index <= $artifactCount; $index++) {
$capsule->table('cloud_migration_bundle_artifacts')->insert([
'id' => $id * 10 + $index,
'migration_id' => $id,
'object_key' => 'virt/vm-migrations/bundle-' . $id
. '/snapshots/snapshot-' . $index . '.qcow2',
'state' => 'restored',
'updated_at' => $now,
]);
}
};
// A completed migration must delete every snapshot transport before the
// root object, then remove its frozen object-store credentials.
$newMigration(1, 'completed');
$history = [];
$storage = $client([
new Response(204),
new Response(204),
new Response(204),
], $history);
for ($step = 0; $step < 3; $step++) {
$assert(CloudMigrationBundleService::cleanupOneEligible(
$storage,
time() + $step
), 'Successful migration bundle cleanup did not advance.');
}
$backup = CloudInstanceBackup::findOrFail(101);
$assert(CloudMigrationBundleArtifact::where('migration_id', 1)
->where('state', 'deleted')->count() === 2
&& (string) $backup->state === 'deleted'
&& (string) $backup->sealed_storage_runtime === ''
&& $backup->sealed_credential === null,
'Successful migration retained object-store artifacts or credentials.');
$assert(array_map(
static fn (array $entry): string =>
$entry['request']->getMethod() . ' '
. $entry['request']->getUri()->getPath(),
$history
) === [
'DELETE /backup-bucket/virt/vm-migrations/bundle-1/snapshots/snapshot-1.qcow2',
'DELETE /backup-bucket/virt/vm-migrations/bundle-1/snapshots/snapshot-2.qcow2',
'DELETE /backup-bucket/virt/vm-migrations/bundle-1/system.qcow2',
], 'Migration transport objects were not deleted snapshot-first/root-last.');
// A deliberately retained source is still a successful target migration;
// its object-store transport must not become a permanent hidden backup.
$newMigration(2, 'source_retained', 0);
$retainedHistory = [];
$assert(CloudMigrationBundleService::cleanupOneEligible(
$client([new Response(204)], $retainedHistory),
time()
), 'Source-retained migration did not clean its transport object.');
$assert((string) CloudInstanceBackup::findOrFail(102)->state === 'deleted'
&& (string) CloudInstanceMigration::findOrFail(2)->state
=== 'source_retained',
'Transport cleanup changed the retained-source migration contract.');
// Once DeleteObject has an unknown effect, every automatic follow-up must
// be read-only HEAD reconciliation; it must never replay DELETE.
$newMigration(3, 'completed', 1);
$unknownHistory = [];
$expectCleanupFailure(static function () use (
$client,
&$unknownHistory
): void {
CloudMigrationBundleService::cleanupOneEligible($client([
new ConnectException('transport lost', new Request(
'DELETE', 'https://objects.example.com'
)),
], $unknownHistory), time());
}, 'Unknown DeleteObject effect was not reported.');
$artifact = CloudMigrationBundleArtifact::findOrFail(31);
$assert((string) $artifact->state === 'delete_uncertain'
&& count($unknownHistory) === 1
&& $unknownHistory[0]['request']->getMethod() === 'DELETE',
'Unknown object deletion did not retain its effect fence.');
CloudInstanceBackup::where('id', 103)->update([
'sealed_storage_runtime' => 'invalid-ciphertext',
]);
$invalidFenceHistory = [];
$expectCleanupFailure(static function () use (
$client,
&$invalidFenceHistory
): void {
CloudMigrationBundleService::cleanupOneEligible(
$client([], $invalidFenceHistory),
time() + 301
);
}, 'Invalid runtime did not surface during ambiguous reconciliation.');
$assert($invalidFenceHistory === []
&& (string) CloudMigrationBundleArtifact::findOrFail(31)->state
=== 'delete_uncertain',
'Invalid runtime downgraded an unknown-effect deletion fence.');
CloudInstanceBackup::where('id', 103)->update([
'sealed_storage_runtime' => $sealedRuntime,
]);
$presentHistory = [];
$expectCleanupFailure(static function () use (
$client,
&$presentHistory
): void {
CloudMigrationBundleService::cleanupOneEligible(
$client([new Response(200)], $presentHistory),
time() + 602
);
}, 'Still-present ambiguous object was not reported.');
$assert((string) CloudMigrationBundleArtifact::findOrFail(31)->state
=== 'delete_uncertain'
&& count($presentHistory) === 1
&& $presentHistory[0]['request']->getMethod() === 'HEAD',
'A present ambiguous object caused DeleteObject to be replayed.');
$absentHistory = [];
$assert(CloudMigrationBundleService::cleanupOneEligible(
$client([new Response(404)], $absentHistory),
time() + 903
), 'Absent ambiguous object did not converge to deleted.');
$assert((string) CloudMigrationBundleArtifact::findOrFail(31)->state
=== 'deleted'
&& $absentHistory[0]['request']->getMethod() === 'HEAD',
'HEAD 404 did not safely finish ambiguous object cleanup.');
// Finish migration 3's root before proving that a later failed migration
// cannot become eligible by accident.
$thirdRootHistory = [];
$assert(CloudMigrationBundleService::cleanupOneEligible(
$client([new Response(204)], $thirdRootHistory),
time() + 904
) && (string) CloudInstanceBackup::findOrFail(103)->state === 'deleted',
'A reconciled bundle did not finish by deleting its root object.');
// A failed migration is not a successful migration and must retain its
// bundle for rollback/recovery investigation.
$newMigration(4, 'rolled_back', 0);
$failedHistory = [];
$assert(!CloudMigrationBundleService::cleanupOneEligible(
$client([], $failedHistory),
time() + 1000
) && $failedHistory === []
&& (string) CloudInstanceBackup::findOrFail(104)->state === 'ready',
'A non-successful migration lost its recovery bundle.');
// Successful migration alone is insufficient: the target must have
// restored every snapshot and the root backup must remain ready.
$newMigration(5, 'completed', 1);
CloudMigrationBundleArtifact::where('migration_id', 5)->update([
'state' => 'ready',
]);
$newMigration(6, 'completed', 0);
CloudInstanceBackup::where('id', 106)->update(['state' => 'failed']);
$newMigration(7, 'cleanup_uncertain', 0);
$failClosedHistory = [];
$assert(!CloudMigrationBundleService::cleanupOneEligible(
$client([], $failClosedHistory),
time() + 1100
) && $failClosedHistory === [],
'An incomplete or uncertain migration bundle became deletable.');
// A fresh deleting claim belongs to another worker. Once stale, it is an
// unknown effect and must be reconciled with HEAD instead of replaying it.
$newMigration(8, 'completed', 1);
$leaseNow = time() + 2000;
CloudMigrationBundleArtifact::where('migration_id', 8)->update([
'state' => 'deleting',
'updated_at' => $leaseNow,
]);
$freshHistory = [];
$assert(!CloudMigrationBundleService::cleanupOneEligible(
$client([], $freshHistory),
$leaseNow + 59
) && $freshHistory === [],
'A fresh object-delete lease was stolen by another worker.');
CloudInstanceBackup::where('id', 108)->update([
'sealed_storage_runtime' => 'invalid-ciphertext',
]);
$staleInvalidHistory = [];
$expectCleanupFailure(static function () use (
$client,
&$staleInvalidHistory,
$leaseNow
): void {
CloudMigrationBundleService::cleanupOneEligible(
$client([], $staleInvalidHistory),
$leaseNow + 60
);
}, 'Invalid runtime did not surface for a stale delete claim.');
$assert($staleInvalidHistory === []
&& (string) CloudMigrationBundleArtifact::where(
'migration_id', 8
)->value('state') === 'delete_uncertain',
'A stale unknown-effect claim was downgraded to a replayable state.');
CloudInstanceBackup::where('id', 108)->update([
'sealed_storage_runtime' => $sealedRuntime,
]);
$staleHistory = [];
$assert(CloudMigrationBundleService::cleanupOneEligible(
$client([new Response(404)], $staleHistory),
$leaseNow + 361
) && $staleHistory[0]['request']->getMethod() === 'HEAD',
'A stale object-delete claim was replayed instead of reconciled.');
$staleRootHistory = [];
$assert(CloudMigrationBundleService::cleanupOneEligible(
$client([new Response(204)], $staleRootHistory),
$leaseNow + 362
) && (string) CloudInstanceBackup::findOrFail(108)->state === 'deleted',
'The reconciled stale bundle did not delete its root last.');
// A definitive provider rejection may be retried, but only after the
// backoff window; this must not form a hot loop in the worker.
$newMigration(9, 'completed', 1);
$rejectedHistory = [];
$expectCleanupFailure(static function () use (
$client,
&$rejectedHistory,
$leaseNow
): void {
CloudMigrationBundleService::cleanupOneEligible(
$client([new Response(403)], $rejectedHistory),
$leaseNow + 400
);
}, 'A definitive DeleteObject rejection was not reported.');
$assert((string) CloudMigrationBundleArtifact::where(
'migration_id', 9
)->value('state') === 'delete_rejected',
'A definitive DeleteObject rejection was not persisted.');
$backoffHistory = [];
$assert(!CloudMigrationBundleService::cleanupOneEligible(
$client([], $backoffHistory),
$leaseNow + 699
) && $backoffHistory === [],
'A rejected DeleteObject entered a retry hot loop.');
$retryHistory = [];
$assert(CloudMigrationBundleService::cleanupOneEligible(
$client([new Response(204)], $retryHistory),
$leaseNow + 700
) && $retryHistory[0]['request']->getMethod() === 'DELETE',
'A definitively rejected delete was not retried after backoff.');
$retryRootHistory = [];
$assert(CloudMigrationBundleService::cleanupOneEligible(
$client([new Response(204)], $retryRootHistory),
$leaseNow + 701
), 'The retry test bundle root was not deleted.');
// Root deletion has the same unknown-effect fence: runtime damage after a
// lost DELETE response cannot turn it back into a replayable operation.
$newMigration(10, 'completed', 0);
$rootUnknownHistory = [];
$expectCleanupFailure(static function () use (
$client,
&$rootUnknownHistory,
$leaseNow
): void {
CloudMigrationBundleService::cleanupOneEligible($client([
new ConnectException('transport lost', new Request(
'DELETE', 'https://objects.example.com'
)),
], $rootUnknownHistory), $leaseNow + 800);
}, 'Unknown root DeleteObject effect was not reported.');
$assert((string) CloudInstanceBackup::findOrFail(110)->state
=== 'delete_uncertain'
&& $rootUnknownHistory[0]['request']->getMethod() === 'DELETE',
'Unknown root deletion did not retain its effect fence.');
CloudInstanceBackup::where('id', 110)->update([
'sealed_storage_runtime' => 'invalid-ciphertext',
]);
$rootInvalidHistory = [];
$expectCleanupFailure(static function () use (
$client,
&$rootInvalidHistory,
$leaseNow
): void {
CloudMigrationBundleService::cleanupOneEligible(
$client([], $rootInvalidHistory),
$leaseNow + 1101
);
}, 'Invalid runtime did not surface for ambiguous root cleanup.');
$assert($rootInvalidHistory === []
&& (string) CloudInstanceBackup::findOrFail(110)->state
=== 'delete_uncertain',
'Invalid runtime downgraded an ambiguous root deletion fence.');
CloudInstanceBackup::where('id', 110)->update([
'sealed_storage_runtime' => $sealedRuntime,
]);
$rootAbsentHistory = [];
$assert(CloudMigrationBundleService::cleanupOneEligible(
$client([new Response(404)], $rootAbsentHistory),
$leaseNow + 1402
) && (string) CloudInstanceBackup::findOrFail(110)->state === 'deleted'
&& $rootAbsentHistory[0]['request']->getMethod() === 'HEAD',
'Ambiguous root deletion did not converge with HEAD only.');
// A corrupt frozen runtime is isolated with backoff and cannot starve a
// later valid bundle forever.
$newMigration(11, 'completed', 0);
CloudInstanceBackup::where('id', 111)->update([
'sealed_storage_runtime' => 'invalid-ciphertext',
]);
$newMigration(12, 'completed', 0);
$invalidHistory = [];
$expectCleanupFailure(static function () use (
$client,
&$invalidHistory,
$leaseNow
): void {
CloudMigrationBundleService::cleanupOneEligible(
$client([], $invalidHistory),
$leaseNow + 1500
);
}, 'An invalid frozen runtime was not reported.');
$assert($invalidHistory === []
&& (string) CloudInstanceBackup::findOrFail(111)->state
=== 'delete_rejected',
'An invalid frozen runtime was not isolated behind backoff.');
$laterHistory = [];
$assert(CloudMigrationBundleService::cleanupOneEligible(
$client([new Response(204)], $laterHistory),
$leaseNow + 1501
) && (string) CloudInstanceBackup::findOrFail(112)->state === 'deleted',
'One invalid runtime starved a later valid migration bundle.');
echo "Maintenance migration bundle cleanup and reconciliation passed.\n";
}