You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
464 lines
19 KiB
PHP
464 lines
19 KiB
PHP
<?php
|
|
|
|
namespace App\Services {
|
|
final class Config
|
|
{
|
|
public static string $dataKey = '';
|
|
|
|
public static function get($name, $default = null)
|
|
{
|
|
return $name === 'cloud_data_key' ? self::$dataKey : $default;
|
|
}
|
|
}
|
|
}
|
|
|
|
namespace {
|
|
require dirname(__DIR__) . '/vendor/autoload.php';
|
|
|
|
use App\Models\CloudInstanceBackup;
|
|
use App\Models\CloudInstanceMigration;
|
|
use App\Models\CloudMigrationBundleArtifact;
|
|
use App\Services\Config;
|
|
use App\Services\Cloud\CloudError;
|
|
use App\Services\Cloud\CloudMigrationBundleService;
|
|
use App\Services\Cloud\CloudObjectStorageS3Client;
|
|
use App\Services\Cloud\CloudObjectStorageTransferError;
|
|
use App\Services\Cloud\CloudSecrets;
|
|
use GuzzleHttp\Client;
|
|
use GuzzleHttp\Exception\ConnectException;
|
|
use GuzzleHttp\Handler\MockHandler;
|
|
use GuzzleHttp\HandlerStack;
|
|
use GuzzleHttp\Middleware;
|
|
use GuzzleHttp\Psr7\Request;
|
|
use GuzzleHttp\Psr7\Response;
|
|
use Illuminate\Database\Capsule\Manager as Capsule;
|
|
use Illuminate\Database\Schema\Blueprint;
|
|
|
|
if (!function_exists('sodium_crypto_aead_xchacha20poly1305_ietf_encrypt')) {
|
|
echo "SKIPPED CloudMigrationBundleCleanupTest: runtime lacks sodium.\n";
|
|
exit(0);
|
|
}
|
|
|
|
$assert = static function (bool $condition, string $message): void {
|
|
if (!$condition) {
|
|
throw new RuntimeException($message);
|
|
}
|
|
};
|
|
$expectCleanupFailure = static function (callable $callback,
|
|
string $message): void {
|
|
try {
|
|
$callback();
|
|
} catch (CloudError|CloudObjectStorageTransferError $error) {
|
|
return;
|
|
}
|
|
throw new RuntimeException($message);
|
|
};
|
|
|
|
$capsule = new Capsule();
|
|
$capsule->addConnection([
|
|
'driver' => 'sqlite', 'database' => ':memory:',
|
|
], 'default');
|
|
$capsule->setAsGlobal();
|
|
$capsule->bootEloquent();
|
|
$schema = $capsule->schema('default');
|
|
$schema->create('cloud_instance_backups',
|
|
static function (Blueprint $table): void {
|
|
$table->increments('id');
|
|
$table->string('purpose');
|
|
$table->string('state');
|
|
$table->string('object_key');
|
|
$table->text('sealed_storage_runtime');
|
|
$table->text('sealed_credential')->nullable();
|
|
$table->integer('storage_revision');
|
|
$table->integer('active_job_id')->nullable();
|
|
$table->string('error_code')->default('');
|
|
$table->integer('updated_at');
|
|
});
|
|
$schema->create('cloud_instance_migrations',
|
|
static function (Blueprint $table): void {
|
|
$table->increments('id');
|
|
$table->integer('backup_id');
|
|
$table->string('state');
|
|
$table->string('error_code')->default('');
|
|
$table->integer('updated_at');
|
|
});
|
|
$schema->create('cloud_migration_bundle_artifacts',
|
|
static function (Blueprint $table): void {
|
|
$table->increments('id');
|
|
$table->integer('migration_id');
|
|
$table->string('object_key');
|
|
$table->string('state');
|
|
$table->integer('updated_at');
|
|
});
|
|
|
|
Config::$dataKey = base64_encode(str_repeat('m', 32));
|
|
$runtime = [
|
|
'config' => [
|
|
'endpoint' => 'https://objects.example.com',
|
|
'region' => 'us-east-1',
|
|
'bucket' => 'backup-bucket',
|
|
'prefix' => 'virt',
|
|
'path_style' => true,
|
|
],
|
|
'credentials' => [
|
|
'access_key_id' => 'MIGRATIONACCESS',
|
|
'secret_access_key' => 'migration-secret-value',
|
|
],
|
|
'revision' => 9,
|
|
];
|
|
$sealedRuntime = CloudSecrets::encrypt(
|
|
$runtime,
|
|
'object-storage-backup-runtime'
|
|
);
|
|
$client = static function (array $responses, array &$history):
|
|
CloudObjectStorageS3Client {
|
|
$stack = HandlerStack::create(new MockHandler($responses));
|
|
$stack->push(Middleware::history($history));
|
|
return new CloudObjectStorageS3Client(new Client(['handler' => $stack]));
|
|
};
|
|
$newMigration = static function (int $id, string $state,
|
|
int $artifactCount = 2) use ($sealedRuntime, $capsule): void {
|
|
$now = time() - 600;
|
|
$capsule->table('cloud_instance_backups')->insert([
|
|
'id' => 100 + $id,
|
|
'purpose' => 'migration_bundle',
|
|
'state' => 'ready',
|
|
'object_key' => 'virt/vm-migrations/bundle-' . $id
|
|
. '/system.qcow2',
|
|
'sealed_storage_runtime' => $sealedRuntime,
|
|
'sealed_credential' => 'sealed-guest-credential',
|
|
'storage_revision' => 9,
|
|
'active_job_id' => null,
|
|
'error_code' => '',
|
|
'updated_at' => $now,
|
|
]);
|
|
$capsule->table('cloud_instance_migrations')->insert([
|
|
'id' => $id,
|
|
'backup_id' => 100 + $id,
|
|
'state' => $state,
|
|
'error_code' => '',
|
|
'updated_at' => $now,
|
|
]);
|
|
for ($index = 1; $index <= $artifactCount; $index++) {
|
|
$capsule->table('cloud_migration_bundle_artifacts')->insert([
|
|
'id' => $id * 10 + $index,
|
|
'migration_id' => $id,
|
|
'object_key' => 'virt/vm-migrations/bundle-' . $id
|
|
. '/snapshots/snapshot-' . $index . '.qcow2',
|
|
'state' => 'restored',
|
|
'updated_at' => $now,
|
|
]);
|
|
}
|
|
};
|
|
|
|
// A completed migration must delete every snapshot transport before the
|
|
// root object, then remove its frozen object-store credentials.
|
|
$newMigration(1, 'completed');
|
|
$history = [];
|
|
$storage = $client([
|
|
new Response(204),
|
|
new Response(204),
|
|
new Response(204),
|
|
], $history);
|
|
for ($step = 0; $step < 3; $step++) {
|
|
$assert(CloudMigrationBundleService::cleanupOneEligible(
|
|
$storage,
|
|
time() + $step
|
|
), 'Successful migration bundle cleanup did not advance.');
|
|
}
|
|
$backup = CloudInstanceBackup::findOrFail(101);
|
|
$assert(CloudMigrationBundleArtifact::where('migration_id', 1)
|
|
->where('state', 'deleted')->count() === 2
|
|
&& (string) $backup->state === 'deleted'
|
|
&& (string) $backup->sealed_storage_runtime === ''
|
|
&& $backup->sealed_credential === null,
|
|
'Successful migration retained object-store artifacts or credentials.');
|
|
$assert(array_map(
|
|
static fn (array $entry): string =>
|
|
$entry['request']->getMethod() . ' '
|
|
. $entry['request']->getUri()->getPath(),
|
|
$history
|
|
) === [
|
|
'DELETE /backup-bucket/virt/vm-migrations/bundle-1/snapshots/snapshot-1.qcow2',
|
|
'DELETE /backup-bucket/virt/vm-migrations/bundle-1/snapshots/snapshot-2.qcow2',
|
|
'DELETE /backup-bucket/virt/vm-migrations/bundle-1/system.qcow2',
|
|
], 'Migration transport objects were not deleted snapshot-first/root-last.');
|
|
|
|
// A deliberately retained source is still a successful target migration;
|
|
// its object-store transport must not become a permanent hidden backup.
|
|
$newMigration(2, 'source_retained', 0);
|
|
$retainedHistory = [];
|
|
$assert(CloudMigrationBundleService::cleanupOneEligible(
|
|
$client([new Response(204)], $retainedHistory),
|
|
time()
|
|
), 'Source-retained migration did not clean its transport object.');
|
|
$assert((string) CloudInstanceBackup::findOrFail(102)->state === 'deleted'
|
|
&& (string) CloudInstanceMigration::findOrFail(2)->state
|
|
=== 'source_retained',
|
|
'Transport cleanup changed the retained-source migration contract.');
|
|
|
|
// Once DeleteObject has an unknown effect, every automatic follow-up must
|
|
// be read-only HEAD reconciliation; it must never replay DELETE.
|
|
$newMigration(3, 'completed', 1);
|
|
$unknownHistory = [];
|
|
$expectCleanupFailure(static function () use (
|
|
$client,
|
|
&$unknownHistory
|
|
): void {
|
|
CloudMigrationBundleService::cleanupOneEligible($client([
|
|
new ConnectException('transport lost', new Request(
|
|
'DELETE', 'https://objects.example.com'
|
|
)),
|
|
], $unknownHistory), time());
|
|
}, 'Unknown DeleteObject effect was not reported.');
|
|
$artifact = CloudMigrationBundleArtifact::findOrFail(31);
|
|
$assert((string) $artifact->state === 'delete_uncertain'
|
|
&& count($unknownHistory) === 1
|
|
&& $unknownHistory[0]['request']->getMethod() === 'DELETE',
|
|
'Unknown object deletion did not retain its effect fence.');
|
|
CloudInstanceBackup::where('id', 103)->update([
|
|
'sealed_storage_runtime' => 'invalid-ciphertext',
|
|
]);
|
|
$invalidFenceHistory = [];
|
|
$expectCleanupFailure(static function () use (
|
|
$client,
|
|
&$invalidFenceHistory
|
|
): void {
|
|
CloudMigrationBundleService::cleanupOneEligible(
|
|
$client([], $invalidFenceHistory),
|
|
time() + 301
|
|
);
|
|
}, 'Invalid runtime did not surface during ambiguous reconciliation.');
|
|
$assert($invalidFenceHistory === []
|
|
&& (string) CloudMigrationBundleArtifact::findOrFail(31)->state
|
|
=== 'delete_uncertain',
|
|
'Invalid runtime downgraded an unknown-effect deletion fence.');
|
|
CloudInstanceBackup::where('id', 103)->update([
|
|
'sealed_storage_runtime' => $sealedRuntime,
|
|
]);
|
|
$presentHistory = [];
|
|
$expectCleanupFailure(static function () use (
|
|
$client,
|
|
&$presentHistory
|
|
): void {
|
|
CloudMigrationBundleService::cleanupOneEligible(
|
|
$client([new Response(200)], $presentHistory),
|
|
time() + 602
|
|
);
|
|
}, 'Still-present ambiguous object was not reported.');
|
|
$assert((string) CloudMigrationBundleArtifact::findOrFail(31)->state
|
|
=== 'delete_uncertain'
|
|
&& count($presentHistory) === 1
|
|
&& $presentHistory[0]['request']->getMethod() === 'HEAD',
|
|
'A present ambiguous object caused DeleteObject to be replayed.');
|
|
$absentHistory = [];
|
|
$assert(CloudMigrationBundleService::cleanupOneEligible(
|
|
$client([new Response(404)], $absentHistory),
|
|
time() + 903
|
|
), 'Absent ambiguous object did not converge to deleted.');
|
|
$assert((string) CloudMigrationBundleArtifact::findOrFail(31)->state
|
|
=== 'deleted'
|
|
&& $absentHistory[0]['request']->getMethod() === 'HEAD',
|
|
'HEAD 404 did not safely finish ambiguous object cleanup.');
|
|
|
|
// Finish migration 3's root before proving that a later failed migration
|
|
// cannot become eligible by accident.
|
|
$thirdRootHistory = [];
|
|
$assert(CloudMigrationBundleService::cleanupOneEligible(
|
|
$client([new Response(204)], $thirdRootHistory),
|
|
time() + 904
|
|
) && (string) CloudInstanceBackup::findOrFail(103)->state === 'deleted',
|
|
'A reconciled bundle did not finish by deleting its root object.');
|
|
|
|
// A failed migration is not a successful migration and must retain its
|
|
// bundle for rollback/recovery investigation.
|
|
$newMigration(4, 'rolled_back', 0);
|
|
$failedHistory = [];
|
|
$assert(!CloudMigrationBundleService::cleanupOneEligible(
|
|
$client([], $failedHistory),
|
|
time() + 1000
|
|
) && $failedHistory === []
|
|
&& (string) CloudInstanceBackup::findOrFail(104)->state === 'ready',
|
|
'A non-successful migration lost its recovery bundle.');
|
|
|
|
// Successful migration alone is insufficient: the target must have
|
|
// restored every snapshot and the root backup must remain ready.
|
|
$newMigration(5, 'completed', 1);
|
|
CloudMigrationBundleArtifact::where('migration_id', 5)->update([
|
|
'state' => 'ready',
|
|
]);
|
|
$newMigration(6, 'completed', 0);
|
|
CloudInstanceBackup::where('id', 106)->update(['state' => 'failed']);
|
|
$newMigration(7, 'cleanup_uncertain', 0);
|
|
$failClosedHistory = [];
|
|
$assert(!CloudMigrationBundleService::cleanupOneEligible(
|
|
$client([], $failClosedHistory),
|
|
time() + 1100
|
|
) && $failClosedHistory === [],
|
|
'An incomplete or uncertain migration bundle became deletable.');
|
|
|
|
// A fresh deleting claim belongs to another worker. Once stale, it is an
|
|
// unknown effect and must be reconciled with HEAD instead of replaying it.
|
|
$newMigration(8, 'completed', 1);
|
|
$leaseNow = time() + 2000;
|
|
CloudMigrationBundleArtifact::where('migration_id', 8)->update([
|
|
'state' => 'deleting',
|
|
'updated_at' => $leaseNow,
|
|
]);
|
|
$freshHistory = [];
|
|
$assert(!CloudMigrationBundleService::cleanupOneEligible(
|
|
$client([], $freshHistory),
|
|
$leaseNow + 59
|
|
) && $freshHistory === [],
|
|
'A fresh object-delete lease was stolen by another worker.');
|
|
CloudInstanceBackup::where('id', 108)->update([
|
|
'sealed_storage_runtime' => 'invalid-ciphertext',
|
|
]);
|
|
$staleInvalidHistory = [];
|
|
$expectCleanupFailure(static function () use (
|
|
$client,
|
|
&$staleInvalidHistory,
|
|
$leaseNow
|
|
): void {
|
|
CloudMigrationBundleService::cleanupOneEligible(
|
|
$client([], $staleInvalidHistory),
|
|
$leaseNow + 60
|
|
);
|
|
}, 'Invalid runtime did not surface for a stale delete claim.');
|
|
$assert($staleInvalidHistory === []
|
|
&& (string) CloudMigrationBundleArtifact::where(
|
|
'migration_id', 8
|
|
)->value('state') === 'delete_uncertain',
|
|
'A stale unknown-effect claim was downgraded to a replayable state.');
|
|
CloudInstanceBackup::where('id', 108)->update([
|
|
'sealed_storage_runtime' => $sealedRuntime,
|
|
]);
|
|
$staleHistory = [];
|
|
$assert(CloudMigrationBundleService::cleanupOneEligible(
|
|
$client([new Response(404)], $staleHistory),
|
|
$leaseNow + 361
|
|
) && $staleHistory[0]['request']->getMethod() === 'HEAD',
|
|
'A stale object-delete claim was replayed instead of reconciled.');
|
|
$staleRootHistory = [];
|
|
$assert(CloudMigrationBundleService::cleanupOneEligible(
|
|
$client([new Response(204)], $staleRootHistory),
|
|
$leaseNow + 362
|
|
) && (string) CloudInstanceBackup::findOrFail(108)->state === 'deleted',
|
|
'The reconciled stale bundle did not delete its root last.');
|
|
|
|
// A definitive provider rejection may be retried, but only after the
|
|
// backoff window; this must not form a hot loop in the worker.
|
|
$newMigration(9, 'completed', 1);
|
|
$rejectedHistory = [];
|
|
$expectCleanupFailure(static function () use (
|
|
$client,
|
|
&$rejectedHistory,
|
|
$leaseNow
|
|
): void {
|
|
CloudMigrationBundleService::cleanupOneEligible(
|
|
$client([new Response(403)], $rejectedHistory),
|
|
$leaseNow + 400
|
|
);
|
|
}, 'A definitive DeleteObject rejection was not reported.');
|
|
$assert((string) CloudMigrationBundleArtifact::where(
|
|
'migration_id', 9
|
|
)->value('state') === 'delete_rejected',
|
|
'A definitive DeleteObject rejection was not persisted.');
|
|
$backoffHistory = [];
|
|
$assert(!CloudMigrationBundleService::cleanupOneEligible(
|
|
$client([], $backoffHistory),
|
|
$leaseNow + 699
|
|
) && $backoffHistory === [],
|
|
'A rejected DeleteObject entered a retry hot loop.');
|
|
$retryHistory = [];
|
|
$assert(CloudMigrationBundleService::cleanupOneEligible(
|
|
$client([new Response(204)], $retryHistory),
|
|
$leaseNow + 700
|
|
) && $retryHistory[0]['request']->getMethod() === 'DELETE',
|
|
'A definitively rejected delete was not retried after backoff.');
|
|
$retryRootHistory = [];
|
|
$assert(CloudMigrationBundleService::cleanupOneEligible(
|
|
$client([new Response(204)], $retryRootHistory),
|
|
$leaseNow + 701
|
|
), 'The retry test bundle root was not deleted.');
|
|
|
|
// Root deletion has the same unknown-effect fence: runtime damage after a
|
|
// lost DELETE response cannot turn it back into a replayable operation.
|
|
$newMigration(10, 'completed', 0);
|
|
$rootUnknownHistory = [];
|
|
$expectCleanupFailure(static function () use (
|
|
$client,
|
|
&$rootUnknownHistory,
|
|
$leaseNow
|
|
): void {
|
|
CloudMigrationBundleService::cleanupOneEligible($client([
|
|
new ConnectException('transport lost', new Request(
|
|
'DELETE', 'https://objects.example.com'
|
|
)),
|
|
], $rootUnknownHistory), $leaseNow + 800);
|
|
}, 'Unknown root DeleteObject effect was not reported.');
|
|
$assert((string) CloudInstanceBackup::findOrFail(110)->state
|
|
=== 'delete_uncertain'
|
|
&& $rootUnknownHistory[0]['request']->getMethod() === 'DELETE',
|
|
'Unknown root deletion did not retain its effect fence.');
|
|
CloudInstanceBackup::where('id', 110)->update([
|
|
'sealed_storage_runtime' => 'invalid-ciphertext',
|
|
]);
|
|
$rootInvalidHistory = [];
|
|
$expectCleanupFailure(static function () use (
|
|
$client,
|
|
&$rootInvalidHistory,
|
|
$leaseNow
|
|
): void {
|
|
CloudMigrationBundleService::cleanupOneEligible(
|
|
$client([], $rootInvalidHistory),
|
|
$leaseNow + 1101
|
|
);
|
|
}, 'Invalid runtime did not surface for ambiguous root cleanup.');
|
|
$assert($rootInvalidHistory === []
|
|
&& (string) CloudInstanceBackup::findOrFail(110)->state
|
|
=== 'delete_uncertain',
|
|
'Invalid runtime downgraded an ambiguous root deletion fence.');
|
|
CloudInstanceBackup::where('id', 110)->update([
|
|
'sealed_storage_runtime' => $sealedRuntime,
|
|
]);
|
|
$rootAbsentHistory = [];
|
|
$assert(CloudMigrationBundleService::cleanupOneEligible(
|
|
$client([new Response(404)], $rootAbsentHistory),
|
|
$leaseNow + 1402
|
|
) && (string) CloudInstanceBackup::findOrFail(110)->state === 'deleted'
|
|
&& $rootAbsentHistory[0]['request']->getMethod() === 'HEAD',
|
|
'Ambiguous root deletion did not converge with HEAD only.');
|
|
|
|
// A corrupt frozen runtime is isolated with backoff and cannot starve a
|
|
// later valid bundle forever.
|
|
$newMigration(11, 'completed', 0);
|
|
CloudInstanceBackup::where('id', 111)->update([
|
|
'sealed_storage_runtime' => 'invalid-ciphertext',
|
|
]);
|
|
$newMigration(12, 'completed', 0);
|
|
$invalidHistory = [];
|
|
$expectCleanupFailure(static function () use (
|
|
$client,
|
|
&$invalidHistory,
|
|
$leaseNow
|
|
): void {
|
|
CloudMigrationBundleService::cleanupOneEligible(
|
|
$client([], $invalidHistory),
|
|
$leaseNow + 1500
|
|
);
|
|
}, 'An invalid frozen runtime was not reported.');
|
|
$assert($invalidHistory === []
|
|
&& (string) CloudInstanceBackup::findOrFail(111)->state
|
|
=== 'delete_rejected',
|
|
'An invalid frozen runtime was not isolated behind backoff.');
|
|
$laterHistory = [];
|
|
$assert(CloudMigrationBundleService::cleanupOneEligible(
|
|
$client([new Response(204)], $laterHistory),
|
|
$leaseNow + 1501
|
|
) && (string) CloudInstanceBackup::findOrFail(112)->state === 'deleted',
|
|
'One invalid runtime starved a later valid migration bundle.');
|
|
|
|
echo "Maintenance migration bundle cleanup and reconciliation passed.\n";
|
|
}
|