|
|
|
|
<?php
|
|
|
|
|
|
|
|
|
|
ob_start();
|
|
|
|
|
require __DIR__ . '/CloudPortableRestoreJobDispatcherTest.php';
|
|
|
|
|
ob_end_clean();
|
|
|
|
|
|
|
|
|
|
use App\Models\CloudBackupRestore;
|
|
|
|
|
use App\Models\CloudBackupRestoreAttempt;
|
|
|
|
|
use App\Models\CloudInstanceBackup;
|
|
|
|
|
use App\Models\CloudJob;
|
|
|
|
|
use App\Services\Cloud\CloudError;
|
|
|
|
|
use App\Services\Cloud\CloudBackupService;
|
|
|
|
|
use App\Services\Cloud\CloudPortableRestoreAttemptCoordinator;
|
|
|
|
|
use App\Services\Cloud\CloudPortableRestoreAdmissionRecoveryService;
|
|
|
|
|
use App\Services\Cloud\CloudPortableRestoreRequestService;
|
|
|
|
|
use App\Services\Cloud\CloudPortableRestoreTargetPlan;
|
|
|
|
|
use App\Services\Cloud\CloudPortableRestoreTargetService;
|
|
|
|
|
use Illuminate\Database\Capsule\Manager as Capsule;
|
|
|
|
|
|
|
|
|
|
$rejects409 = static function (callable $callback, string $message) use (
|
|
|
|
|
$assert
|
|
|
|
|
): void {
|
|
|
|
|
try {
|
|
|
|
|
$callback();
|
|
|
|
|
} catch (CloudError $error) {
|
|
|
|
|
$assert($error->httpStatus === 409, $message . ' (wrong status)');
|
|
|
|
|
return;
|
|
|
|
|
}
|
|
|
|
|
throw new RuntimeException($message);
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
$backup = CloudInstanceBackup::findOrFail(1);
|
|
|
|
|
$plan = null;
|
|
|
|
|
foreach (CloudPortableRestoreTargetService::candidates($backup)
|
|
|
|
|
as $candidate) {
|
|
|
|
|
$target = $candidate->toArray()['target'];
|
|
|
|
|
if ((int) $target['host_id'] === 1
|
|
|
|
|
&& (int) $target['resource_pool_id'] === 100) {
|
|
|
|
|
$plan = $candidate;
|
|
|
|
|
break;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
$assert($plan instanceof CloudPortableRestoreTargetPlan,
|
|
|
|
|
'A fresh request-service target plan is unavailable.');
|
|
|
|
|
$requestActor = $actor;
|
|
|
|
|
$requestActor['request_idempotency_key'] =
|
|
|
|
|
'portable-request-service-00000001';
|
|
|
|
|
$input = [
|
|
|
|
|
'target_host_id' => 1,
|
|
|
|
|
'target_resource_pool_id' => 100,
|
|
|
|
|
'target_plan_sha256' => $plan->sha256(),
|
|
|
|
|
'name' => 'request-service-vm',
|
|
|
|
|
'confirm_new_instance' => true,
|
|
|
|
|
];
|
|
|
|
|
$jobCount = CloudJob::count();
|
|
|
|
|
$restoreCount = CloudBackupRestore::count();
|
|
|
|
|
$attemptCount = CloudBackupRestoreAttempt::count();
|
|
|
|
|
$instanceCount = Capsule::table('cloud_instances')->count();
|
|
|
|
|
|
|
|
|
|
// Simulate a process exit after the durable admission transaction but before
|
|
|
|
|
// job publication. Replaying the exact public request must finish that same
|
|
|
|
|
// generation instead of leaking or replacing its held dual-stack bundle.
|
|
|
|
|
$admitted = CloudPortableRestoreAttemptCoordinator::reserve(
|
|
|
|
|
$requestActor,
|
|
|
|
|
$backup,
|
|
|
|
|
$plan,
|
|
|
|
|
$plan->sha256(),
|
|
|
|
|
'request-service-vm',
|
|
|
|
|
(string) $requestActor['request_idempotency_key']
|
|
|
|
|
)->toArray();
|
|
|
|
|
$assert(($admitted['dispatchable'] ?? false) === true
|
|
|
|
|
&& CloudJob::count() === $jobCount
|
|
|
|
|
&& CloudBackupRestore::count() === $restoreCount + 1
|
|
|
|
|
&& CloudBackupRestoreAttempt::count() === $attemptCount + 1,
|
|
|
|
|
'Admission-only restore did not preserve one dispatchable generation.');
|
|
|
|
|
Capsule::table('cloud_backup_restore_attempts')->where(
|
|
|
|
|
'attempt_key',
|
|
|
|
|
(string) $admitted['attempt_key']
|
|
|
|
|
)->update([
|
|
|
|
|
'last_error_code' => 'portable_restore_admission_unavailable',
|
|
|
|
|
'updated_at' => time() - 360,
|
|
|
|
|
]);
|
|
|
|
|
Capsule::table('user')->whereIn('id', [9, 99])->update(['enable' => 0]);
|
|
|
|
|
$assert(CloudPortableRestoreAdmissionRecoveryService::recoverOne()
|
|
|
|
|
&& CloudJob::count() === $jobCount + 1
|
|
|
|
|
&& (string) CloudBackupRestoreAttempt::where(
|
|
|
|
|
'attempt_key',
|
|
|
|
|
(string) $admitted['attempt_key']
|
|
|
|
|
)->value('last_error_code') === '',
|
|
|
|
|
'Background recovery did not publish the exact undispatched admission.');
|
|
|
|
|
Capsule::table('user')->whereIn('id', [9, 99])->update(['enable' => 1]);
|
|
|
|
|
|
|
|
|
|
$created = CloudPortableRestoreRequestService::request(
|
|
|
|
|
$backup,
|
|
|
|
|
$requestActor,
|
|
|
|
|
$input
|
|
|
|
|
);
|
|
|
|
|
$restore = $created['restore'] ?? null;
|
|
|
|
|
$job = $created['job'] ?? null;
|
|
|
|
|
$assert($restore instanceof CloudBackupRestore
|
|
|
|
|
&& $job instanceof CloudJob
|
|
|
|
|
&& (int) $job->backup_restore_id === (int) $restore->id
|
|
|
|
|
&& (int) $job->backup_id === (int) $backup->id
|
|
|
|
|
&& $restore->target_instance_id === null
|
|
|
|
|
&& $job->instance_id === null
|
|
|
|
|
&& CloudJob::count() === $jobCount + 1
|
|
|
|
|
&& CloudBackupRestore::count() === $restoreCount + 1
|
|
|
|
|
&& CloudBackupRestoreAttempt::count() === $attemptCount + 1
|
|
|
|
|
&& Capsule::table('cloud_instances')->count() === $instanceCount,
|
|
|
|
|
'Portable restore request did not create one unmaterialized attempt/job.');
|
|
|
|
|
$attempt = CloudBackupRestoreAttempt::where(
|
|
|
|
|
'backup_restore_id',
|
|
|
|
|
(int) $restore->id
|
|
|
|
|
)->firstOrFail();
|
|
|
|
|
$claims = Capsule::table('cloud_resource_reservation_ips')
|
|
|
|
|
->where('reservation_id', (int) $attempt->reservation_id)
|
|
|
|
|
->orderBy('address_family')->get();
|
|
|
|
|
$assert($claims->count() === 2
|
|
|
|
|
&& (int) $claims[0]->address_family === 4
|
|
|
|
|
&& (int) $claims[1]->address_family === 6,
|
|
|
|
|
'Portable restore request did not reserve one atomic dual-stack bundle.');
|
|
|
|
|
|
|
|
|
|
$replay = CloudPortableRestoreRequestService::request(
|
|
|
|
|
$backup,
|
|
|
|
|
$requestActor,
|
|
|
|
|
$input
|
|
|
|
|
);
|
|
|
|
|
$assert((int) $replay['job']->id === (int) $job->id
|
|
|
|
|
&& (int) $replay['restore']->id === (int) $restore->id
|
|
|
|
|
&& CloudJob::count() === $jobCount + 1
|
|
|
|
|
&& CloudBackupRestore::count() === $restoreCount + 1
|
|
|
|
|
&& CloudBackupRestoreAttempt::count() === $attemptCount + 1,
|
|
|
|
|
'Exact public request replay created duplicate durable rows.');
|
|
|
|
|
$apiReplay = CloudBackupService::restore(
|
|
|
|
|
$backup,
|
|
|
|
|
$requestActor,
|
|
|
|
|
$input
|
|
|
|
|
);
|
|
|
|
|
$assert((int) $apiReplay['job']->id === (int) $job->id
|
|
|
|
|
&& (int) ($apiReplay['restore']['target_host_id'] ?? 0)
|
|
|
|
|
=== (int) $restore->target_host_id
|
|
|
|
|
&& CloudJob::count() === $jobCount + 1,
|
|
|
|
|
'Public backup restore entry remained closed or duplicated its request.');
|
|
|
|
|
|
|
|
|
|
$conflict = $input;
|
|
|
|
|
$conflict['name'] = 'different-request-name';
|
|
|
|
|
$rejects409(
|
|
|
|
|
static fn () => CloudPortableRestoreRequestService::request(
|
|
|
|
|
$backup,
|
|
|
|
|
$requestActor,
|
|
|
|
|
$conflict
|
|
|
|
|
),
|
|
|
|
|
'The same public idempotency key accepted a different target name.'
|
|
|
|
|
);
|
|
|
|
|
$stale = $input;
|
|
|
|
|
$stale['target_plan_sha256'] = str_repeat('0', 64);
|
|
|
|
|
$rejects409(
|
|
|
|
|
static fn () => CloudPortableRestoreRequestService::request(
|
|
|
|
|
$backup,
|
|
|
|
|
$requestActor,
|
|
|
|
|
$stale
|
|
|
|
|
),
|
|
|
|
|
'The same public idempotency key accepted a different target plan.'
|
|
|
|
|
);
|
|
|
|
|
$assert(CloudJob::count() === $jobCount + 1
|
|
|
|
|
&& CloudBackupRestore::count() === $restoreCount + 1
|
|
|
|
|
&& CloudBackupRestoreAttempt::count() === $attemptCount + 1,
|
|
|
|
|
'Rejected public request replay changed durable restore rows.');
|
|
|
|
|
|
|
|
|
|
$service = (string) file_get_contents(
|
|
|
|
|
dirname(__DIR__) . '/app/Services/Cloud/CloudPortableRestoreRequestService.php'
|
|
|
|
|
);
|
|
|
|
|
$backupService = (string) file_get_contents(
|
|
|
|
|
dirname(__DIR__) . '/app/Services/Cloud/CloudBackupService.php'
|
|
|
|
|
);
|
|
|
|
|
$assert(str_contains($service, 'CloudPortableRestoreTargetService::candidates(')
|
|
|
|
|
&& str_contains($service, 'CloudPortableRestoreAttemptCoordinator::reserve(')
|
|
|
|
|
&& str_contains($service, 'CloudPortableRestoreJobDispatcher::queue(')
|
|
|
|
|
&& !str_contains($service, 'headObject(')
|
|
|
|
|
&& !str_contains($service, 'presignedGet(')
|
|
|
|
|
&& substr_count($service, "'source_url'") === 1
|
|
|
|
|
&& str_contains($backupService,
|
|
|
|
|
'CloudPortableRestoreRequestService::request(')
|
|
|
|
|
&& !str_contains($backupService,
|
|
|
|
|
'CloudPublicNetworkTransitionGate::assertPortableRestoreReady();')
|
|
|
|
|
&& !str_contains($backupService,
|
|
|
|
|
'CloudPlacementService::reserveBackupRestore(\n $restore'),
|
|
|
|
|
'Public portable restore retained a remote side effect or legacy writer.');
|
|
|
|
|
|
|
|
|
|
echo "Portable restore request service behavior tests passed.\n";
|