You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
807 lines
28 KiB
PHP
807 lines
28 KiB
PHP
<?php
|
|
|
|
ob_start();
|
|
require __DIR__ . '/CloudPortableRestoreTargetServiceTest.php';
|
|
ob_end_clean();
|
|
|
|
use App\Models\CloudBackupRestore;
|
|
use App\Models\CloudBackupRestoreAttempt;
|
|
use App\Models\CloudInstanceBackup;
|
|
use App\Models\CloudResourceReservation;
|
|
use App\Services\Cloud\CloudBackupService;
|
|
use App\Services\Cloud\CloudError;
|
|
use App\Services\Cloud\CloudHostCapacityAdmission;
|
|
use App\Services\Cloud\CloudIpAddress;
|
|
use App\Services\Cloud\CloudPortableRestoreAttemptCoordinator;
|
|
use App\Services\Cloud\CloudPortableRestoreTargetPlan;
|
|
use App\Services\Cloud\CloudPortableRestoreTargetService;
|
|
use App\Services\Cloud\CloudPublicNetworkAllocationService;
|
|
use Illuminate\Database\Capsule\Manager as Capsule;
|
|
use Illuminate\Database\Schema\Blueprint;
|
|
|
|
$rejects409 = static function (callable $callback, string $message) use (
|
|
$assert
|
|
): void {
|
|
try {
|
|
$callback();
|
|
} catch (CloudError $error) {
|
|
$assert($error->httpStatus === 409, $message . ' (wrong status)');
|
|
return;
|
|
}
|
|
throw new RuntimeException($message);
|
|
};
|
|
|
|
// Restore the valid source row after the target-scheduler negative cases.
|
|
Capsule::table('cloud_instance_backups')->where('id', 1)->update([
|
|
'virtual_size_bytes' => 20 * 1073741824,
|
|
]);
|
|
|
|
// Extend the target-scheduler fixture with the exact mutable ledgers owned by
|
|
// the attempt coordinator and atomic public-network allocator.
|
|
$schema->create('user', static function (Blueprint $t): void {
|
|
$t->increments('id');
|
|
$t->integer('enable');
|
|
$t->boolean('is_admin')->default(false);
|
|
});
|
|
Capsule::table('user')->insert([
|
|
['id' => 9, 'enable' => 1, 'is_admin' => 0],
|
|
['id' => 99, 'enable' => 1, 'is_admin' => 1],
|
|
]);
|
|
|
|
$schema->table('cloud_instances', static function (Blueprint $t): void {
|
|
$t->string('name')->default('source-vm');
|
|
$t->integer('user_id')->nullable();
|
|
$t->integer('resource_pool_id')->nullable();
|
|
$t->integer('reservation_id')->nullable();
|
|
});
|
|
Capsule::table('cloud_instances')->where('id', 1)->update(['user_id' => 9]);
|
|
|
|
$schema->table('cloud_public_ip_pools', static function (Blueprint $t): void {
|
|
$t->string('allocation_cursor')->nullable();
|
|
});
|
|
$schema->table('cloud_public_ips', static function (Blueprint $t): void {
|
|
$t->binary('address_binary')->nullable();
|
|
$t->integer('reservation_ip_claim_id')->nullable();
|
|
$t->integer('created_at')->default(0);
|
|
$t->integer('updated_at')->default(0);
|
|
});
|
|
foreach (Capsule::table('cloud_public_ips')->orderBy('id')->get() as $row) {
|
|
Capsule::table('cloud_public_ips')->where('id', (int) $row->id)->update([
|
|
'address_binary' => CloudIpAddress::parse(
|
|
(string) $row->address
|
|
)->binary(),
|
|
'created_at' => $now,
|
|
'updated_at' => $now,
|
|
]);
|
|
}
|
|
|
|
$schema->drop('cloud_resource_reservation_ips');
|
|
$schema->drop('cloud_resource_reservations');
|
|
$schema->drop('cloud_backup_restores');
|
|
$schema->drop('cloud_jobs');
|
|
|
|
$schema->create('cloud_resource_reservations', static function (
|
|
Blueprint $t
|
|
): void {
|
|
$t->increments('id');
|
|
$t->string('reservation_key');
|
|
$t->string('operation_id');
|
|
$t->integer('order_id')->nullable();
|
|
$t->integer('backup_restore_id')->nullable();
|
|
$t->string('backup_restore_attempt_key')->nullable();
|
|
$t->integer('migration_source_debt_id')->nullable();
|
|
$t->integer('offer_id')->nullable();
|
|
$t->integer('resource_pool_id');
|
|
$t->integer('host_id');
|
|
$t->integer('user_id');
|
|
$t->integer('instance_id')->nullable();
|
|
$t->integer('network_attachment_id')->nullable();
|
|
$t->integer('public_ip_pool_id')->nullable();
|
|
$t->integer('public_ip_id')->nullable();
|
|
$t->integer('cpu_cores');
|
|
$t->integer('memory_mib');
|
|
$t->integer('disk_gib');
|
|
$t->string('state');
|
|
$t->integer('expires_at')->nullable();
|
|
$t->integer('released_at')->nullable();
|
|
$t->string('release_reason')->default('');
|
|
$t->integer('created_at');
|
|
$t->integer('updated_at');
|
|
$t->unique('operation_id');
|
|
$t->unique('backup_restore_attempt_key');
|
|
});
|
|
$schema->create('cloud_resource_reservation_ips', static function (
|
|
Blueprint $t
|
|
): void {
|
|
$t->increments('id');
|
|
$t->integer('reservation_id');
|
|
$t->integer('address_family');
|
|
$t->integer('claim_generation');
|
|
$t->integer('active_claim')->nullable();
|
|
$t->integer('network_attachment_id');
|
|
$t->integer('public_ip_pool_id');
|
|
$t->integer('public_ip_id');
|
|
$t->string('operation_id');
|
|
$t->string('state');
|
|
$t->integer('created_at');
|
|
$t->integer('updated_at');
|
|
$t->integer('released_at')->nullable();
|
|
$t->string('release_reason')->default('');
|
|
$t->unique(['reservation_id', 'address_family', 'active_claim']);
|
|
});
|
|
$schema->create('cloud_backup_restores', static function (Blueprint $t): void {
|
|
$t->increments('id');
|
|
$t->string('restore_key')->unique();
|
|
$t->integer('backup_id');
|
|
$t->integer('requested_by_user_id');
|
|
$t->string('request_idempotency_key');
|
|
$t->string('request_fingerprint');
|
|
$t->text('selected_plan_json');
|
|
$t->string('selected_plan_sha256');
|
|
$t->integer('target_host_id');
|
|
$t->integer('target_instance_id')->nullable();
|
|
$t->integer('reservation_id')->nullable();
|
|
$t->integer('job_id')->nullable();
|
|
$t->string('operation_id');
|
|
$t->string('new_uuid');
|
|
$t->string('new_name');
|
|
$t->string('state');
|
|
$t->string('error_code')->default('');
|
|
$t->integer('ready_at')->nullable();
|
|
$t->integer('created_at');
|
|
$t->integer('updated_at');
|
|
$t->unique(['requested_by_user_id', 'request_idempotency_key']);
|
|
});
|
|
$schema->create('cloud_backup_restore_attempts', static function (
|
|
Blueprint $t
|
|
): void {
|
|
$t->increments('id');
|
|
$t->string('attempt_key')->unique();
|
|
$t->integer('backup_restore_id');
|
|
$t->integer('generation');
|
|
$t->string('operation_id')->unique();
|
|
$t->integer('job_id')->nullable();
|
|
$t->integer('reservation_id')->unique();
|
|
$t->integer('target_host_id');
|
|
$t->string('target_uuid');
|
|
$t->string('target_name');
|
|
$t->text('target_plan_json');
|
|
$t->string('target_plan_sha256');
|
|
$t->integer('target_network_attachment_id');
|
|
$t->text('target_identity_json');
|
|
$t->string('target_identity_sha256');
|
|
$t->string('target_claim_set_sha256');
|
|
$t->text('target_public_network_json');
|
|
$t->string('target_public_network_sha256');
|
|
$t->string('state');
|
|
$t->string('effect_resolution');
|
|
$t->string('evidence_resolution')->default('not_applicable');
|
|
$t->text('evidence_json')->nullable();
|
|
$t->string('evidence_sha256')->nullable();
|
|
$t->integer('evidence_recorded_at')->nullable();
|
|
$t->integer('active_attempt')->nullable();
|
|
$t->string('last_error_code')->default('');
|
|
$t->string('finalization_error_code')->default('');
|
|
$t->integer('effect_resolved_at')->nullable();
|
|
$t->integer('terminal_at')->nullable();
|
|
$t->integer('created_at');
|
|
$t->integer('updated_at');
|
|
$t->unique(['backup_restore_id', 'generation']);
|
|
$t->unique(['backup_restore_id', 'active_attempt']);
|
|
});
|
|
$schema->create('cloud_jobs', static function (Blueprint $t): void {
|
|
$t->increments('id');
|
|
$t->integer('user_id');
|
|
$t->integer('backup_id')->nullable();
|
|
$t->integer('backup_restore_id')->nullable();
|
|
$t->integer('host_id')->nullable();
|
|
$t->string('kind');
|
|
$t->string('operation_id');
|
|
$t->string('phase');
|
|
$t->string('outcome')->nullable();
|
|
$t->string('effect_resolution');
|
|
$t->integer('terminal_at')->nullable();
|
|
$t->integer('finalization_incident')->default(0);
|
|
});
|
|
$schema->create('cloud_backup_recoveries', static function (Blueprint $t): void {
|
|
$t->increments('id');
|
|
$t->integer('backup_id');
|
|
$t->string('state');
|
|
});
|
|
$schema->create('cloud_deployment_snapshots', static function (
|
|
Blueprint $t
|
|
): void {
|
|
$t->increments('id');
|
|
$t->integer('backup_id');
|
|
$t->string('state');
|
|
});
|
|
$schema->create('cloud_instance_migrations', static function (
|
|
Blueprint $t
|
|
): void {
|
|
$t->increments('id');
|
|
$t->integer('source_host_id');
|
|
$t->integer('source_reservation_id');
|
|
$t->integer('target_host_id');
|
|
$t->integer('target_reservation_id');
|
|
$t->integer('instance_id');
|
|
$t->integer('restore_id');
|
|
$t->string('state');
|
|
});
|
|
|
|
$backup = CloudInstanceBackup::findOrFail(1);
|
|
$plans = CloudPortableRestoreTargetService::candidates($backup);
|
|
$plan = array_values(array_filter(
|
|
$plans,
|
|
static fn (CloudPortableRestoreTargetPlan $candidate): bool =>
|
|
$candidate->toArray()['target']['host_id'] === 2
|
|
&& $candidate->toArray()['target']['resource_pool_id'] === 200
|
|
))[0] ?? null;
|
|
$assert($plan instanceof CloudPortableRestoreTargetPlan,
|
|
'Host-two portable restore plan is missing.');
|
|
|
|
$actor = [
|
|
'user_id' => 99,
|
|
'is_admin' => true,
|
|
'is_super_admin' => true,
|
|
'can_manage' => true,
|
|
];
|
|
$requestKey = 'restore-request-0001';
|
|
$first = CloudPortableRestoreAttemptCoordinator::reserve(
|
|
$actor,
|
|
$backup,
|
|
$plan,
|
|
$plan->sha256(),
|
|
'restored-vm',
|
|
$requestKey
|
|
)->toArray();
|
|
$assert($first['generation'] === 1 && $first['dispatchable'] === true
|
|
&& is_array($first['dispatch_body'])
|
|
&& Capsule::table('cloud_resource_reservations')
|
|
->where('id', $first['reservation_id'])->value('user_id') === 9
|
|
&& Capsule::table('cloud_resource_reservation_ips')
|
|
->where('reservation_id', $first['reservation_id'])->count() === 2,
|
|
'Initial restore did not atomically reserve owner capacity and dual stack.');
|
|
|
|
$same = CloudPortableRestoreAttemptCoordinator::reserve(
|
|
$actor, $backup, $plan, $plan->sha256(), 'restored-vm', $requestKey
|
|
)->toArray();
|
|
$assert($same['attempt_key'] === $first['attempt_key']
|
|
&& $same['reservation_id'] === $first['reservation_id'],
|
|
'Exact reserve replay created another attempt.');
|
|
$rejects409(
|
|
static fn () => CloudPortableRestoreAttemptCoordinator::reserve(
|
|
$actor, $backup, $plan, $plan->sha256(), 'different-vm', $requestKey
|
|
),
|
|
'Same idempotency key accepted a different restore request.'
|
|
);
|
|
|
|
// Exact replay is a lookup of already-frozen evidence. Current eligibility
|
|
// drift must neither reject it nor generate a replacement attempt/bundle.
|
|
Capsule::table('cloud_instance_backups')->where('id', 1)->update([
|
|
'state' => 'failed',
|
|
]);
|
|
Capsule::table('cloud_host_agents')->where('host_id', 2)->update([
|
|
'enabled' => 0,
|
|
'revision' => 99,
|
|
'tested_revision' => 98,
|
|
'last_test_status' => 'failed',
|
|
]);
|
|
Capsule::table('cloud_host_storage_configurations')->where('host_id', 2)
|
|
->update(['revision' => 33, 'state' => 'pending']);
|
|
Capsule::table('cloud_host_resource_pool_memberships')
|
|
->where('host_id', 2)->where('resource_pool_id', 200)->update([
|
|
'scheduler_state' => 'disabled',
|
|
'revision' => 13,
|
|
]);
|
|
$eligibilityReplay = CloudPortableRestoreAttemptCoordinator::reserve(
|
|
$actor,
|
|
CloudInstanceBackup::findOrFail(1),
|
|
$plan,
|
|
$plan->sha256(),
|
|
'restored-vm',
|
|
$requestKey
|
|
)->toArray();
|
|
$assert($eligibilityReplay['attempt_key'] === $first['attempt_key']
|
|
&& $eligibilityReplay['reservation_id'] === $first['reservation_id'],
|
|
'Current backup or target eligibility drift broke exact replay.');
|
|
Capsule::table('cloud_instance_backups')->where('id', 1)->update([
|
|
'state' => 'ready',
|
|
]);
|
|
Capsule::table('cloud_host_agents')->where('host_id', 2)->update([
|
|
'enabled' => 1,
|
|
'revision' => 42,
|
|
'tested_revision' => 42,
|
|
'last_test_status' => 'healthy',
|
|
]);
|
|
Capsule::table('cloud_host_storage_configurations')->where('host_id', 2)
|
|
->update(['revision' => 32, 'state' => 'applied']);
|
|
Capsule::table('cloud_host_resource_pool_memberships')
|
|
->where('host_id', 2)->where('resource_pool_id', 200)->update([
|
|
'scheduler_state' => 'active',
|
|
'revision' => 12,
|
|
]);
|
|
|
|
// A torn durable claim set must never be silently replaced during replay.
|
|
$claimRowsBefore = Capsule::table('cloud_resource_reservation_ips')->count();
|
|
$ipIdsBefore = Capsule::table('cloud_public_ips')->orderBy('id')
|
|
->pluck('id')->all();
|
|
Capsule::connection()->beginTransaction();
|
|
try {
|
|
Capsule::table('cloud_resource_reservation_ips')
|
|
->where('reservation_id', $first['reservation_id'])->delete();
|
|
$rejects409(
|
|
static fn () => CloudPortableRestoreAttemptCoordinator::reserve(
|
|
$actor,
|
|
CloudInstanceBackup::findOrFail(1),
|
|
$plan,
|
|
$plan->sha256(),
|
|
'restored-vm',
|
|
$requestKey
|
|
),
|
|
'Replay allocated a replacement bundle for missing durable claims.'
|
|
);
|
|
$assert(Capsule::table('cloud_resource_reservation_ips')->count()
|
|
=== $claimRowsBefore - 2
|
|
&& Capsule::table('cloud_public_ips')->orderBy('id')
|
|
->pluck('id')->all() === $ipIdsBefore,
|
|
'Failed replay inserted replacement claims or public addresses.');
|
|
} finally {
|
|
Capsule::connection()->rollBack();
|
|
}
|
|
|
|
// Volatile capacity changes must not invalidate an already-reserved replay.
|
|
Capsule::table('cloud_resource_reservations')->insert([
|
|
'reservation_key' => str_repeat('1', 32),
|
|
'operation_id' => str_repeat('2', 32),
|
|
'order_id' => 77,
|
|
'backup_restore_id' => null,
|
|
'backup_restore_attempt_key' => null,
|
|
'migration_source_debt_id' => null,
|
|
'offer_id' => 88,
|
|
'resource_pool_id' => 200,
|
|
'host_id' => 2,
|
|
'user_id' => 9,
|
|
'instance_id' => null,
|
|
'network_attachment_id' => null,
|
|
'public_ip_pool_id' => null,
|
|
'public_ip_id' => null,
|
|
'cpu_cores' => 1,
|
|
'memory_mib' => 256,
|
|
'disk_gib' => 1,
|
|
'state' => 'held',
|
|
'expires_at' => null,
|
|
'released_at' => null,
|
|
'release_reason' => '',
|
|
'created_at' => $now,
|
|
'updated_at' => $now,
|
|
]);
|
|
$capacityReplay = CloudPortableRestoreAttemptCoordinator::reserve(
|
|
$actor, $backup, $plan, $plan->sha256(), 'restored-vm', $requestKey
|
|
)->toArray();
|
|
$assert($capacityReplay['attempt_key'] === $first['attempt_key'],
|
|
'Volatile capacity drift broke exact replay.');
|
|
|
|
$attempt = CloudBackupRestoreAttempt::where(
|
|
'attempt_key',
|
|
$first['attempt_key']
|
|
)->firstOrFail();
|
|
$reservation = CloudResourceReservation::findOrFail(
|
|
$first['reservation_id']
|
|
);
|
|
|
|
// Reservation/claim fencing is not a dispatchable state. A stale reserved
|
|
// attempt must fail closed instead of returning a body that could be sent.
|
|
$reservation->state = 'uncertain';
|
|
$reservation->save();
|
|
Capsule::table('cloud_resource_reservation_ips')
|
|
->where('reservation_id', (int) $reservation->id)->update([
|
|
'state' => 'uncertain', 'updated_at' => $now,
|
|
]);
|
|
Capsule::table('cloud_public_ips')
|
|
->where('reservation_id', (int) $reservation->id)->update([
|
|
'state' => 'uncertain', 'updated_at' => $now,
|
|
]);
|
|
$rejects409(
|
|
static fn () => CloudPortableRestoreAttemptCoordinator::reserve(
|
|
$actor, $backup, $plan, $plan->sha256(), 'restored-vm', $requestKey
|
|
),
|
|
'Reserved attempt accepted an uncertain fenced resource bundle.'
|
|
);
|
|
$reservation->state = 'held';
|
|
$reservation->save();
|
|
Capsule::table('cloud_resource_reservation_ips')
|
|
->where('reservation_id', (int) $reservation->id)->update([
|
|
'state' => 'reserved', 'updated_at' => $now,
|
|
]);
|
|
Capsule::table('cloud_public_ips')
|
|
->where('reservation_id', (int) $reservation->id)->update([
|
|
'state' => 'reserved', 'updated_at' => $now,
|
|
]);
|
|
|
|
$attempt->state = 'effect_unknown';
|
|
$attempt->effect_resolution = 'unknown';
|
|
$attempt->job_id = 500;
|
|
$attempt->save();
|
|
$reservation->state = 'uncertain';
|
|
$reservation->save();
|
|
Capsule::table('cloud_resource_reservation_ips')
|
|
->where('reservation_id', (int) $reservation->id)->update([
|
|
'state' => 'uncertain', 'updated_at' => $now,
|
|
]);
|
|
Capsule::table('cloud_public_ips')
|
|
->where('reservation_id', (int) $reservation->id)->update([
|
|
'state' => 'uncertain', 'updated_at' => $now,
|
|
]);
|
|
$unknown = CloudPortableRestoreAttemptCoordinator::reserve(
|
|
$actor, $backup, $plan, $plan->sha256(), 'restored-vm', $requestKey
|
|
)->toArray();
|
|
$assert($unknown['dispatchable'] === false
|
|
&& $unknown['dispatch_body'] === null,
|
|
'Unknown-effect replay exposed a dispatch instruction.');
|
|
|
|
Capsule::table('cloud_instance_backups')->where('id', 1)->update([
|
|
'state' => 'failed',
|
|
]);
|
|
$unknownAfterBackupDrift = CloudPortableRestoreAttemptCoordinator::reserve(
|
|
$actor,
|
|
CloudInstanceBackup::findOrFail(1),
|
|
$plan,
|
|
$plan->sha256(),
|
|
'restored-vm',
|
|
$requestKey
|
|
)->toArray();
|
|
$assert($unknownAfterBackupDrift['attempt_key'] === $first['attempt_key']
|
|
&& $unknownAfterBackupDrift['dispatchable'] === false
|
|
&& $unknownAfterBackupDrift['dispatch_body'] === null,
|
|
'Backup-state drift broke stable unknown-effect replay.');
|
|
Capsule::table('cloud_instance_backups')->where('id', 1)->update([
|
|
'state' => 'ready',
|
|
]);
|
|
|
|
// The attempt row, not the mutable parent projection, protects the backup.
|
|
Capsule::table('cloud_backup_restores')->where('id', $first['restore_id'])
|
|
->update(['state' => 'failed']);
|
|
$guard = new ReflectionMethod(
|
|
CloudBackupService::class,
|
|
'assertBackupDeletionAvailable'
|
|
);
|
|
$rejects409(
|
|
static fn () => $guard->invoke(null, CloudInstanceBackup::findOrFail(1)),
|
|
'An active attempt did not protect its source backup.'
|
|
);
|
|
|
|
CloudPublicNetworkAllocationService::releaseBundle(
|
|
(int) $reservation->id,
|
|
(string) $attempt->operation_id,
|
|
'restore_not_applied',
|
|
true
|
|
);
|
|
$releasedAt = $now + 10;
|
|
$reservation->state = 'released';
|
|
$reservation->released_at = $releasedAt;
|
|
$reservation->release_reason = 'restore_not_applied';
|
|
$reservation->save();
|
|
$attempt->state = 'failed';
|
|
$attempt->effect_resolution = 'not_applied';
|
|
$attempt->active_attempt = null;
|
|
$attempt->effect_resolved_at = $releasedAt;
|
|
$attempt->terminal_at = $releasedAt;
|
|
$attempt->save();
|
|
Capsule::table('cloud_jobs')->insert([
|
|
'id' => 500,
|
|
'user_id' => 9,
|
|
'backup_id' => 1,
|
|
'backup_restore_id' => $first['restore_id'],
|
|
'host_id' => 2,
|
|
'kind' => 'backup_restore',
|
|
'operation_id' => (string) $attempt->operation_id,
|
|
'phase' => 'terminal',
|
|
'outcome' => 'failed',
|
|
'effect_resolution' => 'not_applied',
|
|
'terminal_at' => $releasedAt,
|
|
'finalization_incident' => 0,
|
|
]);
|
|
|
|
// A fully released terminal not-applied attempt no longer protects the object.
|
|
$guard->invoke(null, CloudInstanceBackup::findOrFail(1));
|
|
|
|
// Retry authority is the exact terminal job evidence, including ownership and
|
|
// finalization status. Each mutation must fail without creating a generation.
|
|
$jobEvidenceMutations = [
|
|
['kind', 'wrong_kind', 'backup_restore'],
|
|
['host_id', 1, 2],
|
|
['user_id', 99, 9],
|
|
['backup_restore_id', 999, $first['restore_id']],
|
|
['operation_id', str_repeat('9', 32), (string) $attempt->operation_id],
|
|
['finalization_incident', 1, 0],
|
|
];
|
|
foreach ($jobEvidenceMutations as [$field, $invalid, $valid]) {
|
|
Capsule::table('cloud_jobs')->where('id', 500)->update([$field => $invalid]);
|
|
$rejects409(
|
|
static fn () => CloudPortableRestoreAttemptCoordinator::retry(
|
|
$actor,
|
|
$backup,
|
|
$plan,
|
|
$plan->sha256(),
|
|
'restored-vm',
|
|
$requestKey,
|
|
(string) $attempt->attempt_key
|
|
),
|
|
'Retry accepted mismatched terminal job evidence: ' . $field . '.'
|
|
);
|
|
$assert(CloudBackupRestoreAttempt::where(
|
|
'backup_restore_id',
|
|
$first['restore_id']
|
|
)->count() === 1, 'Rejected retry created another attempt row.');
|
|
Capsule::table('cloud_jobs')->where('id', 500)->update([$field => $valid]);
|
|
}
|
|
|
|
// Released reservation and address claims remain part of the predecessor's
|
|
// exact operation identity. A torn terminal ledger must never authorize a new
|
|
// dispatch generation even when the job itself says not_applied.
|
|
$ledgerOperationMutations = [
|
|
['cloud_resource_reservations', null],
|
|
['cloud_resource_reservation_ips', 4],
|
|
['cloud_resource_reservation_ips', 6],
|
|
];
|
|
foreach ($ledgerOperationMutations as [$table, $family]) {
|
|
$query = Capsule::table($table)->where(
|
|
'reservation_id',
|
|
(int) $reservation->id
|
|
);
|
|
if ($table === 'cloud_resource_reservations') {
|
|
$query = Capsule::table($table)->where('id', (int) $reservation->id);
|
|
} else {
|
|
$query->where('address_family', $family);
|
|
}
|
|
$query->update(['operation_id' => str_repeat('8', 32)]);
|
|
$rejects409(
|
|
static fn () => CloudPortableRestoreAttemptCoordinator::retry(
|
|
$actor,
|
|
$backup,
|
|
$plan,
|
|
$plan->sha256(),
|
|
'restored-vm',
|
|
$requestKey,
|
|
(string) $attempt->attempt_key
|
|
),
|
|
'Retry accepted torn terminal ledger operation identity: '
|
|
. $table . ':' . (string) $family . '.'
|
|
);
|
|
$assert(CloudBackupRestoreAttempt::where(
|
|
'backup_restore_id',
|
|
$first['restore_id']
|
|
)->count() === 1, 'Rejected ledger retry created another attempt row.');
|
|
$query->update(['operation_id' => (string) $attempt->operation_id]);
|
|
}
|
|
|
|
$ipv4Claim = Capsule::table('cloud_resource_reservation_ips')
|
|
->where('reservation_id', (int) $reservation->id)
|
|
->where('address_family', 4)->first();
|
|
$assert(is_object($ipv4Claim), 'Released IPv4 claim fixture is missing.');
|
|
Capsule::table('cloud_resource_reservation_ips')
|
|
->where('id', (int) $ipv4Claim->id)
|
|
->update(['public_ip_pool_id' => (int) $ipv4Claim->public_ip_pool_id + 1]);
|
|
$rejects409(
|
|
static fn () => CloudPortableRestoreAttemptCoordinator::retry(
|
|
$actor,
|
|
$backup,
|
|
$plan,
|
|
$plan->sha256(),
|
|
'restored-vm',
|
|
$requestKey,
|
|
(string) $attempt->attempt_key
|
|
),
|
|
'Retry accepted a released claim that changed frozen claim-set identity.'
|
|
);
|
|
$assert(CloudBackupRestoreAttempt::where(
|
|
'backup_restore_id',
|
|
$first['restore_id']
|
|
)->count() === 1, 'Rejected claim identity created another attempt row.');
|
|
Capsule::table('cloud_resource_reservation_ips')
|
|
->where('id', (int) $ipv4Claim->id)
|
|
->update(['public_ip_pool_id' => (int) $ipv4Claim->public_ip_pool_id]);
|
|
|
|
foreach ([
|
|
['claim_generation', (int) $ipv4Claim->claim_generation + 1],
|
|
['public_ip_id', (int) $ipv4Claim->public_ip_id + 999999],
|
|
] as [$field, $invalid]) {
|
|
Capsule::table('cloud_resource_reservation_ips')
|
|
->where('id', (int) $ipv4Claim->id)->update([$field => $invalid]);
|
|
$rejects409(
|
|
static fn () => CloudPortableRestoreAttemptCoordinator::retry(
|
|
$actor,
|
|
$backup,
|
|
$plan,
|
|
$plan->sha256(),
|
|
'restored-vm',
|
|
$requestKey,
|
|
(string) $attempt->attempt_key
|
|
),
|
|
'Retry accepted changed released claim identity: ' . $field . '.'
|
|
);
|
|
$assert(CloudBackupRestoreAttempt::where(
|
|
'backup_restore_id',
|
|
$first['restore_id']
|
|
)->count() === 1, 'Rejected claim mutation created another attempt row.');
|
|
Capsule::table('cloud_resource_reservation_ips')
|
|
->where('id', (int) $ipv4Claim->id)
|
|
->update([$field => $ipv4Claim->{$field}]);
|
|
}
|
|
|
|
// A released claim is not sufficient when its concrete address still points
|
|
// at the predecessor. The address may later be legitimately reused by another
|
|
// operation, but all ownership references to this operation must be gone.
|
|
Capsule::table('cloud_public_ips')
|
|
->where('id', (int) $ipv4Claim->public_ip_id)->update([
|
|
'state' => 'reserved',
|
|
'reservation_id' => (int) $reservation->id,
|
|
'reservation_ip_claim_id' => (int) $ipv4Claim->id,
|
|
'instance_id' => null,
|
|
'operation_id' => (string) $attempt->operation_id,
|
|
]);
|
|
$rejects409(
|
|
static fn () => CloudPortableRestoreAttemptCoordinator::retry(
|
|
$actor,
|
|
$backup,
|
|
$plan,
|
|
$plan->sha256(),
|
|
'restored-vm',
|
|
$requestKey,
|
|
(string) $attempt->attempt_key
|
|
),
|
|
'Retry accepted a public address that retained predecessor ownership.'
|
|
);
|
|
$assert(CloudBackupRestoreAttempt::where(
|
|
'backup_restore_id',
|
|
$first['restore_id']
|
|
)->count() === 1, 'Residual address ownership created another attempt row.');
|
|
Capsule::table('cloud_public_ips')
|
|
->where('id', (int) $ipv4Claim->public_ip_id)->update([
|
|
'state' => 'available',
|
|
'reservation_id' => null,
|
|
'reservation_ip_claim_id' => null,
|
|
'instance_id' => null,
|
|
'operation_id' => null,
|
|
]);
|
|
|
|
$second = CloudPortableRestoreAttemptCoordinator::retry(
|
|
$actor,
|
|
$backup,
|
|
$plan,
|
|
$plan->sha256(),
|
|
'restored-vm',
|
|
$requestKey,
|
|
(string) $attempt->attempt_key
|
|
)->toArray();
|
|
$assert($second['generation'] === 2
|
|
&& $second['attempt_key'] !== $first['attempt_key']
|
|
&& $second['reservation_id'] !== $first['reservation_id']
|
|
&& $second['dispatchable'] === true,
|
|
'Explicit not-applied retry did not create one fresh generation.');
|
|
$retryReplay = CloudPortableRestoreAttemptCoordinator::retry(
|
|
$actor,
|
|
$backup,
|
|
$plan,
|
|
$plan->sha256(),
|
|
'restored-vm',
|
|
$requestKey,
|
|
(string) $attempt->attempt_key
|
|
)->toArray();
|
|
$assert($retryReplay['attempt_key'] === $second['attempt_key']
|
|
&& $retryReplay['generation'] === 2,
|
|
'Retry replay skipped to another generation or failed idempotency.');
|
|
|
|
// A queued generation is already owned by a future job and cannot be emitted
|
|
// as another dispatch instruction.
|
|
$secondAttempt = CloudBackupRestoreAttempt::where(
|
|
'attempt_key',
|
|
$second['attempt_key']
|
|
)->firstOrFail();
|
|
$secondAttempt->state = 'queued';
|
|
$secondAttempt->job_id = 501;
|
|
$secondAttempt->save();
|
|
$queued = CloudPortableRestoreAttemptCoordinator::reserve(
|
|
$actor, $backup, $plan, $plan->sha256(), 'restored-vm', $requestKey
|
|
)->toArray();
|
|
$assert($queued['dispatchable'] === false && $queued['dispatch_body'] === null,
|
|
'Queued replay exposed a second dispatch instruction.');
|
|
|
|
// A completed attempt replays only its durable result and never another Agent
|
|
// instruction. Model the already-finalized instance/network ownership chain.
|
|
$secondReservation = CloudResourceReservation::findOrFail(
|
|
$second['reservation_id']
|
|
);
|
|
Capsule::table('cloud_instances')->insert([
|
|
'id' => 2,
|
|
'host_id' => 2,
|
|
'status' => 'active',
|
|
'spec_json' => json_encode([
|
|
'public_network' => $second['dispatch_body']['public_network'],
|
|
], JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES),
|
|
'name' => 'restored-vm',
|
|
'user_id' => 9,
|
|
'resource_pool_id' => 200,
|
|
'reservation_id' => (int) $secondReservation->id,
|
|
]);
|
|
CloudPublicNetworkAllocationService::commitBundle(
|
|
(int) $secondReservation->id,
|
|
2,
|
|
(string) $secondAttempt->operation_id
|
|
);
|
|
$secondReservation->instance_id = 2;
|
|
$secondReservation->state = 'committed';
|
|
$secondReservation->updated_at = $now + 20;
|
|
$secondReservation->save();
|
|
$secondAttempt->state = 'succeeded';
|
|
$secondAttempt->effect_resolution = 'applied';
|
|
$secondAttempt->active_attempt = null;
|
|
$secondAttempt->effect_resolved_at = $now + 20;
|
|
$secondAttempt->terminal_at = $now + 20;
|
|
$secondAttempt->save();
|
|
Capsule::table('cloud_backup_restores')->where('id', $first['restore_id'])
|
|
->update([
|
|
'target_instance_id' => 2,
|
|
'state' => 'ready',
|
|
'ready_at' => $now + 20,
|
|
]);
|
|
$succeeded = CloudPortableRestoreAttemptCoordinator::reserve(
|
|
$actor, $backup, $plan, $plan->sha256(), 'restored-vm', $requestKey
|
|
)->toArray();
|
|
$assert($succeeded['attempt_key'] === $second['attempt_key']
|
|
&& $succeeded['state'] === 'succeeded'
|
|
&& $succeeded['dispatchable'] === false
|
|
&& $succeeded['dispatch_body'] === null,
|
|
'Succeeded replay exposed another dispatch instruction.');
|
|
|
|
$policy = (object) [
|
|
'updated_at' => 1,
|
|
'configured_cpu_cores' => 8,
|
|
'configured_memory_mib' => 8192,
|
|
'configured_disk_gib' => 100,
|
|
'safety_cpu_cores' => 0,
|
|
'safety_memory_mib' => 0,
|
|
'safety_disk_gib' => 0,
|
|
'max_instances' => 10,
|
|
];
|
|
$resources = ['cpu_cores' => 1, 'memory_mib' => 512, 'disk_gib' => 10];
|
|
$validUsage = [
|
|
'unknown' => 0,
|
|
'instance_count' => 0,
|
|
'cpu_sum' => 0,
|
|
'memory_sum' => 0,
|
|
'disk_sum' => 0,
|
|
];
|
|
$missingMaximum = clone $policy;
|
|
unset($missingMaximum->max_instances);
|
|
$missingTimestamp = clone $policy;
|
|
unset($missingTimestamp->updated_at);
|
|
$assert(!CloudHostCapacityAdmission::allows($policy, [], $resources)
|
|
&& !CloudHostCapacityAdmission::allows($policy, [
|
|
'unknown' => 0,
|
|
'instance_count' => -1,
|
|
'cpu_sum' => 0,
|
|
'memory_sum' => 0,
|
|
'disk_sum' => 0,
|
|
], $resources)
|
|
&& !CloudHostCapacityAdmission::allows($policy, [
|
|
'unknown' => 1,
|
|
'instance_count' => 0,
|
|
'cpu_sum' => 0,
|
|
'memory_sum' => 0,
|
|
'disk_sum' => 0,
|
|
], $resources)
|
|
&& !CloudHostCapacityAdmission::allows(
|
|
$missingMaximum,
|
|
$validUsage,
|
|
$resources
|
|
)
|
|
&& !CloudHostCapacityAdmission::allows(
|
|
$missingTimestamp,
|
|
$validUsage,
|
|
$resources
|
|
),
|
|
'Capacity admission accepted incomplete policy or invalid usage.');
|
|
|
|
echo "Portable restore attempt coordinator behavior tests passed.\n";
|