You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
virt/web/tests/CloudPortableRestoreAttempt...

807 lines
28 KiB
PHP

<?php
ob_start();
require __DIR__ . '/CloudPortableRestoreTargetServiceTest.php';
ob_end_clean();
use App\Models\CloudBackupRestore;
use App\Models\CloudBackupRestoreAttempt;
use App\Models\CloudInstanceBackup;
use App\Models\CloudResourceReservation;
use App\Services\Cloud\CloudBackupService;
use App\Services\Cloud\CloudError;
use App\Services\Cloud\CloudHostCapacityAdmission;
use App\Services\Cloud\CloudIpAddress;
use App\Services\Cloud\CloudPortableRestoreAttemptCoordinator;
use App\Services\Cloud\CloudPortableRestoreTargetPlan;
use App\Services\Cloud\CloudPortableRestoreTargetService;
use App\Services\Cloud\CloudPublicNetworkAllocationService;
use Illuminate\Database\Capsule\Manager as Capsule;
use Illuminate\Database\Schema\Blueprint;
$rejects409 = static function (callable $callback, string $message) use (
$assert
): void {
try {
$callback();
} catch (CloudError $error) {
$assert($error->httpStatus === 409, $message . ' (wrong status)');
return;
}
throw new RuntimeException($message);
};
// Restore the valid source row after the target-scheduler negative cases.
Capsule::table('cloud_instance_backups')->where('id', 1)->update([
'virtual_size_bytes' => 20 * 1073741824,
]);
// Extend the target-scheduler fixture with the exact mutable ledgers owned by
// the attempt coordinator and atomic public-network allocator.
$schema->create('user', static function (Blueprint $t): void {
$t->increments('id');
$t->integer('enable');
$t->boolean('is_admin')->default(false);
});
Capsule::table('user')->insert([
['id' => 9, 'enable' => 1, 'is_admin' => 0],
['id' => 99, 'enable' => 1, 'is_admin' => 1],
]);
$schema->table('cloud_instances', static function (Blueprint $t): void {
$t->string('name')->default('source-vm');
$t->integer('user_id')->nullable();
$t->integer('resource_pool_id')->nullable();
$t->integer('reservation_id')->nullable();
});
Capsule::table('cloud_instances')->where('id', 1)->update(['user_id' => 9]);
$schema->table('cloud_public_ip_pools', static function (Blueprint $t): void {
$t->string('allocation_cursor')->nullable();
});
$schema->table('cloud_public_ips', static function (Blueprint $t): void {
$t->binary('address_binary')->nullable();
$t->integer('reservation_ip_claim_id')->nullable();
$t->integer('created_at')->default(0);
$t->integer('updated_at')->default(0);
});
foreach (Capsule::table('cloud_public_ips')->orderBy('id')->get() as $row) {
Capsule::table('cloud_public_ips')->where('id', (int) $row->id)->update([
'address_binary' => CloudIpAddress::parse(
(string) $row->address
)->binary(),
'created_at' => $now,
'updated_at' => $now,
]);
}
$schema->drop('cloud_resource_reservation_ips');
$schema->drop('cloud_resource_reservations');
$schema->drop('cloud_backup_restores');
$schema->drop('cloud_jobs');
$schema->create('cloud_resource_reservations', static function (
Blueprint $t
): void {
$t->increments('id');
$t->string('reservation_key');
$t->string('operation_id');
$t->integer('order_id')->nullable();
$t->integer('backup_restore_id')->nullable();
$t->string('backup_restore_attempt_key')->nullable();
$t->integer('migration_source_debt_id')->nullable();
$t->integer('offer_id')->nullable();
$t->integer('resource_pool_id');
$t->integer('host_id');
$t->integer('user_id');
$t->integer('instance_id')->nullable();
$t->integer('network_attachment_id')->nullable();
$t->integer('public_ip_pool_id')->nullable();
$t->integer('public_ip_id')->nullable();
$t->integer('cpu_cores');
$t->integer('memory_mib');
$t->integer('disk_gib');
$t->string('state');
$t->integer('expires_at')->nullable();
$t->integer('released_at')->nullable();
$t->string('release_reason')->default('');
$t->integer('created_at');
$t->integer('updated_at');
$t->unique('operation_id');
$t->unique('backup_restore_attempt_key');
});
$schema->create('cloud_resource_reservation_ips', static function (
Blueprint $t
): void {
$t->increments('id');
$t->integer('reservation_id');
$t->integer('address_family');
$t->integer('claim_generation');
$t->integer('active_claim')->nullable();
$t->integer('network_attachment_id');
$t->integer('public_ip_pool_id');
$t->integer('public_ip_id');
$t->string('operation_id');
$t->string('state');
$t->integer('created_at');
$t->integer('updated_at');
$t->integer('released_at')->nullable();
$t->string('release_reason')->default('');
$t->unique(['reservation_id', 'address_family', 'active_claim']);
});
$schema->create('cloud_backup_restores', static function (Blueprint $t): void {
$t->increments('id');
$t->string('restore_key')->unique();
$t->integer('backup_id');
$t->integer('requested_by_user_id');
$t->string('request_idempotency_key');
$t->string('request_fingerprint');
$t->text('selected_plan_json');
$t->string('selected_plan_sha256');
$t->integer('target_host_id');
$t->integer('target_instance_id')->nullable();
$t->integer('reservation_id')->nullable();
$t->integer('job_id')->nullable();
$t->string('operation_id');
$t->string('new_uuid');
$t->string('new_name');
$t->string('state');
$t->string('error_code')->default('');
$t->integer('ready_at')->nullable();
$t->integer('created_at');
$t->integer('updated_at');
$t->unique(['requested_by_user_id', 'request_idempotency_key']);
});
$schema->create('cloud_backup_restore_attempts', static function (
Blueprint $t
): void {
$t->increments('id');
$t->string('attempt_key')->unique();
$t->integer('backup_restore_id');
$t->integer('generation');
$t->string('operation_id')->unique();
$t->integer('job_id')->nullable();
$t->integer('reservation_id')->unique();
$t->integer('target_host_id');
$t->string('target_uuid');
$t->string('target_name');
$t->text('target_plan_json');
$t->string('target_plan_sha256');
$t->integer('target_network_attachment_id');
$t->text('target_identity_json');
$t->string('target_identity_sha256');
$t->string('target_claim_set_sha256');
$t->text('target_public_network_json');
$t->string('target_public_network_sha256');
$t->string('state');
$t->string('effect_resolution');
$t->string('evidence_resolution')->default('not_applicable');
$t->text('evidence_json')->nullable();
$t->string('evidence_sha256')->nullable();
$t->integer('evidence_recorded_at')->nullable();
$t->integer('active_attempt')->nullable();
$t->string('last_error_code')->default('');
$t->string('finalization_error_code')->default('');
$t->integer('effect_resolved_at')->nullable();
$t->integer('terminal_at')->nullable();
$t->integer('created_at');
$t->integer('updated_at');
$t->unique(['backup_restore_id', 'generation']);
$t->unique(['backup_restore_id', 'active_attempt']);
});
$schema->create('cloud_jobs', static function (Blueprint $t): void {
$t->increments('id');
$t->integer('user_id');
$t->integer('backup_id')->nullable();
$t->integer('backup_restore_id')->nullable();
$t->integer('host_id')->nullable();
$t->string('kind');
$t->string('operation_id');
$t->string('phase');
$t->string('outcome')->nullable();
$t->string('effect_resolution');
$t->integer('terminal_at')->nullable();
$t->integer('finalization_incident')->default(0);
});
$schema->create('cloud_backup_recoveries', static function (Blueprint $t): void {
$t->increments('id');
$t->integer('backup_id');
$t->string('state');
});
$schema->create('cloud_deployment_snapshots', static function (
Blueprint $t
): void {
$t->increments('id');
$t->integer('backup_id');
$t->string('state');
});
$schema->create('cloud_instance_migrations', static function (
Blueprint $t
): void {
$t->increments('id');
$t->integer('source_host_id');
$t->integer('source_reservation_id');
$t->integer('target_host_id');
$t->integer('target_reservation_id');
$t->integer('instance_id');
$t->integer('restore_id');
$t->string('state');
});
$backup = CloudInstanceBackup::findOrFail(1);
$plans = CloudPortableRestoreTargetService::candidates($backup);
$plan = array_values(array_filter(
$plans,
static fn (CloudPortableRestoreTargetPlan $candidate): bool =>
$candidate->toArray()['target']['host_id'] === 2
&& $candidate->toArray()['target']['resource_pool_id'] === 200
))[0] ?? null;
$assert($plan instanceof CloudPortableRestoreTargetPlan,
'Host-two portable restore plan is missing.');
$actor = [
'user_id' => 99,
'is_admin' => true,
'is_super_admin' => true,
'can_manage' => true,
];
$requestKey = 'restore-request-0001';
$first = CloudPortableRestoreAttemptCoordinator::reserve(
$actor,
$backup,
$plan,
$plan->sha256(),
'restored-vm',
$requestKey
)->toArray();
$assert($first['generation'] === 1 && $first['dispatchable'] === true
&& is_array($first['dispatch_body'])
&& Capsule::table('cloud_resource_reservations')
->where('id', $first['reservation_id'])->value('user_id') === 9
&& Capsule::table('cloud_resource_reservation_ips')
->where('reservation_id', $first['reservation_id'])->count() === 2,
'Initial restore did not atomically reserve owner capacity and dual stack.');
$same = CloudPortableRestoreAttemptCoordinator::reserve(
$actor, $backup, $plan, $plan->sha256(), 'restored-vm', $requestKey
)->toArray();
$assert($same['attempt_key'] === $first['attempt_key']
&& $same['reservation_id'] === $first['reservation_id'],
'Exact reserve replay created another attempt.');
$rejects409(
static fn () => CloudPortableRestoreAttemptCoordinator::reserve(
$actor, $backup, $plan, $plan->sha256(), 'different-vm', $requestKey
),
'Same idempotency key accepted a different restore request.'
);
// Exact replay is a lookup of already-frozen evidence. Current eligibility
// drift must neither reject it nor generate a replacement attempt/bundle.
Capsule::table('cloud_instance_backups')->where('id', 1)->update([
'state' => 'failed',
]);
Capsule::table('cloud_host_agents')->where('host_id', 2)->update([
'enabled' => 0,
'revision' => 99,
'tested_revision' => 98,
'last_test_status' => 'failed',
]);
Capsule::table('cloud_host_storage_configurations')->where('host_id', 2)
->update(['revision' => 33, 'state' => 'pending']);
Capsule::table('cloud_host_resource_pool_memberships')
->where('host_id', 2)->where('resource_pool_id', 200)->update([
'scheduler_state' => 'disabled',
'revision' => 13,
]);
$eligibilityReplay = CloudPortableRestoreAttemptCoordinator::reserve(
$actor,
CloudInstanceBackup::findOrFail(1),
$plan,
$plan->sha256(),
'restored-vm',
$requestKey
)->toArray();
$assert($eligibilityReplay['attempt_key'] === $first['attempt_key']
&& $eligibilityReplay['reservation_id'] === $first['reservation_id'],
'Current backup or target eligibility drift broke exact replay.');
Capsule::table('cloud_instance_backups')->where('id', 1)->update([
'state' => 'ready',
]);
Capsule::table('cloud_host_agents')->where('host_id', 2)->update([
'enabled' => 1,
'revision' => 42,
'tested_revision' => 42,
'last_test_status' => 'healthy',
]);
Capsule::table('cloud_host_storage_configurations')->where('host_id', 2)
->update(['revision' => 32, 'state' => 'applied']);
Capsule::table('cloud_host_resource_pool_memberships')
->where('host_id', 2)->where('resource_pool_id', 200)->update([
'scheduler_state' => 'active',
'revision' => 12,
]);
// A torn durable claim set must never be silently replaced during replay.
$claimRowsBefore = Capsule::table('cloud_resource_reservation_ips')->count();
$ipIdsBefore = Capsule::table('cloud_public_ips')->orderBy('id')
->pluck('id')->all();
Capsule::connection()->beginTransaction();
try {
Capsule::table('cloud_resource_reservation_ips')
->where('reservation_id', $first['reservation_id'])->delete();
$rejects409(
static fn () => CloudPortableRestoreAttemptCoordinator::reserve(
$actor,
CloudInstanceBackup::findOrFail(1),
$plan,
$plan->sha256(),
'restored-vm',
$requestKey
),
'Replay allocated a replacement bundle for missing durable claims.'
);
$assert(Capsule::table('cloud_resource_reservation_ips')->count()
=== $claimRowsBefore - 2
&& Capsule::table('cloud_public_ips')->orderBy('id')
->pluck('id')->all() === $ipIdsBefore,
'Failed replay inserted replacement claims or public addresses.');
} finally {
Capsule::connection()->rollBack();
}
// Volatile capacity changes must not invalidate an already-reserved replay.
Capsule::table('cloud_resource_reservations')->insert([
'reservation_key' => str_repeat('1', 32),
'operation_id' => str_repeat('2', 32),
'order_id' => 77,
'backup_restore_id' => null,
'backup_restore_attempt_key' => null,
'migration_source_debt_id' => null,
'offer_id' => 88,
'resource_pool_id' => 200,
'host_id' => 2,
'user_id' => 9,
'instance_id' => null,
'network_attachment_id' => null,
'public_ip_pool_id' => null,
'public_ip_id' => null,
'cpu_cores' => 1,
'memory_mib' => 256,
'disk_gib' => 1,
'state' => 'held',
'expires_at' => null,
'released_at' => null,
'release_reason' => '',
'created_at' => $now,
'updated_at' => $now,
]);
$capacityReplay = CloudPortableRestoreAttemptCoordinator::reserve(
$actor, $backup, $plan, $plan->sha256(), 'restored-vm', $requestKey
)->toArray();
$assert($capacityReplay['attempt_key'] === $first['attempt_key'],
'Volatile capacity drift broke exact replay.');
$attempt = CloudBackupRestoreAttempt::where(
'attempt_key',
$first['attempt_key']
)->firstOrFail();
$reservation = CloudResourceReservation::findOrFail(
$first['reservation_id']
);
// Reservation/claim fencing is not a dispatchable state. A stale reserved
// attempt must fail closed instead of returning a body that could be sent.
$reservation->state = 'uncertain';
$reservation->save();
Capsule::table('cloud_resource_reservation_ips')
->where('reservation_id', (int) $reservation->id)->update([
'state' => 'uncertain', 'updated_at' => $now,
]);
Capsule::table('cloud_public_ips')
->where('reservation_id', (int) $reservation->id)->update([
'state' => 'uncertain', 'updated_at' => $now,
]);
$rejects409(
static fn () => CloudPortableRestoreAttemptCoordinator::reserve(
$actor, $backup, $plan, $plan->sha256(), 'restored-vm', $requestKey
),
'Reserved attempt accepted an uncertain fenced resource bundle.'
);
$reservation->state = 'held';
$reservation->save();
Capsule::table('cloud_resource_reservation_ips')
->where('reservation_id', (int) $reservation->id)->update([
'state' => 'reserved', 'updated_at' => $now,
]);
Capsule::table('cloud_public_ips')
->where('reservation_id', (int) $reservation->id)->update([
'state' => 'reserved', 'updated_at' => $now,
]);
$attempt->state = 'effect_unknown';
$attempt->effect_resolution = 'unknown';
$attempt->job_id = 500;
$attempt->save();
$reservation->state = 'uncertain';
$reservation->save();
Capsule::table('cloud_resource_reservation_ips')
->where('reservation_id', (int) $reservation->id)->update([
'state' => 'uncertain', 'updated_at' => $now,
]);
Capsule::table('cloud_public_ips')
->where('reservation_id', (int) $reservation->id)->update([
'state' => 'uncertain', 'updated_at' => $now,
]);
$unknown = CloudPortableRestoreAttemptCoordinator::reserve(
$actor, $backup, $plan, $plan->sha256(), 'restored-vm', $requestKey
)->toArray();
$assert($unknown['dispatchable'] === false
&& $unknown['dispatch_body'] === null,
'Unknown-effect replay exposed a dispatch instruction.');
Capsule::table('cloud_instance_backups')->where('id', 1)->update([
'state' => 'failed',
]);
$unknownAfterBackupDrift = CloudPortableRestoreAttemptCoordinator::reserve(
$actor,
CloudInstanceBackup::findOrFail(1),
$plan,
$plan->sha256(),
'restored-vm',
$requestKey
)->toArray();
$assert($unknownAfterBackupDrift['attempt_key'] === $first['attempt_key']
&& $unknownAfterBackupDrift['dispatchable'] === false
&& $unknownAfterBackupDrift['dispatch_body'] === null,
'Backup-state drift broke stable unknown-effect replay.');
Capsule::table('cloud_instance_backups')->where('id', 1)->update([
'state' => 'ready',
]);
// The attempt row, not the mutable parent projection, protects the backup.
Capsule::table('cloud_backup_restores')->where('id', $first['restore_id'])
->update(['state' => 'failed']);
$guard = new ReflectionMethod(
CloudBackupService::class,
'assertBackupDeletionAvailable'
);
$rejects409(
static fn () => $guard->invoke(null, CloudInstanceBackup::findOrFail(1)),
'An active attempt did not protect its source backup.'
);
CloudPublicNetworkAllocationService::releaseBundle(
(int) $reservation->id,
(string) $attempt->operation_id,
'restore_not_applied',
true
);
$releasedAt = $now + 10;
$reservation->state = 'released';
$reservation->released_at = $releasedAt;
$reservation->release_reason = 'restore_not_applied';
$reservation->save();
$attempt->state = 'failed';
$attempt->effect_resolution = 'not_applied';
$attempt->active_attempt = null;
$attempt->effect_resolved_at = $releasedAt;
$attempt->terminal_at = $releasedAt;
$attempt->save();
Capsule::table('cloud_jobs')->insert([
'id' => 500,
'user_id' => 9,
'backup_id' => 1,
'backup_restore_id' => $first['restore_id'],
'host_id' => 2,
'kind' => 'backup_restore',
'operation_id' => (string) $attempt->operation_id,
'phase' => 'terminal',
'outcome' => 'failed',
'effect_resolution' => 'not_applied',
'terminal_at' => $releasedAt,
'finalization_incident' => 0,
]);
// A fully released terminal not-applied attempt no longer protects the object.
$guard->invoke(null, CloudInstanceBackup::findOrFail(1));
// Retry authority is the exact terminal job evidence, including ownership and
// finalization status. Each mutation must fail without creating a generation.
$jobEvidenceMutations = [
['kind', 'wrong_kind', 'backup_restore'],
['host_id', 1, 2],
['user_id', 99, 9],
['backup_restore_id', 999, $first['restore_id']],
['operation_id', str_repeat('9', 32), (string) $attempt->operation_id],
['finalization_incident', 1, 0],
];
foreach ($jobEvidenceMutations as [$field, $invalid, $valid]) {
Capsule::table('cloud_jobs')->where('id', 500)->update([$field => $invalid]);
$rejects409(
static fn () => CloudPortableRestoreAttemptCoordinator::retry(
$actor,
$backup,
$plan,
$plan->sha256(),
'restored-vm',
$requestKey,
(string) $attempt->attempt_key
),
'Retry accepted mismatched terminal job evidence: ' . $field . '.'
);
$assert(CloudBackupRestoreAttempt::where(
'backup_restore_id',
$first['restore_id']
)->count() === 1, 'Rejected retry created another attempt row.');
Capsule::table('cloud_jobs')->where('id', 500)->update([$field => $valid]);
}
// Released reservation and address claims remain part of the predecessor's
// exact operation identity. A torn terminal ledger must never authorize a new
// dispatch generation even when the job itself says not_applied.
$ledgerOperationMutations = [
['cloud_resource_reservations', null],
['cloud_resource_reservation_ips', 4],
['cloud_resource_reservation_ips', 6],
];
foreach ($ledgerOperationMutations as [$table, $family]) {
$query = Capsule::table($table)->where(
'reservation_id',
(int) $reservation->id
);
if ($table === 'cloud_resource_reservations') {
$query = Capsule::table($table)->where('id', (int) $reservation->id);
} else {
$query->where('address_family', $family);
}
$query->update(['operation_id' => str_repeat('8', 32)]);
$rejects409(
static fn () => CloudPortableRestoreAttemptCoordinator::retry(
$actor,
$backup,
$plan,
$plan->sha256(),
'restored-vm',
$requestKey,
(string) $attempt->attempt_key
),
'Retry accepted torn terminal ledger operation identity: '
. $table . ':' . (string) $family . '.'
);
$assert(CloudBackupRestoreAttempt::where(
'backup_restore_id',
$first['restore_id']
)->count() === 1, 'Rejected ledger retry created another attempt row.');
$query->update(['operation_id' => (string) $attempt->operation_id]);
}
$ipv4Claim = Capsule::table('cloud_resource_reservation_ips')
->where('reservation_id', (int) $reservation->id)
->where('address_family', 4)->first();
$assert(is_object($ipv4Claim), 'Released IPv4 claim fixture is missing.');
Capsule::table('cloud_resource_reservation_ips')
->where('id', (int) $ipv4Claim->id)
->update(['public_ip_pool_id' => (int) $ipv4Claim->public_ip_pool_id + 1]);
$rejects409(
static fn () => CloudPortableRestoreAttemptCoordinator::retry(
$actor,
$backup,
$plan,
$plan->sha256(),
'restored-vm',
$requestKey,
(string) $attempt->attempt_key
),
'Retry accepted a released claim that changed frozen claim-set identity.'
);
$assert(CloudBackupRestoreAttempt::where(
'backup_restore_id',
$first['restore_id']
)->count() === 1, 'Rejected claim identity created another attempt row.');
Capsule::table('cloud_resource_reservation_ips')
->where('id', (int) $ipv4Claim->id)
->update(['public_ip_pool_id' => (int) $ipv4Claim->public_ip_pool_id]);
foreach ([
['claim_generation', (int) $ipv4Claim->claim_generation + 1],
['public_ip_id', (int) $ipv4Claim->public_ip_id + 999999],
] as [$field, $invalid]) {
Capsule::table('cloud_resource_reservation_ips')
->where('id', (int) $ipv4Claim->id)->update([$field => $invalid]);
$rejects409(
static fn () => CloudPortableRestoreAttemptCoordinator::retry(
$actor,
$backup,
$plan,
$plan->sha256(),
'restored-vm',
$requestKey,
(string) $attempt->attempt_key
),
'Retry accepted changed released claim identity: ' . $field . '.'
);
$assert(CloudBackupRestoreAttempt::where(
'backup_restore_id',
$first['restore_id']
)->count() === 1, 'Rejected claim mutation created another attempt row.');
Capsule::table('cloud_resource_reservation_ips')
->where('id', (int) $ipv4Claim->id)
->update([$field => $ipv4Claim->{$field}]);
}
// A released claim is not sufficient when its concrete address still points
// at the predecessor. The address may later be legitimately reused by another
// operation, but all ownership references to this operation must be gone.
Capsule::table('cloud_public_ips')
->where('id', (int) $ipv4Claim->public_ip_id)->update([
'state' => 'reserved',
'reservation_id' => (int) $reservation->id,
'reservation_ip_claim_id' => (int) $ipv4Claim->id,
'instance_id' => null,
'operation_id' => (string) $attempt->operation_id,
]);
$rejects409(
static fn () => CloudPortableRestoreAttemptCoordinator::retry(
$actor,
$backup,
$plan,
$plan->sha256(),
'restored-vm',
$requestKey,
(string) $attempt->attempt_key
),
'Retry accepted a public address that retained predecessor ownership.'
);
$assert(CloudBackupRestoreAttempt::where(
'backup_restore_id',
$first['restore_id']
)->count() === 1, 'Residual address ownership created another attempt row.');
Capsule::table('cloud_public_ips')
->where('id', (int) $ipv4Claim->public_ip_id)->update([
'state' => 'available',
'reservation_id' => null,
'reservation_ip_claim_id' => null,
'instance_id' => null,
'operation_id' => null,
]);
$second = CloudPortableRestoreAttemptCoordinator::retry(
$actor,
$backup,
$plan,
$plan->sha256(),
'restored-vm',
$requestKey,
(string) $attempt->attempt_key
)->toArray();
$assert($second['generation'] === 2
&& $second['attempt_key'] !== $first['attempt_key']
&& $second['reservation_id'] !== $first['reservation_id']
&& $second['dispatchable'] === true,
'Explicit not-applied retry did not create one fresh generation.');
$retryReplay = CloudPortableRestoreAttemptCoordinator::retry(
$actor,
$backup,
$plan,
$plan->sha256(),
'restored-vm',
$requestKey,
(string) $attempt->attempt_key
)->toArray();
$assert($retryReplay['attempt_key'] === $second['attempt_key']
&& $retryReplay['generation'] === 2,
'Retry replay skipped to another generation or failed idempotency.');
// A queued generation is already owned by a future job and cannot be emitted
// as another dispatch instruction.
$secondAttempt = CloudBackupRestoreAttempt::where(
'attempt_key',
$second['attempt_key']
)->firstOrFail();
$secondAttempt->state = 'queued';
$secondAttempt->job_id = 501;
$secondAttempt->save();
$queued = CloudPortableRestoreAttemptCoordinator::reserve(
$actor, $backup, $plan, $plan->sha256(), 'restored-vm', $requestKey
)->toArray();
$assert($queued['dispatchable'] === false && $queued['dispatch_body'] === null,
'Queued replay exposed a second dispatch instruction.');
// A completed attempt replays only its durable result and never another Agent
// instruction. Model the already-finalized instance/network ownership chain.
$secondReservation = CloudResourceReservation::findOrFail(
$second['reservation_id']
);
Capsule::table('cloud_instances')->insert([
'id' => 2,
'host_id' => 2,
'status' => 'active',
'spec_json' => json_encode([
'public_network' => $second['dispatch_body']['public_network'],
], JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES),
'name' => 'restored-vm',
'user_id' => 9,
'resource_pool_id' => 200,
'reservation_id' => (int) $secondReservation->id,
]);
CloudPublicNetworkAllocationService::commitBundle(
(int) $secondReservation->id,
2,
(string) $secondAttempt->operation_id
);
$secondReservation->instance_id = 2;
$secondReservation->state = 'committed';
$secondReservation->updated_at = $now + 20;
$secondReservation->save();
$secondAttempt->state = 'succeeded';
$secondAttempt->effect_resolution = 'applied';
$secondAttempt->active_attempt = null;
$secondAttempt->effect_resolved_at = $now + 20;
$secondAttempt->terminal_at = $now + 20;
$secondAttempt->save();
Capsule::table('cloud_backup_restores')->where('id', $first['restore_id'])
->update([
'target_instance_id' => 2,
'state' => 'ready',
'ready_at' => $now + 20,
]);
$succeeded = CloudPortableRestoreAttemptCoordinator::reserve(
$actor, $backup, $plan, $plan->sha256(), 'restored-vm', $requestKey
)->toArray();
$assert($succeeded['attempt_key'] === $second['attempt_key']
&& $succeeded['state'] === 'succeeded'
&& $succeeded['dispatchable'] === false
&& $succeeded['dispatch_body'] === null,
'Succeeded replay exposed another dispatch instruction.');
$policy = (object) [
'updated_at' => 1,
'configured_cpu_cores' => 8,
'configured_memory_mib' => 8192,
'configured_disk_gib' => 100,
'safety_cpu_cores' => 0,
'safety_memory_mib' => 0,
'safety_disk_gib' => 0,
'max_instances' => 10,
];
$resources = ['cpu_cores' => 1, 'memory_mib' => 512, 'disk_gib' => 10];
$validUsage = [
'unknown' => 0,
'instance_count' => 0,
'cpu_sum' => 0,
'memory_sum' => 0,
'disk_sum' => 0,
];
$missingMaximum = clone $policy;
unset($missingMaximum->max_instances);
$missingTimestamp = clone $policy;
unset($missingTimestamp->updated_at);
$assert(!CloudHostCapacityAdmission::allows($policy, [], $resources)
&& !CloudHostCapacityAdmission::allows($policy, [
'unknown' => 0,
'instance_count' => -1,
'cpu_sum' => 0,
'memory_sum' => 0,
'disk_sum' => 0,
], $resources)
&& !CloudHostCapacityAdmission::allows($policy, [
'unknown' => 1,
'instance_count' => 0,
'cpu_sum' => 0,
'memory_sum' => 0,
'disk_sum' => 0,
], $resources)
&& !CloudHostCapacityAdmission::allows(
$missingMaximum,
$validUsage,
$resources
)
&& !CloudHostCapacityAdmission::allows(
$missingTimestamp,
$validUsage,
$resources
),
'Capacity admission accepted incomplete policy or invalid usage.');
echo "Portable restore attempt coordinator behavior tests passed.\n";