You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
virt/web/tests/CloudPortableRestoreFinaliz...

673 lines
26 KiB
PHTML

<?php
ob_start();
require __DIR__ . '/CloudPortableRestoreJobDispatcherTest.php';
ob_end_clean();
use App\Models\CloudBackupRestore;
use App\Models\CloudBackupRestoreAttempt;
use App\Models\CloudInstance;
use App\Models\CloudJob;
use App\Models\CloudResourceReservation;
use App\Services\Cloud\CloudError;
use App\Services\Cloud\CloudInstanceResourceRelease;
use App\Services\Cloud\CloudOperationRecoveryIdentity;
use App\Services\Cloud\CloudPortableRestoreFinalizer;
use App\Services\Cloud\CloudSecrets;
use Illuminate\Database\Capsule\Manager as Capsule;
use Illuminate\Database\Schema\Blueprint;
if (!class_exists(CloudPortableRestoreFinalizer::class)) {
throw new RuntimeException(
'CloudPortableRestoreFinalizer production seam is not implemented.'
);
}
// The target scheduler fixture intentionally has only the columns it needs.
// Extend it with the production instance projection written after a restore
// succeeds; the source row remains valid because every new field is nullable
// or has the same production default.
$schema->table('cloud_instances', static function (Blueprint $t): void {
$t->string('uuid')->nullable();
$t->integer('order_id')->nullable();
$t->integer('plan_id')->nullable();
$t->integer('offer_id')->nullable();
$t->integer('current_period_id')->nullable();
$t->string('service_status')->default('exempt');
$t->string('network_status')->default('active');
$t->text('sealed_credential')->nullable();
$t->integer('expires_at')->nullable();
$t->decimal('renewal_price', 12, 2)->nullable();
$t->integer('renewal_days')->default(0);
$t->integer('renewal_traffic_bytes')->default(0);
$t->integer('traffic_quota_bytes')->default(0);
$t->integer('traffic_download_bytes')->default(0);
$t->integer('traffic_upload_bytes')->default(0);
$t->integer('lifetime_download_bytes')->default(0);
$t->integer('lifetime_upload_bytes')->default(0);
$t->integer('traffic_synced_at')->default(0);
$t->integer('suspended_at')->nullable();
$t->integer('network_suspended_at')->nullable();
$t->integer('created_at')->default(0);
$t->integer('updated_at')->default(0);
});
$attempt = CloudBackupRestoreAttempt::whereNotNull('job_id')
->orderByDesc('id')->firstOrFail();
$job = CloudJob::findOrFail((int) $attempt->job_id);
$restore = CloudBackupRestore::findOrFail((int) $attempt->backup_restore_id);
$reservation = CloudResourceReservation::findOrFail(
(int) $attempt->reservation_id
);
$queuedPayload = CloudSecrets::decrypt(
(string) $job->sealed_payload,
'job-payload'
);
$prepareWire = new ReflectionMethod(
\App\Services\Cloud\CloudWorkerService::class,
'portableRestoreDispatchPayload'
);
[$stablePayload] = $prepareWire->invoke(null, $job, $queuedPayload);
$now = time();
Capsule::table('cloud_jobs')->where('id', (int) $job->id)->update([
'phase' => 'dispatching',
'status' => 'running',
'attempts' => 1,
'dispatch_attempts' => 1,
'dispatch_started_at' => $now,
'dispatched_at' => $now,
'receipt_deadline_at' => $now + 600,
'sealed_payload' => CloudSecrets::encrypt(
$stablePayload,
'job-payload'
),
'updated_at' => $now,
]);
$job = CloudJob::findOrFail((int) $job->id);
$body = $stablePayload['body'];
$result = [
'uuid' => $body['uuid'],
'name' => $body['name'],
'status' => 'running',
'vcpu' => $body['vcpu'],
'memory_mib' => $body['memory_mib'],
'disk_gib' => $body['disk_gib'],
'bandwidth_mbps' => $body['bandwidth_mbps'],
'public_network' => $body['public_network'],
'machine' => 'q35',
'firmware' => 'uefi',
'secure_boot' => false,
'disks' => [['device' => 'vda', 'format' => 'qcow2']],
'interfaces' => [],
'backup_key' => $body['backup_key'],
'sha256' => $body['source_sha256'],
];
$baselineInstances = CloudInstance::count();
$connection = $job->getConnection();
$assertHeld = static function () use ($assert, $reservation): void {
$locked = CloudResourceReservation::findOrFail((int) $reservation->id);
$claims = Capsule::table('cloud_resource_reservation_ips')
->where('reservation_id', (int) $reservation->id)
->orderBy('address_family')->get();
$assert((string) $locked->state === 'held'
&& $locked->instance_id === null
&& $claims->count() === 2
&& $claims->every(static fn ($claim): bool =>
(string) $claim->state === 'reserved'
&& (int) $claim->active_claim === 1),
'Restore finalizer changed the held bundle before a valid conclusion.');
};
// Success creates the panel row only after the Agent effect is proven, then
// commits both address families and every authority row in the same outer
// transaction.
$connection->beginTransaction();
try {
CloudPortableRestoreFinalizer::finishJobLocked(
$job,
'succeeded',
$result,
$stablePayload
);
CloudPortableRestoreFinalizer::finishJobLocked(
$job,
'succeeded',
$result,
$stablePayload
);
$finishedAttempt = CloudBackupRestoreAttempt::findOrFail(
(int) $attempt->id
);
$finishedRestore = CloudBackupRestore::findOrFail((int) $restore->id);
$finishedReservation = CloudResourceReservation::findOrFail(
(int) $reservation->id
);
$instance = CloudInstance::where('uuid', (string) $body['uuid'])->first();
$claims = Capsule::table('cloud_resource_reservation_ips')
->where('reservation_id', (int) $reservation->id)
->orderBy('address_family')->get();
$addresses = Capsule::table('cloud_public_ips')->whereIn(
'id',
$claims->pluck('public_ip_id')->map(
static fn ($value): int => (int) $value
)->all()
)->get();
$spec = $instance instanceof CloudInstance
? json_decode((string) $instance->spec_json, true) : null;
$assert($instance instanceof CloudInstance
&& CloudInstance::count() === $baselineInstances + 1
&& (int) $instance->host_id === (int) $attempt->target_host_id
&& (int) $instance->user_id === (int) $job->user_id
&& (int) $instance->reservation_id === (int) $reservation->id
&& (int) $instance->resource_pool_id
=== (int) $reservation->resource_pool_id
&& (string) $instance->status === 'active'
&& (string) $instance->service_status === 'exempt'
&& (string) $instance->network_status === 'active'
&& $instance->sealed_credential === null
&& is_array($spec)
&& ($spec['public_network'] ?? null) == $body['public_network']
&& ($spec['restored_from_backup'] ?? null) === $body['backup_key']
&& !array_key_exists('root_password', $spec)
&& !array_key_exists('guest_hostname', $spec),
'Successful restore did not create the canonical disk-owned instance projection.');
$assert((string) $finishedReservation->state === 'committed'
&& (int) $finishedReservation->instance_id === (int) $instance->id
&& $claims->count() === 2
&& $claims->every(static fn ($claim): bool =>
(string) $claim->state === 'committed'
&& (int) $claim->active_claim === 1)
&& $addresses->count() === 2
&& $addresses->every(static fn ($address): bool =>
(string) $address->state === 'assigned'
&& (int) $address->instance_id === (int) $instance->id),
'Successful restore did not atomically commit IPv4 and IPv6.');
$assert((string) $finishedAttempt->state === 'succeeded'
&& (string) $finishedAttempt->effect_resolution === 'applied'
&& $finishedAttempt->active_attempt === null
&& $finishedAttempt->effect_resolved_at !== null
&& $finishedAttempt->terminal_at !== null
&& (string) $finishedRestore->state === 'ready'
&& (int) $finishedRestore->target_instance_id === (int) $instance->id
&& (int) $finishedRestore->job_id === (int) $job->id
&& (int) $finishedRestore->reservation_id
=== (int) $reservation->id,
'Successful restore did not settle attempt and generation-one projections.');
// All three delete entry points share this gate and final settlement.
// Admission must recognize the attempt-owned source before any host
// mutation, then confirmed absence releases both families exactly once.
$instance->name = 'renamed-restored-vm';
$instance->save();
CloudInstanceResourceRelease::assertCanDispatchDelete($instance);
$instance->status = 'deleted';
$instance->save();
CloudInstanceResourceRelease::releaseAfterConfirmedDelete(
$instance,
'instance_deleted'
);
$released = CloudResourceReservation::findOrFail(
(int) $reservation->id
);
$releasedClaims = Capsule::table('cloud_resource_reservation_ips')
->where('reservation_id', (int) $reservation->id)
->orderBy('address_family')->get();
$releasedAddresses = Capsule::table('cloud_public_ips')->whereIn(
'id',
$releasedClaims->pluck('public_ip_id')->map(
static fn ($value): int => (int) $value
)->all()
)->get();
$releasedAt = (int) $released->released_at;
$assert((string) $released->state === 'released'
&& (int) $released->instance_id === (int) $instance->id
&& $releasedAt > 0
&& (string) $released->release_reason === 'instance_deleted'
&& $releasedClaims->count() === 2
&& $releasedClaims->every(static fn ($claim): bool =>
(string) $claim->state === 'released'
&& $claim->active_claim === null
&& $claim->released_at !== null)
&& $releasedAddresses->count() === 2
&& $releasedAddresses->every(static fn ($address): bool =>
(string) $address->state === 'available'
&& $address->reservation_id === null
&& $address->reservation_ip_claim_id === null
&& $address->instance_id === null
&& $address->operation_id === null),
'Confirmed restored-instance deletion did not release dual-stack ownership.');
$releasedAt = 123456789;
Capsule::table('cloud_resource_reservations')->where(
'id',
(int) $reservation->id
)->update([
'released_at' => $releasedAt,
]);
CloudInstanceResourceRelease::releaseAfterConfirmedDelete(
$instance->fresh(),
'instance_deleted'
);
$assert((int) CloudResourceReservation::findOrFail(
(int) $reservation->id
)->released_at === $releasedAt
&& (string) CloudBackupRestoreAttempt::findOrFail(
(int) $attempt->id
)->state === 'succeeded'
&& (string) CloudBackupRestore::findOrFail(
(int) $restore->id
)->state === 'ready',
'Restored-instance deletion replay mutated terminal restore history.');
$assert((string) CloudResourceReservation::findOrFail(
(int) $reservation->id
)->release_reason === 'instance_deleted',
'Restored-instance deletion replay rewrote its release reason.');
} finally {
$connection->rollBack();
}
// An unknown or torn restore attempt must be rejected before a delete reaches
// the host, preserving the complete committed pair for reconciliation.
$connection->beginTransaction();
try {
CloudPortableRestoreFinalizer::finishJobLocked(
$job,
'succeeded',
$result,
$stablePayload
);
$guardedInstance = CloudInstance::where(
'uuid',
(string) $body['uuid']
)->firstOrFail();
Capsule::table('cloud_resource_reservations')->where(
'id',
(int) $reservation->id
)->update(['backup_restore_attempt_key' => null]);
$missingSourceRejected = false;
try {
CloudInstanceResourceRelease::assertCanDispatchDelete(
$guardedInstance
);
} catch (CloudError $error) {
$missingSourceRejected = $error->httpStatus === 409;
}
$assert($missingSourceRejected,
'A restore reservation without its attempt key reached deletion.');
Capsule::table('cloud_resource_reservations')->where(
'id',
(int) $reservation->id
)->update([
'backup_restore_attempt_key' => (string) $attempt->attempt_key,
]);
Capsule::table('cloud_backup_restore_attempts')->where(
'id',
(int) $attempt->id
)->update([
'state' => 'effect_unknown',
'effect_resolution' => 'unknown',
'active_attempt' => 1,
'effect_resolved_at' => null,
'terminal_at' => null,
]);
$deleteRejected = false;
try {
CloudInstanceResourceRelease::assertCanDispatchDelete(
$guardedInstance
);
} catch (CloudError $error) {
$deleteRejected = $error->httpStatus === 409;
}
$guardedReservation = CloudResourceReservation::findOrFail(
(int) $reservation->id
);
$guardedClaims = Capsule::table('cloud_resource_reservation_ips')
->where('reservation_id', (int) $reservation->id)->get();
$assert($deleteRejected
&& (string) $guardedReservation->state === 'committed'
&& $guardedClaims->count() === 2
&& $guardedClaims->every(static fn ($claim): bool =>
(string) $claim->state === 'committed'
&& (int) $claim->active_claim === 1),
'Unknown restore attempt reached deletion or released its bundle.');
} finally {
$connection->rollBack();
}
// The restored domain must match the one fixed portable platform contract;
// UUID, disk and network equality cannot authorize a BIOS/i440fx variant.
$connection->beginTransaction();
try {
foreach ([
['machine', 'pc-i440fx'],
['firmware', 'bios'],
['secure_boot', true],
] as [$field, $wrong]) {
$incompatible = $result;
$incompatible[$field] = $wrong;
$rejected = false;
try {
CloudPortableRestoreFinalizer::finishJobLocked(
$job,
'succeeded',
$incompatible,
$stablePayload
);
} catch (CloudError $error) {
$rejected = $error->httpStatus === 502;
}
$assert($rejected,
'Restore accepted incompatible platform field: ' . $field);
$assertHeld();
}
} finally {
$connection->rollBack();
}
// A deterministic not-applied conclusion releases both claims and concrete
// addresses. Replaying the same conclusion is an exact no-op, not a second
// release or a new instance.
$connection->beginTransaction();
try {
CloudPortableRestoreFinalizer::finishJobLocked(
$job,
'failed',
[
'creation_absent' => true,
'managed_artifacts_absent' => true,
],
$stablePayload
);
CloudPortableRestoreFinalizer::finishJobLocked(
$job,
'failed',
[
'creation_absent' => true,
'managed_artifacts_absent' => true,
],
$stablePayload
);
$failedAttempt = CloudBackupRestoreAttempt::findOrFail((int) $attempt->id);
$failedRestore = CloudBackupRestore::findOrFail((int) $restore->id);
$failedReservation = CloudResourceReservation::findOrFail(
(int) $reservation->id
);
$claims = Capsule::table('cloud_resource_reservation_ips')
->where('reservation_id', (int) $reservation->id)
->orderBy('address_family')->get();
$addresses = Capsule::table('cloud_public_ips')->whereIn(
'id',
$claims->pluck('public_ip_id')->map(
static fn ($value): int => (int) $value
)->all()
)->get();
$assert(CloudInstance::count() === $baselineInstances
&& (string) $failedReservation->state === 'released'
&& $failedReservation->instance_id === null
&& $failedReservation->released_at !== null
&& $claims->count() === 2
&& $claims->every(static fn ($claim): bool =>
(string) $claim->state === 'released'
&& $claim->active_claim === null
&& $claim->released_at !== null)
&& $addresses->count() === 2
&& $addresses->every(static fn ($address): bool =>
(string) $address->state === 'available'
&& $address->reservation_id === null
&& $address->reservation_ip_claim_id === null
&& $address->instance_id === null
&& $address->operation_id === null),
'Confirmed-not-applied restore retained a resource or address claim.');
$assert((string) $failedAttempt->state === 'failed'
&& (string) $failedAttempt->effect_resolution === 'not_applied'
&& $failedAttempt->active_attempt === null
&& $failedAttempt->effect_resolved_at !== null
&& $failedAttempt->terminal_at !== null
&& (string) $failedRestore->state === 'failed'
&& $failedRestore->target_instance_id === null
&& (int) $failedRestore->job_id === (int) $job->id
&& (int) $failedRestore->reservation_id
=== (int) $reservation->id,
'Confirmed-not-applied restore did not expose an explicit retry boundary.');
} finally {
$connection->rollBack();
}
// A generic failure is not proof that the host and managed artifacts are
// absent. The deep finalizer itself must reject incomplete absence evidence.
$connection->beginTransaction();
try {
$absenceRejected = false;
try {
CloudPortableRestoreFinalizer::finishJobLocked(
$job,
'failed',
['creation_absent' => true],
$stablePayload
);
} catch (\App\Services\Cloud\CloudRecoveryEvidenceConflict $error) {
$absenceRejected = $error->httpStatus === 409;
}
$assert($absenceRejected, 'Incomplete absence evidence released a restore.');
$assertHeld();
} finally {
$connection->rollBack();
}
// Unknown effect never releases or reassigns anything. It fences both legs,
// keeps the generation active, and therefore cannot be retried automatically.
$connection->beginTransaction();
try {
CloudPortableRestoreFinalizer::finishJobLocked(
$job,
'uncertain',
null,
$stablePayload
);
CloudPortableRestoreFinalizer::finishJobLocked(
$job,
'uncertain',
null,
$stablePayload
);
$unknownAttempt = CloudBackupRestoreAttempt::findOrFail((int) $attempt->id);
$unknownRestore = CloudBackupRestore::findOrFail((int) $restore->id);
$unknownReservation = CloudResourceReservation::findOrFail(
(int) $reservation->id
);
$claims = Capsule::table('cloud_resource_reservation_ips')
->where('reservation_id', (int) $reservation->id)
->orderBy('address_family')->get();
$addresses = Capsule::table('cloud_public_ips')->whereIn(
'id',
$claims->pluck('public_ip_id')->map(
static fn ($value): int => (int) $value
)->all()
)->get();
$assert(CloudInstance::count() === $baselineInstances
&& (string) $unknownReservation->state === 'uncertain'
&& $unknownReservation->instance_id === null
&& $claims->count() === 2
&& $claims->every(static fn ($claim): bool =>
(string) $claim->state === 'uncertain'
&& (int) $claim->active_claim === 1)
&& $addresses->count() === 2
&& $addresses->every(static fn ($address): bool =>
(string) $address->state === 'uncertain'
&& (int) $address->reservation_id
=== (int) $reservation->id),
'Unknown restore effect released or reassigned a fenced resource.');
$assert((string) $unknownAttempt->state === 'effect_unknown'
&& (string) $unknownAttempt->effect_resolution === 'unknown'
&& (int) $unknownAttempt->active_attempt === 1
&& $unknownAttempt->effect_resolved_at === null
&& $unknownAttempt->terminal_at === null
&& (string) $unknownRestore->state === 'uncertain'
&& $unknownRestore->target_instance_id === null
&& (int) $unknownRestore->job_id === (int) $job->id
&& (int) $unknownRestore->reservation_id
=== (int) $reservation->id,
'Unknown restore effect did not retain its active recovery fence.');
} finally {
$connection->rollBack();
}
// Recovery prepare validates and persists exact evidence but must not expose
// an instance or commit either address family before the Agent fence commit.
$connection->beginTransaction();
try {
$preparedJob = CloudJob::findOrFail((int) $job->id);
CloudPortableRestoreFinalizer::finishJobLocked(
$preparedJob,
'succeeded',
$result,
$stablePayload,
true
);
$pendingAttempt = CloudBackupRestoreAttempt::findOrFail((int) $attempt->id);
$assert((string) $pendingAttempt->state === 'finalization_pending'
&& (string) $pendingAttempt->effect_resolution === 'applied'
&& (string) $pendingAttempt->finalization_error_code
=== 'recovery_prepare_pending'
&& CloudInstance::count() === $baselineInstances,
'Prepared restore published the target before fence resolution.');
$assertHeld();
$compactMethod = new ReflectionMethod(
\App\Services\Cloud\CloudWorkerService::class,
'compactPreparedRecoveryPayload'
);
$compactPayload = $compactMethod->invoke(
null,
'backup_restore',
$stablePayload
);
$compactPayload['_operation_recovery'] = [
'version' => 1,
'kind' => 'backup_restore',
'prepared_marker' => 'prepared:test',
'operation' => ['status' => 'succeeded'],
];
Capsule::table('cloud_jobs')->where('id', (int) $job->id)->update([
'sealed_payload' => CloudSecrets::encrypt(
$compactPayload,
'job-payload'
),
]);
$preparedJob = CloudJob::findOrFail((int) $job->id);
$assert(CloudOperationRecoveryIdentity::same(
CloudOperationRecoveryIdentity::expected($preparedJob),
$compactPayload[CloudOperationRecoveryIdentity::payloadKey()]
),
'Compact prepared restore payload lost its recovery identity.');
$tamperedCompact = $compactPayload;
$tamperedCompact['generation'] = (int) $tamperedCompact['generation'] + 1;
$preparedJob->sealed_payload = CloudSecrets::encrypt(
$tamperedCompact,
'job-payload'
);
$preparedTamperRejected = false;
try {
CloudOperationRecoveryIdentity::expected($preparedJob);
} catch (CloudError $error) {
$preparedTamperRejected = $error->httpStatus === 409;
}
$assert($preparedTamperRejected,
'Prepared escrow bypassed portable restore recovery authority.');
$preparedJob->sealed_payload = CloudSecrets::encrypt(
$compactPayload,
'job-payload'
);
CloudPortableRestoreFinalizer::publishPreparedLocked(
$preparedJob,
'applied'
);
CloudPortableRestoreFinalizer::publishPreparedLocked(
$preparedJob,
'applied'
);
$published = CloudBackupRestoreAttempt::findOrFail((int) $attempt->id);
$assert((string) $published->state === 'succeeded'
&& CloudInstance::count() === $baselineInstances + 1,
'Prepared restore did not publish exactly once after fence resolution.');
} finally {
$connection->rollBack();
}
// If exact evidence survived but the ordinary Worker transaction failed, the
// local retry consumes only that evidence and never dispatches the mutation.
$connection->beginTransaction();
try {
$retryJob = CloudJob::findOrFail((int) $job->id);
CloudPortableRestoreFinalizer::finishJobLocked(
$retryJob,
'succeeded',
$result,
$stablePayload,
true
);
Capsule::table('cloud_backup_restore_attempts')
->where('id', (int) $attempt->id)->update([
'last_error_code' => 'local_finalization_failed',
'finalization_error_code' => 'local_finalization_failed',
'updated_at' => time(),
]);
$leaseToken = str_repeat('a', 32);
Capsule::table('cloud_jobs')->where('id', (int) $job->id)->update([
'phase' => 'terminal',
'outcome' => 'failed',
'status' => 'failed',
'terminal_at' => time(),
'effect_may_have_applied' => 1,
'safe_to_resubmit' => 0,
'effect_resolution' => 'unknown',
'error_code' => 'local_finalization_failed',
'finalization_incident' => 1,
'finalization_error_code' => 'local_finalization_failed',
'lease_token' => $leaseToken,
'lease_until' => time() + 60,
'updated_at' => time(),
]);
CloudPortableRestoreFinalizer::retryPendingFinalization(
(int) $job->id,
$leaseToken
);
$retriedJob = CloudJob::findOrFail((int) $job->id);
$retriedAttempt = CloudBackupRestoreAttempt::findOrFail((int) $attempt->id);
$assert((string) $retriedAttempt->state === 'succeeded'
&& (string) $retriedJob->effect_resolution === 'applied'
&& !(bool) $retriedJob->finalization_incident
&& $retriedJob->lease_token === null
&& CloudInstance::count() === $baselineInstances + 1,
'Portable restore PHP-only retry did not converge from durable evidence.');
} finally {
$connection->rollBack();
}
// Any frozen-identity tear must abort the whole outer transaction before an
// instance row or one-sided address mutation can escape.
$connection->beginTransaction();
try {
$torn = CloudBackupRestoreAttempt::findOrFail((int) $attempt->id);
$torn->target_public_network_sha256 = str_repeat('0', 64);
$torn->save();
$rejected = false;
try {
CloudPortableRestoreFinalizer::finishJobLocked(
$job,
'succeeded',
$result,
$stablePayload
);
} catch (CloudError $error) {
$rejected = $error->httpStatus === 409;
}
$assert($rejected && CloudInstance::count() === $baselineInstances,
'Torn restore identity produced a partial instance.');
$assertHeld();
} finally {
$connection->rollBack();
}
echo "Portable restore finalizer behavior tests passed.\n";