You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
virt/web/tests/CloudPortableRestoreJobDisp...

1056 lines
41 KiB
PHP

<?php
ob_start();
require __DIR__ . '/CloudPortableRestoreAttemptCoordinatorTest.php';
ob_end_clean();
use App\Models\CloudBackupRestore;
use App\Models\CloudBackupRestoreAttempt;
use App\Models\CloudInstanceBackup;
use App\Models\CloudJob;
use App\Models\CloudResourceReservation;
use App\Services\Cloud\CloudError;
use App\Services\Cloud\CloudAdminOperationRecoveryService;
use App\Services\Cloud\CloudAgentClient;
use App\Services\Cloud\CloudAgentError;
use App\Services\Cloud\CloudOperationRecoveryIdentity;
use App\Services\Cloud\CloudPortableRestoreAttemptCoordinator;
use App\Services\Cloud\CloudPortableRestoreJobDispatcher;
use App\Services\Cloud\CloudPortableRestoreJobEnvelope;
use App\Services\Cloud\CloudPortableRestoreTargetPlan;
use App\Services\Cloud\CloudPortableRestoreTargetService;
use App\Services\Cloud\CloudSecrets;
use App\Services\Cloud\CloudWorkerService;
use Illuminate\Database\Capsule\Manager as Capsule;
use Illuminate\Database\Schema\Blueprint;
if (!function_exists('sodium_crypto_aead_xchacha20poly1305_ietf_encrypt')) {
echo "SKIPPED CloudPortableRestoreJobDispatcherTest: runtime lacks sodium.\n";
exit(0);
}
if (!class_exists(CloudPortableRestoreJobDispatcher::class)) {
throw new RuntimeException(
'CloudPortableRestoreJobDispatcher production seam is not implemented.'
);
}
$rejects409 = static function (callable $callback, string $message) use (
$assert
): void {
try {
$callback();
} catch (CloudError $error) {
$assert($error->httpStatus === 409, $message . ' (wrong status)');
return;
}
throw new RuntimeException($message);
};
// The coordinator fixture deliberately carries only fields it needs. Extend
// the same in-memory job table with the worker envelope columns exercised by
// this stage; production schema already owns these columns.
$schema->table('cloud_jobs', static function (Blueprint $t): void {
$t->string('request_idempotency_key')->default('');
$t->string('request_fingerprint', 64)->default('');
$t->integer('instance_id')->nullable();
$t->integer('parent_job_id')->nullable();
$t->integer('vpc_repair_root_job_id')->nullable();
$t->string('execution_mode')->default('agent');
$t->string('status')->default('queued');
$t->text('sealed_payload')->default('');
$t->integer('attempts')->default(0);
$t->integer('dispatch_attempts')->default(0);
$t->integer('dispatch_deadline_at')->default(0);
$t->integer('dispatch_started_at')->nullable();
$t->integer('agent_accepted_at')->nullable();
$t->integer('receipt_deadline_at')->nullable();
$t->integer('effect_may_have_applied')->nullable();
$t->integer('safe_to_resubmit')->nullable();
$t->integer('effect_resolved_at')->nullable();
$t->integer('effect_resolved_by')->nullable();
$t->string('effect_resolution_reason')->default('');
$t->string('finalization_error_code')->nullable();
$t->string('completion_event_id')->nullable();
$t->integer('read_at')->nullable();
$t->integer('dispatched_at')->nullable();
$t->string('lease_token')->nullable();
$t->integer('lease_until')->default(0);
$t->integer('available_at')->default(0);
$t->string('error_code')->nullable();
$t->text('result_json')->nullable();
$t->integer('created_at')->default(0);
$t->integer('updated_at')->default(0);
});
$schema->create('cloud_agent_completion_events', static function (
Blueprint $t
): void {
$t->increments('id');
$t->integer('host_id');
$t->string('operation_id');
});
$schema->create('cloud_storage_backends', static function (
Blueprint $t
): void {
$t->increments('id'); $t->string('backend_key');
$t->string('identity_sha256'); $t->string('name');
$t->string('driver'); $t->string('endpoint'); $t->string('region');
$t->string('bucket'); $t->string('prefix'); $t->boolean('path_style');
$t->string('failure_domain'); $t->text('sealed_credentials');
$t->integer('config_revision'); $t->integer('tested_revision');
$t->string('admin_state'); $t->string('health_state');
$t->integer('write_priority');
$t->integer('consecutive_failures')->default(0);
$t->integer('consecutive_successes')->default(0);
$t->string('last_error_code')->default('');
$t->integer('last_tested_at')->default(0);
$t->integer('last_healthy_at')->default(0);
$t->integer('next_probe_at')->default(0);
$t->integer('created_at'); $t->integer('updated_at');
});
$schema->create('cloud_storage_replicas', static function (
Blueprint $t
): void {
$t->increments('id'); $t->string('replica_key');
$t->integer('artifact_id'); $t->integer('backend_id');
$t->string('role'); $t->string('object_key');
$t->string('object_version_id')->default('');
$t->string('object_etag')->default('');
$t->integer('size_bytes'); $t->string('sha256', 64);
$t->string('state'); $t->string('publish_operation_id')->nullable();
$t->string('delete_operation_id')->nullable();
$t->integer('delete_generation')->default(0);
$t->integer('delete_cycle_attempts')->default(0);
$t->integer('delete_presence_confirmations')->default(0);
$t->integer('delete_last_checked_at')->default(0);
$t->integer('delete_next_check_at')->default(0);
$t->integer('delete_attention_at')->nullable();
$t->string('lease_token')->nullable();
$t->integer('lease_until')->default(0);
$t->string('error_code')->default('');
$t->integer('verified_at'); $t->integer('created_at');
$t->integer('updated_at');
});
Capsule::table('sqlite_sequence')->where('name', 'cloud_jobs')->update([
'seq' => 600,
]);
global $System_Config;
$System_Config['cloud_data_key'] = base64_encode(str_repeat('r', 32));
$runtime = [
'config' => [
'endpoint' => 'https://objects.example.com',
'region' => 'us-east-1',
'bucket' => 'portable-backups',
'prefix' => 'virt',
'path_style' => true,
],
'credentials' => [
'access_key_id' => 'RESTOREACCESS',
'secret_access_key' => 'restore-secret-value',
],
'revision' => 7,
];
Capsule::table('cloud_storage_backends')->insert([
'id' => 71,
'backend_key' => str_repeat('1', 32),
'identity_sha256' => str_repeat('2', 64),
'name' => 'portable-primary',
'driver' => 's3',
'endpoint' => 'https://objects.example.com',
'region' => 'us-east-1',
'bucket' => 'portable-backups',
'prefix' => 'virt',
'path_style' => 1,
'failure_domain' => 'portable-primary',
'sealed_credentials' => CloudSecrets::encrypt(
$runtime['credentials'],
'object-storage-backend-credentials'
),
'config_revision' => 1,
'tested_revision' => 1,
'admin_state' => 'active',
'health_state' => 'healthy',
'write_priority' => 10,
'consecutive_failures' => 0,
'consecutive_successes' => 1,
'last_error_code' => '',
'last_tested_at' => time(),
'last_healthy_at' => time(),
'next_probe_at' => time() + 300,
'created_at' => time(),
'updated_at' => time(),
]);
$sourceBackupRow = Capsule::table('cloud_instance_backups')
->where('id', 1)->first();
$assert(is_object($sourceBackupRow), 'Portable source backup is missing.');
Capsule::table('cloud_storage_artifacts')->insert([
'id' => 83,
'artifact_key' => str_repeat('3', 32),
'kind' => 'backup',
'maximum_size_bytes' => 2048,
'size_bytes' => (int) $sourceBackupRow->object_size_bytes,
'sha256' => (string) $sourceBackupRow->sha256,
'durability_policy' => 'mirror_async_2',
'desired_copies' => 2,
'required_copies' => 1,
'state' => 'available',
'last_operation_id' => str_repeat('4', 32),
'verified_at' => time(),
'created_at' => time(),
'updated_at' => time(),
]);
Capsule::table('cloud_storage_publications')->insert([
'artifact_id' => 83,
'backup_id' => 1,
'state' => 'ready_to_finalize',
'created_at' => time(),
'updated_at' => time(),
]);
Capsule::table('cloud_storage_replicas')->insert([
'id' => 73,
'replica_key' => str_repeat('5', 32),
'artifact_id' => 83,
'backend_id' => 71,
'role' => 'primary',
'object_key' => (string) $sourceBackupRow->object_key,
'object_version_id' => '',
'object_etag' => '',
'size_bytes' => (int) $sourceBackupRow->object_size_bytes,
'sha256' => (string) $sourceBackupRow->sha256,
'state' => 'ready',
'publish_operation_id' => str_repeat('6', 32),
'delete_operation_id' => null,
'delete_generation' => 0,
'delete_cycle_attempts' => 0,
'delete_presence_confirmations' => 0,
'delete_last_checked_at' => 0,
'delete_next_check_at' => 0,
'delete_attention_at' => null,
'lease_token' => null,
'lease_until' => 0,
'error_code' => '',
'verified_at' => time(),
'created_at' => time(),
'updated_at' => time(),
]);
Capsule::table('cloud_instance_backups')->where('id', 1)->update([
'storage_artifact_id' => 83,
'sealed_storage_runtime' => '',
'storage_revision' => 0,
]);
$backup = CloudInstanceBackup::findOrFail(1);
$plans = CloudPortableRestoreTargetService::candidates($backup);
$plan = array_values(array_filter(
$plans,
static fn (CloudPortableRestoreTargetPlan $candidate): bool =>
$candidate->toArray()['target']['host_id'] === 3
&& $candidate->toArray()['target']['resource_pool_id'] === 300
))[0] ?? null;
$assert($plan instanceof CloudPortableRestoreTargetPlan,
'Fresh portable restore dispatch plan is unavailable.');
$requestKey = 'dispatch-request-0000000000000001';
$reserved = CloudPortableRestoreAttemptCoordinator::reserve(
$actor,
$backup,
$plan,
$plan->sha256(),
'dispatch-vm',
$requestKey
)->toArray();
$attempt = CloudBackupRestoreAttempt::where(
'attempt_key',
$reserved['attempt_key']
)->firstOrFail();
$reservation = CloudResourceReservation::findOrFail(
$reserved['reservation_id']
);
$jobCount = CloudJob::count();
$instanceCount = Capsule::table('cloud_instances')->count();
$claimSnapshot = Capsule::table('cloud_resource_reservation_ips')
->where('reservation_id', (int) $reservation->id)
->orderBy('address_family')->get()->map(
static fn ($row): array => (array) $row
)->all();
// Any torn input must roll back without an orphan job or changing the attempt.
$reservation->operation_id = str_repeat('7', 32);
$reservation->save();
$rejects409(
static fn () => CloudPortableRestoreJobDispatcher::queue(
$actor,
(string) $attempt->attempt_key
),
'Dispatcher accepted a torn reservation operation.'
);
$assert(CloudJob::count() === $jobCount
&& (string) CloudBackupRestoreAttempt::findOrFail($attempt->id)->state
=== 'reserved',
'Rejected dispatch left a job or changed attempt state.');
$reservation->operation_id = (string) $attempt->operation_id;
$reservation->save();
$restoreBeforeQueue = CloudBackupRestore::findOrFail(
(int) $attempt->backup_restore_id
);
$restoreBeforeQueue->target_instance_id = 777;
$restoreBeforeQueue->save();
$rejects409(
static fn () => CloudPortableRestoreJobDispatcher::queue(
$actor,
(string) $attempt->attempt_key
),
'Dispatcher accepted a pre-created target instance.'
);
$restoreBeforeQueue->target_instance_id = null;
$restoreBeforeQueue->save();
foreach (['backup_restore_id' => 777, 'order_id' => 777] as $field => $value) {
$reservation->{$field} = $value;
$reservation->save();
$rejects409(
static fn () => CloudPortableRestoreJobDispatcher::queue(
$actor,
(string) $attempt->attempt_key
),
'Dispatcher accepted a doubly-owned reservation: ' . $field . '.'
);
$reservation->{$field} = null;
$reservation->save();
}
$assert(CloudJob::count() === $jobCount
&& (string) CloudBackupRestoreAttempt::findOrFail($attempt->id)->state
=== 'reserved',
'Rejected ownership tear left a job or changed attempt state.');
$ipv6Claim = Capsule::table('cloud_resource_reservation_ips')
->where('reservation_id', (int) $reservation->id)
->where('address_family', 6)->first();
$assert(is_object($ipv6Claim), 'IPv6 claim fixture is missing.');
Capsule::table('cloud_resource_reservation_ips')
->where('id', (int) $ipv6Claim->id)
->update(['operation_id' => str_repeat('6', 32)]);
$rejects409(
static fn () => CloudPortableRestoreJobDispatcher::queue(
$actor,
(string) $attempt->attempt_key
),
'Dispatcher accepted a torn IPv6 claim.'
);
$assert(CloudJob::count() === $jobCount
&& Capsule::table('cloud_instances')->count() === $instanceCount,
'Rejected claim dispatch produced a job or instance.');
Capsule::table('cloud_resource_reservation_ips')
->where('id', (int) $ipv6Claim->id)
->update(['operation_id' => (string) $attempt->operation_id]);
$originalPlanSha = (string) $attempt->target_plan_sha256;
$attempt->target_plan_sha256 = str_repeat('5', 64);
$attempt->save();
$rejects409(
static fn () => CloudPortableRestoreJobDispatcher::queue(
$actor,
(string) $attempt->attempt_key
),
'Dispatcher accepted a torn target plan.'
);
$attempt->target_plan_sha256 = $originalPlanSha;
$attempt->save();
$job = CloudPortableRestoreJobDispatcher::queue(
$actor,
(string) $attempt->attempt_key
);
$attempt = CloudBackupRestoreAttempt::findOrFail($attempt->id);
$restore = CloudBackupRestore::findOrFail($attempt->backup_restore_id);
$reservation = CloudResourceReservation::findOrFail($reservation->id);
$assert((int) $job->id > 0
&& (int) $job->user_id === (int) $backup->user_id
&& (int) $job->backup_id === (int) $backup->id
&& (int) $job->backup_restore_id === (int) $restore->id
&& (int) $job->host_id === (int) $attempt->target_host_id
&& (string) $job->kind === 'backup_restore'
&& (string) $job->execution_mode === 'agent'
&& (string) $job->phase === 'accepted'
&& (string) $job->status === 'queued'
&& hash_equals((string) $job->operation_id,
(string) $attempt->operation_id)
&& hash_equals((string) $job->request_idempotency_key,
(string) $attempt->attempt_key)
&& (string) $attempt->state === 'queued'
&& (int) $attempt->job_id === (int) $job->id
&& (string) $attempt->effect_resolution === 'not_applicable'
&& (string) $reservation->state === 'held'
&& Capsule::table('cloud_instances')->count() === $instanceCount,
'Reserved restore attempt was not atomically converted into one queued job.');
$assert((int) $restore->job_id === (int) $job->id,
'Generation-one job projection was not written to the parent restore.');
$payload = CloudSecrets::decrypt((string) $job->sealed_payload, 'job-payload');
$body = is_array($payload['body'] ?? null) ? $payload['body'] : [];
$public = is_array($body['public_network'] ?? null)
? $body['public_network'] : [];
$assert(($payload['method'] ?? null) === 'POST'
&& ($payload['request_target'] ?? null) === '/v1/vms/'
. rawurlencode((string) $attempt->target_uuid) . '/restore-backup'
&& ($payload['attempt_key'] ?? null) === (string) $attempt->attempt_key
&& (int) ($payload['generation'] ?? 0) === (int) $attempt->generation
&& ($payload['target_plan_sha256'] ?? null)
=== (string) $attempt->target_plan_sha256
&& ($payload['target_claim_set_sha256'] ?? null)
=== (string) $attempt->target_claim_set_sha256
&& ($payload['target_public_network_sha256'] ?? null)
=== (string) $attempt->target_public_network_sha256
&& ($payload['owner_id'] ?? null) === (int) $backup->user_id
&& ($body['uuid'] ?? null) === strtolower((string) $attempt->target_uuid)
&& ($body['name'] ?? null) === (string) $attempt->target_name
&& ($body['backup_key'] ?? null) === (string) $backup->backup_key
&& ($body['source_size_bytes'] ?? null)
=== (int) $backup->object_size_bytes
&& ($body['source_sha256'] ?? null) === (string) $backup->sha256
&& !array_key_exists('source_url', $body)
&& ($body['source'] ?? null) === ''
&& ($body['template_id'] ?? null) === ''
&& ($body['template_revision'] ?? null) === 0
&& ($body['template_generation_token'] ?? null) === ''
&& ($body['startup_script'] ?? null) === ''
&& ($body['hostname'] ?? null) === ''
&& ($body['guest_ssh_public_key'] ?? null) === ''
&& ($body['root_password'] ?? null) === ''
&& ($body['debian_12_network_fix'] ?? null) === false
&& is_array($body['platform_network_policy_snapshot'] ?? null)
&& ($public['contract'] ?? null) === 1
&& is_array($public['ipv4'] ?? null)
&& is_array($public['ipv6'] ?? null),
'Queued restore payload lost immutable backup or dual-stack request identity.');
foreach (['network', 'public_pool_key', 'public_pool_cidr',
'public_ip_cidr', 'gateway', 'dns_servers', 'vpc_port'] as $legacy) {
$assert(!array_key_exists($legacy, $body),
'Queued restore body retained legacy or VPC field: ' . $legacy);
}
$encodedPayload = json_encode(
$payload,
JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES
);
foreach (['restore-secret-value', 'RESTOREACCESS', 'sealed_storage_runtime',
'object_key', 'source_url', 'X-Amz-', 'vpc_port_id'] as $secret) {
$assert(!str_contains($encodedPayload, $secret),
'Queued restore payload leaked long-lived or unrelated data: ' . $secret);
}
$secretCheck = new ReflectionMethod(
CloudPortableRestoreJobEnvelope::class,
'assertSecretFree'
);
$legalNamePayload = $payload;
$legalNamePayload['body']['name'] = 'secret_access_key';
$secretCheck->invoke(null, $legalNamePayload);
// A bounded capability belongs only to the one Agent wire request. The worker
// persists only the exact Journal fingerprint before crossing the dispatch
// CAS, never the URL itself.
$prepareWire = new ReflectionMethod(
CloudWorkerService::class,
'portableRestoreDispatchPayload'
);
[$stableDispatchPayload, $wirePayload] = $prepareWire->invoke(
null,
$job,
$payload
);
$assert(str_starts_with(
(string) $wirePayload['body']['source_url'],
'https://'
) && str_contains(
(string) $wirePayload['body']['source_url'],
'X-Amz-Signature='
),
'Dispatch-time wire fixture did not contain a bounded download capability.');
$stableEncoded = json_encode(
$stableDispatchPayload,
JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES
);
$persistedIdentity = $stableDispatchPayload[
CloudOperationRecoveryIdentity::payloadKey()
] ?? null;
$assert(is_array($persistedIdentity)
&& !str_contains($stableEncoded, 'source_url')
&& !str_contains($stableEncoded, 'X-Amz-')
&& CloudOperationRecoveryIdentity::same(
CloudOperationRecoveryIdentity::expectedFromPayload(
$job,
$wirePayload
),
$persistedIdentity
),
'Worker did not separate the stable Journal identity from the wire capability.');
// Reconciliation must recover the exact wire fingerprint from stable durable
// state after dispatch, without retaining or recreating the expired URL.
$originalPhase = (string) $job->phase;
$originalDispatchAttempts = (int) $job->dispatch_attempts;
$originalSealed = (string) $job->sealed_payload;
$job->phase = 'dispatching';
$job->dispatch_attempts = 1;
$job->sealed_payload = CloudSecrets::encrypt(
$stableDispatchPayload,
'job-payload'
);
$assert(CloudOperationRecoveryIdentity::same(
CloudOperationRecoveryIdentity::expected($job),
$persistedIdentity
),
'Dispatched restore identity could not be rebuilt without its source URL.');
// Recovery authority follows the attempt ledger after dispatch and never
// requires the discarded URL. Each non-terminal state keeps the same stable
// Agent identity while enforcing the matching resource lifecycle.
foreach ([
['awaiting_receipt', 'not_applicable', 'held', 'reserved'],
['effect_unknown', 'unknown', 'uncertain', 'uncertain'],
['finalization_pending', 'applied', 'uncertain', 'uncertain'],
] as [$attemptState, $effect, $reservationState, $claimState]) {
$evidenceJson = match ($attemptState) {
'effect_unknown' => json_encode(
['effect_may_have_applied' => true],
JSON_THROW_ON_ERROR
),
'finalization_pending' => json_encode(
['proof' => 'validated'],
JSON_THROW_ON_ERROR
),
default => null,
};
Capsule::table('cloud_backup_restore_attempts')
->where('id', (int) $attempt->id)->update([
'state' => $attemptState,
'effect_resolution' => $effect,
'evidence_resolution' => $evidenceJson === null
? 'not_applicable' : $effect,
'evidence_json' => $evidenceJson,
'evidence_sha256' => $evidenceJson === null
? null : hash('sha256', $evidenceJson),
'evidence_recorded_at' => $evidenceJson === null
? null : time(),
'effect_resolved_at' => $attemptState === 'finalization_pending'
? time() : null,
]);
Capsule::table('cloud_resource_reservations')
->where('id', (int) $reservation->id)->update([
'state' => $reservationState,
]);
Capsule::table('cloud_resource_reservation_ips')
->where('reservation_id', (int) $reservation->id)->update([
'state' => $claimState,
]);
Capsule::table('cloud_public_ips')
->where('reservation_id', (int) $reservation->id)->update([
'state' => $claimState,
]);
$assert(CloudOperationRecoveryIdentity::same(
CloudOperationRecoveryIdentity::expected($job),
$persistedIdentity
),
'Recovery identity rejected valid attempt state: ' . $attemptState);
}
$tamperedRecoveryPayload = $stableDispatchPayload;
$tamperedRecoveryPayload['generation']++;
$job->sealed_payload = CloudSecrets::encrypt(
$tamperedRecoveryPayload,
'job-payload'
);
$rejects409(
static fn () => CloudOperationRecoveryIdentity::expected($job),
'Recovery identity accepted a changed attempt generation.'
);
$originalAttemptOperation = (string) $attempt->operation_id;
Capsule::table('cloud_backup_restore_attempts')
->where('id', (int) $attempt->id)->update([
'operation_id' => str_repeat('f', 32),
]);
$rejects409(
static fn () => CloudOperationRecoveryIdentity::expected($job),
'Portable ownership downgraded to legacy after an operation mismatch.'
);
Capsule::table('cloud_backup_restore_attempts')
->where('id', (int) $attempt->id)->update([
'operation_id' => $originalAttemptOperation,
]);
Capsule::table('cloud_backup_restore_attempts')
->where('id', (int) $attempt->id)->update([
'state' => 'queued',
'effect_resolution' => 'not_applicable',
'evidence_resolution' => 'not_applicable',
'evidence_json' => null,
'evidence_sha256' => null,
'evidence_recorded_at' => null,
'effect_resolved_at' => null,
]);
Capsule::table('cloud_resource_reservations')
->where('id', (int) $reservation->id)->update(['state' => 'held']);
Capsule::table('cloud_resource_reservation_ips')
->where('reservation_id', (int) $reservation->id)->update([
'state' => 'reserved',
]);
Capsule::table('cloud_public_ips')
->where('reservation_id', (int) $reservation->id)->update([
'state' => 'reserved',
]);
$job->sealed_payload = CloudSecrets::encrypt(
$stableDispatchPayload,
'job-payload'
);
// Admin inspection uses the typed dual-stack projection for attempt-owned
// restores while preserving the scalar contract only for legacy jobs.
$inspectProjection = new ReflectionMethod(
CloudAdminOperationRecoveryService::class,
'inspection'
);
$inspectionBase = [
'operation_id' => (string) $job->operation_id,
'action' => $persistedIdentity['action'],
'resource_id' => $persistedIdentity['resource_id'],
'request_fingerprint' => $persistedIdentity['request_fingerprint'],
'operation_status' => 'succeeded',
'checked_at' => time(),
'outcome' => 'succeeded',
'summary_code' => 'operation_succeeded',
'effect_resolution' => 'applied',
'fence_state' => 'not_required',
'fence_resolution_required' => false,
'abort_job' => ['available' => false],
'cleanup' => ['available' => false],
];
$rejectsInspection = static function (callable $callback, string $message) use (
$assert
): void {
try {
$callback();
} catch (CloudAgentError $error) {
$assert($error->httpStatus === 502, $message . ' (wrong status)');
return;
}
throw new RuntimeException($message);
};
$portableProjection = [
'uuid' => strtolower((string) $attempt->target_uuid),
'name' => (string) $attempt->target_name,
'vcpu' => (int) $body['vcpu'],
'memory_mib' => (int) $body['memory_mib'],
'disk_gib' => (int) $body['disk_gib'],
'bandwidth_mbps' => (int) $body['bandwidth_mbps'],
'public_network' => $public,
'machine' => 'q35',
'firmware' => 'uefi',
'secure_boot' => false,
'backup_key' => (string) $backup->backup_key,
'sha256' => (string) $backup->sha256,
];
$portableInspection = $inspectProjection->invoke(
null,
$inspectionBase + ['effect_projection' => $portableProjection],
(string) $job->operation_id,
'backup_restore',
true,
false,
true
);
$assert(($portableInspection['effect_projection']['public_network'] ?? null)
=== $public,
'Portable restore inspection lost its typed dual-stack projection.');
$legacyScalarProjection = [
'uuid' => strtolower((string) $attempt->target_uuid),
'public_ip_cidr' => (string) $public['ipv4']['address_cidr'],
'public_pool_key' => str_repeat('a', 32),
'backup_key' => (string) $backup->backup_key,
'sha256' => (string) $backup->sha256,
];
$rejectsInspection(
static fn () => $inspectProjection->invoke(
null,
$inspectionBase + ['effect_projection' => $legacyScalarProjection],
(string) $job->operation_id,
'backup_restore',
true,
false,
true
),
'Portable restore inspection accepted an IPv4 scalar-only projection.'
);
$rejectsInspection(
static fn () => $inspectProjection->invoke(
null,
$inspectionBase + ['effect_projection' => $legacyScalarProjection],
(string) $job->operation_id,
'backup_restore',
true,
false,
false
),
'Legacy restore inspection retained an IPv4 scalar-only projection.'
);
$notAppliedInspection = $inspectionBase;
$notAppliedInspection['operation_status'] = 'failed';
$notAppliedInspection['outcome'] = 'failed';
$notAppliedInspection['summary_code'] = 'operation_failed';
$notAppliedInspection['effect_resolution'] = 'not_applied';
$notAppliedInspection['effect_projection'] = ['creation_absent' => true];
$rejectsInspection(
static fn () => $inspectProjection->invoke(
null,
$notAppliedInspection,
(string) $job->operation_id,
'backup_restore',
true,
false,
true
),
'Portable not-applied inspection accepted incomplete absence evidence.'
);
$finishIdentity = new ReflectionMethod(
CloudWorkerService::class,
'operationIdentityForFinalization'
);
$assert(CloudOperationRecoveryIdentity::same(
$finishIdentity->invoke(
null,
$job,
$stableDispatchPayload,
false,
false
),
$persistedIdentity
),
'Normal restore finalization required the discarded source URL.');
$job->phase = $originalPhase;
$job->dispatch_attempts = $originalDispatchAttempts;
$job->sealed_payload = $originalSealed;
// Prepared recovery must compact the transient wire fixture while retaining
// the exact restore-attempt identity needed for convergence.
$compact = new ReflectionMethod(
CloudWorkerService::class,
'compactPreparedRecoveryPayload'
);
$prepared = $compact->invoke(null, 'backup_restore', $wirePayload);
$preparedEncoded = json_encode(
$prepared,
JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES
);
$assert(($prepared['attempt_key'] ?? null) === (string) $attempt->attempt_key
&& (int) ($prepared['generation'] ?? 0) === (int) $attempt->generation
&& ($prepared['target_plan_sha256'] ?? null)
=== (string) $attempt->target_plan_sha256
&& ($prepared['target_claim_set_sha256'] ?? null)
=== (string) $attempt->target_claim_set_sha256
&& ($prepared['target_public_network_sha256'] ?? null)
=== (string) $attempt->target_public_network_sha256
&& !str_contains($preparedEncoded, 'source_url')
&& !str_contains($preparedEncoded, 'X-Amz-')
&& !str_contains($preparedEncoded, 'RESTOREACCESS')
&& !str_contains($preparedEncoded, 'restore-secret-value'),
'Prepared recovery lost restore identity or retained a download capability.');
$replay = CloudPortableRestoreJobDispatcher::queue(
$actor,
(string) $attempt->attempt_key
);
$assert((int) $replay->id === (int) $job->id
&& CloudJob::count() === $jobCount + 1
&& Capsule::table('cloud_instances')->count() === $instanceCount
&& Capsule::table('cloud_resource_reservation_ips')
->where('reservation_id', (int) $reservation->id)
->orderBy('address_family')->get()->map(
static fn ($row): array => (array) $row
)->all() === $claimSnapshot,
'Exact queue replay created another job, instance, or changed claims.');
// Once queued, every durable identity must still be checked before returning
// the existing job; replay cannot hide a torn cross-table association.
$jobMutations = [
['operation_id', str_repeat('4', 32), (string) $job->operation_id],
['host_id', 1, (int) $job->host_id],
['user_id', 99, (int) $job->user_id],
['backup_id', 999, (int) $job->backup_id],
['backup_restore_id', 999, (int) $job->backup_restore_id],
];
foreach ($jobMutations as [$field, $invalid, $valid]) {
Capsule::table('cloud_jobs')->where('id', (int) $job->id)
->update([$field => $invalid]);
$rejects409(
static fn () => CloudPortableRestoreJobDispatcher::queue(
$actor,
(string) $attempt->attempt_key
),
'Queue replay accepted torn job identity: ' . $field . '.'
);
$assert(CloudJob::count() === $jobCount + 1,
'Rejected queue replay created another job.');
Capsule::table('cloud_jobs')->where('id', (int) $job->id)
->update([$field => $valid]);
}
// The remaining fault-injection cases run in an outer rollback so this file
// remains a reusable queued-attempt fixture for the finalizer test suite.
Capsule::connection()->beginTransaction();
try {
// Close the valid envelope fixture with exact local absence evidence so its
// addresses can be reused by the independent pre-dispatch failure cases.
$finish = new ReflectionMethod(CloudWorkerService::class, 'finish');
$finish->invoke(
new CloudWorkerService(),
$job,
[
'operation_id' => (string) $job->operation_id,
'status' => 'failed',
'result' => [
'creation_absent' => true,
'managed_artifacts_absent' => true,
],
'effect_may_have_applied' => false,
'safe_to_resubmit' => true,
],
$payload
);
$assert((string) CloudJob::findOrFail((int) $job->id)->effect_resolution
=== 'not_applied'
&& (string) CloudResourceReservation::findOrFail(
(int) $reservation->id
)->state === 'released',
'Valid never-dispatched restore fixture did not release its dual stack.');
// Exercise the public Worker seam rather than its private admission helper:
// a retired restore payload without an attempt identity must fail locally,
// before Agent contact, without changing any canonical resource ledger.
$missingAttemptPayload = $payload;
unset($missingAttemptPayload['attempt_key']);
$retiredRestoreJob = new CloudJob();
$retiredRestoreJob->operation_id = bin2hex(random_bytes(16));
$retiredRestoreJob->request_idempotency_key =
'retired-restore-worker-gate-0001';
$retiredRestoreJob->request_fingerprint = hash(
'sha256',
'retired-restore-worker-gate-0001'
);
$retiredRestoreJob->user_id = (int) $job->user_id;
$retiredRestoreJob->host_id = (int) $job->host_id;
$retiredRestoreJob->instance_id = null;
$retiredRestoreJob->backup_id = (int) $job->backup_id;
$retiredRestoreJob->backup_restore_id = (int) $job->backup_restore_id;
$retiredRestoreJob->kind = 'backup_restore';
$retiredRestoreJob->execution_mode = 'agent';
$retiredRestoreJob->sealed_payload = CloudSecrets::encrypt(
$missingAttemptPayload,
'job-payload'
);
$retiredRestoreJob->available_at = $retiredRestoreJob->created_at =
$retiredRestoreJob->updated_at = time();
$retiredRestoreJob->save();
$retiredRestoreIpLedger = Capsule::table('cloud_public_ips')
->orderBy('id')->get([
'id', 'state', 'reservation_id', 'instance_id',
])->map(static fn ($row): array => (array) $row)->all();
$retiredRestoreClaimLedger = Capsule::table(
'cloud_resource_reservation_ips'
)->orderBy('id')->get([
'id', 'state', 'reservation_id', 'public_ip_id', 'instance_id',
])->map(static fn ($row): array => (array) $row)->all();
$retiredRestoreAgentCalls = 0;
$retiredRestoreTransport = new \GuzzleHttp\Client([
'handler' => static function () use (&$retiredRestoreAgentCalls): void {
$retiredRestoreAgentCalls++;
throw new RuntimeException('Retired restore reached Agent.');
},
]);
$retiredRestoreAgent = new CloudAgentClient(
'3',
str_repeat('a', 32),
$retiredRestoreTransport,
20.0,
'http://127.0.0.1:8001/internal/agent/request',
str_repeat('b', 32)
);
$assert((new CloudWorkerService($retiredRestoreAgent))->tick() === true,
'Worker did not consume the retired restore job.');
$retiredRestoreJob->refresh();
$assert($retiredRestoreAgentCalls === 0
&& (int) $retiredRestoreJob->dispatch_attempts === 0
&& (string) $retiredRestoreJob->phase === 'terminal'
&& (string) $retiredRestoreJob->outcome === 'failed'
&& Capsule::table('cloud_public_ips')->orderBy('id')->get([
'id', 'state', 'reservation_id', 'instance_id',
])->map(static fn ($row): array => (array) $row)->all()
=== $retiredRestoreIpLedger
&& Capsule::table('cloud_resource_reservation_ips')
->orderBy('id')->get([
'id', 'state', 'reservation_id', 'public_ip_id', 'instance_id',
])->map(static fn ($row): array => (array) $row)->all()
=== $retiredRestoreClaimLedger,
'Retired restore reached Agent, remained claimable, or changed the '
. 'canonical address ledgers.');
// The restore remains artifact-backed. Corrupting the selected backend's
// credential envelope after reservation makes capability minting fail at
// the local pre-dispatch boundary without reviving any backup-owned secret.
$failurePlan = $plan;
$failureReserved = CloudPortableRestoreAttemptCoordinator::reserve(
$actor,
$backup,
$failurePlan,
$failurePlan->sha256(),
'predispatch-vm',
'predispatch-request-0000000001'
)->toArray();
$failureJob = CloudPortableRestoreJobDispatcher::queue(
$actor,
(string) $failureReserved['attempt_key']
);
$failureAttempt = CloudBackupRestoreAttempt::where(
'attempt_key',
(string) $failureReserved['attempt_key']
)->firstOrFail();
$failureReservation = CloudResourceReservation::findOrFail(
(int) $failureReserved['reservation_id']
);
$failureRestore = CloudBackupRestore::findOrFail(
(int) $failureAttempt->backup_restore_id
);
// A local capability-minting failure happens before the immutable dispatch
// CAS. It must never contact Agent; its exact not-dispatched evidence must
// atomically release both public-address claims and reserved host capacity.
$agentCalls = 0;
$tripwireTransport = new \GuzzleHttp\Client([
'handler' => static function () use (&$agentCalls): void {
$agentCalls++;
throw new RuntimeException(
'Local portable-restore failure reached Agent.'
);
},
]);
$tripwireAgent = new CloudAgentClient(
'3',
str_repeat('a', 32),
$tripwireTransport,
20.0,
'http://127.0.0.1:8001/internal/agent/request',
str_repeat('b', 32)
);
$worker = new CloudWorkerService($tripwireAgent);
$tickJob = new ReflectionMethod(CloudWorkerService::class, 'tickJob');
$validBackendCredentials = (string) Capsule::table(
'cloud_storage_backends'
)->where('id', 71)->value('sealed_credentials');
Capsule::table('cloud_storage_backends')->where('id', 71)->update([
'sealed_credentials' => 'invalid-backend-credential-envelope',
]);
try {
$assert($tickJob->invoke($worker) === true,
'Worker did not consume the local pre-dispatch restore failure.');
} finally {
Capsule::table('cloud_storage_backends')->where('id', 71)->update([
'sealed_credentials' => $validBackendCredentials,
]);
}
$failedJob = CloudJob::findOrFail((int) $failureJob->id);
$failedAttempt = CloudBackupRestoreAttempt::findOrFail(
(int) $failureAttempt->id
);
$failedReservation = CloudResourceReservation::findOrFail(
(int) $failureReservation->id
);
$failedRestore = CloudBackupRestore::findOrFail((int) $failureRestore->id);
$releasedClaims = Capsule::table('cloud_resource_reservation_ips')
->where('reservation_id', (int) $failureReservation->id)
->orderBy('address_family')->get();
$releasedAddresses = Capsule::table('cloud_public_ips')
->whereIn('id', $releasedClaims->pluck('public_ip_id')->all())
->orderBy('address_family')->get();
$durablePayload = (string) $failedJob->sealed_payload === '' ? []
: CloudSecrets::decrypt(
(string) $failedJob->sealed_payload,
'job-payload'
);
$assert($agentCalls === 0
&& (string) $failedJob->phase === 'terminal'
&& (string) $failedJob->outcome === 'failed'
&& (int) $failedJob->dispatch_attempts === 0
&& (string) $failedJob->effect_resolution === 'not_applied'
&& (int) $failedJob->finalization_incident === 0
&& (string) $failedAttempt->state === 'failed'
&& $failedAttempt->active_attempt === null
&& (string) $failedReservation->state === 'released'
&& (string) $failedRestore->state === 'failed'
&& $releasedClaims->count() === 2
&& $releasedClaims->every(static fn ($claim): bool =>
(string) $claim->state === 'released')
&& $releasedAddresses->count() === 2
&& $releasedAddresses->every(static fn ($address): bool =>
(string) $address->state === 'available'
&& $address->reservation_id === null
&& $address->instance_id === null)
&& !str_contains(json_encode(
$durablePayload,
JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES
), 'source_url'),
'Local pre-dispatch failure contacted Agent, retained a dual-stack claim, '
. 'or persisted its transient capability.');
// Corrupting one immutable envelope field is not an ordinary local failure.
// It must stop before Agent, retain every resource fence and publish an
// operator-visible finalization incident instead of releasing addresses.
$tornPlan = $plan;
$tornReserved = CloudPortableRestoreAttemptCoordinator::reserve(
$actor,
$backup,
$tornPlan,
$tornPlan->sha256(),
'torn-envelope-vm',
'torn-envelope-request-000000001'
)->toArray();
$tornJob = CloudPortableRestoreJobDispatcher::queue(
$actor,
(string) $tornReserved['attempt_key']
);
$tornPayload = CloudSecrets::decrypt(
(string) $tornJob->sealed_payload,
'job-payload'
);
$tornPayload['generation'] = (int) $tornPayload['generation'] + 1;
$tornJob->sealed_payload = CloudSecrets::encrypt(
$tornPayload,
'job-payload'
);
$tornJob->save();
$assert($tickJob->invoke($worker) === true,
'Worker did not consume the torn portable-restore envelope.');
$tornJob->refresh();
$tornAttempt = CloudBackupRestoreAttempt::where(
'attempt_key',
(string) $tornReserved['attempt_key']
)->firstOrFail();
$tornReservation = CloudResourceReservation::findOrFail(
(int) $tornReserved['reservation_id']
);
$tornClaims = Capsule::table('cloud_resource_reservation_ips')
->where('reservation_id', (int) $tornReservation->id)->get();
$assert($agentCalls === 0
&& (string) $tornJob->phase === 'terminal'
&& (string) $tornJob->outcome === 'failed'
&& (int) $tornJob->dispatch_attempts === 0
&& (string) $tornJob->effect_resolution === 'not_applied'
&& (int) $tornJob->finalization_incident === 1
&& (string) $tornAttempt->state === 'queued'
&& (int) $tornAttempt->active_attempt === 1
&& (string) $tornReservation->state === 'held'
&& $tornClaims->count() === 2
&& $tornClaims->every(static fn ($claim): bool =>
(string) $claim->state === 'reserved')
&& Capsule::table('cloud_instances')->count() === $instanceCount,
'Torn restore envelope reached Agent or released fenced dual-stack state.');
} finally {
Capsule::connection()->rollBack();
}
echo "Portable restore job dispatcher behavior tests passed.\n";