You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
virt/web/tests/CloudPortableRestoreJobDisp...

950 lines
38 KiB
PHP

<?php
ob_start();
require __DIR__ . '/CloudPortableRestoreAttemptCoordinatorTest.php';
ob_end_clean();
use App\Models\CloudBackupRestore;
use App\Models\CloudBackupRestoreAttempt;
use App\Models\CloudInstanceBackup;
use App\Models\CloudJob;
use App\Models\CloudResourceReservation;
use App\Services\Cloud\CloudError;
use App\Services\Cloud\CloudAdminOperationRecoveryService;
use App\Services\Cloud\CloudAgentClient;
use App\Services\Cloud\CloudAgentError;
use App\Services\Cloud\CloudOperationRecoveryIdentity;
use App\Services\Cloud\CloudPortableRestoreAttemptCoordinator;
use App\Services\Cloud\CloudPortableRestoreJobDispatcher;
use App\Services\Cloud\CloudPortableRestoreJobEnvelope;
use App\Services\Cloud\CloudPortableRestoreTargetPlan;
use App\Services\Cloud\CloudPortableRestoreTargetService;
use App\Services\Cloud\CloudSecrets;
use App\Services\Cloud\CloudWorkerService;
use Illuminate\Database\Capsule\Manager as Capsule;
use Illuminate\Database\Schema\Blueprint;
if (!function_exists('sodium_crypto_aead_xchacha20poly1305_ietf_encrypt')) {
echo "SKIPPED CloudPortableRestoreJobDispatcherTest: runtime lacks sodium.\n";
exit(0);
}
if (!class_exists(CloudPortableRestoreJobDispatcher::class)) {
throw new RuntimeException(
'CloudPortableRestoreJobDispatcher production seam is not implemented.'
);
}
$rejects409 = static function (callable $callback, string $message) use (
$assert
): void {
try {
$callback();
} catch (CloudError $error) {
$assert($error->httpStatus === 409, $message . ' (wrong status)');
return;
}
throw new RuntimeException($message);
};
// The coordinator fixture deliberately carries only fields it needs. Extend
// the same in-memory job table with the worker envelope columns exercised by
// this stage; production schema already owns these columns.
$schema->table('cloud_jobs', static function (Blueprint $t): void {
$t->string('request_idempotency_key')->default('');
$t->string('request_fingerprint', 64)->default('');
$t->integer('instance_id')->nullable();
$t->integer('parent_job_id')->nullable();
$t->integer('vpc_repair_root_job_id')->nullable();
$t->string('execution_mode')->default('agent');
$t->string('status')->default('queued');
$t->text('sealed_payload')->default('');
$t->integer('attempts')->default(0);
$t->integer('dispatch_attempts')->default(0);
$t->integer('dispatch_deadline_at')->default(0);
$t->integer('dispatch_started_at')->nullable();
$t->integer('agent_accepted_at')->nullable();
$t->integer('receipt_deadline_at')->nullable();
$t->integer('effect_may_have_applied')->nullable();
$t->integer('safe_to_resubmit')->nullable();
$t->integer('effect_resolved_at')->nullable();
$t->integer('effect_resolved_by')->nullable();
$t->string('effect_resolution_reason')->default('');
$t->string('finalization_error_code')->nullable();
$t->string('completion_event_id')->nullable();
$t->integer('read_at')->nullable();
$t->integer('dispatched_at')->nullable();
$t->string('lease_token')->nullable();
$t->integer('lease_until')->default(0);
$t->integer('available_at')->default(0);
$t->string('error_code')->nullable();
$t->text('result_json')->nullable();
$t->integer('created_at')->default(0);
$t->integer('updated_at')->default(0);
});
$schema->create('cloud_agent_completion_events', static function (
Blueprint $t
): void {
$t->increments('id');
$t->integer('host_id');
$t->string('operation_id');
});
Capsule::table('sqlite_sequence')->where('name', 'cloud_jobs')->update([
'seq' => 600,
]);
global $System_Config;
$System_Config['cloud_data_key'] = base64_encode(str_repeat('r', 32));
$runtime = [
'config' => [
'endpoint' => 'https://objects.example.com',
'region' => 'us-east-1',
'bucket' => 'portable-backups',
'prefix' => 'virt',
'path_style' => true,
],
'credentials' => [
'access_key_id' => 'RESTOREACCESS',
'secret_access_key' => 'restore-secret-value',
],
'revision' => 7,
];
Capsule::table('cloud_instance_backups')->where('id', 1)->update([
'sealed_storage_runtime' => CloudSecrets::encrypt(
$runtime,
'object-storage-backup-runtime'
),
]);
$backup = CloudInstanceBackup::findOrFail(1);
$plans = CloudPortableRestoreTargetService::candidates($backup);
$plan = array_values(array_filter(
$plans,
static fn (CloudPortableRestoreTargetPlan $candidate): bool =>
$candidate->toArray()['target']['host_id'] === 3
&& $candidate->toArray()['target']['resource_pool_id'] === 300
))[0] ?? null;
$assert($plan instanceof CloudPortableRestoreTargetPlan,
'Fresh portable restore dispatch plan is unavailable.');
$requestKey = 'dispatch-request-0000000000000001';
$reserved = CloudPortableRestoreAttemptCoordinator::reserve(
$actor,
$backup,
$plan,
$plan->sha256(),
'dispatch-vm',
$requestKey
)->toArray();
$attempt = CloudBackupRestoreAttempt::where(
'attempt_key',
$reserved['attempt_key']
)->firstOrFail();
$reservation = CloudResourceReservation::findOrFail(
$reserved['reservation_id']
);
$jobCount = CloudJob::count();
$instanceCount = Capsule::table('cloud_instances')->count();
$claimSnapshot = Capsule::table('cloud_resource_reservation_ips')
->where('reservation_id', (int) $reservation->id)
->orderBy('address_family')->get()->map(
static fn ($row): array => (array) $row
)->all();
// Any torn input must roll back without an orphan job or changing the attempt.
$reservation->operation_id = str_repeat('7', 32);
$reservation->save();
$rejects409(
static fn () => CloudPortableRestoreJobDispatcher::queue(
$actor,
(string) $attempt->attempt_key
),
'Dispatcher accepted a torn reservation operation.'
);
$assert(CloudJob::count() === $jobCount
&& (string) CloudBackupRestoreAttempt::findOrFail($attempt->id)->state
=== 'reserved',
'Rejected dispatch left a job or changed attempt state.');
$reservation->operation_id = (string) $attempt->operation_id;
$reservation->save();
$restoreBeforeQueue = CloudBackupRestore::findOrFail(
(int) $attempt->backup_restore_id
);
$restoreBeforeQueue->target_instance_id = 777;
$restoreBeforeQueue->save();
$rejects409(
static fn () => CloudPortableRestoreJobDispatcher::queue(
$actor,
(string) $attempt->attempt_key
),
'Dispatcher accepted a pre-created target instance.'
);
$restoreBeforeQueue->target_instance_id = null;
$restoreBeforeQueue->save();
foreach (['backup_restore_id' => 777, 'order_id' => 777] as $field => $value) {
$reservation->{$field} = $value;
$reservation->save();
$rejects409(
static fn () => CloudPortableRestoreJobDispatcher::queue(
$actor,
(string) $attempt->attempt_key
),
'Dispatcher accepted a doubly-owned reservation: ' . $field . '.'
);
$reservation->{$field} = null;
$reservation->save();
}
$assert(CloudJob::count() === $jobCount
&& (string) CloudBackupRestoreAttempt::findOrFail($attempt->id)->state
=== 'reserved',
'Rejected ownership tear left a job or changed attempt state.');
$ipv6Claim = Capsule::table('cloud_resource_reservation_ips')
->where('reservation_id', (int) $reservation->id)
->where('address_family', 6)->first();
$assert(is_object($ipv6Claim), 'IPv6 claim fixture is missing.');
Capsule::table('cloud_resource_reservation_ips')
->where('id', (int) $ipv6Claim->id)
->update(['operation_id' => str_repeat('6', 32)]);
$rejects409(
static fn () => CloudPortableRestoreJobDispatcher::queue(
$actor,
(string) $attempt->attempt_key
),
'Dispatcher accepted a torn IPv6 claim.'
);
$assert(CloudJob::count() === $jobCount
&& Capsule::table('cloud_instances')->count() === $instanceCount,
'Rejected claim dispatch produced a job or instance.');
Capsule::table('cloud_resource_reservation_ips')
->where('id', (int) $ipv6Claim->id)
->update(['operation_id' => (string) $attempt->operation_id]);
$originalPlanSha = (string) $attempt->target_plan_sha256;
$attempt->target_plan_sha256 = str_repeat('5', 64);
$attempt->save();
$rejects409(
static fn () => CloudPortableRestoreJobDispatcher::queue(
$actor,
(string) $attempt->attempt_key
),
'Dispatcher accepted a torn target plan.'
);
$attempt->target_plan_sha256 = $originalPlanSha;
$attempt->save();
$job = CloudPortableRestoreJobDispatcher::queue(
$actor,
(string) $attempt->attempt_key
);
$attempt = CloudBackupRestoreAttempt::findOrFail($attempt->id);
$restore = CloudBackupRestore::findOrFail($attempt->backup_restore_id);
$reservation = CloudResourceReservation::findOrFail($reservation->id);
$assert((int) $job->id > 0
&& (int) $job->user_id === (int) $backup->user_id
&& (int) $job->backup_id === (int) $backup->id
&& (int) $job->backup_restore_id === (int) $restore->id
&& (int) $job->host_id === (int) $attempt->target_host_id
&& (string) $job->kind === 'backup_restore'
&& (string) $job->execution_mode === 'agent'
&& (string) $job->phase === 'accepted'
&& (string) $job->status === 'queued'
&& hash_equals((string) $job->operation_id,
(string) $attempt->operation_id)
&& hash_equals((string) $job->request_idempotency_key,
(string) $attempt->attempt_key)
&& (string) $attempt->state === 'queued'
&& (int) $attempt->job_id === (int) $job->id
&& (string) $attempt->effect_resolution === 'not_applicable'
&& (string) $reservation->state === 'held'
&& Capsule::table('cloud_instances')->count() === $instanceCount,
'Reserved restore attempt was not atomically converted into one queued job.');
$assert((int) $restore->job_id === (int) $job->id,
'Generation-one job projection was not written to the parent restore.');
$payload = CloudSecrets::decrypt((string) $job->sealed_payload, 'job-payload');
$body = is_array($payload['body'] ?? null) ? $payload['body'] : [];
$public = is_array($body['public_network'] ?? null)
? $body['public_network'] : [];
$assert(($payload['method'] ?? null) === 'POST'
&& ($payload['request_target'] ?? null) === '/v1/vms/'
. rawurlencode((string) $attempt->target_uuid) . '/restore-backup'
&& ($payload['attempt_key'] ?? null) === (string) $attempt->attempt_key
&& (int) ($payload['generation'] ?? 0) === (int) $attempt->generation
&& ($payload['target_plan_sha256'] ?? null)
=== (string) $attempt->target_plan_sha256
&& ($payload['target_claim_set_sha256'] ?? null)
=== (string) $attempt->target_claim_set_sha256
&& ($payload['target_public_network_sha256'] ?? null)
=== (string) $attempt->target_public_network_sha256
&& ($payload['owner_id'] ?? null) === (int) $backup->user_id
&& ($body['uuid'] ?? null) === strtolower((string) $attempt->target_uuid)
&& ($body['name'] ?? null) === (string) $attempt->target_name
&& ($body['backup_key'] ?? null) === (string) $backup->backup_key
&& ($body['source_size_bytes'] ?? null)
=== (int) $backup->object_size_bytes
&& ($body['source_sha256'] ?? null) === (string) $backup->sha256
&& !array_key_exists('source_url', $body)
&& ($body['source'] ?? null) === ''
&& ($body['template_id'] ?? null) === ''
&& ($body['template_revision'] ?? null) === 0
&& ($body['template_generation_token'] ?? null) === ''
&& ($body['startup_script'] ?? null) === ''
&& ($body['hostname'] ?? null) === ''
&& ($body['guest_ssh_public_key'] ?? null) === ''
&& ($body['root_password'] ?? null) === ''
&& ($body['debian_12_network_fix'] ?? null) === false
&& is_array($body['platform_network_policy_snapshot'] ?? null)
&& ($public['contract'] ?? null) === 1
&& is_array($public['ipv4'] ?? null)
&& is_array($public['ipv6'] ?? null),
'Queued restore payload lost immutable backup or dual-stack request identity.');
foreach (['network', 'public_pool_key', 'public_pool_cidr',
'public_ip_cidr', 'gateway', 'dns_servers', 'vpc_port'] as $legacy) {
$assert(!array_key_exists($legacy, $body),
'Queued restore body retained legacy or VPC field: ' . $legacy);
}
$encodedPayload = json_encode(
$payload,
JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES
);
foreach (['restore-secret-value', 'RESTOREACCESS', 'sealed_storage_runtime',
'object_key', 'source_url', 'X-Amz-', 'vpc_port_id'] as $secret) {
$assert(!str_contains($encodedPayload, $secret),
'Queued restore payload leaked long-lived or unrelated data: ' . $secret);
}
$secretCheck = new ReflectionMethod(
CloudPortableRestoreJobEnvelope::class,
'assertSecretFree'
);
$legalNamePayload = $payload;
$legalNamePayload['body']['name'] = 'secret_access_key';
$secretCheck->invoke(null, $legalNamePayload);
// A bounded capability belongs only to the one Agent wire request. The worker
// persists only the exact Journal fingerprint before crossing the dispatch
// CAS, never the URL itself.
$prepareWire = new ReflectionMethod(
CloudWorkerService::class,
'portableRestoreDispatchPayload'
);
[$stableDispatchPayload, $wirePayload] = $prepareWire->invoke(
null,
$job,
$payload
);
$assert(str_starts_with(
(string) $wirePayload['body']['source_url'],
'https://'
) && str_contains(
(string) $wirePayload['body']['source_url'],
'X-Amz-Signature='
),
'Dispatch-time wire fixture did not contain a bounded download capability.');
$stableEncoded = json_encode(
$stableDispatchPayload,
JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES
);
$persistedIdentity = $stableDispatchPayload[
CloudOperationRecoveryIdentity::payloadKey()
] ?? null;
$assert(is_array($persistedIdentity)
&& !str_contains($stableEncoded, 'source_url')
&& !str_contains($stableEncoded, 'X-Amz-')
&& CloudOperationRecoveryIdentity::same(
CloudOperationRecoveryIdentity::expectedFromPayload(
$job,
$wirePayload
),
$persistedIdentity
),
'Worker did not separate the stable Journal identity from the wire capability.');
// Reconciliation must recover the exact wire fingerprint from stable durable
// state after dispatch, without retaining or recreating the expired URL.
$originalPhase = (string) $job->phase;
$originalDispatchAttempts = (int) $job->dispatch_attempts;
$originalSealed = (string) $job->sealed_payload;
$job->phase = 'dispatching';
$job->dispatch_attempts = 1;
$job->sealed_payload = CloudSecrets::encrypt(
$stableDispatchPayload,
'job-payload'
);
$assert(CloudOperationRecoveryIdentity::same(
CloudOperationRecoveryIdentity::expected($job),
$persistedIdentity
),
'Dispatched restore identity could not be rebuilt without its source URL.');
// Recovery authority follows the attempt ledger after dispatch and never
// requires the discarded URL. Each non-terminal state keeps the same stable
// Agent identity while enforcing the matching resource lifecycle.
foreach ([
['awaiting_receipt', 'not_applicable', 'held', 'reserved'],
['effect_unknown', 'unknown', 'uncertain', 'uncertain'],
['finalization_pending', 'applied', 'uncertain', 'uncertain'],
] as [$attemptState, $effect, $reservationState, $claimState]) {
$evidenceJson = match ($attemptState) {
'effect_unknown' => json_encode(
['effect_may_have_applied' => true],
JSON_THROW_ON_ERROR
),
'finalization_pending' => json_encode(
['proof' => 'validated'],
JSON_THROW_ON_ERROR
),
default => null,
};
Capsule::table('cloud_backup_restore_attempts')
->where('id', (int) $attempt->id)->update([
'state' => $attemptState,
'effect_resolution' => $effect,
'evidence_resolution' => $evidenceJson === null
? 'not_applicable' : $effect,
'evidence_json' => $evidenceJson,
'evidence_sha256' => $evidenceJson === null
? null : hash('sha256', $evidenceJson),
'evidence_recorded_at' => $evidenceJson === null
? null : time(),
'effect_resolved_at' => $attemptState === 'finalization_pending'
? time() : null,
]);
Capsule::table('cloud_resource_reservations')
->where('id', (int) $reservation->id)->update([
'state' => $reservationState,
]);
Capsule::table('cloud_resource_reservation_ips')
->where('reservation_id', (int) $reservation->id)->update([
'state' => $claimState,
]);
Capsule::table('cloud_public_ips')
->where('reservation_id', (int) $reservation->id)->update([
'state' => $claimState,
]);
$assert(CloudOperationRecoveryIdentity::same(
CloudOperationRecoveryIdentity::expected($job),
$persistedIdentity
),
'Recovery identity rejected valid attempt state: ' . $attemptState);
}
$tamperedRecoveryPayload = $stableDispatchPayload;
$tamperedRecoveryPayload['generation']++;
$job->sealed_payload = CloudSecrets::encrypt(
$tamperedRecoveryPayload,
'job-payload'
);
$rejects409(
static fn () => CloudOperationRecoveryIdentity::expected($job),
'Recovery identity accepted a changed attempt generation.'
);
$originalAttemptOperation = (string) $attempt->operation_id;
Capsule::table('cloud_backup_restore_attempts')
->where('id', (int) $attempt->id)->update([
'operation_id' => str_repeat('f', 32),
]);
$rejects409(
static fn () => CloudOperationRecoveryIdentity::expected($job),
'Portable ownership downgraded to legacy after an operation mismatch.'
);
Capsule::table('cloud_backup_restore_attempts')
->where('id', (int) $attempt->id)->update([
'operation_id' => $originalAttemptOperation,
]);
Capsule::table('cloud_backup_restore_attempts')
->where('id', (int) $attempt->id)->update([
'state' => 'queued',
'effect_resolution' => 'not_applicable',
'evidence_resolution' => 'not_applicable',
'evidence_json' => null,
'evidence_sha256' => null,
'evidence_recorded_at' => null,
'effect_resolved_at' => null,
]);
Capsule::table('cloud_resource_reservations')
->where('id', (int) $reservation->id)->update(['state' => 'held']);
Capsule::table('cloud_resource_reservation_ips')
->where('reservation_id', (int) $reservation->id)->update([
'state' => 'reserved',
]);
Capsule::table('cloud_public_ips')
->where('reservation_id', (int) $reservation->id)->update([
'state' => 'reserved',
]);
$job->sealed_payload = CloudSecrets::encrypt(
$stableDispatchPayload,
'job-payload'
);
// Admin inspection uses the typed dual-stack projection for attempt-owned
// restores while preserving the scalar contract only for legacy jobs.
$inspectProjection = new ReflectionMethod(
CloudAdminOperationRecoveryService::class,
'inspection'
);
$inspectionBase = [
'operation_id' => (string) $job->operation_id,
'action' => $persistedIdentity['action'],
'resource_id' => $persistedIdentity['resource_id'],
'request_fingerprint' => $persistedIdentity['request_fingerprint'],
'operation_status' => 'succeeded',
'checked_at' => time(),
'outcome' => 'succeeded',
'summary_code' => 'operation_succeeded',
'effect_resolution' => 'applied',
'fence_state' => 'not_required',
'fence_resolution_required' => false,
'abort_job' => ['available' => false],
'cleanup' => ['available' => false],
];
$rejectsInspection = static function (callable $callback, string $message) use (
$assert
): void {
try {
$callback();
} catch (CloudAgentError $error) {
$assert($error->httpStatus === 502, $message . ' (wrong status)');
return;
}
throw new RuntimeException($message);
};
$portableProjection = [
'uuid' => strtolower((string) $attempt->target_uuid),
'name' => (string) $attempt->target_name,
'vcpu' => (int) $body['vcpu'],
'memory_mib' => (int) $body['memory_mib'],
'disk_gib' => (int) $body['disk_gib'],
'bandwidth_mbps' => (int) $body['bandwidth_mbps'],
'public_network' => $public,
'machine' => 'q35',
'firmware' => 'uefi',
'secure_boot' => false,
'backup_key' => (string) $backup->backup_key,
'sha256' => (string) $backup->sha256,
];
$portableInspection = $inspectProjection->invoke(
null,
$inspectionBase + ['effect_projection' => $portableProjection],
(string) $job->operation_id,
'backup_restore',
true,
false,
true
);
$assert(($portableInspection['effect_projection']['public_network'] ?? null)
=== $public,
'Portable restore inspection lost its typed dual-stack projection.');
$legacyScalarProjection = [
'uuid' => strtolower((string) $attempt->target_uuid),
'public_ip_cidr' => (string) $public['ipv4']['address_cidr'],
'public_pool_key' => str_repeat('a', 32),
'backup_key' => (string) $backup->backup_key,
'sha256' => (string) $backup->sha256,
];
$rejectsInspection(
static fn () => $inspectProjection->invoke(
null,
$inspectionBase + ['effect_projection' => $legacyScalarProjection],
(string) $job->operation_id,
'backup_restore',
true,
false,
true
),
'Portable restore inspection accepted an IPv4 scalar-only projection.'
);
$rejectsInspection(
static fn () => $inspectProjection->invoke(
null,
$inspectionBase + ['effect_projection' => $legacyScalarProjection],
(string) $job->operation_id,
'backup_restore',
true,
false,
false
),
'Legacy restore inspection retained an IPv4 scalar-only projection.'
);
$notAppliedInspection = $inspectionBase;
$notAppliedInspection['operation_status'] = 'failed';
$notAppliedInspection['outcome'] = 'failed';
$notAppliedInspection['summary_code'] = 'operation_failed';
$notAppliedInspection['effect_resolution'] = 'not_applied';
$notAppliedInspection['effect_projection'] = ['creation_absent' => true];
$rejectsInspection(
static fn () => $inspectProjection->invoke(
null,
$notAppliedInspection,
(string) $job->operation_id,
'backup_restore',
true,
false,
true
),
'Portable not-applied inspection accepted incomplete absence evidence.'
);
$finishIdentity = new ReflectionMethod(
CloudWorkerService::class,
'operationIdentityForFinalization'
);
$assert(CloudOperationRecoveryIdentity::same(
$finishIdentity->invoke(
null,
$job,
$stableDispatchPayload,
false,
false
),
$persistedIdentity
),
'Normal restore finalization required the discarded source URL.');
$job->phase = $originalPhase;
$job->dispatch_attempts = $originalDispatchAttempts;
$job->sealed_payload = $originalSealed;
// Prepared recovery must compact the transient wire fixture while retaining
// the exact restore-attempt identity needed for convergence.
$compact = new ReflectionMethod(
CloudWorkerService::class,
'compactPreparedRecoveryPayload'
);
$prepared = $compact->invoke(null, 'backup_restore', $wirePayload);
$preparedEncoded = json_encode(
$prepared,
JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES
);
$assert(($prepared['attempt_key'] ?? null) === (string) $attempt->attempt_key
&& (int) ($prepared['generation'] ?? 0) === (int) $attempt->generation
&& ($prepared['target_plan_sha256'] ?? null)
=== (string) $attempt->target_plan_sha256
&& ($prepared['target_claim_set_sha256'] ?? null)
=== (string) $attempt->target_claim_set_sha256
&& ($prepared['target_public_network_sha256'] ?? null)
=== (string) $attempt->target_public_network_sha256
&& !str_contains($preparedEncoded, 'source_url')
&& !str_contains($preparedEncoded, 'X-Amz-')
&& !str_contains($preparedEncoded, 'RESTOREACCESS')
&& !str_contains($preparedEncoded, 'restore-secret-value'),
'Prepared recovery lost restore identity or retained a download capability.');
$replay = CloudPortableRestoreJobDispatcher::queue(
$actor,
(string) $attempt->attempt_key
);
$assert((int) $replay->id === (int) $job->id
&& CloudJob::count() === $jobCount + 1
&& Capsule::table('cloud_instances')->count() === $instanceCount
&& Capsule::table('cloud_resource_reservation_ips')
->where('reservation_id', (int) $reservation->id)
->orderBy('address_family')->get()->map(
static fn ($row): array => (array) $row
)->all() === $claimSnapshot,
'Exact queue replay created another job, instance, or changed claims.');
// Once queued, every durable identity must still be checked before returning
// the existing job; replay cannot hide a torn cross-table association.
$jobMutations = [
['operation_id', str_repeat('4', 32), (string) $job->operation_id],
['host_id', 1, (int) $job->host_id],
['user_id', 99, (int) $job->user_id],
['backup_id', 999, (int) $job->backup_id],
['backup_restore_id', 999, (int) $job->backup_restore_id],
];
foreach ($jobMutations as [$field, $invalid, $valid]) {
Capsule::table('cloud_jobs')->where('id', (int) $job->id)
->update([$field => $invalid]);
$rejects409(
static fn () => CloudPortableRestoreJobDispatcher::queue(
$actor,
(string) $attempt->attempt_key
),
'Queue replay accepted torn job identity: ' . $field . '.'
);
$assert(CloudJob::count() === $jobCount + 1,
'Rejected queue replay created another job.');
Capsule::table('cloud_jobs')->where('id', (int) $job->id)
->update([$field => $valid]);
}
// The remaining fault-injection cases run in an outer rollback so this file
// remains a reusable queued-attempt fixture for the finalizer test suite.
Capsule::connection()->beginTransaction();
try {
// Close the valid envelope fixture with exact local absence evidence so its
// addresses can be reused by the independent pre-dispatch failure cases.
$finish = new ReflectionMethod(CloudWorkerService::class, 'finish');
$finish->invoke(
new CloudWorkerService(),
$job,
[
'operation_id' => (string) $job->operation_id,
'status' => 'failed',
'result' => [
'creation_absent' => true,
'managed_artifacts_absent' => true,
],
'effect_may_have_applied' => false,
'safe_to_resubmit' => true,
],
$payload
);
$assert((string) CloudJob::findOrFail((int) $job->id)->effect_resolution
=== 'not_applied'
&& (string) CloudResourceReservation::findOrFail(
(int) $reservation->id
)->state === 'released',
'Valid never-dispatched restore fixture did not release its dual stack.');
// Exercise the public Worker seam rather than its private admission helper:
// a legacy restore payload without an attempt identity must fail locally,
// before Agent contact, without changing any canonical resource ledger.
$missingAttemptPayload = $payload;
unset($missingAttemptPayload['attempt_key']);
$legacyRestoreJob = new CloudJob();
$legacyRestoreJob->operation_id = bin2hex(random_bytes(16));
$legacyRestoreJob->request_idempotency_key =
'legacy-restore-worker-gate-0001';
$legacyRestoreJob->request_fingerprint = hash(
'sha256',
'legacy-restore-worker-gate-0001'
);
$legacyRestoreJob->user_id = (int) $job->user_id;
$legacyRestoreJob->host_id = (int) $job->host_id;
$legacyRestoreJob->instance_id = null;
$legacyRestoreJob->backup_id = (int) $job->backup_id;
$legacyRestoreJob->backup_restore_id = (int) $job->backup_restore_id;
$legacyRestoreJob->kind = 'backup_restore';
$legacyRestoreJob->execution_mode = 'agent';
$legacyRestoreJob->sealed_payload = CloudSecrets::encrypt(
$missingAttemptPayload,
'job-payload'
);
$legacyRestoreJob->available_at = $legacyRestoreJob->created_at =
$legacyRestoreJob->updated_at = time();
$legacyRestoreJob->save();
$legacyRestoreIpLedger = Capsule::table('cloud_public_ips')
->orderBy('id')->get([
'id', 'state', 'reservation_id', 'instance_id',
])->map(static fn ($row): array => (array) $row)->all();
$legacyRestoreClaimLedger = Capsule::table(
'cloud_resource_reservation_ips'
)->orderBy('id')->get([
'id', 'state', 'reservation_id', 'public_ip_id', 'instance_id',
])->map(static fn ($row): array => (array) $row)->all();
$legacyRestoreAgentCalls = 0;
$legacyRestoreTransport = new \GuzzleHttp\Client([
'handler' => static function () use (&$legacyRestoreAgentCalls): void {
$legacyRestoreAgentCalls++;
throw new RuntimeException('Legacy restore reached Agent.');
},
]);
$legacyRestoreAgent = new CloudAgentClient(
'3',
str_repeat('a', 32),
$legacyRestoreTransport,
20.0,
'http://127.0.0.1:8001/internal/agent/request',
str_repeat('b', 32)
);
$assert((new CloudWorkerService($legacyRestoreAgent))->tick() === true,
'Worker did not consume the legacy restore job.');
$legacyRestoreJob->refresh();
$assert($legacyRestoreAgentCalls === 0
&& (int) $legacyRestoreJob->dispatch_attempts === 0
&& (string) $legacyRestoreJob->phase === 'terminal'
&& (string) $legacyRestoreJob->outcome === 'failed'
&& Capsule::table('cloud_public_ips')->orderBy('id')->get([
'id', 'state', 'reservation_id', 'instance_id',
])->map(static fn ($row): array => (array) $row)->all()
=== $legacyRestoreIpLedger
&& Capsule::table('cloud_resource_reservation_ips')
->orderBy('id')->get([
'id', 'state', 'reservation_id', 'public_ip_id', 'instance_id',
])->map(static fn ($row): array => (array) $row)->all()
=== $legacyRestoreClaimLedger,
'Legacy restore reached Agent, remained claimable, or changed the '
. 'canonical address ledgers.');
// Freeze an internally consistent backup whose stored object-runtime shape
// cannot mint a capability. The immutable plan binds that exact runtime;
// changing an already-frozen ciphertext would be ledger damage instead.
$invalidRuntime = [
'config' => [],
'credentials' => [],
'revision' => (int) $backup->storage_revision,
];
$invalidBackup = $backup->replicate();
$invalidBackup->backup_key = str_repeat('d', 32);
$invalidBackup->object_key = (string) $backup->object_key
. '.predispatch';
$invalidBackup->sealed_storage_runtime = CloudSecrets::encrypt(
$invalidRuntime,
'object-storage-backup-runtime'
);
$invalidBackup->save();
$failurePlan = array_values(array_filter(
CloudPortableRestoreTargetService::candidates($invalidBackup),
static fn (CloudPortableRestoreTargetPlan $candidate): bool =>
(int) $candidate->toArray()['target']['host_id'] === 3
))[0] ?? null;
$assert($failurePlan instanceof CloudPortableRestoreTargetPlan,
'Pre-dispatch failure target plan is unavailable.');
$failureReserved = CloudPortableRestoreAttemptCoordinator::reserve(
$actor,
$invalidBackup,
$failurePlan,
$failurePlan->sha256(),
'predispatch-vm',
'predispatch-request-0000000001'
)->toArray();
$failureJob = CloudPortableRestoreJobDispatcher::queue(
$actor,
(string) $failureReserved['attempt_key']
);
$failureAttempt = CloudBackupRestoreAttempt::where(
'attempt_key',
(string) $failureReserved['attempt_key']
)->firstOrFail();
$failureReservation = CloudResourceReservation::findOrFail(
(int) $failureReserved['reservation_id']
);
$failureRestore = CloudBackupRestore::findOrFail(
(int) $failureAttempt->backup_restore_id
);
// A local capability-minting failure happens before the immutable dispatch
// CAS. It must never contact Agent; its exact not-dispatched evidence must
// atomically release both public-address claims and reserved host capacity.
$agentCalls = 0;
$tripwireTransport = new \GuzzleHttp\Client([
'handler' => static function () use (&$agentCalls): void {
$agentCalls++;
throw new RuntimeException(
'Local portable-restore failure reached Agent.'
);
},
]);
$tripwireAgent = new CloudAgentClient(
'3',
str_repeat('a', 32),
$tripwireTransport,
20.0,
'http://127.0.0.1:8001/internal/agent/request',
str_repeat('b', 32)
);
$worker = new CloudWorkerService($tripwireAgent);
$tickJob = new ReflectionMethod(CloudWorkerService::class, 'tickJob');
$assert($tickJob->invoke($worker) === true,
'Worker did not consume the local pre-dispatch restore failure.');
$failedJob = CloudJob::findOrFail((int) $failureJob->id);
$failedAttempt = CloudBackupRestoreAttempt::findOrFail(
(int) $failureAttempt->id
);
$failedReservation = CloudResourceReservation::findOrFail(
(int) $failureReservation->id
);
$failedRestore = CloudBackupRestore::findOrFail((int) $failureRestore->id);
$releasedClaims = Capsule::table('cloud_resource_reservation_ips')
->where('reservation_id', (int) $failureReservation->id)
->orderBy('address_family')->get();
$releasedAddresses = Capsule::table('cloud_public_ips')
->whereIn('id', $releasedClaims->pluck('public_ip_id')->all())
->orderBy('address_family')->get();
$durablePayload = (string) $failedJob->sealed_payload === '' ? []
: CloudSecrets::decrypt(
(string) $failedJob->sealed_payload,
'job-payload'
);
$assert($agentCalls === 0
&& (string) $failedJob->phase === 'terminal'
&& (string) $failedJob->outcome === 'failed'
&& (int) $failedJob->dispatch_attempts === 0
&& (string) $failedJob->effect_resolution === 'not_applied'
&& (int) $failedJob->finalization_incident === 0
&& (string) $failedAttempt->state === 'failed'
&& $failedAttempt->active_attempt === null
&& (string) $failedReservation->state === 'released'
&& (string) $failedRestore->state === 'failed'
&& $releasedClaims->count() === 2
&& $releasedClaims->every(static fn ($claim): bool =>
(string) $claim->state === 'released')
&& $releasedAddresses->count() === 2
&& $releasedAddresses->every(static fn ($address): bool =>
(string) $address->state === 'available'
&& $address->reservation_id === null
&& $address->instance_id === null)
&& !str_contains(json_encode(
$durablePayload,
JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES
), 'source_url'),
'Local pre-dispatch failure contacted Agent, retained a dual-stack claim, '
. 'or persisted its transient capability.');
// Corrupting one immutable envelope field is not an ordinary local failure.
// It must stop before Agent, retain every resource fence and publish an
// operator-visible finalization incident instead of releasing addresses.
$tornBackup = $backup->replicate();
$tornBackup->backup_key = str_repeat('c', 32);
$tornBackup->object_key = (string) $backup->object_key . '.torn';
$tornBackup->save();
$tornPlan = array_values(array_filter(
CloudPortableRestoreTargetService::candidates($tornBackup),
static fn (CloudPortableRestoreTargetPlan $candidate): bool =>
(int) $candidate->toArray()['target']['host_id'] === 3
))[0] ?? null;
$assert($tornPlan instanceof CloudPortableRestoreTargetPlan,
'Torn-envelope target plan is unavailable.');
$tornReserved = CloudPortableRestoreAttemptCoordinator::reserve(
$actor,
$tornBackup,
$tornPlan,
$tornPlan->sha256(),
'torn-envelope-vm',
'torn-envelope-request-000000001'
)->toArray();
$tornJob = CloudPortableRestoreJobDispatcher::queue(
$actor,
(string) $tornReserved['attempt_key']
);
$tornPayload = CloudSecrets::decrypt(
(string) $tornJob->sealed_payload,
'job-payload'
);
$tornPayload['generation'] = (int) $tornPayload['generation'] + 1;
$tornJob->sealed_payload = CloudSecrets::encrypt(
$tornPayload,
'job-payload'
);
$tornJob->save();
$assert($tickJob->invoke($worker) === true,
'Worker did not consume the torn portable-restore envelope.');
$tornJob->refresh();
$tornAttempt = CloudBackupRestoreAttempt::where(
'attempt_key',
(string) $tornReserved['attempt_key']
)->firstOrFail();
$tornReservation = CloudResourceReservation::findOrFail(
(int) $tornReserved['reservation_id']
);
$tornClaims = Capsule::table('cloud_resource_reservation_ips')
->where('reservation_id', (int) $tornReservation->id)->get();
$assert($agentCalls === 0
&& (string) $tornJob->phase === 'terminal'
&& (string) $tornJob->outcome === 'failed'
&& (int) $tornJob->dispatch_attempts === 0
&& (string) $tornJob->effect_resolution === 'not_applied'
&& (int) $tornJob->finalization_incident === 1
&& (string) $tornAttempt->state === 'queued'
&& (int) $tornAttempt->active_attempt === 1
&& (string) $tornReservation->state === 'held'
&& $tornClaims->count() === 2
&& $tornClaims->every(static fn ($claim): bool =>
(string) $claim->state === 'reserved')
&& Capsule::table('cloud_instances')->count() === $instanceCount,
'Torn restore envelope reached Agent or released fenced dual-stack state.');
} finally {
Capsule::connection()->rollBack();
}
echo "Portable restore job dispatcher behavior tests passed.\n";