You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
1082 lines
42 KiB
PHP
1082 lines
42 KiB
PHP
<?php
|
|
|
|
ob_start();
|
|
require __DIR__ . '/CloudPortableRestoreAttemptCoordinatorTest.php';
|
|
ob_end_clean();
|
|
|
|
use App\Models\CloudBackupRestore;
|
|
use App\Models\CloudBackupRestoreAttempt;
|
|
use App\Models\CloudInstanceBackup;
|
|
use App\Models\CloudJob;
|
|
use App\Models\CloudResourceReservation;
|
|
use App\Services\Cloud\CloudError;
|
|
use App\Services\Cloud\CloudAdminOperationRecoveryService;
|
|
use App\Services\Cloud\CloudAgentClient;
|
|
use App\Services\Cloud\CloudAgentError;
|
|
use App\Services\Cloud\CloudOperationRecoveryIdentity;
|
|
use App\Services\Cloud\CloudPortableRestoreAttemptCoordinator;
|
|
use App\Services\Cloud\CloudPortableRestoreJobDispatcher;
|
|
use App\Services\Cloud\CloudPortableRestoreJobEnvelope;
|
|
use App\Services\Cloud\CloudPortableRestoreTargetPlan;
|
|
use App\Services\Cloud\CloudPortableRestoreTargetService;
|
|
use App\Services\Cloud\CloudSecrets;
|
|
use App\Services\Cloud\CloudWorkerService;
|
|
use Illuminate\Database\Capsule\Manager as Capsule;
|
|
use Illuminate\Database\Schema\Blueprint;
|
|
|
|
if (!function_exists('sodium_crypto_aead_xchacha20poly1305_ietf_encrypt')) {
|
|
echo "SKIPPED CloudPortableRestoreJobDispatcherTest: runtime lacks sodium.\n";
|
|
exit(0);
|
|
}
|
|
if (!class_exists(CloudPortableRestoreJobDispatcher::class)) {
|
|
throw new RuntimeException(
|
|
'CloudPortableRestoreJobDispatcher production seam is not implemented.'
|
|
);
|
|
}
|
|
|
|
$rejects409 = static function (callable $callback, string $message) use (
|
|
$assert
|
|
): void {
|
|
try {
|
|
$callback();
|
|
} catch (CloudError $error) {
|
|
$assert($error->httpStatus === 409, $message . ' (wrong status)');
|
|
return;
|
|
}
|
|
throw new RuntimeException($message);
|
|
};
|
|
|
|
// The coordinator fixture deliberately carries only fields it needs. Extend
|
|
// the same in-memory job table with the worker envelope columns exercised by
|
|
// this stage; production schema already owns these columns.
|
|
$schema->table('cloud_jobs', static function (Blueprint $t): void {
|
|
$t->string('request_idempotency_key')->default('');
|
|
$t->string('request_fingerprint', 64)->default('');
|
|
$t->integer('instance_id')->nullable();
|
|
$t->integer('parent_job_id')->nullable();
|
|
$t->integer('vpc_repair_root_job_id')->nullable();
|
|
$t->string('execution_mode')->default('agent');
|
|
$t->string('status')->default('queued');
|
|
$t->text('sealed_payload')->default('');
|
|
$t->integer('attempts')->default(0);
|
|
$t->integer('dispatch_attempts')->default(0);
|
|
$t->integer('dispatch_deadline_at')->default(0);
|
|
$t->integer('dispatch_started_at')->nullable();
|
|
$t->integer('agent_accepted_at')->nullable();
|
|
$t->integer('receipt_deadline_at')->nullable();
|
|
$t->integer('effect_may_have_applied')->nullable();
|
|
$t->integer('safe_to_resubmit')->nullable();
|
|
$t->integer('effect_resolved_at')->nullable();
|
|
$t->integer('effect_resolved_by')->nullable();
|
|
$t->string('effect_resolution_reason')->default('');
|
|
$t->string('finalization_error_code')->nullable();
|
|
$t->string('completion_event_id')->nullable();
|
|
$t->integer('read_at')->nullable();
|
|
$t->integer('dispatched_at')->nullable();
|
|
$t->string('lease_token')->nullable();
|
|
$t->integer('lease_until')->default(0);
|
|
$t->integer('available_at')->default(0);
|
|
$t->string('error_code')->nullable();
|
|
$t->text('result_json')->nullable();
|
|
$t->integer('created_at')->default(0);
|
|
$t->integer('updated_at')->default(0);
|
|
});
|
|
$schema->create('cloud_agent_completion_events', static function (
|
|
Blueprint $t
|
|
): void {
|
|
$t->increments('id');
|
|
$t->integer('host_id');
|
|
$t->string('operation_id');
|
|
});
|
|
$schema->create('cloud_storage_backends', static function (
|
|
Blueprint $t
|
|
): void {
|
|
$t->increments('id'); $t->string('backend_key');
|
|
$t->string('identity_sha256'); $t->string('name');
|
|
$t->string('driver'); $t->string('endpoint'); $t->string('region');
|
|
$t->string('bucket'); $t->string('prefix'); $t->boolean('path_style');
|
|
$t->string('failure_domain'); $t->text('sealed_credentials');
|
|
$t->integer('config_revision'); $t->integer('tested_revision');
|
|
$t->string('admin_state'); $t->string('health_state');
|
|
$t->integer('write_priority');
|
|
$t->integer('consecutive_failures')->default(0);
|
|
$t->integer('consecutive_successes')->default(0);
|
|
$t->string('last_error_code')->default('');
|
|
$t->integer('last_tested_at')->default(0);
|
|
$t->integer('last_healthy_at')->default(0);
|
|
$t->integer('next_probe_at')->default(0);
|
|
$t->integer('created_at'); $t->integer('updated_at');
|
|
});
|
|
$schema->create('cloud_storage_replicas', static function (
|
|
Blueprint $t
|
|
): void {
|
|
$t->increments('id'); $t->string('replica_key');
|
|
$t->integer('artifact_id'); $t->integer('backend_id');
|
|
$t->string('role'); $t->string('object_key');
|
|
$t->string('object_version_id')->default('');
|
|
$t->string('object_etag')->default('');
|
|
$t->integer('size_bytes'); $t->string('sha256', 64);
|
|
$t->string('state'); $t->string('publish_operation_id')->nullable();
|
|
$t->string('delete_operation_id')->nullable();
|
|
$t->integer('delete_generation')->default(0);
|
|
$t->integer('delete_cycle_attempts')->default(0);
|
|
$t->integer('delete_presence_confirmations')->default(0);
|
|
$t->integer('delete_last_checked_at')->default(0);
|
|
$t->integer('delete_next_check_at')->default(0);
|
|
$t->integer('delete_attention_at')->nullable();
|
|
$t->string('lease_token')->nullable();
|
|
$t->integer('lease_until')->default(0);
|
|
$t->string('error_code')->default('');
|
|
$t->integer('verified_at'); $t->integer('created_at');
|
|
$t->integer('updated_at');
|
|
});
|
|
Capsule::table('sqlite_sequence')->where('name', 'cloud_jobs')->update([
|
|
'seq' => 600,
|
|
]);
|
|
|
|
global $System_Config;
|
|
$System_Config['cloud_data_key'] = base64_encode(str_repeat('r', 32));
|
|
$runtime = [
|
|
'config' => [
|
|
'endpoint' => 'https://objects.example.com',
|
|
'region' => 'us-east-1',
|
|
'bucket' => 'portable-backups',
|
|
'prefix' => 'virt',
|
|
'path_style' => true,
|
|
],
|
|
'credentials' => [
|
|
'access_key_id' => 'RESTOREACCESS',
|
|
'secret_access_key' => 'restore-secret-value',
|
|
],
|
|
'revision' => 7,
|
|
];
|
|
Capsule::table('cloud_storage_backends')->insert([
|
|
'id' => 71,
|
|
'backend_key' => str_repeat('1', 32),
|
|
'identity_sha256' => str_repeat('2', 64),
|
|
'name' => 'portable-primary',
|
|
'driver' => 's3',
|
|
'endpoint' => 'https://objects.example.com',
|
|
'region' => 'us-east-1',
|
|
'bucket' => 'portable-backups',
|
|
'prefix' => 'virt',
|
|
'path_style' => 1,
|
|
'failure_domain' => 'portable-primary',
|
|
'sealed_credentials' => CloudSecrets::encrypt(
|
|
$runtime['credentials'],
|
|
'object-storage-backend-credentials'
|
|
),
|
|
'config_revision' => 1,
|
|
'tested_revision' => 1,
|
|
'admin_state' => 'active',
|
|
'health_state' => 'healthy',
|
|
'write_priority' => 10,
|
|
'consecutive_failures' => 0,
|
|
'consecutive_successes' => 1,
|
|
'last_error_code' => '',
|
|
'last_tested_at' => time(),
|
|
'last_healthy_at' => time(),
|
|
'next_probe_at' => time() + 300,
|
|
'created_at' => time(),
|
|
'updated_at' => time(),
|
|
]);
|
|
$sourceBackupRow = Capsule::table('cloud_instance_backups')
|
|
->where('id', 1)->first();
|
|
$assert(is_object($sourceBackupRow), 'Portable source backup is missing.');
|
|
Capsule::table('cloud_storage_artifacts')->insert([
|
|
'id' => 83,
|
|
'artifact_key' => str_repeat('3', 32),
|
|
'kind' => 'backup',
|
|
'maximum_size_bytes' => 2048,
|
|
'size_bytes' => (int) $sourceBackupRow->object_size_bytes,
|
|
'sha256' => (string) $sourceBackupRow->sha256,
|
|
'durability_policy' => 'mirror_async_2',
|
|
'desired_copies' => 2,
|
|
'required_copies' => 1,
|
|
'state' => 'available',
|
|
'last_operation_id' => str_repeat('4', 32),
|
|
'verified_at' => time(),
|
|
'created_at' => time(),
|
|
'updated_at' => time(),
|
|
]);
|
|
Capsule::table('cloud_storage_publications')->insert([
|
|
'artifact_id' => 83,
|
|
'backup_id' => 1,
|
|
'state' => 'ready_to_finalize',
|
|
'created_at' => time(),
|
|
'updated_at' => time(),
|
|
]);
|
|
Capsule::table('cloud_storage_replicas')->insert([
|
|
'id' => 73,
|
|
'replica_key' => str_repeat('5', 32),
|
|
'artifact_id' => 83,
|
|
'backend_id' => 71,
|
|
'role' => 'primary',
|
|
'object_key' => (string) $sourceBackupRow->object_key,
|
|
'object_version_id' => '',
|
|
'object_etag' => '',
|
|
'size_bytes' => (int) $sourceBackupRow->object_size_bytes,
|
|
'sha256' => (string) $sourceBackupRow->sha256,
|
|
'state' => 'ready',
|
|
'publish_operation_id' => str_repeat('6', 32),
|
|
'delete_operation_id' => null,
|
|
'delete_generation' => 0,
|
|
'delete_cycle_attempts' => 0,
|
|
'delete_presence_confirmations' => 0,
|
|
'delete_last_checked_at' => 0,
|
|
'delete_next_check_at' => 0,
|
|
'delete_attention_at' => null,
|
|
'lease_token' => null,
|
|
'lease_until' => 0,
|
|
'error_code' => '',
|
|
'verified_at' => time(),
|
|
'created_at' => time(),
|
|
'updated_at' => time(),
|
|
]);
|
|
Capsule::table('cloud_instance_backups')->where('id', 1)->update([
|
|
'storage_artifact_id' => 83,
|
|
'sealed_storage_runtime' => '',
|
|
'storage_revision' => 0,
|
|
]);
|
|
$backup = CloudInstanceBackup::findOrFail(1);
|
|
$plans = CloudPortableRestoreTargetService::candidates($backup);
|
|
$plan = array_values(array_filter(
|
|
$plans,
|
|
static fn (CloudPortableRestoreTargetPlan $candidate): bool =>
|
|
$candidate->toArray()['target']['host_id'] === 3
|
|
&& $candidate->toArray()['target']['resource_pool_id'] === 300
|
|
))[0] ?? null;
|
|
$assert($plan instanceof CloudPortableRestoreTargetPlan,
|
|
'Fresh portable restore dispatch plan is unavailable.');
|
|
|
|
$requestKey = 'dispatch-request-0000000000000001';
|
|
$reserved = CloudPortableRestoreAttemptCoordinator::reserve(
|
|
$actor,
|
|
$backup,
|
|
$plan,
|
|
$plan->sha256(),
|
|
'dispatch-vm',
|
|
$requestKey
|
|
)->toArray();
|
|
$attempt = CloudBackupRestoreAttempt::where(
|
|
'attempt_key',
|
|
$reserved['attempt_key']
|
|
)->firstOrFail();
|
|
$reservation = CloudResourceReservation::findOrFail(
|
|
$reserved['reservation_id']
|
|
);
|
|
$jobCount = CloudJob::count();
|
|
$instanceCount = Capsule::table('cloud_instances')->count();
|
|
$claimSnapshot = Capsule::table('cloud_resource_reservation_ips')
|
|
->where('reservation_id', (int) $reservation->id)
|
|
->orderBy('address_family')->get()->map(
|
|
static fn ($row): array => (array) $row
|
|
)->all();
|
|
|
|
// Any torn input must roll back without an orphan job or changing the attempt.
|
|
$reservation->operation_id = str_repeat('7', 32);
|
|
$reservation->save();
|
|
$rejects409(
|
|
static fn () => CloudPortableRestoreJobDispatcher::queue(
|
|
$actor,
|
|
(string) $attempt->attempt_key
|
|
),
|
|
'Dispatcher accepted a torn reservation operation.'
|
|
);
|
|
$assert(CloudJob::count() === $jobCount
|
|
&& (string) CloudBackupRestoreAttempt::findOrFail($attempt->id)->state
|
|
=== 'reserved',
|
|
'Rejected dispatch left a job or changed attempt state.');
|
|
$reservation->operation_id = (string) $attempt->operation_id;
|
|
$reservation->save();
|
|
|
|
$restoreBeforeQueue = CloudBackupRestore::findOrFail(
|
|
(int) $attempt->backup_restore_id
|
|
);
|
|
$restoreBeforeQueue->target_instance_id = 777;
|
|
$restoreBeforeQueue->save();
|
|
$rejects409(
|
|
static fn () => CloudPortableRestoreJobDispatcher::queue(
|
|
$actor,
|
|
(string) $attempt->attempt_key
|
|
),
|
|
'Dispatcher accepted a pre-created target instance.'
|
|
);
|
|
$restoreBeforeQueue->target_instance_id = null;
|
|
$restoreBeforeQueue->save();
|
|
|
|
foreach (['backup_restore_id' => 777, 'order_id' => 777] as $field => $value) {
|
|
$reservation->{$field} = $value;
|
|
$reservation->save();
|
|
$rejects409(
|
|
static fn () => CloudPortableRestoreJobDispatcher::queue(
|
|
$actor,
|
|
(string) $attempt->attempt_key
|
|
),
|
|
'Dispatcher accepted a doubly-owned reservation: ' . $field . '.'
|
|
);
|
|
$reservation->{$field} = null;
|
|
$reservation->save();
|
|
}
|
|
$assert(CloudJob::count() === $jobCount
|
|
&& (string) CloudBackupRestoreAttempt::findOrFail($attempt->id)->state
|
|
=== 'reserved',
|
|
'Rejected ownership tear left a job or changed attempt state.');
|
|
|
|
$ipv6Claim = Capsule::table('cloud_resource_reservation_ips')
|
|
->where('reservation_id', (int) $reservation->id)
|
|
->where('address_family', 6)->first();
|
|
$assert(is_object($ipv6Claim), 'IPv6 claim fixture is missing.');
|
|
Capsule::table('cloud_resource_reservation_ips')
|
|
->where('id', (int) $ipv6Claim->id)
|
|
->update(['operation_id' => str_repeat('6', 32)]);
|
|
$rejects409(
|
|
static fn () => CloudPortableRestoreJobDispatcher::queue(
|
|
$actor,
|
|
(string) $attempt->attempt_key
|
|
),
|
|
'Dispatcher accepted a torn IPv6 claim.'
|
|
);
|
|
$assert(CloudJob::count() === $jobCount
|
|
&& Capsule::table('cloud_instances')->count() === $instanceCount,
|
|
'Rejected claim dispatch produced a job or instance.');
|
|
Capsule::table('cloud_resource_reservation_ips')
|
|
->where('id', (int) $ipv6Claim->id)
|
|
->update(['operation_id' => (string) $attempt->operation_id]);
|
|
|
|
$originalPlanSha = (string) $attempt->target_plan_sha256;
|
|
$attempt->target_plan_sha256 = str_repeat('5', 64);
|
|
$attempt->save();
|
|
$rejects409(
|
|
static fn () => CloudPortableRestoreJobDispatcher::queue(
|
|
$actor,
|
|
(string) $attempt->attempt_key
|
|
),
|
|
'Dispatcher accepted a torn target plan.'
|
|
);
|
|
$attempt->target_plan_sha256 = $originalPlanSha;
|
|
$attempt->save();
|
|
|
|
$job = CloudPortableRestoreJobDispatcher::queue(
|
|
$actor,
|
|
(string) $attempt->attempt_key
|
|
);
|
|
$attempt = CloudBackupRestoreAttempt::findOrFail($attempt->id);
|
|
$restore = CloudBackupRestore::findOrFail($attempt->backup_restore_id);
|
|
$reservation = CloudResourceReservation::findOrFail($reservation->id);
|
|
$assert((int) $job->id > 0
|
|
&& (int) $job->user_id === (int) $backup->user_id
|
|
&& (int) $job->backup_id === (int) $backup->id
|
|
&& (int) $job->backup_restore_id === (int) $restore->id
|
|
&& (int) $job->host_id === (int) $attempt->target_host_id
|
|
&& (string) $job->kind === 'backup_restore'
|
|
&& (string) $job->execution_mode === 'agent'
|
|
&& (string) $job->phase === 'accepted'
|
|
&& (string) $job->status === 'queued'
|
|
&& hash_equals((string) $job->operation_id,
|
|
(string) $attempt->operation_id)
|
|
&& hash_equals((string) $job->request_idempotency_key,
|
|
(string) $attempt->attempt_key)
|
|
&& (string) $attempt->state === 'queued'
|
|
&& (int) $attempt->job_id === (int) $job->id
|
|
&& (string) $attempt->effect_resolution === 'not_applicable'
|
|
&& (string) $reservation->state === 'held'
|
|
&& Capsule::table('cloud_instances')->count() === $instanceCount,
|
|
'Reserved restore attempt was not atomically converted into one queued job.');
|
|
$assert((int) $restore->job_id === (int) $job->id,
|
|
'Generation-one job projection was not written to the parent restore.');
|
|
|
|
$payload = CloudSecrets::decrypt((string) $job->sealed_payload, 'job-payload');
|
|
$body = is_array($payload['body'] ?? null) ? $payload['body'] : [];
|
|
$public = is_array($body['public_network'] ?? null)
|
|
? $body['public_network'] : [];
|
|
$assert(($payload['method'] ?? null) === 'POST'
|
|
&& ($payload['request_target'] ?? null) === '/v1/vms/'
|
|
. rawurlencode((string) $attempt->target_uuid) . '/restore-backup'
|
|
&& ($payload['attempt_key'] ?? null) === (string) $attempt->attempt_key
|
|
&& (int) ($payload['generation'] ?? 0) === (int) $attempt->generation
|
|
&& ($payload['target_plan_sha256'] ?? null)
|
|
=== (string) $attempt->target_plan_sha256
|
|
&& ($payload['target_claim_set_sha256'] ?? null)
|
|
=== (string) $attempt->target_claim_set_sha256
|
|
&& ($payload['target_public_network_sha256'] ?? null)
|
|
=== (string) $attempt->target_public_network_sha256
|
|
&& ($payload['owner_id'] ?? null) === (int) $backup->user_id
|
|
&& ($body['uuid'] ?? null) === strtolower((string) $attempt->target_uuid)
|
|
&& ($body['name'] ?? null) === (string) $attempt->target_name
|
|
&& ($body['backup_key'] ?? null) === (string) $backup->backup_key
|
|
&& ($body['source_size_bytes'] ?? null)
|
|
=== (int) $backup->object_size_bytes
|
|
&& ($body['source_sha256'] ?? null) === (string) $backup->sha256
|
|
&& !array_key_exists('source_url', $body)
|
|
&& ($body['source'] ?? null) === ''
|
|
&& ($body['template_id'] ?? null) === ''
|
|
&& ($body['template_revision'] ?? null) === 0
|
|
&& ($body['template_generation_token'] ?? null) === ''
|
|
&& ($body['startup_script'] ?? null) === ''
|
|
&& ($body['hostname'] ?? null) === ''
|
|
&& ($body['guest_ssh_public_key'] ?? null) === ''
|
|
&& ($body['root_password'] ?? null) === ''
|
|
&& ($body['debian_12_network_fix'] ?? null) === false
|
|
&& is_array($body['platform_network_policy_snapshot'] ?? null)
|
|
&& ($public['contract'] ?? null) === 1
|
|
&& is_array($public['ipv4'] ?? null)
|
|
&& is_array($public['ipv6'] ?? null),
|
|
'Queued restore payload lost immutable backup or dual-stack request identity.');
|
|
foreach (['network', 'public_pool_key', 'public_pool_cidr',
|
|
'public_ip_cidr', 'gateway', 'dns_servers', 'vpc_port'] as $legacy) {
|
|
$assert(!array_key_exists($legacy, $body),
|
|
'Queued restore body retained legacy or VPC field: ' . $legacy);
|
|
}
|
|
$encodedPayload = json_encode(
|
|
$payload,
|
|
JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES
|
|
);
|
|
foreach (['restore-secret-value', 'RESTOREACCESS', 'sealed_storage_runtime',
|
|
'object_key', 'source_url', 'X-Amz-', 'vpc_port_id'] as $secret) {
|
|
$assert(!str_contains($encodedPayload, $secret),
|
|
'Queued restore payload leaked long-lived or unrelated data: ' . $secret);
|
|
}
|
|
$secretCheck = new ReflectionMethod(
|
|
CloudPortableRestoreJobEnvelope::class,
|
|
'assertSecretFree'
|
|
);
|
|
$legalNamePayload = $payload;
|
|
$legalNamePayload['body']['name'] = 'secret_access_key';
|
|
$secretCheck->invoke(null, $legalNamePayload);
|
|
|
|
// A bounded capability belongs only to the one Agent wire request. The worker
|
|
// persists only the exact Journal fingerprint before crossing the dispatch
|
|
// CAS, never the URL itself.
|
|
$prepareWire = new ReflectionMethod(
|
|
CloudWorkerService::class,
|
|
'portableRestoreDispatchPayload'
|
|
);
|
|
[$stableDispatchPayload, $wirePayload] = $prepareWire->invoke(
|
|
null,
|
|
$job,
|
|
$payload
|
|
);
|
|
$assert(str_starts_with(
|
|
(string) $wirePayload['body']['source_url'],
|
|
'https://'
|
|
) && str_contains(
|
|
(string) $wirePayload['body']['source_url'],
|
|
'X-Amz-Signature='
|
|
),
|
|
'Dispatch-time wire fixture did not contain a bounded download capability.');
|
|
$stableEncoded = json_encode(
|
|
$stableDispatchPayload,
|
|
JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES
|
|
);
|
|
$persistedIdentity = $stableDispatchPayload[
|
|
CloudOperationRecoveryIdentity::payloadKey()
|
|
] ?? null;
|
|
$assert(is_array($persistedIdentity)
|
|
&& !str_contains($stableEncoded, 'source_url')
|
|
&& !str_contains($stableEncoded, 'X-Amz-')
|
|
&& CloudOperationRecoveryIdentity::same(
|
|
CloudOperationRecoveryIdentity::expectedFromPayload(
|
|
$job,
|
|
$wirePayload
|
|
),
|
|
$persistedIdentity
|
|
),
|
|
'Worker did not separate the stable Journal identity from the wire capability.');
|
|
|
|
// Reconciliation must recover the exact wire fingerprint from stable durable
|
|
// state after dispatch, without retaining or recreating the expired URL.
|
|
$originalPhase = (string) $job->phase;
|
|
$originalDispatchAttempts = (int) $job->dispatch_attempts;
|
|
$originalSealed = (string) $job->sealed_payload;
|
|
$job->phase = 'dispatching';
|
|
$job->dispatch_attempts = 1;
|
|
$job->sealed_payload = CloudSecrets::encrypt(
|
|
$stableDispatchPayload,
|
|
'job-payload'
|
|
);
|
|
$assert(CloudOperationRecoveryIdentity::same(
|
|
CloudOperationRecoveryIdentity::expected($job),
|
|
$persistedIdentity
|
|
),
|
|
'Dispatched restore identity could not be rebuilt without its source URL.');
|
|
|
|
// Recovery authority follows the attempt ledger after dispatch and never
|
|
// requires the discarded URL. Each non-terminal state keeps the same stable
|
|
// Agent identity while enforcing the matching resource lifecycle.
|
|
foreach ([
|
|
['awaiting_receipt', 'not_applicable', 'held', 'reserved'],
|
|
['effect_unknown', 'unknown', 'uncertain', 'uncertain'],
|
|
['finalization_pending', 'applied', 'uncertain', 'uncertain'],
|
|
] as [$attemptState, $effect, $reservationState, $claimState]) {
|
|
$evidenceJson = match ($attemptState) {
|
|
'effect_unknown' => json_encode(
|
|
['effect_may_have_applied' => true],
|
|
JSON_THROW_ON_ERROR
|
|
),
|
|
'finalization_pending' => json_encode(
|
|
['proof' => 'validated'],
|
|
JSON_THROW_ON_ERROR
|
|
),
|
|
default => null,
|
|
};
|
|
Capsule::table('cloud_backup_restore_attempts')
|
|
->where('id', (int) $attempt->id)->update([
|
|
'state' => $attemptState,
|
|
'effect_resolution' => $effect,
|
|
'evidence_resolution' => $evidenceJson === null
|
|
? 'not_applicable' : $effect,
|
|
'evidence_json' => $evidenceJson,
|
|
'evidence_sha256' => $evidenceJson === null
|
|
? null : hash('sha256', $evidenceJson),
|
|
'evidence_recorded_at' => $evidenceJson === null
|
|
? null : time(),
|
|
'effect_resolved_at' => $attemptState === 'finalization_pending'
|
|
? time() : null,
|
|
]);
|
|
Capsule::table('cloud_resource_reservations')
|
|
->where('id', (int) $reservation->id)->update([
|
|
'state' => $reservationState,
|
|
]);
|
|
Capsule::table('cloud_resource_reservation_ips')
|
|
->where('reservation_id', (int) $reservation->id)->update([
|
|
'state' => $claimState,
|
|
]);
|
|
Capsule::table('cloud_public_ips')
|
|
->where('reservation_id', (int) $reservation->id)->update([
|
|
'state' => $claimState,
|
|
]);
|
|
$assert(CloudOperationRecoveryIdentity::same(
|
|
CloudOperationRecoveryIdentity::expected($job),
|
|
$persistedIdentity
|
|
),
|
|
'Recovery identity rejected valid attempt state: ' . $attemptState);
|
|
}
|
|
$tamperedRecoveryPayload = $stableDispatchPayload;
|
|
$tamperedRecoveryPayload['generation']++;
|
|
$job->sealed_payload = CloudSecrets::encrypt(
|
|
$tamperedRecoveryPayload,
|
|
'job-payload'
|
|
);
|
|
$rejects409(
|
|
static fn () => CloudOperationRecoveryIdentity::expected($job),
|
|
'Recovery identity accepted a changed attempt generation.'
|
|
);
|
|
$originalAttemptOperation = (string) $attempt->operation_id;
|
|
Capsule::table('cloud_backup_restore_attempts')
|
|
->where('id', (int) $attempt->id)->update([
|
|
'operation_id' => str_repeat('f', 32),
|
|
]);
|
|
$rejects409(
|
|
static fn () => CloudOperationRecoveryIdentity::expected($job),
|
|
'Portable ownership downgraded to legacy after an operation mismatch.'
|
|
);
|
|
Capsule::table('cloud_backup_restore_attempts')
|
|
->where('id', (int) $attempt->id)->update([
|
|
'operation_id' => $originalAttemptOperation,
|
|
]);
|
|
Capsule::table('cloud_backup_restore_attempts')
|
|
->where('id', (int) $attempt->id)->update([
|
|
'state' => 'queued',
|
|
'effect_resolution' => 'not_applicable',
|
|
'evidence_resolution' => 'not_applicable',
|
|
'evidence_json' => null,
|
|
'evidence_sha256' => null,
|
|
'evidence_recorded_at' => null,
|
|
'effect_resolved_at' => null,
|
|
]);
|
|
Capsule::table('cloud_resource_reservations')
|
|
->where('id', (int) $reservation->id)->update(['state' => 'held']);
|
|
Capsule::table('cloud_resource_reservation_ips')
|
|
->where('reservation_id', (int) $reservation->id)->update([
|
|
'state' => 'reserved',
|
|
]);
|
|
Capsule::table('cloud_public_ips')
|
|
->where('reservation_id', (int) $reservation->id)->update([
|
|
'state' => 'reserved',
|
|
]);
|
|
$job->sealed_payload = CloudSecrets::encrypt(
|
|
$stableDispatchPayload,
|
|
'job-payload'
|
|
);
|
|
|
|
// Admin inspection uses the typed dual-stack projection for attempt-owned
|
|
// restores while preserving the scalar contract only for legacy jobs.
|
|
$inspectProjection = new ReflectionMethod(
|
|
CloudAdminOperationRecoveryService::class,
|
|
'inspection'
|
|
);
|
|
$inspectionBase = [
|
|
'operation_id' => (string) $job->operation_id,
|
|
'action' => $persistedIdentity['action'],
|
|
'resource_id' => $persistedIdentity['resource_id'],
|
|
'request_fingerprint' => $persistedIdentity['request_fingerprint'],
|
|
'operation_status' => 'succeeded',
|
|
'checked_at' => time(),
|
|
'outcome' => 'succeeded',
|
|
'summary_code' => 'operation_succeeded',
|
|
'effect_resolution' => 'applied',
|
|
'fence_state' => 'not_required',
|
|
'fence_resolution_required' => false,
|
|
'abort_job' => ['available' => false],
|
|
'cleanup' => ['available' => false],
|
|
];
|
|
$rejectsInspection = static function (callable $callback, string $message) use (
|
|
$assert
|
|
): void {
|
|
try {
|
|
$callback();
|
|
} catch (CloudAgentError $error) {
|
|
$assert($error->httpStatus === 502, $message . ' (wrong status)');
|
|
return;
|
|
}
|
|
throw new RuntimeException($message);
|
|
};
|
|
$portableProjection = [
|
|
'uuid' => strtolower((string) $attempt->target_uuid),
|
|
'name' => (string) $attempt->target_name,
|
|
'vcpu' => (int) $body['vcpu'],
|
|
'memory_mib' => (int) $body['memory_mib'],
|
|
'disk_gib' => (int) $body['disk_gib'],
|
|
'bandwidth_mbps' => (int) $body['bandwidth_mbps'],
|
|
'public_network' => $public,
|
|
'machine' => 'q35',
|
|
'firmware' => 'uefi',
|
|
'secure_boot' => false,
|
|
'backup_key' => (string) $backup->backup_key,
|
|
'sha256' => (string) $backup->sha256,
|
|
];
|
|
$portableInspection = $inspectProjection->invoke(
|
|
null,
|
|
$inspectionBase + ['effect_projection' => $portableProjection],
|
|
(string) $job->operation_id,
|
|
'backup_restore',
|
|
true,
|
|
false,
|
|
true
|
|
);
|
|
$assert(($portableInspection['effect_projection']['public_network'] ?? null)
|
|
=== $public,
|
|
'Portable restore inspection lost its typed dual-stack projection.');
|
|
$legacyScalarProjection = [
|
|
'uuid' => strtolower((string) $attempt->target_uuid),
|
|
'public_ip_cidr' => (string) $public['ipv4']['address_cidr'],
|
|
'public_pool_key' => str_repeat('a', 32),
|
|
'backup_key' => (string) $backup->backup_key,
|
|
'sha256' => (string) $backup->sha256,
|
|
];
|
|
$rejectsInspection(
|
|
static fn () => $inspectProjection->invoke(
|
|
null,
|
|
$inspectionBase + ['effect_projection' => $legacyScalarProjection],
|
|
(string) $job->operation_id,
|
|
'backup_restore',
|
|
true,
|
|
false,
|
|
true
|
|
),
|
|
'Portable restore inspection accepted an IPv4 scalar-only projection.'
|
|
);
|
|
$rejectsInspection(
|
|
static fn () => $inspectProjection->invoke(
|
|
null,
|
|
$inspectionBase + ['effect_projection' => $legacyScalarProjection],
|
|
(string) $job->operation_id,
|
|
'backup_restore',
|
|
true,
|
|
false,
|
|
false
|
|
),
|
|
'Legacy restore inspection retained an IPv4 scalar-only projection.'
|
|
);
|
|
$notAppliedInspection = $inspectionBase;
|
|
$notAppliedInspection['operation_status'] = 'failed';
|
|
$notAppliedInspection['outcome'] = 'failed';
|
|
$notAppliedInspection['summary_code'] = 'operation_failed';
|
|
$notAppliedInspection['effect_resolution'] = 'not_applied';
|
|
$notAppliedInspection['effect_projection'] = ['creation_absent' => true];
|
|
$rejectsInspection(
|
|
static fn () => $inspectProjection->invoke(
|
|
null,
|
|
$notAppliedInspection,
|
|
(string) $job->operation_id,
|
|
'backup_restore',
|
|
true,
|
|
false,
|
|
true
|
|
),
|
|
'Portable not-applied inspection accepted incomplete absence evidence.'
|
|
);
|
|
$finishIdentity = new ReflectionMethod(
|
|
CloudWorkerService::class,
|
|
'operationIdentityForFinalization'
|
|
);
|
|
$assert(CloudOperationRecoveryIdentity::same(
|
|
$finishIdentity->invoke(
|
|
null,
|
|
$job,
|
|
$stableDispatchPayload,
|
|
false,
|
|
false
|
|
),
|
|
$persistedIdentity
|
|
),
|
|
'Normal restore finalization required the discarded source URL.');
|
|
$job->phase = $originalPhase;
|
|
$job->dispatch_attempts = $originalDispatchAttempts;
|
|
$job->sealed_payload = $originalSealed;
|
|
|
|
// Prepared recovery must compact the transient wire fixture while retaining
|
|
// the exact restore-attempt identity needed for convergence.
|
|
$compact = new ReflectionMethod(
|
|
CloudWorkerService::class,
|
|
'compactPreparedRecoveryPayload'
|
|
);
|
|
$prepared = $compact->invoke(null, 'backup_restore', $wirePayload);
|
|
$preparedEncoded = json_encode(
|
|
$prepared,
|
|
JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES
|
|
);
|
|
$assert(($prepared['attempt_key'] ?? null) === (string) $attempt->attempt_key
|
|
&& (int) ($prepared['generation'] ?? 0) === (int) $attempt->generation
|
|
&& ($prepared['target_plan_sha256'] ?? null)
|
|
=== (string) $attempt->target_plan_sha256
|
|
&& ($prepared['target_claim_set_sha256'] ?? null)
|
|
=== (string) $attempt->target_claim_set_sha256
|
|
&& ($prepared['target_public_network_sha256'] ?? null)
|
|
=== (string) $attempt->target_public_network_sha256
|
|
&& !str_contains($preparedEncoded, 'source_url')
|
|
&& !str_contains($preparedEncoded, 'X-Amz-')
|
|
&& !str_contains($preparedEncoded, 'RESTOREACCESS')
|
|
&& !str_contains($preparedEncoded, 'restore-secret-value'),
|
|
'Prepared recovery lost restore identity or retained a download capability.');
|
|
|
|
$replay = CloudPortableRestoreJobDispatcher::queue(
|
|
$actor,
|
|
(string) $attempt->attempt_key
|
|
);
|
|
$assert((int) $replay->id === (int) $job->id
|
|
&& CloudJob::count() === $jobCount + 1
|
|
&& Capsule::table('cloud_instances')->count() === $instanceCount
|
|
&& Capsule::table('cloud_resource_reservation_ips')
|
|
->where('reservation_id', (int) $reservation->id)
|
|
->orderBy('address_family')->get()->map(
|
|
static fn ($row): array => (array) $row
|
|
)->all() === $claimSnapshot,
|
|
'Exact queue replay created another job, instance, or changed claims.');
|
|
|
|
// Once queued, every durable identity must still be checked before returning
|
|
// the existing job; replay cannot hide a torn cross-table association.
|
|
$jobMutations = [
|
|
['operation_id', str_repeat('4', 32), (string) $job->operation_id],
|
|
['host_id', 1, (int) $job->host_id],
|
|
['user_id', 99, (int) $job->user_id],
|
|
['backup_id', 999, (int) $job->backup_id],
|
|
['backup_restore_id', 999, (int) $job->backup_restore_id],
|
|
];
|
|
foreach ($jobMutations as [$field, $invalid, $valid]) {
|
|
Capsule::table('cloud_jobs')->where('id', (int) $job->id)
|
|
->update([$field => $invalid]);
|
|
$rejects409(
|
|
static fn () => CloudPortableRestoreJobDispatcher::queue(
|
|
$actor,
|
|
(string) $attempt->attempt_key
|
|
),
|
|
'Queue replay accepted torn job identity: ' . $field . '.'
|
|
);
|
|
$assert(CloudJob::count() === $jobCount + 1,
|
|
'Rejected queue replay created another job.');
|
|
Capsule::table('cloud_jobs')->where('id', (int) $job->id)
|
|
->update([$field => $valid]);
|
|
}
|
|
|
|
// The remaining fault-injection cases run in an outer rollback so this file
|
|
// remains a reusable queued-attempt fixture for the finalizer test suite.
|
|
Capsule::connection()->beginTransaction();
|
|
try {
|
|
// Close the valid envelope fixture with exact local absence evidence so its
|
|
// addresses can be reused by the independent pre-dispatch failure cases.
|
|
$finish = new ReflectionMethod(CloudWorkerService::class, 'finish');
|
|
$finish->invoke(
|
|
new CloudWorkerService(),
|
|
$job,
|
|
[
|
|
'operation_id' => (string) $job->operation_id,
|
|
'status' => 'failed',
|
|
'result' => [
|
|
'creation_absent' => true,
|
|
'managed_artifacts_absent' => true,
|
|
],
|
|
'effect_may_have_applied' => false,
|
|
'safe_to_resubmit' => true,
|
|
],
|
|
$payload
|
|
);
|
|
$assert((string) CloudJob::findOrFail((int) $job->id)->effect_resolution
|
|
=== 'not_applied'
|
|
&& (string) CloudResourceReservation::findOrFail(
|
|
(int) $reservation->id
|
|
)->state === 'released',
|
|
'Valid never-dispatched restore fixture did not release its dual stack.');
|
|
|
|
// Exercise the public Worker seam rather than its private admission helper:
|
|
// a legacy restore payload without an attempt identity must fail locally,
|
|
// before Agent contact, without changing any canonical resource ledger.
|
|
$missingAttemptPayload = $payload;
|
|
unset($missingAttemptPayload['attempt_key']);
|
|
$legacyRestoreJob = new CloudJob();
|
|
$legacyRestoreJob->operation_id = bin2hex(random_bytes(16));
|
|
$legacyRestoreJob->request_idempotency_key =
|
|
'legacy-restore-worker-gate-0001';
|
|
$legacyRestoreJob->request_fingerprint = hash(
|
|
'sha256',
|
|
'legacy-restore-worker-gate-0001'
|
|
);
|
|
$legacyRestoreJob->user_id = (int) $job->user_id;
|
|
$legacyRestoreJob->host_id = (int) $job->host_id;
|
|
$legacyRestoreJob->instance_id = null;
|
|
$legacyRestoreJob->backup_id = (int) $job->backup_id;
|
|
$legacyRestoreJob->backup_restore_id = (int) $job->backup_restore_id;
|
|
$legacyRestoreJob->kind = 'backup_restore';
|
|
$legacyRestoreJob->execution_mode = 'agent';
|
|
$legacyRestoreJob->sealed_payload = CloudSecrets::encrypt(
|
|
$missingAttemptPayload,
|
|
'job-payload'
|
|
);
|
|
$legacyRestoreJob->available_at = $legacyRestoreJob->created_at =
|
|
$legacyRestoreJob->updated_at = time();
|
|
$legacyRestoreJob->save();
|
|
$legacyRestoreIpLedger = Capsule::table('cloud_public_ips')
|
|
->orderBy('id')->get([
|
|
'id', 'state', 'reservation_id', 'instance_id',
|
|
])->map(static fn ($row): array => (array) $row)->all();
|
|
$legacyRestoreClaimLedger = Capsule::table(
|
|
'cloud_resource_reservation_ips'
|
|
)->orderBy('id')->get([
|
|
'id', 'state', 'reservation_id', 'public_ip_id', 'instance_id',
|
|
])->map(static fn ($row): array => (array) $row)->all();
|
|
$legacyRestoreAgentCalls = 0;
|
|
$legacyRestoreTransport = new \GuzzleHttp\Client([
|
|
'handler' => static function () use (&$legacyRestoreAgentCalls): void {
|
|
$legacyRestoreAgentCalls++;
|
|
throw new RuntimeException('Legacy restore reached Agent.');
|
|
},
|
|
]);
|
|
$legacyRestoreAgent = new CloudAgentClient(
|
|
'3',
|
|
str_repeat('a', 32),
|
|
$legacyRestoreTransport,
|
|
20.0,
|
|
'http://127.0.0.1:8001/internal/agent/request',
|
|
str_repeat('b', 32)
|
|
);
|
|
$assert((new CloudWorkerService($legacyRestoreAgent))->tick() === true,
|
|
'Worker did not consume the legacy restore job.');
|
|
$legacyRestoreJob->refresh();
|
|
$assert($legacyRestoreAgentCalls === 0
|
|
&& (int) $legacyRestoreJob->dispatch_attempts === 0
|
|
&& (string) $legacyRestoreJob->phase === 'terminal'
|
|
&& (string) $legacyRestoreJob->outcome === 'failed'
|
|
&& Capsule::table('cloud_public_ips')->orderBy('id')->get([
|
|
'id', 'state', 'reservation_id', 'instance_id',
|
|
])->map(static fn ($row): array => (array) $row)->all()
|
|
=== $legacyRestoreIpLedger
|
|
&& Capsule::table('cloud_resource_reservation_ips')
|
|
->orderBy('id')->get([
|
|
'id', 'state', 'reservation_id', 'public_ip_id', 'instance_id',
|
|
])->map(static fn ($row): array => (array) $row)->all()
|
|
=== $legacyRestoreClaimLedger,
|
|
'Legacy restore reached Agent, remained claimable, or changed the '
|
|
. 'canonical address ledgers.');
|
|
|
|
// Freeze an internally consistent backup whose stored object-runtime shape
|
|
// cannot mint a capability. The immutable plan binds that exact runtime;
|
|
// changing an already-frozen ciphertext would be ledger damage instead.
|
|
$invalidRuntime = [
|
|
'config' => [],
|
|
'credentials' => [],
|
|
'revision' => (int) $backup->storage_revision,
|
|
];
|
|
$invalidBackup = $backup->replicate();
|
|
$invalidBackup->backup_key = str_repeat('d', 32);
|
|
$invalidBackup->storage_artifact_id = null;
|
|
$invalidBackup->object_key = (string) $backup->object_key
|
|
. '.predispatch';
|
|
$invalidBackup->storage_revision = 7;
|
|
$invalidBackup->sealed_storage_runtime = CloudSecrets::encrypt(
|
|
$invalidRuntime,
|
|
'object-storage-backup-runtime'
|
|
);
|
|
$invalidBackup->save();
|
|
$failurePlan = array_values(array_filter(
|
|
CloudPortableRestoreTargetService::candidates($invalidBackup),
|
|
static fn (CloudPortableRestoreTargetPlan $candidate): bool =>
|
|
(int) $candidate->toArray()['target']['host_id'] === 3
|
|
))[0] ?? null;
|
|
$assert($failurePlan instanceof CloudPortableRestoreTargetPlan,
|
|
'Pre-dispatch failure target plan is unavailable.');
|
|
$failureReserved = CloudPortableRestoreAttemptCoordinator::reserve(
|
|
$actor,
|
|
$invalidBackup,
|
|
$failurePlan,
|
|
$failurePlan->sha256(),
|
|
'predispatch-vm',
|
|
'predispatch-request-0000000001'
|
|
)->toArray();
|
|
$failureJob = CloudPortableRestoreJobDispatcher::queue(
|
|
$actor,
|
|
(string) $failureReserved['attempt_key']
|
|
);
|
|
$failureAttempt = CloudBackupRestoreAttempt::where(
|
|
'attempt_key',
|
|
(string) $failureReserved['attempt_key']
|
|
)->firstOrFail();
|
|
$failureReservation = CloudResourceReservation::findOrFail(
|
|
(int) $failureReserved['reservation_id']
|
|
);
|
|
$failureRestore = CloudBackupRestore::findOrFail(
|
|
(int) $failureAttempt->backup_restore_id
|
|
);
|
|
|
|
// A local capability-minting failure happens before the immutable dispatch
|
|
// CAS. It must never contact Agent; its exact not-dispatched evidence must
|
|
// atomically release both public-address claims and reserved host capacity.
|
|
$agentCalls = 0;
|
|
$tripwireTransport = new \GuzzleHttp\Client([
|
|
'handler' => static function () use (&$agentCalls): void {
|
|
$agentCalls++;
|
|
throw new RuntimeException(
|
|
'Local portable-restore failure reached Agent.'
|
|
);
|
|
},
|
|
]);
|
|
$tripwireAgent = new CloudAgentClient(
|
|
'3',
|
|
str_repeat('a', 32),
|
|
$tripwireTransport,
|
|
20.0,
|
|
'http://127.0.0.1:8001/internal/agent/request',
|
|
str_repeat('b', 32)
|
|
);
|
|
$worker = new CloudWorkerService($tripwireAgent);
|
|
$tickJob = new ReflectionMethod(CloudWorkerService::class, 'tickJob');
|
|
$assert($tickJob->invoke($worker) === true,
|
|
'Worker did not consume the local pre-dispatch restore failure.');
|
|
$failedJob = CloudJob::findOrFail((int) $failureJob->id);
|
|
$failedAttempt = CloudBackupRestoreAttempt::findOrFail(
|
|
(int) $failureAttempt->id
|
|
);
|
|
$failedReservation = CloudResourceReservation::findOrFail(
|
|
(int) $failureReservation->id
|
|
);
|
|
$failedRestore = CloudBackupRestore::findOrFail((int) $failureRestore->id);
|
|
$releasedClaims = Capsule::table('cloud_resource_reservation_ips')
|
|
->where('reservation_id', (int) $failureReservation->id)
|
|
->orderBy('address_family')->get();
|
|
$releasedAddresses = Capsule::table('cloud_public_ips')
|
|
->whereIn('id', $releasedClaims->pluck('public_ip_id')->all())
|
|
->orderBy('address_family')->get();
|
|
$durablePayload = (string) $failedJob->sealed_payload === '' ? []
|
|
: CloudSecrets::decrypt(
|
|
(string) $failedJob->sealed_payload,
|
|
'job-payload'
|
|
);
|
|
$assert($agentCalls === 0
|
|
&& (string) $failedJob->phase === 'terminal'
|
|
&& (string) $failedJob->outcome === 'failed'
|
|
&& (int) $failedJob->dispatch_attempts === 0
|
|
&& (string) $failedJob->effect_resolution === 'not_applied'
|
|
&& (int) $failedJob->finalization_incident === 0
|
|
&& (string) $failedAttempt->state === 'failed'
|
|
&& $failedAttempt->active_attempt === null
|
|
&& (string) $failedReservation->state === 'released'
|
|
&& (string) $failedRestore->state === 'failed'
|
|
&& $releasedClaims->count() === 2
|
|
&& $releasedClaims->every(static fn ($claim): bool =>
|
|
(string) $claim->state === 'released')
|
|
&& $releasedAddresses->count() === 2
|
|
&& $releasedAddresses->every(static fn ($address): bool =>
|
|
(string) $address->state === 'available'
|
|
&& $address->reservation_id === null
|
|
&& $address->instance_id === null)
|
|
&& !str_contains(json_encode(
|
|
$durablePayload,
|
|
JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES
|
|
), 'source_url'),
|
|
'Local pre-dispatch failure contacted Agent, retained a dual-stack claim, '
|
|
. 'or persisted its transient capability.');
|
|
|
|
// Corrupting one immutable envelope field is not an ordinary local failure.
|
|
// It must stop before Agent, retain every resource fence and publish an
|
|
// operator-visible finalization incident instead of releasing addresses.
|
|
$tornBackup = $backup->replicate();
|
|
$tornBackup->backup_key = str_repeat('c', 32);
|
|
$tornBackup->storage_artifact_id = null;
|
|
$tornBackup->object_key = (string) $backup->object_key . '.torn';
|
|
$tornBackup->sealed_storage_runtime = CloudSecrets::encrypt(
|
|
$runtime,
|
|
'object-storage-backup-runtime'
|
|
);
|
|
$tornBackup->storage_revision = 7;
|
|
$tornBackup->save();
|
|
$tornPlan = array_values(array_filter(
|
|
CloudPortableRestoreTargetService::candidates($tornBackup),
|
|
static fn (CloudPortableRestoreTargetPlan $candidate): bool =>
|
|
(int) $candidate->toArray()['target']['host_id'] === 3
|
|
))[0] ?? null;
|
|
$assert($tornPlan instanceof CloudPortableRestoreTargetPlan,
|
|
'Torn-envelope target plan is unavailable.');
|
|
$tornReserved = CloudPortableRestoreAttemptCoordinator::reserve(
|
|
$actor,
|
|
$tornBackup,
|
|
$tornPlan,
|
|
$tornPlan->sha256(),
|
|
'torn-envelope-vm',
|
|
'torn-envelope-request-000000001'
|
|
)->toArray();
|
|
$tornJob = CloudPortableRestoreJobDispatcher::queue(
|
|
$actor,
|
|
(string) $tornReserved['attempt_key']
|
|
);
|
|
$tornPayload = CloudSecrets::decrypt(
|
|
(string) $tornJob->sealed_payload,
|
|
'job-payload'
|
|
);
|
|
$tornPayload['generation'] = (int) $tornPayload['generation'] + 1;
|
|
$tornJob->sealed_payload = CloudSecrets::encrypt(
|
|
$tornPayload,
|
|
'job-payload'
|
|
);
|
|
$tornJob->save();
|
|
$assert($tickJob->invoke($worker) === true,
|
|
'Worker did not consume the torn portable-restore envelope.');
|
|
$tornJob->refresh();
|
|
$tornAttempt = CloudBackupRestoreAttempt::where(
|
|
'attempt_key',
|
|
(string) $tornReserved['attempt_key']
|
|
)->firstOrFail();
|
|
$tornReservation = CloudResourceReservation::findOrFail(
|
|
(int) $tornReserved['reservation_id']
|
|
);
|
|
$tornClaims = Capsule::table('cloud_resource_reservation_ips')
|
|
->where('reservation_id', (int) $tornReservation->id)->get();
|
|
$assert($agentCalls === 0
|
|
&& (string) $tornJob->phase === 'terminal'
|
|
&& (string) $tornJob->outcome === 'failed'
|
|
&& (int) $tornJob->dispatch_attempts === 0
|
|
&& (string) $tornJob->effect_resolution === 'not_applied'
|
|
&& (int) $tornJob->finalization_incident === 1
|
|
&& (string) $tornAttempt->state === 'queued'
|
|
&& (int) $tornAttempt->active_attempt === 1
|
|
&& (string) $tornReservation->state === 'held'
|
|
&& $tornClaims->count() === 2
|
|
&& $tornClaims->every(static fn ($claim): bool =>
|
|
(string) $claim->state === 'reserved')
|
|
&& Capsule::table('cloud_instances')->count() === $instanceCount,
|
|
'Torn restore envelope reached Agent or released fenced dual-stack state.');
|
|
} finally {
|
|
Capsule::connection()->rollBack();
|
|
}
|
|
|
|
echo "Portable restore job dispatcher behavior tests passed.\n";
|