You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
virt/web/tests/CloudAgentMutationIdentityT...

651 lines
25 KiB
PHTML

<?php
declare(strict_types=1);
namespace App\Services\Cloud {
/** Isolated encrypted envelope for Agent mutation identity tests. */
final class CloudSecrets
{
private static array $values = [];
public static function encrypt(array $value, string $purpose): string
{
$key = bin2hex(random_bytes(16));
self::$values[$key] = $value;
return 'agent-mutation-identity-test:' . $key;
}
public static function decrypt(string $value, string $purpose): array
{
$key = substr(
$value,
strlen('agent-mutation-identity-test:')
);
if (!array_key_exists($key, self::$values)) {
throw new \RuntimeException('Missing identity test payload.');
}
return self::$values[$key];
}
}
}
namespace {
require dirname(__DIR__) . '/vendor/autoload.php';
use App\Models\CloudJob;
use App\Services\Cloud\CloudError;
use App\Services\Cloud\CloudOperationCompletionService;
use App\Services\Cloud\CloudOperationRecoveryIdentity;
use App\Services\Cloud\CloudOperationRecoveryPolicy;
use App\Services\Cloud\CloudRecoveryOperationIdentityConflict;
use App\Services\Cloud\CloudSecrets;
use App\Services\Cloud\CloudWorkerService;
use Illuminate\Database\Capsule\Manager as Capsule;
use Illuminate\Database\Schema\Blueprint;
$assert = static function (bool $condition, string $message): void {
if (!$condition) {
throw new RuntimeException($message);
}
};
$expectCloudError = static function (
callable $callback,
int $status
) use ($assert): void {
try {
$callback();
} catch (CloudError $error) {
$assert($error->httpStatus === $status,
'Unexpected CloudError status.');
return;
}
throw new RuntimeException('Expected CloudError was not thrown.');
};
$capsule = new Capsule();
$capsule->addConnection([
'driver' => 'sqlite', 'database' => ':memory:',
], 'default');
$capsule->setAsGlobal();
$capsule->bootEloquent();
$schema = $capsule->schema('default');
$schema->create('cloud_jobs', static function (Blueprint $table): void {
$table->increments('id');
$table->integer('host_id');
$table->integer('user_id');
$table->integer('order_id')->nullable();
$table->integer('instance_id')->nullable();
$table->integer('snapshot_id')->nullable();
$table->string('operation_id')->unique();
$table->string('execution_mode');
$table->string('kind');
$table->string('phase');
$table->string('outcome')->nullable();
$table->string('status');
$table->integer('dispatch_attempts')->default(1);
$table->integer('terminal_at')->nullable();
$table->boolean('effect_may_have_applied')->nullable();
$table->boolean('safe_to_resubmit')->nullable();
$table->string('effect_resolution')->default('not_applicable');
$table->text('sealed_payload');
$table->string('completion_event_id')->nullable();
$table->boolean('finalization_incident')->default(0);
$table->string('finalization_error_code')->nullable();
$table->integer('read_at')->nullable();
$table->string('lease_token')->nullable();
$table->integer('lease_until')->default(0);
$table->integer('available_at')->default(0);
$table->string('error_code')->nullable();
$table->integer('created_at');
$table->integer('updated_at');
});
$schema->create('cloud_agent_completion_events',
static function (Blueprint $table): void {
$table->string('event_id')->primary();
$table->integer('host_id');
$table->string('operation_id');
$table->string('payload_sha256');
$table->string('outcome');
$table->boolean('effect_may_have_applied');
$table->boolean('safe_to_resubmit');
$table->string('error_code')->default('');
$table->text('result_json')->nullable();
$table->string('disposition')->default('received');
$table->integer('received_at');
$table->integer('applied_at')->nullable();
$table->unique(['host_id', 'operation_id']);
});
$schema->create('cloud_host_storage_configurations',
static function (Blueprint $table): void {
$table->integer('host_id')->primary();
$table->integer('revision');
foreach (['managed', 'template', 'seed', 'user_iso', 'rescue']
as $role) {
$table->text($role . '_directory');
}
$table->string('state');
$table->string('operation_id')->nullable();
$table->integer('job_id')->nullable();
$table->integer('created_at');
$table->integer('updated_at');
});
$now = time();
$createBody = static function (string $operationId, int $suffix): array {
return [
'operation_id' => $operationId,
'source' => 'cloud',
'name' => 'identity-vm-' . $suffix,
'vcpu' => 2,
'memory_mib' => 2048,
'disk_gib' => 40,
'bandwidth_mbps' => 100,
'network' => 'bridge:br0',
'template_id' => 'debian-13',
'template_revision' => 1,
'template_generation_token' => str_repeat('1', 32),
'root_password' => 'Identity-test9!Pass',
'public_pool_key' => str_repeat('2', 32),
'public_pool_cidr' => '203.0.113.0/24',
'public_ip_cidr' => '203.0.113.' . $suffix . '/24',
'gateway' => '203.0.113.1',
'dns_servers' => ['1.1.1.1'],
'platform_network_policy_snapshot' => ['schema_version' => 1],
];
};
$insertCreate = static function (
int $id,
string $operationId
) use ($createBody, $now): void {
Capsule::table('cloud_jobs')->insert([
'id' => $id,
'host_id' => 7,
'user_id' => 11,
'order_id' => 1000 + $id,
'operation_id' => $operationId,
'execution_mode' => 'agent',
'kind' => 'create',
'phase' => 'awaiting_receipt',
'outcome' => null,
'status' => 'running',
'dispatch_attempts' => 1,
'effect_resolution' => 'not_applicable',
'sealed_payload' => CloudSecrets::encrypt([
'method' => 'POST',
'request_target' => '/v1/vms',
'body' => $createBody($operationId, 20 + $id),
'credential' => [
'password' => 'Identity-test9!Pass',
'login_name' => 'root',
'auth_mode' => 'password',
],
'template_metadata' => [
'template_id' => 'debian-13',
'os_source' => 'cloud_template',
'os_name' => 'Debian 13',
'login_name' => 'root',
],
], 'job-payload'),
'created_at' => $now - 30,
'updated_at' => $now - 10,
]);
};
$createOperation = str_repeat('1a', 16);
$storageOperation = str_repeat('2b', 16);
$invalidCreateOperation = str_repeat('3c', 16);
$hashCreateOperation = str_repeat('4d', 16);
$secretCreateOperation = str_repeat('5e', 16);
$activeLedgerConflictOperation = str_repeat('6f', 16);
$terminalLedgerConflictOperation = str_repeat('70', 16);
$repairedDuringDeferOperation = str_repeat('81', 16);
$insertCreate(1, $createOperation);
$insertCreate(3, $invalidCreateOperation);
$insertCreate(4, $hashCreateOperation);
$insertCreate(5, $secretCreateOperation);
$insertCreate(6, $activeLedgerConflictOperation);
$insertCreate(7, $terminalLedgerConflictOperation);
$insertCreate(8, $repairedDuringDeferOperation);
$storageBody = [
'operation_id' => $storageOperation,
'managed_directory' => '/srv/virt/managed',
'template_directory' => '/srv/virt/templates',
'seed_directory' => '/srv/virt/seeds',
'user_iso_directory' => '/srv/virt/user-isos',
'rescue_directory' => '/srv/virt/rescue',
];
Capsule::table('cloud_jobs')->insert([
'id' => 2,
'host_id' => 8,
'user_id' => 11,
'operation_id' => $storageOperation,
'execution_mode' => 'agent',
'kind' => 'host_storage_configure',
'phase' => 'dispatching',
'outcome' => null,
'status' => 'running',
'dispatch_attempts' => 1,
'effect_resolution' => 'not_applicable',
'sealed_payload' => CloudSecrets::encrypt([
'method' => 'POST',
'request_target' => '/v1/storage/configuration',
'body' => $storageBody,
'owner_id' => 11,
'can_manage' => true,
'storage_revision' => 4,
], 'job-payload'),
'created_at' => $now - 30,
'updated_at' => $now - 10,
]);
Capsule::table('cloud_host_storage_configurations')->insert([
'host_id' => 8,
'revision' => 4,
'managed_directory' => $storageBody['managed_directory'],
'template_directory' => $storageBody['template_directory'],
'seed_directory' => $storageBody['seed_directory'],
'user_iso_directory' => $storageBody['user_iso_directory'],
'rescue_directory' => $storageBody['rescue_directory'],
'state' => 'applying',
'operation_id' => $storageOperation,
'job_id' => 2,
'created_at' => $now - 30,
'updated_at' => $now - 10,
]);
$createJob = CloudJob::findOrFail(1);
$storageJob = CloudJob::findOrFail(2);
$createIdentity = CloudOperationRecoveryIdentity::expected($createJob);
$storageIdentity = CloudOperationRecoveryIdentity::expected($storageJob);
$assert(CloudOperationRecoveryIdentity::supports('create')
&& CloudOperationRecoveryIdentity::supports(
'host_storage_configure'
)
&& !CloudOperationRecoveryPolicy::supports('create')
&& !CloudOperationRecoveryPolicy::supports(
'host_storage_configure'
)
&& $createIdentity['action'] === 'create-vm'
&& preg_match(
'/^[a-f0-9]{8}-[a-f0-9]{4}-5[a-f0-9]{3}-[89ab][a-f0-9]{3}-[a-f0-9]{12}$/D',
$createIdentity['resource_id']
) === 1
&& $storageIdentity['action'] === 'configure-storage'
&& $storageIdentity['resource_id'] === 'host-storage',
'Agent identity support was incorrectly coupled to recovery policy.');
$operationForJob = new ReflectionMethod(
CloudWorkerService::class,
'operationForJob'
);
$validSync = [
'operation_id' => $createOperation,
'status' => 'succeeded',
] + $createIdentity;
$assert($operationForJob->invoke(null, $createJob, $validSync)
=== $validSync,
'A correctly bound synchronous create response was rejected.');
$wrongSync = $validSync;
$wrongSync['resource_id'] = 'different-create-resource';
try {
$operationForJob->invoke(null, $createJob, $wrongSync);
throw new RuntimeException(
'A cross-operation synchronous create response was accepted.'
);
} catch (CloudRecoveryOperationIdentityConflict $expected) {
// Expected: no result from the mismatched response can reach ledgers.
}
$validStorageSync = [
'operation_id' => $storageOperation,
'status' => 'succeeded',
] + $storageIdentity;
$wrongStorageSync = $validStorageSync;
$wrongStorageSync['request_fingerprint'] = str_repeat('f', 64);
try {
$operationForJob->invoke(null, $storageJob, $wrongStorageSync);
throw new RuntimeException(
'A cross-operation synchronous storage response was accepted.'
);
} catch (CloudRecoveryOperationIdentityConflict $expected) {
// Expected.
}
$receipt = static function (
string $eventId,
int $hostId,
string $operationId,
array $identity
): array {
return [
'type' => 'completion',
'id' => $eventId,
'agent_id' => (string) $hostId,
'operation_id' => $operationId,
'action' => $identity['action'],
'resource_id' => $identity['resource_id'],
'request_fingerprint' => $identity['request_fingerprint'],
'status' => 'succeeded',
'effect_may_have_applied' => false,
'safe_to_resubmit' => false,
'error_code' => '',
'result' => [],
];
};
// Completion ingestion may persist evidence for a corrupt dispatch
// ledger, but it must not ACK/consume that evidence. Both an active task
// and an apparently settled terminal task stay visibly repairable until
// dispatch_attempts is restored to one of its only valid states (0/1).
Capsule::table('cloud_jobs')->where('id', 6)->update([
'dispatch_attempts' => 2,
]);
$activeLedgerConflictJob = CloudJob::findOrFail(6);
$activeLedgerConflictReceipt = $receipt(
'active-dispatch-ledger-conflict',
7,
$activeLedgerConflictOperation,
CloudOperationRecoveryIdentity::expected(
$activeLedgerConflictJob,
true
)
);
$activeLedgerConflictAccepted =
CloudOperationCompletionService::accept(
$activeLedgerConflictReceipt
);
$activeLedgerConflictJob->refresh();
$activeLedgerConflictEvent = Capsule::table(
'cloud_agent_completion_events'
)->where(
'event_id',
$activeLedgerConflictReceipt['id']
)->first();
$assert($activeLedgerConflictAccepted === [
'accepted' => true,
'disposition' => 'received',
]
&& (string) $activeLedgerConflictJob->phase === 'terminal'
&& (string) $activeLedgerConflictJob->outcome === 'failed'
&& (bool) $activeLedgerConflictJob->effect_may_have_applied
&& !(bool) $activeLedgerConflictJob->safe_to_resubmit
&& (string) $activeLedgerConflictJob->effect_resolution === 'unknown'
&& (bool) $activeLedgerConflictJob->finalization_incident
&& (string) $activeLedgerConflictJob->finalization_error_code
=== 'recovery_dispatch_ledger_conflict'
&& (string) $activeLedgerConflictEvent->disposition === 'received'
&& $activeLedgerConflictEvent->applied_at === null,
'An active invalid dispatch ledger consumed its immutable completion: '
. json_encode([
'accepted' => $activeLedgerConflictAccepted,
'job' => $activeLedgerConflictJob->toArray(),
'event' => (array) $activeLedgerConflictEvent,
], JSON_THROW_ON_ERROR));
$terminalLedgerConflictEventId =
'terminal-dispatch-ledger-conflict';
Capsule::table('cloud_jobs')->where('id', 7)->update([
'phase' => 'terminal',
'outcome' => 'succeeded',
'status' => 'succeeded',
'dispatch_attempts' => 2,
'terminal_at' => time(),
'effect_may_have_applied' => 0,
'safe_to_resubmit' => 0,
'effect_resolution' => 'applied',
'completion_event_id' => $terminalLedgerConflictEventId,
]);
$terminalLedgerConflictJob = CloudJob::findOrFail(7);
$terminalLedgerConflictTerminalAt =
(int) $terminalLedgerConflictJob->terminal_at;
$terminalLedgerConflictReceipt = $receipt(
$terminalLedgerConflictEventId,
7,
$terminalLedgerConflictOperation,
CloudOperationRecoveryIdentity::expected(
$terminalLedgerConflictJob,
true
)
);
$terminalLedgerConflictAccepted =
CloudOperationCompletionService::accept(
$terminalLedgerConflictReceipt
);
$terminalLedgerConflictJob->refresh();
$terminalLedgerConflictEvent =
\App\Models\CloudAgentCompletionEvent::where(
'event_id',
$terminalLedgerConflictEventId
)->firstOrFail();
$assert($terminalLedgerConflictAccepted === [
'accepted' => true,
'disposition' => 'received',
]
&& (string) $terminalLedgerConflictJob->phase === 'terminal'
&& (string) $terminalLedgerConflictJob->outcome === 'succeeded'
&& (string) $terminalLedgerConflictJob->status === 'succeeded'
&& !(bool) $terminalLedgerConflictJob->effect_may_have_applied
&& !(bool) $terminalLedgerConflictJob->safe_to_resubmit
&& (string) $terminalLedgerConflictJob->effect_resolution === 'applied'
&& $terminalLedgerConflictJob->error_code === null
&& (int) $terminalLedgerConflictJob->terminal_at
=== $terminalLedgerConflictTerminalAt
&& (string) $terminalLedgerConflictJob->completion_event_id
=== $terminalLedgerConflictEventId
&& (bool) $terminalLedgerConflictJob->finalization_incident
&& (string) $terminalLedgerConflictJob->finalization_error_code
=== 'recovery_dispatch_ledger_conflict'
&& (string) $terminalLedgerConflictEvent->disposition === 'received'
&& $terminalLedgerConflictEvent->applied_at === null
&& !CloudOperationCompletionService::settledTaskWithEvent(
$terminalLedgerConflictJob,
$terminalLedgerConflictEvent
),
'A terminal invalid dispatch ledger ACKed a received completion.');
// Simulate an operator repairing the counter after a worker observed the
// invalid value but before deferDispatchLedgerConflict acquired its lock.
// The repaired value must remain sticky until background validation.
Capsule::table('cloud_jobs')->where('id', 8)->update([
'dispatch_attempts' => 2,
]);
$repairedDuringDefer = CloudJob::findOrFail(8);
Capsule::table('cloud_jobs')->where('id', 8)->update([
'dispatch_attempts' => 1,
]);
CloudOperationCompletionService::deferDispatchLedgerConflict(
$repairedDuringDefer
);
$repairedDuringDefer->refresh();
$assert((int) $repairedDuringDefer->dispatch_attempts === 1
&& (string) $repairedDuringDefer->phase === 'terminal'
&& (string) $repairedDuringDefer->outcome === 'failed'
&& (bool) $repairedDuringDefer->effect_may_have_applied
&& (string) $repairedDuringDefer->effect_resolution === 'unknown'
&& (bool) $repairedDuringDefer->finalization_incident
&& (string) $repairedDuringDefer->finalization_error_code
=== 'recovery_dispatch_ledger_conflict',
'A lock-window dispatch repair bypassed the sticky safety incident.');
$wrongCreate = $receipt(
'create-wrong-identity-event',
7,
$createOperation,
$createIdentity
);
$wrongCreate['resource_id'] = 'different-create-resource';
$createAccepted = CloudOperationCompletionService::accept($wrongCreate);
$wrongStorage = $receipt(
'storage-wrong-identity-event',
8,
$storageOperation,
$storageIdentity
);
$wrongStorage['request_fingerprint'] = str_repeat('f', 64);
$storageAccepted = CloudOperationCompletionService::accept($wrongStorage);
$createAfterWrong = CloudJob::findOrFail(1);
$storageAfterWrong = CloudJob::findOrFail(2);
$assert($createAccepted === [
'accepted' => true, 'disposition' => 'rejected',
]
&& $storageAccepted === [
'accepted' => true, 'disposition' => 'rejected',
]
&& (string) $createAfterWrong->phase === 'awaiting_receipt'
&& $createAfterWrong->outcome === null
&& (string) $createAfterWrong->effect_resolution === 'not_applicable'
&& (string) $storageAfterWrong->phase === 'dispatching'
&& $storageAfterWrong->outcome === null
&& (string) $storageAfterWrong->effect_resolution
=== 'not_applicable'
&& (string) Capsule::table('cloud_host_storage_configurations')
->where('host_id', 8)->value('state') === 'applying',
'A wrong identity bypassed create or storage deadline finalization.');
$invalidCreate = $receipt(
'create-invalid-receipt-event',
7,
$invalidCreateOperation,
CloudOperationRecoveryIdentity::expected(CloudJob::findOrFail(3))
);
unset($invalidCreate['action']);
$expectCloudError(
static fn () => CloudOperationCompletionService::accept(
$invalidCreate
),
400
);
$invalidStorage = $receipt(
'storage-invalid-receipt-event',
8,
$storageOperation,
$storageIdentity
);
unset($invalidStorage['action']);
$expectCloudError(
static fn () => CloudOperationCompletionService::accept(
$invalidStorage
),
400
);
$invalidCreateJob = CloudJob::findOrFail(3);
$invalidStorageJob = CloudJob::findOrFail(2);
$assert((string) $invalidCreateJob->phase === 'awaiting_receipt'
&& $invalidCreateJob->outcome === null
&& (bool) $invalidCreateJob->finalization_incident
&& (string) $invalidCreateJob->finalization_error_code
=== 'recovery_receipt_invalid'
&& (string) $invalidStorageJob->phase === 'dispatching'
&& $invalidStorageJob->outcome === null
&& (bool) $invalidStorageJob->finalization_incident
&& (string) $invalidStorageJob->finalization_error_code
=== 'recovery_receipt_invalid'
&& (string) Capsule::table('cloud_host_storage_configurations')
->where('host_id', 8)->value('state') === 'applying',
'An attributable invalid receipt bypassed create/storage finalizers.');
$hashIdentity = CloudOperationRecoveryIdentity::expected(
CloudJob::findOrFail(4)
);
$hashReceipt = $receipt(
'create-hash-conflict-event',
7,
$hashCreateOperation,
$hashIdentity
);
Capsule::table('cloud_agent_completion_events')->insert([
'event_id' => $hashReceipt['id'],
'host_id' => 7,
'operation_id' => $hashCreateOperation,
'payload_sha256' => str_repeat('0', 64),
'outcome' => 'failed',
'effect_may_have_applied' => 1,
'safe_to_resubmit' => 0,
'error_code' => 'old_conflicting_receipt',
'result_json' => null,
'disposition' => 'rejected',
'received_at' => $now - 5,
'applied_at' => null,
]);
$expectCloudError(
static fn () => CloudOperationCompletionService::accept($hashReceipt),
409
);
$hashJob = CloudJob::findOrFail(4);
$assert((string) $hashJob->phase === 'awaiting_receipt'
&& $hashJob->outcome === null
&& (bool) $hashJob->finalization_incident
&& (string) $hashJob->finalization_error_code
=== 'recovery_operation_identity_conflict',
'A payload-hash conflict was invisible or bypassed create finalization.');
$secretReceipt = $receipt(
'create-secret-result-event',
7,
$secretCreateOperation,
CloudOperationRecoveryIdentity::expected(
CloudJob::findOrFail(5),
true
)
);
$secretReceipt['result'] = [
'vm' => ['ssh_private_key' => 'must-never-be-persisted'],
];
$expectCloudError(
static fn () => CloudOperationCompletionService::accept(
$secretReceipt
),
400
);
$secretJob = CloudJob::findOrFail(5);
$assert(!Capsule::table('cloud_agent_completion_events')
->where('event_id', $secretReceipt['id'])->exists()
&& (string) $secretJob->phase === 'awaiting_receipt'
&& $secretJob->outcome === null
&& (bool) $secretJob->finalization_incident
&& (string) $secretJob->finalization_error_code
=== 'recovery_receipt_invalid',
'A create completion secret was persisted or bypassed its finalizer.');
$unknownSecretCases = [
'unknown-generated-password' => [
'generatedPassword' => 'must-never-persist',
],
'unknown-root-password' => [
'nested' => ['root-password' => 'must-never-persist'],
],
'unknown-access-token' => [
'access_token' => 'must-never-persist',
],
'unknown-passphrase' => [
'diskPassphrase' => 'must-never-persist',
],
'unknown-plain-token' => [
'token' => 'must-never-persist',
],
];
$unknownIndex = 0;
foreach ($unknownSecretCases as $eventId => $secretResult) {
$unknownIndex++;
$unknownOperation = substr(hash(
'sha256',
'unknown-secret-operation:' . $unknownIndex
), 0, 32);
$unknownReceipt = $receipt(
$eventId . '-event',
77,
$unknownOperation,
$createIdentity
);
$unknownReceipt['result'] = $secretResult;
$expectCloudError(
static fn () => CloudOperationCompletionService::accept(
$unknownReceipt
),
400
);
$assert(!Capsule::table('cloud_agent_completion_events')
->where('event_id', $unknownReceipt['id'])->exists(),
'An unknown-operation secret completion was persisted.');
}
echo "Agent mutation identity binding tests passed.\n";
}