You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
virt/web/tests/CloudFirewallWorkflowTest.php

534 lines
22 KiB
PHTML

<?php
namespace App\Services {
final class Config
{
public static function get($name, $default = null)
{
return $name === 'cloud_engine_url'
? 'http://127.0.0.1:8000' : $default;
}
}
}
namespace App\Services\Cloud {
// Isolated test cipher only; production CloudSecrets is never loaded here.
final class CloudSecrets
{
private static array $values = [];
public static function key(string $purpose): string
{
return str_repeat('k', 32);
}
public static function encrypt(array $value): string
{
$key = bin2hex(random_bytes(16));
self::$values[$key] = $value;
return 'firewall-test:' . $key;
}
public static function decrypt(string $value): array
{
return self::$values[substr($value, 14)];
}
}
}
namespace {
require dirname(__DIR__) . '/vendor/autoload.php';
use App\Models\CloudInstance;
use App\Models\CloudInstanceFirewall;
use App\Models\CloudInstanceFirewallVersion;
use App\Models\CloudJob;
use App\Models\User;
use App\Services\Cloud\CloudEngineClient;
use App\Services\Cloud\CloudError;
use App\Services\Cloud\CloudFirewallService;
use App\Services\Cloud\CloudJobService;
use App\Services\Cloud\CloudLifecycleService;
use App\Services\Cloud\CloudSecrets;
use App\Services\Cloud\CloudWorkerService;
use GuzzleHttp\Client;
use GuzzleHttp\Handler\HandlerStack;
use GuzzleHttp\Handler\MockHandler;
use GuzzleHttp\Psr7\Response;
use Illuminate\Database\Capsule\Manager as Capsule;
use Illuminate\Database\Schema\Blueprint;
$assert = static function ($condition, string $message): void {
if (!$condition) {
throw new RuntimeException($message);
}
};
$expect = static function (callable $callable, int $status,
string $message) use ($assert): void {
try {
$callable();
} catch (CloudError $error) {
$assert($error->httpStatus === $status,
$message . ' Unexpected status: ' . $error->httpStatus);
return;
}
throw new RuntimeException($message);
};
$capsule = new Capsule();
$capsule->addConnection([
'driver' => 'sqlite', 'database' => ':memory:',
], 'default');
$capsule->setAsGlobal();
$capsule->bootEloquent();
$schema = $capsule->schema('default');
$schema->create('user', static function (Blueprint $table): void {
$table->increments('id');
$table->string('money')->default('0.00');
$table->integer('enable')->default(1);
$table->integer('is_admin')->default(0);
});
$schema->create('cloud_instances', static function (Blueprint $table): void {
$table->increments('id');
$table->string('uuid');
$table->integer('host_id');
$table->integer('user_id');
$table->string('name');
$table->string('status');
$table->string('service_status');
$table->string('network_status');
$table->integer('expires_at')->nullable();
$table->text('spec_json');
$table->text('credential_cipher')->nullable();
$table->integer('created_at');
$table->integer('updated_at');
});
$schema->create('cloud_instance_firewalls', static function (
Blueprint $table): void {
$table->integer('instance_id')->primary();
$table->integer('user_id');
$table->integer('host_id');
$table->boolean('enabled');
$table->integer('desired_revision');
$table->integer('applied_revision');
$table->string('desired_digest')->nullable();
$table->string('applied_digest')->nullable();
$table->string('sync_state');
$table->string('last_operation_id')->nullable();
$table->integer('last_success_at');
$table->integer('last_failure_at');
$table->string('last_error_code')->nullable();
$table->integer('created_at');
$table->integer('updated_at');
});
$schema->create('cloud_instance_firewall_versions', static function (
Blueprint $table): void {
$table->increments('id');
$table->integer('instance_id');
$table->integer('revision');
$table->boolean('enabled');
$table->string('policy_digest');
$table->string('request_hash');
$table->string('idempotency_key');
$table->string('state');
$table->integer('job_id')->nullable();
$table->string('operation_id')->nullable();
$table->string('error_code')->nullable();
$table->integer('applied_at')->nullable();
$table->integer('created_at');
$table->integer('updated_at');
$table->unique(['instance_id', 'revision']);
$table->unique(['instance_id', 'idempotency_key']);
});
$schema->create('cloud_instance_firewall_rules', static function (
Blueprint $table): void {
$table->increments('id');
$table->integer('version_id');
$table->integer('sequence');
$table->string('direction');
$table->string('protocol');
$table->string('source_cidr');
$table->integer('port_from')->nullable();
$table->integer('port_to')->nullable();
$table->string('description');
$table->integer('created_at');
});
$schema->create('cloud_jobs', static function (Blueprint $table): void {
$table->increments('id');
$table->string('operation_id')->unique();
$table->integer('order_id')->nullable();
$table->integer('instance_id')->nullable();
$table->integer('snapshot_id')->nullable();
$table->integer('user_id');
$table->integer('host_id');
$table->string('kind');
$table->string('status');
$table->text('payload_cipher');
$table->integer('attempts')->default(0);
$table->integer('dispatched_at')->nullable();
$table->string('lease_token')->nullable();
$table->integer('lease_until')->default(0);
$table->integer('available_at');
$table->string('error_code')->nullable();
$table->text('result_json')->nullable();
$table->integer('created_at');
$table->integer('updated_at');
});
foreach ([10, 20, 30] as $userId) {
$user = new User();
$user->id = $userId;
$user->money = '0.00';
$user->enable = 1;
$user->is_admin = 0;
$user->save();
}
$newInstance = static function (int $owner, string $source = 'cloud_template'):
CloudInstance {
$instance = new CloudInstance();
$instance->uuid = sprintf('11111111-1111-4111-8111-%012d', $owner);
$instance->host_id = 7;
$instance->user_id = $owner;
$instance->name = 'vm-firewall-' . $owner;
$instance->status = 'active';
$instance->service_status = 'exempt';
$instance->network_status = 'active';
$instance->expires_at = null;
$instance->spec_json = json_encode([
'os_source' => $source, 'public_ip_cidr' => '192.0.2.10/24',
], JSON_THROW_ON_ERROR);
$instance->created_at = $instance->updated_at = time();
$instance->save();
return $instance;
};
$instance = $newInstance(10);
$actor = ['user_id' => 10, 'is_admin' => false,
'is_super_admin' => false, 'can_manage' => false];
$empty = CloudFirewallService::listing($instance, $actor);
$assert($empty === [
'enabled' => false, 'desired_revision' => 0,
'applied_revision' => 0, 'sync_state' => 'disabled',
'last_success_at' => null, 'last_failure_at' => null,
'last_error_code' => null, 'can_mutate' => true, 'rules' => [],
], 'A fresh managed instance did not expose the disabled database-only policy.');
$request = [
'enabled' => true,
'expected_revision' => 0,
'idempotency_key' => 'fw-request-00000001',
// Deliberately unsorted; canonicalization is part of the digest spec.
'rules' => [[
'protocol' => 'TCP', 'source_cidr' => '203.0.113.55',
'port_from' => 22, 'port_to' => 22, 'description' => 'SSH',
], [
'protocol' => 'icmp', 'source_cidr' => '0.0.0.123/0',
'port_from' => null, 'port_to' => null, 'description' => 'Ping',
]],
];
$queued = CloudFirewallService::apply($instance, $actor, $request);
$job = $queued['job'];
$payload = CloudSecrets::decrypt((string) $job->payload_cipher);
$expectedDigest = '46c4d40eef4410edaf998e17981ae91d161348e2fc6572b2bd96c8b5cbf9bfc3';
$assert($payload['method'] === 'PUT'
&& $payload['resource'] === 'vms/' . $instance->uuid . '/firewall'
&& $payload['body']['revision'] === 1
&& $payload['body']['policy_digest'] === $expectedDigest
&& $payload['body']['rules'][0] === [
'protocol' => 'icmp', 'source_cidr' => '0.0.0.0/0',
'port_from' => null, 'port_to' => null, 'description' => 'Ping',
]
&& $payload['body']['rules'][1]['source_cidr'] === '203.0.113.55/32',
'Firewall apply lost its typed engine contract or fixed v1 digest vector.');
$assert($queued['policy']['sync_state'] === 'pending'
&& $queued['policy']['desired_revision'] === 1
&& $queued['policy']['applied_revision'] === 0
&& $queued['policy']['can_mutate'] === false,
'Queued desired state was presented as already applied.');
$retried = CloudFirewallService::apply($instance, $actor, $request);
$assert((int) $retried['job']->id === (int) $job->id
&& CloudJob::count() === 1,
'An identical idempotent retry created a second engine operation.');
$changedReuse = $request;
$changedReuse['enabled'] = false;
$expect(static fn () => CloudFirewallService::apply(
$instance, $actor, $changedReuse
), 409, 'One idempotency key was accepted for two different requests.');
$nextRequest = [
'enabled' => true,
'expected_revision' => 1,
'idempotency_key' => 'fw-request-00000002',
'rules' => [[
'protocol' => 'tcp', 'source_cidr' => '198.51.100.0/24',
'port_from' => 443, 'port_to' => 443, 'description' => 'HTTPS',
]],
];
$expect(static fn () => CloudFirewallService::apply(
$instance, $actor, $nextRequest
), 409, 'A queued firewall target accepted an unbounded replacement.');
$assert(CloudJob::count() === 1
&& CloudInstanceFirewallVersion::count() === 1,
'A rejected queued replacement still amplified durable history.');
$job->status = 'running';
$job->dispatched_at = time();
$job->save();
$assert(CloudFirewallService::assertJobRunnable($job, $payload),
'The current firewall target was rejected before dispatch.');
CloudFirewallService::markJobRunning($job);
CloudFirewallService::finishJobLocked($job, 'succeeded', [
'revision' => 1,
'policy_digest' => $payload['body']['policy_digest'],
], $payload, [
'operation_id' => (string) $job->operation_id,
'status' => 'succeeded',
]);
$job->status = 'succeeded';
$job->save();
$replacement = CloudFirewallService::apply(
$instance, $actor, $nextRequest
);
$replacementJob = $replacement['job'];
$replacementPayload = CloudSecrets::decrypt(
(string) $replacementJob->payload_cipher
);
$assert((string) CloudInstanceFirewallVersion::where('revision', 1)
->value('state') === 'applied'
&& $replacement['policy']['desired_revision'] === 2
&& CloudJob::where('status', 'queued')->count() === 1,
'A confirmed firewall target did not allow the next revision.');
$replacementJob->lease_token = str_repeat('a', 32);
$replacementJob->lease_until = time() + 60;
$replacementJob->save();
$expect(static fn () => CloudFirewallService::apply($instance, $actor, [
'enabled' => false, 'expected_revision' => 2,
'idempotency_key' => 'fw-request-00000003', 'rules' => [],
]), 409, 'A leased firewall job was replaced under its worker.');
$replacementJob->lease_token = null;
$replacementJob->lease_until = 0;
$replacementJob->status = 'running';
$replacementJob->dispatched_at = time();
$replacementJob->save();
CloudFirewallService::assertJobRunnable($replacementJob, $replacementPayload);
CloudFirewallService::markJobRunning($replacementJob);
$expect(static fn () => CloudFirewallService::apply($instance, $actor, [
'enabled' => false, 'expected_revision' => 2,
'idempotency_key' => 'fw-request-00000004', 'rules' => [],
]), 409, 'A running firewall operation accepted a competing target.');
$expect(static fn () => CloudFirewallService::finishJobLocked(
$replacementJob, 'succeeded', [
'revision' => 2,
'policy_digest' => $replacementPayload['body']['policy_digest'],
], $replacementPayload, [
'operation_id' => str_repeat('f', 32),
'status' => 'succeeded',
]
), 502, 'A firewall result from another operation was accepted.');
CloudFirewallService::finishJobLocked($replacementJob, 'succeeded', [
'revision' => 2,
'policy_digest' => $replacementPayload['body']['policy_digest'],
], $replacementPayload, [
'operation_id' => (string) $replacementJob->operation_id,
'status' => 'succeeded',
]);
$replacementJob->status = 'succeeded';
$replacementJob->lease_token = null;
$replacementJob->save();
$applied = CloudFirewallService::listing($instance, $actor);
$assert($applied['sync_state'] === 'active'
&& $applied['desired_revision'] === 2
&& $applied['applied_revision'] === 2
&& $applied['last_success_at'] !== null
&& $applied['can_mutate'] === true,
'A confirmed result did not converge desired and applied revisions.');
$disable = CloudFirewallService::apply($instance, $actor, [
'enabled' => false, 'expected_revision' => 2,
'idempotency_key' => 'fw-request-00000005',
'rules' => $applied['rules'],
]);
$disableJob = $disable['job'];
$disablePayload = CloudSecrets::decrypt((string) $disableJob->payload_cipher);
$disableJob->status = 'uncertain';
$disableJob->dispatched_at = time();
$disableJob->save();
CloudFirewallService::markJobUncertain($disableJob);
$uncertain = CloudFirewallService::listing($instance, $actor);
$assert($uncertain['sync_state'] === 'uncertain'
&& $uncertain['applied_revision'] === 2
&& $uncertain['last_error_code'] === 'firewall_result_uncertain'
&& $uncertain['can_mutate'] === false,
'A lost response overwrote the last confirmed policy or allowed replay.');
CloudFirewallService::finishJobLocked($disableJob, 'succeeded', [
'revision' => 3,
'policy_digest' => $disablePayload['body']['policy_digest'],
], $disablePayload, [
'operation_id' => (string) $disableJob->operation_id,
'status' => 'succeeded',
]);
$disableJob->status = 'succeeded';
$disableJob->save();
$disabled = CloudFirewallService::listing($instance, $actor);
$assert($disabled['sync_state'] === 'disabled'
&& $disabled['enabled'] === false
&& $disabled['applied_revision'] === 3,
'A reconciled disable did not preserve rules and confirm the revision.');
$completedJobCount = CloudJob::count();
$completedVersionCount = CloudInstanceFirewallVersion::count();
$samePolicy = CloudFirewallService::apply($instance, $actor, [
'enabled' => false, 'expected_revision' => 3,
'idempotency_key' => 'fw-request-same-policy-0001',
'rules' => $disabled['rules'],
]);
$assert($samePolicy['job'] === null
&& $samePolicy['unchanged'] === true
&& CloudJob::count() === $completedJobCount
&& CloudInstanceFirewallVersion::count() === $completedVersionCount,
'An already-converged policy created duplicate durable history.');
$expect(static fn () => CloudFirewallService::listing($instance, [
'user_id' => 99, 'is_admin' => true, 'is_super_admin' => true,
'can_manage' => true,
]), 404, 'A non-owner received the tenant firewall policy.');
$iso = $newInstance(20, 'iso');
$expect(static fn () => CloudFirewallService::listing($iso, [
'user_id' => 20, 'is_admin' => false,
]), 409, 'An ISO instance received unsupported firewall controls.');
$expect(static fn () => CloudFirewallService::apply($instance, $actor, [
'enabled' => true, 'expected_revision' => 3,
'idempotency_key' => 'fw-request-invalid-1',
'rules' => [[
'protocol' => 'icmp', 'source_cidr' => '::/0',
'port_from' => null, 'port_to' => null, 'description' => '',
]],
]), 422, 'An IPv6 rule bypassed the IPv4-only contract.');
// An exact firewall reconciliation may legitimately stay uncertain until
// an administrator repairs host evidence. It must never grant free service
// by permanently blocking traffic enforcement or expiry destruction.
$overlap = CloudFirewallService::apply($instance, $actor, [
'enabled' => true, 'expected_revision' => 3,
'idempotency_key' => 'fw-request-00000006',
'rules' => [[
'protocol' => 'tcp', 'source_cidr' => '192.0.2.44/32',
'port_from' => 22, 'port_to' => 22,
'description' => 'Deletion overlap',
]],
]);
$overlapJob = $overlap['job'];
$overlapPayload = CloudSecrets::decrypt(
(string) $overlapJob->payload_cipher
);
$lifecycleConflict = new ReflectionMethod(
CloudLifecycleService::class, 'hasConflictingJob'
);
$lifecycle = new CloudLifecycleService();
foreach (['queued', 'running', 'uncertain'] as $firewallJobState) {
$overlapJob->status = $firewallJobState;
$overlapJob->save();
foreach ([
['active', 'blocking'],
['active', 'blocking_expired'],
['purge_stopping', 'active'],
['resume_pending', 'restore_pending'],
] as [$serviceState, $networkState]) {
$instance->service_status = $serviceState;
$instance->network_status = $networkState;
$instance->save();
$assert($lifecycleConflict->invoke($lifecycle, $instance) === false,
'A firewall operation blocked lifecycle convergence.');
}
}
$overlapJob->status = 'uncertain';
$overlapJob->dispatched_at = time();
$overlapJob->save();
$manualDelete = CloudJobService::enqueue(
$instance, $actor, 'delete', 'DELETE', '?delete_disks=true', []
);
$assert((string) $manualDelete->kind === 'delete'
&& (string) $manualDelete->status === 'queued',
'An uncertain firewall operation blocked an owner-requested delete.');
$manualDelete->status = 'failed';
$manualDelete->save();
$instance->status = 'deleting';
$instance->service_status = 'purge_deleting';
$instance->network_status = 'blocked_expired';
$instance->save();
$expiryDelete = CloudJobService::enqueueExpiryDelete($instance);
$assert((string) $expiryDelete->kind === 'expiry_delete'
&& (string) $expiryDelete->status === 'queued',
'An uncertain firewall operation blocked irreversible expiry cleanup.');
CloudFirewallService::finishInstanceDeleteLocked(
$expiryDelete, 'succeeded'
);
$instance->status = 'deleted';
$instance->save();
$expiryDelete->status = 'succeeded';
$expiryDelete->save();
$assert(CloudFirewallService::assertJobRunnable(
$overlapJob, $overlapPayload
) === false,
'A firewall job remained runnable after confirmed instance deletion.');
$overlapJob->available_at = 0;
$overlapJob->lease_token = null;
$overlapJob->lease_until = 0;
$overlapJob->save();
(new CloudWorkerService())->tick();
$overlapJob->refresh();
$deletedPolicy = CloudInstanceFirewall::find((int) $instance->id);
$assert((string) $overlapJob->status === 'failed'
&& (string) $overlapJob->error_code === 'firewall_apply_superseded'
&& (string) $deletedPolicy->sync_state === 'deleted'
&& (string) $deletedPolicy->last_operation_id
=== (string) $expiryDelete->operation_id
&& (string) CloudInstanceFirewallVersion::where('revision', 4)
->value('state') === 'superseded',
'A late firewall result revived or obscured a deleted policy.');
// Once a dispatched operation is authoritatively absent from the engine,
// it is safe to terminate locally; omitting the original operation ID from
// that terminal result would otherwise fail the reconciliation guard and
// leave the policy uncertain forever.
$missingInstance = $newInstance(30);
$missingActor = ['user_id' => 30, 'is_admin' => false,
'is_super_admin' => false, 'can_manage' => false];
$missingApply = CloudFirewallService::apply(
$missingInstance, $missingActor, [
'enabled' => true, 'expected_revision' => 0,
'idempotency_key' => 'fw-request-missing-0001',
'rules' => [[
'protocol' => 'tcp', 'source_cidr' => '192.0.2.30/32',
'port_from' => 22, 'port_to' => 22,
'description' => 'Missing operation',
]],
]
);
$missingJob = $missingApply['job'];
$missingJob->status = 'uncertain';
$missingJob->dispatched_at = time() - 30;
$missingJob->available_at = 0;
$missingJob->lease_token = null;
$missingJob->lease_until = 0;
$missingJob->save();
CloudFirewallService::markJobUncertain($missingJob);
$missingEngine = new CloudEngineClient(new Client([
'handler' => HandlerStack::create(new MockHandler([
new Response(404, [], '{}'),
])),
]));
(new CloudWorkerService($missingEngine))->tick();
$missingJob->refresh();
$missingPolicy = CloudFirewallService::listing(
$missingInstance, $missingActor
);
$assert((string) $missingJob->status === 'failed'
&& (string) $missingJob->error_code === 'firewall_apply_failed'
&& $missingPolicy['sync_state'] === 'failed'
&& $missingPolicy['can_mutate'] === true,
'An authoritatively missing engine operation remained uncertain.');
echo "Cloud firewall desired/applied, idempotency and queue workflow passed.\n";
}