You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
953 lines
41 KiB
PHP
953 lines
41 KiB
PHP
<?php
|
|
namespace App\Services\Cloud {
|
|
// Isolated test cipher only; production CloudSecrets is never loaded here.
|
|
final class CloudSecrets
|
|
{
|
|
private static array $values = [];
|
|
public static function key(string $purpose): string
|
|
{
|
|
return str_repeat('k', 32);
|
|
}
|
|
public static function encrypt(array $value, string $purpose): string
|
|
{
|
|
$key = bin2hex(random_bytes(16));
|
|
self::$values[$key] = $value;
|
|
return 'firewall-test:' . $key;
|
|
}
|
|
public static function decrypt(string $value, string $purpose): array
|
|
{
|
|
return self::$values[substr($value, 14)];
|
|
}
|
|
}
|
|
}
|
|
|
|
namespace {
|
|
require dirname(__DIR__) . '/vendor/autoload.php';
|
|
|
|
use App\Models\CloudInstance;
|
|
use App\Models\CloudInstanceFirewall;
|
|
use App\Models\CloudInstanceFirewallVersion;
|
|
use App\Models\CloudInstanceMigration;
|
|
use App\Models\CloudJob;
|
|
use App\Models\User;
|
|
use App\Services\Cloud\CloudAgentClient;
|
|
use App\Services\Cloud\CloudError;
|
|
use App\Services\Cloud\CloudFirewallService;
|
|
use App\Services\Cloud\CloudJobService;
|
|
use App\Services\Cloud\CloudLifecycleService;
|
|
use App\Services\Cloud\CloudSecrets;
|
|
use App\Services\Cloud\CloudWorkerService;
|
|
use GuzzleHttp\Client;
|
|
use GuzzleHttp\HandlerStack;
|
|
use GuzzleHttp\Handler\MockHandler;
|
|
use GuzzleHttp\Middleware;
|
|
use GuzzleHttp\Psr7\Response;
|
|
use Illuminate\Database\Capsule\Manager as Capsule;
|
|
use Illuminate\Database\Schema\Blueprint;
|
|
|
|
$assert = static function ($condition, string $message): void {
|
|
if (!$condition) {
|
|
throw new RuntimeException($message);
|
|
}
|
|
};
|
|
$markDispatched = static function (CloudJob $job): void {
|
|
$now = time();
|
|
$job->phase = 'awaiting_receipt';
|
|
$job->status = 'running';
|
|
$job->attempts = 1;
|
|
$job->dispatch_attempts = 1;
|
|
$job->dispatch_started_at = $now;
|
|
$job->dispatched_at = $now;
|
|
$job->agent_accepted_at = $now;
|
|
$job->receipt_deadline_at = $now + 960;
|
|
$job->save();
|
|
};
|
|
$terminalize = static function (CloudJob $job, string $outcome,
|
|
bool $effect = false, bool $safe = false): void {
|
|
$now = time();
|
|
$job->phase = 'terminal';
|
|
$job->outcome = $outcome;
|
|
$job->status = $outcome;
|
|
$job->attempts = max(1, (int) $job->attempts);
|
|
$job->dispatch_attempts = max(1, (int) $job->dispatch_attempts);
|
|
$job->dispatch_started_at ??= $now - 1;
|
|
$job->dispatched_at ??= $now - 1;
|
|
$job->agent_accepted_at ??= $now - 1;
|
|
$job->receipt_deadline_at ??= $now + 900;
|
|
$job->terminal_at = $now;
|
|
$job->effect_may_have_applied = $effect ? 1 : 0;
|
|
$job->safe_to_resubmit = $safe ? 1 : 0;
|
|
$job->effect_resolution = $outcome === 'succeeded' ? 'applied'
|
|
: ($effect ? 'unknown' : 'not_applied');
|
|
$job->lease_token = null;
|
|
$job->lease_until = 0;
|
|
$job->available_at = 0;
|
|
$job->updated_at = $now;
|
|
$job->save();
|
|
};
|
|
$setTestState = static function (CloudJob $job, string $state): void {
|
|
$now = time();
|
|
if ($state === 'accepted') {
|
|
$job->phase = 'accepted';
|
|
$job->outcome = null;
|
|
$job->status = 'queued';
|
|
$job->attempts = 0;
|
|
$job->dispatch_attempts = 0;
|
|
$job->dispatch_started_at = null;
|
|
$job->agent_accepted_at = null;
|
|
$job->receipt_deadline_at = null;
|
|
$job->terminal_at = null;
|
|
$job->dispatched_at = null;
|
|
$job->effect_may_have_applied = null;
|
|
$job->safe_to_resubmit = null;
|
|
$job->effect_resolution = 'not_applicable';
|
|
} elseif ($state === 'awaiting_receipt') {
|
|
$job->phase = 'awaiting_receipt';
|
|
$job->outcome = null;
|
|
$job->status = 'running';
|
|
$job->attempts = 1;
|
|
$job->dispatch_attempts = 1;
|
|
$job->dispatch_started_at = $now - 1;
|
|
$job->agent_accepted_at = $now - 1;
|
|
$job->receipt_deadline_at = $now + 900;
|
|
$job->terminal_at = null;
|
|
$job->dispatched_at = $now - 1;
|
|
$job->effect_may_have_applied = null;
|
|
$job->safe_to_resubmit = null;
|
|
$job->effect_resolution = 'not_applicable';
|
|
} elseif ($state === 'failed_unknown') {
|
|
$job->phase = 'terminal';
|
|
$job->outcome = 'failed';
|
|
$job->status = 'failed';
|
|
$job->attempts = 1;
|
|
$job->dispatch_attempts = 1;
|
|
$job->terminal_at = $now;
|
|
$job->effect_may_have_applied = 1;
|
|
$job->safe_to_resubmit = 0;
|
|
$job->effect_resolution = 'unknown';
|
|
} else {
|
|
throw new RuntimeException('Invalid test task state');
|
|
}
|
|
$job->lease_token = null;
|
|
$job->lease_until = 0;
|
|
$job->save();
|
|
};
|
|
$expect = static function (callable $callable, int $status,
|
|
string $message) use ($assert): void {
|
|
try {
|
|
$callable();
|
|
} catch (CloudError $error) {
|
|
$assert($error->httpStatus === $status,
|
|
$message . ' Unexpected status: ' . $error->httpStatus);
|
|
return;
|
|
}
|
|
throw new RuntimeException($message);
|
|
};
|
|
|
|
$capsule = new Capsule();
|
|
$capsule->addConnection([
|
|
'driver' => 'sqlite', 'database' => ':memory:',
|
|
], 'default');
|
|
$capsule->setAsGlobal();
|
|
$capsule->bootEloquent();
|
|
$schema = $capsule->schema('default');
|
|
$schema->create('user', static function (Blueprint $table): void {
|
|
$table->increments('id');
|
|
$table->string('money')->default('0.00');
|
|
$table->integer('enable')->default(1);
|
|
$table->integer('is_admin')->default(0);
|
|
});
|
|
$schema->create('cloud_instances', static function (Blueprint $table): void {
|
|
$table->increments('id');
|
|
$table->string('uuid');
|
|
$table->integer('host_id');
|
|
$table->integer('user_id');
|
|
$table->string('name');
|
|
$table->string('status');
|
|
$table->string('service_status');
|
|
$table->string('network_status');
|
|
$table->integer('expires_at')->nullable();
|
|
$table->text('spec_json');
|
|
$table->text('sealed_credential')->nullable();
|
|
$table->integer('created_at');
|
|
$table->integer('updated_at');
|
|
});
|
|
$schema->create('cloud_instance_firewalls', static function (
|
|
Blueprint $table): void {
|
|
$table->integer('instance_id')->primary();
|
|
$table->integer('user_id');
|
|
$table->integer('host_id');
|
|
$table->boolean('enabled');
|
|
$table->integer('desired_revision');
|
|
$table->integer('applied_revision');
|
|
$table->string('desired_digest')->nullable();
|
|
$table->string('applied_digest')->nullable();
|
|
$table->string('sync_state');
|
|
$table->string('last_operation_id')->nullable();
|
|
$table->integer('last_success_at');
|
|
$table->integer('last_failure_at');
|
|
$table->string('last_error_code')->nullable();
|
|
$table->integer('created_at');
|
|
$table->integer('updated_at');
|
|
});
|
|
$schema->create('cloud_instance_firewall_versions', static function (
|
|
Blueprint $table): void {
|
|
$table->increments('id');
|
|
$table->integer('instance_id');
|
|
$table->integer('revision');
|
|
$table->boolean('enabled');
|
|
$table->string('policy_digest');
|
|
$table->string('request_hash');
|
|
$table->string('idempotency_key');
|
|
$table->string('state');
|
|
$table->integer('job_id')->nullable();
|
|
$table->string('operation_id')->nullable();
|
|
$table->string('error_code')->nullable();
|
|
$table->integer('applied_at')->nullable();
|
|
$table->integer('created_at');
|
|
$table->integer('updated_at');
|
|
$table->unique(['instance_id', 'revision']);
|
|
$table->unique(['instance_id', 'idempotency_key']);
|
|
});
|
|
$schema->create('cloud_instance_firewall_rules', static function (
|
|
Blueprint $table): void {
|
|
$table->increments('id');
|
|
$table->integer('version_id');
|
|
$table->integer('sequence');
|
|
$table->string('direction');
|
|
$table->string('protocol');
|
|
$table->string('source_cidr');
|
|
$table->integer('port_from')->nullable();
|
|
$table->integer('port_to')->nullable();
|
|
$table->string('description');
|
|
$table->integer('created_at');
|
|
});
|
|
$schema->create('cloud_jobs', static function (Blueprint $table): void {
|
|
$table->increments('id');
|
|
$table->string('operation_id')->unique();
|
|
$table->string('request_idempotency_key');
|
|
$table->string('request_fingerprint');
|
|
$table->integer('parent_job_id')->nullable();
|
|
$table->integer('order_id')->nullable();
|
|
$table->integer('invoice_id')->nullable();
|
|
$table->integer('instance_id')->nullable();
|
|
$table->integer('snapshot_id')->nullable();
|
|
$table->integer('vpc_repair_root_job_id')->nullable();
|
|
$table->integer('user_id');
|
|
$table->integer('host_id')->nullable();
|
|
$table->string('kind');
|
|
$table->string('execution_mode');
|
|
$table->string('phase');
|
|
$table->string('outcome')->nullable();
|
|
$table->string('status');
|
|
$table->text('sealed_payload');
|
|
$table->integer('attempts');
|
|
$table->integer('dispatch_attempts');
|
|
$table->integer('dispatch_deadline_at');
|
|
$table->integer('dispatch_started_at')->nullable();
|
|
$table->integer('agent_accepted_at')->nullable();
|
|
$table->integer('receipt_deadline_at')->nullable();
|
|
$table->integer('terminal_at')->nullable();
|
|
$table->boolean('effect_may_have_applied')->nullable();
|
|
$table->boolean('safe_to_resubmit')->nullable();
|
|
$table->string('effect_resolution');
|
|
$table->integer('effect_resolved_at')->nullable();
|
|
$table->integer('effect_resolved_by')->nullable();
|
|
$table->text('effect_resolution_reason')->nullable();
|
|
$table->string('completion_event_id')->nullable();
|
|
$table->integer('read_at')->nullable();
|
|
$table->integer('dispatched_at')->nullable();
|
|
$table->string('lease_token')->nullable();
|
|
$table->integer('lease_until')->default(0);
|
|
$table->integer('available_at');
|
|
$table->string('error_code')->nullable();
|
|
$table->boolean('finalization_incident')->default(0);
|
|
$table->string('finalization_error_code')->nullable();
|
|
$table->text('result_json')->nullable();
|
|
$table->integer('created_at');
|
|
$table->integer('updated_at');
|
|
});
|
|
$schema->create('cloud_agent_completion_events', static function (
|
|
Blueprint $table
|
|
): void {
|
|
$table->string('event_id')->primary();
|
|
$table->integer('host_id');
|
|
$table->string('operation_id');
|
|
$table->string('payload_sha256');
|
|
$table->string('outcome');
|
|
$table->boolean('effect_may_have_applied');
|
|
$table->boolean('safe_to_resubmit');
|
|
$table->string('error_code')->default('');
|
|
$table->text('result_json')->nullable();
|
|
$table->string('disposition')->default('received');
|
|
$table->integer('received_at');
|
|
$table->integer('applied_at')->nullable();
|
|
$table->unique(['host_id', 'operation_id']);
|
|
});
|
|
foreach ([10, 20, 30, 40] as $userId) {
|
|
$user = new User();
|
|
$user->id = $userId;
|
|
$user->money = '0.00';
|
|
$user->enable = 1;
|
|
$user->is_admin = 0;
|
|
$user->save();
|
|
}
|
|
|
|
$newInstance = static function (int $owner, string $source = 'cloud_template'):
|
|
CloudInstance {
|
|
$instance = new CloudInstance();
|
|
$instance->uuid = sprintf('11111111-1111-4111-8111-%012d', $owner);
|
|
$instance->host_id = 7;
|
|
$instance->user_id = $owner;
|
|
$instance->name = 'vm-firewall-' . $owner;
|
|
$instance->status = 'active';
|
|
$instance->service_status = 'exempt';
|
|
$instance->network_status = 'active';
|
|
$instance->expires_at = null;
|
|
$instance->spec_json = json_encode([
|
|
'os_source' => $source, 'public_ip_cidr' => '192.0.2.10/24',
|
|
], JSON_THROW_ON_ERROR);
|
|
$instance->created_at = $instance->updated_at = time();
|
|
$instance->save();
|
|
return $instance;
|
|
};
|
|
|
|
$instance = $newInstance(10);
|
|
$actor = ['user_id' => 10, 'is_admin' => false,
|
|
'is_super_admin' => false, 'can_manage' => false];
|
|
$empty = CloudFirewallService::listing($instance, $actor);
|
|
$assert($empty === [
|
|
'enabled' => false, 'desired_revision' => 0,
|
|
'applied_revision' => 0, 'sync_state' => 'disabled',
|
|
'last_success_at' => null, 'last_failure_at' => null,
|
|
'last_error_code' => null, 'can_mutate' => true, 'rules' => [],
|
|
], 'A fresh managed instance did not expose the disabled database-only policy.');
|
|
|
|
$request = [
|
|
'enabled' => true,
|
|
'expected_revision' => 0,
|
|
'idempotency_key' => 'fw-request-00000001',
|
|
// Deliberately unsorted; canonicalization is part of the digest spec.
|
|
'rules' => [[
|
|
'protocol' => 'TCP', 'source_cidr' => '203.0.113.55',
|
|
'port_from' => 22, 'port_to' => 22, 'description' => 'SSH',
|
|
], [
|
|
'protocol' => 'icmp', 'source_cidr' => '0.0.0.123/0',
|
|
'port_from' => null, 'port_to' => null, 'description' => 'Ping',
|
|
], [
|
|
'protocol' => 'UDP',
|
|
'source_cidr' => '::192.0.2.1/128',
|
|
'port_from' => 53, 'port_to' => 53,
|
|
'description' => 'IPv6 DNS',
|
|
]],
|
|
];
|
|
$queued = CloudFirewallService::apply($instance, $actor, $request);
|
|
$job = $queued['job'];
|
|
$payload = CloudSecrets::decrypt(
|
|
(string) $job->sealed_payload, 'job-payload'
|
|
);
|
|
$expectedDigest =
|
|
'c2769d658fa8ddb6c3cd437da397ccae1e4122cfbae94f4cf81da10198969546';
|
|
$expectedAgentDigest =
|
|
'ad6621eb15e30da61eeb4379a27cc7c9eda192f1a41b6cdc5ec00e1c2f8a0398';
|
|
$assert($payload['method'] === 'PUT'
|
|
&& $payload['request_target']
|
|
=== '/v1/vms/' . $instance->uuid . '/firewall'
|
|
&& $payload['body']['revision'] === 1
|
|
&& $payload['body']['default_in'] === 'drop'
|
|
&& $payload['body']['default_out'] === 'accept'
|
|
&& $payload['firewall_policy_digest'] === $expectedDigest
|
|
&& $payload['agent_firewall_policy_digest'] === $expectedAgentDigest
|
|
&& $payload['body']['rules'][0] === [
|
|
'direction' => 'ingress', 'action' => 'accept',
|
|
'protocol' => 'icmp', 'cidr' => '0.0.0.0/0',
|
|
]
|
|
&& $payload['body']['rules'][1]['cidr'] === '203.0.113.55/32'
|
|
&& $payload['body']['rules'][2] === [
|
|
'direction' => 'ingress', 'action' => 'accept',
|
|
'protocol' => 'udp', 'cidr' => '::c000:201/128',
|
|
'port_from' => 53, 'port_to' => 53,
|
|
],
|
|
'Firewall apply lost its typed Agent contract or fixed v1 digest vector.');
|
|
$assert($queued['policy']['sync_state'] === 'pending'
|
|
&& $queued['policy']['desired_revision'] === 1
|
|
&& $queued['policy']['applied_revision'] === 0
|
|
&& $queued['policy']['can_mutate'] === false,
|
|
'Queued desired state was presented as already applied.');
|
|
|
|
$retried = CloudFirewallService::apply($instance, $actor, $request);
|
|
$assert((int) $retried['job']->id === (int) $job->id
|
|
&& CloudJob::count() === 1,
|
|
'An identical idempotent retry created a second Agent operation.');
|
|
$changedReuse = $request;
|
|
$changedReuse['enabled'] = false;
|
|
$expect(static fn () => CloudFirewallService::apply(
|
|
$instance, $actor, $changedReuse
|
|
), 409, 'One idempotency key was accepted for two different requests.');
|
|
|
|
$nextRequest = [
|
|
'enabled' => true,
|
|
'expected_revision' => 1,
|
|
'idempotency_key' => 'fw-request-00000002',
|
|
'rules' => [[
|
|
'protocol' => 'tcp', 'source_cidr' => '198.51.100.0/24',
|
|
'port_from' => 443, 'port_to' => 443, 'description' => 'HTTPS',
|
|
]],
|
|
];
|
|
$expect(static fn () => CloudFirewallService::apply(
|
|
$instance, $actor, $nextRequest
|
|
), 409, 'A queued firewall target accepted an unbounded replacement.');
|
|
$assert(CloudJob::count() === 1
|
|
&& CloudInstanceFirewallVersion::count() === 1,
|
|
'A rejected queued replacement still amplified durable history.');
|
|
|
|
$markDispatched($job);
|
|
$assert(CloudFirewallService::assertJobRunnable($job, $payload),
|
|
'The current firewall target was rejected before dispatch.');
|
|
CloudFirewallService::markJobRunning($job);
|
|
CloudFirewallService::finishJobLocked($job, 'succeeded', [
|
|
'revision' => 1,
|
|
'policy_digest' => $payload['agent_firewall_policy_digest'],
|
|
], $payload, [
|
|
'operation_id' => (string) $job->operation_id,
|
|
'status' => 'succeeded',
|
|
]);
|
|
$terminalize($job, 'succeeded');
|
|
|
|
$replacement = CloudFirewallService::apply(
|
|
$instance, $actor, $nextRequest
|
|
);
|
|
$replacementJob = $replacement['job'];
|
|
$replacementPayload = CloudSecrets::decrypt(
|
|
(string) $replacementJob->sealed_payload,
|
|
'job-payload'
|
|
);
|
|
$assert((string) CloudInstanceFirewallVersion::where('revision', 1)
|
|
->value('state') === 'applied'
|
|
&& $replacement['policy']['desired_revision'] === 2
|
|
&& CloudJob::where('status', 'queued')->count() === 1,
|
|
'A confirmed firewall target did not allow the next revision.');
|
|
|
|
$replacementJob->lease_token = str_repeat('a', 32);
|
|
$replacementJob->lease_until = time() + 60;
|
|
$replacementJob->save();
|
|
$expect(static fn () => CloudFirewallService::apply($instance, $actor, [
|
|
'enabled' => false, 'expected_revision' => 2,
|
|
'idempotency_key' => 'fw-request-00000003', 'rules' => [],
|
|
]), 409, 'A leased firewall job was replaced under its worker.');
|
|
$replacementJob->lease_token = null;
|
|
$replacementJob->lease_until = 0;
|
|
$markDispatched($replacementJob);
|
|
CloudFirewallService::assertJobRunnable($replacementJob, $replacementPayload);
|
|
CloudFirewallService::markJobRunning($replacementJob);
|
|
$expect(static fn () => CloudFirewallService::apply($instance, $actor, [
|
|
'enabled' => false, 'expected_revision' => 2,
|
|
'idempotency_key' => 'fw-request-00000004', 'rules' => [],
|
|
]), 409, 'A running firewall operation accepted a competing target.');
|
|
|
|
$expect(static fn () => CloudFirewallService::finishJobLocked(
|
|
$replacementJob, 'succeeded', [
|
|
'revision' => 2,
|
|
'policy_digest' => $replacementPayload['agent_firewall_policy_digest'],
|
|
], $replacementPayload, [
|
|
'operation_id' => str_repeat('f', 32),
|
|
'status' => 'succeeded',
|
|
]
|
|
), 502, 'A firewall result from another operation was accepted.');
|
|
|
|
CloudFirewallService::finishJobLocked($replacementJob, 'succeeded', [
|
|
'revision' => 2,
|
|
'policy_digest' => $replacementPayload['agent_firewall_policy_digest'],
|
|
], $replacementPayload, [
|
|
'operation_id' => (string) $replacementJob->operation_id,
|
|
'status' => 'succeeded',
|
|
]);
|
|
$terminalize($replacementJob, 'succeeded');
|
|
$applied = CloudFirewallService::listing($instance, $actor);
|
|
$assert($applied['sync_state'] === 'active'
|
|
&& $applied['desired_revision'] === 2
|
|
&& $applied['applied_revision'] === 2
|
|
&& $applied['last_success_at'] !== null
|
|
&& $applied['can_mutate'] === true,
|
|
'A confirmed result did not converge desired and applied revisions.');
|
|
|
|
$disable = CloudFirewallService::apply($instance, $actor, [
|
|
'enabled' => false, 'expected_revision' => 2,
|
|
'idempotency_key' => 'fw-request-00000005',
|
|
'rules' => $applied['rules'],
|
|
]);
|
|
$disableJob = $disable['job'];
|
|
$disablePayload = CloudSecrets::decrypt(
|
|
(string) $disableJob->sealed_payload, 'job-payload'
|
|
);
|
|
$markDispatched($disableJob);
|
|
CloudFirewallService::markJobUncertain($disableJob);
|
|
$terminalize($disableJob, 'failed', true, false);
|
|
$uncertain = CloudFirewallService::listing($instance, $actor);
|
|
$assert($disableJob->phase === 'terminal'
|
|
&& $disableJob->outcome === 'failed'
|
|
&& $disableJob->effect_resolution === 'unknown'
|
|
&& $uncertain['sync_state'] === 'uncertain'
|
|
&& $uncertain['applied_revision'] === 2
|
|
&& $uncertain['last_error_code'] === 'firewall_result_uncertain'
|
|
&& $uncertain['can_mutate'] === false,
|
|
'An unknown-effect receipt deadline did not end the task and preserve the resource fence.');
|
|
// Simulate an explicit read-only inspection proving that the requested
|
|
// policy is active. The task remains failed; only its resource fence and
|
|
// catalog projection are resolved.
|
|
$disableJob->effect_resolution = 'applied';
|
|
$disableJob->effect_resolved_at = time();
|
|
$disableJob->effect_resolved_by = 99;
|
|
$disableJob->effect_resolution_reason =
|
|
'Read-only host inspection confirmed firewall policy effect.';
|
|
$disableJob->save();
|
|
CloudFirewallService::finishJobLocked($disableJob, 'succeeded', [
|
|
'revision' => 3,
|
|
'policy_digest' => $disablePayload['agent_firewall_policy_digest'],
|
|
], $disablePayload, [
|
|
'operation_id' => (string) $disableJob->operation_id,
|
|
'status' => 'succeeded',
|
|
]);
|
|
$disabled = CloudFirewallService::listing($instance, $actor);
|
|
$assert($disabled['sync_state'] === 'disabled'
|
|
&& $disabled['enabled'] === false
|
|
&& $disabled['applied_revision'] === 3
|
|
&& $disableJob->fresh()->outcome === 'failed',
|
|
'Explicit fence resolution rewrote the immutable task outcome or lost the inspected policy.');
|
|
$completedJobCount = CloudJob::count();
|
|
$completedVersionCount = CloudInstanceFirewallVersion::count();
|
|
$samePolicy = CloudFirewallService::apply($instance, $actor, [
|
|
'enabled' => false, 'expected_revision' => 3,
|
|
'idempotency_key' => 'fw-request-same-policy-0001',
|
|
'rules' => $disabled['rules'],
|
|
]);
|
|
$samePolicyJob = $samePolicy['job'];
|
|
$samePolicyResult = json_decode(
|
|
(string) $samePolicyJob->result_json, true, 512, JSON_THROW_ON_ERROR
|
|
);
|
|
$assert($samePolicyJob instanceof CloudJob
|
|
&& $samePolicy['unchanged'] === true
|
|
&& (string) $samePolicyJob->kind === 'firewall_apply'
|
|
&& (string) $samePolicyJob->execution_mode === 'agent'
|
|
&& (string) $samePolicyJob->phase === 'terminal'
|
|
&& (string) $samePolicyJob->outcome === 'succeeded'
|
|
&& (string) $samePolicyJob->status === 'succeeded'
|
|
&& (int) $samePolicyJob->attempts === 0
|
|
&& (int) $samePolicyJob->dispatch_attempts === 0
|
|
&& $samePolicyJob->dispatch_started_at === null
|
|
&& $samePolicyJob->dispatched_at === null
|
|
&& $samePolicyJob->agent_accepted_at === null
|
|
&& $samePolicyJob->receipt_deadline_at === null
|
|
&& (int) $samePolicyJob->effect_may_have_applied === 0
|
|
&& (int) $samePolicyJob->safe_to_resubmit === 0
|
|
&& (string) $samePolicyJob->effect_resolution === 'applied'
|
|
&& $samePolicyResult === [
|
|
'unchanged' => true,
|
|
'policy_digest' => (string) CloudInstanceFirewall::find(
|
|
(int) $instance->id
|
|
)->applied_digest,
|
|
'revision' => 3,
|
|
]
|
|
&& CloudSecrets::decrypt(
|
|
(string) $samePolicyJob->sealed_payload, 'job-payload'
|
|
) === []
|
|
&& CloudJob::count() === $completedJobCount + 1
|
|
&& CloudInstanceFirewallVersion::count() === $completedVersionCount,
|
|
'An already-converged mutation did not receive an undispatched terminal task.');
|
|
|
|
$samePolicyReplay = CloudFirewallService::apply($instance, $actor, [
|
|
'enabled' => false, 'expected_revision' => 3,
|
|
'idempotency_key' => 'fw-request-same-policy-0001',
|
|
'rules' => $disabled['rules'],
|
|
]);
|
|
$assert((int) $samePolicyReplay['job']->id === (int) $samePolicyJob->id
|
|
&& $samePolicyReplay['unchanged'] === true
|
|
&& CloudJob::count() === $completedJobCount + 1
|
|
&& CloudInstanceFirewallVersion::count() === $completedVersionCount,
|
|
'Replaying an already-converged mutation created a second task or policy version.');
|
|
|
|
$samePolicyHistory = [];
|
|
$samePolicyStack = HandlerStack::create(new MockHandler([]));
|
|
$samePolicyStack->push(Middleware::history($samePolicyHistory));
|
|
$samePolicyAgent = new CloudAgentClient(
|
|
'1',
|
|
'test-secret-which-is-at-least-32-bytes',
|
|
new Client(['handler' => $samePolicyStack]),
|
|
20.0,
|
|
'http://127.0.0.1:8001/internal/agent/request',
|
|
'test-gateway-secret-at-least-32-bytes'
|
|
);
|
|
$assert((new CloudWorkerService($samePolicyAgent))->tick() === false
|
|
&& $samePolicyHistory === []
|
|
&& (string) $samePolicyJob->fresh()->phase === 'terminal'
|
|
&& (int) $samePolicyJob->fresh()->dispatch_attempts === 0,
|
|
'An already-converged firewall task entered the worker or reached Agent.');
|
|
|
|
$expect(static fn () => CloudFirewallService::listing($instance, [
|
|
'user_id' => 99, 'is_admin' => true, 'is_super_admin' => true,
|
|
'can_manage' => true,
|
|
]), 404, 'A non-owner received the tenant firewall policy.');
|
|
$iso = $newInstance(20, 'iso');
|
|
$expect(static fn () => CloudFirewallService::listing($iso, [
|
|
'user_id' => 20, 'is_admin' => false,
|
|
]), 409, 'An ISO instance received unsupported firewall controls.');
|
|
foreach ([
|
|
'fe80::1%eth0/64', '2001:db8::/129',
|
|
'2001:db8::/64/1', '2001:db8:::1/64',
|
|
] as $index => $invalidCidr) {
|
|
$expect(static fn () => CloudFirewallService::apply(
|
|
$instance,
|
|
$actor,
|
|
[
|
|
'enabled' => true, 'expected_revision' => 3,
|
|
'idempotency_key' => 'fw-request-invalid-' . $index,
|
|
'rules' => [[
|
|
'protocol' => 'icmp', 'source_cidr' => $invalidCidr,
|
|
'port_from' => null, 'port_to' => null,
|
|
'description' => '',
|
|
]],
|
|
]
|
|
), 422, 'An invalid IPv6 CIDR crossed the PHP contract.');
|
|
}
|
|
$expect(static fn () => CloudFirewallService::apply($instance, $actor, [
|
|
'enabled' => true, 'expected_revision' => 3,
|
|
'idempotency_key' => 'fw-request-duplicate-v6',
|
|
'rules' => [[
|
|
'protocol' => 'icmp', 'source_cidr' => '2001:db8::1/64',
|
|
'port_from' => null, 'port_to' => null, 'description' => '',
|
|
], [
|
|
'protocol' => 'icmp', 'source_cidr' => '2001:db8::2/64',
|
|
'port_from' => null, 'port_to' => null, 'description' => '',
|
|
]],
|
|
]), 422, 'Two IPv6 hosts in one canonical prefix were not deduplicated.');
|
|
|
|
$cidr = new \ReflectionMethod(CloudFirewallService::class, 'cidr');
|
|
$canonicalVectors = [
|
|
'::192.0.2.1/128' => '::c000:201/128',
|
|
'::1:0/112' => '::1:0/112',
|
|
'::FFFF:192.0.2.129/120' => '::ffff:192.0.2.0/120',
|
|
'::ffff:192.0.2.129/95' => '::fffe:0:0/95',
|
|
'64:ff9b::192.0.2.1/96' => '64:ff9b::/96',
|
|
'ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff/0' => '::/0',
|
|
'ffff::/1' => '8000::/1',
|
|
'2001:db8:abcd:ef01:ffff:ffff:ffff:ffff/73'
|
|
=> '2001:db8:abcd:ef01:ff80::/73',
|
|
'2001:db8::3/127' => '2001:db8::2/127',
|
|
'2001:0DB8::1/128' => '2001:db8::1/128',
|
|
'ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff/128'
|
|
=> 'ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff/128',
|
|
];
|
|
foreach ($canonicalVectors as $input => $expected) {
|
|
$actual = $cidr->invoke(null, $input);
|
|
$assert($actual === $expected,
|
|
"CIDR {$input} normalized to {$actual}, expected {$expected}.");
|
|
}
|
|
$assert(strlen(end($canonicalVectors)) === 43,
|
|
'The maximum canonical IPv6 CIDR storage boundary is not 43 bytes.');
|
|
|
|
// Network-only enforcement can converge around firewall work. Destructive
|
|
// graceful purge and disk deletion retain the firewall row as a blocker so
|
|
// they cannot race the same domain/filter resources.
|
|
$overlap = CloudFirewallService::apply($instance, $actor, [
|
|
'enabled' => true, 'expected_revision' => 3,
|
|
'idempotency_key' => 'fw-request-00000006',
|
|
'rules' => [[
|
|
'protocol' => 'tcp', 'source_cidr' => '192.0.2.44/32',
|
|
'port_from' => 22, 'port_to' => 22,
|
|
'description' => 'Deletion overlap',
|
|
]],
|
|
]);
|
|
$overlapJob = $overlap['job'];
|
|
$overlapPayload = CloudSecrets::decrypt(
|
|
(string) $overlapJob->sealed_payload,
|
|
'job-payload'
|
|
);
|
|
$lifecycleConflict = new ReflectionMethod(
|
|
CloudLifecycleService::class, 'hasConflictingJob'
|
|
);
|
|
$lifecycle = new CloudLifecycleService();
|
|
foreach (['accepted', 'awaiting_receipt', 'failed_unknown']
|
|
as $firewallJobState) {
|
|
$setTestState($overlapJob, $firewallJobState);
|
|
foreach ([
|
|
['active', 'blocking'],
|
|
['active', 'blocking_expired'],
|
|
['resume_pending', 'restore_pending'],
|
|
] as [$serviceState, $networkState]) {
|
|
$instance->service_status = $serviceState;
|
|
$instance->network_status = $networkState;
|
|
$instance->save();
|
|
$assert($lifecycleConflict->invoke($lifecycle, $instance) === true,
|
|
'Active or unknown-effect firewall work did not serialize lifecycle mutation.');
|
|
}
|
|
$instance->service_status = 'purge_stopping';
|
|
$instance->network_status = 'active';
|
|
$instance->save();
|
|
$assert($lifecycleConflict->invoke($lifecycle, $instance) === true,
|
|
'A firewall operation did not block destructive graceful purge.');
|
|
}
|
|
$setTestState($overlapJob, 'failed_unknown');
|
|
$expect(static fn () => CloudJobService::enqueue(
|
|
$instance, $actor, 'delete', 'DELETE', '',
|
|
['remove_managed_disks' => true]
|
|
), 409, 'An unresolved firewall operation did not block an owner-requested delete.');
|
|
$overlapJob->effect_resolution = 'not_applied';
|
|
$overlapJob->effect_resolved_at = time();
|
|
$overlapJob->effect_resolved_by = 99;
|
|
$overlapJob->effect_resolution_reason =
|
|
'Read-only host inspection confirmed policy was not changed.';
|
|
$overlapJob->save();
|
|
$manualDelete = CloudJobService::enqueue(
|
|
$instance, $actor, 'delete', 'DELETE', '',
|
|
['remove_managed_disks' => true]
|
|
);
|
|
$manualDeletePayload = CloudSecrets::decrypt(
|
|
(string) $manualDelete->sealed_payload,
|
|
'job-payload'
|
|
);
|
|
$assert((string) $manualDelete->kind === 'delete'
|
|
&& (string) $manualDelete->status === 'queued'
|
|
&& ($manualDeletePayload['request_target'] ?? null)
|
|
=== '/v1/vms/' . $instance->uuid
|
|
&& ($manualDeletePayload['body'] ?? null)
|
|
=== ['remove_managed_disks' => true],
|
|
'A resolved firewall operation still blocked an owner-requested delete.');
|
|
$terminalize($manualDelete, 'failed', false, true);
|
|
$instance->status = 'deleting';
|
|
$instance->service_status = 'purge_deleting';
|
|
$instance->network_status = 'blocked_expired';
|
|
$instance->save();
|
|
$overlapJob->effect_resolution = 'unknown';
|
|
$overlapJob->save();
|
|
$expect(static fn () => CloudJobService::enqueueExpiryDelete($instance),
|
|
409, 'An unresolved firewall operation did not block irreversible expiry cleanup.');
|
|
$overlapJob->effect_resolution = 'not_applied';
|
|
$overlapJob->effect_resolved_at = time();
|
|
$overlapJob->effect_resolved_by = 99;
|
|
$overlapJob->save();
|
|
$expiryDelete = CloudJobService::enqueueExpiryDelete($instance);
|
|
$assert((string) $expiryDelete->kind === 'expiry_delete'
|
|
&& (string) $expiryDelete->status === 'queued',
|
|
'A resolved firewall operation still blocked irreversible expiry cleanup.');
|
|
CloudFirewallService::finishInstanceDeleteLocked(
|
|
$expiryDelete, 'succeeded'
|
|
);
|
|
$instance->status = 'deleted';
|
|
$instance->save();
|
|
$terminalize($expiryDelete, 'succeeded');
|
|
$assert(CloudFirewallService::assertJobRunnable(
|
|
$overlapJob, $overlapPayload
|
|
) === false,
|
|
'A firewall job remained runnable after confirmed instance deletion.');
|
|
$overlapJob->refresh();
|
|
$deletedPolicy = CloudInstanceFirewall::find((int) $instance->id);
|
|
$assert((string) $overlapJob->phase === 'terminal'
|
|
&& (string) $overlapJob->outcome === 'failed'
|
|
&& (string) $deletedPolicy->sync_state === 'deleted'
|
|
&& (string) $deletedPolicy->last_operation_id
|
|
=== (string) $expiryDelete->operation_id
|
|
&& (string) CloudInstanceFirewallVersion::where('revision', 4)
|
|
->value('state') !== 'applied',
|
|
'A terminal firewall failure revived or obscured a deleted policy.');
|
|
|
|
// Once a mutation crossed the durable dispatch boundary, its absolute
|
|
// receipt deadline closes it locally. The fallback never polls Agent and
|
|
// never replays the original firewall mutation.
|
|
$missingInstance = $newInstance(30);
|
|
$missingActor = ['user_id' => 30, 'is_admin' => false,
|
|
'is_super_admin' => false, 'can_manage' => false];
|
|
$missingApply = CloudFirewallService::apply(
|
|
$missingInstance, $missingActor, [
|
|
'enabled' => true, 'expected_revision' => 0,
|
|
'idempotency_key' => 'fw-request-missing-0001',
|
|
'rules' => [[
|
|
'protocol' => 'tcp', 'source_cidr' => '192.0.2.30/32',
|
|
'port_from' => 22, 'port_to' => 22,
|
|
'description' => 'Missing operation',
|
|
]],
|
|
]
|
|
);
|
|
$missingJob = $missingApply['job'];
|
|
$markDispatched($missingJob);
|
|
$missingJob->receipt_deadline_at = time() - 1;
|
|
$missingJob->lease_token = null;
|
|
$missingJob->lease_until = 0;
|
|
$missingJob->save();
|
|
$missingHistory = [];
|
|
$missingStack = HandlerStack::create(new MockHandler([]));
|
|
$missingStack->push(Middleware::history($missingHistory));
|
|
$assert(CloudWorkerService::expireDueTasksForOwner(
|
|
30, (int) $missingJob->id, 1
|
|
) === 1, 'The firewall receipt deadline was not claimed locally.');
|
|
$missingJob->refresh();
|
|
$missingPolicy = CloudFirewallService::listing(
|
|
$missingInstance, $missingActor
|
|
);
|
|
$assert($missingHistory === []
|
|
&& (string) $missingJob->phase === 'terminal'
|
|
&& (string) $missingJob->outcome === 'failed'
|
|
&& (string) $missingJob->effect_resolution === 'unknown'
|
|
&& (int) $missingJob->dispatch_attempts === 1
|
|
&& $missingPolicy['sync_state'] === 'uncertain'
|
|
&& $missingPolicy['can_mutate'] === false,
|
|
'A firewall deadline contacted Agent, replayed the mutation, or remained active.');
|
|
|
|
// A strict Agent validation response carrying the exact not-dispatched
|
|
// marker is terminal for firewall work as well. The panel-generated
|
|
// failure remains bound to the original operation so the firewall
|
|
// projection does not reject it and strand the job as uncertain.
|
|
$validationInstance = $newInstance(40);
|
|
$validationActor = ['user_id' => 40, 'is_admin' => false,
|
|
'is_super_admin' => false, 'can_manage' => false];
|
|
$validationApply = CloudFirewallService::apply(
|
|
$validationInstance, $validationActor, [
|
|
'enabled' => true, 'expected_revision' => 0,
|
|
'idempotency_key' => 'fw-request-validation-0001',
|
|
'rules' => [[
|
|
'protocol' => 'tcp', 'source_cidr' => '192.0.2.40/32',
|
|
'port_from' => 22, 'port_to' => 22,
|
|
'description' => 'Strict validation failure',
|
|
]],
|
|
]
|
|
);
|
|
$validationJob = $validationApply['job'];
|
|
$validationHistory = [];
|
|
$validationStack = HandlerStack::create(new MockHandler([
|
|
new Response(422, [], json_encode(['error' => [
|
|
'code' => 'invalid_json',
|
|
'message' => 'request contains an unknown field',
|
|
'retryable' => false,
|
|
'dispatch_state' => 'not_dispatched',
|
|
]], JSON_THROW_ON_ERROR)),
|
|
]));
|
|
$validationStack->push(Middleware::history($validationHistory));
|
|
$validationAgent = new CloudAgentClient(
|
|
'1',
|
|
'test-secret-which-is-at-least-32-bytes',
|
|
new Client(['handler' => $validationStack]),
|
|
20.0,
|
|
'http://127.0.0.1:8001/internal/agent/request',
|
|
'test-gateway-secret-at-least-32-bytes'
|
|
);
|
|
(new CloudWorkerService($validationAgent))->tick();
|
|
$validationJob->refresh();
|
|
$validationPolicy = CloudFirewallService::listing(
|
|
$validationInstance, $validationActor
|
|
);
|
|
$assert(count($validationHistory) === 1
|
|
&& (string) $validationJob->status === 'failed'
|
|
&& (string) $validationJob->error_code === 'firewall_apply_failed'
|
|
&& $validationPolicy['sync_state'] === 'failed'
|
|
&& $validationPolicy['can_mutate'] === true
|
|
&& (string) CloudInstanceFirewallVersion::where(
|
|
'instance_id', (int) $validationInstance->id
|
|
)->value('state') === 'failed',
|
|
'A proven pre-dispatch firewall rejection was stranded as uncertain.');
|
|
|
|
$gatewayApply = CloudFirewallService::apply(
|
|
$validationInstance, $validationActor, [
|
|
'enabled' => false, 'expected_revision' => 1,
|
|
'idempotency_key' => 'fw-request-gateway-auth-0001',
|
|
'rules' => [],
|
|
]
|
|
);
|
|
$gatewayJob = $gatewayApply['job'];
|
|
$gatewayHistory = [];
|
|
$gatewayStack = HandlerStack::create(new MockHandler([
|
|
new Response(403, [], json_encode(['error' => [
|
|
'code' => 'gateway_forbidden',
|
|
'message' => 'Agent gateway request is forbidden',
|
|
'retryable' => false,
|
|
'dispatch_state' => 'not_dispatched',
|
|
]], JSON_THROW_ON_ERROR)),
|
|
]));
|
|
$gatewayStack->push(Middleware::history($gatewayHistory));
|
|
$gatewayAgent = new CloudAgentClient(
|
|
'1',
|
|
'test-secret-which-is-at-least-32-bytes',
|
|
new Client(['handler' => $gatewayStack]),
|
|
20.0,
|
|
'http://127.0.0.1:8001/internal/agent/request',
|
|
'test-gateway-secret-at-least-32-bytes'
|
|
);
|
|
(new CloudWorkerService($gatewayAgent))->tick();
|
|
$gatewayJob->refresh();
|
|
$gatewayPolicy = CloudFirewallService::listing(
|
|
$validationInstance, $validationActor
|
|
);
|
|
$assert(count($gatewayHistory) === 1
|
|
&& (string) $gatewayJob->status === 'failed'
|
|
&& $gatewayPolicy['sync_state'] === 'failed'
|
|
&& $gatewayPolicy['can_mutate'] === true
|
|
&& (string) CloudInstanceFirewallVersion::where(
|
|
'instance_id', (int) $validationInstance->id
|
|
)->where('revision', 2)->value('state') === 'failed',
|
|
'A pre-dispatch gateway rejection stranded firewall work as uncertain.');
|
|
|
|
$migrationInstance = $newInstance(50);
|
|
$migrationActor = ['user_id' => 50, 'is_admin' => false,
|
|
'is_super_admin' => false, 'can_manage' => false];
|
|
$migrationApply = CloudFirewallService::apply(
|
|
$migrationInstance, $migrationActor, [
|
|
'enabled' => true, 'expected_revision' => 0,
|
|
'idempotency_key' => 'fw-request-migration-source-0001',
|
|
'rules' => [[
|
|
'protocol' => 'tcp', 'source_cidr' => '192.0.2.50/32',
|
|
'port_from' => 22, 'port_to' => 22,
|
|
'description' => 'Preserved during migration',
|
|
]],
|
|
]
|
|
);
|
|
$migrationSourceJob = $migrationApply['job'];
|
|
$migrationSourcePayload = CloudSecrets::decrypt(
|
|
(string) $migrationSourceJob->sealed_payload,
|
|
'job-payload'
|
|
);
|
|
CloudFirewallService::finishJobLocked(
|
|
$migrationSourceJob,
|
|
'succeeded',
|
|
[
|
|
'revision' => 1,
|
|
'policy_digest' =>
|
|
(string) $migrationSourcePayload['body']['policy_digest'],
|
|
],
|
|
$migrationSourcePayload,
|
|
['operation_id' => (string) $migrationSourceJob->operation_id]
|
|
);
|
|
CloudFirewallService::assertMaintenanceMigrationReady($migrationInstance);
|
|
$migrationInstance->host_id = 8;
|
|
$migrationInstance->save();
|
|
$migration = new CloudInstanceMigration();
|
|
$migration->id = 99;
|
|
$migration->migration_key = str_repeat('a', 32);
|
|
$migration->source_host_id = 7;
|
|
$migration->target_host_id = 8;
|
|
$firewallJob = CloudFirewallService::
|
|
queueMaintenanceMigrationReapplyLocked(
|
|
$migrationInstance,
|
|
$migration,
|
|
99
|
|
);
|
|
$firewallPayload = CloudSecrets::decrypt(
|
|
(string) $firewallJob->sealed_payload,
|
|
'job-payload'
|
|
);
|
|
$migratedPolicy = CloudInstanceFirewall::findOrFail(
|
|
(int) $migrationInstance->id
|
|
);
|
|
$assert($firewallJob !== null
|
|
&& (int) $firewallJob->host_id === 8
|
|
&& (string) $firewallJob->kind === 'firewall_apply'
|
|
&& ($firewallPayload['maintenance_migration_id'] ?? null) === 99
|
|
&& ($firewallPayload['request_target'] ?? null)
|
|
=== '/v1/vms/' . $migrationInstance->uuid . '/firewall'
|
|
&& count($firewallPayload['body']['rules'] ?? []) === 1
|
|
&& (int) $migratedPolicy->host_id === 8
|
|
&& (int) $migratedPolicy->desired_revision === 2
|
|
&& (int) $migratedPolicy->applied_revision === 1
|
|
&& (string) $migratedPolicy->sync_state === 'pending',
|
|
'Maintenance migration did not preserve and requeue the settled user firewall on the target host.');
|
|
|
|
echo "Cloud firewall desired/applied, idempotency and queue workflow passed.\n";
|
|
}
|