You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
virt/web/tests/CloudVpcHostRetirementServi...

587 lines
24 KiB
PHP

<?php
namespace App\Services\Cloud {
final class CloudSecrets
{
public static function encrypt(array $value, string $purpose): string
{
return 'retirement-test:' . hash('sha256', json_encode(
$value,
JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES
));
}
}
}
namespace {
require dirname(__DIR__) . '/vendor/autoload.php';
use App\Models\CloudJob;
use App\Services\Cloud\CloudError;
use App\Services\Cloud\CloudVpcHostRetirementService;
use App\Services\Cloud\CloudVpcPortService;
use Illuminate\Database\Capsule\Manager as Capsule;
use Illuminate\Database\Schema\Blueprint;
$assert = static function (bool $condition, string $message): void {
if (!$condition) {
throw new RuntimeException($message);
}
};
$reject = static function (callable $callback, string $message) use (
$assert
): void {
try {
$callback();
} catch (CloudError $error) {
$assert($error->httpStatus === 409,
$message . ': HTTP ' . $error->httpStatus);
return;
}
throw new RuntimeException($message . ': request was accepted');
};
$capsule = new Capsule();
$capsule->addConnection([
'driver' => 'sqlite', 'database' => ':memory:',
], 'default');
$capsule->setAsGlobal();
$capsule->bootEloquent();
$schema = $capsule->schema('default');
$schema->create('cloud_hosts', static function (Blueprint $table): void {
$table->increments('id');
$table->string('name');
$table->string('region')->default('');
$table->boolean('enabled')->default(false);
$table->integer('created_at');
$table->integer('updated_at');
});
$schema->create('cloud_host_agents', static function (Blueprint $table): void {
$table->integer('host_id')->primary();
$table->boolean('enabled')->default(false);
});
$schema->create('cloud_host_resource_pool_memberships', static function (
Blueprint $table
): void {
$table->increments('id');
$table->integer('host_id');
$table->integer('resource_pool_id');
$table->string('scheduler_state');
$table->integer('priority')->default(0);
$table->integer('weight')->default(100);
$table->integer('revision')->default(1);
$table->integer('created_at');
$table->integer('updated_at');
});
$schema->create('cloud_instances', static function (Blueprint $table): void {
$table->increments('id');
$table->string('uuid');
$table->integer('host_id');
$table->integer('user_id');
$table->string('name');
$table->string('status');
$table->string('service_status');
$table->string('network_status');
$table->text('spec_json');
$table->integer('created_at');
$table->integer('updated_at');
});
$schema->create('cloud_resource_reservations', static function (
Blueprint $table
): void {
$table->increments('id');
$table->integer('host_id');
$table->string('state');
});
$schema->create('cloud_instance_migrations', static function (
Blueprint $table
): void {
$table->increments('id');
$table->integer('instance_id');
$table->integer('source_host_id');
$table->integer('target_host_id');
$table->string('state');
});
$schema->create('cloud_vpcs', static function (Blueprint $table): void {
$table->increments('id');
$table->string('vpc_key');
$table->integer('user_id');
$table->integer('region_id');
$table->integer('cidr_pool_id');
$table->string('name');
$table->string('cidr');
$table->integer('prefix_length');
$table->string('gateway');
$table->integer('active_claim')->nullable();
$table->string('state');
$table->integer('desired_revision');
$table->integer('deleted_at')->nullable();
$table->integer('created_at');
$table->integer('updated_at');
});
$schema->create('cloud_vpc_ports', static function (Blueprint $table): void {
$table->increments('id');
$table->string('port_key');
$table->integer('vpc_id');
$table->integer('instance_id');
$table->integer('migration_host_id')->nullable();
$table->integer('user_id');
$table->string('private_ip');
$table->string('mac_address');
$table->integer('active_claim')->nullable();
$table->string('state');
$table->integer('desired_revision');
$table->integer('applied_revision');
$table->string('last_operation_id')->nullable();
$table->string('last_error_code')->default('');
$table->integer('attached_at')->nullable();
$table->integer('released_at')->nullable();
$table->integer('created_at');
$table->integer('updated_at');
});
$schema->create('cloud_vpc_host_states', static function (
Blueprint $table
): void {
$table->increments('id');
$table->integer('vpc_id');
$table->integer('host_id');
$table->integer('desired_revision');
$table->integer('applied_revision');
$table->string('desired_digest');
$table->string('applied_digest')->nullable();
$table->string('sync_state');
$table->string('last_operation_id')->nullable();
$table->string('last_error_code')->default('');
$table->integer('last_success_at')->default(0);
$table->integer('last_failure_at')->default(0);
$table->integer('created_at');
$table->integer('updated_at');
});
$schema->create('cloud_vpc_membership_batches', static function (
Blueprint $table
): void {
$table->increments('id');
$table->string('batch_key');
$table->integer('vpc_id');
$table->integer('user_id');
$table->integer('parent_job_id');
$table->string('kind');
$table->string('state');
$table->integer('desired_revision');
$table->integer('active_claim')->nullable();
$table->string('last_error_code')->default('');
$table->integer('terminal_at')->nullable();
$table->integer('created_at');
$table->integer('updated_at');
});
$schema->create('cloud_vpc_membership_batch_items', static function (
Blueprint $table
): void {
$table->increments('id');
$table->integer('batch_id');
$table->integer('user_id');
$table->integer('instance_id');
$table->integer('vpc_port_id');
$table->integer('host_id');
$table->integer('child_job_id')->nullable();
$table->string('state');
$table->string('last_error_code')->default('');
$table->integer('created_at');
$table->integer('updated_at');
});
$schema->create('cloud_jobs', static function (Blueprint $table): void {
$table->increments('id');
$table->string('operation_id');
$table->string('request_idempotency_key');
$table->string('request_fingerprint');
$table->integer('parent_job_id')->nullable();
$table->integer('supersedes_job_id')->nullable();
$table->integer('vpc_repair_root_job_id')->nullable();
$table->integer('order_id')->nullable();
$table->integer('invoice_id')->nullable();
$table->integer('instance_id')->nullable();
$table->integer('snapshot_id')->nullable();
$table->integer('backup_id')->nullable();
$table->integer('backup_restore_id')->nullable();
$table->integer('backup_recovery_id')->nullable();
$table->integer('vpc_id')->nullable();
$table->integer('vpc_port_id')->nullable();
$table->integer('vpc_batch_id')->nullable();
$table->integer('user_id');
$table->integer('host_id')->nullable();
$table->string('kind');
$table->string('execution_mode');
$table->string('phase');
$table->string('outcome')->nullable();
$table->string('status');
$table->text('sealed_payload');
$table->integer('attempts');
$table->integer('dispatch_attempts');
$table->integer('dispatch_deadline_at');
$table->integer('dispatch_started_at')->nullable();
$table->integer('agent_accepted_at')->nullable();
$table->integer('receipt_deadline_at')->nullable();
$table->integer('terminal_at')->nullable();
$table->boolean('effect_may_have_applied')->nullable();
$table->boolean('safe_to_resubmit')->nullable();
$table->string('effect_resolution');
$table->integer('effect_resolved_at')->nullable();
$table->integer('effect_resolved_by')->nullable();
$table->text('effect_resolution_reason')->default('');
$table->boolean('finalization_incident')->default(false);
$table->string('finalization_error_code')->nullable();
$table->string('completion_event_id')->nullable();
$table->integer('read_at')->nullable();
$table->integer('dispatched_at')->nullable();
$table->string('lease_token')->nullable();
$table->integer('lease_until')->default(0);
$table->integer('available_at');
$table->string('error_code')->nullable();
$table->text('result_json')->nullable();
$table->integer('created_at');
$table->integer('updated_at');
});
$now = time();
$actor = [
'user_id' => 99, 'is_admin' => true,
'is_super_admin' => true, 'can_manage' => true,
];
Capsule::table('cloud_hosts')->insert([
['id' => 1, 'name' => 'retiring-host', 'region' => 'hk',
'enabled' => 1, 'created_at' => $now, 'updated_at' => $now],
['id' => 2, 'name' => 'surviving-host', 'region' => 'hk',
'enabled' => 1, 'created_at' => $now, 'updated_at' => $now],
]);
Capsule::table('cloud_host_agents')->insert([
'host_id' => 1, 'enabled' => 1,
]);
Capsule::table('cloud_host_resource_pool_memberships')->insert([
'id' => 1, 'host_id' => 1, 'resource_pool_id' => 1,
'scheduler_state' => 'active', 'priority' => 0, 'weight' => 100,
'created_at' => $now, 'updated_at' => $now,
]);
$instanceRows = [
['id' => 10, 'uuid' => '11111111-1111-4111-8111-111111111110',
'host_id' => 1, 'user_id' => 7, 'name' => 'retired-reference',
'status' => 'deleted'],
['id' => 11, 'uuid' => '11111111-1111-4111-8111-111111111111',
'host_id' => 1, 'user_id' => 7, 'name' => 'host-blocker',
'status' => 'active'],
];
foreach ($instanceRows as &$row) {
$row += ['service_status' => 'active', 'network_status' => 'active',
'spec_json' => '{}', 'created_at' => $now, 'updated_at' => $now];
}
unset($row);
Capsule::table('cloud_instances')->insert($instanceRows);
Capsule::table('cloud_resource_reservations')->insert([
'id' => 1, 'host_id' => 1, 'state' => 'held',
]);
Capsule::table('cloud_instance_migrations')->insert([
'id' => 1, 'instance_id' => 11, 'source_host_id' => 1,
'target_host_id' => 2, 'state' => 'preparing_source',
]);
Capsule::table('cloud_vpcs')->insert([
'id' => 20, 'vpc_key' => str_repeat('a', 32), 'user_id' => 7,
'region_id' => 1, 'cidr_pool_id' => 1, 'name' => 'retire-test',
'cidr' => '10.10.0.0/24', 'prefix_length' => 24,
'gateway' => '10.10.0.1', 'active_claim' => 1,
'state' => 'provisioning', 'desired_revision' => 3,
'deleted_at' => null, 'created_at' => $now, 'updated_at' => $now,
]);
Capsule::table('cloud_vpc_ports')->insert([
[
'id' => 30, 'port_key' => str_repeat('b', 32), 'vpc_id' => 20,
'instance_id' => 10, 'migration_host_id' => null, 'user_id' => 7,
'private_ip' => '10.10.0.2', 'mac_address' => '02:00:00:00:00:02',
'active_claim' => 1, 'state' => 'release_pending',
'desired_revision' => 3, 'applied_revision' => 2,
'last_operation_id' => null, 'last_error_code' => '',
'attached_at' => $now - 100, 'released_at' => null,
'created_at' => $now, 'updated_at' => $now,
],
[
'id' => 31, 'port_key' => str_repeat('c', 32), 'vpc_id' => 20,
'instance_id' => 11, 'migration_host_id' => null, 'user_id' => 7,
'private_ip' => '10.10.0.3', 'mac_address' => '02:00:00:00:00:03',
'active_claim' => 1, 'state' => 'attached',
'desired_revision' => 2, 'applied_revision' => 2,
'last_operation_id' => null, 'last_error_code' => '',
'attached_at' => $now - 100, 'released_at' => null,
'created_at' => $now, 'updated_at' => $now,
],
]);
$rootOperation = str_repeat('d', 32);
$tipOperation = str_repeat('e', 32);
Capsule::table('cloud_vpc_host_states')->insert([
'id' => 40, 'vpc_id' => 20, 'host_id' => 1,
'desired_revision' => 3, 'applied_revision' => 2,
'desired_digest' => str_repeat('f', 64),
'applied_digest' => str_repeat('1', 64), 'sync_state' => 'unknown',
'last_operation_id' => $tipOperation,
'last_error_code' => 'operation_effect_unknown',
'last_success_at' => $now - 100, 'last_failure_at' => $now - 10,
'created_at' => $now, 'updated_at' => $now,
]);
$job = static function (
int $id,
string $operationId,
?int $supersedes,
?int $root
) use ($now): array {
return [
'id' => $id, 'operation_id' => $operationId,
'request_idempotency_key' => $operationId,
'request_fingerprint' => hash('sha256', $operationId),
'parent_job_id' => null, 'supersedes_job_id' => $supersedes,
'vpc_repair_root_job_id' => $root, 'order_id' => null,
'invoice_id' => null, 'instance_id' => 10, 'snapshot_id' => null,
'backup_id' => null, 'backup_restore_id' => null,
'backup_recovery_id' => null, 'vpc_id' => 20,
'vpc_port_id' => 30, 'vpc_batch_id' => null,
'user_id' => 7, 'host_id' => 1, 'kind' => 'vpc_host_remove',
'execution_mode' => 'agent', 'phase' => 'terminal',
'outcome' => 'failed', 'status' => 'failed', 'sealed_payload' => '{}',
'attempts' => 1, 'dispatch_attempts' => 1,
'dispatch_deadline_at' => $now - 200,
'dispatch_started_at' => $now - 190,
'agent_accepted_at' => $now - 185,
'receipt_deadline_at' => $now - 100, 'terminal_at' => $now - 20,
'effect_may_have_applied' => 1, 'safe_to_resubmit' => 0,
'effect_resolution' => 'unknown', 'effect_resolved_at' => null,
'effect_resolved_by' => null, 'effect_resolution_reason' => '',
'finalization_incident' => 0, 'finalization_error_code' => null,
'completion_event_id' => null, 'read_at' => null,
'dispatched_at' => $now - 185, 'lease_token' => null,
'lease_until' => 0, 'available_at' => 0,
'error_code' => 'operation_effect_unknown', 'result_json' => null,
'created_at' => $now - 200, 'updated_at' => $now - 20,
];
};
Capsule::table('cloud_jobs')->insert([
$job(100, $rootOperation, null, null),
$job(101, $tipOperation, 100, 100),
array_replace(
$job(102, str_repeat('9', 32), 101, 100),
[
'effect_may_have_applied' => 0,
'safe_to_resubmit' => 1,
'effect_resolution' => 'not_applied',
'error_code' => 'agent_rejected',
]
),
]);
$retire = static fn (): array => CloudVpcHostRetirementService::retire(
$actor,
101,
'The physical node was decommissioned after all tenant data was removed.'
);
$reject($retire, 'enabled host must block permanent retirement');
Capsule::table('cloud_hosts')->where('id', 1)->update(['enabled' => 0]);
$reject($retire, 'enabled Agent must block permanent retirement');
Capsule::table('cloud_host_agents')->where('host_id', 1)
->update(['enabled' => 0]);
$reject($retire, 'active scheduler membership must block retirement');
Capsule::table('cloud_host_resource_pool_memberships')->where('id', 1)
->update(['scheduler_state' => 'disabled']);
$reject($retire, 'nondeleted instance must block permanent retirement');
Capsule::table('cloud_instances')->where('id', 11)
->update(['status' => 'deleted']);
$reject($retire, 'active resource reservation must block retirement');
Capsule::table('cloud_resource_reservations')->where('id', 1)
->update(['state' => 'released']);
$reject($retire, 'active maintenance migration must block retirement');
Capsule::table('cloud_instance_migrations')->where('id', 1)
->update(['state' => 'completed']);
$reject($retire, 'attached VPC port must block permanent retirement');
Capsule::table('cloud_vpc_ports')->where('id', 31)->update([
'active_claim' => null, 'state' => 'released', 'released_at' => $now,
]);
Capsule::table('cloud_jobs')->where('id', 102)->update([
'effect_may_have_applied' => 1, 'safe_to_resubmit' => 0,
'effect_resolution' => 'unknown',
]);
$reject($retire, 'parallel unknown repair generation must block retirement');
Capsule::table('cloud_jobs')->where('id', 102)->update([
'effect_may_have_applied' => 0, 'safe_to_resubmit' => 1,
'effect_resolution' => 'not_applied',
]);
Capsule::table('cloud_jobs')->where('id', 101)->update([
'effect_may_have_applied' => 0, 'safe_to_resubmit' => 1,
'effect_resolution' => 'not_applied',
]);
$reject($retire, 'known no-effect job must not be rewritten as applied');
Capsule::table('cloud_jobs')->where('id', 101)->update([
'effect_may_have_applied' => 1, 'safe_to_resubmit' => 0,
'effect_resolution' => 'unknown',
]);
$result = $retire();
$assert($result['retired'] === true && $result['task_id'] === '101'
&& $result['host_id'] === '1' && $result['vpc_id'] === '20'
&& $result['resolved_generations'] === 2,
'retirement result must identify the exact converged chain');
$assert(Capsule::table('cloud_vpc_host_states')->where('id', 40)
->doesntExist(), 'retired host projection must be deleted');
$resolved = CloudJob::query()->whereIn('id', [100, 101])
->orderBy('id')->get();
$assert($resolved->count() === 2, 'repair chain audit rows must be retained');
foreach ($resolved as $generation) {
$assert((string) $generation->outcome === 'failed'
&& (string) $generation->status === 'failed'
&& (string) $generation->effect_resolution === 'applied'
&& (int) $generation->effect_resolved_by === 99
&& str_starts_with(
(string) $generation->effect_resolution_reason,
'host_permanently_retired:'
), 'each failed generation must retain outcome and gain audit proof');
}
$knownNoEffectSibling = CloudJob::find(102);
$assert($knownNoEffectSibling !== null
&& (string) $knownNoEffectSibling->effect_resolution === 'not_applied'
&& $knownNoEffectSibling->effect_resolved_by === null,
'known no-effect repair sibling must retain its original conclusion');
$port = Capsule::table('cloud_vpc_ports')->where('id', 30)->first();
$vpc = Capsule::table('cloud_vpcs')->where('id', 20)->first();
$assert($port !== null && $port->active_claim === null
&& $port->state === 'released' && (int) $port->released_at > 0,
'reference VPC port must finish its pending release');
$assert($vpc !== null && $vpc->state === 'active',
'VPC must converge after its last host projection is retired');
Capsule::table('cloud_instances')->insert([
'id' => 12, 'uuid' => '11111111-1111-4111-8111-111111111112',
'host_id' => 1, 'user_id' => 7, 'name' => 'batch-reference',
'status' => 'deleted', 'service_status' => 'active',
'network_status' => 'active', 'spec_json' => '{}',
'created_at' => $now, 'updated_at' => $now,
]);
Capsule::table('cloud_vpcs')->insert([
'id' => 21, 'vpc_key' => str_repeat('4', 32), 'user_id' => 7,
'region_id' => 1, 'cidr_pool_id' => 1, 'name' => 'batch-retire-test',
'cidr' => '10.20.0.0/24', 'prefix_length' => 24,
'gateway' => '10.20.0.1', 'active_claim' => 1,
'state' => 'degraded', 'desired_revision' => 4,
'deleted_at' => null, 'created_at' => $now, 'updated_at' => $now,
]);
Capsule::table('cloud_vpc_ports')->insert([
'id' => 32, 'port_key' => str_repeat('5', 32), 'vpc_id' => 21,
'instance_id' => 12, 'migration_host_id' => null, 'user_id' => 7,
'private_ip' => '10.20.0.2', 'mac_address' => '02:00:00:00:00:04',
'active_claim' => 1, 'state' => 'release_pending',
'desired_revision' => 4, 'applied_revision' => 3,
'last_operation_id' => null, 'last_error_code' => '',
'attached_at' => $now - 100, 'released_at' => null,
'created_at' => $now, 'updated_at' => $now,
]);
$batchOperation = str_repeat('6', 32);
Capsule::table('cloud_vpc_host_states')->insert([
'id' => 41, 'vpc_id' => 21, 'host_id' => 1,
'desired_revision' => 4, 'applied_revision' => 3,
'desired_digest' => str_repeat('7', 64),
'applied_digest' => str_repeat('8', 64), 'sync_state' => 'unknown',
'last_operation_id' => $batchOperation,
'last_error_code' => 'operation_effect_unknown',
'last_success_at' => $now - 100, 'last_failure_at' => $now - 10,
'created_at' => $now, 'updated_at' => $now,
]);
$batchPrefix = CloudVpcPortService::batchProjectionRequestPrefix(50, 4);
$parent = array_replace($job(200, str_repeat('2', 32), null, null), [
'request_idempotency_key' => str_repeat('2', 32),
'instance_id' => null, 'vpc_id' => 21, 'vpc_port_id' => null,
'vpc_batch_id' => null, 'host_id' => null,
'kind' => 'vpc_membership_batch', 'execution_mode' => 'local',
'phase' => 'accepted', 'outcome' => null, 'status' => 'queued',
'attempts' => 0, 'dispatch_attempts' => 0,
'dispatch_deadline_at' => $now + 3600,
'dispatch_started_at' => null, 'agent_accepted_at' => null,
'receipt_deadline_at' => null, 'terminal_at' => null,
'effect_may_have_applied' => null, 'safe_to_resubmit' => null,
'effect_resolution' => 'not_applicable', 'dispatched_at' => null,
'error_code' => null,
]);
$batchChild = array_replace(
$job(201, $batchOperation, null, null),
[
'request_idempotency_key' => $batchPrefix
. 'vpc_host_remove-1-' . $batchOperation,
'instance_id' => 12, 'vpc_id' => 21, 'vpc_port_id' => 32,
'vpc_batch_id' => 50,
]
);
Capsule::table('cloud_jobs')->insert([$parent, $batchChild]);
Capsule::table('cloud_vpc_membership_batches')->insert([
'id' => 50, 'batch_key' => str_repeat('3', 32), 'vpc_id' => 21,
'user_id' => 7, 'parent_job_id' => 200, 'kind' => 'detach',
'state' => 'blocked', 'desired_revision' => 4, 'active_claim' => 1,
'last_error_code' => 'vpc_result_uncertain', 'terminal_at' => null,
'created_at' => $now, 'updated_at' => $now,
]);
Capsule::table('cloud_vpc_membership_batch_items')->insert([
'id' => 60, 'batch_id' => 50, 'user_id' => 7, 'instance_id' => 12,
'vpc_port_id' => 32, 'host_id' => 1, 'child_job_id' => null,
'state' => 'succeeded', 'last_error_code' => '',
'created_at' => $now, 'updated_at' => $now,
]);
$batchResult = CloudVpcHostRetirementService::retire(
$actor,
201,
'The second physical node projection was retired after the batch was audited.'
);
$retiredBatch = Capsule::table('cloud_vpc_membership_batches')
->where('id', 50)->first();
$retiredParent = CloudJob::find(200);
$retiredBatchPort = Capsule::table('cloud_vpc_ports')
->where('id', 32)->first();
$assert($batchResult['retired'] === true
&& $batchResult['resolved_generations'] === 1,
'blocked batch host retirement must resolve one exact child generation');
$assert($retiredBatch !== null && $retiredBatch->state === 'succeeded'
&& $retiredBatch->active_claim === null
&& (int) $retiredBatch->terminal_at > 0,
'blocked membership batch must reach a terminal success state');
$assert($retiredParent !== null
&& (string) $retiredParent->phase === 'terminal'
&& (string) $retiredParent->outcome === 'succeeded',
'membership batch parent task must converge with the retired child');
$assert($retiredBatchPort !== null
&& $retiredBatchPort->active_claim === null
&& $retiredBatchPort->state === 'released',
'batch release-pending port must be released after host retirement');
$root = dirname(__DIR__);
$routes = file_get_contents($root . '/config/routes_cloud.php');
$controller = file_get_contents(
$root . '/app/Controllers/CloudAdminOperationsController.php'
);
$recovery = file_get_contents(
$root . '/app/Services/Cloud/CloudAdminOperationRecoveryService.php'
);
$script = file_get_contents(
$root . '/public/cloud/admin-operations-recovery.js'
);
$view = file_get_contents(
$root . '/resources/views/venix/admin/cloud/operations.tpl'
);
$assert(is_string($routes) && is_string($controller)
&& is_string($recovery) && is_string($script) && is_string($view)
&& str_contains($routes, '/retire-vpc-host')
&& str_contains($controller, 'function retireVpcHost(')
&& str_contains($recovery, 'function retireVpcHost(')
&& str_contains($script, 'async function retireVpcHost(')
&& str_contains($view, 'operation-detail-retire-vpc-host')
&& str_contains($view, 'operation-vpc-retirement-reason')
&& str_contains($script, 'reason: reason.value')
&& !str_contains($script, 'window.prompt('),
'permanent host-retirement route, controller, UI, or service wiring is incomplete');
echo "CloudVpcHostRetirementServiceTest passed\n";
}