You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
virt/web/tests/CloudAbuseMonitoringSchemaC...

432 lines
19 KiB
PHP

<?php
$root = dirname(__DIR__);
$read = static function (string $path) use ($root): string {
$value = file_get_contents($root . '/' . ltrim($path, '/'));
if (!is_string($value)) {
throw new RuntimeException('Unable to read ' . $path);
}
return $value;
};
$assert = static function (bool $condition, string $message): void {
if (!$condition) {
throw new RuntimeException($message);
}
};
$tableSql = static function (string $schema, string $table) use ($assert): string {
$matched = preg_match(
'/CREATE TABLE ' . preg_quote($table, '/')
. ' \(([\s\S]*?)\n\) ENGINE=InnoDB/',
$schema,
$match
) === 1;
$assert($matched, 'Missing abuse-monitoring table: ' . $table);
return $match[1];
};
$main = $read('sql/cloud.sql');
$log = $read('sql/cloud_log.sql');
$setup = $read('app/Command/CloudSetup.php');
$traffic = $read('app/Services/Cloud/CloudTrafficService.php');
$lifecycle = $read('app/Services/Cloud/CloudLifecycleService.php');
$deletion = $read('app/Services/UserDataDeletionService.php');
$projection = $read('app/Services/Cloud/CloudAbuseProjectionService.php');
$mainTables = [
'cloud_abuse_instance_states' => [
'instance_id', 'user_id', 'host_id', 'rule_version',
'policy_revision', 'policy_hash', 'risk_state',
'collection_state', 'latest_direction', 'latest_rule_code',
'latest_severity', 'summary_json', 'last_projection_key',
'last_bucket_start', 'last_sample_at', 'last_evaluated_at',
'created_at', 'updated_at',
],
'cloud_abuse_findings' => [
'id', 'finding_key', 'instance_id', 'user_id', 'host_id',
'rule_code', 'rule_version', 'policy_revision', 'policy_hash',
'direction', 'ip_family', 'severity',
'state', 'first_seen_at', 'last_seen_at', 'consecutive_windows',
'occurrence_generation', 'transition_revision', 'evidence_json',
'resolved_at', 'resolution_reason',
'created_at', 'updated_at',
],
'cloud_abuse_alert_events' => [
'id', 'event_key', 'sample_key', 'instance_id', 'user_id',
'host_id', 'bucket_start', 'rule_version', 'policy_revision',
'policy_hash', 'transition_digest', 'transition_count',
'highest_severity', 'created_at',
],
'cloud_abuse_alert_event_items' => [
'id', 'event_id', 'item_key', 'finding_key',
'occurrence_generation', 'transition_revision', 'rule_code',
'direction', 'ip_family', 'transition_type', 'from_severity',
'to_severity', 'evidence_json', 'resolution_reason',
'occurred_at', 'created_at',
],
'cloud_abuse_alert_deliveries' => [
'id', 'event_id', 'binding_generation', 'bound_admin_id',
'chat_id', 'telegram_message_id', 'state', 'message_text',
'content_hash', 'error_code', 'skip_reason',
'dispatch_started_at', 'sent_at', 'terminal_at',
'destination_redacted_at', 'created_at', 'updated_at',
],
'administrator_telegram_delivery_gate' => [
'id', 'next_send_at', 'updated_at',
],
'cloud_abuse_projection_receipts' => [
'sample_key', 'instance_id', 'rule_version', 'policy_revision',
'policy_hash', 'bucket_start', 'projected_at',
],
];
$logTables = [
'cloud_vm_network_observation_rollups' => [
'instance_id', 'user_id', 'host_id', 'interval_seconds',
'bucket_start', 'sample_count', 'complete_count', 'quality_mask',
'rx_bytes_min', 'rx_bytes_max', 'tx_bytes_min', 'tx_bytes_max',
'rx_packets_min', 'rx_packets_max', 'tx_packets_min',
'tx_packets_max', 'tcp_active_peak', 'tcp_new_total',
'tcp_syn_total', 'tcp_unreplied_total', 'udp_active_peak',
'udp_new_total', 'peer_ip_cardinality_peak',
'peer_port_cardinality_peak',
'first_sample_at', 'last_sample_at', 'created_at', 'updated_at',
],
'cloud_vm_network_observation_samples' => [
'sample_key', 'instance_id', 'user_id', 'host_id', 'vm_uuid',
'placement_key', 'collector_generation', 'window_id',
'contract_version', 'bucket_start', 'bucket_seconds', 'complete',
'metrics_json',
'quality_flags_json', 'received_at',
],
'cloud_vm_network_projection_outbox' => [
'sample_key', 'instance_id', 'bucket_start', 'placement_key',
'rule_version', 'policy_revision', 'policy_hash', 'state',
'attempts', 'available_at', 'lease_token', 'lease_until',
'last_error_code', 'projected_at', 'terminal_at',
'discard_reason_code', 'created_at', 'updated_at',
],
'cloud_vm_network_monitor_cursors' => [
'host_id', 'collector_generation', 'cursor_value',
'oldest_cursor', 'gap_before', 'poll_revision', 'next_poll_at',
'poll_lease_token', 'poll_lease_until', 'collection_state',
'consecutive_failures', 'last_window_start',
'last_success_at', 'last_error_code', 'updated_at',
],
'cloud_vm_network_baselines' => [
'instance_id', 'user_id', 'rule_version', 'policy_revision',
'policy_hash', 'sample_count',
'warmup_state', 'metrics_json', 'last_projection_key',
'last_bucket_start', 'projection_lease_token',
'projection_lease_until', 'revision', 'created_at', 'updated_at',
],
];
$definitions = [];
foreach ($mainTables as $table => $columns) {
$definitions[$table] = $tableSql($main, $table);
$assert(str_contains($setup, "'" . $table . "'"),
'CloudSetup does not require main abuse table: ' . $table);
foreach ($columns as $column) {
$assert(preg_match('/\b' . preg_quote($column, '/') . '\b/',
$definitions[$table]) === 1,
'Missing main abuse column: ' . $table . '.' . $column);
$assert(str_contains($setup, "'" . $column . "'"),
'CloudSetup does not require main abuse column: '
. $table . '.' . $column);
}
}
foreach ($logTables as $table => $columns) {
$definitions[$table] = $tableSql($log, $table);
$assert(str_contains($setup, "'" . $table . "'"),
'CloudSetup does not require log abuse table: ' . $table);
foreach ($columns as $column) {
$assert(preg_match('/\b' . preg_quote($column, '/') . '\b/',
$definitions[$table]) === 1,
'Missing log abuse column: ' . $table . '.' . $column);
$assert(str_contains($setup, "'" . $column . "'"),
'CloudSetup does not require log abuse column: '
. $table . '.' . $column);
}
}
foreach ([
'cloud_abuse_state_risk', 'cloud_abuse_state_collection',
'cloud_abuse_state_host', 'cloud_abuse_state_owner',
] as $index) {
$assert(str_contains($definitions['cloud_abuse_instance_states'], $index)
&& str_contains($setup, "'" . $index . "'"),
'Missing abuse state index: ' . $index);
}
foreach ([
'cloud_abuse_alert_event_key', 'cloud_abuse_alert_event_sample',
'cloud_abuse_alert_event_instance', 'cloud_abuse_alert_event_owner',
'cloud_abuse_alert_event_host', 'cloud_abuse_alert_event_retention',
] as $index) {
$assert(str_contains($definitions['cloud_abuse_alert_events'], $index)
&& str_contains($setup, "'" . $index . "'"),
'Missing abuse alert event index: ' . $index);
}
foreach ([
'cloud_abuse_alert_item_key', 'cloud_abuse_alert_item_finding_once',
'cloud_abuse_alert_item_event', 'cloud_abuse_alert_item_finding',
] as $index) {
$assert(str_contains(
$definitions['cloud_abuse_alert_event_items'], $index
) && str_contains($setup, "'" . $index . "'"),
'Missing abuse alert item index: ' . $index);
}
foreach ([
'cloud_abuse_alert_delivery_event',
'cloud_abuse_alert_delivery_message',
'cloud_abuse_alert_delivery_due',
'cloud_abuse_alert_delivery_admin',
'cloud_abuse_alert_delivery_terminal',
] as $index) {
$assert(str_contains(
$definitions['cloud_abuse_alert_deliveries'], $index
) && str_contains($setup, "'" . $index . "'"),
'Missing abuse alert delivery index: ' . $index);
}
foreach ([
'cloud_abuse_finding_key', 'cloud_abuse_finding_instance',
'cloud_abuse_finding_owner', 'cloud_abuse_finding_host',
'cloud_abuse_finding_rule', 'cloud_abuse_finding_retention',
] as $index) {
$assert(str_contains($definitions['cloud_abuse_findings'], $index)
&& str_contains($setup, "'" . $index . "'"),
'Missing abuse finding index: ' . $index);
}
foreach ([
'cloud_network_rollup_owner',
'cloud_network_rollup_host',
'cloud_network_rollup_retention',
] as $index) {
$assert(str_contains(
$definitions['cloud_vm_network_observation_rollups'], $index
) && str_contains($setup, "'" . $index . "'"),
'Missing observation rollup index: ' . $index);
}
foreach ([
'cloud_network_observation_source',
'cloud_network_observation_instance',
'cloud_network_observation_retention',
] as $index) {
$assert(str_contains(
$definitions['cloud_vm_network_observation_samples'], $index
) && str_contains($setup, "'" . $index . "'"),
'Missing observation index: ' . $index);
}
$assert(!str_contains(
$definitions['cloud_vm_network_observation_samples'],
'cloud_network_observation_placement'
) && !str_contains(
$definitions['cloud_vm_network_observation_samples'],
'cloud_network_observation_host'
),
'Observation storage retained redundant wide placement/host indexes.');
foreach ([
'cloud_network_projection_due',
'cloud_network_projection_instance',
'cloud_network_projection_lease',
'cloud_network_projection_retention',
] as $index) {
$assert(str_contains(
$definitions['cloud_vm_network_projection_outbox'], $index
) && str_contains($setup, "'" . $index . "'"),
'Missing projection index: ' . $index);
}
foreach ([
'cloud_network_monitor_cursor_health',
'cloud_network_monitor_cursor_due',
'cloud_network_monitor_cursor_lease',
] as $index) {
$assert(str_contains(
$definitions['cloud_vm_network_monitor_cursors'], $index
) && str_contains($setup, "'" . $index . "'"),
'Missing cursor index: ' . $index);
}
foreach ([
'cloud_network_baseline_owner',
'cloud_network_baseline_warmup',
'cloud_network_baseline_projection_lease',
] as $index) {
$assert(str_contains(
$definitions['cloud_vm_network_baselines'], $index
) && str_contains($setup, "'" . $index . "'"),
'Missing baseline index: ' . $index);
}
foreach ([
'chk_cloud_network_rollup_bucket',
'chk_cloud_network_rollup_counts',
'chk_cloud_network_rollup_counters',
] as $check) {
$assert(str_contains($log, 'CONSTRAINT ' . $check . ' CHECK')
&& str_contains($setup, "'" . $check . "'"),
'CloudSetup does not require rollup CHECK: ' . $check);
}
foreach ([
'chk_cloud_network_observation_contract',
'chk_cloud_network_observation_bucket',
'chk_cloud_network_observation_complete',
'chk_cloud_network_projection_key',
'chk_cloud_network_projection_policy',
'chk_cloud_network_projection_state',
'chk_cloud_network_projection_lease',
'chk_cloud_network_monitor_cursor_state',
'chk_cloud_network_monitor_cursor_generation',
'chk_cloud_network_monitor_cursor_lease',
'chk_cloud_network_baseline_warmup',
'chk_cloud_network_baseline_policy',
'chk_cloud_network_baseline_projection',
'chk_cloud_network_baseline_projection_lease',
] as $check) {
$assert(str_contains($log, 'CONSTRAINT ' . $check . ' CHECK')
&& str_contains($setup, "'" . $check . "'"),
'CloudSetup does not require log CHECK: ' . $check);
}
$assert(str_contains($definitions['cloud_abuse_instance_states'],
"risk_state IN ('observing','normal','suspicious','high_risk')")
&& str_contains($definitions['cloud_abuse_instance_states'],
"'unknown','healthy','degraded','stale','unavailable'")
&& str_contains($definitions['cloud_abuse_findings'],
"direction IN ('inbound','outbound')")
&& str_contains($definitions['cloud_abuse_findings'],
"ip_family IN ('ipv4','ipv6','mixed')")
&& str_contains($definitions['cloud_abuse_findings'],
"state IN ('open','resolved')")
&& str_contains($definitions['cloud_abuse_instance_states'],
'chk_cloud_abuse_state_policy CHECK')
&& str_contains($definitions['cloud_abuse_findings'],
'chk_cloud_abuse_finding_policy CHECK')
&& str_contains($definitions['cloud_abuse_projection_receipts'],
'chk_cloud_abuse_projection_policy CHECK')
&& str_contains($definitions['cloud_abuse_findings'],
"resolution_reason IN (\n 'evidence_clear','policy_replaced','owner_changed'"),
'Abuse-monitoring state constraints are incomplete.');
$assert(str_contains($definitions['cloud_abuse_alert_events'],
'chk_cloud_abuse_alert_event_identity CHECK')
&& str_contains($definitions['cloud_abuse_alert_events'],
"highest_severity IN ('suspicious','high')")
&& str_contains($definitions['cloud_abuse_alert_event_items'],
'chk_cloud_abuse_alert_item_transition CHECK')
&& str_contains($definitions['cloud_abuse_alert_event_items'],
"transition_type = 'opened'")
&& str_contains($definitions['cloud_abuse_alert_event_items'],
"transition_type = 'escalated'")
&& str_contains($definitions['cloud_abuse_alert_event_items'],
"transition_type = 'resolved'"),
'Abuse alert event transition constraints are incomplete.');
$delivery = $definitions['cloud_abuse_alert_deliveries'];
$assert(str_contains($delivery,
'chk_cloud_abuse_alert_delivery_destination CHECK')
&& str_contains($delivery, 'binding_generation IS NOT NULL')
&& str_contains($delivery, 'chat_id IS NOT NULL')
&& str_contains($delivery, 'OR telegram_message_id > 0')
&& str_contains($delivery,
"state = 'failed'\n AND telegram_message_id IS NULL\n AND dispatch_started_at IS NULL")
&& str_contains($delivery,
"state = 'unknown'\n AND dispatch_started_at IS NOT NULL")
&& str_contains($definitions['administrator_telegram_delivery_gate'],
'chk_administrator_telegram_delivery_gate CHECK')
&& str_contains($setup,
"'chk_cloud_abuse_alert_delivery_destination'")
&& str_contains($setup,
"'chk_administrator_telegram_delivery_gate'"),
'Abuse delivery or shared Telegram gate constraints are incomplete.');
$assert(str_contains($definitions['cloud_abuse_instance_states'],
'fk_cloud_abuse_state_instance FOREIGN KEY (instance_id)')
&& str_contains($definitions['cloud_abuse_instance_states'],
'REFERENCES cloud_instances (id) ON DELETE CASCADE')
&& str_contains($definitions['cloud_abuse_findings'],
'fk_cloud_abuse_finding_instance FOREIGN KEY (instance_id)')
&& str_contains($definitions['cloud_abuse_findings'],
'REFERENCES cloud_instances (id) ON DELETE CASCADE')
&& str_contains($definitions['cloud_abuse_projection_receipts'],
'fk_cloud_abuse_projection_instance FOREIGN KEY (instance_id)')
&& str_contains($definitions['cloud_abuse_projection_receipts'],
'REFERENCES cloud_instances (id) ON DELETE CASCADE')
&& str_contains($setup, "'fk_cloud_abuse_state_instance'")
&& str_contains($setup, "'fk_cloud_abuse_finding_instance'")
&& str_contains($setup, "'fk_cloud_abuse_projection_instance'"),
'Current state or finding rows can outlive a deleted instance.');
$assert(!str_contains($definitions['cloud_abuse_alert_events'],
'REFERENCES cloud_instances')
&& str_contains($definitions['cloud_abuse_alert_event_items'],
'fk_cloud_abuse_alert_item_event FOREIGN KEY (event_id)')
&& str_contains($definitions['cloud_abuse_alert_event_items'],
'ON DELETE CASCADE ON UPDATE RESTRICT'),
'Alert events are not independent or item cleanup is not bounded.');
$assert(str_contains($delivery,
'fk_cloud_abuse_alert_delivery_event FOREIGN KEY (event_id)')
&& str_contains($delivery,
'ON DELETE CASCADE ON UPDATE RESTRICT')
&& str_contains($setup,
"'fk_cloud_abuse_alert_delivery_event'"),
'Abuse delivery does not cascade only from its immutable event.');
$outboxDelete = strpos(
$deletion, "'cloud_vm_network_projection_outbox'"
);
$sampleDelete = strrpos(
$deletion, "'cloud_vm_network_observation_samples'"
);
$assert($outboxDelete !== false && $sampleDelete !== false
&& $outboxDelete < $sampleDelete
&& str_contains($deletion, "'cloud_vm_network_baselines'")
&& str_contains($deletion, "'cloud_vm_network_observation_rollups'"),
'Account deletion leaves tenant observation history in the log database.');
$assert(str_contains($deletion,
"['cloud_abuse_alert_events', 'user_id']")
&& str_contains($deletion,
"['cloud_abuse_findings', 'user_id']")
&& str_contains($deletion,
"['cloud_abuse_instance_states', 'user_id']"),
'Account deletion does not erase frozen old-owner abuse data.');
$ownerLock = strpos($projection, "->table('user')");
$instanceLock = strpos($projection, "->table('cloud_instances')");
$assert($ownerLock !== false && $instanceLock !== false
&& $ownerLock < $instanceLock,
'Projection does not share the account deletion user-before-instance lock order.');
$assert(str_contains(
$definitions['cloud_vm_network_projection_outbox'],
'ON DELETE RESTRICT ON UPDATE RESTRICT'
) && !str_contains(
$definitions['cloud_vm_network_projection_outbox'],
'ON DELETE CASCADE'
) && str_contains($setup, "'delete_rule' => 'RESTRICT'")
&& str_contains($setup, "'update_rule' => 'RESTRICT'"),
'Pending or leased projection samples are not deletion-protected.');
$assert(str_contains(
$definitions['cloud_vm_network_observation_samples'],
'contract_version = 1'
) && str_contains(
$definitions['cloud_vm_network_observation_samples'],
'bucket_seconds = 60'
) && str_contains(
$definitions['cloud_vm_network_monitor_cursors'],
"'unknown','healthy','degraded','stale','unavailable'"
) && str_contains(
$definitions['cloud_vm_network_baselines'],
"warmup_state IN ('warming','ready')"
), 'Observation quality or baseline constraints are incomplete.');
$assert(str_contains(
$setup,
"SCHEMA_VERSION = 'go-agent-wss-v53-traffic-settlement-receipts'"
), 'CloudSetup schema version was not advanced for abuse observation.');
foreach (array_keys($mainTables + $logTables) as $table) {
$assert(!str_contains($traffic, $table)
&& !str_contains($lifecycle, $table),
'Abuse monitoring leaked into billing/lifecycle sampling: ' . $table);
}
$forbiddenIdentityColumn = '/^\s*`?remote_(?:ip|port)[a-z0-9_]*`?\s+/mi';
$assert(preg_match($forbiddenIdentityColumn, $main) !== 1
&& preg_match($forbiddenIdentityColumn, $log) !== 1,
'Schema stores forbidden remote flow identities.');
echo "Abuse monitoring schema contract passed.\n";