From 0991bf9efae3651b2df11e78df5bb433b7496ea1 Mon Sep 17 00:00:00 2001 From: codex Date: Tue, 17 Nov 2026 03:35:31 +0800 Subject: [PATCH] refactor(abuse): remove legacy telemetry pipeline --- docs/DEPLOYMENT.md | 12 +- docs/SERVICE_COMPENSATION.md | 2 +- web/app/Command/CloudSetup.php | 548 ++---- .../Cloud/CloudAbuseAlertEventCoordinator.php | 301 ---- .../CloudAbuseObservationRollupStore.php | 354 ---- .../Cloud/CloudAbuseProjectionService.php | 1539 ----------------- web/app/Services/TicketTelegramNotifier.php | 4 +- web/app/Services/UserDataDeletionService.php | 86 +- web/sql/cloud.sql | 350 ---- web/sql/cloud_log.sql | 276 --- .../CloudAbuseAlertEventCoordinatorTest.php | 198 --- ...CloudAbuseMonitoringSchemaContractTest.php | 431 ----- .../CloudAbusePolicySchemaContractTest.php | 2 +- ...CloudAbuseProjectionLedgerContractTest.php | 366 ---- web/tests/CloudAbuseProjectionServiceTest.php | 1387 --------------- ...blicationPreparationSchemaContractTest.php | 2 +- .../CloudCompactAbuseSchemaContractTest.php | 52 +- web/tests/CloudFreshSchemaContractTest.php | 6 +- ...oudMaintenanceNoticeSchemaContractTest.php | 2 +- ...licIpPoolReplacementSchemaContractTest.php | 2 +- ...licNetworkTransitionSchemaContractTest.php | 2 +- .../CloudRescueCatalogSchemaContractTest.php | 2 +- ...loudRestoreMigrationLedgerContractTest.php | 2 +- ...dServiceCompensationSchemaContractTest.php | 2 +- ...udStoragePublicationSchemaContractTest.php | 2 +- .../CloudTemplateImportIntentContractTest.php | 2 +- .../CloudTrafficPoolSchemaContractTest.php | 2 +- web/tests/CloudVpcSchemaContractTest.php | 2 +- web/tests/ManualAccountDeletionSqliteTest.php | 175 +- .../TicketTelegramNotifierSecurityTest.php | 4 +- web/tests/UiCopyCatalogSchemaContractTest.php | 2 +- 31 files changed, 333 insertions(+), 5784 deletions(-) delete mode 100644 web/app/Services/Cloud/CloudAbuseAlertEventCoordinator.php delete mode 100644 web/app/Services/Cloud/CloudAbuseObservationRollupStore.php delete mode 100644 web/app/Services/Cloud/CloudAbuseProjectionService.php delete mode 100644 web/tests/CloudAbuseAlertEventCoordinatorTest.php delete mode 100644 web/tests/CloudAbuseMonitoringSchemaContractTest.php delete mode 100644 web/tests/CloudAbuseProjectionLedgerContractTest.php delete mode 100644 web/tests/CloudAbuseProjectionServiceTest.php diff --git a/docs/DEPLOYMENT.md b/docs/DEPLOYMENT.md index 1eccec44..141d2430 100644 --- a/docs/DEPLOYMENT.md +++ b/docs/DEPLOYMENT.md @@ -160,17 +160,17 @@ libvirt 的持久 pool 定义就是 Agent 后续使用的映射,不另写存 公网网络接入在宿主机的“网络接入”中按资源池线路独立配置,可使用 `bridge:br0`、其他 Linux 网桥或 `network:default` 这样的 libvirt network。名称以字母或数字开头,后续只能包含字母、数字、点、下划线和短横线,bridge 名称为 1~15 字符,libvirt network 名称为 1~64 字符。宿主机上的公网网桥或 libvirt network 必须由运维预先准备。Agent 只发现、验证和使用现有公网网络,不创建公网网桥、不启动 libvirt network,也不修改公网地址、上游网关或挂载。`cloud_hosts.default_network` 仅保留为兼容展示值,不参与 Agent 就绪、调度、恢复、迁移或换 IP 决策;真正的运行时权威是资源池线路、当前 Agent 代次已验证的网络接入,以及绑定该接入的公网地址池。VPC 使用下述独立的 Agent 受管 bridge 和 L3 路由,不改变这一公网边界。 -当前完整 schema 版本为 `go-agent-wss-v56-daily-traffic-settlement`。v48~v55 已完成滥用证据分层、性能与流量日志压缩、UTC 每日结算、紧凑结算凭据、7 天安全窗压缩、每日累计检查点及服务周期边界锚点。v56 将结算窗口固定为 UTC 日,删除仅为旧小时结算桶保留的可变窗口字段和分支;`seal_at` 继续作为到期调度权威,分钟级 FEFO 扣减、游标、不可变账本、每日检查点和周期锚点均保持不变。 +当前完整 schema 版本为 `go-agent-wss-v57-compact-abuse-ledger`。v48~v55 已完成滥用证据分层、性能与流量日志压缩、UTC 每日结算、紧凑结算凭据、7 天安全窗压缩、每日累计检查点及服务周期边界锚点。v56 将结算窗口固定为 UTC 日,删除仅为旧小时结算桶保留的可变窗口字段和分支;`seal_at` 继续作为到期调度权威,分钟级 FEFO 扣减、游标、不可变账本、每日检查点和周期锚点均保持不变。v57 将网络滥用监测改为主库内的实例级紧凑账本:每个实例一条可变检测状态、每个实例每天至多一条汇总、仅在 finding 转移时写入稀疏事件和 Telegram 投递安全账本,不再保存逐分钟样本、小时汇总、投影 outbox、projection receipt 或单独 baseline。 测试环境从 v55 升级 v56 时,必须先停止生命周期 worker,清空 `cloud_traffic_usage_settlement_buckets` 与 `cloud_traffic_settlement_receipts` 的旧结算细节,再删除旧窗口字段、重建两个身份唯一索引和日窗口检查约束,最后写入当前 schema marker 与真实 information_schema 指纹。旧结算细节可以丢弃,但不可变账本、池投影、实例游标、额度、每日检查点和周期锚点不得删除。完成 `php xcat cloudInit` 全量校验后才能恢复 worker;不得让 v55 代码与 v56 表结构混跑。全新部署只允许直接使用当前 `cloud.sql` 与 `cloud_log.sql`,不保留旧窗口兼容路径。 -v47 为 `cloud_offers` 增加季度、年度及对应自动备份价格;既有方案升级时必须在恢复销售前显式回填 90/365 天价格,不能让数据库默认零价进入购买目录。从 v44 升级时,必须先停止滥用观测与投影 worker,为既有 finding 回填 `occurrence_generation = 1` 与 `transition_revision = 1`,再创建 `cloud_abuse_alert_events` 和 `cloud_abuse_alert_event_items`;从 v45 升级还必须停止通知 worker,创建 `cloud_abuse_alert_deliveries`、`administrator_telegram_delivery_gate` 并写入唯一 gate 行。历史 finding 不反向合成事件;事件只从升级后主库事务内真实发生的 `opened`、`escalated` 和 `evidence_clear` 解决转移产生。完成列、约束、索引、外键和 gate 核对后,先恢复滥用观测,再恢复通知 worker。 +v47 为 `cloud_offers` 增加季度、年度及对应自动备份价格;既有方案升级时必须在恢复销售前显式回填 90/365 天价格,不能让数据库默认零价进入购买目录。v57 的滥用账本是 clean-break:发布前停止滥用观测与通知 worker,备份数据库,只删除旧主库 `cloud_abuse_instance_states`、`cloud_abuse_findings`、`cloud_abuse_alert_events`、`cloud_abuse_alert_event_items`、`cloud_abuse_alert_deliveries`、`cloud_abuse_projection_receipts`,以及旧日志库 `cloud_vm_network_observation_samples`、`cloud_vm_network_observation_rollups`、`cloud_vm_network_projection_outbox`、`cloud_vm_network_monitor_cursors`、`cloud_vm_network_baselines`;`cloud_abuse_policy_revisions` 与 `cloud_abuse_policy_rules` 仍是当前策略权威,不能删除。随后用经过审核的一次性脚本创建当前六张 `cloud_network_*` 表并更新 schema marker 与实际结构指纹。旧样本、finding、事件和投递不导入、不双写、不回退读取;仓库中的 `cloudInit` 会明确拒绝新旧表并存或指纹不匹配的半升级数据库。完成列、约束、索引、外键和单例 Telegram gate 核对后,先恢复滥用观测,再恢复通知 worker。 -事件 header 与 projection receipt 在同一个主库事务中写入,且事件先于 receipt;同一实例同一采样桶的多规则、双协议族转移合并为一个 header 和多个 item。`continued` 不产生事件,`policy_replaced` 与 `owner_changed` 只关闭 finding,不能伪造“恢复”事件。item 已冻结当时的聚合 evidence、规则身份和 finding 代次,后续投递不得回读会变化的 finding。v46 为每个事件建立至多一条 Telegram 投递决策;远端调用开始后的中断或回执不确定统一进入 `unknown`,禁止自动重放。删除用户时按事件冻结的 `user_id` 显式删除 header,item 与投递账本按当前外键和账户清理规则收敛,避免实例转交后遗留原租户数据。 +Agent UUID、采集 generation 和 cursor 只用于把本次窗口映射到逻辑实例;每个宿主机游标、所有受影响实例账本、UTC 日切汇总、finding 转移事件和游标推进在同一个主库事务中提交。重复或倒序窗口不会推进实例账本,事务失败也不会只推进宿主机 cursor。`continued` 不产生事件,`policy_replaced` 与 `owner_changed` 只关闭 finding,不能伪造“恢复”事件;真正的 `opened`、`escalated` 与 `evidence_clear` 解决各写一条不可变事件,并冻结该时刻的白名单聚合 evidence。每个事件至多建立一条 Telegram 投递决策;远端调用开始后的中断或回执不确定统一进入 `unknown`,禁止自动重放。实例删除通过外键级联清理账本;删除曾绑定 Telegram 的管理员时保留事件唯一处置并匿名化目的地,避免同一事件再次发送。 -管理员“网络滥用监控 -> 滥用告警历史”通过 `GET /api/v1/admin/abuse-monitoring/alerts` 异步读取只读投影。没有投递行的事件显示为 `queued`,其余只显示 `pending`、`dispatching`、`sent`、`skipped`、`failed` 或 `unknown` 的安全文案;页面不提供重试、重新发送或任何写操作。接口不得返回 Telegram chat、绑定代次、token、冻结消息、内容/策略 hash、内部事件 key 或原始 evidence JSON,只允许展示白名单内的非负聚合计数。部署验收必须用只读管理员验证事件/实例/用户/节点独立筛选和分页,并确认移动端卡片可读;再用 `unknown` 测试记录确认页面明确提示“禁止自动重放”。 +管理员“网络滥用监控 -> 滥用告警历史”通过 `GET /api/v1/admin/abuse-monitoring/alerts` 异步读取紧凑主库账本。没有投递行的事件显示为 `queued`,其余只显示 `pending`、`dispatching`、`sent`、`skipped`、`failed` 或 `unknown` 的安全文案;页面不提供重试、重新发送或任何写操作。接口不得返回 Telegram chat、绑定代次、token、冻结消息、内容 hash 或原始 evidence JSON,只允许展示白名单内的非负聚合计数。部署验收必须用只读管理员验证事件/实例/用户/节点独立筛选和分页,并确认移动端卡片可读;再用 `unknown` 测试记录确认页面明确提示“禁止自动重放”。 -`occurrence_generation` 与 `transition_revision` 只描述当前仍保留的 finding 生命周期,不是跨 retention 的全局幂等键;旧 finding 被清理后,同一规则以后允许从 `1/1` 重新建立。未来投递只能以不可变的 `event_id` / `item_key` 建立幂等边界,不能使用 generation/revision 猜测是否已经外发。账户删除与投影都必须遵循“先锁 user、再锁 instance”的顺序;正式部署还应在真实 MySQL 双连接下验收删除和投影并发,SQLite 行为测试不等价于 InnoDB 行锁门禁。 +`occurrence_generation` 与 `transition_revision` 只描述当前仍保留的 finding 生命周期,不是跨 retention 的全局幂等键;旧 finding 被清理后,同一规则以后允许从 `1/1` 重新建立。投递只能以不可变 `event_id` 的唯一投递行为幂等边界,不能使用 generation/revision 猜测是否已经外发。账户删除与观测写入都先锁用户/实例所有权,再锁相应事件和投递行;正式部署还应在真实 MySQL 双连接下验收删除、采集和通知并发,SQLite 行为测试不等价于 InnoDB 行锁门禁。 自动备份订阅通过独立的延期字段保留原逻辑执行时隙;只有已确认未应用且可安全重试的救援或磁盘切换竞态,才会退休空白制品并以递增的 `schedule_attempt` 建立新代次。每个资源预留的当前公网权威只来自 `cloud_resource_reservation_ips.active_claim = 1` 的一组 IPv4+IPv6 精确声明;两族允许具有各自的 generation 和 operation ID,但必须绑定同一资源预留和网络接入,并各自与具体 IP、地址池、宿主机、实例状态双向一致。`cloud_resource_reservations` 中旧的单地址列只保留审计形态,不参与当前地址读取、释放或管理员投影。释放时历史声明保留并把 `active_claim` 置空,具体 IP 同时清空 claim、operation、reservation 和 instance 归属。普通换 IP、独立恢复和维护迁移分别只允许 transition ledger、restore attempt 与 migration source debt 所有的完整双栈流程;旧 `CloudPlacementService` 单地址恢复、旧换 IP writer,以及缺少或损坏冻结身份的任务仍会在派发 CAS 前拒绝,不能绕过新账本。 @@ -272,7 +272,7 @@ systemctl enable --now virt-cloud-worker.service \ `virt-cloud-storage-health-worker.service` 与副本复制 worker 彻底隔离:它每次只领取一个到期后端,在数据库事务外执行最长 15 秒的 `HeadBucket`,并且只能按领取时冻结的配置版本结算。成功只恢复健康证据,不会把人工设置的 `read_only`、`draining` 或 `disabled` 改回 `active`;失败使正在 `active` 的后端转为 `read_only`、按 60 秒到 1 小时退避,新写入随即停止选择它。这个 worker 不执行 PUT、COPY、DELETE 或 Abort,也不把凭据写入心跳、日志或命令行。 -`virt-cloud-abuse-monitor.service` 是独立、只读宿主机的网络滥用观测进程。Agent 仅发布每分钟公网接口与 conntrack 的有界聚合计数,不采集报文正文、远端 IP 列表或远端端口列表;PHP 将样本、投影意图与宿主机游标原子写入日志库,再用幂等 receipt 投影管理员只读状态和不可变转移事件。该服务不进入 `cloud_jobs`,不会自动断网、限速、修改防火墙、实例生命周期或账单。Telegram 外发由通知 worker 根据不可变事件和独立投递账本处理;不能从事件存在本身推断已经通知,也不能把 `dispatching` 或 `unknown` 当作可安全重发。采集缺口、Agent generation 变化及不完整样本均按异常证据显示,不会当作正常零值参与判定。正式启用前需确认宿主机 conntrack 可读、Agent 网络观测能力可用,并同时检查 `abuse_monitor` 心跳、通知 worker 与后台采集健康状态。 +`virt-cloud-abuse-monitor.service` 是独立、只读宿主机的网络滥用观测进程。Agent 仅发布每分钟公网接口与 conntrack 的有界聚合计数,不采集报文正文、远端 IP 列表或远端端口列表;PHP 在一个主库事务中直接更新宿主机 cursor、实例紧凑账本、日切汇总和稀疏 finding/事件,不再把逐分钟样本写入日志库,也没有第二阶段投影 worker。该服务不进入 `cloud_jobs`,不会自动断网、限速、修改防火墙、实例生命周期或账单。Telegram 外发由通知 worker 根据不可变事件和独立投递账本处理;不能从事件存在本身推断已经通知,也不能把 `dispatching` 或 `unknown` 当作可安全重发。采集缺口、Agent generation 变化及不完整样本均按异常证据显示,不会当作正常零值参与判定。正式启用前需确认宿主机 conntrack 可读、Agent 网络观测能力可用,并同时检查 `abuse_monitor` 心跳、通知 worker 与后台采集健康状态。 实例补偿的批次、明细和收件人账本已包含在当前完整 schema 中。`php xcat cloudInit` 只能为全新空数据库创建和验证当前全部结构,不能冒充既有数据库的增量升级。既有安装发布前必须备份数据库,使用经过审核的一次性迁移落地当前完整结构并核对 schema 指纹;升级成功后删除迁移脚本。然后按本节重启普通、通知及对象存储 workers。补偿本身不调用 Agent/Gateway,因此仅补偿功能不要求更新或重新编译宿主机 Agent 和面板 Gateway。完整语义见 [`SERVICE_COMPENSATION.md`](SERVICE_COMPENSATION.md)。 diff --git a/docs/SERVICE_COMPENSATION.md b/docs/SERVICE_COMPENSATION.md index 3b4f6e7f..c6179a08 100644 --- a/docs/SERVICE_COMPENSATION.md +++ b/docs/SERVICE_COMPENSATION.md @@ -47,7 +47,7 @@ ## 部署 -全新数据库必须由当前 `CloudSetup` 创建并验证 `go-agent-wss-v56-daily-traffic-settlement` 完整 schema;其中补偿模块包括: +全新数据库必须由当前 `CloudSetup` 创建并验证 `go-agent-wss-v57-compact-abuse-ledger` 完整 schema;其中补偿模块包括: - `cloud_service_compensations`; - `cloud_service_compensation_items`; diff --git a/web/app/Command/CloudSetup.php b/web/app/Command/CloudSetup.php index 897edc4f..45260326 100644 --- a/web/app/Command/CloudSetup.php +++ b/web/app/Command/CloudSetup.php @@ -8,7 +8,7 @@ use App\Services\Config; /** Initializes only the clean-break PHP business and Agent log schemas. */ final class CloudSetup { - private const SCHEMA_VERSION = 'go-agent-wss-v56-daily-traffic-settlement'; + private const SCHEMA_VERSION = 'go-agent-wss-v57-compact-abuse-ledger'; /** Tables installed by ss.sql before cloudInit and owned by this schema. */ private const PREINSTALLED_MAIN_TABLES = [ 'ui_copy_locales', 'ui_copy_entries', 'ui_copy_drafts', @@ -69,11 +69,10 @@ final class CloudSetup 'cloud_service_compensation_items', 'cloud_service_compensation_recipients', 'cloud_abuse_policy_revisions', 'cloud_abuse_policy_rules', - 'cloud_abuse_instance_states', 'cloud_abuse_findings', - 'cloud_abuse_alert_events', 'cloud_abuse_alert_event_items', - 'cloud_abuse_alert_deliveries', + 'cloud_network_monitor_hosts', 'cloud_network_instance_ledger', + 'cloud_network_daily_summaries', 'cloud_network_findings', + 'cloud_network_alert_events', 'cloud_network_alert_deliveries', 'administrator_telegram_delivery_gate', - 'cloud_abuse_projection_receipts', 'cloud_jobs', 'cloud_agent_completion_events', 'cloud_ssh_keys', 'cloud_sessions', 'cloud_console_tickets', @@ -85,17 +84,25 @@ final class CloudSetup 'cloud_vm_traffic_boundary_anchors', 'cloud_vm_traffic_totals', 'cloud_vm_daily_traffic_cursors', 'cloud_vm_daily_traffic', + ]; + /** Refusal-only denylists: never read, import, rename or migrate these rows. */ + private const RETIRED_MAIN_TABLES = [ + 'cloud_abuse_instance_states', + 'cloud_abuse_findings', + 'cloud_abuse_alert_events', + 'cloud_abuse_alert_event_items', + 'cloud_abuse_alert_deliveries', + 'cloud_abuse_projection_receipts', + ]; + private const RETIRED_LOG_TABLES = [ + 'engine_vm_metric_samples', + 'engine_vm_traffic_checkpoints', 'engine_vm_traffic_totals', 'cloud_vm_network_observation_samples', 'cloud_vm_network_observation_rollups', 'cloud_vm_network_projection_outbox', 'cloud_vm_network_monitor_cursors', 'cloud_vm_network_baselines', ]; - /** Refusal-only denylist: never read, import, rename or migrate these rows. */ - private const RETIRED_LOG_TABLES = [ - 'engine_vm_metric_samples', - 'engine_vm_traffic_checkpoints', 'engine_vm_traffic_totals', - ]; private const REQUIRED_MAIN_COLUMNS = [ 'ui_copy_locales' => [ 'id', 'locale', 'display_name', 'enabled', 'source_claim', @@ -627,38 +634,41 @@ final class CloudSetup 'enabled', 'suspicious_thresholds_json', 'high_thresholds_json', 'created_at', ], - 'cloud_abuse_instance_states' => [ - 'instance_id', 'user_id', 'host_id', 'rule_version', - 'policy_revision', 'policy_hash', 'risk_state', - 'collection_state', 'latest_direction', 'latest_rule_code', - 'latest_severity', 'summary_json', 'last_projection_key', - 'last_bucket_start', 'last_sample_at', 'last_evaluated_at', - 'created_at', 'updated_at', + 'cloud_network_monitor_hosts' => [ + 'host_id', 'collector_generation', 'cursor_value', + 'oldest_cursor', 'gap_before', 'poll_revision', 'next_poll_at', + 'poll_lease_token', 'poll_lease_until', 'collection_state', + 'consecutive_failures', 'last_window_start', 'last_success_at', + 'last_error_code', 'updated_at', ], - 'cloud_abuse_findings' => [ - 'id', 'finding_key', 'instance_id', 'user_id', 'host_id', - 'rule_code', 'rule_version', 'policy_revision', 'policy_hash', - 'direction', 'ip_family', - 'severity', 'state', 'first_seen_at', 'last_seen_at', + 'cloud_network_instance_ledger' => [ + 'instance_id', 'policy_revision', 'risk_state', + 'collection_state', 'latest_direction', 'latest_rule_code', + 'latest_severity', 'detector_state_json', 'latest_metrics_json', + 'current_day_start', 'current_day_json', 'last_bucket_start', + 'last_sample_at', 'last_evaluated_at', 'created_at', 'updated_at', + ], + 'cloud_network_daily_summaries' => [ + 'instance_id', 'day_start', 'sample_count', 'complete_count', + 'quality_mask', 'rx_bps_peak', 'tx_bps_peak', 'rx_pps_peak', + 'tx_pps_peak', 'tcp_active_peak', 'tcp_new_peak', 'tcp_syn_peak', + 'tcp_unreplied_peak', 'udp_active_peak', 'udp_new_peak', + 'peer_ip_peak', 'peer_port_peak', 'first_sample_at', + 'last_sample_at', 'created_at', 'updated_at', + ], + 'cloud_network_findings' => [ + 'id', 'instance_id', 'policy_revision', 'rule_code', 'direction', + 'ip_family', 'severity', 'state', 'first_seen_at', 'last_seen_at', 'consecutive_windows', 'occurrence_generation', 'transition_revision', 'evidence_json', 'resolved_at', - 'resolution_reason', - 'created_at', 'updated_at', - ], - 'cloud_abuse_alert_events' => [ - 'id', 'event_key', 'sample_key', 'instance_id', 'user_id', - 'host_id', 'bucket_start', 'rule_version', 'policy_revision', - 'policy_hash', 'transition_digest', 'transition_count', - 'highest_severity', 'created_at', + 'resolution_reason', 'created_at', 'updated_at', ], - 'cloud_abuse_alert_event_items' => [ - 'id', 'event_id', 'item_key', 'finding_key', - 'occurrence_generation', 'transition_revision', 'rule_code', - 'direction', 'ip_family', 'transition_type', 'from_severity', - 'to_severity', 'evidence_json', 'resolution_reason', - 'occurred_at', 'created_at', + 'cloud_network_alert_events' => [ + 'id', 'finding_id', 'transition_revision', 'transition_type', + 'from_severity', 'to_severity', 'evidence_json', + 'resolution_reason', 'occurred_at', 'created_at', ], - 'cloud_abuse_alert_deliveries' => [ + 'cloud_network_alert_deliveries' => [ 'id', 'event_id', 'binding_generation', 'bound_admin_id', 'chat_id', 'telegram_message_id', 'state', 'message_text', 'content_hash', 'error_code', 'skip_reason', @@ -668,10 +678,6 @@ final class CloudSetup 'administrator_telegram_delivery_gate' => [ 'id', 'next_send_at', 'updated_at', ], - 'cloud_abuse_projection_receipts' => [ - 'sample_key', 'instance_id', 'rule_version', 'policy_revision', - 'policy_hash', 'bucket_start', 'projected_at', - ], ]; private const REQUIRED_LOG_COLUMNS = [ 'cloud_vm_metric_samples' => [ @@ -724,48 +730,6 @@ final class CloudSetup 'upload_bytes', 'sample_count', 'first_observed_at', 'last_observed_at', 'created_at', 'updated_at', ], - 'cloud_vm_network_observation_samples' => [ - 'sample_key', 'instance_id', 'user_id', 'host_id', 'vm_uuid', - 'placement_key', 'collector_generation', 'window_id', - 'contract_version', 'bucket_start', 'bucket_seconds', - 'complete', 'metrics_json', - 'quality_flags_json', 'received_at', - ], - 'cloud_vm_network_observation_rollups' => [ - 'instance_id', 'user_id', 'host_id', 'interval_seconds', - 'bucket_start', 'sample_count', 'complete_count', 'quality_mask', - 'rx_bytes_min', 'rx_bytes_max', 'tx_bytes_min', 'tx_bytes_max', - 'rx_packets_min', 'rx_packets_max', 'tx_packets_min', - 'tx_packets_max', 'tcp_active_peak', 'tcp_new_total', - 'tcp_syn_total', 'tcp_unreplied_total', 'udp_active_peak', - 'udp_new_total', 'peer_ip_cardinality_peak', - 'peer_port_cardinality_peak', - 'first_sample_at', 'last_sample_at', 'created_at', 'updated_at', - ], - 'cloud_vm_network_projection_outbox' => [ - 'sample_key', 'instance_id', 'bucket_start', 'placement_key', - 'rule_version', 'policy_revision', 'policy_hash', 'state', - 'attempts', 'available_at', 'lease_token', - 'lease_until', 'last_error_code', 'projected_at', - 'terminal_at', 'discard_reason_code', 'created_at', - 'updated_at', - ], - 'cloud_vm_network_monitor_cursors' => [ - 'host_id', 'collector_generation', 'cursor_value', - 'oldest_cursor', 'gap_before', 'poll_revision', - 'next_poll_at', 'poll_lease_token', 'poll_lease_until', - 'collection_state', 'consecutive_failures', - 'last_window_start', 'last_success_at', 'last_error_code', - 'updated_at', - ], - 'cloud_vm_network_baselines' => [ - 'instance_id', 'user_id', 'rule_version', 'policy_revision', - 'policy_hash', 'sample_count', - 'warmup_state', 'metrics_json', 'last_projection_key', - 'last_bucket_start', 'projection_lease_token', - 'projection_lease_until', 'revision', 'created_at', - 'updated_at', - ], ]; private const REQUIRED_CHECKS = [ 'ui_copy_locales' => [ @@ -890,48 +854,38 @@ final class CloudSetup 'chk_cloud_abuse_policy_rule_protocol', 'chk_cloud_abuse_policy_rule_enabled', ], - 'cloud_abuse_instance_states' => [ - 'chk_cloud_abuse_state_risk', - 'chk_cloud_abuse_state_collection', - 'chk_cloud_abuse_state_direction', - 'chk_cloud_abuse_state_severity', - 'chk_cloud_abuse_state_policy', - 'chk_cloud_abuse_state_projection', - ], - 'cloud_abuse_findings' => [ - 'chk_cloud_abuse_finding_key', - 'chk_cloud_abuse_finding_direction', - 'chk_cloud_abuse_finding_family', - 'chk_cloud_abuse_finding_severity', - 'chk_cloud_abuse_finding_state', - 'chk_cloud_abuse_finding_policy', - 'chk_cloud_abuse_finding_window', - 'chk_cloud_abuse_finding_resolution', - ], - 'cloud_abuse_alert_events' => [ - 'chk_cloud_abuse_alert_event_identity', - 'chk_cloud_abuse_alert_event_policy', - 'chk_cloud_abuse_alert_event_shape', - ], - 'cloud_abuse_alert_event_items' => [ - 'chk_cloud_abuse_alert_item_identity', - 'chk_cloud_abuse_alert_item_scope', - 'chk_cloud_abuse_alert_item_transition', - ], - 'cloud_abuse_alert_deliveries' => [ - 'chk_cloud_abuse_alert_delivery_state', - 'chk_cloud_abuse_alert_delivery_content', - 'chk_cloud_abuse_alert_delivery_destination', - 'chk_cloud_abuse_alert_delivery_shape', + 'cloud_network_monitor_hosts' => [ + 'chk_cloud_network_monitor_host_state', + 'chk_cloud_network_monitor_host_generation', + 'chk_cloud_network_monitor_host_lease', + ], + 'cloud_network_instance_ledger' => [ + 'chk_cloud_network_ledger_risk', + 'chk_cloud_network_ledger_collection', + 'chk_cloud_network_ledger_direction', + 'chk_cloud_network_ledger_severity', + 'chk_cloud_network_ledger_window', + ], + 'cloud_network_daily_summaries' => [ + 'chk_cloud_network_daily_window', + ], + 'cloud_network_findings' => [ + 'chk_cloud_network_finding_scope', + 'chk_cloud_network_finding_state', + 'chk_cloud_network_finding_resolution', + ], + 'cloud_network_alert_events' => [ + 'chk_cloud_network_alert_transition', + ], + 'cloud_network_alert_deliveries' => [ + 'chk_cloud_network_alert_delivery_state', + 'chk_cloud_network_alert_delivery_content', + 'chk_cloud_network_alert_delivery_destination', + 'chk_cloud_network_alert_delivery_shape', ], 'administrator_telegram_delivery_gate' => [ 'chk_administrator_telegram_delivery_gate', ], - 'cloud_abuse_projection_receipts' => [ - 'chk_cloud_abuse_projection_key', - 'chk_cloud_abuse_projection_policy', - 'chk_cloud_abuse_projection_bucket', - ], 'cloud_template_import_intents' => [ 'chk_cloud_template_import_source', 'chk_cloud_template_import_state', @@ -1095,33 +1049,6 @@ final class CloudSetup 'chk_vm_metric_rollup_bucket', 'chk_vm_metric_rollup_counts', ], - 'cloud_vm_network_observation_rollups' => [ - 'chk_cloud_network_rollup_bucket', - 'chk_cloud_network_rollup_counts', - 'chk_cloud_network_rollup_counters', - ], - 'cloud_vm_network_observation_samples' => [ - 'chk_cloud_network_observation_contract', - 'chk_cloud_network_observation_bucket', - 'chk_cloud_network_observation_complete', - ], - 'cloud_vm_network_projection_outbox' => [ - 'chk_cloud_network_projection_key', - 'chk_cloud_network_projection_policy', - 'chk_cloud_network_projection_state', - 'chk_cloud_network_projection_lease', - ], - 'cloud_vm_network_monitor_cursors' => [ - 'chk_cloud_network_monitor_cursor_state', - 'chk_cloud_network_monitor_cursor_generation', - 'chk_cloud_network_monitor_cursor_lease', - ], - 'cloud_vm_network_baselines' => [ - 'chk_cloud_network_baseline_warmup', - 'chk_cloud_network_baseline_policy', - 'chk_cloud_network_baseline_projection', - 'chk_cloud_network_baseline_projection_lease', - ], ]; private const REQUIRED_INDEXES = [ 'ui_copy_locales' => [ @@ -1259,43 +1186,34 @@ final class CloudSetup 'PRIMARY', 'cloud_abuse_policy_rule_revision', 'cloud_abuse_policy_rule_code', ], - 'cloud_abuse_instance_states' => [ - 'PRIMARY', 'cloud_abuse_state_risk', - 'cloud_abuse_state_collection', 'cloud_abuse_state_host', - 'cloud_abuse_state_owner', - ], - 'cloud_abuse_findings' => [ - 'PRIMARY', 'cloud_abuse_finding_key', - 'cloud_abuse_finding_instance', 'cloud_abuse_finding_owner', - 'cloud_abuse_finding_host', 'cloud_abuse_finding_rule', - 'cloud_abuse_finding_retention', - ], - 'cloud_abuse_alert_events' => [ - 'PRIMARY', 'cloud_abuse_alert_event_key', - 'cloud_abuse_alert_event_sample', - 'cloud_abuse_alert_event_instance', - 'cloud_abuse_alert_event_owner', - 'cloud_abuse_alert_event_host', - 'cloud_abuse_alert_event_retention', - ], - 'cloud_abuse_alert_event_items' => [ - 'PRIMARY', 'cloud_abuse_alert_item_key', - 'cloud_abuse_alert_item_finding_once', - 'cloud_abuse_alert_item_event', - 'cloud_abuse_alert_item_finding', - ], - 'cloud_abuse_alert_deliveries' => [ - 'PRIMARY', 'cloud_abuse_alert_delivery_event', - 'cloud_abuse_alert_delivery_message', - 'cloud_abuse_alert_delivery_due', - 'cloud_abuse_alert_delivery_admin', - 'cloud_abuse_alert_delivery_terminal', + 'cloud_network_monitor_hosts' => [ + 'PRIMARY', 'cloud_network_monitor_host_due', + 'cloud_network_monitor_host_health', + 'cloud_network_monitor_host_lease', ], - 'administrator_telegram_delivery_gate' => ['PRIMARY'], - 'cloud_abuse_projection_receipts' => [ - 'PRIMARY', 'cloud_abuse_projection_instance', - 'cloud_abuse_projection_retention', + 'cloud_network_instance_ledger' => [ + 'PRIMARY', 'cloud_network_ledger_risk', + 'cloud_network_ledger_collection', + ], + 'cloud_network_daily_summaries' => [ + 'PRIMARY', 'cloud_network_daily_retention', + ], + 'cloud_network_findings' => [ + 'PRIMARY', 'cloud_network_finding_scope', + 'cloud_network_finding_state', 'cloud_network_finding_retention', + ], + 'cloud_network_alert_events' => [ + 'PRIMARY', 'cloud_network_alert_finding_revision', + 'cloud_network_alert_retention', ], + 'cloud_network_alert_deliveries' => [ + 'PRIMARY', 'cloud_network_alert_delivery_event', + 'cloud_network_alert_delivery_message', + 'cloud_network_alert_delivery_due', + 'cloud_network_alert_delivery_admin', + 'cloud_network_alert_delivery_terminal', + ], + 'administrator_telegram_delivery_gate' => ['PRIMARY'], 'cloud_template_import_intents' => [ 'PRIMARY', 'cloud_template_import_request', 'cloud_template_import_template', @@ -1592,31 +1510,6 @@ final class CloudSetup 'PRIMARY', 'vm_traffic_boundary_instance', 'vm_traffic_boundary_owner', ], - 'cloud_vm_network_observation_rollups' => [ - 'PRIMARY', 'cloud_network_rollup_owner', - 'cloud_network_rollup_host', 'cloud_network_rollup_retention', - ], - 'cloud_vm_network_observation_samples' => [ - 'PRIMARY', 'cloud_network_observation_source', - 'cloud_network_observation_instance', - 'cloud_network_observation_retention', - ], - 'cloud_vm_network_projection_outbox' => [ - 'PRIMARY', 'cloud_network_projection_due', - 'cloud_network_projection_instance', - 'cloud_network_projection_lease', - 'cloud_network_projection_retention', - ], - 'cloud_vm_network_monitor_cursors' => [ - 'PRIMARY', 'cloud_network_monitor_cursor_health', - 'cloud_network_monitor_cursor_due', - 'cloud_network_monitor_cursor_lease', - ], - 'cloud_vm_network_baselines' => [ - 'PRIMARY', 'cloud_network_baseline_owner', - 'cloud_network_baseline_warmup', - 'cloud_network_baseline_projection_lease', - ], ]; /** * Safety-critical indexes whose name alone is not a sufficient contract. @@ -1947,75 +1840,88 @@ final class CloudSetup 'columns' => ['rule_code', 'policy_revision_id'], ], ], - 'cloud_abuse_findings' => [ - 'cloud_abuse_finding_key' => [ - 'non_unique' => 0, - 'columns' => ['finding_key'], + 'cloud_network_monitor_hosts' => [ + 'cloud_network_monitor_host_due' => [ + 'non_unique' => 1, + 'columns' => ['next_poll_at', 'poll_revision', 'host_id'], ], - 'cloud_abuse_finding_retention' => [ + 'cloud_network_monitor_host_health' => [ + 'non_unique' => 1, + 'columns' => ['collection_state', 'last_success_at', 'host_id'], + ], + 'cloud_network_monitor_host_lease' => [ + 'non_unique' => 1, + 'columns' => ['poll_lease_until', 'host_id'], + ], + ], + 'cloud_network_instance_ledger' => [ + 'cloud_network_ledger_risk' => [ + 'non_unique' => 1, + 'columns' => ['risk_state', 'updated_at', 'instance_id'], + ], + 'cloud_network_ledger_collection' => [ 'non_unique' => 1, 'columns' => [ - 'state', 'resolved_at', 'last_seen_at', 'id', + 'collection_state', 'last_sample_at', 'instance_id', ], ], ], - 'cloud_abuse_alert_events' => [ - 'cloud_abuse_alert_event_key' => [ - 'non_unique' => 0, - 'columns' => ['event_key'], + 'cloud_network_daily_summaries' => [ + 'cloud_network_daily_retention' => [ + 'non_unique' => 1, + 'columns' => ['day_start', 'instance_id'], ], - 'cloud_abuse_alert_event_sample' => [ + ], + 'cloud_network_findings' => [ + 'cloud_network_finding_scope' => [ 'non_unique' => 0, - 'columns' => ['sample_key'], + 'columns' => [ + 'instance_id', 'policy_revision', 'rule_code', + 'direction', 'ip_family', + ], ], - 'cloud_abuse_alert_event_retention' => [ + 'cloud_network_finding_state' => [ 'non_unique' => 1, - 'columns' => ['created_at', 'id'], + 'columns' => ['state', 'last_seen_at', 'instance_id', 'id'], ], - ], - 'cloud_abuse_alert_event_items' => [ - 'cloud_abuse_alert_item_key' => [ - 'non_unique' => 0, - 'columns' => ['item_key'], + 'cloud_network_finding_retention' => [ + 'non_unique' => 1, + 'columns' => ['state', 'resolved_at', 'id'], ], - 'cloud_abuse_alert_item_finding_once' => [ + ], + 'cloud_network_alert_events' => [ + 'cloud_network_alert_finding_revision' => [ 'non_unique' => 0, - 'columns' => ['event_id', 'finding_key'], + 'columns' => ['finding_id', 'transition_revision'], ], - 'cloud_abuse_alert_item_event' => [ + 'cloud_network_alert_retention' => [ 'non_unique' => 1, - 'columns' => ['event_id', 'id'], + 'columns' => ['created_at', 'id'], ], ], - 'cloud_abuse_alert_deliveries' => [ - 'cloud_abuse_alert_delivery_event' => [ + 'cloud_network_alert_deliveries' => [ + 'cloud_network_alert_delivery_event' => [ 'non_unique' => 0, 'columns' => ['event_id'], ], - 'cloud_abuse_alert_delivery_message' => [ + 'cloud_network_alert_delivery_message' => [ 'non_unique' => 0, 'columns' => [ 'binding_generation', 'chat_id', 'telegram_message_id', ], ], - 'cloud_abuse_alert_delivery_due' => [ + 'cloud_network_alert_delivery_due' => [ 'non_unique' => 1, 'columns' => ['state', 'updated_at', 'id'], ], - 'cloud_abuse_alert_delivery_terminal' => [ + 'cloud_network_alert_delivery_admin' => [ 'non_unique' => 1, - 'columns' => ['terminal_at', 'state', 'id'], + 'columns' => ['bound_admin_id', 'created_at', 'id'], ], - ], - 'cloud_abuse_projection_receipts' => [ - 'PRIMARY' => [ - 'non_unique' => 0, - 'columns' => ['sample_key'], - ], - 'cloud_abuse_projection_retention' => [ + 'cloud_network_alert_delivery_terminal' => [ 'non_unique' => 1, - 'columns' => ['projected_at', 'sample_key'], + 'columns' => ['terminal_at', 'state', 'id'], ], ], 'cloud_jobs' => [ @@ -2102,130 +2008,25 @@ final class CloudSetup ], ], ], - 'cloud_vm_network_observation_rollups' => [ - 'PRIMARY' => [ - 'non_unique' => 0, - 'columns' => [ - 'instance_id', 'interval_seconds', 'bucket_start', - ], - ], - 'cloud_network_rollup_owner' => [ - 'non_unique' => 1, - 'columns' => [ - 'user_id', 'interval_seconds', 'bucket_start', - 'instance_id', - ], - ], - 'cloud_network_rollup_retention' => [ - 'non_unique' => 1, - 'columns' => [ - 'interval_seconds', 'bucket_start', 'instance_id', - ], - ], - ], - 'cloud_vm_network_observation_samples' => [ - 'PRIMARY' => [ - 'non_unique' => 0, - 'columns' => ['sample_key'], - ], - 'cloud_network_observation_source' => [ - 'non_unique' => 0, - 'columns' => [ - 'host_id', 'collector_generation', 'window_id', 'vm_uuid', - ], - ], - 'cloud_network_observation_instance' => [ - 'non_unique' => 1, - 'columns' => ['instance_id', 'bucket_start'], - ], - 'cloud_network_observation_retention' => [ - 'non_unique' => 1, - 'columns' => ['bucket_start'], - ], - ], - 'cloud_vm_network_projection_outbox' => [ - 'PRIMARY' => [ - 'non_unique' => 0, - 'columns' => ['sample_key'], - ], - 'cloud_network_projection_due' => [ - 'non_unique' => 1, - 'columns' => [ - 'state', 'available_at', 'bucket_start', 'sample_key', - ], - ], - 'cloud_network_projection_instance' => [ - 'non_unique' => 1, - 'columns' => [ - 'instance_id', 'state', 'bucket_start', 'sample_key', - ], - ], - 'cloud_network_projection_lease' => [ - 'non_unique' => 1, - 'columns' => ['state', 'lease_until', 'sample_key'], - ], - 'cloud_network_projection_retention' => [ - 'non_unique' => 1, - 'columns' => ['state', 'terminal_at', 'sample_key'], - ], - ], - 'cloud_vm_network_monitor_cursors' => [ - 'PRIMARY' => [ - 'non_unique' => 0, - 'columns' => ['host_id'], - ], - 'cloud_network_monitor_cursor_health' => [ - 'non_unique' => 1, - 'columns' => [ - 'collection_state', 'last_success_at', 'host_id', - ], - ], - 'cloud_network_monitor_cursor_due' => [ - 'non_unique' => 1, - 'columns' => ['next_poll_at', 'poll_revision', 'host_id'], - ], - 'cloud_network_monitor_cursor_lease' => [ - 'non_unique' => 1, - 'columns' => ['poll_lease_until', 'host_id'], - ], - ], - 'cloud_vm_network_baselines' => [ - 'PRIMARY' => [ - 'non_unique' => 0, - 'columns' => ['instance_id'], - ], - 'cloud_network_baseline_owner' => [ - 'non_unique' => 1, - 'columns' => ['user_id', 'updated_at'], - ], - 'cloud_network_baseline_warmup' => [ - 'non_unique' => 1, - 'columns' => ['warmup_state', 'sample_count'], - ], - 'cloud_network_baseline_projection_lease' => [ - 'non_unique' => 1, - 'columns' => ['projection_lease_until', 'instance_id'], - ], - ], ]; private const REQUIRED_FOREIGN_KEYS = [ 'cloud_abuse_policy_rules' => [ 'fk_cloud_abuse_policy_rule_revision', ], - 'cloud_abuse_instance_states' => [ - 'fk_cloud_abuse_state_instance', + 'cloud_network_instance_ledger' => [ + 'fk_cloud_network_ledger_instance', ], - 'cloud_abuse_findings' => [ - 'fk_cloud_abuse_finding_instance', + 'cloud_network_daily_summaries' => [ + 'fk_cloud_network_daily_instance', ], - 'cloud_abuse_alert_event_items' => [ - 'fk_cloud_abuse_alert_item_event', + 'cloud_network_findings' => [ + 'fk_cloud_network_finding_instance', ], - 'cloud_abuse_alert_deliveries' => [ - 'fk_cloud_abuse_alert_delivery_event', + 'cloud_network_alert_events' => [ + 'fk_cloud_network_alert_finding', ], - 'cloud_abuse_projection_receipts' => [ - 'fk_cloud_abuse_projection_instance', + 'cloud_network_alert_deliveries' => [ + 'fk_cloud_network_alert_delivery_event', ], 'cloud_storage_replicas' => [ 'fk_cloud_storage_replica_artifact', @@ -2272,19 +2073,8 @@ final class CloudSetup ], 'cloud_jobs' => ['fk_cloud_job_user'], ]; - private const REQUIRED_LOG_FOREIGN_KEYS = [ - 'cloud_vm_network_projection_outbox' => [ - 'fk_cloud_network_projection_sample', - ], - ]; - private const REQUIRED_LOG_FOREIGN_KEY_RULES = [ - 'cloud_vm_network_projection_outbox' => [ - 'fk_cloud_network_projection_sample' => [ - 'delete_rule' => 'RESTRICT', - 'update_rule' => 'RESTRICT', - ], - ], - ]; + private const REQUIRED_LOG_FOREIGN_KEYS = []; + private const REQUIRED_LOG_FOREIGN_KEY_RULES = []; public static function initialize(): bool { @@ -2407,10 +2197,17 @@ final class CloudSetup '虚拟化数据库版本不兼容,请使用全新数据库' ); } + foreach (self::RETIRED_MAIN_TABLES as $retiredTable) { + if ($connection->getSchemaBuilder()->hasTable($retiredTable)) { + throw new \RuntimeException( + '主库包含已废弃的网络滥用表,请完成 clean-break 清理' + ); + } + } foreach (self::RETIRED_LOG_TABLES as $retiredTable) { if ($logConnection->getSchemaBuilder()->hasTable($retiredTable)) { throw new \RuntimeException( - '日志库包含已废弃的 Python Engine 表,请使用全新日志库' + '日志库包含已废弃的虚拟化表,请完成 clean-break 清理' ); } } @@ -2439,7 +2236,8 @@ final class CloudSetup ); } - foreach (self::MAIN_TABLES as $table) { + foreach (array_merge(self::MAIN_TABLES, self::RETIRED_MAIN_TABLES) + as $table) { if ($connection->getSchemaBuilder()->hasTable($table)) { throw new \RuntimeException( '检测到未标记的旧虚拟化数据库,不支持原地升级' diff --git a/web/app/Services/Cloud/CloudAbuseAlertEventCoordinator.php b/web/app/Services/Cloud/CloudAbuseAlertEventCoordinator.php deleted file mode 100644 index 0e91fb09..00000000 --- a/web/app/Services/Cloud/CloudAbuseAlertEventCoordinator.php +++ /dev/null @@ -1,301 +0,0 @@ - $claim - * @param list> $transitions - */ - public function record( - object $main, - array $claim, - array $transitions, - int $now - ): void { - if ($transitions === []) { - return; - } - - $items = self::canonicalTransitions($transitions); - foreach ($items as $item) { - if ($item['occurred_at'] !== (int) $claim['bucket_start']) { - throw new \LogicException( - 'Abuse alert transition bucket is inconsistent' - ); - } - } - $canonicalJson = self::json($items); - $digest = hash('sha256', $canonicalJson); - $eventKey = hash('sha256', implode("\n", [ - 'cloud-abuse-alert-event-v1', - (string) $claim['sample_key'], - (string) $claim['instance_id'], - (string) $claim['bucket_start'], - (string) $claim['rule_version'], - (string) $claim['policy_revision'], - (string) $claim['policy_hash'], - $digest, - ])); - $highestSeverity = self::highestSeverity($items); - $eventValues = [ - 'event_key' => $eventKey, - 'sample_key' => (string) $claim['sample_key'], - 'instance_id' => (int) $claim['instance_id'], - 'user_id' => (int) $claim['user_id'], - 'host_id' => (int) $claim['host_id'], - 'bucket_start' => (int) $claim['bucket_start'], - 'rule_version' => (int) $claim['rule_version'], - 'policy_revision' => (int) $claim['policy_revision'], - 'policy_hash' => (string) $claim['policy_hash'], - 'transition_digest' => $digest, - 'transition_count' => count($items), - 'highest_severity' => $highestSeverity, - 'created_at' => $now, - ]; - - $existing = $main->table('cloud_abuse_alert_events') - ->where('sample_key', $eventValues['sample_key']) - ->lockForUpdate()->first(); - if ($existing === null) { - $eventId = (int) $main->table('cloud_abuse_alert_events') - ->insertGetId($eventValues); - if ($eventId < 1) { - throw new \RuntimeException( - 'Abuse alert event insert did not return an id' - ); - } - foreach ($items as $item) { - $main->table('cloud_abuse_alert_event_items')->insert( - self::itemValues($eventId, $eventKey, $item, $now) - ); - } - return; - } - - self::assertEventReplay($existing, $eventValues); - $eventId = (int) $existing->id; - $storedItems = $main->table('cloud_abuse_alert_event_items') - ->where('event_id', $eventId)->orderBy('item_key')->get(); - if (count($storedItems) !== count($items)) { - throw new \RuntimeException( - 'Abuse alert event item count conflicts with replay' - ); - } - $byKey = []; - foreach ($storedItems as $stored) { - $byKey[(string) $stored->item_key] = $stored; - } - foreach ($items as $item) { - $expected = self::itemValues( - $eventId, - $eventKey, - $item, - $now - ); - $stored = $byKey[$expected['item_key']] ?? null; - if ($stored === null) { - throw new \RuntimeException( - 'Abuse alert event item is missing during replay' - ); - } - self::assertItemReplay($stored, $expected); - } - } - - /** - * @param list> $transitions - * @return list> - */ - private static function canonicalTransitions(array $transitions): array - { - $canonical = []; - foreach ($transitions as $transition) { - $type = (string) ($transition['transition_type'] ?? ''); - $from = (string) ($transition['from_severity'] ?? ''); - $to = (string) ($transition['to_severity'] ?? ''); - $reason = (string) ($transition['resolution_reason'] ?? ''); - $validShape = ($type === 'opened' && $from === 'none' - && in_array($to, ['suspicious', 'high'], true) - && $reason === '') - || ($type === 'escalated' && $from === 'suspicious' - && $to === 'high' && $reason === '') - || ($type === 'resolved' - && in_array($from, ['suspicious', 'high'], true) - && $to === 'none' && $reason === 'evidence_clear'); - $findingKey = (string) ($transition['finding_key'] ?? ''); - $direction = (string) ($transition['direction'] ?? ''); - $family = (string) ($transition['ip_family'] ?? ''); - $ruleCode = (string) ($transition['rule_code'] ?? ''); - $generation = (int) ($transition['occurrence_generation'] ?? 0); - $revision = (int) ($transition['transition_revision'] ?? 0); - $occurredAt = (int) ($transition['occurred_at'] ?? 0); - if (!$validShape - || preg_match('/^[a-f0-9]{64}$/D', $findingKey) !== 1 - || !in_array($direction, ['inbound', 'outbound'], true) - || !in_array($family, ['ipv4', 'ipv6', 'mixed'], true) - || preg_match('/^[a-z][a-z0-9_]{0,47}$/D', - $ruleCode) !== 1 - || $generation < 1 || $revision < 1 || $occurredAt < 1 - || !self::validEvidence($transition['evidence'] ?? null)) { - throw new \LogicException( - 'Abuse alert transition is invalid' - ); - } - $canonical[] = [ - 'finding_key' => $findingKey, - 'occurrence_generation' => $generation, - 'transition_revision' => $revision, - 'rule_code' => $ruleCode, - 'direction' => $direction, - 'ip_family' => $family, - 'transition_type' => $type, - 'from_severity' => $from, - 'to_severity' => $to, - 'evidence' => self::canonicalEvidence( - $transition['evidence'] - ), - 'resolution_reason' => $reason, - 'occurred_at' => $occurredAt, - ]; - } - usort($canonical, static function (array $left, array $right): int { - return [ - $left['finding_key'], - $left['occurrence_generation'], - $left['transition_revision'], - $left['transition_type'], - ] <=> [ - $right['finding_key'], - $right['occurrence_generation'], - $right['transition_revision'], - $right['transition_type'], - ]; - }); - return $canonical; - } - - /** @param list> $items */ - private static function highestSeverity(array $items): string - { - $highest = 'suspicious'; - foreach ($items as $item) { - if ($item['from_severity'] === 'high' - || $item['to_severity'] === 'high') { - $highest = 'high'; - break; - } - } - return $highest; - } - - /** - * @param array $item - * @return array - */ - private static function itemValues( - int $eventId, - string $eventKey, - array $item, - int $now - ): array { - $itemKey = hash('sha256', implode("\n", [ - 'cloud-abuse-alert-item-v1', - $eventKey, - $item['finding_key'], - (string) $item['occurrence_generation'], - (string) $item['transition_revision'], - $item['transition_type'], - ])); - return [ - 'event_id' => $eventId, - 'item_key' => $itemKey, - 'finding_key' => $item['finding_key'], - 'occurrence_generation' => $item['occurrence_generation'], - 'transition_revision' => $item['transition_revision'], - 'rule_code' => $item['rule_code'], - 'direction' => $item['direction'], - 'ip_family' => $item['ip_family'], - 'transition_type' => $item['transition_type'], - 'from_severity' => $item['from_severity'], - 'to_severity' => $item['to_severity'], - 'evidence_json' => self::json($item['evidence']), - 'resolution_reason' => $item['resolution_reason'], - 'occurred_at' => $item['occurred_at'], - 'created_at' => $now, - ]; - } - - /** @param array $expected */ - private static function assertEventReplay( - object $stored, - array $expected - ): void { - foreach ([ - 'event_key', 'sample_key', 'instance_id', 'user_id', 'host_id', - 'bucket_start', 'rule_version', 'policy_revision', 'policy_hash', - 'transition_digest', 'transition_count', 'highest_severity', - ] as $column) { - if ((string) $stored->{$column} !== (string) $expected[$column]) { - throw new \RuntimeException( - 'Abuse alert event conflicts with replay' - ); - } - } - } - - /** @param array $expected */ - private static function assertItemReplay( - object $stored, - array $expected - ): void { - foreach ([ - 'event_id', 'item_key', 'finding_key', 'occurrence_generation', - 'transition_revision', 'rule_code', 'direction', 'ip_family', - 'transition_type', 'from_severity', 'to_severity', - 'evidence_json', 'resolution_reason', 'occurred_at', - ] as $column) { - if ((string) $stored->{$column} !== (string) $expected[$column]) { - throw new \RuntimeException( - 'Abuse alert event item conflicts with replay' - ); - } - } - } - - private static function validEvidence(mixed $value): bool - { - if (!is_array($value)) { - return false; - } - foreach ($value as $key => $item) { - if (!is_string($key) - || preg_match('/^[a-z][a-z0-9_]{0,63}$/D', $key) !== 1 - || !is_int($item) || $item < 0) { - return false; - } - } - return true; - } - - /** @param array $value @return array */ - private static function canonicalEvidence(array $value): array - { - ksort($value, SORT_STRING); - return $value; - } - - private static function json(mixed $value): string - { - return json_encode( - $value, - JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE - | JSON_PRESERVE_ZERO_FRACTION | JSON_THROW_ON_ERROR - ); - } -} diff --git a/web/app/Services/Cloud/CloudAbuseObservationRollupStore.php b/web/app/Services/Cloud/CloudAbuseObservationRollupStore.php deleted file mode 100644 index 84cd62f0..00000000 --- a/web/app/Services/Cloud/CloudAbuseObservationRollupStore.php +++ /dev/null @@ -1,354 +0,0 @@ - 1 << 0, - 'cold_start' => 1 << 1, - 'conntrack_unavailable' => 1 << 2, - 'counter_reset' => 1 << 3, - 'flow_limit_reached' => 1 << 4, - 'incomplete' => 1 << 5, - 'interface_unavailable' => 1 << 6, - 'local_port_cardinality_saturated' => 1 << 7, - 'mapping_changed' => 1 << 8, - 'overflow' => 1 << 9, - 'packet_accounting_unavailable' => 1 << 10, - 'relation_cardinality_saturated' => 1 << 11, - 'remote_ip_cardinality_saturated' => 1 << 12, - 'remote_port_cardinality_saturated' => 1 << 13, - 'sampling_gap' => 1 << 14, - 'panel_ring_gap' => 1 << 15, - ]; - - /** - * @param array $metrics canonical observation metrics - * @param list $qualityFlags canonical quality flags - */ - public function record( - object $log, - int $instanceId, - int $userId, - int $hostId, - int $bucketStart, - bool $complete, - array $metrics, - array $qualityFlags, - int $now - ): void { - if ($instanceId < 1 || $userId < 1 || $hostId < 1 - || $bucketStart < 1 || $bucketStart % 60 !== 0 || $now < 1) { - throw new \InvalidArgumentException( - 'Abuse observation rollup identity is invalid' - ); - } - $summary = self::summary($metrics, $qualityFlags); - foreach ([self::FIVE_MINUTES, self::ONE_HOUR] as $interval) { - $this->recordInterval( - $log, - $instanceId, - $userId, - $hostId, - $bucketStart, - $interval, - $complete, - $summary, - $now - ); - } - } - - /** @param array $summary */ - private function recordInterval( - object $log, - int $instanceId, - int $userId, - int $hostId, - int $sampleStart, - int $interval, - bool $complete, - array $summary, - int $now - ): void { - $bucketStart = intdiv($sampleStart, $interval) * $interval; - $query = $log->table('cloud_vm_network_observation_rollups') - ->where('instance_id', $instanceId) - ->where('interval_seconds', $interval) - ->where('bucket_start', $bucketStart); - $existing = $query->lockForUpdate()->first(); - if ($existing === null) { - $log->table('cloud_vm_network_observation_rollups')->insert([ - 'instance_id' => $instanceId, - 'user_id' => $userId, - 'host_id' => $hostId, - 'interval_seconds' => $interval, - 'bucket_start' => $bucketStart, - 'sample_count' => 1, - 'complete_count' => $complete ? 1 : 0, - 'quality_mask' => $summary['quality_mask'], - 'rx_bytes_min' => $summary['rx_bytes'], - 'rx_bytes_max' => $summary['rx_bytes'], - 'tx_bytes_min' => $summary['tx_bytes'], - 'tx_bytes_max' => $summary['tx_bytes'], - 'rx_packets_min' => $summary['rx_packets'], - 'rx_packets_max' => $summary['rx_packets'], - 'tx_packets_min' => $summary['tx_packets'], - 'tx_packets_max' => $summary['tx_packets'], - 'tcp_active_peak' => $summary['tcp_active'], - 'tcp_new_total' => $summary['tcp_new'], - 'tcp_syn_total' => $summary['tcp_syn'], - 'tcp_unreplied_total' => $summary['tcp_unreplied'], - 'udp_active_peak' => $summary['udp_active'], - 'udp_new_total' => $summary['udp_new'], - 'peer_ip_cardinality_peak' => $summary['remote_ips'], - 'peer_port_cardinality_peak' => $summary['remote_ports'], - 'first_sample_at' => $sampleStart, - 'last_sample_at' => $sampleStart, - 'created_at' => $now, - 'updated_at' => $now, - ]); - return; - } - if ((int) $existing->user_id !== $userId) { - throw new \RuntimeException( - 'Abuse observation rollup owner changed inside one bucket' - ); - } - - $query->update([ - // Host may legitimately change during maintenance migration. - 'host_id' => $hostId, - 'sample_count' => (int) $existing->sample_count + 1, - 'complete_count' => (int) $existing->complete_count - + ($complete ? 1 : 0), - 'quality_mask' => ((int) $existing->quality_mask) - | $summary['quality_mask'], - 'rx_bytes_min' => self::minimum( - $existing->rx_bytes_min, - $summary['rx_bytes'] - ), - 'rx_bytes_max' => self::maximum( - $existing->rx_bytes_max, - $summary['rx_bytes'] - ), - 'tx_bytes_min' => self::minimum( - $existing->tx_bytes_min, - $summary['tx_bytes'] - ), - 'tx_bytes_max' => self::maximum( - $existing->tx_bytes_max, - $summary['tx_bytes'] - ), - 'rx_packets_min' => self::minimum( - $existing->rx_packets_min, - $summary['rx_packets'] - ), - 'rx_packets_max' => self::maximum( - $existing->rx_packets_max, - $summary['rx_packets'] - ), - 'tx_packets_min' => self::minimum( - $existing->tx_packets_min, - $summary['tx_packets'] - ), - 'tx_packets_max' => self::maximum( - $existing->tx_packets_max, - $summary['tx_packets'] - ), - 'tcp_active_peak' => max( - (int) $existing->tcp_active_peak, - $summary['tcp_active'] - ), - 'tcp_new_total' => self::add( - (int) $existing->tcp_new_total, - $summary['tcp_new'] - ), - 'tcp_syn_total' => self::add( - (int) $existing->tcp_syn_total, - $summary['tcp_syn'] - ), - 'tcp_unreplied_total' => self::add( - (int) $existing->tcp_unreplied_total, - $summary['tcp_unreplied'] - ), - 'udp_active_peak' => max( - (int) $existing->udp_active_peak, - $summary['udp_active'] - ), - 'udp_new_total' => self::add( - (int) $existing->udp_new_total, - $summary['udp_new'] - ), - 'peer_ip_cardinality_peak' => max( - (int) $existing->peer_ip_cardinality_peak, - $summary['remote_ips'] - ), - 'peer_port_cardinality_peak' => max( - (int) $existing->peer_port_cardinality_peak, - $summary['remote_ports'] - ), - 'first_sample_at' => min( - (int) $existing->first_sample_at, - $sampleStart - ), - 'last_sample_at' => max( - (int) $existing->last_sample_at, - $sampleStart - ), - 'updated_at' => $now, - ]); - } - - /** @return array */ - private static function summary( - array $metrics, - array $qualityFlags - ): array { - $interface = is_array($metrics['interface'] ?? null) - ? $metrics['interface'] : []; - $result = [ - 'quality_mask' => self::qualityMask($qualityFlags), - 'rx_bytes' => self::counter($interface['rx_bytes'] ?? null), - 'tx_bytes' => self::counter($interface['tx_bytes'] ?? null), - 'rx_packets' => self::counter($interface['rx_packets'] ?? null), - 'tx_packets' => self::counter($interface['tx_packets'] ?? null), - 'tcp_active' => 0, - 'tcp_new' => 0, - 'tcp_syn' => 0, - 'tcp_unreplied' => 0, - 'udp_active' => 0, - 'udp_new' => 0, - 'remote_ips' => 0, - 'remote_ports' => 0, - ]; - foreach (['ipv4', 'ipv6'] as $family) { - $families = $metrics[$family] ?? null; - if (!is_array($families)) { - continue; - } - foreach (['inbound', 'outbound'] as $direction) { - $flow = $families[$direction] ?? null; - if (!is_array($flow)) { - continue; - } - $result['tcp_active'] = self::add( - $result['tcp_active'], - self::requiredCounter($flow, 'tcp_active_flows') - ); - $result['tcp_new'] = self::add( - $result['tcp_new'], - self::requiredCounter($flow, 'tcp_new_flows') - ); - $result['tcp_syn'] = self::add( - $result['tcp_syn'], - self::requiredCounter($flow, 'tcp_syn_flows') - ); - $result['tcp_unreplied'] = self::add( - $result['tcp_unreplied'], - self::requiredCounter($flow, 'tcp_unreplied_flows') - ); - $result['udp_active'] = self::add( - $result['udp_active'], - self::requiredCounter($flow, 'udp_active_flows') - ); - $result['udp_new'] = self::add( - $result['udp_new'], - self::requiredCounter($flow, 'udp_new_flows') - ); - $result['remote_ips'] = self::add( - $result['remote_ips'], - self::requiredCounter($flow, 'unique_remote_ips') - ); - $result['remote_ports'] = self::add( - $result['remote_ports'], - self::requiredCounter($flow, 'unique_remote_ports') - ); - } - } - return $result; - } - - /** @param list $flags */ - private static function qualityMask(array $flags): int - { - $mask = 0; - foreach ($flags as $flag) { - if (!is_string($flag) || !isset(self::QUALITY_BITS[$flag])) { - throw new \RuntimeException( - 'Abuse observation rollup quality flag is invalid' - ); - } - $mask |= self::QUALITY_BITS[$flag]; - } - return $mask; - } - - private static function requiredCounter(array $values, string $key): int - { - $value = self::counter($values[$key] ?? null); - if ($value === null) { - throw new \RuntimeException( - 'Abuse observation rollup flow counter is missing' - ); - } - return $value; - } - - private static function counter(mixed $value): ?int - { - if ($value === null) { - return null; - } - if (!is_int($value) || $value < 0) { - throw new \RuntimeException( - 'Abuse observation rollup counter is invalid' - ); - } - return $value; - } - - private static function minimum(mixed $left, ?int $right): ?int - { - if ($left === null) { - return $right; - } - if ($right === null) { - return (int) $left; - } - return min((int) $left, $right); - } - - private static function maximum(mixed $left, ?int $right): ?int - { - if ($left === null) { - return $right; - } - if ($right === null) { - return (int) $left; - } - return max((int) $left, $right); - } - - private static function add(int $left, int $right): int - { - if ($right > PHP_INT_MAX - $left) { - throw new \OverflowException( - 'Abuse observation rollup counter overflowed' - ); - } - return $left + $right; - } -} diff --git a/web/app/Services/Cloud/CloudAbuseProjectionService.php b/web/app/Services/Cloud/CloudAbuseProjectionService.php deleted file mode 100644 index 462adb53..00000000 --- a/web/app/Services/Cloud/CloudAbuseProjectionService.php +++ /dev/null @@ -1,1539 +0,0 @@ - */ - private array $errors = []; - - /** - * The optional hook is a test seam for the only unavoidable cross-database - * crash window. Production callers must leave it null. - */ - public function __construct( - ?callable $clock = null, - ?callable $afterMainCommit = null, - ?callable $policySnapshot = null, - ?CloudAbuseAlertEventCoordinator $alertEvents = null - ) { - $this->clock = \Closure::fromCallable( - $clock ?? static fn (): int => time() - ); - $this->afterMainCommit = $afterMainCommit === null - ? null : \Closure::fromCallable($afterMainCommit); - $this->policySnapshot = \Closure::fromCallable($policySnapshot - ?? static fn (int $revision, string $hash): - CloudAbusePolicySnapshot => (new CloudAbusePolicyService()) - ->snapshotByIdentity($revision, $hash)); - $this->alertEvents = $alertEvents - ?? new CloudAbuseAlertEventCoordinator(); - } - - /** Project at most one ordered sample. */ - public function tick(): bool - { - $this->errors = []; - $now = ($this->clock)(); - if (!is_int($now) || $now < 1) { - throw new \LogicException('Abuse projection clock is invalid'); - } - $claim = $this->claim($now); - if ($claim === null) { - return false; - } - if (isset($claim['claim_error_code'])) { - $this->errors[] = (string) $claim['claim_error_code']; - return true; - } - try { - $claim = self::hydrateClaim($claim); - $claim['policy_snapshot'] = $this->loadPolicySnapshot($claim); - $decision = $this->projectMain($claim, $now); - if ($this->afterMainCommit !== null) { - ($this->afterMainCommit)($claim['sample_key']); - } - if ($decision['state'] === 'discarded') { - $reason = self::boundedCode( - (string) $decision['reason'], - 'projection_discarded' - ); - $this->discard( - $claim, - $reason, - $now - ); - $this->errors[] = $reason; - } else { - $this->finalize($claim, $decision['baseline'], $now); - } - } catch (\Throwable $error) { - $this->errors[] = $this->retryOrDiscard( - $claim, - $error, - $now - ); - } - return true; - } - - /** @return list */ - public function observationErrors(): array - { - return $this->errors; - } - - /** @return null|array */ - private function claim(int $now): ?array - { - $log = User::getConnectionResolver()->connection('log'); - return $log->transaction(function () use ($log, $now): ?array { - $candidate = $log->table( - 'cloud_vm_network_projection_outbox as candidate' - )->where(static function ($due) use ($now): void { - $due->where(static function ($pending) use ($now): void { - $pending->where('candidate.state', 'pending') - ->where('candidate.available_at', '<=', $now); - })->orWhere(static function ($expired) use ($now): void { - $expired->where('candidate.state', 'leased') - ->where('candidate.lease_until', '<=', $now); - }); - })->whereNotExists(static function ($earlier): void { - $earlier->selectRaw('1')->from( - 'cloud_vm_network_projection_outbox as prior' - )->whereColumn( - 'prior.instance_id', - 'candidate.instance_id' - )->whereIn('prior.state', ['pending', 'leased']) - ->where(static function ($order): void { - $order->whereColumn( - 'prior.bucket_start', - '<', - 'candidate.bucket_start' - )->orWhere(static function ($same): void { - $same->whereColumn( - 'prior.bucket_start', - 'candidate.bucket_start' - )->whereColumn( - 'prior.sample_key', - '<', - 'candidate.sample_key' - ); - }); - }); - })->orderBy('candidate.bucket_start') - ->orderBy('candidate.sample_key')->first(); - if ($candidate === null) { - return null; - } - $outbox = $log->table('cloud_vm_network_projection_outbox') - ->where('sample_key', (string) $candidate->sample_key) - ->lockForUpdate()->first(); - if ($outbox === null - || !in_array((string) $outbox->state, - ['pending', 'leased'], true) - || ((string) $outbox->state === 'pending' - && (int) $outbox->available_at > $now) - || ((string) $outbox->state === 'leased' - && (int) $outbox->lease_until > $now)) { - return null; - } - if (!self::validPolicyIdentity( - (int) $outbox->rule_version, - (int) $outbox->policy_revision, - (string) $outbox->policy_hash - )) { - $reported = $this->recordClaimFailure( - $log, - $outbox, - 'projection_policy_identity_invalid', - $now - ); - return ['claim_error_code' => $reported]; - } - $sample = $log->table( - 'cloud_vm_network_observation_samples' - )->where('sample_key', (string) $outbox->sample_key) - ->lockForUpdate()->first(); - if ($sample === null - || (int) $sample->instance_id !== (int) $outbox->instance_id - || (int) $sample->bucket_start !== (int) $outbox->bucket_start - || !hash_equals( - (string) $sample->placement_key, - (string) $outbox->placement_key - )) { - $code = 'projection_sample_ledger_inconsistent'; - $reported = $this->recordClaimFailure( - $log, - $outbox, - $code, - $now - ); - return ['claim_error_code' => $reported]; - } - $baseline = $log->table('cloud_vm_network_baselines') - ->where('instance_id', (int) $sample->instance_id) - ->lockForUpdate()->first(); - if ($baseline === null) { - $log->table('cloud_vm_network_baselines')->insert([ - 'instance_id' => (int) $sample->instance_id, - 'user_id' => (int) $sample->user_id, - 'rule_version' => (int) $outbox->rule_version, - 'policy_revision' => (int) $outbox->policy_revision, - 'policy_hash' => (string) $outbox->policy_hash, - 'sample_count' => 0, - 'warmup_state' => 'warming', - 'metrics_json' => self::json( - CloudAbuseDetector::initialBaseline() - ), - 'last_projection_key' => null, - 'last_bucket_start' => null, - 'projection_lease_token' => null, - 'projection_lease_until' => 0, - 'revision' => 0, - 'created_at' => $now, - 'updated_at' => $now, - ]); - $baseline = $log->table('cloud_vm_network_baselines') - ->where('instance_id', (int) $sample->instance_id) - ->lockForUpdate()->first(); - } - if ($baseline === null - || (int) $baseline->user_id !== (int) $sample->user_id - || !self::validPolicyIdentity( - (int) $baseline->rule_version, - (int) $baseline->policy_revision, - (string) $baseline->policy_hash - )) { - $code = 'projection_baseline_identity_inconsistent'; - $reported = $this->recordClaimFailure( - $log, - $outbox, - $code, - $now - ); - return ['claim_error_code' => $reported]; - } - if ($baseline->projection_lease_token !== null - && (int) $baseline->projection_lease_until > $now) { - return null; - } - $token = bin2hex(random_bytes(16)); - $attempts = (int) $outbox->attempts + 1; - $outboxUpdated = $log->table( - 'cloud_vm_network_projection_outbox' - )->where('sample_key', (string) $outbox->sample_key) - ->where('state', (string) $outbox->state) - ->update([ - 'state' => 'leased', - 'attempts' => $attempts, - 'lease_token' => $token, - 'lease_until' => $now + self::LEASE_SECONDS, - 'last_error_code' => '', - 'updated_at' => $now, - ]); - $baselineUpdated = $log->table( - 'cloud_vm_network_baselines' - )->where('instance_id', (int) $sample->instance_id) - ->where('revision', (int) $baseline->revision) - ->update([ - 'projection_lease_token' => $token, - 'projection_lease_until' => $now - + self::LEASE_SECONDS, - 'updated_at' => $now, - ]); - if ($outboxUpdated !== 1 || $baselineUpdated !== 1) { - throw new \RuntimeException( - 'Abuse projection lease compare-and-swap failed' - ); - } - $complete = (int) $sample->complete === 1; - return [ - 'sample_key' => (string) $sample->sample_key, - 'instance_id' => (int) $sample->instance_id, - 'user_id' => (int) $sample->user_id, - 'host_id' => (int) $sample->host_id, - 'placement_key' => (string) $sample->placement_key, - 'bucket_start' => (int) $sample->bucket_start, - 'complete' => $complete, - 'metrics_json' => (string) $sample->metrics_json, - 'quality_flags_json' => - (string) $sample->quality_flags_json, - 'baseline_json' => (string) $baseline->metrics_json, - 'rule_version' => (int) $outbox->rule_version, - 'policy_revision' => (int) $outbox->policy_revision, - 'policy_hash' => (string) $outbox->policy_hash, - 'baseline_rule_version' => - (int) $baseline->rule_version, - 'baseline_policy_revision' => - (int) $baseline->policy_revision, - 'baseline_policy_hash' => - (string) $baseline->policy_hash, - 'baseline_revision' => (int) $baseline->revision, - 'last_projection_key' => $baseline->last_projection_key, - 'last_bucket_start' => $baseline->last_bucket_start === null - ? null : (int) $baseline->last_bucket_start, - 'attempts' => $attempts, - 'lease_token' => $token, - ]; - }, 3); - } - - private function recordClaimFailure( - object $log, - object $outbox, - string $code, - int $now - ): string { - $code = self::boundedCode($code, 'projection_claim_failed'); - $attempts = (int) $outbox->attempts + 1; - $terminal = $attempts >= self::MAX_ATTEMPTS; - if ($terminal) { - $receiptState = $this->receiptState([ - 'sample_key' => (string) $outbox->sample_key, - 'instance_id' => (int) $outbox->instance_id, - 'bucket_start' => (int) $outbox->bucket_start, - 'rule_version' => (int) $outbox->rule_version, - 'policy_revision' => (int) $outbox->policy_revision, - 'policy_hash' => (string) $outbox->policy_hash, - ]); - if ($receiptState === 'matching') { - $baseline = $log->table('cloud_vm_network_baselines') - ->where('instance_id', (int) $outbox->instance_id) - ->lockForUpdate()->first(); - if ($baseline !== null - && $baseline->projection_lease_token !== null - && (int) $baseline->projection_lease_until > $now) { - $terminal = false; - $code = 'projection_receipt_finalize_pending'; - } else { - try { - $snapshot = $this->loadPolicySnapshot([ - 'rule_version' => (int) $outbox->rule_version, - 'policy_revision' => - (int) $outbox->policy_revision, - 'policy_hash' => (string) $outbox->policy_hash, - ]); - $seed = $this->currentOwnerBaselineSeed( - (int) $outbox->instance_id, - $snapshot - ); - } catch (\Throwable) { - $seed = false; - } - if ($seed === false) { - $terminal = false; - $code = 'projection_baseline_seed_pending'; - } else { - // The main read model already committed this sample. - // Use its current open findings to reset statistics - // without losing the evidence required for resolution. - if ($seed === null) { - $deleted = $log->table( - 'cloud_vm_network_baselines' - )->where('instance_id', - (int) $outbox->instance_id)->delete(); - if ($baseline !== null && $deleted !== 1) { - throw new \RuntimeException( - 'Abuse baseline delete CAS failed' - ); - } - } else { - $seedBaseline = $seed['baseline']; - $seedBaseline['last_bucket_start'] = - (int) $outbox->bucket_start; - $values = [ - 'user_id' => $seed['user_id'], - 'rule_version' => - (int) $outbox->rule_version, - 'policy_revision' => - (int) $outbox->policy_revision, - 'policy_hash' => - (string) $outbox->policy_hash, - 'sample_count' => 0, - 'warmup_state' => 'warming', - 'metrics_json' => self::json($seedBaseline), - 'last_projection_key' => - (string) $outbox->sample_key, - 'last_bucket_start' => - (int) $outbox->bucket_start, - 'projection_lease_token' => null, - 'projection_lease_until' => 0, - 'updated_at' => $now, - ]; - if ($baseline === null) { - $log->table( - 'cloud_vm_network_baselines' - )->insert([ - 'instance_id' => - (int) $outbox->instance_id, - 'revision' => 0, - 'created_at' => $now, - ] + $values); - } else { - $values['revision'] = - (int) $baseline->revision + 1; - $baselineUpdated = $log->table( - 'cloud_vm_network_baselines' - )->where('instance_id', - (int) $outbox->instance_id) - ->where('revision', - (int) $baseline->revision) - ->update($values); - if ($baselineUpdated !== 1) { - throw new \RuntimeException( - 'Abuse baseline reset CAS failed' - ); - } - } - } - $updated = $log->table( - 'cloud_vm_network_projection_outbox' - )->where('sample_key', (string) $outbox->sample_key) - ->where('state', (string) $outbox->state) - ->where('attempts', (int) $outbox->attempts) - ->update([ - 'state' => 'done', - 'attempts' => min( - $attempts, - self::MAX_ATTEMPTS - ), - 'lease_token' => null, - 'lease_until' => 0, - 'last_error_code' => '', - 'projected_at' => $now, - 'terminal_at' => $now, - 'discard_reason_code' => '', - 'updated_at' => $now, - ]); - if ($updated !== 1) { - throw new \RuntimeException( - 'Abuse receipt claim recovery CAS failed' - ); - } - return 'projection_receipt_baseline_reset'; - } - } - } - if ($receiptState === 'conflict') { - $terminal = false; - $code = 'projection_receipt_conflict'; - } elseif ($receiptState === 'unknown') { - $terminal = false; - $code = 'projection_receipt_check_failed'; - } - } - $values = $terminal ? [ - 'state' => 'discarded', - 'attempts' => $attempts, - 'lease_token' => null, - 'lease_until' => 0, - 'last_error_code' => $code, - 'projected_at' => null, - 'terminal_at' => $now, - 'discard_reason_code' => 'projection_retry_exhausted', - 'updated_at' => $now, - ] : [ - 'state' => 'pending', - 'attempts' => min($attempts, self::MAX_ATTEMPTS), - 'available_at' => $now + self::retryDelay($attempts), - 'lease_token' => null, - 'lease_until' => 0, - 'last_error_code' => $code, - 'projected_at' => null, - 'terminal_at' => null, - 'discard_reason_code' => '', - 'updated_at' => $now, - ]; - $updated = $log->table('cloud_vm_network_projection_outbox') - ->where('sample_key', (string) $outbox->sample_key) - ->where('state', (string) $outbox->state) - ->where('attempts', (int) $outbox->attempts) - ->update($values); - if ($updated !== 1) { - throw new \RuntimeException( - 'Abuse projection claim failure compare-and-swap failed' - ); - } - return $terminal ? 'projection_retry_exhausted' : $code; - } - - /** @param array $claim @return array */ - private static function hydrateClaim(array $claim): array - { - $claim['metrics'] = self::canonicalMetrics(self::decodeObject( - (string) $claim['metrics_json'], - 'sample metrics' - )); - $claim['quality_flags'] = self::canonicalQuality(self::decodeList( - (string) $claim['quality_flags_json'], - 'sample quality flags' - )); - $claim['baseline'] = self::decodeObject( - (string) $claim['baseline_json'], - 'network baseline' - ); - unset( - $claim['metrics_json'], - $claim['quality_flags_json'], - $claim['baseline_json'] - ); - if ($claim['complete'] === in_array( - 'incomplete', - $claim['quality_flags'], - true - )) { - throw new \RuntimeException( - 'Sample completeness and quality evidence disagree' - ); - } - self::assertStoredSampleSemantics( - $claim['metrics'], - $claim['quality_flags'], - $claim['complete'] - ); - return $claim; - } - - /** @param array $identity */ - private function loadPolicySnapshot( - array $identity - ): CloudAbusePolicySnapshot { - $ruleVersion = (int) ($identity['rule_version'] ?? 0); - $revision = (int) ($identity['policy_revision'] ?? 0); - $hash = (string) ($identity['policy_hash'] ?? ''); - if (!self::validPolicyIdentity($ruleVersion, $revision, $hash)) { - throw new \RuntimeException( - 'Abuse projection policy identity is invalid' - ); - } - $snapshot = ($this->policySnapshot)($revision, $hash); - if (!$snapshot instanceof CloudAbusePolicySnapshot - || $snapshot->revision !== $revision - || !hash_equals($snapshot->policyHash, $hash) - || $snapshot->policy->engineVersion() !== $ruleVersion) { - throw new \RuntimeException( - 'Abuse projection policy snapshot identity conflict' - ); - } - return $snapshot; - } - - private static function validPolicyIdentity( - int $ruleVersion, - int $revision, - string $hash - ): bool { - return $ruleVersion > 0 - && $revision > 0 - && preg_match('/^[a-f0-9]{64}$/D', $hash) === 1; - } - - /** - * Recheck the semantic invariants that made the stored sample canonical. - * The log database is a durable boundary: corrupt rows must never be - * promoted to healthy evidence merely because their JSON shape is valid. - * - * @param array $metrics - * @param array $quality - */ - private static function assertStoredSampleSemantics( - array $metrics, - array $quality, - bool $complete - ): void { - if ($metrics['ipv4'] === null && $metrics['ipv6'] === null) { - throw new \RuntimeException( - 'Sample has no observable address family' - ); - } - $interfaceUnavailable = in_array( - 'interface_unavailable', - $quality, - true - ); - $hasMissingCounter = in_array( - null, - $metrics['interface'], - true - ); - if ($hasMissingCounter !== $interfaceUnavailable) { - throw new \RuntimeException( - 'Sample interface evidence is inconsistent' - ); - } - if (in_array('panel_ring_gap', $quality, true) - && $complete) { - throw new \RuntimeException( - 'Panel ring gap sample cannot be complete' - ); - } - if ($complete && array_intersect($quality, [ - 'clock_regression', 'cold_start', 'conntrack_unavailable', - 'counter_reset', 'flow_limit_reached', - 'interface_unavailable', - 'local_port_cardinality_saturated', 'mapping_changed', - 'overflow', 'relation_cardinality_saturated', - 'remote_ip_cardinality_saturated', - 'remote_port_cardinality_saturated', 'sampling_gap', - ]) !== []) { - throw new \RuntimeException( - 'Complete sample contains risk-unsafe evidence' - ); - } - } - - /** - * @param array $claim - * @return array{state:string,reason?:string,baseline?:array} - */ - private function projectMain(array $claim, int $now): array - { - if ($claim['last_bucket_start'] !== null - && $claim['bucket_start'] <= $claim['last_bucket_start']) { - if (is_string($claim['last_projection_key']) - && hash_equals( - $claim['last_projection_key'], - $claim['sample_key'] - )) { - return [ - 'state' => 'done', - 'baseline' => $claim['baseline'], - ]; - } - return ['state' => 'discarded', - 'reason' => 'out_of_order_sample']; - } - $snapshot = $claim['policy_snapshot'] ?? null; - if (!$snapshot instanceof CloudAbusePolicySnapshot) { - throw new \RuntimeException( - 'Abuse projection policy snapshot is missing' - ); - } - $policy = $snapshot->policy; - $policyChanged = $claim['baseline_rule_version'] - !== $claim['rule_version'] - || !hash_equals( - $claim['baseline_policy_hash'], - $claim['policy_hash'] - ); - $base = $policyChanged - ? self::rebaseBaselineForPolicy($claim['baseline'], $policy) - : $claim['baseline']; - $reset = in_array( - 'panel_ring_gap', - $claim['quality_flags'], - true - ); - $base = $reset - ? CloudAbuseDetector::resetBaselineFromPrevious( - $base, - $policy - ) : $base; - $detected = CloudAbuseDetector::evaluate([ - 'bucket_start' => $claim['bucket_start'], - 'complete' => $claim['complete'], - 'quality_flags' => $claim['quality_flags'], - 'metrics' => $claim['metrics'], - ], $base, $policy); - $baseline = $detected['baseline']; - $evaluation = $detected['evaluation']; - if (!$claim['complete']) { - // Do not derive a per-second rate across an unobserved minute. - // Statistical history and finding streaks stay frozen, while the - // next complete sample establishes a fresh interface baseline. - $baseline['last_interface'] = null; - } - - $main = User::getConnectionResolver()->connection('default'); - return $main->transaction(function () use ( - $main, - $claim, - $baseline, - $evaluation, - $policyChanged, - $now - ): array { - // Account deletion locks the tenant row before deleting any - // instance-owned data. Take the same lock first so a projection - // cannot insert a durable event after deletion has swept that - // user's alert rows. - $owner = $main->table('user') - ->where('id', $claim['user_id']) - ->lockForUpdate()->first(['id']); - if ($owner === null) { - return ['state' => 'discarded', - 'reason' => 'instance_owner_missing']; - } - $instance = $main->table('cloud_instances') - ->where('id', $claim['instance_id']) - ->lockForUpdate()->first(); - if ($instance === null) { - return ['state' => 'discarded', - 'reason' => 'instance_deleted']; - } - if ((int) $instance->user_id !== $claim['user_id']) { - $this->retireOwnerReadModel( - $main, - $claim['instance_id'], - $claim['user_id'], - $now - ); - return ['state' => 'discarded', - 'reason' => 'instance_owner_changed']; - } - $receipt = $main->table('cloud_abuse_projection_receipts') - ->where('sample_key', $claim['sample_key']) - ->lockForUpdate()->first(); - if ($receipt !== null) { - if ((int) $receipt->instance_id !== $claim['instance_id'] - || (int) $receipt->bucket_start - !== $claim['bucket_start'] - || (int) $receipt->rule_version - !== $claim['rule_version'] - || (int) $receipt->policy_revision - !== $claim['policy_revision'] - || !hash_equals( - (string) $receipt->policy_hash, - $claim['policy_hash'] - )) { - throw new \RuntimeException( - 'Abuse projection receipt identity conflict' - ); - } - return ['state' => 'done', 'baseline' => $baseline]; - } - $state = $main->table('cloud_abuse_instance_states') - ->where('instance_id', $claim['instance_id']) - ->lockForUpdate()->first(); - if ($state !== null && $state->last_bucket_start !== null - && (int) $state->last_bucket_start - >= $claim['bucket_start']) { - return ['state' => 'discarded', - 'reason' => 'projection_order_conflict']; - } - - $this->retireReplacedPolicyFindings( - $main, - $claim, - $now - ); - - $healthy = ($evaluation['collection_state'] ?? null) - === 'healthy'; - $summary = [ - 'rates' => $evaluation['rates'] ?? [], - 'sample_count' => $evaluation['sample_count'] ?? null, - 'warmup_remaining' => - $evaluation['warmup_remaining'] ?? null, - 'quality_flags' => $claim['quality_flags'], - 'metrics' => $claim['metrics'], - ]; - $stateValues = [ - 'instance_id' => $claim['instance_id'], - 'user_id' => $claim['user_id'], - 'host_id' => (int) $instance->host_id, - 'rule_version' => $claim['rule_version'], - 'policy_revision' => $claim['policy_revision'], - 'policy_hash' => $claim['policy_hash'], - 'risk_state' => $healthy - ? (string) $evaluation['risk_state'] - : ($policyChanged - ? 'observing' - : (string) ($state->risk_state ?? 'observing')), - 'collection_state' => $healthy ? 'healthy' : 'degraded', - 'latest_direction' => $healthy - ? (string) $evaluation['direction'] - : ($policyChanged - ? 'none' - : (string) ($state->latest_direction ?? 'none')), - 'latest_rule_code' => $healthy - ? (string) $evaluation['latest_rule_code'] - : ($policyChanged - ? '' - : (string) ($state->latest_rule_code ?? '')), - 'latest_severity' => $healthy - ? (string) $evaluation['latest_severity'] - : ($policyChanged - ? 'none' - : (string) ($state->latest_severity ?? 'none')), - 'summary_json' => self::json($summary), - 'last_projection_key' => $claim['sample_key'], - 'last_bucket_start' => $claim['bucket_start'], - 'last_sample_at' => $claim['bucket_start'] + 60, - 'last_evaluated_at' => $healthy - ? $now : ($policyChanged - ? null : ($state->last_evaluated_at ?? null)), - 'updated_at' => $now, - ]; - if ($state === null) { - $stateValues['created_at'] = $now; - $main->table('cloud_abuse_instance_states') - ->insert($stateValues); - } else { - $main->table('cloud_abuse_instance_states') - ->where('instance_id', $claim['instance_id']) - ->update($stateValues); - } - $transitions = []; - if ($healthy) { - $transitions = $this->applyFindings( - $main, - $claim, - (int) $instance->host_id, - (array) ($evaluation['findings'] ?? []), - $now - ); - } - $eventClaim = $claim; - $eventClaim['host_id'] = (int) $instance->host_id; - $this->alertEvents->record( - $main, - $eventClaim, - $transitions, - $now - ); - $main->table('cloud_abuse_projection_receipts')->insert([ - 'sample_key' => $claim['sample_key'], - 'instance_id' => $claim['instance_id'], - 'rule_version' => $claim['rule_version'], - 'policy_revision' => $claim['policy_revision'], - 'policy_hash' => $claim['policy_hash'], - 'bucket_start' => $claim['bucket_start'], - 'projected_at' => $now, - ]); - return ['state' => 'done', 'baseline' => $baseline]; - }, 3); - } - - /** - * A changed detector policy keeps the long-lived statistical history but - * cannot inherit threshold streaks earned under another immutable policy. - * - * @param array $previous - * @return array - */ - private static function rebaseBaselineForPolicy( - array $previous, - CloudAbuseDetectionPolicy $policy - ): array { - $baseline = CloudAbuseDetector::initialBaseline($policy); - $baseline['sample_count'] = max( - 0, - (int) ($previous['sample_count'] ?? 0) - ); - $baseline['last_bucket_start'] = - is_int($previous['last_bucket_start'] ?? null) - ? $previous['last_bucket_start'] : null; - $baseline['last_interface'] = - is_array($previous['last_interface'] ?? null) - ? $previous['last_interface'] : null; - $baseline['stats'] = is_array($previous['stats'] ?? null) - ? $previous['stats'] : []; - return $baseline; - } - - /** @param array $claim */ - private function retireReplacedPolicyFindings( - object $main, - array $claim, - int $now - ): void { - $rows = $main->table('cloud_abuse_findings') - ->where('instance_id', $claim['instance_id']) - ->where('user_id', $claim['user_id']) - ->where('state', 'open') - ->where(static function ($query) use ($claim): void { - $query->where('rule_version', '<>', $claim['rule_version']) - ->orWhere('policy_hash', '<>', $claim['policy_hash']); - })->orderBy('id')->lockForUpdate()->get(); - foreach ($rows as $row) { - $resolvedAt = max( - $claim['bucket_start'], - (int) $row->last_seen_at - ); - $updated = $main->table('cloud_abuse_findings') - ->where('id', (int) $row->id) - ->where('state', 'open')->update([ - 'state' => 'resolved', - 'resolved_at' => $resolvedAt, - 'resolution_reason' => 'policy_replaced', - 'transition_revision' => - (int) $row->transition_revision + 1, - 'updated_at' => $now, - ]); - if ($updated !== 1) { - throw new \RuntimeException( - 'Abuse policy replacement finding CAS failed' - ); - } - } - } - - private function retireOwnerReadModel( - object $main, - int $instanceId, - int $oldUserId, - int $now - ): void { - $main->table('cloud_abuse_instance_states') - ->where('instance_id', $instanceId) - ->where('user_id', $oldUserId) - ->delete(); - $findings = $main->table('cloud_abuse_findings') - ->where('instance_id', $instanceId) - ->where('user_id', $oldUserId) - ->where('state', 'open') - ->orderBy('id')->lockForUpdate()->get(); - foreach ($findings as $finding) { - $resolvedAt = max($now, (int) $finding->last_seen_at); - $updated = $main->table('cloud_abuse_findings') - ->where('id', (int) $finding->id) - ->where('instance_id', $instanceId) - ->where('user_id', $oldUserId) - ->where('state', 'open')->update([ - 'state' => 'resolved', - 'resolved_at' => $resolvedAt, - 'resolution_reason' => 'owner_changed', - 'transition_revision' => - (int) $finding->transition_revision + 1, - 'updated_at' => $now, - ]); - if ($updated !== 1) { - throw new \RuntimeException( - 'Abuse projection owner cleanup compare-and-swap failed' - ); - } - } - } - - /** - * @return null|array{user_id:int,baseline:array} - */ - private function currentOwnerBaselineSeed( - int $instanceId, - CloudAbusePolicySnapshot $snapshot - ): ?array { - $main = User::getConnectionResolver()->connection('default'); - $instance = $main->table('cloud_instances') - ->where('id', $instanceId)->first(['user_id']); - if ($instance === null) { - return null; - } - $userId = (int) $instance->user_id; - $rows = $main->table('cloud_abuse_findings') - ->where('instance_id', $instanceId) - ->where('user_id', $userId) - ->where('rule_version', $snapshot->policy->engineVersion()) - ->where('policy_hash', $snapshot->policyHash) - ->where('state', 'open') - ->orderBy('id')->get([ - 'rule_code', 'direction', 'ip_family', 'severity', - ]); - $findings = []; - foreach ($rows as $row) { - $findings[] = [ - 'rule_code' => (string) $row->rule_code, - 'direction' => (string) $row->direction, - 'ip_family' => (string) $row->ip_family, - 'severity' => (string) $row->severity, - ]; - } - return [ - 'user_id' => $userId, - 'baseline' => CloudAbuseDetector::resetBaseline( - $findings, - $snapshot->policy - ), - ]; - } - - /** - * @param array $claim - * @param array> $findings - * @return list> - */ - private function applyFindings( - object $main, - array $claim, - int $currentHostId, - array $findings, - int $now - ): array { - $transitions = []; - foreach ($findings as $finding) { - $key = hash('sha256', implode("\n", [ - $claim['instance_id'], - $claim['rule_version'], - $claim['policy_hash'], - (string) $finding['rule_code'], - (string) $finding['ip_family'], - (string) $finding['direction'], - ])); - $row = $main->table('cloud_abuse_findings') - ->where('finding_key', $key)->lockForUpdate()->first(); - if (($finding['active'] ?? false) === true) { - $opened = $row === null - || (string) $row->state === 'resolved'; - $escalated = $row !== null - && (string) $row->state === 'open' - && (string) $row->severity === 'suspicious' - && (string) $finding['severity'] === 'high'; - $generation = $row === null ? 1 - : (int) $row->occurrence_generation - + ($opened ? 1 : 0); - $revision = $row === null ? 1 - : (int) $row->transition_revision - + (($opened || $escalated) ? 1 : 0); - $values = [ - 'instance_id' => $claim['instance_id'], - 'user_id' => $claim['user_id'], - 'host_id' => $currentHostId, - 'rule_code' => (string) $finding['rule_code'], - 'rule_version' => $claim['rule_version'], - 'policy_revision' => $claim['policy_revision'], - 'policy_hash' => $claim['policy_hash'], - 'direction' => (string) $finding['direction'], - 'ip_family' => (string) $finding['ip_family'], - 'severity' => (string) $finding['severity'], - 'state' => 'open', - 'first_seen_at' => $row === null - || (string) $row->state === 'resolved' - ? $claim['bucket_start'] - : (int) $row->first_seen_at, - 'last_seen_at' => $claim['bucket_start'], - 'consecutive_windows' => max( - 1, - (int) $finding['consecutive_windows'] - ), - 'occurrence_generation' => $generation, - 'transition_revision' => $revision, - 'evidence_json' => self::json( - (array) $finding['evidence'] - ), - 'resolved_at' => null, - 'resolution_reason' => '', - 'updated_at' => $now, - ]; - if ($row === null) { - $values['finding_key'] = $key; - $values['created_at'] = $now; - $main->table('cloud_abuse_findings')->insert($values); - } else { - $updated = $main->table('cloud_abuse_findings') - ->where('finding_key', $key) - ->where('transition_revision', - (int) $row->transition_revision) - ->update($values); - if ($updated !== 1) { - throw new \RuntimeException( - 'Abuse finding transition CAS failed' - ); - } - } - if ($opened || $escalated) { - $transitions[] = self::findingTransition( - $key, - $finding, - $generation, - $revision, - $opened ? 'opened' : 'escalated', - $opened ? 'none' : 'suspicious', - (string) $finding['severity'], - '', - $claim['bucket_start'] - ); - } - } elseif (($finding['transition'] ?? '') === 'resolved' - && $row !== null && (string) $row->state === 'open') { - $revision = (int) $row->transition_revision + 1; - $updated = $main->table('cloud_abuse_findings') - ->where('finding_key', $key) - ->where('state', 'open') - ->where('transition_revision', - (int) $row->transition_revision)->update([ - 'state' => 'resolved', - 'resolved_at' => $claim['bucket_start'], - 'resolution_reason' => 'evidence_clear', - 'transition_revision' => $revision, - 'updated_at' => $now, - ]); - if ($updated !== 1) { - throw new \RuntimeException( - 'Abuse finding resolution CAS failed' - ); - } - $transitions[] = self::findingTransition( - $key, - $finding, - (int) $row->occurrence_generation, - $revision, - 'resolved', - (string) $row->severity, - 'none', - 'evidence_clear', - $claim['bucket_start'] - ); - } - } - return $transitions; - } - - /** - * @param array $finding - * @return array - */ - private static function findingTransition( - string $findingKey, - array $finding, - int $generation, - int $revision, - string $type, - string $fromSeverity, - string $toSeverity, - string $resolutionReason, - int $occurredAt - ): array { - return [ - 'finding_key' => $findingKey, - 'occurrence_generation' => $generation, - 'transition_revision' => $revision, - 'rule_code' => (string) $finding['rule_code'], - 'direction' => (string) $finding['direction'], - 'ip_family' => (string) $finding['ip_family'], - 'transition_type' => $type, - 'from_severity' => $fromSeverity, - 'to_severity' => $toSeverity, - 'evidence' => (array) ($finding['evidence'] ?? []), - 'resolution_reason' => $resolutionReason, - 'occurred_at' => $occurredAt, - ]; - } - - /** @param array $claim @param array $baseline */ - private function finalize(array $claim, array $baseline, int $now): void - { - $log = User::getConnectionResolver()->connection('log'); - $log->transaction(function () use ( - $log, - $claim, - $baseline, - $now - ): void { - $base = $log->table('cloud_vm_network_baselines') - ->where('instance_id', $claim['instance_id']) - ->lockForUpdate()->first(); - $outbox = $log->table('cloud_vm_network_projection_outbox') - ->where('sample_key', $claim['sample_key']) - ->lockForUpdate()->first(); - if ($base === null || $outbox === null - || (string) $base->projection_lease_token - !== $claim['lease_token'] - || (string) $outbox->lease_token !== $claim['lease_token'] - || (string) $outbox->state !== 'leased') { - throw new \RuntimeException( - 'Abuse projection lease was lost before finalization' - ); - } - $sampleCount = max(0, (int) ($baseline['sample_count'] ?? 0)); - $snapshot = $claim['policy_snapshot'] ?? null; - if (!$snapshot instanceof CloudAbusePolicySnapshot) { - throw new \RuntimeException( - 'Abuse projection policy snapshot is missing' - ); - } - $baselineUpdated = $log->table('cloud_vm_network_baselines') - ->where('instance_id', $claim['instance_id']) - ->where('revision', $claim['baseline_revision']) - ->where('projection_lease_token', $claim['lease_token']) - ->update([ - 'rule_version' => $claim['rule_version'], - 'policy_revision' => $claim['policy_revision'], - 'policy_hash' => $claim['policy_hash'], - 'sample_count' => $sampleCount, - 'warmup_state' => $sampleCount - >= $snapshot->policy->windows()['warmup'] - ? 'ready' : 'warming', - 'metrics_json' => self::json($baseline), - 'last_projection_key' => $claim['sample_key'], - 'last_bucket_start' => $claim['bucket_start'], - 'projection_lease_token' => null, - 'projection_lease_until' => 0, - 'revision' => $claim['baseline_revision'] + 1, - 'updated_at' => $now, - ]); - $updated = $log->table('cloud_vm_network_projection_outbox') - ->where('sample_key', $claim['sample_key']) - ->where('state', 'leased') - ->where('lease_token', $claim['lease_token']) - ->update([ - 'state' => 'done', - 'lease_token' => null, - 'lease_until' => 0, - 'last_error_code' => '', - 'projected_at' => $now, - 'terminal_at' => $now, - 'discard_reason_code' => '', - 'updated_at' => $now, - ]); - if ($baselineUpdated !== 1 || $updated !== 1) { - throw new \RuntimeException( - 'Abuse projection finalization compare-and-swap failed' - ); - } - }, 3); - } - - /** @param array $claim */ - private function discard(array $claim, string $reason, int $now): void - { - if (preg_match('/^[a-z0-9_]{3,64}$/D', $reason) !== 1) { - $reason = 'projection_discarded'; - } - $log = User::getConnectionResolver()->connection('log'); - $log->transaction(function () use ($log, $claim, $reason, $now): void { - if ($reason === 'instance_owner_changed') { - $released = $log->table('cloud_vm_network_baselines') - ->where('instance_id', $claim['instance_id']) - ->where('user_id', $claim['user_id']) - ->where('revision', $claim['baseline_revision']) - ->where('projection_lease_token', $claim['lease_token']) - ->delete(); - } else { - $released = $this->releaseBaselineLease( - $log, - $claim, - $now - ); - } - $updated = $log->table('cloud_vm_network_projection_outbox') - ->where('sample_key', $claim['sample_key']) - ->where('state', 'leased') - ->where('lease_token', $claim['lease_token']) - ->update([ - 'state' => 'discarded', - 'lease_token' => null, - 'lease_until' => 0, - 'last_error_code' => '', - 'projected_at' => null, - 'terminal_at' => $now, - 'discard_reason_code' => $reason, - 'updated_at' => $now, - ]); - if ($released !== 1 || $updated !== 1) { - throw new \RuntimeException( - 'Abuse projection discard finalization failed' - ); - } - }, 3); - } - - /** @param array $claim */ - private function retryOrDiscard( - array $claim, - \Throwable $error, - int $now - ): string { - $code = self::boundedCode( - $error instanceof CloudAgentError - ? $error->agentCode : 'projection_failed', - 'projection_failed' - ); - $terminal = $claim['attempts'] >= self::MAX_ATTEMPTS; - if ($terminal) { - $receiptState = $this->receiptState($claim); - if ($receiptState === 'matching') { - $terminal = false; - $code = 'projection_receipt_finalize_pending'; - } elseif ($receiptState === 'conflict') { - $terminal = false; - $code = 'projection_receipt_conflict'; - } elseif ($receiptState === 'unknown') { - $terminal = false; - $code = 'projection_receipt_check_failed'; - } - } - $log = User::getConnectionResolver()->connection('log'); - $log->transaction(function () use ( - $log, - $claim, - $now, - $code, - $terminal - ): void { - $this->releaseBaselineLease($log, $claim, $now); - $values = $terminal ? [ - 'state' => 'discarded', - 'lease_token' => null, - 'lease_until' => 0, - 'last_error_code' => $code, - 'projected_at' => null, - 'terminal_at' => $now, - 'discard_reason_code' => 'projection_retry_exhausted', - 'updated_at' => $now, - ] : [ - 'state' => 'pending', - 'available_at' => $now - + self::retryDelay($claim['attempts']), - 'lease_token' => null, - 'lease_until' => 0, - 'last_error_code' => $code, - 'projected_at' => null, - 'terminal_at' => null, - 'discard_reason_code' => '', - 'updated_at' => $now, - ]; - $log->table('cloud_vm_network_projection_outbox') - ->where('sample_key', $claim['sample_key']) - ->where('state', 'leased') - ->where('lease_token', $claim['lease_token']) - ->update($values); - }, 3); - return $code; - } - - /** @param array $claim */ - private function receiptState(array $claim): string - { - try { - $receipt = User::getConnectionResolver() - ->connection('default') - ->table('cloud_abuse_projection_receipts') - ->where('sample_key', $claim['sample_key'])->first(); - } catch (\Throwable) { - return 'unknown'; - } - if ($receipt === null) { - return 'absent'; - } - if ((int) $receipt->instance_id !== $claim['instance_id'] - || (int) $receipt->bucket_start !== $claim['bucket_start'] - || (int) $receipt->rule_version - !== (int) ($claim['rule_version'] ?? 0) - || (int) $receipt->policy_revision - !== (int) ($claim['policy_revision'] ?? 0) - || !hash_equals( - (string) $receipt->policy_hash, - (string) ($claim['policy_hash'] ?? '') - ) - || (int) $receipt->projected_at < 1) { - return 'conflict'; - } - return 'matching'; - } - - /** @param array $claim */ - private function releaseBaselineLease( - object $log, - array $claim, - int $now - ): int { - return $log->table('cloud_vm_network_baselines') - ->where('instance_id', $claim['instance_id']) - ->where('revision', $claim['baseline_revision']) - ->where('projection_lease_token', $claim['lease_token']) - ->update([ - 'projection_lease_token' => null, - 'projection_lease_until' => 0, - 'updated_at' => $now, - ]); - } - - /** @return array */ - private static function decodeObject(string $json, string $name): array - { - try { - $value = json_decode($json, true, 16, JSON_THROW_ON_ERROR); - } catch (\JsonException $error) { - throw new \RuntimeException("Invalid {$name}", 0, $error); - } - if (!is_array($value) || $value === [] || array_is_list($value)) { - throw new \RuntimeException("Invalid {$name}"); - } - return $value; - } - - /** @return array */ - private static function decodeList(string $json, string $name): array - { - try { - $value = json_decode($json, true, 8, JSON_THROW_ON_ERROR); - } catch (\JsonException $error) { - throw new \RuntimeException("Invalid {$name}", 0, $error); - } - if (!is_array($value) || !array_is_list($value)) { - throw new \RuntimeException("Invalid {$name}"); - } - return $value; - } - - /** @param array $metrics @return array */ - private static function canonicalMetrics(array $metrics): array - { - self::exactKeys($metrics, ['interface', 'ipv4', 'ipv6'], - 'sample metrics'); - $interfaceFields = [ - 'rx_bytes', 'tx_bytes', 'rx_packets', 'tx_packets', - 'rx_errors', 'tx_errors', 'rx_drops', 'tx_drops', - ]; - if (!is_array($metrics['interface']) - || array_is_list($metrics['interface'])) { - throw new \RuntimeException('Invalid sample interface metrics'); - } - self::exactKeys($metrics['interface'], $interfaceFields, - 'sample interface metrics'); - $interface = []; - foreach ($interfaceFields as $field) { - $value = $metrics['interface'][$field]; - if ($value !== null && (!is_int($value) || $value < 0)) { - throw new \RuntimeException( - 'Invalid sample interface counter' - ); - } - $interface[$field] = $value; - } - return [ - 'interface' => $interface, - 'ipv4' => self::canonicalFamily($metrics['ipv4'], 'ipv4'), - 'ipv6' => self::canonicalFamily($metrics['ipv6'], 'ipv6'), - ]; - } - - private static function canonicalFamily(mixed $family, string $name): ?array - { - if ($family === null) { - return null; - } - if (!is_array($family) || array_is_list($family)) { - throw new \RuntimeException("Invalid {$name} sample metrics"); - } - self::exactKeys($family, ['inbound', 'outbound'], - "{$name} sample metrics"); - return [ - 'inbound' => self::canonicalFlow( - $family['inbound'], - "{$name} inbound" - ), - 'outbound' => self::canonicalFlow( - $family['outbound'], - "{$name} outbound" - ), - ]; - } - - /** @return array */ - private static function canonicalFlow(mixed $flow, string $name): array - { - $fields = [ - 'tcp_active_flows', 'tcp_new_flows', 'tcp_syn_flows', - 'tcp_unreplied_flows', 'udp_active_flows', 'udp_new_flows', - 'other_active_flows', 'unique_remote_ips', - 'unique_remote_ports', 'unique_local_ports', - 'max_remote_ports_per_remote_ip', - 'max_remote_ips_per_remote_port', - 'max_local_ports_per_remote_ip', - 'max_remote_ips_per_local_port', - ]; - if (!is_array($flow) || array_is_list($flow)) { - throw new \RuntimeException("Invalid {$name} flow metrics"); - } - self::exactKeys($flow, $fields, "{$name} flow metrics"); - $result = []; - foreach ($fields as $field) { - if (!is_int($flow[$field]) || $flow[$field] < 0) { - throw new \RuntimeException("Invalid {$name} flow counter"); - } - $result[$field] = $flow[$field]; - } - return $result; - } - - /** @param array $quality @return array */ - private static function canonicalQuality(array $quality): array - { - $allowed = [ - 'clock_regression', 'cold_start', 'conntrack_unavailable', - 'counter_reset', 'flow_limit_reached', 'incomplete', - 'interface_unavailable', 'local_port_cardinality_saturated', - 'mapping_changed', 'overflow', 'packet_accounting_unavailable', - 'panel_ring_gap', 'relation_cardinality_saturated', - 'remote_ip_cardinality_saturated', - 'remote_port_cardinality_saturated', 'sampling_gap', - ]; - if (count($quality) > 17) { - throw new \RuntimeException('Too many sample quality flags'); - } - foreach ($quality as $flag) { - if (!is_string($flag) || !in_array($flag, $allowed, true)) { - throw new \RuntimeException('Invalid sample quality flag'); - } - } - $canonical = array_values(array_unique($quality)); - sort($canonical, SORT_STRING); - if ($canonical !== $quality) { - throw new \RuntimeException( - 'Sample quality flags are not canonical' - ); - } - return $quality; - } - - /** @param array $value @param array $keys */ - private static function exactKeys( - array $value, - array $keys, - string $name - ): void { - $actual = array_keys($value); - sort($actual, SORT_STRING); - $expected = $keys; - sort($expected, SORT_STRING); - if ($actual !== $expected) { - throw new \RuntimeException("Invalid {$name} shape"); - } - } - - private static function boundedCode(string $code, string $fallback): string - { - return preg_match('/^[a-z0-9_]{3,64}$/D', $code) === 1 - ? $code : $fallback; - } - - private static function retryDelay(int $attempts): int - { - return min( - self::MAX_BACKOFF_SECONDS, - 15 * (2 ** min(6, max(0, $attempts - 1))) - ); - } - - private static function json(array $value): string - { - return json_encode( - $value, - JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES - ); - } -} diff --git a/web/app/Services/TicketTelegramNotifier.php b/web/app/Services/TicketTelegramNotifier.php index 442f0783..c74e25f3 100644 --- a/web/app/Services/TicketTelegramNotifier.php +++ b/web/app/Services/TicketTelegramNotifier.php @@ -314,11 +314,11 @@ final class TicketTelegramNotifier ): ?object { $connection = (new Configurations())->getConnection(); if (!$connection->getSchemaBuilder()->hasTable( - 'cloud_abuse_alert_deliveries' + 'cloud_network_alert_deliveries' )) { return null; } - return $connection->table('cloud_abuse_alert_deliveries') + return $connection->table('cloud_network_alert_deliveries') ->where('binding_generation', $destination->bindingGeneration) ->where('bound_admin_id', $destination->administratorId) ->where('chat_id', $destination->chatId) diff --git a/web/app/Services/UserDataDeletionService.php b/web/app/Services/UserDataDeletionService.php index d94a33a1..ce5cf193 100644 --- a/web/app/Services/UserDataDeletionService.php +++ b/web/app/Services/UserDataDeletionService.php @@ -34,12 +34,6 @@ class UserDataDeletionService ['cloud_backup_period_entitlements', 'user_id'], ['cloud_instance_network_policy_states', 'user_id'], ['cloud_instance_network_restrictions', 'user_id'], - // Alert items cascade from the immutable event header. Keep this - // explicit owner reference because an event may predate an instance - // ownership transfer and must be erased with its original tenant. - ['cloud_abuse_alert_events', 'user_id'], - ['cloud_abuse_findings', 'user_id'], - ['cloud_abuse_instance_states', 'user_id'], ['cloud_traffic_usage_settlement_buckets', 'user_id'], ['cloud_traffic_settlement_receipts', 'user_id'], ['cloud_traffic_instance_cursors', 'user_id'], @@ -500,11 +494,15 @@ class UserDataDeletionService } } - if (!$this->hasColumns($connection, 'cloud_abuse_alert_events', [ + if (!$this->hasColumns($connection, 'cloud_network_alert_events', [ + 'id', 'finding_id', + ]) || !$this->hasColumns($connection, 'cloud_network_findings', [ + 'id', 'instance_id', + ]) || !$this->hasColumns($connection, 'cloud_instances', [ 'id', 'user_id', ]) || !$this->hasColumns( $connection, - 'cloud_abuse_alert_deliveries', + 'cloud_network_alert_deliveries', [ 'id', 'event_id', 'bound_admin_id', 'state', 'error_code', 'terminal_at', 'updated_at', @@ -512,9 +510,20 @@ class UserDataDeletionService )) { return; } - $owned = $connection->table('cloud_abuse_alert_events') - ->whereIn('user_id', $userIds)->pluck('id')->all(); - $bound = $connection->table('cloud_abuse_alert_deliveries') + $owned = $connection->table('cloud_network_alert_events as event') + ->join( + 'cloud_network_findings as finding', + 'finding.id', + '=', + 'event.finding_id' + )->join( + 'cloud_instances as instance', + 'instance.id', + '=', + 'finding.instance_id' + )->whereIn('instance.user_id', $userIds) + ->pluck('event.id')->all(); + $bound = $connection->table('cloud_network_alert_deliveries') ->whereIn('bound_admin_id', $userIds)->pluck('event_id')->all(); $eventIds = array_values(array_unique(array_map( 'intval', @@ -525,13 +534,13 @@ class UserDataDeletionService return; } - $lockedEventIds = $connection->table('cloud_abuse_alert_events') + $lockedEventIds = $connection->table('cloud_network_alert_events') ->whereIn('id', $eventIds)->orderBy('id')->lockForUpdate() ->pluck('id')->map(static fn ($id): int => (int) $id)->all(); if ($lockedEventIds === []) { return; } - $deliveryRows = $connection->table('cloud_abuse_alert_deliveries') + $deliveryRows = $connection->table('cloud_network_alert_deliveries') ->whereIn('event_id', $lockedEventIds)->orderBy('id') ->lockForUpdate()->get(['id', 'state', 'updated_at']); $now = time(); @@ -539,7 +548,7 @@ class UserDataDeletionService if ((string) $row->state === 'dispatching') { if ((int) $row->updated_at <= $now - 30) { $settled = $connection->table( - 'cloud_abuse_alert_deliveries' + 'cloud_network_alert_deliveries' )->where('id', (int) $row->id) ->where('state', 'dispatching')->update([ 'state' => 'unknown', @@ -622,23 +631,16 @@ class UserDataDeletionService array $userIds ): int { if (!$this->hasColumns($connection, - 'cloud_abuse_alert_deliveries', [ + 'cloud_network_alert_deliveries', [ 'id', 'bound_admin_id', 'state', 'binding_generation', 'chat_id', 'telegram_message_id', 'dispatch_started_at', 'terminal_at', 'destination_redacted_at', 'updated_at', ])) { return 0; } - $rows = $connection->table('cloud_abuse_alert_deliveries') + $rows = $connection->table('cloud_network_alert_deliveries') ->whereIn('bound_admin_id', $userIds) - ->whereNotExists(static function ($query) use ($userIds): void { - $query->selectRaw('1') - ->from('cloud_abuse_alert_events as alert_event') - ->whereColumn( - 'alert_event.id', - 'cloud_abuse_alert_deliveries.event_id' - )->whereIn('alert_event.user_id', $userIds); - })->orderBy('id')->get(['id', 'state']); + ->orderBy('id')->get(['id', 'state']); if ($rows->isEmpty()) { return 0; } @@ -660,7 +662,7 @@ class UserDataDeletionService 'terminal_at' => $now, ]; } - $count = $connection->table('cloud_abuse_alert_deliveries') + $count = $connection->table('cloud_network_alert_deliveries') ->where('id', (int) $row->id) ->where('state', (string) $row->state) ->whereIn('bound_admin_id', $userIds) @@ -773,40 +775,6 @@ class UserDataDeletionService $ids ); } - $observationSampleKeys = $this->pluckByUsers( - $connection, - 'cloud_vm_network_observation_samples', - 'user_id', - $ids, - 'sample_key' - ); - // Projection intents deliberately RESTRICT sample deletion. Account - // erasure is authoritative, so remove every intent before its - // tenant-owned sample instead of relying on a cascading shortcut. - $deleted += $this->deleteWhereIn( - $connection, - 'cloud_vm_network_projection_outbox', - 'sample_key', - $observationSampleKeys - ); - $deleted += $this->deleteWhereIn( - $connection, - 'cloud_vm_network_observation_samples', - 'sample_key', - $observationSampleKeys - ); - $deleted += $this->deleteWhereIn( - $connection, - 'cloud_vm_network_observation_rollups', - 'user_id', - $ids - ); - $deleted += $this->deleteWhereIn( - $connection, - 'cloud_vm_network_baselines', - 'user_id', - $ids - ); $deleted += $this->deleteWhereIn($connection, 'user_email_log', 'email', $emails); return $deleted; }, 3); diff --git a/web/sql/cloud.sql b/web/sql/cloud.sql index 1a912106..3c61a7b8 100644 --- a/web/sql/cloud.sql +++ b/web/sql/cloud.sql @@ -3374,326 +3374,6 @@ CREATE TABLE cloud_network_alert_deliveries ( ON DELETE CASCADE ON UPDATE RESTRICT ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; --- Current read model for the observe-only abuse monitor. These rows never --- participate in instance lifecycle, billing, traffic-pool accounting or --- operation locks; stale/degraded collection is an observation, not a reason --- to change service_status or network_status. -CREATE TABLE cloud_abuse_instance_states ( - instance_id BIGINT UNSIGNED NOT NULL PRIMARY KEY, - user_id BIGINT UNSIGNED NOT NULL, - host_id BIGINT UNSIGNED NOT NULL, - rule_version INT UNSIGNED NOT NULL, - policy_revision BIGINT UNSIGNED NOT NULL, - policy_hash CHAR(64) CHARACTER SET ascii COLLATE ascii_bin NOT NULL, - risk_state VARCHAR(16) CHARACTER SET ascii COLLATE ascii_bin NOT NULL, - collection_state VARCHAR(16) CHARACTER SET ascii COLLATE ascii_bin NOT NULL, - latest_direction VARCHAR(16) CHARACTER SET ascii COLLATE ascii_bin NOT NULL DEFAULT 'none', - latest_rule_code VARCHAR(48) CHARACTER SET ascii COLLATE ascii_bin NOT NULL DEFAULT '', - latest_severity VARCHAR(16) CHARACTER SET ascii COLLATE ascii_bin NOT NULL DEFAULT 'none', - summary_json JSON NOT NULL, - last_projection_key CHAR(64) CHARACTER SET ascii COLLATE ascii_bin DEFAULT NULL, - last_bucket_start BIGINT UNSIGNED DEFAULT NULL, - last_sample_at BIGINT UNSIGNED DEFAULT NULL, - last_evaluated_at BIGINT UNSIGNED DEFAULT NULL, - created_at BIGINT UNSIGNED NOT NULL, - updated_at BIGINT UNSIGNED NOT NULL, - KEY cloud_abuse_state_risk (risk_state, updated_at, instance_id), - KEY cloud_abuse_state_collection ( - collection_state, last_sample_at, instance_id - ), - KEY cloud_abuse_state_host (host_id, risk_state, instance_id), - KEY cloud_abuse_state_owner (user_id, risk_state, instance_id), - CONSTRAINT chk_cloud_abuse_state_risk CHECK ( - risk_state IN ('observing','normal','suspicious','high_risk') - ), - CONSTRAINT chk_cloud_abuse_state_collection CHECK ( - collection_state IN ( - 'unknown','healthy','degraded','stale','unavailable' - ) - ), - CONSTRAINT chk_cloud_abuse_state_direction CHECK ( - latest_direction IN ('none','inbound','outbound','both') - ), - CONSTRAINT chk_cloud_abuse_state_severity CHECK ( - latest_severity IN ('none','info','suspicious','high') - ), - CONSTRAINT chk_cloud_abuse_state_policy CHECK ( - rule_version > 0 - AND policy_revision > 0 - AND policy_hash REGEXP '^[a-f0-9]{64}$' - ), - CONSTRAINT chk_cloud_abuse_state_projection CHECK ( - (last_projection_key IS NULL AND last_bucket_start IS NULL) - OR (last_projection_key IS NOT NULL - AND last_projection_key REGEXP '^[a-f0-9]{64}$' - AND last_bucket_start IS NOT NULL) - ), - CONSTRAINT fk_cloud_abuse_state_instance FOREIGN KEY (instance_id) - REFERENCES cloud_instances (id) ON DELETE CASCADE ON UPDATE RESTRICT -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; - --- Findings contain only rule identifiers and aggregate evidence. The --- deterministic key makes projection from the log database idempotent even --- though the two databases cannot share a transaction. -CREATE TABLE cloud_abuse_findings ( - id BIGINT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY, - finding_key CHAR(64) CHARACTER SET ascii COLLATE ascii_bin NOT NULL, - instance_id BIGINT UNSIGNED NOT NULL, - user_id BIGINT UNSIGNED NOT NULL, - host_id BIGINT UNSIGNED NOT NULL, - rule_code VARCHAR(48) CHARACTER SET ascii COLLATE ascii_bin NOT NULL, - rule_version INT UNSIGNED NOT NULL, - policy_revision BIGINT UNSIGNED NOT NULL, - policy_hash CHAR(64) CHARACTER SET ascii COLLATE ascii_bin NOT NULL, - direction VARCHAR(16) CHARACTER SET ascii COLLATE ascii_bin NOT NULL, - ip_family VARCHAR(8) CHARACTER SET ascii COLLATE ascii_bin NOT NULL, - severity VARCHAR(16) CHARACTER SET ascii COLLATE ascii_bin NOT NULL, - state VARCHAR(16) CHARACTER SET ascii COLLATE ascii_bin NOT NULL, - first_seen_at BIGINT UNSIGNED NOT NULL, - last_seen_at BIGINT UNSIGNED NOT NULL, - consecutive_windows INT UNSIGNED NOT NULL DEFAULT 1, - occurrence_generation INT UNSIGNED NOT NULL DEFAULT 1, - transition_revision INT UNSIGNED NOT NULL DEFAULT 1, - evidence_json JSON NOT NULL, - resolved_at BIGINT UNSIGNED DEFAULT NULL, - resolution_reason VARCHAR(32) CHARACTER SET ascii COLLATE ascii_bin NOT NULL DEFAULT '', - created_at BIGINT UNSIGNED NOT NULL, - updated_at BIGINT UNSIGNED NOT NULL, - UNIQUE KEY cloud_abuse_finding_key (finding_key), - KEY cloud_abuse_finding_instance ( - instance_id, state, last_seen_at, id - ), - KEY cloud_abuse_finding_owner (user_id, state, last_seen_at, id), - KEY cloud_abuse_finding_host (host_id, state, last_seen_at, id), - KEY cloud_abuse_finding_rule ( - rule_code, severity, state, last_seen_at, id - ), - KEY cloud_abuse_finding_retention ( - state, resolved_at, last_seen_at, id - ), - CONSTRAINT chk_cloud_abuse_finding_key CHECK ( - finding_key REGEXP '^[a-f0-9]{64}$' - ), - CONSTRAINT chk_cloud_abuse_finding_direction CHECK ( - direction IN ('inbound','outbound') - ), - CONSTRAINT chk_cloud_abuse_finding_family CHECK ( - ip_family IN ('ipv4','ipv6','mixed') - ), - CONSTRAINT chk_cloud_abuse_finding_severity CHECK ( - severity IN ('info','suspicious','high') - ), - CONSTRAINT chk_cloud_abuse_finding_state CHECK ( - state IN ('open','resolved') - ), - CONSTRAINT chk_cloud_abuse_finding_policy CHECK ( - rule_version > 0 - AND policy_revision > 0 - AND policy_hash REGEXP '^[a-f0-9]{64}$' - ), - CONSTRAINT chk_cloud_abuse_finding_window CHECK ( - first_seen_at <= last_seen_at - AND consecutive_windows >= 1 - AND occurrence_generation >= 1 - AND transition_revision >= 1 - ), - CONSTRAINT chk_cloud_abuse_finding_resolution CHECK ( - (state = 'open' AND resolved_at IS NULL AND resolution_reason = '') - OR (state = 'resolved' AND resolved_at IS NOT NULL - AND resolved_at >= last_seen_at - AND resolution_reason IN ( - 'evidence_clear','policy_replaced','owner_changed' - )) - ), - CONSTRAINT fk_cloud_abuse_finding_instance FOREIGN KEY (instance_id) - REFERENCES cloud_instances (id) ON DELETE CASCADE ON UPDATE RESTRICT -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; - --- Immutable alert facts produced from finding state transitions. One header --- represents one instance/sample bucket; multiple rule/family transitions are --- stored as items. This phase deliberately records no Telegram delivery state. -CREATE TABLE cloud_abuse_alert_events ( - id BIGINT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY, - event_key CHAR(64) CHARACTER SET ascii COLLATE ascii_bin NOT NULL, - sample_key CHAR(64) CHARACTER SET ascii COLLATE ascii_bin NOT NULL, - instance_id BIGINT UNSIGNED NOT NULL, - user_id BIGINT UNSIGNED NOT NULL, - host_id BIGINT UNSIGNED NOT NULL, - bucket_start BIGINT UNSIGNED NOT NULL, - rule_version INT UNSIGNED NOT NULL, - policy_revision BIGINT UNSIGNED NOT NULL, - policy_hash CHAR(64) CHARACTER SET ascii COLLATE ascii_bin NOT NULL, - transition_digest CHAR(64) CHARACTER SET ascii COLLATE ascii_bin NOT NULL, - transition_count INT UNSIGNED NOT NULL, - highest_severity VARCHAR(16) CHARACTER SET ascii COLLATE ascii_bin NOT NULL, - created_at BIGINT UNSIGNED NOT NULL, - UNIQUE KEY cloud_abuse_alert_event_key (event_key), - UNIQUE KEY cloud_abuse_alert_event_sample (sample_key), - KEY cloud_abuse_alert_event_instance ( - instance_id, bucket_start, id - ), - KEY cloud_abuse_alert_event_owner (user_id, created_at, id), - KEY cloud_abuse_alert_event_host (host_id, created_at, id), - KEY cloud_abuse_alert_event_retention (created_at, id), - CONSTRAINT chk_cloud_abuse_alert_event_identity CHECK ( - event_key REGEXP '^[a-f0-9]{64}$' - AND sample_key REGEXP '^[a-f0-9]{64}$' - AND transition_digest REGEXP '^[a-f0-9]{64}$' - ), - CONSTRAINT chk_cloud_abuse_alert_event_policy CHECK ( - rule_version > 0 - AND policy_revision > 0 - AND policy_hash REGEXP '^[a-f0-9]{64}$' - ), - CONSTRAINT chk_cloud_abuse_alert_event_shape CHECK ( - bucket_start % 60 = 0 - AND transition_count > 0 - AND highest_severity IN ('suspicious','high') - ) -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; - -CREATE TABLE cloud_abuse_alert_event_items ( - id BIGINT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY, - event_id BIGINT UNSIGNED NOT NULL, - item_key CHAR(64) CHARACTER SET ascii COLLATE ascii_bin NOT NULL, - finding_key CHAR(64) CHARACTER SET ascii COLLATE ascii_bin NOT NULL, - occurrence_generation INT UNSIGNED NOT NULL, - transition_revision INT UNSIGNED NOT NULL, - rule_code VARCHAR(48) CHARACTER SET ascii COLLATE ascii_bin NOT NULL, - direction VARCHAR(16) CHARACTER SET ascii COLLATE ascii_bin NOT NULL, - ip_family VARCHAR(8) CHARACTER SET ascii COLLATE ascii_bin NOT NULL, - transition_type VARCHAR(16) CHARACTER SET ascii COLLATE ascii_bin NOT NULL, - from_severity VARCHAR(16) CHARACTER SET ascii COLLATE ascii_bin NOT NULL, - to_severity VARCHAR(16) CHARACTER SET ascii COLLATE ascii_bin NOT NULL, - evidence_json JSON NOT NULL, - resolution_reason VARCHAR(32) CHARACTER SET ascii COLLATE ascii_bin NOT NULL DEFAULT '', - occurred_at BIGINT UNSIGNED NOT NULL, - created_at BIGINT UNSIGNED NOT NULL, - UNIQUE KEY cloud_abuse_alert_item_key (item_key), - UNIQUE KEY cloud_abuse_alert_item_finding_once (event_id, finding_key), - KEY cloud_abuse_alert_item_event (event_id, id), - KEY cloud_abuse_alert_item_finding ( - finding_key, occurrence_generation, transition_revision - ), - CONSTRAINT chk_cloud_abuse_alert_item_identity CHECK ( - item_key REGEXP '^[a-f0-9]{64}$' - AND finding_key REGEXP '^[a-f0-9]{64}$' - AND occurrence_generation >= 1 - AND transition_revision >= 1 - ), - CONSTRAINT chk_cloud_abuse_alert_item_scope CHECK ( - direction IN ('inbound','outbound') - AND ip_family IN ('ipv4','ipv6','mixed') - ), - CONSTRAINT chk_cloud_abuse_alert_item_transition CHECK ( - (transition_type = 'opened' - AND from_severity = 'none' - AND to_severity IN ('suspicious','high') - AND resolution_reason = '') - OR (transition_type = 'escalated' - AND from_severity = 'suspicious' - AND to_severity = 'high' - AND resolution_reason = '') - OR (transition_type = 'resolved' - AND from_severity IN ('suspicious','high') - AND to_severity = 'none' - AND resolution_reason = 'evidence_clear') - ), - CONSTRAINT fk_cloud_abuse_alert_item_event FOREIGN KEY (event_id) - REFERENCES cloud_abuse_alert_events (id) - ON DELETE CASCADE ON UPDATE RESTRICT -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; - --- One immutable Telegram delivery decision per alert event. The message body --- is frozen before the row becomes pending, so a restarted worker never --- reformats an event with changed application code. Once dispatching begins, --- every uncertain outcome is terminal and must never be replayed. -CREATE TABLE cloud_abuse_alert_deliveries ( - id BIGINT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY, - event_id BIGINT UNSIGNED NOT NULL, - binding_generation CHAR(32) CHARACTER SET ascii COLLATE ascii_bin DEFAULT NULL, - bound_admin_id BIGINT UNSIGNED DEFAULT NULL, - chat_id VARCHAR(20) CHARACTER SET ascii COLLATE ascii_bin DEFAULT NULL, - telegram_message_id BIGINT UNSIGNED DEFAULT NULL, - state VARCHAR(16) CHARACTER SET ascii COLLATE ascii_bin NOT NULL, - message_text TEXT NOT NULL, - content_hash CHAR(64) CHARACTER SET ascii COLLATE ascii_bin NOT NULL, - error_code VARCHAR(64) CHARACTER SET ascii COLLATE ascii_bin NOT NULL DEFAULT '', - skip_reason VARCHAR(64) CHARACTER SET ascii COLLATE ascii_bin NOT NULL DEFAULT '', - dispatch_started_at BIGINT UNSIGNED DEFAULT NULL, - sent_at BIGINT UNSIGNED DEFAULT NULL, - terminal_at BIGINT UNSIGNED DEFAULT NULL, - destination_redacted_at BIGINT UNSIGNED DEFAULT NULL, - created_at BIGINT UNSIGNED NOT NULL, - updated_at BIGINT UNSIGNED NOT NULL, - UNIQUE KEY cloud_abuse_alert_delivery_event (event_id), - UNIQUE KEY cloud_abuse_alert_delivery_message ( - binding_generation, chat_id, telegram_message_id - ), - KEY cloud_abuse_alert_delivery_due (state, updated_at, id), - KEY cloud_abuse_alert_delivery_admin (bound_admin_id, created_at, id), - KEY cloud_abuse_alert_delivery_terminal (terminal_at, state, id), - CONSTRAINT chk_cloud_abuse_alert_delivery_state CHECK ( - state IN ('pending','dispatching','sent','failed','unknown','skipped') - ), - CONSTRAINT chk_cloud_abuse_alert_delivery_content CHECK ( - CHAR_LENGTH(message_text) BETWEEN 1 AND 4096 - AND content_hash REGEXP '^[a-f0-9]{64}$' - ), - CONSTRAINT chk_cloud_abuse_alert_delivery_destination CHECK ( - ( - destination_redacted_at IS NULL - AND ( - (binding_generation IS NULL AND bound_admin_id IS NULL - AND chat_id IS NULL AND telegram_message_id IS NULL) - OR (binding_generation IS NOT NULL - AND binding_generation REGEXP '^[a-f0-9]{32}$' - AND bound_admin_id IS NOT NULL AND bound_admin_id > 0 - AND chat_id IS NOT NULL - AND chat_id REGEXP '^[1-9][0-9]{0,18}$' - AND (telegram_message_id IS NULL - OR telegram_message_id > 0)) - ) - ) - OR ( - destination_redacted_at IS NOT NULL - AND state IN ('sent','failed','unknown','skipped') - AND binding_generation IS NULL AND bound_admin_id IS NULL - AND chat_id IS NULL AND telegram_message_id IS NULL - ) - ), - CONSTRAINT chk_cloud_abuse_alert_delivery_shape CHECK ( - (state = 'pending' - AND binding_generation IS NOT NULL AND telegram_message_id IS NULL - AND dispatch_started_at IS NULL AND sent_at IS NULL - AND terminal_at IS NULL AND error_code = '' AND skip_reason = '') - OR (state = 'dispatching' - AND binding_generation IS NOT NULL AND telegram_message_id IS NULL - AND dispatch_started_at IS NOT NULL AND sent_at IS NULL - AND terminal_at IS NULL AND error_code = '' AND skip_reason = '') - OR (state = 'sent' - AND dispatch_started_at IS NOT NULL AND sent_at IS NOT NULL - AND terminal_at IS NOT NULL AND error_code = '' AND skip_reason = '' - AND (destination_redacted_at IS NOT NULL - OR telegram_message_id IS NOT NULL)) - OR (state = 'failed' - AND telegram_message_id IS NULL - AND dispatch_started_at IS NULL AND sent_at IS NULL - AND terminal_at IS NOT NULL - AND error_code <> '' AND skip_reason = '') - OR (state = 'unknown' - AND dispatch_started_at IS NOT NULL AND sent_at IS NULL - AND terminal_at IS NOT NULL - AND error_code <> '' AND skip_reason = '') - OR (state = 'skipped' - AND dispatch_started_at IS NULL AND sent_at IS NULL - AND terminal_at IS NOT NULL AND error_code = '' AND skip_reason <> '') - ), - CONSTRAINT fk_cloud_abuse_alert_delivery_event FOREIGN KEY (event_id) - REFERENCES cloud_abuse_alert_events (id) - ON DELETE CASCADE ON UPDATE RESTRICT -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; - -- Cross-process per-chat rate gate. The singleton is advanced before a remote -- request so concurrent notification workers cannot exceed the provider rate. CREATE TABLE administrator_telegram_delivery_gate ( @@ -3709,36 +3389,6 @@ INSERT INTO administrator_telegram_delivery_gate ( id, next_send_at, updated_at ) VALUES (1, 0, 0); --- Main-database idempotency receipt for one log-database outbox projection. --- The state row's last_projection_key/last_bucket_start pair is the ordering --- fence; a replay may observe this receipt and safely mark the outbox done. -CREATE TABLE cloud_abuse_projection_receipts ( - sample_key CHAR(64) CHARACTER SET ascii COLLATE ascii_bin NOT NULL PRIMARY KEY, - instance_id BIGINT UNSIGNED NOT NULL, - rule_version INT UNSIGNED NOT NULL, - policy_revision BIGINT UNSIGNED NOT NULL, - policy_hash CHAR(64) CHARACTER SET ascii COLLATE ascii_bin NOT NULL, - bucket_start BIGINT UNSIGNED NOT NULL, - projected_at BIGINT UNSIGNED NOT NULL, - KEY cloud_abuse_projection_instance ( - instance_id, projected_at, sample_key - ), - KEY cloud_abuse_projection_retention (projected_at, sample_key), - CONSTRAINT chk_cloud_abuse_projection_key CHECK ( - sample_key REGEXP '^[a-f0-9]{64}$' - ), - CONSTRAINT chk_cloud_abuse_projection_policy CHECK ( - rule_version > 0 - AND policy_revision > 0 - AND policy_hash REGEXP '^[a-f0-9]{64}$' - ), - CONSTRAINT chk_cloud_abuse_projection_bucket CHECK ( - bucket_start % 60 = 0 - ), - CONSTRAINT fk_cloud_abuse_projection_instance FOREIGN KEY (instance_id) - REFERENCES cloud_instances (id) ON DELETE CASCADE ON UPDATE RESTRICT -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; - CREATE TABLE cloud_jobs ( id BIGINT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY, operation_id CHAR(32) NOT NULL, diff --git a/web/sql/cloud_log.sql b/web/sql/cloud_log.sql index 17ec1aed..17826ef2 100644 --- a/web/sql/cloud_log.sql +++ b/web/sql/cloud_log.sql @@ -197,279 +197,3 @@ CREATE TABLE cloud_vm_daily_traffic ( KEY cloud_daily_traffic_retention (day_start), KEY cloud_daily_traffic_owner (user_id, day_start) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; - --- Network-abuse observations are deliberately isolated from billing traffic. --- The Agent publishes bounded, aggregate one-minute windows only: no packet --- payload, remote-address list or remote-port list is stored here. A stable --- sample key makes retries idempotent while instance_id keeps history attached --- to the logical VM across maintenance migration. -CREATE TABLE cloud_vm_network_observation_samples ( - sample_key CHAR(64) CHARACTER SET ascii COLLATE ascii_bin NOT NULL, - instance_id BIGINT UNSIGNED NOT NULL, - user_id BIGINT UNSIGNED NOT NULL, - host_id BIGINT UNSIGNED NOT NULL, - vm_uuid CHAR(36) CHARACTER SET ascii COLLATE ascii_bin NOT NULL, - placement_key CHAR(64) CHARACTER SET ascii COLLATE ascii_bin NOT NULL, - collector_generation CHAR(32) CHARACTER SET ascii COLLATE ascii_bin NOT NULL, - window_id CHAR(32) CHARACTER SET ascii COLLATE ascii_bin NOT NULL, - contract_version SMALLINT UNSIGNED NOT NULL, - bucket_start BIGINT UNSIGNED NOT NULL, - bucket_seconds SMALLINT UNSIGNED NOT NULL, - complete TINYINT(1) NOT NULL, - metrics_json JSON NOT NULL, - quality_flags_json JSON NOT NULL, - received_at BIGINT UNSIGNED NOT NULL, - PRIMARY KEY (sample_key), - UNIQUE KEY cloud_network_observation_source ( - host_id, collector_generation, window_id, vm_uuid - ), - KEY cloud_network_observation_instance ( - instance_id, bucket_start - ), - KEY cloud_network_observation_retention (bucket_start), - CONSTRAINT chk_cloud_network_observation_contract CHECK ( - contract_version = 1 - AND sample_key REGEXP '^[a-f0-9]{64}$' - AND placement_key REGEXP '^[a-f0-9]{64}$' - AND collector_generation REGEXP '^[a-f0-9]{32}$' - AND window_id REGEXP '^[a-f0-9]{32}$' - AND vm_uuid REGEXP - '^[a-f0-9]{8}-[a-f0-9]{4}-[1-5][a-f0-9]{3}-[89ab][a-f0-9]{3}-[a-f0-9]{12}$' - ), - CONSTRAINT chk_cloud_network_observation_bucket CHECK ( - bucket_seconds = 60 AND bucket_start % 60 = 0 - ), - CONSTRAINT chk_cloud_network_observation_complete CHECK ( - complete IN (0, 1) - ) -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; - --- Bounded numeric history derived exactly once from each accepted raw sample. --- Five-minute rows support recent investigation; hourly rows retain long-term --- capacity and abuse trends without keeping the much wider observation JSON. -CREATE TABLE cloud_vm_network_observation_rollups ( - instance_id BIGINT UNSIGNED NOT NULL, - user_id BIGINT UNSIGNED NOT NULL, - host_id BIGINT UNSIGNED NOT NULL, - interval_seconds SMALLINT UNSIGNED NOT NULL, - bucket_start BIGINT UNSIGNED NOT NULL, - sample_count SMALLINT UNSIGNED NOT NULL, - complete_count SMALLINT UNSIGNED NOT NULL, - quality_mask INT UNSIGNED NOT NULL DEFAULT 0, - rx_bytes_min BIGINT UNSIGNED DEFAULT NULL, - rx_bytes_max BIGINT UNSIGNED DEFAULT NULL, - tx_bytes_min BIGINT UNSIGNED DEFAULT NULL, - tx_bytes_max BIGINT UNSIGNED DEFAULT NULL, - rx_packets_min BIGINT UNSIGNED DEFAULT NULL, - rx_packets_max BIGINT UNSIGNED DEFAULT NULL, - tx_packets_min BIGINT UNSIGNED DEFAULT NULL, - tx_packets_max BIGINT UNSIGNED DEFAULT NULL, - tcp_active_peak BIGINT UNSIGNED NOT NULL DEFAULT 0, - tcp_new_total BIGINT UNSIGNED NOT NULL DEFAULT 0, - tcp_syn_total BIGINT UNSIGNED NOT NULL DEFAULT 0, - tcp_unreplied_total BIGINT UNSIGNED NOT NULL DEFAULT 0, - udp_active_peak BIGINT UNSIGNED NOT NULL DEFAULT 0, - udp_new_total BIGINT UNSIGNED NOT NULL DEFAULT 0, - peer_ip_cardinality_peak BIGINT UNSIGNED NOT NULL DEFAULT 0, - peer_port_cardinality_peak BIGINT UNSIGNED NOT NULL DEFAULT 0, - first_sample_at BIGINT UNSIGNED NOT NULL, - last_sample_at BIGINT UNSIGNED NOT NULL, - created_at BIGINT UNSIGNED NOT NULL, - updated_at BIGINT UNSIGNED NOT NULL, - PRIMARY KEY (instance_id, interval_seconds, bucket_start), - KEY cloud_network_rollup_owner ( - user_id, interval_seconds, bucket_start, instance_id - ), - KEY cloud_network_rollup_host ( - host_id, interval_seconds, bucket_start, instance_id - ), - KEY cloud_network_rollup_retention ( - interval_seconds, bucket_start, instance_id - ), - CONSTRAINT chk_cloud_network_rollup_bucket CHECK ( - interval_seconds IN (300, 3600) - AND bucket_start % interval_seconds = 0 - ), - CONSTRAINT chk_cloud_network_rollup_counts CHECK ( - sample_count > 0 AND complete_count <= sample_count - AND first_sample_at >= bucket_start - AND last_sample_at >= first_sample_at - AND last_sample_at < bucket_start + interval_seconds - ), - CONSTRAINT chk_cloud_network_rollup_counters CHECK ( - (rx_bytes_min IS NULL AND rx_bytes_max IS NULL - OR rx_bytes_min IS NOT NULL AND rx_bytes_max >= rx_bytes_min) - AND (tx_bytes_min IS NULL AND tx_bytes_max IS NULL - OR tx_bytes_min IS NOT NULL AND tx_bytes_max >= tx_bytes_min) - AND (rx_packets_min IS NULL AND rx_packets_max IS NULL - OR rx_packets_min IS NOT NULL AND rx_packets_max >= rx_packets_min) - AND (tx_packets_min IS NULL AND tx_packets_max IS NULL - OR tx_packets_min IS NOT NULL AND tx_packets_max >= tx_packets_min) - ) -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; - --- Durable cross-database projection intent. Ingestion can commit a sample, --- its pending projection and the advanced host cursor in one log-database --- transaction. The main database records a separate idempotent receipt. -CREATE TABLE cloud_vm_network_projection_outbox ( - sample_key CHAR(64) CHARACTER SET ascii COLLATE ascii_bin NOT NULL, - instance_id BIGINT UNSIGNED NOT NULL, - bucket_start BIGINT UNSIGNED NOT NULL, - placement_key CHAR(64) CHARACTER SET ascii COLLATE ascii_bin NOT NULL, - rule_version INT UNSIGNED NOT NULL, - policy_revision BIGINT UNSIGNED NOT NULL, - policy_hash CHAR(64) CHARACTER SET ascii COLLATE ascii_bin NOT NULL, - state VARCHAR(16) CHARACTER SET ascii COLLATE ascii_bin NOT NULL, - attempts INT UNSIGNED NOT NULL DEFAULT 0, - available_at BIGINT UNSIGNED NOT NULL, - lease_token CHAR(32) CHARACTER SET ascii COLLATE ascii_bin DEFAULT NULL, - lease_until BIGINT UNSIGNED NOT NULL DEFAULT 0, - last_error_code VARCHAR(64) CHARACTER SET ascii COLLATE ascii_bin NOT NULL DEFAULT '', - projected_at BIGINT UNSIGNED DEFAULT NULL, - terminal_at BIGINT UNSIGNED DEFAULT NULL, - discard_reason_code VARCHAR(64) CHARACTER SET ascii COLLATE ascii_bin NOT NULL DEFAULT '', - created_at BIGINT UNSIGNED NOT NULL, - updated_at BIGINT UNSIGNED NOT NULL, - PRIMARY KEY (sample_key), - KEY cloud_network_projection_due ( - state, available_at, bucket_start, sample_key - ), - KEY cloud_network_projection_instance ( - instance_id, state, bucket_start, sample_key - ), - KEY cloud_network_projection_lease (state, lease_until, sample_key), - KEY cloud_network_projection_retention (state, terminal_at, sample_key), - CONSTRAINT chk_cloud_network_projection_key CHECK ( - sample_key REGEXP '^[a-f0-9]{64}$' - AND placement_key REGEXP '^[a-f0-9]{64}$' - AND instance_id > 0 - AND bucket_start % 60 = 0 - ), - CONSTRAINT chk_cloud_network_projection_policy CHECK ( - rule_version > 0 - AND policy_revision > 0 - AND policy_hash REGEXP '^[a-f0-9]{64}$' - ), - CONSTRAINT chk_cloud_network_projection_state CHECK ( - state IN ('pending','leased','done','discarded') - ), - CONSTRAINT chk_cloud_network_projection_lease CHECK ( - (state = 'pending' AND lease_token IS NULL AND lease_until = 0 - AND projected_at IS NULL AND terminal_at IS NULL - AND discard_reason_code = '') - OR (state = 'leased' - AND lease_token IS NOT NULL - AND lease_token REGEXP '^[a-f0-9]{32}$' - AND lease_until > 0 AND projected_at IS NULL - AND terminal_at IS NULL AND discard_reason_code = '') - OR (state = 'done' AND lease_token IS NULL AND lease_until = 0 - AND projected_at IS NOT NULL AND terminal_at IS NOT NULL - AND terminal_at >= projected_at AND discard_reason_code = '') - OR (state = 'discarded' AND lease_token IS NULL AND lease_until = 0 - AND projected_at IS NULL AND terminal_at IS NOT NULL - AND discard_reason_code REGEXP '^[a-z0-9_]{3,64}$') - ), - CONSTRAINT fk_cloud_network_projection_sample FOREIGN KEY (sample_key) - REFERENCES cloud_vm_network_observation_samples (sample_key) - ON DELETE RESTRICT ON UPDATE RESTRICT -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; - --- One cursor per host. Collector generation changes after an Agent restart; --- consumers must never compare opaque cursor values across generations. -CREATE TABLE cloud_vm_network_monitor_cursors ( - host_id BIGINT UNSIGNED NOT NULL, - collector_generation CHAR(32) CHARACTER SET ascii COLLATE ascii_bin DEFAULT NULL, - cursor_value BIGINT UNSIGNED NOT NULL DEFAULT 0, - oldest_cursor BIGINT UNSIGNED NOT NULL DEFAULT 0, - gap_before TINYINT(1) NOT NULL DEFAULT 0, - poll_revision BIGINT UNSIGNED NOT NULL DEFAULT 0, - next_poll_at BIGINT UNSIGNED NOT NULL DEFAULT 0, - poll_lease_token CHAR(32) CHARACTER SET ascii COLLATE ascii_bin DEFAULT NULL, - poll_lease_until BIGINT UNSIGNED NOT NULL DEFAULT 0, - collection_state VARCHAR(16) CHARACTER SET ascii COLLATE ascii_bin NOT NULL, - consecutive_failures INT UNSIGNED NOT NULL DEFAULT 0, - last_window_start BIGINT UNSIGNED DEFAULT NULL, - last_success_at BIGINT UNSIGNED DEFAULT NULL, - last_error_code VARCHAR(64) CHARACTER SET ascii COLLATE ascii_bin NOT NULL DEFAULT '', - updated_at BIGINT UNSIGNED NOT NULL, - PRIMARY KEY (host_id), - KEY cloud_network_monitor_cursor_health ( - collection_state, last_success_at, host_id - ), - KEY cloud_network_monitor_cursor_due ( - next_poll_at, poll_revision, host_id - ), - KEY cloud_network_monitor_cursor_lease ( - poll_lease_until, host_id - ), - CONSTRAINT chk_cloud_network_monitor_cursor_state CHECK ( - collection_state IN ( - 'unknown','healthy','degraded','stale','unavailable' - ) - ), - CONSTRAINT chk_cloud_network_monitor_cursor_generation CHECK ( - (collector_generation IS NULL AND cursor_value = 0 - AND oldest_cursor = 0 AND gap_before = 0 - AND collection_state = 'unknown' AND consecutive_failures = 0 - AND last_window_start IS NULL AND last_success_at IS NULL) - OR (collector_generation IS NOT NULL - AND collector_generation REGEXP '^[a-f0-9]{32}$' - AND gap_before IN (0, 1) - AND ((oldest_cursor = 0 AND cursor_value = 0 AND gap_before = 0) - OR (oldest_cursor > 0 AND oldest_cursor <= cursor_value))) - ), - CONSTRAINT chk_cloud_network_monitor_cursor_lease CHECK ( - (poll_lease_token IS NULL AND poll_lease_until = 0) - OR (poll_lease_token IS NOT NULL - AND poll_lease_token REGEXP '^[a-f0-9]{32}$' - AND poll_lease_until > 0) - ) -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; - --- Compact historical baseline owned by the logical instance rather than the --- current host. metrics_json contains aggregate statistical state only and --- therefore survives maintenance migration without carrying flow identities. -CREATE TABLE cloud_vm_network_baselines ( - instance_id BIGINT UNSIGNED NOT NULL, - user_id BIGINT UNSIGNED NOT NULL, - rule_version INT UNSIGNED NOT NULL, - policy_revision BIGINT UNSIGNED NOT NULL, - policy_hash CHAR(64) CHARACTER SET ascii COLLATE ascii_bin NOT NULL, - sample_count INT UNSIGNED NOT NULL DEFAULT 0, - warmup_state VARCHAR(16) CHARACTER SET ascii COLLATE ascii_bin NOT NULL, - metrics_json JSON NOT NULL, - last_projection_key CHAR(64) CHARACTER SET ascii COLLATE ascii_bin DEFAULT NULL, - last_bucket_start BIGINT UNSIGNED DEFAULT NULL, - projection_lease_token CHAR(32) CHARACTER SET ascii COLLATE ascii_bin DEFAULT NULL, - projection_lease_until BIGINT UNSIGNED NOT NULL DEFAULT 0, - revision BIGINT UNSIGNED NOT NULL DEFAULT 0, - created_at BIGINT UNSIGNED NOT NULL, - updated_at BIGINT UNSIGNED NOT NULL, - PRIMARY KEY (instance_id), - KEY cloud_network_baseline_owner (user_id, updated_at), - KEY cloud_network_baseline_warmup (warmup_state, sample_count), - KEY cloud_network_baseline_projection_lease ( - projection_lease_until, instance_id - ), - CONSTRAINT chk_cloud_network_baseline_warmup CHECK ( - warmup_state IN ('warming','ready') - ), - CONSTRAINT chk_cloud_network_baseline_policy CHECK ( - rule_version > 0 - AND policy_revision > 0 - AND policy_hash REGEXP '^[a-f0-9]{64}$' - ), - CONSTRAINT chk_cloud_network_baseline_projection CHECK ( - (last_projection_key IS NULL AND last_bucket_start IS NULL) - OR (last_projection_key IS NOT NULL - AND last_projection_key REGEXP '^[a-f0-9]{64}$' - AND last_bucket_start IS NOT NULL - AND last_bucket_start % 60 = 0) - ), - CONSTRAINT chk_cloud_network_baseline_projection_lease CHECK ( - (projection_lease_token IS NULL AND projection_lease_until = 0) - OR (projection_lease_token IS NOT NULL - AND projection_lease_token REGEXP '^[a-f0-9]{32}$' - AND projection_lease_until > 0) - ) -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; diff --git a/web/tests/CloudAbuseAlertEventCoordinatorTest.php b/web/tests/CloudAbuseAlertEventCoordinatorTest.php deleted file mode 100644 index 8697195e..00000000 --- a/web/tests/CloudAbuseAlertEventCoordinatorTest.php +++ /dev/null @@ -1,198 +0,0 @@ -addConnection([ - 'driver' => 'sqlite', 'database' => ':memory:', -], 'default'); -$capsule->setAsGlobal(); -$capsule->bootEloquent(); -$main = $capsule->connection('default'); -$main->statement('PRAGMA foreign_keys = ON'); -$schema = $capsule->schema('default'); -$schema->create('cloud_instances', static function (Blueprint $table): void { - $table->increments('id'); -}); -$schema->create('cloud_abuse_alert_events', - static function (Blueprint $table): void { - $table->increments('id'); - $table->string('event_key')->unique(); - $table->string('sample_key')->unique(); - $table->integer('instance_id'); - $table->integer('user_id'); - $table->integer('host_id'); - $table->integer('bucket_start'); - $table->integer('rule_version'); - $table->integer('policy_revision'); - $table->string('policy_hash'); - $table->string('transition_digest'); - $table->integer('transition_count'); - $table->string('highest_severity'); - $table->integer('created_at'); - }); -$schema->create('cloud_abuse_alert_event_items', - static function (Blueprint $table): void { - $table->increments('id'); - $table->integer('event_id'); - $table->string('item_key')->unique(); - $table->string('finding_key'); - $table->integer('occurrence_generation'); - $table->integer('transition_revision'); - $table->string('rule_code'); - $table->string('direction'); - $table->string('ip_family'); - $table->string('transition_type'); - $table->string('from_severity'); - $table->string('to_severity'); - $table->text('evidence_json'); - $table->string('resolution_reason')->default(''); - $table->integer('occurred_at'); - $table->integer('created_at'); - $table->foreign('event_id')->references('id') - ->on('cloud_abuse_alert_events')->cascadeOnDelete(); - }); - -$main->table('cloud_instances')->insert(['id' => 41]); -$policyHash = str_repeat('a', 64); -$claim = [ - 'sample_key' => str_repeat('b', 64), - 'instance_id' => 41, - 'user_id' => 9, - 'host_id' => 3, - 'bucket_start' => 120, - 'rule_version' => 1, - 'policy_revision' => 7, - 'policy_hash' => $policyHash, -]; -$transitions = [[ - 'finding_key' => str_repeat('c', 64), - 'occurrence_generation' => 1, - 'transition_revision' => 1, - 'rule_code' => 'outbound_tcp_syn_sweep', - 'direction' => 'outbound', - 'ip_family' => 'ipv4', - 'transition_type' => 'opened', - 'from_severity' => 'none', - 'to_severity' => 'suspicious', - 'evidence' => ['tcp_syn_flows' => 900, 'unique_remote_ips' => 140], - 'resolution_reason' => '', - 'occurred_at' => 120, -], [ - 'finding_key' => str_repeat('d', 64), - 'occurrence_generation' => 2, - 'transition_revision' => 4, - 'rule_code' => 'inbound_udp_flood', - 'direction' => 'inbound', - 'ip_family' => 'ipv6', - 'transition_type' => 'resolved', - 'from_severity' => 'high', - 'to_severity' => 'none', - 'evidence' => ['udp_active_flows' => 0, 'udp_new_flows' => 0], - 'resolution_reason' => 'evidence_clear', - 'occurred_at' => 120, -]]; - -$coordinator = new CloudAbuseAlertEventCoordinator(); -$main->transaction(static function () use ( - $coordinator, - $main, - $claim, - $transitions -): void { - $coordinator->record($main, $claim, array_reverse($transitions), 1000); -}); -$event = $main->table('cloud_abuse_alert_events')->first(); -$items = $main->table('cloud_abuse_alert_event_items') - ->orderBy('finding_key')->get(); -$assert($event !== null - && (int) $event->transition_count === 2 - && $event->highest_severity === 'high' - && count($items) === 2 - && $items[0]->finding_key === str_repeat('c', 64) - && $items[0]->evidence_json - === '{"tcp_syn_flows":900,"unique_remote_ips":140}', - 'Multiple transitions were not frozen in one canonical event batch.'); - -$coordinator->record($main, $claim, $transitions, 1001); -$assert($main->table('cloud_abuse_alert_events')->count() === 1 - && $main->table('cloud_abuse_alert_event_items')->count() === 2, - 'An identical event replay duplicated the durable ledger.'); - -$invalidEvidence = $transitions; -$invalidEvidence[0]['evidence'] = ['remote_ip' => '203.0.113.4']; -$threw = false; -try { - $coordinator->record($main, $claim + [ - 'sample_key' => str_repeat('e', 64), - ], $invalidEvidence, 1002); -} catch (LogicException) { - $threw = true; -} -$assert($threw && $main->table('cloud_abuse_alert_events')->count() === 1, - 'Non-aggregate evidence was accepted by the alert ledger.'); - -$wrongBucket = $transitions; -$wrongBucket[0]['occurred_at'] = 60; -$threw = false; -try { - $coordinator->record($main, $claim + [ - 'sample_key' => str_repeat('f', 64), - ], $wrongBucket, 1003); -} catch (LogicException) { - $threw = true; -} -$assert($threw, 'A transition from another sample bucket was accepted.'); - -$rollbackClaim = $claim; -$rollbackClaim['sample_key'] = str_repeat('1', 64); -$rolledBack = false; -try { - $main->transaction(static function () use ( - $coordinator, - $main, - $rollbackClaim, - $transitions - ): void { - $coordinator->record( - $main, - $rollbackClaim, - $transitions, - 1004 - ); - throw new RuntimeException('force rollback'); - }); -} catch (RuntimeException $error) { - $rolledBack = $error->getMessage() === 'force rollback'; -} -$assert($rolledBack - && !$main->table('cloud_abuse_alert_events') - ->where('sample_key', $rollbackClaim['sample_key'])->exists(), - 'A rolled-back alert event left a partial durable ledger.'); - -$main->table('cloud_instances')->where('id', 41)->delete(); -$assert($main->table('cloud_abuse_alert_events')->count() === 1 - && $main->table('cloud_abuse_alert_event_items')->count() === 2, - 'Deleting an instance erased an alert before delivery finality exists.'); -$main->table('cloud_abuse_alert_events')->where('id', $event->id)->delete(); -$assert($main->table('cloud_abuse_alert_event_items')->count() === 0, - 'Deleting an alert event did not cascade its immutable items.'); -$source = file_get_contents(dirname(__DIR__) - . '/app/Services/Cloud/CloudAbuseAlertEventCoordinator.php'); -$assert(is_string($source) - && !str_contains($source, 'Telegram') - && !str_contains($source, 'TicketTelegram') - && !str_contains($source, 'Http::'), - 'Phase-6 event recording gained an external delivery dependency.'); - -echo "Cloud abuse alert event coordinator tests passed.\n"; diff --git a/web/tests/CloudAbuseMonitoringSchemaContractTest.php b/web/tests/CloudAbuseMonitoringSchemaContractTest.php deleted file mode 100644 index 45b3279a..00000000 --- a/web/tests/CloudAbuseMonitoringSchemaContractTest.php +++ /dev/null @@ -1,431 +0,0 @@ - [ - 'instance_id', 'user_id', 'host_id', 'rule_version', - 'policy_revision', 'policy_hash', 'risk_state', - 'collection_state', 'latest_direction', 'latest_rule_code', - 'latest_severity', 'summary_json', 'last_projection_key', - 'last_bucket_start', 'last_sample_at', 'last_evaluated_at', - 'created_at', 'updated_at', - ], - 'cloud_abuse_findings' => [ - 'id', 'finding_key', 'instance_id', 'user_id', 'host_id', - 'rule_code', 'rule_version', 'policy_revision', 'policy_hash', - 'direction', 'ip_family', 'severity', - 'state', 'first_seen_at', 'last_seen_at', 'consecutive_windows', - 'occurrence_generation', 'transition_revision', 'evidence_json', - 'resolved_at', 'resolution_reason', - 'created_at', 'updated_at', - ], - 'cloud_abuse_alert_events' => [ - 'id', 'event_key', 'sample_key', 'instance_id', 'user_id', - 'host_id', 'bucket_start', 'rule_version', 'policy_revision', - 'policy_hash', 'transition_digest', 'transition_count', - 'highest_severity', 'created_at', - ], - 'cloud_abuse_alert_event_items' => [ - 'id', 'event_id', 'item_key', 'finding_key', - 'occurrence_generation', 'transition_revision', 'rule_code', - 'direction', 'ip_family', 'transition_type', 'from_severity', - 'to_severity', 'evidence_json', 'resolution_reason', - 'occurred_at', 'created_at', - ], - 'cloud_abuse_alert_deliveries' => [ - 'id', 'event_id', 'binding_generation', 'bound_admin_id', - 'chat_id', 'telegram_message_id', 'state', 'message_text', - 'content_hash', 'error_code', 'skip_reason', - 'dispatch_started_at', 'sent_at', 'terminal_at', - 'destination_redacted_at', 'created_at', 'updated_at', - ], - 'administrator_telegram_delivery_gate' => [ - 'id', 'next_send_at', 'updated_at', - ], - 'cloud_abuse_projection_receipts' => [ - 'sample_key', 'instance_id', 'rule_version', 'policy_revision', - 'policy_hash', 'bucket_start', 'projected_at', - ], -]; -$logTables = [ - 'cloud_vm_network_observation_rollups' => [ - 'instance_id', 'user_id', 'host_id', 'interval_seconds', - 'bucket_start', 'sample_count', 'complete_count', 'quality_mask', - 'rx_bytes_min', 'rx_bytes_max', 'tx_bytes_min', 'tx_bytes_max', - 'rx_packets_min', 'rx_packets_max', 'tx_packets_min', - 'tx_packets_max', 'tcp_active_peak', 'tcp_new_total', - 'tcp_syn_total', 'tcp_unreplied_total', 'udp_active_peak', - 'udp_new_total', 'peer_ip_cardinality_peak', - 'peer_port_cardinality_peak', - 'first_sample_at', 'last_sample_at', 'created_at', 'updated_at', - ], - 'cloud_vm_network_observation_samples' => [ - 'sample_key', 'instance_id', 'user_id', 'host_id', 'vm_uuid', - 'placement_key', 'collector_generation', 'window_id', - 'contract_version', 'bucket_start', 'bucket_seconds', 'complete', - 'metrics_json', - 'quality_flags_json', 'received_at', - ], - 'cloud_vm_network_projection_outbox' => [ - 'sample_key', 'instance_id', 'bucket_start', 'placement_key', - 'rule_version', 'policy_revision', 'policy_hash', 'state', - 'attempts', 'available_at', 'lease_token', 'lease_until', - 'last_error_code', 'projected_at', 'terminal_at', - 'discard_reason_code', 'created_at', 'updated_at', - ], - 'cloud_vm_network_monitor_cursors' => [ - 'host_id', 'collector_generation', 'cursor_value', - 'oldest_cursor', 'gap_before', 'poll_revision', 'next_poll_at', - 'poll_lease_token', 'poll_lease_until', 'collection_state', - 'consecutive_failures', 'last_window_start', - 'last_success_at', 'last_error_code', 'updated_at', - ], - 'cloud_vm_network_baselines' => [ - 'instance_id', 'user_id', 'rule_version', 'policy_revision', - 'policy_hash', 'sample_count', - 'warmup_state', 'metrics_json', 'last_projection_key', - 'last_bucket_start', 'projection_lease_token', - 'projection_lease_until', 'revision', 'created_at', 'updated_at', - ], -]; - -$definitions = []; -foreach ($mainTables as $table => $columns) { - $definitions[$table] = $tableSql($main, $table); - $assert(str_contains($setup, "'" . $table . "'"), - 'CloudSetup does not require main abuse table: ' . $table); - foreach ($columns as $column) { - $assert(preg_match('/\b' . preg_quote($column, '/') . '\b/', - $definitions[$table]) === 1, - 'Missing main abuse column: ' . $table . '.' . $column); - $assert(str_contains($setup, "'" . $column . "'"), - 'CloudSetup does not require main abuse column: ' - . $table . '.' . $column); - } -} -foreach ($logTables as $table => $columns) { - $definitions[$table] = $tableSql($log, $table); - $assert(str_contains($setup, "'" . $table . "'"), - 'CloudSetup does not require log abuse table: ' . $table); - foreach ($columns as $column) { - $assert(preg_match('/\b' . preg_quote($column, '/') . '\b/', - $definitions[$table]) === 1, - 'Missing log abuse column: ' . $table . '.' . $column); - $assert(str_contains($setup, "'" . $column . "'"), - 'CloudSetup does not require log abuse column: ' - . $table . '.' . $column); - } -} - -foreach ([ - 'cloud_abuse_state_risk', 'cloud_abuse_state_collection', - 'cloud_abuse_state_host', 'cloud_abuse_state_owner', -] as $index) { - $assert(str_contains($definitions['cloud_abuse_instance_states'], $index) - && str_contains($setup, "'" . $index . "'"), - 'Missing abuse state index: ' . $index); -} -foreach ([ - 'cloud_abuse_alert_event_key', 'cloud_abuse_alert_event_sample', - 'cloud_abuse_alert_event_instance', 'cloud_abuse_alert_event_owner', - 'cloud_abuse_alert_event_host', 'cloud_abuse_alert_event_retention', -] as $index) { - $assert(str_contains($definitions['cloud_abuse_alert_events'], $index) - && str_contains($setup, "'" . $index . "'"), - 'Missing abuse alert event index: ' . $index); -} -foreach ([ - 'cloud_abuse_alert_item_key', 'cloud_abuse_alert_item_finding_once', - 'cloud_abuse_alert_item_event', 'cloud_abuse_alert_item_finding', -] as $index) { - $assert(str_contains( - $definitions['cloud_abuse_alert_event_items'], $index - ) && str_contains($setup, "'" . $index . "'"), - 'Missing abuse alert item index: ' . $index); -} -foreach ([ - 'cloud_abuse_alert_delivery_event', - 'cloud_abuse_alert_delivery_message', - 'cloud_abuse_alert_delivery_due', - 'cloud_abuse_alert_delivery_admin', - 'cloud_abuse_alert_delivery_terminal', -] as $index) { - $assert(str_contains( - $definitions['cloud_abuse_alert_deliveries'], $index - ) && str_contains($setup, "'" . $index . "'"), - 'Missing abuse alert delivery index: ' . $index); -} -foreach ([ - 'cloud_abuse_finding_key', 'cloud_abuse_finding_instance', - 'cloud_abuse_finding_owner', 'cloud_abuse_finding_host', - 'cloud_abuse_finding_rule', 'cloud_abuse_finding_retention', -] as $index) { - $assert(str_contains($definitions['cloud_abuse_findings'], $index) - && str_contains($setup, "'" . $index . "'"), - 'Missing abuse finding index: ' . $index); -} -foreach ([ - 'cloud_network_rollup_owner', - 'cloud_network_rollup_host', - 'cloud_network_rollup_retention', -] as $index) { - $assert(str_contains( - $definitions['cloud_vm_network_observation_rollups'], $index - ) && str_contains($setup, "'" . $index . "'"), - 'Missing observation rollup index: ' . $index); -} -foreach ([ - 'cloud_network_observation_source', - 'cloud_network_observation_instance', - 'cloud_network_observation_retention', -] as $index) { - $assert(str_contains( - $definitions['cloud_vm_network_observation_samples'], $index - ) && str_contains($setup, "'" . $index . "'"), - 'Missing observation index: ' . $index); -} -$assert(!str_contains( - $definitions['cloud_vm_network_observation_samples'], - 'cloud_network_observation_placement' - ) && !str_contains( - $definitions['cloud_vm_network_observation_samples'], - 'cloud_network_observation_host' - ), - 'Observation storage retained redundant wide placement/host indexes.'); -foreach ([ - 'cloud_network_projection_due', - 'cloud_network_projection_instance', - 'cloud_network_projection_lease', - 'cloud_network_projection_retention', -] as $index) { - $assert(str_contains( - $definitions['cloud_vm_network_projection_outbox'], $index - ) && str_contains($setup, "'" . $index . "'"), - 'Missing projection index: ' . $index); -} -foreach ([ - 'cloud_network_monitor_cursor_health', - 'cloud_network_monitor_cursor_due', - 'cloud_network_monitor_cursor_lease', -] as $index) { - $assert(str_contains( - $definitions['cloud_vm_network_monitor_cursors'], $index - ) && str_contains($setup, "'" . $index . "'"), - 'Missing cursor index: ' . $index); -} -foreach ([ - 'cloud_network_baseline_owner', - 'cloud_network_baseline_warmup', - 'cloud_network_baseline_projection_lease', -] as $index) { - $assert(str_contains( - $definitions['cloud_vm_network_baselines'], $index - ) && str_contains($setup, "'" . $index . "'"), - 'Missing baseline index: ' . $index); -} -foreach ([ - 'chk_cloud_network_rollup_bucket', - 'chk_cloud_network_rollup_counts', - 'chk_cloud_network_rollup_counters', -] as $check) { - $assert(str_contains($log, 'CONSTRAINT ' . $check . ' CHECK') - && str_contains($setup, "'" . $check . "'"), - 'CloudSetup does not require rollup CHECK: ' . $check); -} -foreach ([ - 'chk_cloud_network_observation_contract', - 'chk_cloud_network_observation_bucket', - 'chk_cloud_network_observation_complete', - 'chk_cloud_network_projection_key', - 'chk_cloud_network_projection_policy', - 'chk_cloud_network_projection_state', - 'chk_cloud_network_projection_lease', - 'chk_cloud_network_monitor_cursor_state', - 'chk_cloud_network_monitor_cursor_generation', - 'chk_cloud_network_monitor_cursor_lease', - 'chk_cloud_network_baseline_warmup', - 'chk_cloud_network_baseline_policy', - 'chk_cloud_network_baseline_projection', - 'chk_cloud_network_baseline_projection_lease', -] as $check) { - $assert(str_contains($log, 'CONSTRAINT ' . $check . ' CHECK') - && str_contains($setup, "'" . $check . "'"), - 'CloudSetup does not require log CHECK: ' . $check); -} - -$assert(str_contains($definitions['cloud_abuse_instance_states'], - "risk_state IN ('observing','normal','suspicious','high_risk')") - && str_contains($definitions['cloud_abuse_instance_states'], - "'unknown','healthy','degraded','stale','unavailable'") - && str_contains($definitions['cloud_abuse_findings'], - "direction IN ('inbound','outbound')") - && str_contains($definitions['cloud_abuse_findings'], - "ip_family IN ('ipv4','ipv6','mixed')") - && str_contains($definitions['cloud_abuse_findings'], - "state IN ('open','resolved')") - && str_contains($definitions['cloud_abuse_instance_states'], - 'chk_cloud_abuse_state_policy CHECK') - && str_contains($definitions['cloud_abuse_findings'], - 'chk_cloud_abuse_finding_policy CHECK') - && str_contains($definitions['cloud_abuse_projection_receipts'], - 'chk_cloud_abuse_projection_policy CHECK') - && str_contains($definitions['cloud_abuse_findings'], - "resolution_reason IN (\n 'evidence_clear','policy_replaced','owner_changed'"), - 'Abuse-monitoring state constraints are incomplete.'); -$assert(str_contains($definitions['cloud_abuse_alert_events'], - 'chk_cloud_abuse_alert_event_identity CHECK') - && str_contains($definitions['cloud_abuse_alert_events'], - "highest_severity IN ('suspicious','high')") - && str_contains($definitions['cloud_abuse_alert_event_items'], - 'chk_cloud_abuse_alert_item_transition CHECK') - && str_contains($definitions['cloud_abuse_alert_event_items'], - "transition_type = 'opened'") - && str_contains($definitions['cloud_abuse_alert_event_items'], - "transition_type = 'escalated'") - && str_contains($definitions['cloud_abuse_alert_event_items'], - "transition_type = 'resolved'"), - 'Abuse alert event transition constraints are incomplete.'); -$delivery = $definitions['cloud_abuse_alert_deliveries']; -$assert(str_contains($delivery, - 'chk_cloud_abuse_alert_delivery_destination CHECK') - && str_contains($delivery, 'binding_generation IS NOT NULL') - && str_contains($delivery, 'chat_id IS NOT NULL') - && str_contains($delivery, 'OR telegram_message_id > 0') - && str_contains($delivery, - "state = 'failed'\n AND telegram_message_id IS NULL\n AND dispatch_started_at IS NULL") - && str_contains($delivery, - "state = 'unknown'\n AND dispatch_started_at IS NOT NULL") - && str_contains($definitions['administrator_telegram_delivery_gate'], - 'chk_administrator_telegram_delivery_gate CHECK') - && str_contains($setup, - "'chk_cloud_abuse_alert_delivery_destination'") - && str_contains($setup, - "'chk_administrator_telegram_delivery_gate'"), - 'Abuse delivery or shared Telegram gate constraints are incomplete.'); - -$assert(str_contains($definitions['cloud_abuse_instance_states'], - 'fk_cloud_abuse_state_instance FOREIGN KEY (instance_id)') - && str_contains($definitions['cloud_abuse_instance_states'], - 'REFERENCES cloud_instances (id) ON DELETE CASCADE') - && str_contains($definitions['cloud_abuse_findings'], - 'fk_cloud_abuse_finding_instance FOREIGN KEY (instance_id)') - && str_contains($definitions['cloud_abuse_findings'], - 'REFERENCES cloud_instances (id) ON DELETE CASCADE') - && str_contains($definitions['cloud_abuse_projection_receipts'], - 'fk_cloud_abuse_projection_instance FOREIGN KEY (instance_id)') - && str_contains($definitions['cloud_abuse_projection_receipts'], - 'REFERENCES cloud_instances (id) ON DELETE CASCADE') - && str_contains($setup, "'fk_cloud_abuse_state_instance'") - && str_contains($setup, "'fk_cloud_abuse_finding_instance'") - && str_contains($setup, "'fk_cloud_abuse_projection_instance'"), - 'Current state or finding rows can outlive a deleted instance.'); -$assert(!str_contains($definitions['cloud_abuse_alert_events'], - 'REFERENCES cloud_instances') - && str_contains($definitions['cloud_abuse_alert_event_items'], - 'fk_cloud_abuse_alert_item_event FOREIGN KEY (event_id)') - && str_contains($definitions['cloud_abuse_alert_event_items'], - 'ON DELETE CASCADE ON UPDATE RESTRICT'), - 'Alert events are not independent or item cleanup is not bounded.'); -$assert(str_contains($delivery, - 'fk_cloud_abuse_alert_delivery_event FOREIGN KEY (event_id)') - && str_contains($delivery, - 'ON DELETE CASCADE ON UPDATE RESTRICT') - && str_contains($setup, - "'fk_cloud_abuse_alert_delivery_event'"), - 'Abuse delivery does not cascade only from its immutable event.'); - -$outboxDelete = strpos( - $deletion, "'cloud_vm_network_projection_outbox'" -); -$sampleDelete = strrpos( - $deletion, "'cloud_vm_network_observation_samples'" -); -$assert($outboxDelete !== false && $sampleDelete !== false - && $outboxDelete < $sampleDelete - && str_contains($deletion, "'cloud_vm_network_baselines'") - && str_contains($deletion, "'cloud_vm_network_observation_rollups'"), - 'Account deletion leaves tenant observation history in the log database.'); -$assert(str_contains($deletion, - "['cloud_abuse_alert_events', 'user_id']") - && str_contains($deletion, - "['cloud_abuse_findings', 'user_id']") - && str_contains($deletion, - "['cloud_abuse_instance_states', 'user_id']"), - 'Account deletion does not erase frozen old-owner abuse data.'); -$ownerLock = strpos($projection, "->table('user')"); -$instanceLock = strpos($projection, "->table('cloud_instances')"); -$assert($ownerLock !== false && $instanceLock !== false - && $ownerLock < $instanceLock, - 'Projection does not share the account deletion user-before-instance lock order.'); - -$assert(str_contains( - $definitions['cloud_vm_network_projection_outbox'], - 'ON DELETE RESTRICT ON UPDATE RESTRICT' - ) && !str_contains( - $definitions['cloud_vm_network_projection_outbox'], - 'ON DELETE CASCADE' - ) && str_contains($setup, "'delete_rule' => 'RESTRICT'") - && str_contains($setup, "'update_rule' => 'RESTRICT'"), - 'Pending or leased projection samples are not deletion-protected.'); - -$assert(str_contains( - $definitions['cloud_vm_network_observation_samples'], - 'contract_version = 1' - ) && str_contains( - $definitions['cloud_vm_network_observation_samples'], - 'bucket_seconds = 60' - ) && str_contains( - $definitions['cloud_vm_network_monitor_cursors'], - "'unknown','healthy','degraded','stale','unavailable'" - ) && str_contains( - $definitions['cloud_vm_network_baselines'], - "warmup_state IN ('warming','ready')" - ), 'Observation quality or baseline constraints are incomplete.'); - -$assert(str_contains( - $setup, - "SCHEMA_VERSION = 'go-agent-wss-v56-daily-traffic-settlement'" - ), 'CloudSetup schema version was not advanced for abuse observation.'); - -foreach (array_keys($mainTables + $logTables) as $table) { - $assert(!str_contains($traffic, $table) - && !str_contains($lifecycle, $table), - 'Abuse monitoring leaked into billing/lifecycle sampling: ' . $table); -} -$forbiddenIdentityColumn = '/^\s*`?remote_(?:ip|port)[a-z0-9_]*`?\s+/mi'; -$assert(preg_match($forbiddenIdentityColumn, $main) !== 1 - && preg_match($forbiddenIdentityColumn, $log) !== 1, - 'Schema stores forbidden remote flow identities.'); - -echo "Abuse monitoring schema contract passed.\n"; diff --git a/web/tests/CloudAbusePolicySchemaContractTest.php b/web/tests/CloudAbusePolicySchemaContractTest.php index b632377b..87c2d5cd 100644 --- a/web/tests/CloudAbusePolicySchemaContractTest.php +++ b/web/tests/CloudAbusePolicySchemaContractTest.php @@ -96,7 +96,7 @@ foreach ($defaults->rules() as $code => $rule) { 'The fresh schema rule seed drifted from PHP: ' . $code); } $assert(str_contains($setup, - "SCHEMA_VERSION = 'go-agent-wss-v56-daily-traffic-settlement'") + "SCHEMA_VERSION = 'go-agent-wss-v57-compact-abuse-ledger'") && str_contains($service, "->where('active_slot', 1)->lockForUpdate()") && str_contains($service, "->where('active_slot', 1)->update") && str_contains($service, 'insertGetId'), diff --git a/web/tests/CloudAbuseProjectionLedgerContractTest.php b/web/tests/CloudAbuseProjectionLedgerContractTest.php deleted file mode 100644 index 42c31561..00000000 --- a/web/tests/CloudAbuseProjectionLedgerContractTest.php +++ /dev/null @@ -1,366 +0,0 @@ - 0 AND oldest_cursor <= cursor_value') - && str_contains($cursor, - "poll_lease_token REGEXP '^[a-f0-9]{32}$'") - && preg_match( - '/cloud_network_monitor_cursor_due \(\s*next_poll_at, poll_revision, host_id\s*\)/', - $cursor - ) === 1 - && preg_match( - '/cloud_network_monitor_cursor_lease \(\s*poll_lease_until, host_id\s*\)/', - $cursor - ) === 1, - 'Host cursor lacks strict initial, CAS, due or lease semantics.'); - -$assert(str_contains($baseline, 'last_projection_key CHAR(64)') - && str_contains($baseline, 'rule_version INT UNSIGNED NOT NULL') - && str_contains($baseline, 'policy_revision BIGINT UNSIGNED NOT NULL') - && str_contains($baseline, - 'policy_hash CHAR(64) CHARACTER SET ascii COLLATE ascii_bin NOT NULL') - && str_contains($baseline, - 'chk_cloud_network_baseline_policy CHECK') - && str_contains($baseline, 'last_bucket_start BIGINT UNSIGNED') - && str_contains($baseline, - 'last_projection_key IS NULL AND last_bucket_start IS NULL') - && str_contains($baseline, - "last_projection_key REGEXP '^[a-f0-9]{64}$'") - && str_contains($baseline, 'last_bucket_start % 60 = 0') - && str_contains($baseline, - 'projection_lease_token CHAR(32) CHARACTER SET ascii COLLATE ascii_bin DEFAULT NULL') - && str_contains($baseline, - 'projection_lease_until BIGINT UNSIGNED NOT NULL DEFAULT 0') - && str_contains($baseline, 'revision BIGINT UNSIGNED NOT NULL DEFAULT 0') - && str_contains($baseline, - "projection_lease_token REGEXP '^[a-f0-9]{32}$'") - && preg_match( - '/cloud_network_baseline_projection_lease \(\s*projection_lease_until, instance_id\s*\)/', - $baseline - ) === 1, - 'Baseline lacks a paired projection fence or recoverable instance lease.'); - -$assert(str_contains($receipt, - 'sample_key CHAR(64) CHARACTER SET ascii COLLATE ascii_bin NOT NULL PRIMARY KEY') - && str_contains($receipt, 'rule_version INT UNSIGNED NOT NULL') - && str_contains($receipt, 'policy_revision BIGINT UNSIGNED NOT NULL') - && str_contains($receipt, - 'policy_hash CHAR(64) CHARACTER SET ascii COLLATE ascii_bin NOT NULL') - && str_contains($receipt, 'chk_cloud_abuse_projection_policy CHECK') - && str_contains($receipt, - 'cloud_abuse_projection_instance') - && str_contains($receipt, - 'cloud_abuse_projection_retention (projected_at, sample_key)') - && str_contains($receipt, - 'fk_cloud_abuse_projection_instance FOREIGN KEY (instance_id)') - && str_contains($receipt, - 'ON DELETE CASCADE ON UPDATE RESTRICT'), - 'Main projection receipt is not idempotent, retained or owner-bound.'); - -$assert(str_contains($state, 'last_projection_key CHAR(64)') - && str_contains($state, 'rule_version INT UNSIGNED NOT NULL') - && str_contains($state, 'policy_revision BIGINT UNSIGNED NOT NULL') - && str_contains($state, - 'policy_hash CHAR(64) CHARACTER SET ascii COLLATE ascii_bin NOT NULL') - && str_contains($state, 'chk_cloud_abuse_state_policy CHECK') - && str_contains($state, 'last_bucket_start BIGINT UNSIGNED') - && str_contains($state, - 'last_projection_key IS NOT NULL') - && str_contains($state, - "last_projection_key REGEXP '^[a-f0-9]{64}$'") - && str_contains($state, - 'last_projection_key IS NULL AND last_bucket_start IS NULL'), - 'Current state lacks the projection ordering fence.'); - -$assert(str_contains($finding, - 'cloud_abuse_finding_retention (') - && str_contains($finding, 'policy_revision BIGINT UNSIGNED NOT NULL') - && str_contains($finding, - 'policy_hash CHAR(64) CHARACTER SET ascii COLLATE ascii_bin NOT NULL') - && str_contains($finding, 'chk_cloud_abuse_finding_policy CHECK') - && str_contains($finding, - "resolution_reason IN (\n 'evidence_clear','policy_replaced','owner_changed'") - && str_contains($finding, - 'state, resolved_at, last_seen_at, id') - && str_contains($finding, - "finding_key REGEXP '^[a-f0-9]{64}$'") - && str_contains($finding, - "state = 'resolved' AND resolved_at IS NOT NULL") - && str_contains($finding, - 'resolved_at >= last_seen_at'), - 'Finding retention or terminal-time contract is incomplete.'); - -$findingIdentityStart = strpos( - $projection, - '$key = hash(\'sha256\', implode("\\n", [' -); -$findingIdentityEnd = $findingIdentityStart === false - ? false - : strpos($projection, ']));', $findingIdentityStart); -$findingIdentity = $findingIdentityStart === false - || $findingIdentityEnd === false - ? '' - : substr( - $projection, - $findingIdentityStart, - $findingIdentityEnd - $findingIdentityStart - ); -$lastIdentityOffset = -1; -foreach ([ - '$claim[\'instance_id\']', - '$claim[\'rule_version\']', - '$claim[\'policy_hash\']', - '$finding[\'rule_code\']', - '$finding[\'ip_family\']', - '$finding[\'direction\']', -] as $identityPart) { - $identityOffset = strpos($findingIdentity, $identityPart); - $assert($identityOffset !== false && $identityOffset > $lastIdentityOffset, - 'Finding identity does not include ordered part: ' . $identityPart); - $lastIdentityOffset = $identityOffset; -} -$assert(str_contains($projection, '->snapshotByIdentity($revision, $hash)') - && !str_contains($projection, '->activeSnapshot()'), - 'Projection can drift from the outbox identity to the current policy.'); -$assert(str_contains($deployment, - '`go-agent-wss-v56-daily-traffic-settlement`') - && str_contains($deployment, - '`occurrence_generation = 1` 与 `transition_revision = 1`') - && str_contains($deployment, - '历史 finding 不反向合成事件') - && str_contains($deployment, - '`policy_replaced` 与 `owner_changed` 只关闭 finding'), - 'Deployment contract does not define the v45 event-ledger cutover.'); - -foreach ([ - 'cloud_vm_network_projection_outbox', - 'cloud_abuse_projection_receipts', -] as $requiredTable) { - $assert(str_contains($setup, "'" . $requiredTable . "'"), - 'CloudSetup does not require projection table: ' . $requiredTable); -} -foreach ([ - 'cloud_abuse_projection_retention', - 'fk_cloud_abuse_projection_instance', - 'chk_cloud_abuse_state_projection', - 'chk_cloud_abuse_state_policy', - 'chk_cloud_abuse_finding_key', - 'chk_cloud_abuse_finding_policy', - 'chk_cloud_abuse_projection_policy', - 'chk_cloud_network_projection_lease', - 'chk_cloud_network_projection_policy', - 'chk_cloud_network_monitor_cursor_generation', - 'chk_cloud_network_baseline_projection_lease', - 'chk_cloud_network_baseline_policy', - 'cloud_network_projection_due', - 'cloud_network_projection_instance', - 'cloud_network_projection_retention', - 'cloud_network_monitor_cursor_due', - 'cloud_network_monitor_cursor_lease', - 'cloud_network_baseline_projection_lease', - 'fk_cloud_network_projection_sample', -] as $requiredObject) { - $assert(str_contains($setup, "'" . $requiredObject . "'"), - 'CloudSetup does not verify projection object: ' . $requiredObject); -} - -foreach ([ - "'cloud_network_observation_source'=>['non_unique'=>0,'columns'=>['host_id','collector_generation','window_id','vm_uuid',]", - "'cloud_network_observation_instance'=>['non_unique'=>1,'columns'=>['instance_id','bucket_start'],]", - "'cloud_network_observation_retention'=>['non_unique'=>1,'columns'=>['bucket_start'],]", - "'cloud_network_projection_due'=>['non_unique'=>1,'columns'=>['state','available_at','bucket_start','sample_key',]", - "'cloud_network_projection_instance'=>['non_unique'=>1,'columns'=>['instance_id','state','bucket_start','sample_key',]", - "'cloud_network_projection_lease'=>['non_unique'=>1,'columns'=>['state','lease_until','sample_key'],]", - "'cloud_network_projection_retention'=>['non_unique'=>1,'columns'=>['state','terminal_at','sample_key'],]", - "'cloud_network_monitor_cursor_health'=>['non_unique'=>1,'columns'=>['collection_state','last_success_at','host_id',]", - "'cloud_network_monitor_cursor_due'=>['non_unique'=>1,'columns'=>['next_poll_at','poll_revision','host_id'],]", - "'cloud_network_monitor_cursor_lease'=>['non_unique'=>1,'columns'=>['poll_lease_until','host_id'],]", - "'cloud_network_baseline_owner'=>['non_unique'=>1,'columns'=>['user_id','updated_at'],]", - "'cloud_network_baseline_warmup'=>['non_unique'=>1,'columns'=>['warmup_state','sample_count'],]", - "'cloud_network_baseline_projection_lease'=>['non_unique'=>1,'columns'=>['projection_lease_until','instance_id'],]", -] as $requiredShape) { - $assert(str_contains($logShapeSetup, $requiredShape), - 'CloudSetup log index order is incomplete: ' . $requiredShape); -} - -$assert(str_contains($sample, 'PRIMARY KEY (sample_key)') - && str_contains($cursor, 'PRIMARY KEY (host_id)') - && str_contains($baseline, 'PRIMARY KEY (instance_id)'), - 'A log-ledger primary identity is incomplete.'); - -$outboxDelete = strpos( - $deletion, "'cloud_vm_network_projection_outbox'" -); -$sampleDelete = strrpos( - $deletion, "'cloud_vm_network_observation_samples'" -); -$assert($outboxDelete !== false && $sampleDelete !== false - && $outboxDelete < $sampleDelete - && str_contains($deletion, "'sample_key'") - && str_contains($outbox, - 'REFERENCES cloud_vm_network_observation_samples (sample_key)') - && str_contains($outbox, - 'ON DELETE RESTRICT ON UPDATE RESTRICT') - && str_contains($receipt, - 'REFERENCES cloud_instances (id) ON DELETE CASCADE'), - 'Account deletion does not explicitly clear RESTRICTed intents first.'); - -$assert(str_contains($setup, - '$logConnection, \'TABLE_CONSTRAINTS\', \'CONSTRAINT_NAME\'') - && str_contains($setup, 'self::REQUIRED_LOG_CHECKS') - && str_contains($setup, 'self::REQUIRED_LOG_INDEXES') - && str_contains($setup, 'self::REQUIRED_LOG_INDEX_SHAPES') - && str_contains($setup, 'self::REQUIRED_LOG_FOREIGN_KEYS') - && str_contains($setup, 'self::REQUIRED_LOG_FOREIGN_KEY_RULES') - && str_contains($setup, - "'delete_rule' => 'RESTRICT'") - && str_contains($setup, - "'update_rule' => 'RESTRICT'") - && str_contains($setup, - "'columns' => [\n 'state', 'available_at', 'bucket_start', 'sample_key',") - && str_contains($setup, - "'columns' => [\n 'instance_id', 'state', 'bucket_start', 'sample_key',") - && str_contains($setup, - "'columns' => ['next_poll_at', 'poll_revision', 'host_id']") - && str_contains($setup, - "'columns' => ['projection_lease_until', 'instance_id']"), - 'CloudSetup does not enforce log-schema object shapes on logConnection.'); - -echo "Abuse projection and retention ledger contracts passed.\n"; diff --git a/web/tests/CloudAbuseProjectionServiceTest.php b/web/tests/CloudAbuseProjectionServiceTest.php deleted file mode 100644 index dbded2c1..00000000 --- a/web/tests/CloudAbuseProjectionServiceTest.php +++ /dev/null @@ -1,1387 +0,0 @@ -addConnection([ - 'driver' => 'sqlite', 'database' => ':memory:', -], 'default'); -$capsule->addConnection([ - 'driver' => 'sqlite', 'database' => ':memory:', -], 'log'); -$capsule->setAsGlobal(); -$capsule->bootEloquent(); - -$mainSchema = $capsule->schema('default'); -$mainSchema->create('user', static function (Blueprint $table): void { - $table->increments('id'); -}); -$mainSchema->create('cloud_instances', static function (Blueprint $table): void { - $table->increments('id'); - $table->integer('host_id'); - $table->integer('user_id'); -}); -$mainSchema->create('cloud_abuse_instance_states', - static function (Blueprint $table): void { - $table->integer('instance_id')->primary(); - $table->integer('user_id'); - $table->integer('host_id'); - $table->integer('rule_version')->default(1); - $table->integer('policy_revision')->default(1); - $table->string('policy_hash')->default( - '511828bde950c504f92bb7a0603ca8b14453f575ead276c1889baa764f384dae' - ); - $table->string('risk_state'); - $table->string('collection_state'); - $table->string('latest_direction'); - $table->string('latest_rule_code'); - $table->string('latest_severity'); - $table->text('summary_json'); - $table->string('last_projection_key')->nullable(); - $table->integer('last_bucket_start')->nullable(); - $table->integer('last_sample_at')->nullable(); - $table->integer('last_evaluated_at')->nullable(); - $table->integer('created_at'); - $table->integer('updated_at'); - }); -$mainSchema->create('cloud_abuse_findings', - static function (Blueprint $table): void { - $table->increments('id'); - $table->string('finding_key')->unique(); - $table->integer('instance_id'); - $table->integer('user_id'); - $table->integer('host_id'); - $table->string('rule_code'); - $table->integer('rule_version'); - $table->integer('policy_revision')->default(1); - $table->string('policy_hash')->default( - '511828bde950c504f92bb7a0603ca8b14453f575ead276c1889baa764f384dae' - ); - $table->string('direction'); - $table->string('ip_family'); - $table->string('severity'); - $table->string('state'); - $table->integer('first_seen_at'); - $table->integer('last_seen_at'); - $table->integer('consecutive_windows'); - $table->integer('occurrence_generation')->default(1); - $table->integer('transition_revision')->default(1); - $table->text('evidence_json'); - $table->integer('resolved_at')->nullable(); - $table->string('resolution_reason')->default(''); - $table->integer('created_at'); - $table->integer('updated_at'); - }); -$mainSchema->create('cloud_abuse_alert_events', - static function (Blueprint $table): void { - $table->increments('id'); - $table->string('event_key')->unique(); - $table->string('sample_key')->unique(); - $table->integer('instance_id'); - $table->integer('user_id'); - $table->integer('host_id'); - $table->integer('bucket_start'); - $table->integer('rule_version'); - $table->integer('policy_revision'); - $table->string('policy_hash'); - $table->string('transition_digest'); - $table->integer('transition_count'); - $table->string('highest_severity'); - $table->integer('created_at'); - }); -$mainSchema->create('cloud_abuse_alert_event_items', - static function (Blueprint $table): void { - $table->increments('id'); - $table->integer('event_id'); - $table->string('item_key')->unique(); - $table->string('finding_key'); - $table->integer('occurrence_generation'); - $table->integer('transition_revision'); - $table->string('rule_code'); - $table->string('direction'); - $table->string('ip_family'); - $table->string('transition_type'); - $table->string('from_severity'); - $table->string('to_severity'); - $table->text('evidence_json'); - $table->string('resolution_reason')->default(''); - $table->integer('occurred_at'); - $table->integer('created_at'); - }); -$mainSchema->create('cloud_abuse_projection_receipts', - static function (Blueprint $table): void { - $table->string('sample_key')->primary(); - $table->integer('instance_id'); - $table->integer('rule_version'); - $table->integer('policy_revision')->default(1); - $table->string('policy_hash')->default( - '511828bde950c504f92bb7a0603ca8b14453f575ead276c1889baa764f384dae' - ); - $table->integer('bucket_start'); - $table->integer('projected_at'); - }); - -$logSchema = $capsule->schema('log'); -$logSchema->create('cloud_vm_network_observation_samples', - static function (Blueprint $table): void { - $table->string('sample_key')->primary(); - $table->integer('instance_id'); - $table->integer('user_id'); - $table->integer('host_id'); - $table->string('vm_uuid'); - $table->string('placement_key'); - $table->string('collector_generation'); - $table->string('window_id'); - $table->integer('contract_version'); - $table->integer('bucket_start'); - $table->integer('bucket_seconds'); - $table->integer('complete'); - $table->text('metrics_json'); - $table->text('quality_flags_json'); - $table->integer('received_at'); - }); -$logSchema->create('cloud_vm_network_projection_outbox', - static function (Blueprint $table): void { - $table->string('sample_key')->primary(); - $table->integer('instance_id'); - $table->integer('bucket_start'); - $table->string('placement_key'); - $table->integer('rule_version')->default(1); - $table->integer('policy_revision')->default(1); - $table->string('policy_hash')->default( - '511828bde950c504f92bb7a0603ca8b14453f575ead276c1889baa764f384dae' - ); - $table->string('state'); - $table->integer('attempts')->default(0); - $table->integer('available_at'); - $table->string('lease_token')->nullable(); - $table->integer('lease_until')->default(0); - $table->string('last_error_code')->default(''); - $table->integer('projected_at')->nullable(); - $table->integer('terminal_at')->nullable(); - $table->string('discard_reason_code')->default(''); - $table->integer('created_at'); - $table->integer('updated_at'); - }); -$logSchema->create('cloud_vm_network_baselines', - static function (Blueprint $table): void { - $table->integer('instance_id')->primary(); - $table->integer('user_id'); - $table->integer('rule_version'); - $table->integer('policy_revision')->default(1); - $table->string('policy_hash')->default( - '511828bde950c504f92bb7a0603ca8b14453f575ead276c1889baa764f384dae' - ); - $table->integer('sample_count'); - $table->string('warmup_state'); - $table->text('metrics_json'); - $table->string('last_projection_key')->nullable(); - $table->integer('last_bucket_start')->nullable(); - $table->string('projection_lease_token')->nullable(); - $table->integer('projection_lease_until')->default(0); - $table->integer('revision')->default(0); - $table->integer('created_at'); - $table->integer('updated_at'); - }); - -$main = $capsule->connection('default'); -$log = $capsule->connection('log'); -$now = 1000; -$defaultPolicy = CloudAbuseDetectionPolicy::defaults(); -$policySnapshot = new CloudAbusePolicySnapshot( - 1, - $defaultPolicy->hash(), - $defaultPolicy, - null, - 0 -); -$changedDefinition = $defaultPolicy->definition(); -$changedDefinition['windows']['suspicious'] = 4; -$changedDefinition['rules']['outbound_udp_fanout']['enabled'] = false; -$changedPolicy = CloudAbuseDetectionPolicy::fromDefinition( - $changedDefinition -); -$changedPolicySnapshot = new CloudAbusePolicySnapshot( - 2, - $changedPolicy->hash(), - $changedPolicy, - 7, - 900 -); -$sameDetectorSnapshot = new CloudAbusePolicySnapshot( - 3, - $defaultPolicy->hash(), - $defaultPolicy, - 7, - 950 -); -$policySnapshots = [ - '1:' . $policySnapshot->policyHash => $policySnapshot, - '2:' . $changedPolicySnapshot->policyHash => $changedPolicySnapshot, - '3:' . $sameDetectorSnapshot->policyHash => $sameDetectorSnapshot, -]; -$uuid = static fn (int $id): string => sprintf( - '123e4567-e89b-42d3-a456-%012d', - $id -); -$addInstance = static function (int $id, int $user, int $host = 7) use ( - $main -): void { - if (!$main->table('user')->where('id', $user)->exists()) { - $main->table('user')->insert(['id' => $user]); - } - $main->table('cloud_instances')->insert([ - 'id' => $id, 'host_id' => $host, 'user_id' => $user, - ]); -}; -$flow = static fn (array $values = []): array => array_merge([ - 'tcp_active_flows' => 0, 'tcp_new_flows' => 0, - 'tcp_syn_flows' => 0, 'tcp_unreplied_flows' => 0, - 'udp_active_flows' => 0, 'udp_new_flows' => 0, - 'other_active_flows' => 0, 'unique_remote_ips' => 0, - 'unique_remote_ports' => 0, 'unique_local_ports' => 0, - 'max_remote_ports_per_remote_ip' => 0, - 'max_remote_ips_per_remote_port' => 0, - 'max_local_ports_per_remote_ip' => 0, - 'max_remote_ips_per_local_port' => 0, -], $values); -$metrics = static function (array $outbound = [], array $inbound = []) use ( - $flow -): array { - return [ - 'interface' => [ - 'rx_bytes' => 1000, 'tx_bytes' => 2000, - 'rx_packets' => 10, 'tx_packets' => 20, - 'rx_errors' => 0, 'tx_errors' => 0, - 'rx_drops' => 0, 'tx_drops' => 0, - ], - 'ipv4' => ['inbound' => $flow($inbound), - 'outbound' => $flow($outbound)], - 'ipv6' => ['inbound' => $flow(), 'outbound' => $flow()], - ]; -}; -$insertBaseline = static function ( - int $instanceId, - int $userId, - array $baseline, - ?int $lastBucket = null, - ?string $lastKey = null, - ?string $lease = null, - int $leaseUntil = 0, - int $revision = 0, - ?CloudAbusePolicySnapshot $snapshot = null -) use ($log, &$now, $policySnapshot): void { - $snapshot ??= $policySnapshot; - $log->table('cloud_vm_network_baselines')->insert([ - 'instance_id' => $instanceId, - 'user_id' => $userId, - 'rule_version' => $snapshot->policy->engineVersion(), - 'policy_revision' => $snapshot->revision, - 'policy_hash' => $snapshot->policyHash, - 'sample_count' => (int) ($baseline['sample_count'] ?? 0), - 'warmup_state' => (int) ($baseline['sample_count'] ?? 0) - >= CloudAbuseDetector::WARMUP_WINDOWS ? 'ready' : 'warming', - 'metrics_json' => json_encode($baseline, JSON_THROW_ON_ERROR), - 'last_projection_key' => $lastKey, - 'last_bucket_start' => $lastBucket, - 'projection_lease_token' => $lease, - 'projection_lease_until' => $leaseUntil, - 'revision' => $revision, - 'created_at' => $now, - 'updated_at' => $now, - ]); -}; -$insertSample = static function ( - int $instanceId, - int $userId, - int $bucket, - array $sampleMetrics, - bool $complete = true, - array $quality = [], - string $state = 'pending', - ?string $lease = null, - int $leaseUntil = 0, - int $attempts = 0, - ?CloudAbusePolicySnapshot $snapshot = null -) use ($log, $uuid, &$now, $policySnapshot): string { - $snapshot ??= $policySnapshot; - $key = hash('sha256', $instanceId . "\n" . $bucket); - $placement = hash('sha256', 'placement-' . $instanceId); - $log->table('cloud_vm_network_observation_samples')->insert([ - 'sample_key' => $key, - 'instance_id' => $instanceId, - 'user_id' => $userId, - 'host_id' => 7, - 'vm_uuid' => $uuid($instanceId), - 'placement_key' => $placement, - 'collector_generation' => str_repeat('a', 32), - 'window_id' => substr($key, 0, 32), - 'contract_version' => 1, - 'bucket_start' => $bucket, - 'bucket_seconds' => 60, - 'complete' => $complete ? 1 : 0, - 'metrics_json' => json_encode($sampleMetrics, - JSON_THROW_ON_ERROR), - 'quality_flags_json' => json_encode($quality, - JSON_THROW_ON_ERROR), - 'received_at' => $now, - ]); - $log->table('cloud_vm_network_projection_outbox')->insert([ - 'sample_key' => $key, - 'instance_id' => $instanceId, - 'bucket_start' => $bucket, - 'placement_key' => $placement, - 'rule_version' => $snapshot->policy->engineVersion(), - 'policy_revision' => $snapshot->revision, - 'policy_hash' => $snapshot->policyHash, - 'state' => $state, - 'attempts' => $attempts, - 'available_at' => $now, - 'lease_token' => $lease, - 'lease_until' => $leaseUntil, - 'last_error_code' => '', - 'projected_at' => null, - 'terminal_at' => null, - 'discard_reason_code' => '', - 'created_at' => $now, - 'updated_at' => $now, - ]); - return $key; -}; -$worker = static function (?callable $hook = null) use ( - &$now, - &$policySnapshots -): object { - return new CloudAbuseProjectionService( - static function () use (&$now): int { return $now; }, - $hook, - static function (int $revision, string $hash) use ( - &$policySnapshots - ): CloudAbusePolicySnapshot { - $key = $revision . ':' . $hash; - if (!isset($policySnapshots[$key])) { - throw new RuntimeException('Unknown frozen policy identity'); - } - return $policySnapshots[$key]; - } - ); -}; - -// Normal success atomically creates the current state, receipt and log finality. -$addInstance(1, 11); -$key1 = $insertSample(1, 11, 120, $metrics()); -$successWorker = $worker(); -$assert($successWorker->tick(), 'A due projection was not claimed.'); -$assert($log->table('cloud_vm_network_projection_outbox') - ->where('sample_key', $key1)->value('state') === 'done' - && $main->table('cloud_abuse_projection_receipts') - ->where('sample_key', $key1)->count() === 1 - && $main->table('cloud_abuse_instance_states') - ->where('instance_id', 1)->value('risk_state') === 'observing' - && (int) $log->table('cloud_vm_network_baselines') - ->where('instance_id', 1)->value('sample_count') === 1, - 'Successful projection did not converge both databases.'); -$assert($successWorker->observationErrors() === [], - 'Successful projection retained a stale worker error.'); - -// Crash after the main commit must replay from its receipt without duplicating. -$addInstance(2, 22); -$key2 = $insertSample(2, 22, 180, $metrics()); -$crashed = false; -$worker(static function () use (&$crashed): void { - if (!$crashed) { - $crashed = true; - throw new RuntimeException('simulated cross-database crash'); - } -})->tick(); -$assert($main->table('cloud_abuse_projection_receipts') - ->where('sample_key', $key2)->count() === 1 - && $log->table('cloud_vm_network_projection_outbox') - ->where('sample_key', $key2)->value('state') === 'pending' - && (int) $log->table('cloud_vm_network_baselines') - ->where('instance_id', 2)->value('sample_count') === 0, - 'Simulated crash did not preserve the explicit replay seam.'); -$now += 15; -$assert($worker()->tick(), 'Receipt-backed projection did not replay.'); -$assert($log->table('cloud_vm_network_projection_outbox') - ->where('sample_key', $key2)->value('state') === 'done' - && $main->table('cloud_abuse_projection_receipts') - ->where('sample_key', $key2)->count() === 1 - && (int) $log->table('cloud_vm_network_baselines') - ->where('instance_id', 2)->value('sample_count') === 1 - && (int) $main->table('cloud_abuse_projection_receipts') - ->where('sample_key', $key2)->value('policy_revision') === 1 - && (int) $main->table('cloud_abuse_instance_states') - ->where('instance_id', 2)->value('policy_revision') === 1, - 'Receipt replay duplicated or failed to finalize projection.'); - -// A committed main receipt is authoritative even when the cross-database -// finalization seam keeps failing beyond the ordinary retry budget. -$addInstance(12, 122); -$receiptKey = $insertSample(12, 122, 240, $metrics()); -$finalizeFailure = static function (): void { - throw new RuntimeException('simulated repeated finalization failure'); -}; -for ($attempt = 1; $attempt <= 8; ++$attempt) { - $failingWorker = $worker($finalizeFailure); - $assert($failingWorker->tick(), - 'Receipt-backed retry was not consumed.'); - $receiptOutbox = $log->table( - 'cloud_vm_network_projection_outbox' - )->where('sample_key', $receiptKey)->first(); - $assert((int) $receiptOutbox->attempts === $attempt, - 'Receipt-backed retry count was not durable.'); - $expectedError = $attempt === 8 - ? 'projection_receipt_finalize_pending' : 'projection_failed'; - $assert($failingWorker->observationErrors() === [$expectedError], - 'Receipt-backed retry exposed an unstable worker error.'); - $now += 1000; -} -$receiptOutbox = $log->table('cloud_vm_network_projection_outbox') - ->where('sample_key', $receiptKey)->first(); -$assert($receiptOutbox->state === 'pending' - && $receiptOutbox->discard_reason_code === '' - && $main->table('cloud_abuse_projection_receipts') - ->where('sample_key', $receiptKey)->count() === 1 - && (int) $log->table('cloud_vm_network_baselines') - ->where('instance_id', 12)->value('sample_count') === 0, - 'Committed receipt was discarded after finalization retries.'); -$receiptRecoveryWorker = $worker(); -$assert($receiptRecoveryWorker->tick(), - 'Receipt-backed recovery was not consumed.'); -$assert($log->table('cloud_vm_network_projection_outbox') - ->where('sample_key', $receiptKey)->value('state') === 'done' - && (int) $log->table('cloud_vm_network_baselines') - ->where('instance_id', 12)->value('sample_count') === 1 - && $main->table('cloud_abuse_projection_receipts') - ->where('sample_key', $receiptKey)->count() === 1, - 'Recovered finalization did not converge from the committed receipt.'); -$assert($receiptRecoveryWorker->observationErrors() === [], - 'Successful receipt recovery retained a worker error.'); - -// A matching receipt also protects the earlier claim/ledger-failure path. -// Because damaged identity evidence cannot reproduce the exact baseline, the -// safe convergence is to finish the already-applied outbox and warm up again. -$addInstance(13, 133); -$claimReceiptKey = $insertSample( - 13, - 133, - 300, - $metrics(), - true, - [], - 'pending', - null, - 0, - 7 -); -$claimResetFindingKey = hash('sha256', implode("\n", [ - 13, CloudAbuseDetector::RULE_VERSION, - $policySnapshot->policyHash, - 'outbound_tcp_syn_sweep', 'ipv4', 'outbound', -])); -$main->table('cloud_abuse_findings')->insert([ - 'finding_key' => $claimResetFindingKey, - 'instance_id' => 13, 'user_id' => 133, 'host_id' => 7, - 'rule_code' => 'outbound_tcp_syn_sweep', - 'rule_version' => CloudAbuseDetector::RULE_VERSION, - 'direction' => 'outbound', 'ip_family' => 'ipv4', - 'severity' => 'high', 'state' => 'open', - 'first_seen_at' => 120, 'last_seen_at' => 240, - 'consecutive_windows' => 2, 'evidence_json' => '{}', - 'resolved_at' => null, 'created_at' => $now, 'updated_at' => $now, -]); -$main->table('cloud_abuse_projection_receipts')->insert([ - 'sample_key' => $claimReceiptKey, - 'instance_id' => 13, - 'rule_version' => CloudAbuseDetector::RULE_VERSION, - 'bucket_start' => 300, - 'projected_at' => $now, -]); -$log->table('cloud_vm_network_projection_outbox') - ->where('sample_key', $claimReceiptKey)->update([ - 'placement_key' => str_repeat('1', 64), - ]); -$claimReceiptWorker = $worker(); -$assert($claimReceiptWorker->tick(), - 'Receipt-backed claim recovery was not consumed.'); -$claimReceiptOutbox = $log->table('cloud_vm_network_projection_outbox') - ->where('sample_key', $claimReceiptKey)->first(); -$assert($claimReceiptOutbox->state === 'done' - && $claimReceiptOutbox->discard_reason_code === '' - && (int) $claimReceiptOutbox->attempts === 8 - && $log->table('cloud_vm_network_baselines') - ->where('instance_id', 13)->count() === 1 - && (int) $log->table('cloud_vm_network_baselines') - ->where('instance_id', 13)->value('sample_count') === 0, - 'Matching claim receipt was discarded or left a divergent baseline.'); -$assert($claimReceiptWorker->observationErrors() - === ['projection_receipt_baseline_reset'], - 'Receipt-backed claim recovery exposed no bounded diagnostic.'); -$claimNextKey = $insertSample(13, 133, 360, $metrics()); -$assert($worker()->tick() - && $log->table('cloud_vm_network_projection_outbox') - ->where('sample_key', $claimNextKey)->value('state') === 'done' - && (int) $log->table('cloud_vm_network_baselines') - ->where('instance_id', 13)->value('sample_count') === 1, - 'Receipt-backed claim recovery did not restart safe warm-up.'); -$assert($main->table('cloud_abuse_findings') - ->where('finding_key', $claimResetFindingKey)->value('state') - === 'open', - 'Receipt-backed reset cleared an open finding without evidence.'); -for ($step = 1; $step <= 63; ++$step) { - $insertSample(13, 133, 360 + ($step * 60), $metrics()); - $assert($worker()->tick(), - 'Receipt-reset clear convergence did not run.'); -} -$assert($main->table('cloud_abuse_findings') - ->where('finding_key', $claimResetFindingKey)->value('state') - === 'resolved' - && $main->table('cloud_abuse_instance_states') - ->where('instance_id', 13)->value('risk_state') === 'normal', - 'Receipt reset left an open finding after verified clear evidence.'); - -// A conflicting receipt is never treated as proof that the main effect did -// not happen. It remains attributable and backed off for operator repair. -$addInstance(14, 144); -$claimConflictKey = $insertSample( - 14, - 144, - 300, - $metrics(), - true, - [], - 'pending', - null, - 0, - 7 -); -$main->table('cloud_abuse_projection_receipts')->insert([ - 'sample_key' => $claimConflictKey, - 'instance_id' => 14, - 'rule_version' => CloudAbuseDetector::RULE_VERSION, - 'policy_revision' => 2, - 'policy_hash' => $changedPolicySnapshot->policyHash, - 'bucket_start' => 300, - 'projected_at' => $now, -]); -$log->table('cloud_vm_network_projection_outbox') - ->where('sample_key', $claimConflictKey)->update([ - 'placement_key' => str_repeat('2', 64), - ]); -$claimConflictWorker = $worker(); -$assert($claimConflictWorker->tick(), - 'Conflicting claim receipt was not inspected.'); -$claimConflictOutbox = $log->table('cloud_vm_network_projection_outbox') - ->where('sample_key', $claimConflictKey)->first(); -$assert($claimConflictOutbox->state === 'pending' - && (int) $claimConflictOutbox->attempts === 8 - && $claimConflictOutbox->last_error_code - === 'projection_receipt_conflict' - && $claimConflictOutbox->discard_reason_code === '', - 'Conflicting claim receipt was incorrectly declared not applied.'); -$assert($claimConflictWorker->observationErrors() - === ['projection_receipt_conflict'], - 'Conflicting claim receipt lost its bounded claim diagnostic.'); -$log->table('cloud_vm_network_projection_outbox') - ->where('sample_key', $claimConflictKey)->delete(); -$log->table('cloud_vm_network_observation_samples') - ->where('sample_key', $claimConflictKey)->delete(); -$main->table('cloud_abuse_projection_receipts') - ->where('sample_key', $claimConflictKey)->delete(); - -// If current findings cannot be read, a matching receipt stays pending rather -// than losing the reset boundary; it converges after the read model recovers. -$addInstance(15, 155); -$seedPendingKey = $insertSample( - 15, - 155, - 300, - $metrics(), - true, - [], - 'pending', - null, - 0, - 7 -); -$main->table('cloud_abuse_projection_receipts')->insert([ - 'sample_key' => $seedPendingKey, - 'instance_id' => 15, - 'rule_version' => CloudAbuseDetector::RULE_VERSION, - 'bucket_start' => 300, - 'projected_at' => $now, -]); -$log->table('cloud_vm_network_projection_outbox') - ->where('sample_key', $seedPendingKey)->update([ - 'placement_key' => str_repeat('3', 64), - ]); -$main->statement( - 'ALTER TABLE cloud_abuse_findings ' - . 'RENAME TO cloud_abuse_findings_unavailable' -); -$seedPendingWorker = $worker(); -$assert($seedPendingWorker->tick(), - 'Unavailable baseline seed was not consumed.'); -$seedPending = $log->table('cloud_vm_network_projection_outbox') - ->where('sample_key', $seedPendingKey)->first(); -$assert($seedPending->state === 'pending' - && (int) $seedPending->attempts === 8 - && $seedPending->last_error_code - === 'projection_baseline_seed_pending' - && $seedPending->discard_reason_code === '', - 'Unavailable baseline seed was discarded or left unbounded.'); -$main->statement( - 'ALTER TABLE cloud_abuse_findings_unavailable ' - . 'RENAME TO cloud_abuse_findings' -); -$now += 1000; -$assert($worker()->tick() - && $log->table('cloud_vm_network_projection_outbox') - ->where('sample_key', $seedPendingKey)->value('state') === 'done' - && (int) $log->table('cloud_vm_network_baselines') - ->where('instance_id', 15)->value('sample_count') === 0, - 'Baseline seed did not converge after its read model recovered.'); - -// Missing owners and stale buckets end explicitly instead of locking forever. -$main->table('user')->insert(['id' => 99]); -$deletedKey = $insertSample(99, 99, 240, $metrics()); -$assert($worker()->tick(), 'Deleted-instance sample was not consumed.'); -$deleted = $log->table('cloud_vm_network_projection_outbox') - ->where('sample_key', $deletedKey)->first(); -$assert($deleted->state === 'discarded' - && $deleted->discard_reason_code === 'instance_deleted', - 'Deleted instance left a retrying or locked outbox row.'); - -$main->table('cloud_instances')->insert([ - 'id' => 21, 'host_id' => 7, 'user_id' => 211, -]); -$missingOwnerKey = $insertSample(21, 211, 240, $metrics()); -$assert($worker()->tick(), 'Missing-owner sample was not consumed.'); -$assert($log->table('cloud_vm_network_projection_outbox') - ->where('sample_key', $missingOwnerKey) - ->value('discard_reason_code') === 'instance_owner_missing' - && $main->table('cloud_abuse_instance_states') - ->where('instance_id', 21)->count() === 0 - && $main->table('cloud_abuse_findings') - ->where('instance_id', 21)->count() === 0 - && $main->table('cloud_abuse_alert_events') - ->where('instance_id', 21)->count() === 0 - && $main->table('cloud_abuse_projection_receipts') - ->where('sample_key', $missingOwnerKey)->count() === 0, - 'Missing owner allowed late read-model or event insertion.'); - -$addInstance(8, 88); -$main->table('user')->insert(['id' => 89]); -$oldOwnerFindingKey = hash('sha256', implode("\n", [ - 8, CloudAbuseDetector::RULE_VERSION, - $policySnapshot->policyHash, - 'outbound_tcp_syn_sweep', 'ipv4', 'outbound', -])); -$main->table('cloud_abuse_instance_states')->insert([ - 'instance_id' => 8, 'user_id' => 89, 'host_id' => 7, - 'risk_state' => 'high_risk', 'collection_state' => 'healthy', - 'latest_direction' => 'outbound', - 'latest_rule_code' => 'outbound_tcp_syn_sweep', - 'latest_severity' => 'high', 'summary_json' => '{}', - 'last_projection_key' => str_repeat('8', 64), - 'last_bucket_start' => 180, 'last_sample_at' => 240, - 'last_evaluated_at' => $now, - 'created_at' => $now, 'updated_at' => $now, -]); -$main->table('cloud_abuse_findings')->insert([ - 'finding_key' => $oldOwnerFindingKey, - 'instance_id' => 8, 'user_id' => 89, 'host_id' => 7, - 'rule_code' => 'outbound_tcp_syn_sweep', - 'rule_version' => CloudAbuseDetector::RULE_VERSION, - 'direction' => 'outbound', 'ip_family' => 'ipv4', - 'severity' => 'high', 'state' => 'open', - 'first_seen_at' => 120, 'last_seen_at' => 180, - 'consecutive_windows' => 2, 'evidence_json' => '{}', - 'resolved_at' => null, 'created_at' => $now, 'updated_at' => $now, -]); -$ownerKey = $insertSample(8, 89, 240, $metrics()); -$ownerEventCount = $main->table('cloud_abuse_alert_events')->count(); -$ownerWorker = $worker(); -$assert($ownerWorker->tick(), 'Owner-mismatch sample was not consumed.'); -$ownerMismatch = $log->table('cloud_vm_network_projection_outbox') - ->where('sample_key', $ownerKey)->first(); -$assert($ownerMismatch->state === 'discarded' - && $ownerMismatch->discard_reason_code === 'instance_owner_changed' - && $log->table('cloud_vm_network_baselines') - ->where('instance_id', 8)->count() === 0 - && $main->table('cloud_abuse_instance_states') - ->where('instance_id', 8)->count() === 0 - && $main->table('cloud_abuse_findings') - ->where('finding_key', $oldOwnerFindingKey)->value('state') - === 'resolved' - && (int) $main->table('cloud_abuse_findings') - ->where('finding_key', $oldOwnerFindingKey)->value('user_id') === 89 - && (int) $main->table('cloud_abuse_findings') - ->where('finding_key', $oldOwnerFindingKey) - ->value('transition_revision') === 2 - && $main->table('cloud_abuse_alert_events')->count() - === $ownerEventCount - && (int) $main->table('cloud_instances') - ->where('id', 8)->value('user_id') === 88, - 'Owner mismatch left a retrying or locked outbox row.'); -$assert($ownerWorker->observationErrors() === ['instance_owner_changed'], - 'Owner mismatch did not expose its stable discard reason.'); -$correctOwnerKey = $insertSample(8, 88, 300, $metrics()); -$assert($worker()->tick(), 'Correct owner sample did not recover.'); -$assert($log->table('cloud_vm_network_projection_outbox') - ->where('sample_key', $correctOwnerKey)->value('state') === 'done' - && (int) $log->table('cloud_vm_network_baselines') - ->where('instance_id', 8)->value('user_id') === 88 - && (int) $main->table('cloud_abuse_instance_states') - ->where('instance_id', 8)->value('user_id') === 88, - 'Owner mismatch contaminated the next owner baseline.'); - -$addInstance(3, 33); -$lateBaseline = CloudAbuseDetector::initialBaseline(); -$lateBaseline['last_bucket_start'] = 360; -$insertBaseline(3, 33, $lateBaseline, 360, str_repeat('b', 64)); -$lateKey = $insertSample(3, 33, 300, $metrics()); -$assert($worker()->tick(), 'Out-of-order sample was not consumed.'); -$assert($log->table('cloud_vm_network_projection_outbox') - ->where('sample_key', $lateKey)->value('discard_reason_code') - === 'out_of_order_sample', - 'Out-of-order sample changed or blocked the baseline.'); - -// An expired outbox+baseline lease is safely reclaimed. -$addInstance(4, 44); -$expiredToken = str_repeat('c', 32); -$insertBaseline(4, 44, CloudAbuseDetector::initialBaseline(), - null, null, $expiredToken, $now - 1, 4); -$expiredKey = $insertSample(4, 44, 420, $metrics(), true, [], - 'leased', $expiredToken, $now - 1, 1); -$assert($worker()->tick(), 'Expired projection lease was not reclaimed.'); -$assert($log->table('cloud_vm_network_projection_outbox') - ->where('sample_key', $expiredKey)->value('state') === 'done' - && (int) $log->table('cloud_vm_network_baselines') - ->where('instance_id', 4)->value('revision') === 5, - 'Expired lease recovery did not preserve baseline CAS ordering.'); - -// Warm baseline: two high windows open a finding; five clear windows resolve it. -$addInstance(5, 55); -$warm = CloudAbuseDetector::initialBaseline(); -$warm['sample_count'] = CloudAbuseDetector::WARMUP_WINDOWS; -$warm['last_bucket_start'] = 60; -$insertBaseline(5, 55, $warm, 60, str_repeat('d', 64)); -$high = [ - 'tcp_active_flows' => 6000, 'tcp_new_flows' => 6000, - 'tcp_syn_flows' => 5000, 'tcp_unreplied_flows' => 5000, - 'unique_remote_ips' => 700, 'unique_remote_ports' => 1, - 'unique_local_ports' => 6000, - 'max_remote_ports_per_remote_ip' => 1, - 'max_remote_ips_per_remote_port' => 700, - 'max_local_ports_per_remote_ip' => 1, - 'max_remote_ips_per_local_port' => 700, -]; -foreach ([120, 180] as $bucket) { - $insertSample(5, 55, $bucket, $metrics($high)); - $assert($worker()->tick(), 'High-risk projection did not run.'); -} -$finding = $main->table('cloud_abuse_findings') - ->where('instance_id', 5)->where('state', 'open')->first(); -$assert($finding !== null && $finding->severity === 'high' - && $finding->direction === 'outbound' - && (int) $finding->occurrence_generation === 1 - && (int) $finding->transition_revision === 1, - 'Two high windows did not open the deterministic finding.'); -$openedEvent = $main->table('cloud_abuse_alert_events') - ->where('instance_id', 5)->first(); -$openedItem = $openedEvent === null ? null - : $main->table('cloud_abuse_alert_event_items') - ->where('event_id', (int) $openedEvent->id)->first(); -$assert($openedEvent !== null - && (int) $openedEvent->bucket_start === 180 - && (int) $openedEvent->transition_count === 1 - && $openedEvent->highest_severity === 'high' - && $openedItem !== null - && $openedItem->transition_type === 'opened' - && $openedItem->from_severity === 'none' - && $openedItem->to_severity === 'high' - && (int) $openedItem->occurrence_generation === 1 - && (int) $openedItem->transition_revision === 1, - 'Finding open was not frozen as one immutable event item.'); -$insertSample(5, 55, 240, $metrics($high)); -$assert($worker()->tick(), 'Continued high-risk projection did not run.'); -$assert($main->table('cloud_abuse_alert_events') - ->where('instance_id', 5)->count() === 1 - && (int) $main->table('cloud_abuse_findings') - ->where('instance_id', 5)->value('transition_revision') === 1, - 'Continued finding created an event or advanced its transition revision.'); -foreach ([300, 360, 420, 480, 540] as $bucket) { - $insertSample(5, 55, $bucket, $metrics()); - $assert($worker()->tick(), 'Clear projection did not run.'); -} -$finding = $main->table('cloud_abuse_findings') - ->where('instance_id', 5)->first(); -$assert($finding->state === 'resolved' - && (int) $finding->resolved_at === 540 - && (int) $finding->transition_revision === 2, - 'Five verified clear windows did not resolve the finding.'); -$resolvedItem = $main->table('cloud_abuse_alert_event_items') - ->where('finding_key', (string) $finding->finding_key) - ->where('transition_type', 'resolved')->first(); -$assert($main->table('cloud_abuse_alert_events') - ->where('instance_id', 5)->count() === 2 - && $resolvedItem !== null - && $resolvedItem->from_severity === 'high' - && $resolvedItem->to_severity === 'none' - && $resolvedItem->resolution_reason === 'evidence_clear' - && (int) $resolvedItem->transition_revision === 2, - 'Evidence-clear resolution was not frozen as an event.'); -foreach ([600, 660] as $bucket) { - $insertSample(5, 55, $bucket, $metrics($high)); - $assert($worker()->tick(), 'Reopen projection did not run.'); -} -$reopened = $main->table('cloud_abuse_findings') - ->where('instance_id', 5)->first(); -$reopenedItem = $main->table('cloud_abuse_alert_event_items') - ->where('finding_key', (string) $reopened->finding_key) - ->where('occurrence_generation', 2)->first(); -$assert($reopened->state === 'open' - && (int) $reopened->occurrence_generation === 2 - && (int) $reopened->transition_revision === 3 - && $reopenedItem !== null - && $reopenedItem->transition_type === 'opened' - && (int) $reopenedItem->transition_revision === 3, - 'Resolved finding did not reopen in a new occurrence generation.'); - -// A panel ring gap resets its baseline but never clears an existing finding. -$addInstance(6, 66); -$gapBaseline = CloudAbuseDetector::resetBaseline([[ - 'rule_code' => 'outbound_tcp_syn_sweep', - 'direction' => 'outbound', - 'ip_family' => 'ipv4', - 'severity' => 'high', -]]); -$gapBaseline['sample_count'] = 60; -$gapBaseline['last_bucket_start'] = 60; -$gapBaseline['last_interface'] = $metrics()['interface']; -$insertBaseline(6, 66, $gapBaseline, 60, str_repeat('e', 64)); -$findingKey = hash('sha256', implode("\n", [ - 6, CloudAbuseDetector::RULE_VERSION, - $policySnapshot->policyHash, - 'outbound_tcp_syn_sweep', 'ipv4', 'outbound', -])); -$main->table('cloud_abuse_findings')->insert([ - 'finding_key' => $findingKey, - 'instance_id' => 6, 'user_id' => 66, 'host_id' => 7, - 'rule_code' => 'outbound_tcp_syn_sweep', - 'rule_version' => CloudAbuseDetector::RULE_VERSION, - 'direction' => 'outbound', 'ip_family' => 'ipv4', - 'severity' => 'high', 'state' => 'open', - 'first_seen_at' => 60, 'last_seen_at' => 60, - 'consecutive_windows' => 2, 'evidence_json' => '{}', - 'resolved_at' => null, 'created_at' => $now, 'updated_at' => $now, -]); -$main->table('cloud_abuse_instance_states')->insert([ - 'instance_id' => 6, 'user_id' => 66, 'host_id' => 7, - 'risk_state' => 'high_risk', 'collection_state' => 'healthy', - 'latest_direction' => 'outbound', - 'latest_rule_code' => 'outbound_tcp_syn_sweep', - 'latest_severity' => 'high', 'summary_json' => '{}', - 'last_projection_key' => str_repeat('e', 64), - 'last_bucket_start' => 60, 'last_sample_at' => 120, - 'last_evaluated_at' => $now, - 'created_at' => $now, 'updated_at' => $now, -]); -$gapKey = $insertSample(6, 66, 120, $metrics(), false, - ['incomplete', 'panel_ring_gap']); -$assert($worker()->tick(), 'Panel-gap projection did not run.'); -$gapStored = json_decode((string) $log->table( - 'cloud_vm_network_baselines' -)->where('instance_id', 6)->value('metrics_json'), true, - 32, JSON_THROW_ON_ERROR); -$state6 = $main->table('cloud_abuse_instance_states') - ->where('instance_id', 6)->first(); -$assert($log->table('cloud_vm_network_projection_outbox') - ->where('sample_key', $gapKey)->value('state') === 'done' - && $gapStored['sample_count'] === 0 - && $gapStored['last_interface'] === null - && $main->table('cloud_abuse_findings') - ->where('finding_key', $findingKey)->value('state') === 'open' - && $state6->risk_state === 'high_risk' - && $state6->collection_state === 'degraded', - 'Incomplete ring-gap evidence advanced or cleared trusted state.'); -for ($step = 1; $step <= 64; ++$step) { - $insertSample(6, 66, 120 + ($step * 60), $metrics()); - $assert($worker()->tick(), 'Panel-gap clear convergence did not run.'); -} -$assert($main->table('cloud_abuse_findings') - ->where('finding_key', $findingKey)->value('state') === 'resolved' - && $main->table('cloud_abuse_instance_states') - ->where('instance_id', 6)->value('risk_state') === 'normal', - 'Panel-gap reset left an open finding after verified clear evidence.'); - -// A changed immutable detector policy keeps historical statistics/sample -// count, clears threshold streaks, and retires old findings explicitly rather -// than pretending that new evidence cleared them. -$addInstance(16, 166); -$history = CloudAbuseDetector::evaluate([ - 'bucket_start' => 60, - 'complete' => true, - 'quality_flags' => [], - 'metrics' => $metrics(), -], CloudAbuseDetector::initialBaseline(), $defaultPolicy)['baseline']; -$history['sample_count'] = 60; -$history['rules']['ipv4:outbound_tcp_syn_sweep'] = [ - 'active' => true, - 'severity' => 'high', - 'suspicious_streak' => 0, - 'high_streak' => 2, - 'clear_streak' => 0, - 'transition' => 'continued', -]; -$statsBeforeSwitch = $history['stats']; -$insertBaseline( - 16, - 166, - $history, - 60, - str_repeat('1', 64), - null, - 0, - 0, - $policySnapshot -); -$replacedFindingKey = hash('sha256', implode("\n", [ - 16, - CloudAbuseDetector::RULE_VERSION, - $policySnapshot->policyHash, - 'outbound_tcp_syn_sweep', - 'ipv4', - 'outbound', -])); -$main->table('cloud_abuse_findings')->insert([ - 'finding_key' => $replacedFindingKey, - 'instance_id' => 16, 'user_id' => 166, 'host_id' => 7, - 'rule_code' => 'outbound_tcp_syn_sweep', - 'rule_version' => CloudAbuseDetector::RULE_VERSION, - 'policy_revision' => 1, - 'policy_hash' => $policySnapshot->policyHash, - 'direction' => 'outbound', 'ip_family' => 'ipv4', - 'severity' => 'high', 'state' => 'open', - 'first_seen_at' => 60, 'last_seen_at' => 60, - 'consecutive_windows' => 2, - 'evidence_json' => '{"tcp_syn_flows":5000}', - 'resolved_at' => null, 'resolution_reason' => '', - 'created_at' => $now, 'updated_at' => $now, -]); -$main->table('cloud_abuse_instance_states')->insert([ - 'instance_id' => 16, 'user_id' => 166, 'host_id' => 7, - 'rule_version' => CloudAbuseDetector::RULE_VERSION, - 'policy_revision' => 1, - 'policy_hash' => $policySnapshot->policyHash, - 'risk_state' => 'high_risk', 'collection_state' => 'healthy', - 'latest_direction' => 'outbound', - 'latest_rule_code' => 'outbound_tcp_syn_sweep', - 'latest_severity' => 'high', 'summary_json' => '{}', - 'last_projection_key' => str_repeat('1', 64), - 'last_bucket_start' => 60, 'last_sample_at' => 120, - 'last_evaluated_at' => $now, - 'created_at' => $now, 'updated_at' => $now, -]); -$switchKey = $insertSample( - 16, - 166, - 120, - $metrics(), - false, - ['incomplete', 'sampling_gap'], - 'pending', - null, - 0, - 0, - $changedPolicySnapshot -); -$policyEventCount = $main->table('cloud_abuse_alert_events')->count(); -$assert($worker()->tick(), 'Changed-policy projection did not run.'); -$switchedBaseline = json_decode((string) $log->table( - 'cloud_vm_network_baselines' -)->where('instance_id', 16)->value('metrics_json'), true, - 32, JSON_THROW_ON_ERROR); -$replacedFinding = $main->table('cloud_abuse_findings') - ->where('finding_key', $replacedFindingKey)->first(); -$assert($log->table('cloud_vm_network_projection_outbox') - ->where('sample_key', $switchKey)->value('state') === 'done' - && (int) $log->table('cloud_vm_network_baselines') - ->where('instance_id', 16)->value('policy_revision') === 2 - && hash_equals( - $changedPolicySnapshot->policyHash, - (string) $log->table('cloud_vm_network_baselines') - ->where('instance_id', 16)->value('policy_hash') - ) - && $switchedBaseline['sample_count'] === 60 - && $switchedBaseline['stats'] == $statsBeforeSwitch - && $switchedBaseline['rules'] === [] - && $replacedFinding->state === 'resolved' - && $replacedFinding->resolution_reason === 'policy_replaced' - && (int) $replacedFinding->transition_revision === 2 - && $replacedFinding->evidence_json === '{"tcp_syn_flows":5000}' - && $main->table('cloud_abuse_alert_events')->count() - === $policyEventCount - && $main->table('cloud_abuse_instance_states') - ->where('instance_id', 16)->value('risk_state') === 'observing', - 'Detector policy switch lost statistics or forged evidence recovery.'); - -// A newer revision with the same detector hash may carry unrelated delivery -// metadata in a later schema. It updates identity without clearing streaks. -$addInstance(17, 177); -$sameHashBaseline = CloudAbuseDetector::resetBaseline([[ - 'rule_code' => 'outbound_tcp_syn_sweep', - 'direction' => 'outbound', - 'ip_family' => 'ipv4', - 'severity' => 'high', -]], $defaultPolicy); -$sameHashBaseline['sample_count'] = 60; -$sameHashBaseline['last_bucket_start'] = 60; -$sameHashBaseline['last_interface'] = $metrics()['interface']; -$sameHashBaseline['stats'] = $statsBeforeSwitch; -$insertBaseline( - 17, - 177, - $sameHashBaseline, - 60, - str_repeat('2', 64), - null, - 0, - 0, - $policySnapshot -); -$sameHashFindingKey = hash('sha256', implode("\n", [ - 17, - CloudAbuseDetector::RULE_VERSION, - $policySnapshot->policyHash, - 'outbound_tcp_syn_sweep', - 'ipv4', - 'outbound', -])); -$main->table('cloud_abuse_findings')->insert([ - 'finding_key' => $sameHashFindingKey, - 'instance_id' => 17, 'user_id' => 177, 'host_id' => 7, - 'rule_code' => 'outbound_tcp_syn_sweep', - 'rule_version' => CloudAbuseDetector::RULE_VERSION, - 'policy_revision' => 1, - 'policy_hash' => $policySnapshot->policyHash, - 'direction' => 'outbound', 'ip_family' => 'ipv4', - 'severity' => 'high', 'state' => 'open', - 'first_seen_at' => 60, 'last_seen_at' => 60, - 'consecutive_windows' => 2, 'evidence_json' => '{}', - 'resolved_at' => null, 'resolution_reason' => '', - 'created_at' => $now, 'updated_at' => $now, -]); -$main->table('cloud_abuse_instance_states')->insert([ - 'instance_id' => 17, 'user_id' => 177, 'host_id' => 7, - 'rule_version' => CloudAbuseDetector::RULE_VERSION, - 'policy_revision' => 1, - 'policy_hash' => $policySnapshot->policyHash, - 'risk_state' => 'high_risk', 'collection_state' => 'healthy', - 'latest_direction' => 'outbound', - 'latest_rule_code' => 'outbound_tcp_syn_sweep', - 'latest_severity' => 'high', 'summary_json' => '{}', - 'last_projection_key' => str_repeat('2', 64), - 'last_bucket_start' => 60, 'last_sample_at' => 120, - 'last_evaluated_at' => $now, - 'created_at' => $now, 'updated_at' => $now, -]); -$insertSample( - 17, - 177, - 120, - $metrics(), - false, - ['incomplete', 'sampling_gap'], - 'pending', - null, - 0, - 0, - $sameDetectorSnapshot -); -$assert($worker()->tick(), 'Same-detector revision projection did not run.'); -$sameHashStored = json_decode((string) $log->table( - 'cloud_vm_network_baselines' -)->where('instance_id', 17)->value('metrics_json'), true, - 32, JSON_THROW_ON_ERROR); -$assert((int) $log->table('cloud_vm_network_baselines') - ->where('instance_id', 17)->value('policy_revision') === 3 - && ($sameHashStored['rules']['ipv4:outbound_tcp_syn_sweep']['active'] - ?? false) === true - && $main->table('cloud_abuse_findings') - ->where('finding_key', $sameHashFindingKey)->value('state') === 'open' - && $main->table('cloud_abuse_instance_states') - ->where('instance_id', 17)->value('risk_state') === 'high_risk', - 'Same detector hash incorrectly cleared durable finding streaks.'); - -// Corrupt/non-canonical evidence has bounded retries and an explicit terminal -// state; it can neither leak into the main read model nor hold the queue. -$addInstance(7, 77); -$invalidKey = $insertSample(7, 77, 120, $metrics()); -$invalidMetrics = $metrics(); -$invalidMetrics['ipv4']['outbound']['remote_addresses'] = ['203.0.113.8']; -$log->table('cloud_vm_network_observation_samples') - ->where('sample_key', $invalidKey)->update([ - 'metrics_json' => json_encode($invalidMetrics, - JSON_THROW_ON_ERROR), - ]); -for ($attempt = 1; $attempt <= 8; ++$attempt) { - $assert($worker()->tick(), 'Invalid sample retry was not consumed.'); - $row = $log->table('cloud_vm_network_projection_outbox') - ->where('sample_key', $invalidKey)->first(); - $assert((int) $row->attempts === $attempt, - 'Invalid sample attempt count was not durable.'); - $now += 1000; -} -$invalidOutbox = $log->table('cloud_vm_network_projection_outbox') - ->where('sample_key', $invalidKey)->first(); -$assert($invalidOutbox->state === 'discarded' - && $invalidOutbox->discard_reason_code - === 'projection_retry_exhausted' - && $main->table('cloud_abuse_projection_receipts') - ->where('sample_key', $invalidKey)->count() === 0, - 'Invalid aggregate evidence leaked or left a permanent projection lock.'); - -// A corrupt, attributable sample/outbox pair backs off and terminates without -// monopolising the globally earliest due-candidate position. -$addInstance(9, 99); -$badLedgerKey = $insertSample(9, 99, 60, $metrics()); -$log->table('cloud_vm_network_projection_outbox') - ->where('sample_key', $badLedgerKey)->update([ - 'placement_key' => str_repeat('f', 64), - ]); -$addInstance(10, 100); -$nextInstanceKey = $insertSample(10, 100, 120, $metrics()); -$badLedgerWorker = $worker(); -$assert($badLedgerWorker->tick(), 'Bad ledger candidate was not bounded.'); -$assert((int) $log->table('cloud_vm_network_projection_outbox') - ->where('sample_key', $badLedgerKey)->value('attempts') === 1, - 'Bad ledger candidate did not record its first attempt.'); -$assert($badLedgerWorker->observationErrors() - === ['projection_sample_ledger_inconsistent'], - 'Bad ledger candidate exposed no stable worker error.'); -$assert($worker()->tick(), 'Next instance did not bypass backed-off damage.'); -$assert($log->table('cloud_vm_network_projection_outbox') - ->where('sample_key', $nextInstanceKey)->value('state') === 'done', - 'Bad global candidate blocked an unrelated instance.'); -for ($attempt = 2; $attempt <= 8; ++$attempt) { - $now += 1000; - $assert($worker()->tick(), 'Bad ledger retry was not consumed.'); - $badOutbox = $log->table('cloud_vm_network_projection_outbox') - ->where('sample_key', $badLedgerKey)->first(); - $assert((int) $badOutbox->attempts === $attempt, - 'Bad ledger retry count was not durable.'); -} -$badOutbox = $log->table('cloud_vm_network_projection_outbox') - ->where('sample_key', $badLedgerKey)->first(); -$assert($badOutbox->state === 'discarded' - && $badOutbox->discard_reason_code === 'projection_retry_exhausted' - && $badOutbox->last_error_code - === 'projection_sample_ledger_inconsistent', - 'Bad ledger candidate did not reach an attributable terminal state.'); - -// Event-item persistence is in the same main transaction as finding/state and -// receipt writes. A storage failure must roll all four back; retrying the same -// outbox after repair then creates exactly one event. -$addInstance(19, 199); -$rollbackBaseline = CloudAbuseDetector::initialBaseline(); -$rollbackBaseline['sample_count'] = CloudAbuseDetector::WARMUP_WINDOWS; -$rollbackBaseline['last_bucket_start'] = 780; -$insertBaseline(19, 199, $rollbackBaseline, 780, str_repeat('4', 64)); -$insertSample(19, 199, 840, $metrics($high)); -$assert($worker()->tick(), 'Alert rollback setup projection did not run.'); -$rollbackAlertKey = $insertSample(19, 199, 900, $metrics($high)); -$main->statement( - 'CREATE TRIGGER reject_abuse_alert_item ' - . 'BEFORE INSERT ON cloud_abuse_alert_event_items ' - . "BEGIN SELECT RAISE(ABORT, 'forced item failure'); END" -); -$failedAlertWorker = $worker(); -$assert($failedAlertWorker->tick(), 'Alert item failure was not consumed.'); -$assert($failedAlertWorker->observationErrors() === ['projection_failed'] - && $main->table('cloud_abuse_findings') - ->where('instance_id', 19)->count() === 0 - && $main->table('cloud_abuse_alert_events') - ->where('instance_id', 19)->count() === 0 - && $main->table('cloud_abuse_projection_receipts') - ->where('sample_key', $rollbackAlertKey)->count() === 0 - && (int) $main->table('cloud_abuse_instance_states') - ->where('instance_id', 19)->value('last_bucket_start') === 840, - 'Alert item failure left partial state, finding, event or receipt data.'); -$main->statement('DROP TRIGGER reject_abuse_alert_item'); -$now += 1000; -$assert($worker()->tick(), 'Repaired alert event projection did not retry.'); -$assert($main->table('cloud_abuse_findings') - ->where('instance_id', 19)->count() === 1 - && $main->table('cloud_abuse_alert_events') - ->where('instance_id', 19)->count() === 1 - && $main->table('cloud_abuse_alert_event_items') - ->whereIn('event_id', $main->table('cloud_abuse_alert_events') - ->where('instance_id', 19)->pluck('id')->all())->count() === 1 - && $main->table('cloud_abuse_projection_receipts') - ->where('sample_key', $rollbackAlertKey)->count() === 1, - 'Alert event retry did not converge to one durable batch.'); - -// A crash after the main commit leaves the event and receipt authoritative; -// log finalization replay must not duplicate either immutable row. -$addInstance(20, 200); -$eventReplayBaseline = CloudAbuseDetector::initialBaseline(); -$eventReplayBaseline['sample_count'] = CloudAbuseDetector::WARMUP_WINDOWS; -$eventReplayBaseline['last_bucket_start'] = 960; -$insertBaseline(20, 200, $eventReplayBaseline, 960, str_repeat('5', 64)); -$insertSample(20, 200, 1020, $metrics($high)); -$assert($worker()->tick(), 'Event replay setup projection did not run.'); -$eventReplayKey = $insertSample(20, 200, 1080, $metrics($high)); -$eventCommitCrashed = false; -$worker(static function () use (&$eventCommitCrashed): void { - if (!$eventCommitCrashed) { - $eventCommitCrashed = true; - throw new RuntimeException('simulated event finalization crash'); - } -})->tick(); -$assert($main->table('cloud_abuse_alert_events') - ->where('instance_id', 20)->count() === 1 - && $main->table('cloud_abuse_projection_receipts') - ->where('sample_key', $eventReplayKey)->count() === 1 - && $log->table('cloud_vm_network_projection_outbox') - ->where('sample_key', $eventReplayKey)->value('state') === 'pending', - 'Post-commit crash did not retain the authoritative alert event.'); -$now += 15; -$assert($worker()->tick(), 'Event receipt replay did not run.'); -$replayEventId = (int) $main->table('cloud_abuse_alert_events') - ->where('instance_id', 20)->value('id'); -$assert($log->table('cloud_vm_network_projection_outbox') - ->where('sample_key', $eventReplayKey)->value('state') === 'done' - && $main->table('cloud_abuse_alert_events') - ->where('instance_id', 20)->count() === 1 - && $main->table('cloud_abuse_alert_event_items') - ->where('event_id', $replayEventId)->count() === 1 - && $main->table('cloud_abuse_projection_receipts') - ->where('sample_key', $eventReplayKey)->count() === 1, - 'Receipt replay duplicated the committed alert event batch.'); - -$idleWorker = $worker(); -$assert(!$idleWorker->tick() && $idleWorker->observationErrors() === [], - 'Idle projection worker retained an earlier observation error.'); - -// Detector reports suspicious-to-high as `continued`; the locked finding row -// is the transition authority. A later silent high-to-suspicious downgrade is -// not an alert transition and must not advance the transition revision. -$apply = new ReflectionMethod( - CloudAbuseProjectionService::class, - 'applyFindings' -); -$escalationClaim = [ - 'sample_key' => str_repeat('9', 64), - 'instance_id' => 18, - 'user_id' => 188, - 'host_id' => 7, - 'bucket_start' => 720, - 'rule_version' => CloudAbuseDetector::RULE_VERSION, - 'policy_revision' => 1, - 'policy_hash' => $policySnapshot->policyHash, -]; -$escalationKey = hash('sha256', implode("\n", [ - 18, - CloudAbuseDetector::RULE_VERSION, - $policySnapshot->policyHash, - 'outbound_tcp_syn_sweep', - 'ipv4', - 'outbound', -])); -$main->table('cloud_abuse_findings')->insert([ - 'finding_key' => $escalationKey, - 'instance_id' => 18, 'user_id' => 188, 'host_id' => 7, - 'rule_code' => 'outbound_tcp_syn_sweep', - 'rule_version' => CloudAbuseDetector::RULE_VERSION, - 'policy_revision' => 1, - 'policy_hash' => $policySnapshot->policyHash, - 'direction' => 'outbound', 'ip_family' => 'ipv4', - 'severity' => 'suspicious', 'state' => 'open', - 'first_seen_at' => 660, 'last_seen_at' => 660, - 'consecutive_windows' => 3, - 'occurrence_generation' => 1, 'transition_revision' => 1, - 'evidence_json' => '{"tcp_syn_flows":900}', - 'resolved_at' => null, 'resolution_reason' => '', - 'created_at' => $now, 'updated_at' => $now, -]); -$continuedHigh = [[ - 'rule_code' => 'outbound_tcp_syn_sweep', - 'direction' => 'outbound', 'ip_family' => 'ipv4', - 'severity' => 'high', 'active' => true, - 'transition' => 'continued', 'consecutive_windows' => 2, - 'evidence' => ['tcp_syn_flows' => 5000, - 'unique_remote_ips' => 700], -]]; -$projectionWorker = $worker(); -$escalated = $apply->invoke( - $projectionWorker, - $main, - $escalationClaim, - 7, - $continuedHigh, - $now -); -$assert(count($escalated) === 1 - && $escalated[0]['transition_type'] === 'escalated' - && $escalated[0]['from_severity'] === 'suspicious' - && $escalated[0]['to_severity'] === 'high' - && (int) $main->table('cloud_abuse_findings') - ->where('finding_key', $escalationKey) - ->value('transition_revision') === 2, - 'Locked finding state did not derive suspicious-to-high escalation.'); -(new App\Services\Cloud\CloudAbuseAlertEventCoordinator())->record( - $main, - $escalationClaim, - $escalated, - $now -); -$escalationClaim['sample_key'] = str_repeat('a', 64); -$escalationClaim['bucket_start'] = 780; -$silentDowngrade = $continuedHigh; -$silentDowngrade[0]['severity'] = 'suspicious'; -$silentDowngrade[0]['evidence'] = ['tcp_syn_flows' => 900, - 'unique_remote_ips' => 140]; -$beforeDowngradeEvents = $main->table( - 'cloud_abuse_alert_events' -)->count(); -$downgraded = $apply->invoke( - $projectionWorker, - $main, - $escalationClaim, - 7, - $silentDowngrade, - $now -); -$assert($downgraded === [] - && (int) $main->table('cloud_abuse_findings') - ->where('finding_key', $escalationKey) - ->value('transition_revision') === 2 - && $main->table('cloud_abuse_alert_events')->count() - === $beforeDowngradeEvents, - 'Silent severity downgrade advanced revision or created an event.'); - -$source = file_get_contents(dirname(__DIR__) - . '/app/Services/Cloud/CloudAbuseProjectionService.php'); -$assert(is_string($source) - && !str_contains($source, 'CloudJob') - && !str_contains($source, 'service_status') - && !str_contains($source, 'network_status'), - 'Projection service gained a task, lifecycle or enforcement dependency.'); - -echo "Cloud abuse projection convergence tests passed.\n"; diff --git a/web/tests/CloudBackupPublicationPreparationSchemaContractTest.php b/web/tests/CloudBackupPublicationPreparationSchemaContractTest.php index a9a2a029..cb466cc0 100644 --- a/web/tests/CloudBackupPublicationPreparationSchemaContractTest.php +++ b/web/tests/CloudBackupPublicationPreparationSchemaContractTest.php @@ -64,7 +64,7 @@ foreach ([ 'CloudSetup omits backup preparation contract: ' . $required); } $assert(str_contains($setup, - "SCHEMA_VERSION = 'go-agent-wss-v56-daily-traffic-settlement'") + "SCHEMA_VERSION = 'go-agent-wss-v57-compact-abuse-ledger'") && str_contains($setup, "'columns' => [\n 'create_request_actor_id',\n 'create_request_idempotency_key',") && str_contains($setup, "'non_unique' => 0"), diff --git a/web/tests/CloudCompactAbuseSchemaContractTest.php b/web/tests/CloudCompactAbuseSchemaContractTest.php index a096047b..ea07a197 100644 --- a/web/tests/CloudCompactAbuseSchemaContractTest.php +++ b/web/tests/CloudCompactAbuseSchemaContractTest.php @@ -2,7 +2,11 @@ $root = dirname(__DIR__); $mainSql = file_get_contents($root . '/sql/cloud.sql'); -if (!is_string($mainSql)) { +$logSql = file_get_contents($root . '/sql/cloud_log.sql'); +$setup = file_get_contents($root . '/app/Command/CloudSetup.php'); +$deployment = file_get_contents(dirname($root) . '/docs/DEPLOYMENT.md'); +if (!is_string($mainSql) || !is_string($logSql) + || !is_string($setup) || !is_string($deployment)) { throw new RuntimeException('Unable to read cloud schema'); } @@ -71,4 +75,50 @@ $assert(str_contains($definitions['cloud_network_alert_deliveries'], 'chk_cloud_network_alert_delivery_shape'), 'Telegram delivery lost its conservative unknown-effect fence.'); +$legacyMainTables = [ + 'cloud_abuse_instance_states', + 'cloud_abuse_findings', + 'cloud_abuse_alert_events', + 'cloud_abuse_alert_event_items', + 'cloud_abuse_alert_deliveries', + 'cloud_abuse_projection_receipts', +]; +$legacyLogTables = [ + 'cloud_vm_network_observation_samples', + 'cloud_vm_network_observation_rollups', + 'cloud_vm_network_projection_outbox', + 'cloud_vm_network_monitor_cursors', + 'cloud_vm_network_baselines', +]; +foreach ($legacyMainTables as $table) { + $assert(!str_contains($mainSql, 'CREATE TABLE ' . $table . ' ('), + 'Fresh main schema retained legacy abuse table ' . $table . '.'); + $assert(str_contains($setup, "'" . $table . "'"), + 'CloudSetup no longer refuses retired main table ' . $table . '.'); +} +foreach ($legacyLogTables as $table) { + $assert(!str_contains($logSql, 'CREATE TABLE ' . $table . ' ('), + 'Fresh log schema retained legacy abuse table ' . $table . '.'); + $assert(str_contains($setup, "'" . $table . "'"), + 'CloudSetup no longer refuses retired log table ' . $table . '.'); +} + +foreach ([ + 'CloudAbuseProjectionService.php', + 'CloudAbuseObservationRollupStore.php', + 'CloudAbuseAlertEventCoordinator.php', +] as $legacyClass) { + $assert(!is_file($root . '/app/Services/Cloud/' . $legacyClass), + 'Legacy abuse pipeline class still exists: ' . $legacyClass); +} + +$assert(str_contains($setup, + "SCHEMA_VERSION = 'go-agent-wss-v57-compact-abuse-ledger'") + && str_contains($setup, 'private const RETIRED_MAIN_TABLES') + && str_contains($setup, 'private const RETIRED_LOG_TABLES') + && str_contains($setup, "'cloud_network_alert_delivery_admin' => [") + && str_contains($deployment, '不再保存逐分钟样本') + && str_contains($deployment, '没有第二阶段投影 worker'), + 'Compact clean-break schema gate or deployment contract is incomplete.'); + echo "Compact abuse monitoring schema contract passed.\n"; diff --git a/web/tests/CloudFreshSchemaContractTest.php b/web/tests/CloudFreshSchemaContractTest.php index e9d1a018..0cf865ff 100644 --- a/web/tests/CloudFreshSchemaContractTest.php +++ b/web/tests/CloudFreshSchemaContractTest.php @@ -27,10 +27,10 @@ $assert(stripos($schema, 'compatibility import') === false $assert(!preg_match('/\bengine_[A-Za-z0-9_]*\b/', $schema), 'cloud.sql retained an engine-prefixed identifier.'); $assert(str_contains($setup, - "SCHEMA_VERSION = 'go-agent-wss-v56-daily-traffic-settlement'") + "SCHEMA_VERSION = 'go-agent-wss-v57-compact-abuse-ledger'") && !str_contains($setup, "SCHEMA_VERSION = 'go-v1'") - && str_contains($deployment, '`go-agent-wss-v56-daily-traffic-settlement`') - && str_contains($compensation, '`go-agent-wss-v56-daily-traffic-settlement`'), + && str_contains($deployment, '`go-agent-wss-v57-compact-abuse-ledger`') + && str_contains($compensation, '`go-agent-wss-v57-compact-abuse-ledger`'), 'Outbound Agent schema must reject the incompatible earlier Go database.'); $assert(str_contains($schema, "timezone VARCHAR(64) CHARACTER SET ascii COLLATE ascii_bin NOT NULL DEFAULT 'UTC'") diff --git a/web/tests/CloudMaintenanceNoticeSchemaContractTest.php b/web/tests/CloudMaintenanceNoticeSchemaContractTest.php index b243f73f..cc7a0361 100644 --- a/web/tests/CloudMaintenanceNoticeSchemaContractTest.php +++ b/web/tests/CloudMaintenanceNoticeSchemaContractTest.php @@ -56,7 +56,7 @@ foreach ([ } $assert(str_contains($setup, - "SCHEMA_VERSION = 'go-agent-wss-v56-daily-traffic-settlement'"), + "SCHEMA_VERSION = 'go-agent-wss-v57-compact-abuse-ledger'"), 'CloudSetup schema version was not advanced for maintenance notices.'); echo "Maintenance notice schema contracts passed.\n"; diff --git a/web/tests/CloudPublicIpPoolReplacementSchemaContractTest.php b/web/tests/CloudPublicIpPoolReplacementSchemaContractTest.php index 3b668583..bc143945 100644 --- a/web/tests/CloudPublicIpPoolReplacementSchemaContractTest.php +++ b/web/tests/CloudPublicIpPoolReplacementSchemaContractTest.php @@ -53,7 +53,7 @@ $assert(is_string($schema) 'Public-pool replacement lacks durable batch, child, or claim schema.'); $assert(is_string($setup) && str_contains($setup, - "SCHEMA_VERSION = 'go-agent-wss-v56-daily-traffic-settlement'") + "SCHEMA_VERSION = 'go-agent-wss-v57-compact-abuse-ledger'") && str_contains($setup, "'cloud_public_ip_pool_replacement_batches'") && str_contains($setup, diff --git a/web/tests/CloudPublicNetworkTransitionSchemaContractTest.php b/web/tests/CloudPublicNetworkTransitionSchemaContractTest.php index 0f1499c5..c8f1061f 100644 --- a/web/tests/CloudPublicNetworkTransitionSchemaContractTest.php +++ b/web/tests/CloudPublicNetworkTransitionSchemaContractTest.php @@ -170,7 +170,7 @@ $assert(str_contains($setup, 'CloudSetup does not verify index uniqueness and ordered columns.'); $assert(str_contains($setup, - "SCHEMA_VERSION = 'go-agent-wss-v56-daily-traffic-settlement'") + "SCHEMA_VERSION = 'go-agent-wss-v57-compact-abuse-ledger'") && str_contains($setup, "'cloud_public_network_transitions'") && str_contains($setup, "'cloud_public_network_transition_ips'") && str_contains($setup, "'reservation_ip_claim_id'") diff --git a/web/tests/CloudRescueCatalogSchemaContractTest.php b/web/tests/CloudRescueCatalogSchemaContractTest.php index d20278ed..d62692b9 100644 --- a/web/tests/CloudRescueCatalogSchemaContractTest.php +++ b/web/tests/CloudRescueCatalogSchemaContractTest.php @@ -57,7 +57,7 @@ $assert($replica !== '' && str_contains($replica, 'cleanup_resolved_at BIGINT UNSIGNED'), 'Rescue replica cannot prove exact identity or retain cleanup debt.'); $assert(str_contains($setup, - "SCHEMA_VERSION = 'go-agent-wss-v56-daily-traffic-settlement'") + "SCHEMA_VERSION = 'go-agent-wss-v57-compact-abuse-ledger'") && str_contains($setup, "'cloud_rescue_artifact_delete_due'") && str_contains($setup, "'cloud_rescue_replica_cleanup'"), 'CloudSetup version or rescue catalog indexes are incomplete.'); diff --git a/web/tests/CloudRestoreMigrationLedgerContractTest.php b/web/tests/CloudRestoreMigrationLedgerContractTest.php index b8145eac..5b92f87f 100644 --- a/web/tests/CloudRestoreMigrationLedgerContractTest.php +++ b/web/tests/CloudRestoreMigrationLedgerContractTest.php @@ -218,7 +218,7 @@ foreach ([ 'CloudSetup omits restore/migration ledger contract: ' . $required); } $assert(str_contains($setup, - "SCHEMA_VERSION = 'go-agent-wss-v56-daily-traffic-settlement'") + "SCHEMA_VERSION = 'go-agent-wss-v57-compact-abuse-ledger'") && str_contains($attemptModel, "protected \$table = 'cloud_backup_restore_attempts';") && str_contains($attemptModel, "'active_attempt' => 'int'") diff --git a/web/tests/CloudServiceCompensationSchemaContractTest.php b/web/tests/CloudServiceCompensationSchemaContractTest.php index c9bccc32..8129a98d 100644 --- a/web/tests/CloudServiceCompensationSchemaContractTest.php +++ b/web/tests/CloudServiceCompensationSchemaContractTest.php @@ -105,7 +105,7 @@ $assert(str_contains($setup, 'CloudSetup lost the open-compensation fence index contract.'); $assert(str_contains($setup, - "SCHEMA_VERSION = 'go-agent-wss-v56-daily-traffic-settlement'"), + "SCHEMA_VERSION = 'go-agent-wss-v57-compact-abuse-ledger'"), 'CloudSetup schema version was not advanced for service compensation.'); $appliedShape = preg_match( diff --git a/web/tests/CloudStoragePublicationSchemaContractTest.php b/web/tests/CloudStoragePublicationSchemaContractTest.php index ec1e2ac8..0c44bd3b 100644 --- a/web/tests/CloudStoragePublicationSchemaContractTest.php +++ b/web/tests/CloudStoragePublicationSchemaContractTest.php @@ -138,7 +138,7 @@ $assert(is_string($publicationModel) && str_contains($artifactModel, "'maximum_size_bytes' => 'int'"), 'Storage publication/artifact model contracts are incomplete.'); $assert(str_contains($setup, - "SCHEMA_VERSION = 'go-agent-wss-v56-daily-traffic-settlement'"), + "SCHEMA_VERSION = 'go-agent-wss-v57-compact-abuse-ledger'"), 'CloudSetup schema version was not advanced for durable publications.'); echo "Storage publication schema contracts passed.\n"; diff --git a/web/tests/CloudTemplateImportIntentContractTest.php b/web/tests/CloudTemplateImportIntentContractTest.php index eb5dc996..db72b3a0 100644 --- a/web/tests/CloudTemplateImportIntentContractTest.php +++ b/web/tests/CloudTemplateImportIntentContractTest.php @@ -63,7 +63,7 @@ $assert(str_contains($schema, && str_contains($schema, 'CONSTRAINT chk_cloud_template_import_completion CHECK (') && str_contains($setup, - "SCHEMA_VERSION = 'go-agent-wss-v56-daily-traffic-settlement'"), + "SCHEMA_VERSION = 'go-agent-wss-v57-compact-abuse-ledger'"), 'Template imports are missing durable identity or terminal-state constraints.'); $assert(str_contains($routes, "'/admin/cloud-template-imports'") && str_contains($routes, "'/admin/cloud-template-imports/{id}'") diff --git a/web/tests/CloudTrafficPoolSchemaContractTest.php b/web/tests/CloudTrafficPoolSchemaContractTest.php index 18fe2855..6cd7df8c 100644 --- a/web/tests/CloudTrafficPoolSchemaContractTest.php +++ b/web/tests/CloudTrafficPoolSchemaContractTest.php @@ -86,7 +86,7 @@ $assert(str_contains($schema, 'Service-period traffic-pool identity is not frozen or indexed.'); $assert(str_contains($setup, - "SCHEMA_VERSION = 'go-agent-wss-v56-daily-traffic-settlement'") + "SCHEMA_VERSION = 'go-agent-wss-v57-compact-abuse-ledger'") && str_contains($setup, "'chk_cloud_traffic_entitlement_balance'") && str_contains($setup, "'cloud_traffic_ledger_sequence'") && str_contains($setup, "'chk_cloud_traffic_settlement_state'") diff --git a/web/tests/CloudVpcSchemaContractTest.php b/web/tests/CloudVpcSchemaContractTest.php index b997b71b..b74cecbb 100644 --- a/web/tests/CloudVpcSchemaContractTest.php +++ b/web/tests/CloudVpcSchemaContractTest.php @@ -101,7 +101,7 @@ $assert(str_contains($jobs, 'vpc_id BIGINT UNSIGNED DEFAULT NULL') 'Cloud jobs cannot be correlated to VPC or VPC-port work.'); $assert(str_contains($setup, - "private const SCHEMA_VERSION = 'go-agent-wss-v56-daily-traffic-settlement'") + "private const SCHEMA_VERSION = 'go-agent-wss-v57-compact-abuse-ledger'") && str_contains($setup, "'name' => 'cloud_vpc_enabled'") && str_contains($setup, "'value' => 'false'") && str_contains($setup, diff --git a/web/tests/ManualAccountDeletionSqliteTest.php b/web/tests/ManualAccountDeletionSqliteTest.php index cb540f2b..cf551479 100644 --- a/web/tests/ManualAccountDeletionSqliteTest.php +++ b/web/tests/ManualAccountDeletionSqliteTest.php @@ -40,37 +40,6 @@ $logSchema->create('site_security_events', static function (Blueprint $table) { $logSchema->create('user_email_log', static function (Blueprint $table) { $table->increments('id'); $table->string('email'); }); -$logSchema->create( - 'cloud_vm_network_observation_samples', - static function (Blueprint $table) { - $table->string('sample_key')->primary(); - $table->integer('user_id'); - } -); -$logSchema->create( - 'cloud_vm_network_projection_outbox', - static function (Blueprint $table) { - $table->string('sample_key')->primary(); - $table->string('state'); - $table->integer('rule_version')->default(1); - $table->integer('policy_revision')->default(1); - $table->string('policy_hash')->default( - '511828bde950c504f92bb7a0603ca8b14453f575ead276c1889baa764f384dae' - ); - $table->foreign('sample_key') - ->references('sample_key') - ->on('cloud_vm_network_observation_samples') - ->onDelete('restrict'); - } -); -$logSchema->create('cloud_vm_network_baselines', static function (Blueprint $table) { - $table->integer('instance_id')->primary(); $table->integer('user_id'); - $table->integer('rule_version')->default(1); - $table->integer('policy_revision')->default(1); - $table->string('policy_hash')->default( - '511828bde950c504f92bb7a0603ca8b14453f575ead276c1889baa764f384dae' - ); -}); $schema->create('user', static function (Blueprint $table) { $table->increments('id'); $table->string('email')->unique(); @@ -111,31 +80,21 @@ foreach (['yipay_callback_events', 'yipay_refunds'] as $tableName) { $schema->create('cloud_instances', static function (Blueprint $table) { $table->increments('id'); $table->integer('user_id'); $table->string('status'); }); -$schema->create('cloud_abuse_instance_states', - static function (Blueprint $table) { - $table->integer('instance_id')->primary(); - $table->integer('user_id'); - }); -$schema->create('cloud_abuse_findings', - static function (Blueprint $table) { - $table->increments('id'); - $table->integer('instance_id'); - $table->integer('user_id'); - }); -$schema->create('cloud_abuse_alert_events', +$schema->create('cloud_network_findings', static function (Blueprint $table) { $table->increments('id'); $table->integer('instance_id'); - $table->integer('user_id'); + $table->foreign('instance_id')->references('id') + ->on('cloud_instances')->cascadeOnDelete(); }); -$schema->create('cloud_abuse_alert_event_items', +$schema->create('cloud_network_alert_events', static function (Blueprint $table) { $table->increments('id'); - $table->integer('event_id'); - $table->foreign('event_id')->references('id') - ->on('cloud_abuse_alert_events')->cascadeOnDelete(); + $table->integer('finding_id'); + $table->foreign('finding_id')->references('id') + ->on('cloud_network_findings')->cascadeOnDelete(); }); -$schema->create('cloud_abuse_alert_deliveries', +$schema->create('cloud_network_alert_deliveries', static function (Blueprint $table) { $table->increments('id'); $table->integer('event_id')->unique(); @@ -151,7 +110,7 @@ $schema->create('cloud_abuse_alert_deliveries', $table->integer('destination_redacted_at')->nullable(); $table->integer('updated_at')->default(0); $table->foreign('event_id')->references('id') - ->on('cloud_abuse_alert_events')->cascadeOnDelete(); + ->on('cloud_network_alert_events')->cascadeOnDelete(); }); $schema->create('cloud_orders', static function (Blueprint $table) { $table->increments('id'); $table->integer('user_id'); $table->string('status'); @@ -284,26 +243,15 @@ Capsule::table('cloud_instances')->insert([ ['id' => 101, 'user_id' => 1, 'status' => 'deleted'], ['id' => 301, 'user_id' => 3, 'status' => 'active'], ]); -// Instance 301 has already been transferred to user 3. Historical abuse rows -// still frozen to old owner 1 must be erased with owner 1 without deleting the -// transferred instance or user 3's unrelated abuse history. -Capsule::table('cloud_abuse_instance_states')->insert([ - ['instance_id' => 301, 'user_id' => 1], - ['instance_id' => 302, 'user_id' => 3], +Capsule::table('cloud_network_findings')->insert([ + ['id' => 1, 'instance_id' => 101], + ['id' => 2, 'instance_id' => 301], ]); -Capsule::table('cloud_abuse_findings')->insert([ - ['id' => 1, 'instance_id' => 301, 'user_id' => 1], - ['id' => 2, 'instance_id' => 302, 'user_id' => 3], +Capsule::table('cloud_network_alert_events')->insert([ + ['id' => 1, 'finding_id' => 1], + ['id' => 2, 'finding_id' => 2], ]); -Capsule::table('cloud_abuse_alert_events')->insert([ - ['id' => 1, 'instance_id' => 301, 'user_id' => 1], - ['id' => 2, 'instance_id' => 302, 'user_id' => 3], -]); -Capsule::table('cloud_abuse_alert_event_items')->insert([ - ['id' => 1, 'event_id' => 1], - ['id' => 2, 'event_id' => 2], -]); -Capsule::table('cloud_abuse_alert_deliveries')->insert([ +Capsule::table('cloud_network_alert_deliveries')->insert([ [ 'id' => 1, 'event_id' => 1, 'binding_generation' => str_repeat('a', 32), @@ -402,38 +350,17 @@ Capsule::table('email_verify')->insert([ Capsule::table('configurations')->insert([ 'name' => 'email_blacklist', 'value' => "deleted@example.com\n@blocked.example", ]); -$logConnection->table('cloud_vm_network_observation_samples')->insert([ - ['sample_key' => str_repeat('a', 64), 'user_id' => 1], - ['sample_key' => str_repeat('b', 64), 'user_id' => 1], - ['sample_key' => str_repeat('c', 64), 'user_id' => 3], -]); -$logConnection->table('cloud_vm_network_projection_outbox')->insert([ - ['sample_key' => str_repeat('a', 64), 'state' => 'pending'], - ['sample_key' => str_repeat('b', 64), 'state' => 'leased'], - ['sample_key' => str_repeat('c', 64), 'state' => 'done'], -]); -$logConnection->table('cloud_vm_network_baselines')->insert([ - ['instance_id' => 101, 'user_id' => 1], - ['instance_id' => 301, 'user_id' => 3], -]); - $result = (new UserDataDeletionService())->deleteUsers([1, '2', 999], false); $assert($result['deleted_user_ids'] === [1], 'Expected only the ordinary selected account to be deleted.'); $assert($result['skipped_user_ids'] === [2, 999], 'Administrator or missing account was not excluded.'); $assert(User::find(1) === null && User::find(2) !== null && User::find(3) !== null, 'User deletion was not isolated.'); $assert(Capsule::table('cloud_instances')->where('id', 301)->exists() - && Capsule::table('cloud_abuse_instance_states') - ->orderBy('instance_id')->pluck('user_id')->all() === [3] - && Capsule::table('cloud_abuse_findings') - ->orderBy('id')->pluck('user_id')->all() === [3] - && Capsule::table('cloud_abuse_alert_events') - ->orderBy('id')->pluck('user_id')->all() === [3] - && Capsule::table('cloud_abuse_alert_event_items') - ->orderBy('id')->pluck('event_id')->all() === [2], - 'Transferred instance retained old-owner abuse data or was deleted.'); -$retainedAbuseDelivery = Capsule::table('cloud_abuse_alert_deliveries') + && Capsule::table('cloud_network_findings')->pluck('id')->all() === [2] + && Capsule::table('cloud_network_alert_events')->pluck('id')->all() === [2], + 'Instance-centric abuse data did not follow tenant instance deletion.'); +$retainedAbuseDelivery = Capsule::table('cloud_network_alert_deliveries') ->where('event_id', 2)->first(); -$assert(Capsule::table('cloud_abuse_alert_deliveries')->count() === 1 +$assert(Capsule::table('cloud_network_alert_deliveries')->count() === 1 && $retainedAbuseDelivery !== null && $retainedAbuseDelivery->state === 'sent' && $retainedAbuseDelivery->bound_admin_id === null @@ -495,15 +422,8 @@ $assert(Capsule::table('cloud_service_compensation_items')->pluck('user_id')->al 'Compensation tenant details were not deleted in isolation.'); $assert(Capsule::table('email_verify')->pluck('email')->all() === ['other@example.com'], 'Email verification cleanup was not isolated.'); $assert(Capsule::table('configurations')->where('name', 'email_blacklist')->value('value') === "deleted@example.com\n@blocked.example", 'Manual blacklist was changed.'); -$assert($logConnection->table('cloud_vm_network_observation_samples') - ->pluck('sample_key')->all() === [str_repeat('c', 64)] - && $logConnection->table('cloud_vm_network_projection_outbox') - ->pluck('sample_key')->all() === [str_repeat('c', 64)] - && $logConnection->table('cloud_vm_network_baselines') - ->pluck('user_id')->all() === [3], - 'RESTRICTed network-observation rows were not deleted in owner-safe order.'); $assert($result['cache_error'] === false - && $result['log_rows_deleted'] === 5, + && $result['log_rows_deleted'] === 0, 'Log account history was not deleted in isolation.'); Capsule::table('user')->insert([ @@ -670,10 +590,13 @@ Capsule::table('user')->insert([ 'id' => 12, 'email' => 'former-alert-admin@example.com', 'is_admin' => 0, 'ref_by' => 0, ]); -Capsule::table('cloud_abuse_alert_events')->insert([ - 'id' => 12, 'instance_id' => 301, 'user_id' => 3, +Capsule::table('cloud_network_findings')->insert([ + 'id' => 12, 'instance_id' => 301, +]); +Capsule::table('cloud_network_alert_events')->insert([ + 'id' => 12, 'finding_id' => 12, ]); -Capsule::table('cloud_abuse_alert_deliveries')->insert([ +Capsule::table('cloud_network_alert_deliveries')->insert([ 'id' => 12, 'event_id' => 12, 'binding_generation' => str_repeat('c', 32), 'bound_admin_id' => 12, 'chat_id' => '121212', @@ -701,19 +624,19 @@ try { 'Dispatching abuse alert deletion returned the wrong error.'); } $assert(User::find(12) !== null - && Capsule::table('cloud_abuse_alert_deliveries') + && Capsule::table('cloud_network_alert_deliveries') ->where('id', 12)->value('state') === 'dispatching', 'Dispatching abuse alert deletion fence did not roll back.'); $assert($logConnection->table('site_visit_events') ->where('user_id', 12)->exists() && Capsule::table('configurations')->where('id', 12)->exists(), 'Dispatching abuse guard ran after irreversible log or binding cleanup.'); -Capsule::table('cloud_abuse_alert_deliveries')->where('id', 12)->update([ +Capsule::table('cloud_network_alert_deliveries')->where('id', 12)->update([ 'state' => 'unknown', 'terminal_at' => time(), ]); $formerAdminDeletion = (new UserDataDeletionService()) ->deleteUsers([12], false); -$formerAdminDelivery = Capsule::table('cloud_abuse_alert_deliveries') +$formerAdminDelivery = Capsule::table('cloud_network_alert_deliveries') ->where('id', 12)->first(); $assert($formerAdminDeletion['deleted_user_ids'] === [12] && $formerAdminDelivery?->state === 'unknown' @@ -831,10 +754,16 @@ Capsule::table('user')->insert([ 'id' => 15, 'email' => 'alert-owner@example.com', 'is_admin' => 0, 'ref_by' => 0, ]); -Capsule::table('cloud_abuse_alert_events')->insert([ - 'id' => 15, 'instance_id' => 301, 'user_id' => 15, +Capsule::table('cloud_instances')->insert([ + 'id' => 315, 'user_id' => 15, 'status' => 'deleted', +]); +Capsule::table('cloud_network_findings')->insert([ + 'id' => 15, 'instance_id' => 315, ]); -Capsule::table('cloud_abuse_alert_deliveries')->insert([ +Capsule::table('cloud_network_alert_events')->insert([ + 'id' => 15, 'finding_id' => 15, +]); +Capsule::table('cloud_network_alert_deliveries')->insert([ 'id' => 15, 'event_id' => 15, 'binding_generation' => str_repeat('f', 32), 'bound_admin_id' => 2, 'chat_id' => '222222', @@ -853,27 +782,33 @@ try { 'Owner-event dispatch deletion returned the wrong error.'); } $assert(User::find(15) !== null - && Capsule::table('cloud_abuse_alert_events')->where('id', 15)->exists() + && Capsule::table('cloud_network_alert_events')->where('id', 15)->exists() && $logConnection->table('site_visit_events') ->where('user_id', 15)->exists(), 'Owner-event dispatch fence allowed partial account deletion.'); -Capsule::table('cloud_abuse_alert_deliveries')->where('id', 15)->update([ +Capsule::table('cloud_network_alert_deliveries')->where('id', 15)->update([ 'state' => 'unknown', 'terminal_at' => time(), ]); (new UserDataDeletionService())->deleteUsers([15], false); $assert(User::find(15) === null - && !Capsule::table('cloud_abuse_alert_events')->where('id', 15)->exists() - && !Capsule::table('cloud_abuse_alert_deliveries')->where('id', 15)->exists(), + && !Capsule::table('cloud_network_alert_events')->where('id', 15)->exists() + && !Capsule::table('cloud_network_alert_deliveries')->where('id', 15)->exists(), 'Terminal owner event did not cascade with tenant deletion.'); Capsule::table('user')->insert([ 'id' => 17, 'email' => 'stale-alert-owner@example.com', 'is_admin' => 0, 'ref_by' => 0, ]); -Capsule::table('cloud_abuse_alert_events')->insert([ - 'id' => 17, 'instance_id' => 301, 'user_id' => 17, +Capsule::table('cloud_instances')->insert([ + 'id' => 317, 'user_id' => 17, 'status' => 'deleted', +]); +Capsule::table('cloud_network_findings')->insert([ + 'id' => 17, 'instance_id' => 317, +]); +Capsule::table('cloud_network_alert_events')->insert([ + 'id' => 17, 'finding_id' => 17, ]); -Capsule::table('cloud_abuse_alert_deliveries')->insert([ +Capsule::table('cloud_network_alert_deliveries')->insert([ 'id' => 17, 'event_id' => 17, 'binding_generation' => str_repeat('a', 32), 'bound_admin_id' => 2, 'chat_id' => '222222', @@ -883,8 +818,8 @@ Capsule::table('cloud_abuse_alert_deliveries')->insert([ ]); (new UserDataDeletionService())->deleteUsers([17], false); $assert(User::find(17) === null - && !Capsule::table('cloud_abuse_alert_events')->where('id', 17)->exists() - && !Capsule::table('cloud_abuse_alert_deliveries')->where('id', 17)->exists(), + && !Capsule::table('cloud_network_alert_events')->where('id', 17)->exists() + && !Capsule::table('cloud_network_alert_deliveries')->where('id', 17)->exists(), 'Stale abuse dispatch permanently blocked owner deletion.'); Capsule::table('user')->insert([ diff --git a/web/tests/TicketTelegramNotifierSecurityTest.php b/web/tests/TicketTelegramNotifierSecurityTest.php index ff896a2b..919d9977 100644 --- a/web/tests/TicketTelegramNotifierSecurityTest.php +++ b/web/tests/TicketTelegramNotifierSecurityTest.php @@ -393,7 +393,7 @@ namespace { if ($table === 'ticket') { return new TelegramTestTicketRootQuery(); } - if ($table === 'cloud_abuse_alert_deliveries') { + if ($table === 'cloud_network_alert_deliveries') { if (TelegramTestAbuseStore::$queryFails) { throw new RuntimeException('synthetic abuse ledger failure'); } @@ -407,7 +407,7 @@ namespace { return new class { public function hasTable(string $table): bool { - return $table === 'cloud_abuse_alert_deliveries'; + return $table === 'cloud_network_alert_deliveries'; } }; } diff --git a/web/tests/UiCopyCatalogSchemaContractTest.php b/web/tests/UiCopyCatalogSchemaContractTest.php index 09496c3b..920cbec3 100644 --- a/web/tests/UiCopyCatalogSchemaContractTest.php +++ b/web/tests/UiCopyCatalogSchemaContractTest.php @@ -104,7 +104,7 @@ foreach ($models as $class => $modelTable) { } $assert(str_contains($setup, - "SCHEMA_VERSION = 'go-agent-wss-v56-daily-traffic-settlement'") + "SCHEMA_VERSION = 'go-agent-wss-v57-compact-abuse-ledger'") && str_contains($setup, "self::schema('ss.sql') . \"\\0\" . self::schema('cloud.sql')") && str_contains($setup,