You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
virt/web/sql/cloud_log.sql

405 lines
19 KiB
MySQL

-- Fresh-install Agent metric and traffic schema. Agents expose current signed
-- counters; PHP owns sampling, billing totals, chart retention and all writes.
CREATE TABLE cloud_vm_metric_samples (
instance_id BIGINT UNSIGNED NOT NULL,
user_id BIGINT UNSIGNED NOT NULL,
sampled_at BIGINT UNSIGNED NOT NULL,
cpu_percent DOUBLE NULL,
memory_used_mib DOUBLE NULL,
memory_total_mib DOUBLE NULL,
disk_read_bps DOUBLE NULL,
disk_write_bps DOUBLE NULL,
network_rx_bps DOUBLE NULL,
network_tx_bps DOUBLE NULL,
network_rx_bytes BIGINT UNSIGNED NULL,
network_tx_bytes BIGINT UNSIGNED NULL,
PRIMARY KEY (instance_id, sampled_at),
KEY vm_metric_retention (sampled_at),
KEY vm_metric_owner (user_id, sampled_at, instance_id)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
-- Hierarchical chart history. Every accepted raw sample contributes once to
-- one five-minute and one hourly row; sums and non-null counts preserve exact
-- averages when a larger display bucket merges several stored rows.
CREATE TABLE cloud_vm_metric_rollups (
instance_id BIGINT UNSIGNED NOT NULL,
user_id BIGINT UNSIGNED NOT NULL,
interval_seconds SMALLINT UNSIGNED NOT NULL,
bucket_start BIGINT UNSIGNED NOT NULL,
sample_count SMALLINT UNSIGNED NOT NULL,
last_sampled_at BIGINT UNSIGNED NOT NULL,
cpu_percent_sum DOUBLE NULL,
cpu_percent_count SMALLINT UNSIGNED NOT NULL,
memory_used_mib_sum DOUBLE NULL,
memory_used_mib_count SMALLINT UNSIGNED NOT NULL,
memory_total_mib_sum DOUBLE NULL,
memory_total_mib_count SMALLINT UNSIGNED NOT NULL,
disk_read_bps_sum DOUBLE NULL,
disk_read_bps_count SMALLINT UNSIGNED NOT NULL,
disk_write_bps_sum DOUBLE NULL,
disk_write_bps_count SMALLINT UNSIGNED NOT NULL,
network_rx_bps_sum DOUBLE NULL,
network_rx_bps_count SMALLINT UNSIGNED NOT NULL,
network_tx_bps_sum DOUBLE NULL,
network_tx_bps_count SMALLINT UNSIGNED NOT NULL,
PRIMARY KEY (instance_id, interval_seconds, bucket_start),
KEY vm_metric_rollup_retention (
interval_seconds, bucket_start, instance_id
),
KEY vm_metric_rollup_owner (user_id, instance_id, bucket_start),
CONSTRAINT chk_vm_metric_rollup_bucket CHECK (
interval_seconds IN (300, 3600)
AND bucket_start % interval_seconds = 0
AND last_sampled_at >= bucket_start
AND last_sampled_at < bucket_start + interval_seconds
),
CONSTRAINT chk_vm_metric_rollup_counts CHECK (
sample_count > 0
AND cpu_percent_count <= sample_count
AND memory_used_mib_count <= sample_count
AND memory_total_mib_count <= sample_count
AND disk_read_bps_count <= sample_count
AND disk_write_bps_count <= sample_count
AND network_rx_bps_count <= sample_count
AND network_tx_bps_count <= sample_count
)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
CREATE TABLE cloud_vm_traffic_totals (
host_id BIGINT UNSIGNED NOT NULL,
vm_uuid BINARY(16) NOT NULL,
user_id BIGINT UNSIGNED NOT NULL,
last_observed_at BIGINT UNSIGNED NULL,
network_signature BINARY(32) NOT NULL,
network_baseline_json JSON NOT NULL,
download_rx_bytes BIGINT UNSIGNED NOT NULL DEFAULT 0,
upload_tx_bytes BIGINT UNSIGNED NOT NULL DEFAULT 0,
updated_at BIGINT UNSIGNED NOT NULL,
PRIMARY KEY (host_id, vm_uuid),
KEY vm_traffic_uuid (vm_uuid, host_id),
KEY vm_traffic_owner (user_id, host_id)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
-- Compact, durable billing ledger. Raw chart samples are pruned after 24
-- hours and rollups are bounded; the first cumulative counter observed in
-- every hour is not. Host IDs remain numeric and UUIDs use their native
-- 16-byte representation because these rows grow for the life of a VM.
CREATE TABLE cloud_vm_traffic_checkpoints (
host_id BIGINT UNSIGNED NOT NULL,
vm_uuid BINARY(16) NOT NULL,
checkpoint_hour BIGINT UNSIGNED NOT NULL,
user_id BIGINT UNSIGNED NOT NULL,
first_observed_at BIGINT UNSIGNED NOT NULL,
download_rx_bytes BIGINT UNSIGNED NOT NULL,
upload_tx_bytes BIGINT UNSIGNED NOT NULL,
PRIMARY KEY (host_id, vm_uuid, checkpoint_hour),
KEY vm_traffic_checkpoint_owner (user_id, host_id, first_observed_at)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
-- Logical-instance daily traffic survives maintenance migration because its
-- identity does not depend on the current host or libvirt UUID. The cursor is
-- retained beyond the 30-day chart window so an old cumulative total can
-- never be charged into a new day after pruning.
CREATE TABLE cloud_vm_daily_traffic_cursors (
instance_id BIGINT UNSIGNED NOT NULL,
user_id BIGINT UNSIGNED NOT NULL,
last_observed_at BIGINT UNSIGNED NOT NULL,
download_total_bytes BIGINT UNSIGNED NOT NULL DEFAULT 0,
upload_total_bytes BIGINT UNSIGNED NOT NULL DEFAULT 0,
updated_at BIGINT UNSIGNED NOT NULL,
PRIMARY KEY (instance_id),
KEY cloud_daily_traffic_cursor_owner (user_id, updated_at)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
CREATE TABLE cloud_vm_daily_traffic (
instance_id BIGINT UNSIGNED NOT NULL,
user_id BIGINT UNSIGNED NOT NULL,
day_start BIGINT UNSIGNED NOT NULL,
download_bytes BIGINT UNSIGNED NOT NULL DEFAULT 0,
upload_bytes BIGINT UNSIGNED NOT NULL DEFAULT 0,
sample_count INT UNSIGNED NOT NULL DEFAULT 0,
first_observed_at BIGINT UNSIGNED NOT NULL,
last_observed_at BIGINT UNSIGNED NOT NULL,
created_at BIGINT UNSIGNED NOT NULL,
updated_at BIGINT UNSIGNED NOT NULL,
PRIMARY KEY (instance_id, day_start),
KEY cloud_daily_traffic_retention (day_start),
KEY cloud_daily_traffic_owner (user_id, day_start)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
-- Network-abuse observations are deliberately isolated from billing traffic.
-- The Agent publishes bounded, aggregate one-minute windows only: no packet
-- payload, remote-address list or remote-port list is stored here. A stable
-- sample key makes retries idempotent while instance_id keeps history attached
-- to the logical VM across maintenance migration.
CREATE TABLE cloud_vm_network_observation_samples (
sample_key CHAR(64) CHARACTER SET ascii COLLATE ascii_bin NOT NULL,
instance_id BIGINT UNSIGNED NOT NULL,
user_id BIGINT UNSIGNED NOT NULL,
host_id BIGINT UNSIGNED NOT NULL,
vm_uuid CHAR(36) CHARACTER SET ascii COLLATE ascii_bin NOT NULL,
placement_key CHAR(64) CHARACTER SET ascii COLLATE ascii_bin NOT NULL,
collector_generation CHAR(32) CHARACTER SET ascii COLLATE ascii_bin NOT NULL,
window_id CHAR(32) CHARACTER SET ascii COLLATE ascii_bin NOT NULL,
contract_version SMALLINT UNSIGNED NOT NULL,
bucket_start BIGINT UNSIGNED NOT NULL,
bucket_seconds SMALLINT UNSIGNED NOT NULL,
complete TINYINT(1) NOT NULL,
metrics_json JSON NOT NULL,
quality_flags_json JSON NOT NULL,
received_at BIGINT UNSIGNED NOT NULL,
PRIMARY KEY (sample_key),
UNIQUE KEY cloud_network_observation_source (
host_id, collector_generation, window_id, vm_uuid
),
KEY cloud_network_observation_instance (
instance_id, bucket_start
),
KEY cloud_network_observation_retention (bucket_start),
CONSTRAINT chk_cloud_network_observation_contract CHECK (
contract_version = 1
AND sample_key REGEXP '^[a-f0-9]{64}$'
AND placement_key REGEXP '^[a-f0-9]{64}$'
AND collector_generation REGEXP '^[a-f0-9]{32}$'
AND window_id REGEXP '^[a-f0-9]{32}$'
AND vm_uuid REGEXP
'^[a-f0-9]{8}-[a-f0-9]{4}-[1-5][a-f0-9]{3}-[89ab][a-f0-9]{3}-[a-f0-9]{12}$'
),
CONSTRAINT chk_cloud_network_observation_bucket CHECK (
bucket_seconds = 60 AND bucket_start % 60 = 0
),
CONSTRAINT chk_cloud_network_observation_complete CHECK (
complete IN (0, 1)
)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
-- Bounded numeric history derived exactly once from each accepted raw sample.
-- Five-minute rows support recent investigation; hourly rows retain long-term
-- capacity and abuse trends without keeping the much wider observation JSON.
CREATE TABLE cloud_vm_network_observation_rollups (
instance_id BIGINT UNSIGNED NOT NULL,
user_id BIGINT UNSIGNED NOT NULL,
host_id BIGINT UNSIGNED NOT NULL,
interval_seconds SMALLINT UNSIGNED NOT NULL,
bucket_start BIGINT UNSIGNED NOT NULL,
sample_count SMALLINT UNSIGNED NOT NULL,
complete_count SMALLINT UNSIGNED NOT NULL,
quality_mask INT UNSIGNED NOT NULL DEFAULT 0,
rx_bytes_min BIGINT UNSIGNED DEFAULT NULL,
rx_bytes_max BIGINT UNSIGNED DEFAULT NULL,
tx_bytes_min BIGINT UNSIGNED DEFAULT NULL,
tx_bytes_max BIGINT UNSIGNED DEFAULT NULL,
rx_packets_min BIGINT UNSIGNED DEFAULT NULL,
rx_packets_max BIGINT UNSIGNED DEFAULT NULL,
tx_packets_min BIGINT UNSIGNED DEFAULT NULL,
tx_packets_max BIGINT UNSIGNED DEFAULT NULL,
tcp_active_peak BIGINT UNSIGNED NOT NULL DEFAULT 0,
tcp_new_total BIGINT UNSIGNED NOT NULL DEFAULT 0,
tcp_syn_total BIGINT UNSIGNED NOT NULL DEFAULT 0,
tcp_unreplied_total BIGINT UNSIGNED NOT NULL DEFAULT 0,
udp_active_peak BIGINT UNSIGNED NOT NULL DEFAULT 0,
udp_new_total BIGINT UNSIGNED NOT NULL DEFAULT 0,
peer_ip_cardinality_peak BIGINT UNSIGNED NOT NULL DEFAULT 0,
peer_port_cardinality_peak BIGINT UNSIGNED NOT NULL DEFAULT 0,
first_sample_at BIGINT UNSIGNED NOT NULL,
last_sample_at BIGINT UNSIGNED NOT NULL,
created_at BIGINT UNSIGNED NOT NULL,
updated_at BIGINT UNSIGNED NOT NULL,
PRIMARY KEY (instance_id, interval_seconds, bucket_start),
KEY cloud_network_rollup_owner (
user_id, interval_seconds, bucket_start, instance_id
),
KEY cloud_network_rollup_host (
host_id, interval_seconds, bucket_start, instance_id
),
KEY cloud_network_rollup_retention (
interval_seconds, bucket_start, instance_id
),
CONSTRAINT chk_cloud_network_rollup_bucket CHECK (
interval_seconds IN (300, 3600)
AND bucket_start % interval_seconds = 0
),
CONSTRAINT chk_cloud_network_rollup_counts CHECK (
sample_count > 0 AND complete_count <= sample_count
AND first_sample_at >= bucket_start
AND last_sample_at >= first_sample_at
AND last_sample_at < bucket_start + interval_seconds
),
CONSTRAINT chk_cloud_network_rollup_counters CHECK (
(rx_bytes_min IS NULL AND rx_bytes_max IS NULL
OR rx_bytes_min IS NOT NULL AND rx_bytes_max >= rx_bytes_min)
AND (tx_bytes_min IS NULL AND tx_bytes_max IS NULL
OR tx_bytes_min IS NOT NULL AND tx_bytes_max >= tx_bytes_min)
AND (rx_packets_min IS NULL AND rx_packets_max IS NULL
OR rx_packets_min IS NOT NULL AND rx_packets_max >= rx_packets_min)
AND (tx_packets_min IS NULL AND tx_packets_max IS NULL
OR tx_packets_min IS NOT NULL AND tx_packets_max >= tx_packets_min)
)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
-- Durable cross-database projection intent. Ingestion can commit a sample,
-- its pending projection and the advanced host cursor in one log-database
-- transaction. The main database records a separate idempotent receipt.
CREATE TABLE cloud_vm_network_projection_outbox (
sample_key CHAR(64) CHARACTER SET ascii COLLATE ascii_bin NOT NULL,
instance_id BIGINT UNSIGNED NOT NULL,
bucket_start BIGINT UNSIGNED NOT NULL,
placement_key CHAR(64) CHARACTER SET ascii COLLATE ascii_bin NOT NULL,
rule_version INT UNSIGNED NOT NULL,
policy_revision BIGINT UNSIGNED NOT NULL,
policy_hash CHAR(64) CHARACTER SET ascii COLLATE ascii_bin NOT NULL,
state VARCHAR(16) CHARACTER SET ascii COLLATE ascii_bin NOT NULL,
attempts INT UNSIGNED NOT NULL DEFAULT 0,
available_at BIGINT UNSIGNED NOT NULL,
lease_token CHAR(32) CHARACTER SET ascii COLLATE ascii_bin DEFAULT NULL,
lease_until BIGINT UNSIGNED NOT NULL DEFAULT 0,
last_error_code VARCHAR(64) CHARACTER SET ascii COLLATE ascii_bin NOT NULL DEFAULT '',
projected_at BIGINT UNSIGNED DEFAULT NULL,
terminal_at BIGINT UNSIGNED DEFAULT NULL,
discard_reason_code VARCHAR(64) CHARACTER SET ascii COLLATE ascii_bin NOT NULL DEFAULT '',
created_at BIGINT UNSIGNED NOT NULL,
updated_at BIGINT UNSIGNED NOT NULL,
PRIMARY KEY (sample_key),
KEY cloud_network_projection_due (
state, available_at, bucket_start, sample_key
),
KEY cloud_network_projection_instance (
instance_id, state, bucket_start, sample_key
),
KEY cloud_network_projection_lease (state, lease_until, sample_key),
KEY cloud_network_projection_retention (state, terminal_at, sample_key),
CONSTRAINT chk_cloud_network_projection_key CHECK (
sample_key REGEXP '^[a-f0-9]{64}$'
AND placement_key REGEXP '^[a-f0-9]{64}$'
AND instance_id > 0
AND bucket_start % 60 = 0
),
CONSTRAINT chk_cloud_network_projection_policy CHECK (
rule_version > 0
AND policy_revision > 0
AND policy_hash REGEXP '^[a-f0-9]{64}$'
),
CONSTRAINT chk_cloud_network_projection_state CHECK (
state IN ('pending','leased','done','discarded')
),
CONSTRAINT chk_cloud_network_projection_lease CHECK (
(state = 'pending' AND lease_token IS NULL AND lease_until = 0
AND projected_at IS NULL AND terminal_at IS NULL
AND discard_reason_code = '')
OR (state = 'leased'
AND lease_token IS NOT NULL
AND lease_token REGEXP '^[a-f0-9]{32}$'
AND lease_until > 0 AND projected_at IS NULL
AND terminal_at IS NULL AND discard_reason_code = '')
OR (state = 'done' AND lease_token IS NULL AND lease_until = 0
AND projected_at IS NOT NULL AND terminal_at IS NOT NULL
AND terminal_at >= projected_at AND discard_reason_code = '')
OR (state = 'discarded' AND lease_token IS NULL AND lease_until = 0
AND projected_at IS NULL AND terminal_at IS NOT NULL
AND discard_reason_code REGEXP '^[a-z0-9_]{3,64}$')
),
CONSTRAINT fk_cloud_network_projection_sample FOREIGN KEY (sample_key)
REFERENCES cloud_vm_network_observation_samples (sample_key)
ON DELETE RESTRICT ON UPDATE RESTRICT
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
-- One cursor per host. Collector generation changes after an Agent restart;
-- consumers must never compare opaque cursor values across generations.
CREATE TABLE cloud_vm_network_monitor_cursors (
host_id BIGINT UNSIGNED NOT NULL,
collector_generation CHAR(32) CHARACTER SET ascii COLLATE ascii_bin DEFAULT NULL,
cursor_value BIGINT UNSIGNED NOT NULL DEFAULT 0,
oldest_cursor BIGINT UNSIGNED NOT NULL DEFAULT 0,
gap_before TINYINT(1) NOT NULL DEFAULT 0,
poll_revision BIGINT UNSIGNED NOT NULL DEFAULT 0,
next_poll_at BIGINT UNSIGNED NOT NULL DEFAULT 0,
poll_lease_token CHAR(32) CHARACTER SET ascii COLLATE ascii_bin DEFAULT NULL,
poll_lease_until BIGINT UNSIGNED NOT NULL DEFAULT 0,
collection_state VARCHAR(16) CHARACTER SET ascii COLLATE ascii_bin NOT NULL,
consecutive_failures INT UNSIGNED NOT NULL DEFAULT 0,
last_window_start BIGINT UNSIGNED DEFAULT NULL,
last_success_at BIGINT UNSIGNED DEFAULT NULL,
last_error_code VARCHAR(64) CHARACTER SET ascii COLLATE ascii_bin NOT NULL DEFAULT '',
updated_at BIGINT UNSIGNED NOT NULL,
PRIMARY KEY (host_id),
KEY cloud_network_monitor_cursor_health (
collection_state, last_success_at, host_id
),
KEY cloud_network_monitor_cursor_due (
next_poll_at, poll_revision, host_id
),
KEY cloud_network_monitor_cursor_lease (
poll_lease_until, host_id
),
CONSTRAINT chk_cloud_network_monitor_cursor_state CHECK (
collection_state IN (
'unknown','healthy','degraded','stale','unavailable'
)
),
CONSTRAINT chk_cloud_network_monitor_cursor_generation CHECK (
(collector_generation IS NULL AND cursor_value = 0
AND oldest_cursor = 0 AND gap_before = 0
AND collection_state = 'unknown' AND consecutive_failures = 0
AND last_window_start IS NULL AND last_success_at IS NULL)
OR (collector_generation IS NOT NULL
AND collector_generation REGEXP '^[a-f0-9]{32}$'
AND gap_before IN (0, 1)
AND ((oldest_cursor = 0 AND cursor_value = 0 AND gap_before = 0)
OR (oldest_cursor > 0 AND oldest_cursor <= cursor_value)))
),
CONSTRAINT chk_cloud_network_monitor_cursor_lease CHECK (
(poll_lease_token IS NULL AND poll_lease_until = 0)
OR (poll_lease_token IS NOT NULL
AND poll_lease_token REGEXP '^[a-f0-9]{32}$'
AND poll_lease_until > 0)
)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
-- Compact historical baseline owned by the logical instance rather than the
-- current host. metrics_json contains aggregate statistical state only and
-- therefore survives maintenance migration without carrying flow identities.
CREATE TABLE cloud_vm_network_baselines (
instance_id BIGINT UNSIGNED NOT NULL,
user_id BIGINT UNSIGNED NOT NULL,
rule_version INT UNSIGNED NOT NULL,
policy_revision BIGINT UNSIGNED NOT NULL,
policy_hash CHAR(64) CHARACTER SET ascii COLLATE ascii_bin NOT NULL,
sample_count INT UNSIGNED NOT NULL DEFAULT 0,
warmup_state VARCHAR(16) CHARACTER SET ascii COLLATE ascii_bin NOT NULL,
metrics_json JSON NOT NULL,
last_projection_key CHAR(64) CHARACTER SET ascii COLLATE ascii_bin DEFAULT NULL,
last_bucket_start BIGINT UNSIGNED DEFAULT NULL,
projection_lease_token CHAR(32) CHARACTER SET ascii COLLATE ascii_bin DEFAULT NULL,
projection_lease_until BIGINT UNSIGNED NOT NULL DEFAULT 0,
revision BIGINT UNSIGNED NOT NULL DEFAULT 0,
created_at BIGINT UNSIGNED NOT NULL,
updated_at BIGINT UNSIGNED NOT NULL,
PRIMARY KEY (instance_id),
KEY cloud_network_baseline_owner (user_id, updated_at),
KEY cloud_network_baseline_warmup (warmup_state, sample_count),
KEY cloud_network_baseline_projection_lease (
projection_lease_until, instance_id
),
CONSTRAINT chk_cloud_network_baseline_warmup CHECK (
warmup_state IN ('warming','ready')
),
CONSTRAINT chk_cloud_network_baseline_policy CHECK (
rule_version > 0
AND policy_revision > 0
AND policy_hash REGEXP '^[a-f0-9]{64}$'
),
CONSTRAINT chk_cloud_network_baseline_projection CHECK (
(last_projection_key IS NULL AND last_bucket_start IS NULL)
OR (last_projection_key IS NOT NULL
AND last_projection_key REGEXP '^[a-f0-9]{64}$'
AND last_bucket_start IS NOT NULL
AND last_bucket_start % 60 = 0)
),
CONSTRAINT chk_cloud_network_baseline_projection_lease CHECK (
(projection_lease_token IS NULL AND projection_lease_until = 0)
OR (projection_lease_token IS NOT NULL
AND projection_lease_token REGEXP '^[a-f0-9]{32}$'
AND projection_lease_until > 0)
)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;