Rotate salts on a daily basis (#224)

* Add daily salt rotation for user ids

* Rotate salts on a cron schedule daily

* Keep old salt fo 48 hours

* Double write for plausible.io to test salt rotation

* Rotate salts for every site
pull/225/head
Uku Taht 6 years ago committed by GitHub
parent 0054904ede
commit 4d55c4657a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23

@ -96,7 +96,12 @@ config :plausible,
cron_enabled = String.to_existing_atom(System.get_env("CRON_ENABLED", "false"))
crontab = [
base_cron = [
# Daily at midnight
{"0 0 * * *", Plausible.Workers.RotateSalts},
]
extra_cron = [
# hourly
{"0 * * * *", Plausible.Workers.SendSiteSetupEmails},
#  hourly
@ -111,7 +116,8 @@ crontab = [
{"*/10 * * * *", Plausible.Workers.ProvisionSslCertificates}
]
queues = [
base_queues = [rotate_salts: 1]
extra_queues = [
provision_ssl_certificates: 1,
fetch_tweets: 1,
check_stats_emails: 1,
@ -123,8 +129,8 @@ queues = [
config :plausible, Oban,
repo: Plausible.Repo,
queues: if(cron_enabled, do: queues, else: []),
crontab: if(cron_enabled, do: crontab, else: false)
queues: if(cron_enabled, do: base_queues ++ extra_queues, else: base_queues),
crontab: if(cron_enabled, do: base_cron ++ extra_cron, else: base_cron)
config :plausible, :google,
client_id: System.get_env("GOOGLE_CLIENT_ID"),

@ -146,7 +146,12 @@ config :plausible, :custom_domain_server,
password: custom_domain_server_password,
ip: custom_domain_server_ip
crontab = [
base_cron = [
# Daily at midnight
{"0 0 * * *", Plausible.Workers.RotateSalts},
]
extra_cron = [
# hourly
{"0 * * * *", Plausible.Workers.SendSiteSetupEmails},
#  hourly
@ -161,7 +166,8 @@ crontab = [
{"*/10 * * * *", Plausible.Workers.ProvisionSslCertificates}
]
queues = [
base_queues = [rotate_salts: 1]
extra_queues = [
provision_ssl_certificates: 1,
fetch_tweets: 1,
check_stats_emails: 1,
@ -173,8 +179,8 @@ queues = [
config :plausible, Oban,
repo: Plausible.Repo,
queues: if(cron_enabled, do: queues, else: []),
crontab: if(cron_enabled, do: crontab, else: false)
queues: if(cron_enabled, do: base_queues ++ extra_queues, else: base_queues),
crontab: if(cron_enabled, do: base_cron ++ extra_cron, else: base_cron)
config :ref_inspector,
init: {Plausible.Release, :configure_ref_inspector}

@ -15,6 +15,7 @@ defmodule Plausible.Application do
Keyword.merge([scheme: :http, port: 8123, name: :clickhouse], clickhouse_config)
),
Plausible.Session.Store,
Plausible.Session.Salts,
{Oban, Application.get_env(:plausible, Oban)}
]

@ -0,0 +1,46 @@
defmodule Plausible.Session.Salts do
use Agent
use Plausible.Repo
def start_link(_opts) do
Agent.start_link(fn ->
clean_old_salts()
salts = Repo.all(from s in "salts", select: s.salt, order_by: [desc: s.inserted_at], limit: 2)
case salts do
[current, prev] ->
%{previous: prev, current: current}
[current] ->
%{previous: nil, current: current}
[] ->
new = generate_and_persist_new_salt()
%{previous: nil, current: new}
end
end, name: __MODULE__)
end
def fetch() do
Agent.get(__MODULE__, & &1)
end
def rotate() do
Agent.update(__MODULE__, fn %{current: current} ->
clean_old_salts()
%{
current: generate_and_persist_new_salt(),
previous: current
}
end)
end
defp generate_and_persist_new_salt() do
salt = :crypto.strong_rand_bytes(16)
Repo.insert_all("salts", [%{salt: salt, inserted_at: Timex.now()}])
salt
end
defp clean_old_salts() do
Repo.delete_all(from s in "salts", where: s.inserted_at < fragment("now() - '48 hours'::interval"))
end
end

@ -6,7 +6,6 @@ defmodule Plausible.Session.Store do
require Logger
@session_length_seconds Application.get_env(:plausible, :session_length_minutes) * 60
# Remember session for longer in case of upstream latency
@forget_session_after @session_length_seconds * 2
@garbage_collect_interval_milliseconds 60 * 1000
@ -43,12 +42,12 @@ defmodule Plausible.Session.Store do
{:ok, %{timer: timer, sessions: sessions}}
end
def on_event(event) do
GenServer.call(__MODULE__, {:on_event, event})
def on_event(event, prev_user_id) do
GenServer.call(__MODULE__, {:on_event, event, prev_user_id})
end
def handle_call({:on_event, event}, _from, %{sessions: sessions} = state) do
found_session = sessions[event.user_id]
def handle_call({:on_event, event, prev_user_id}, _from, %{sessions: sessions} = state) do
found_session = sessions[event.user_id] || (prev_user_id && sessions[prev_user_id])
active = is_active?(found_session, event)
updated_sessions =
@ -80,7 +79,8 @@ defmodule Plausible.Session.Store do
defp update_session(session, event) do
%{
session
| timestamp: event.timestamp,
| user_id: event.user_id,
timestamp: event.timestamp,
exit_page: event.pathname,
is_bounce: false,
duration: Timex.diff(event.timestamp, session.start, :second),

@ -42,6 +42,7 @@ defmodule PlausibleWeb.Api.ExternalController do
ref = parse_referrer(uri, params["referrer"])
country_code = visitor_country(conn)
salts = Plausible.Session.Salts.fetch()
event_attrs = %{
timestamp: NaiveDateTime.utc_now(),
@ -49,7 +50,7 @@ defmodule PlausibleWeb.Api.ExternalController do
hostname: strip_www(uri && uri.host),
domain: strip_www(params["domain"]) || strip_www(uri && uri.host),
pathname: uri && (uri.path || "/"),
user_id: generate_user_id(conn, params),
user_id: generate_user_id(conn, params, salts[:current]),
country_code: country_code,
operating_system: ua && os_name(ua),
browser: ua && browser_name(ua),
@ -61,8 +62,9 @@ defmodule PlausibleWeb.Api.ExternalController do
changeset = Plausible.ClickhouseEvent.changeset(%Plausible.ClickhouseEvent{}, event_attrs)
if changeset.valid? do
previous_user_id = salts[:previous] && generate_user_id(conn, params, salts[:previous])
event = struct(Plausible.ClickhouseEvent, event_attrs)
session_id = Plausible.Session.Store.on_event(event)
session_id = Plausible.Session.Store.on_event(event, previous_user_id)
Map.put(event, :session_id, session_id)
|> Plausible.Event.WriteBuffer.insert()
@ -105,16 +107,12 @@ defmodule PlausibleWeb.Api.ExternalController do
end
end
defp generate_user_id(conn, params) do
hash_key =
Keyword.fetch!(Application.get_env(:plausible, PlausibleWeb.Endpoint), :secret_key_base)
|> binary_part(0, 16)
defp generate_user_id(conn, params, salt) do
user_agent = List.first(Plug.Conn.get_req_header(conn, "user-agent")) || ""
ip_address = get_ip(conn)
domain = strip_www(params["domain"]) || ""
SipHash.hash!(hash_key, user_agent <> ip_address <> domain)
SipHash.hash!(salt, user_agent <> ip_address <> domain)
end
defp calculate_screen_size(nil), do: nil

@ -0,0 +1,9 @@
defmodule Plausible.Workers.RotateSalts do
use Plausible.Repo
use Oban.Worker, queue: :rotate_salts
@impl Oban.Worker
def perform(_args, _job) do
Plausible.Session.Salts.rotate()
end
end

@ -0,0 +1,11 @@
defmodule Plausible.Repo.Migrations.CreateSaltsTable do
use Ecto.Migration
def change do
create table(:salts) do
add :salt, :bytea, null: false
timestamps(updated_at: false)
end
end
end
Loading…
Cancel
Save