From f7d752988adbe5d583fb0daa3d7e4c518b2f04fe Mon Sep 17 00:00:00 2001 From: Uku Taht Date: Tue, 3 Dec 2019 17:42:17 +0800 Subject: [PATCH] Add sessions --- config/config.exs | 3 + config/test.exs | 3 + lib/plausible/application.ex | 1 + lib/plausible/ingest/session.ex | 71 +++++++++++++++++++ lib/plausible/session/schema.ex | 28 ++++++++ .../controllers/api/external_controller.ex | 5 +- lib/plausible_web/router.ex | 2 +- .../20191128082207_add_sessions.exs | 23 ++++++ test/plausible/ingest/session_test.exs | 69 ++++++++++++++++++ .../api/external_controller_test.exs | 22 ++++++ 10 files changed, 225 insertions(+), 2 deletions(-) create mode 100644 lib/plausible/ingest/session.ex create mode 100644 lib/plausible/session/schema.ex create mode 100644 priv/repo/migrations/20191128082207_add_sessions.exs create mode 100644 test/plausible/ingest/session_test.exs diff --git a/config/config.exs b/config/config.exs index 88d96281f..ac51e693d 100644 --- a/config/config.exs +++ b/config/config.exs @@ -45,6 +45,9 @@ config :plausible, paddle_api: Plausible.Billing.PaddleApi, google_api: Plausible.Google.Api +config :plausible, + session_timeout: 1000 * 60 * 30 # 30 minutes + # Import environment specific config. This must remain at the bottom # of this file so it overrides the configuration defined above. import_config "#{Mix.env()}.exs" diff --git a/config/test.exs b/config/test.exs index 1bbd3e5ca..d0ca5221d 100644 --- a/config/test.exs +++ b/config/test.exs @@ -26,3 +26,6 @@ config :plausible, Plausible.Mailer, config :plausible, paddle_api: Plausible.PaddleApi.Mock, google_api: Plausible.Google.Api.Mock + +config :plausible, + session_timeout: 0 diff --git a/lib/plausible/application.ex b/lib/plausible/application.ex index 4d32c02dd..14186bdeb 100644 --- a/lib/plausible/application.ex +++ b/lib/plausible/application.ex @@ -11,6 +11,7 @@ defmodule Plausible.Application do opts = [strategy: :one_for_one, name: Plausible.Supervisor] {:ok, _} = Logger.add_backend(Sentry.LoggerBackend) + Application.put_env(:plausible, :server_start, Timex.now()) Supervisor.start_link(children, opts) end diff --git a/lib/plausible/ingest/session.ex b/lib/plausible/ingest/session.ex new file mode 100644 index 000000000..15806442a --- /dev/null +++ b/lib/plausible/ingest/session.ex @@ -0,0 +1,71 @@ +defmodule Plausible.Ingest.Session do + use GenServer + use Plausible.Repo + + @session_timeout Application.get_env(:plausible, :session_timeout) + + def on_event(event) do + user_session = :global.whereis_name(event.user_id) + + if is_pid(user_session) do + GenServer.cast(user_session, {:on_event, event}) + else + GenServer.start_link(__MODULE__, event, name: {:global, event.user_id}) + end + end + + def on_unload(user_id, timestamp) do + user_session = :global.whereis_name(user_id) + + if is_pid(user_session) do + GenServer.cast(user_session, {:on_unload, timestamp}) + end + end + + def init(event) do + timer = Process.send_after(self(), :finalize, @session_timeout) + {:ok, %{first_event: event, timer: timer, is_bounce: true, last_unload: nil}} + end + + def handle_cast({:on_event, _event}, state) do + Process.cancel_timer(state[:timer]) + new_timer = Process.send_after(self(), :finalize, @session_timeout) + {:noreply, %{state | timer: new_timer, is_bounce: false, last_unload: nil}} + end + + def handle_cast({:on_unload, timestamp}, state) do + {:noreply, %{state | last_unload: timestamp}} + end + + def handle_info(:finalize, state) do + event = state[:first_event] + + if !is_potential_leftover?(event) do + length = if state[:last_unload] do + Timex.diff(state[:last_unload], event.timestamp, :seconds) + end + + Plausible.Session.changeset(%Plausible.Session{}, %{ + hostname: event.hostname, + user_id: event.user_id, + new_visitor: event.new_visitor, + is_bounce: state[:is_bounce], + length: length, + referrer: event.referrer, + referrer_source: event.referrer_source, + country_code: event.country_code, + operating_system: event.operating_system, + browser: event.browser + }) |> Repo.insert! + end + + {:stop, :normal, state} + end + + defp is_potential_leftover?(%{new_visitor: true}), do: false + defp is_potential_leftover?(%{timestamp: timestamp}) do + server_start_time = Application.get_env(:plausible, :server_start) + Timex.diff(timestamp, server_start_time, :milliseconds) < @session_timeout + end + +end diff --git a/lib/plausible/session/schema.ex b/lib/plausible/session/schema.ex new file mode 100644 index 000000000..5f76ef2bb --- /dev/null +++ b/lib/plausible/session/schema.ex @@ -0,0 +1,28 @@ +defmodule Plausible.Session do + use Ecto.Schema + import Ecto.Changeset + + schema "sessions" do + field :hostname, :string + field :new_visitor, :boolean + field :user_id, :binary_id + + field :length, :integer + field :is_bounce, :boolean + + field :referrer, :string + field :referrer_source, :string + field :country_code, :string + field :screen_size, :string + field :operating_system, :string + field :browser, :string + + timestamps(inserted_at: :timestamp, updated_at: false) + end + + def changeset(session, attrs) do + session + |> cast(attrs, [:hostname, :referrer, :new_visitor, :user_id, :length, :is_bounce, :operating_system, :browser, :referrer_source, :country_code, :screen_size]) + |> validate_required([:hostname, :new_visitor, :user_id, :is_bounce]) + end +end diff --git a/lib/plausible_web/controllers/api/external_controller.ex b/lib/plausible_web/controllers/api/external_controller.ex index c4cf51ec7..1f46733fc 100644 --- a/lib/plausible_web/controllers/api/external_controller.ex +++ b/lib/plausible_web/controllers/api/external_controller.ex @@ -6,7 +6,10 @@ defmodule PlausibleWeb.Api.ExternalController do params = parse_body(conn) case create_event(conn, params) do - {:ok, _event} -> + {:ok, nil} -> + conn |> send_resp(202, "") + {:ok, event} -> + Plausible.Ingest.Session.on_event(event) conn |> send_resp(202, "") {:error, changeset} -> request = Sentry.Plug.build_request_interface_data(conn, []) diff --git a/lib/plausible_web/router.ex b/lib/plausible_web/router.ex index 93fec002a..60a2157d3 100644 --- a/lib/plausible_web/router.ex +++ b/lib/plausible_web/router.ex @@ -17,7 +17,7 @@ defmodule PlausibleWeb.Router do end pipeline :api do - plug :accepts, ["application/json"] + plug :accepts, ["json"] plug :fetch_session plug PlausibleWeb.AuthPlug end diff --git a/priv/repo/migrations/20191128082207_add_sessions.exs b/priv/repo/migrations/20191128082207_add_sessions.exs new file mode 100644 index 000000000..a8dc68c11 --- /dev/null +++ b/priv/repo/migrations/20191128082207_add_sessions.exs @@ -0,0 +1,23 @@ +defmodule Plausible.Repo.Migrations.AddSessions do + use Ecto.Migration + + def change do + create table(:sessions) do + add :hostname, :text, null: false + add :new_visitor, :boolean, null: false + add :user_id, :binary_id, null: false + + add :is_bounce, :boolean, null: false + add :length, :integer + + add :referrer, :string + add :referrer_source, :string + add :country_code, :string + add :screen_size, :string + add :operating_system, :string + add :browser, :string + + timestamps(inserted_at: :timestamp, updated_at: false) + end + end +end diff --git a/test/plausible/ingest/session_test.exs b/test/plausible/ingest/session_test.exs new file mode 100644 index 000000000..2e7ca79d1 --- /dev/null +++ b/test/plausible/ingest/session_test.exs @@ -0,0 +1,69 @@ +defmodule Plausible.Ingest.SessionTest do + use Plausible.DataCase + alias Plausible.Ingest + + defp capture_session(user_id) do + session_pid = :global.whereis_name(user_id) + Process.monitor(session_pid) + + assert_receive({:DOWN, session_pid, :process, _, :normal}) + + Repo.one(Plausible.Session) + end + + describe "on_event/1" do + test "starts a new session if there is no session for user id" do + pageview = insert(:pageview) + + refute is_pid(:global.whereis_name(pageview.user_id)) + + Ingest.Session.on_event(pageview) + + assert is_pid(:global.whereis_name(pageview.user_id)) + end + + test "copies event data to session" do + pageview = insert(:pageview) + + Ingest.Session.on_event(pageview) + + session = capture_session(pageview.user_id) + + assert session.user_id == pageview.user_id + assert session.new_visitor == pageview.new_visitor + end + + test "inserts bounced session when timeout fires after one pageview" do + pageview = insert(:pageview) + + Ingest.Session.on_event(pageview) + + session = capture_session(pageview.user_id) + assert session.is_bounce + end + + test "session with two events is not a bounce" do + pageview = insert(:pageview) + pageview2 = insert(:pageview, user_id: pageview.user_id) + + Ingest.Session.on_event(pageview) + Ingest.Session.on_event(pageview2) + + session = capture_session(pageview.user_id) + refute session.is_bounce + end + end + + describe "on_unload/1" do + test "uses the unload timestamp to calculate session length" do + pageview = insert(:pageview) + unload_timestamp = Timex.shift(pageview.timestamp, seconds: 30) + + Ingest.Session.on_event(pageview) + Ingest.Session.on_unload(pageview.user_id, unload_timestamp) + + session = capture_session(pageview.user_id) + assert session.length == 30 + end + end +end diff --git a/test/plausible_web/controllers/api/external_controller_test.exs b/test/plausible_web/controllers/api/external_controller_test.exs index f8b10101c..84e1db5e9 100644 --- a/test/plausible_web/controllers/api/external_controller_test.exs +++ b/test/plausible_web/controllers/api/external_controller_test.exs @@ -2,6 +2,13 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do use PlausibleWeb.ConnCase use Plausible.Repo + defp finalize_session(user_id) do + session_pid = :global.whereis_name(user_id) + Process.monitor(session_pid) + + assert_receive({:DOWN, session_pid, _, _, _}) + end + @user_agent "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.110 Safari/537.36" @country_code "EE" @@ -23,6 +30,7 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do |> post("/api/event", Jason.encode!(params)) pageview = Repo.one(Plausible.Event) + finalize_session(pageview.user_id) assert response(conn, 202) == "" assert pageview.hostname == "gigride.live" @@ -44,6 +52,7 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do |> post("/api/event", Jason.encode!(params)) pageview = Repo.one(Plausible.Event) + finalize_session(pageview.user_id) assert pageview.hostname == "example.com" end @@ -79,6 +88,7 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do |> post("/api/event", Jason.encode!(params)) pageview = Repo.one(Plausible.Event) + finalize_session(pageview.user_id) assert response(conn, 202) == "" assert pageview.operating_system == "Mac" @@ -100,6 +110,7 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do |> post("/api/event", Jason.encode!(params)) pageview = Repo.one(Plausible.Event) + finalize_session(pageview.user_id) assert response(conn, 202) == "" assert pageview.referrer_source == "Facebook" @@ -120,6 +131,7 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do |> post("/api/event", Jason.encode!(params)) pageview = Repo.one(Plausible.Event) + finalize_session(pageview.user_id) assert response(conn, 202) == "" assert pageview.referrer_source == nil @@ -140,6 +152,7 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do |> post("/api/event", Jason.encode!(params)) pageview = Repo.one(Plausible.Event) + finalize_session(pageview.user_id) assert response(conn, 202) == "" assert pageview.referrer_source == nil @@ -160,6 +173,7 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do |> post("/api/event", Jason.encode!(params)) pageview = Repo.one(Plausible.Event) + finalize_session(pageview.user_id) assert response(conn, 202) == "" assert pageview.referrer_source == "blog.gigride.live" @@ -179,6 +193,7 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do |> post("/api/event", Jason.encode!(params)) pageview = Repo.one(Plausible.Event) + finalize_session(pageview.user_id) assert pageview.referrer == "indiehackers.com/page" end @@ -197,6 +212,7 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do |> post("/api/event", Jason.encode!(params)) pageview = Repo.one(Plausible.Event) + finalize_session(pageview.user_id) assert pageview.referrer_source == "traffic-source" end @@ -215,6 +231,7 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do |> post("/api/event", Jason.encode!(params)) pageview = Repo.one(Plausible.Event) + finalize_session(pageview.user_id) assert pageview.referrer_source == "traffic-source" end @@ -234,6 +251,7 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do |> post("/api/event", Jason.encode!(params)) pageview = Repo.one(Plausible.Event) + finalize_session(pageview.user_id) assert response(conn, 202) == "" assert pageview.referrer_source == "indiehackers.com" @@ -254,6 +272,7 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do |> post("/api/event", Jason.encode!(params)) pageview = Repo.one(Plausible.Event) + finalize_session(pageview.user_id) assert response(conn, 202) == "" assert is_nil(pageview.referrer_source) @@ -276,6 +295,7 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do |> post("/api/event", Jason.encode!(params)) pageview = Repo.one(Plausible.Event) + finalize_session(pageview.user_id) assert response(conn, 202) == "" assert pageview.screen_size == "Mobile" @@ -295,6 +315,7 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do |> post("/api/event", Jason.encode!(params)) pageview = Repo.one(Plausible.Event) + finalize_session(pageview.user_id) assert response(conn, 202) == "" assert pageview.screen_size == nil @@ -314,6 +335,7 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do |> post("/api/event", Jason.encode!(params)) event = Repo.one(Plausible.Event) + finalize_session(event.user_id) assert response(conn, 202) == "" assert event.name == "custom event"