diff --git a/lib/plausible/clickhouse_event_v2.ex b/lib/plausible/clickhouse_event_v2.ex index 7252f3410..3071f2539 100644 --- a/lib/plausible/clickhouse_event_v2.ex +++ b/lib/plausible/clickhouse_event_v2.ex @@ -15,25 +15,6 @@ defmodule Plausible.ClickhouseEventV2 do field :session_id, Ch, type: "UInt64" field :timestamp, :naive_datetime - field :referrer, :string - field :referrer_source, :string - field :utm_medium, :string - field :utm_source, :string - field :utm_campaign, :string - field :utm_content, :string - field :utm_term, :string - - field :country_code, Ch, type: "FixedString(2)" - field :subdivision1_code, Ch, type: "LowCardinality(String)" - field :subdivision2_code, Ch, type: "LowCardinality(String)" - field :city_geoname_id, Ch, type: "UInt32" - - field :screen_size, Ch, type: "LowCardinality(String)" - field :operating_system, Ch, type: "LowCardinality(String)" - field :operating_system_version, Ch, type: "LowCardinality(String)" - field :browser, Ch, type: "LowCardinality(String)" - field :browser_version, Ch, type: "LowCardinality(String)" - field :"meta.key", {:array, :string} field :"meta.value", {:array, :string} @@ -42,7 +23,26 @@ defmodule Plausible.ClickhouseEventV2 do field :revenue_reporting_amount, Ch, type: "Nullable(Decimal64(3))" field :revenue_reporting_currency, Ch, type: "FixedString(3)" - field :transferred_from, :string + # Fields which are in the schema but not managed by us anymore. + # field :referrer, :string + # field :referrer_source, :string + # field :utm_medium, :string + # field :utm_source, :string + # field :utm_campaign, :string + # field :utm_content, :string + # field :utm_term, :string + + # field :country_code, Ch, type: "FixedString(2)" + # field :subdivision1_code, Ch, type: "LowCardinality(String)" + # field :subdivision2_code, Ch, type: "LowCardinality(String)" + # field :city_geoname_id, Ch, type: "UInt32" + + # field :screen_size, Ch, type: "LowCardinality(String)" + # field :operating_system, Ch, type: "LowCardinality(String)" + # field :operating_system_version, Ch, type: "LowCardinality(String)" + # field :browser, Ch, type: "LowCardinality(String)" + # field :browser_version, Ch, type: "LowCardinality(String)" + # field :transferred_from, :string end def new(attrs) do @@ -56,22 +56,6 @@ defmodule Plausible.ClickhouseEventV2 do :pathname, :user_id, :timestamp, - :operating_system, - :operating_system_version, - :browser, - :browser_version, - :referrer, - :referrer_source, - :utm_medium, - :utm_source, - :utm_campaign, - :utm_content, - :utm_term, - :country_code, - :subdivision1_code, - :subdivision2_code, - :city_geoname_id, - :screen_size, :"meta.key", :"meta.value", :revenue_source_amount, diff --git a/lib/plausible/ingestion/event.ex b/lib/plausible/ingestion/event.ex index 78d988ba2..59c0765c5 100644 --- a/lib/plausible/ingestion/event.ex +++ b/lib/plausible/ingestion/event.ex @@ -13,6 +13,7 @@ defmodule Plausible.Ingestion.Event do defstruct domain: nil, site: nil, clickhouse_event_attrs: %{}, + clickhouse_session_attrs: %{}, clickhouse_event: nil, dropped?: false, drop_reason: nil, @@ -33,6 +34,7 @@ defmodule Plausible.Ingestion.Event do domain: String.t() | nil, site: %Plausible.Site{} | nil, clickhouse_event_attrs: map(), + clickhouse_session_attrs: map(), clickhouse_event: %ClickhouseEventV2{} | nil, dropped?: boolean(), drop_reason: drop_reason(), @@ -141,10 +143,14 @@ defmodule Plausible.Ingestion.Event do struct!(event, fields) end - defp update_attrs(%__MODULE__{} = event, %{} = attrs) do + defp update_event_attrs(%__MODULE__{} = event, %{} = attrs) do struct!(event, clickhouse_event_attrs: Map.merge(event.clickhouse_event_attrs, attrs)) end + defp update_session_attrs(%__MODULE__{} = event, %{} = attrs) do + struct!(event, clickhouse_session_attrs: Map.merge(event.clickhouse_session_attrs, attrs)) + end + defp drop_datacenter_ip(%__MODULE__{} = event) do case event.request.ip_classification do "dc_ip" -> @@ -177,7 +183,7 @@ defmodule Plausible.Ingestion.Event do drop(event, :bot) %UAInspector.Result{} = user_agent -> - update_attrs(event, %{ + update_session_attrs(event, %{ operating_system: os_name(user_agent), operating_system_version: os_version(user_agent), browser: browser_name(user_agent), @@ -191,7 +197,7 @@ defmodule Plausible.Ingestion.Event do end defp put_basic_info(%__MODULE__{} = event) do - update_attrs(event, %{ + update_event_attrs(event, %{ domain: event.domain, site_id: event.site.id, timestamp: event.request.timestamp, @@ -204,7 +210,7 @@ defmodule Plausible.Ingestion.Event do defp put_referrer(%__MODULE__{} = event) do ref = parse_referrer(event.request.uri, event.request.referrer) - update_attrs(event, %{ + update_session_attrs(event, %{ referrer_source: get_referrer_source(event.request, ref), referrer: clean_referrer(ref) }) @@ -213,7 +219,7 @@ defmodule Plausible.Ingestion.Event do defp put_utm_tags(%__MODULE__{} = event) do query_params = event.request.query_params - update_attrs(event, %{ + update_session_attrs(event, %{ utm_medium: query_params["utm_medium"], utm_source: query_params["utm_source"], utm_campaign: query_params["utm_campaign"], @@ -225,17 +231,16 @@ defmodule Plausible.Ingestion.Event do defp put_geolocation(%__MODULE__{} = event) do case event.request.ip_classification do "anonymous_vpn_ip" -> - result = %{country_code: "A1"} - update_attrs(event, result) + update_session_attrs(event, %{country_code: "A1"}) _any -> result = Plausible.Ingestion.Geolocation.lookup(event.request.remote_ip) || %{} - update_attrs(event, result) + update_session_attrs(event, result) end end defp drop_shield_rule_country( - %__MODULE__{domain: domain, clickhouse_event_attrs: %{country_code: cc}} = event + %__MODULE__{domain: domain, clickhouse_session_attrs: %{country_code: cc}} = event ) when is_binary(domain) and is_binary(cc) do case Plausible.Shield.CountryRuleCache.get({domain, String.upcase(cc)}) do @@ -253,7 +258,7 @@ defmodule Plausible.Ingestion.Event do # defensive: ensuring the keys/values are always in the same order {keys, values} = Enum.unzip(props) - update_attrs(event, %{ + update_event_attrs(event, %{ "meta.key": keys, "meta.value": values }) @@ -264,7 +269,7 @@ defmodule Plausible.Ingestion.Event do defp put_revenue(event) do on_full_build do attrs = Plausible.Ingestion.Event.Revenue.get_revenue_attrs(event) - update_attrs(event, attrs) + update_event_attrs(event, attrs) else event end @@ -275,7 +280,7 @@ defmodule Plausible.Ingestion.Event do end defp put_user_id(%__MODULE__{} = event) do - update_attrs(event, %{ + update_event_attrs(event, %{ user_id: generate_user_id( event.request, @@ -310,7 +315,12 @@ defmodule Plausible.Ingestion.Event do event.salts.previous ) - session_id = Plausible.Session.CacheStore.on_event(event.clickhouse_event, previous_user_id) + session_id = + Plausible.Session.CacheStore.on_event( + event.clickhouse_event, + event.clickhouse_session_attrs, + previous_user_id + ) clickhouse_event = Map.put(event.clickhouse_event, :session_id, session_id) %{event | clickhouse_event: clickhouse_event} diff --git a/lib/plausible/ingestion/write_buffer.ex b/lib/plausible/ingestion/write_buffer.ex index e62580cd5..1e33fe958 100644 --- a/lib/plausible/ingestion/write_buffer.ex +++ b/lib/plausible/ingestion/write_buffer.ex @@ -130,7 +130,8 @@ defmodule Plausible.Ingestion.WriteBuffer do |> Ch.RowBinary.encode_names_and_types(types) |> IO.iodata_to_binary() - insert_sql = "INSERT INTO #{schema.__schema__(:source)} FORMAT RowBinaryWithNamesAndTypes" + insert_sql = + "INSERT INTO #{schema.__schema__(:source)} (#{Enum.join(fields, ", ")}) FORMAT RowBinaryWithNamesAndTypes" %{ fields: fields, diff --git a/lib/plausible/session/cache_store.ex b/lib/plausible/session/cache_store.ex index 9329db372..565698b8c 100644 --- a/lib/plausible/session/cache_store.ex +++ b/lib/plausible/session/cache_store.ex @@ -2,7 +2,7 @@ defmodule Plausible.Session.CacheStore do require Logger alias Plausible.Session.WriteBuffer - def on_event(event, prev_user_id, buffer \\ WriteBuffer) do + def on_event(event, session_attributes, prev_user_id, buffer \\ WriteBuffer) do found_session = find_session(event, event.user_id) || find_session(event, prev_user_id) session = @@ -11,7 +11,7 @@ defmodule Plausible.Session.CacheStore do buffer.insert([%{found_session | sign: -1}, %{updated_session | sign: 1}]) persist_session(updated_session) else - new_session = new_session_from_event(event) + new_session = new_session_from_event(event, session_attributes) buffer.insert([new_session]) persist_session(new_session) end @@ -59,7 +59,7 @@ defmodule Plausible.Session.CacheStore do } end - defp new_session_from_event(event) do + defp new_session_from_event(event, session_attributes) do %Plausible.ClickhouseSessionV2{ sign: 1, session_id: Plausible.ClickhouseSessionV2.random_uint64(), @@ -72,22 +72,22 @@ defmodule Plausible.Session.CacheStore do duration: 0, pageviews: if(event.name == "pageview", do: 1, else: 0), events: 1, - referrer: event.referrer, - referrer_source: event.referrer_source, - utm_medium: event.utm_medium, - utm_source: event.utm_source, - utm_campaign: event.utm_campaign, - utm_content: event.utm_content, - utm_term: event.utm_term, - country_code: event.country_code, - subdivision1_code: event.subdivision1_code, - subdivision2_code: event.subdivision2_code, - city_geoname_id: event.city_geoname_id, - screen_size: event.screen_size, - operating_system: event.operating_system, - operating_system_version: event.operating_system_version, - browser: event.browser, - browser_version: event.browser_version, + referrer: Map.get(session_attributes, :referrer), + referrer_source: Map.get(session_attributes, :referrer_source), + utm_medium: Map.get(session_attributes, :utm_medium), + utm_source: Map.get(session_attributes, :utm_source), + utm_campaign: Map.get(session_attributes, :utm_campaign), + utm_content: Map.get(session_attributes, :utm_content), + utm_term: Map.get(session_attributes, :utm_term), + country_code: Map.get(session_attributes, :country_code), + subdivision1_code: Map.get(session_attributes, :subdivision1_code), + subdivision2_code: Map.get(session_attributes, :subdivision2_code), + city_geoname_id: Map.get(session_attributes, :city_geoname_id), + screen_size: Map.get(session_attributes, :screen_size), + operating_system: Map.get(session_attributes, :operating_system), + operating_system_version: Map.get(session_attributes, :operating_system_version), + browser: Map.get(session_attributes, :browser), + browser_version: Map.get(session_attributes, :browser_version), timestamp: event.timestamp, start: event.timestamp, "entry_meta.key": Map.get(event, :"meta.key"), diff --git a/test/plausible/session/cache_store_test.exs b/test/plausible/session/cache_store_test.exs index a0f4f176d..a48781420 100644 --- a/test/plausible/session/cache_store_test.exs +++ b/test/plausible/session/cache_store_test.exs @@ -17,24 +17,27 @@ defmodule Plausible.Session.CacheStoreTest do event = build(:event, name: "pageview", - referrer: "ref", - referrer_source: "refsource", - utm_medium: "medium", - utm_source: "source", - utm_campaign: "campaign", - utm_content: "content", - utm_term: "term", - browser: "browser", - browser_version: "55", - country_code: "EE", - screen_size: "Desktop", - operating_system: "Mac", - operating_system_version: "11", "meta.key": ["logged_in", "darkmode"], "meta.value": ["true", "false"] ) - CacheStore.on_event(event, nil, buffer) + session_params = %{ + referrer: "ref", + referrer_source: "refsource", + utm_medium: "medium", + utm_source: "source", + utm_campaign: "campaign", + utm_content: "content", + utm_term: "term", + browser: "browser", + browser_version: "55", + country_code: "EE", + screen_size: "Desktop", + operating_system: "Mac", + operating_system_version: "11" + } + + CacheStore.on_event(event, session_params, nil, buffer) assert_receive({WriteBuffer, :insert, [sessions]}) assert [session] = sessions @@ -49,19 +52,19 @@ defmodule Plausible.Session.CacheStoreTest do assert session.duration == 0 assert session.pageviews == 1 assert session.events == 1 - assert session.referrer == event.referrer - assert session.referrer_source == event.referrer_source - assert session.utm_medium == event.utm_medium - assert session.utm_source == event.utm_source - assert session.utm_campaign == event.utm_campaign - assert session.utm_content == event.utm_content - assert session.utm_term == event.utm_term - assert session.country_code == event.country_code - assert session.screen_size == event.screen_size - assert session.operating_system == event.operating_system - assert session.operating_system_version == event.operating_system_version - assert session.browser == event.browser - assert session.browser_version == event.browser_version + assert session.referrer == Map.get(session_params, :referrer) + assert session.referrer_source == Map.get(session_params, :referrer_source) + assert session.utm_medium == Map.get(session_params, :utm_medium) + assert session.utm_source == Map.get(session_params, :utm_source) + assert session.utm_campaign == Map.get(session_params, :utm_campaign) + assert session.utm_content == Map.get(session_params, :utm_content) + assert session.utm_term == Map.get(session_params, :utm_term) + assert session.country_code == Map.get(session_params, :country_code) + assert session.screen_size == Map.get(session_params, :screen_size) + assert session.operating_system == Map.get(session_params, :operating_system) + assert session.operating_system_version == Map.get(session_params, :operating_system_version) + assert session.browser == Map.get(session_params, :browser) + assert session.browser_version == Map.get(session_params, :browser_version) assert session.timestamp == event.timestamp assert session.start === event.timestamp # assert Map.get(session, :"entry.meta.key") == ["logged_in", "darkmode"] @@ -77,8 +80,8 @@ defmodule Plausible.Session.CacheStoreTest do | timestamp: timestamp } - CacheStore.on_event(event1, nil, buffer) - CacheStore.on_event(event2, nil, buffer) + CacheStore.on_event(event1, %{}, nil, buffer) + CacheStore.on_event(event2, %{}, nil, buffer) assert_receive({WriteBuffer, :insert, [[_negative_record, session]]}) assert session.is_bounce == false assert session.duration == 10 @@ -92,8 +95,8 @@ defmodule Plausible.Session.CacheStoreTest do event2 = %{event1 | timestamp: timestamp} - CacheStore.on_event(event1, nil, buffer) - CacheStore.on_event(event2, nil, buffer) + CacheStore.on_event(event1, %{}, nil, buffer) + CacheStore.on_event(event2, %{}, nil, buffer) assert_receive({WriteBuffer, :insert, [[_negative_record, session]]}) assert session.duration == 10 @@ -108,7 +111,7 @@ defmodule Plausible.Session.CacheStoreTest do end defp flush(events) do - for e <- events, do: CacheStore.on_event(e, nil) + for e <- events, do: CacheStore.on_event(e, %{}, nil) Plausible.Session.WriteBuffer.flush() end diff --git a/test/plausible/stats/clickhouse_test.exs b/test/plausible/stats/clickhouse_test.exs index 3b553a4c0..dfd847c57 100644 --- a/test/plausible/stats/clickhouse_test.exs +++ b/test/plausible/stats/clickhouse_test.exs @@ -228,22 +228,22 @@ defmodule Plausible.Stats.ClickhouseTest do populate_stats(site, [ build(:pageview, pathname: "/", - referrer_source: "Twitter" + session_referrer_source: "Twitter" ), build(:pageview, pathname: "/plausible.io" ), build(:pageview, pathname: "/plausible.io", - referrer_source: "Google" + session_referrer_source: "Google" ), build(:pageview, pathname: "/plausible.io", - referrer_source: "Google" + session_referrer_source: "Google" ), build(:pageview, pathname: "/plausible.io", - referrer_source: "Bing" + session_referrer_source: "Bing" ) ]) diff --git a/test/plausible_web/controllers/api/external_controller_test.exs b/test/plausible_web/controllers/api/external_controller_test.exs index 87cdbbdd5..3583e3272 100644 --- a/test/plausible_web/controllers/api/external_controller_test.exs +++ b/test/plausible_web/controllers/api/external_controller_test.exs @@ -223,13 +223,13 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do |> put_req_header("user-agent", @user_agent) |> post("/api/event", params) - pageview = get_event(site) + session = get_created_session(site) assert response(conn, 202) == "ok" - assert pageview.operating_system == "Mac" - assert pageview.operating_system_version == "10.13" - assert pageview.browser == "Chrome" - assert pageview.browser_version == "70.0" + assert session.operating_system == "Mac" + assert session.operating_system_version == "10.13" + assert session.browser == "Chrome" + assert session.browser_version == "70.0" end test "parses referrer", %{conn: conn, site: site} do @@ -245,10 +245,10 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do |> put_req_header("user-agent", @user_agent) |> post("/api/event", params) - pageview = get_event(site) + session = get_created_session(site) assert response(conn, 202) == "ok" - assert pageview.referrer_source == "Facebook" + assert session.referrer_source == "Facebook" end test "strips trailing slash from referrer", %{conn: conn, site: site} do @@ -264,11 +264,11 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do |> put_req_header("user-agent", @user_agent) |> post("/api/event", params) - pageview = get_event(site) + session = get_created_session(site) assert response(conn, 202) == "ok" - assert pageview.referrer == "facebook.com/page" - assert pageview.referrer_source == "Facebook" + assert session.referrer == "facebook.com/page" + assert session.referrer_source == "Facebook" end test "ignores event when referrer is a spammer", %{conn: conn, site: site} do @@ -321,10 +321,10 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do |> put_req_header("user-agent", @user_agent) |> post("/api/event", params) - pageview = get_event(site) + session = get_created_session(site) assert response(conn, 202) == "ok" - assert pageview.referrer_source == "" + assert session.referrer_source == "" end test "ignores localhost referrer", %{conn: conn, site: site} do @@ -340,10 +340,10 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do |> put_req_header("user-agent", @user_agent) |> post("/api/event", params) - pageview = get_event(site) + session = get_created_session(site) assert response(conn, 202) == "ok" - assert pageview.referrer_source == "" + assert session.referrer_source == "" end test "parses subdomain referrer", %{conn: conn, site: site} do @@ -359,10 +359,10 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do |> put_req_header("user-agent", @user_agent) |> post("/api/event", params) - pageview = get_event(site) + session = get_created_session(site) assert response(conn, 202) == "ok" - assert pageview.referrer_source == "blog.example.com" + assert session.referrer_source == "blog.example.com" end test "referrer is cleaned", %{conn: conn, site: site} do @@ -376,9 +376,9 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do conn |> post("/api/event", params) - pageview = get_event(site) + session = get_created_session(site) - assert pageview.referrer == "indiehackers.com/page" + assert session.referrer == "indiehackers.com/page" end test "utm_source overrides referrer source", %{conn: conn, site: site} do @@ -392,9 +392,9 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do conn |> post("/api/event", params) - pageview = get_event(site) + session = get_created_session(site) - assert pageview.referrer_source == "betalist" + assert session.referrer_source == "betalist" end test "utm tags are stored", %{conn: conn, site: site} do @@ -408,11 +408,11 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do conn |> post("/api/event", params) - pageview = get_event(site) + session = get_created_session(site) - assert pageview.utm_medium == "ads" - assert pageview.utm_source == "instagram" - assert pageview.utm_campaign == "video_story" + assert session.utm_medium == "ads" + assert session.utm_source == "instagram" + assert session.utm_campaign == "video_story" end test "if it's an :unknown referrer, just the domain is used", %{conn: conn, site: site} do @@ -428,10 +428,10 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do |> put_req_header("user-agent", @user_agent) |> post("/api/event", params) - pageview = get_event(site) + session = get_created_session(site) assert response(conn, 202) == "ok" - assert pageview.referrer_source == "indiehackers.com" + assert session.referrer_source == "indiehackers.com" end test "if the referrer is not http or https, it is ignored", %{conn: conn, site: site} do @@ -447,10 +447,10 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do |> put_req_header("user-agent", @user_agent) |> post("/api/event", params) - pageview = get_event(site) + session = get_created_session(site) assert response(conn, 202) == "ok" - assert pageview.referrer_source == "" + assert session.referrer_source == "" end test "screen size is calculated from user agent", %{conn: conn, site: site} do @@ -465,10 +465,10 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do |> put_req_header("user-agent", @user_agent_mobile) |> post("/api/event", params) - pageview = get_event(site) + session = get_created_session(site) assert response(conn, 202) == "ok" - assert pageview.screen_size == "Mobile" + assert session.screen_size == "Mobile" end test "screen size is nil if user agent is unknown", %{conn: conn, site: site} do @@ -483,10 +483,10 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do |> put_req_header("user-agent", "unknown UA") |> post("/api/event", params) - pageview = get_event(site) + session = get_created_session(site) assert response(conn, 202) == "ok" - assert pageview.screen_size == "" + assert session.screen_size == "" end test "screen size is calculated from user_agent when is tablet", %{conn: conn, site: site} do @@ -501,10 +501,10 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do |> put_req_header("user-agent", @user_agent_tablet) |> post("/api/event", params) - pageview = get_event(site) + session = get_created_session(site) assert response(conn, 202) == "ok" - assert pageview.screen_size == "Tablet" + assert session.screen_size == "Tablet" end test "screen size is calculated from user_agent when is desktop", %{ @@ -522,10 +522,10 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do |> put_req_header("user-agent", @user_agent) |> post("/api/event", params) - pageview = get_event(site) + session = get_created_session(site) assert response(conn, 202) == "ok" - assert pageview.screen_size == "Desktop" + assert session.screen_size == "Desktop" end test "can trigger a custom event", %{conn: conn, site: site} do @@ -792,10 +792,10 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do |> put_req_header("user-agent", @user_agent) |> post("/api/event", params) - event = get_event(site) + session = get_created_session(site) assert response(conn, 202) == "ok" - assert event.referrer == "" + assert session.referrer == "" end # Fake geo is loaded from test/priv/GeoLite2-City-Test.mmdb @@ -810,12 +810,12 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do |> put_req_header("x-forwarded-for", "2.125.160.216") |> post("/api/event", params) - pageview = get_event(site) + session = get_created_session(site) - assert pageview.country_code == "GB" - assert pageview.subdivision1_code == "GB-ENG" - assert pageview.subdivision2_code == "GB-WBK" - assert pageview.city_geoname_id == 2_655_045 + assert session.country_code == "GB" + assert session.subdivision1_code == "GB-ENG" + assert session.subdivision2_code == "GB-WBK" + assert session.city_geoname_id == 2_655_045 end test "ignores unknown country code ZZ", %{conn: conn, site: site} do @@ -829,12 +829,12 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do |> put_req_header("x-forwarded-for", "0.0.0.0") |> post("/api/event", params) - pageview = get_event(site) + session = get_created_session(site) - assert pageview.country_code == <<0, 0>> - assert pageview.subdivision1_code == "" - assert pageview.subdivision2_code == "" - assert pageview.city_geoname_id == 0 + assert session.country_code == <<0, 0>> + assert session.subdivision1_code == "" + assert session.subdivision2_code == "" + assert session.city_geoname_id == 0 end test "ignores disputed territory code XX", %{conn: conn, site: site} do @@ -848,12 +848,12 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do |> put_req_header("x-forwarded-for", "0.0.0.1") |> post("/api/event", params) - pageview = get_event(site) + session = get_created_session(site) - assert pageview.country_code == <<0, 0>> - assert pageview.subdivision1_code == "" - assert pageview.subdivision2_code == "" - assert pageview.city_geoname_id == 0 + assert session.country_code == <<0, 0>> + assert session.subdivision1_code == "" + assert session.subdivision2_code == "" + assert session.city_geoname_id == 0 end test "ignores TOR exit node country code T1", %{conn: conn, site: site} do @@ -867,12 +867,12 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do |> put_req_header("x-forwarded-for", "0.0.0.2") |> post("/api/event", params) - pageview = get_event(site) + session = get_created_session(site) - assert pageview.country_code == <<0, 0>> - assert pageview.subdivision1_code == "" - assert pageview.subdivision2_code == "" - assert pageview.city_geoname_id == 0 + assert session.country_code == <<0, 0>> + assert session.subdivision1_code == "" + assert session.subdivision2_code == "" + assert session.city_geoname_id == 0 end test "scrubs port from x-forwarded-for", %{conn: conn, site: site} do @@ -886,9 +886,9 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do |> put_req_header("x-forwarded-for", "216.160.83.56:123") |> post("/api/event", params) - pageview = get_event(site) + session = get_created_session(site) - assert pageview.country_code == "US" + assert session.country_code == "US" end test "works with ipv6 without port in x-forwarded-for", %{conn: conn, site: site} do @@ -902,9 +902,9 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do |> put_req_header("x-forwarded-for", "2001:218:1:1:1:1:1:1") |> post("/api/event", params) - pageview = get_event(site) + session = get_created_session(site) - assert pageview.country_code == "JP" + assert session.country_code == "JP" end test "works with ipv6 with a port number in x-forwarded-for", %{conn: conn, site: site} do @@ -918,9 +918,9 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do |> put_req_header("x-forwarded-for", "[2001:218:1:1:1:1:1:1]:123") |> post("/api/event", params) - pageview = get_event(site) + session = get_created_session(site) - assert pageview.country_code == "JP" + assert session.country_code == "JP" end test "uses cloudflare's special header for client IP address if present", %{ @@ -938,9 +938,9 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do |> put_req_header("cf-connecting-ip", "216.160.83.56") |> post("/api/event", params) - pageview = get_event(site) + session = get_created_session(site) - assert pageview.country_code == "US" + assert session.country_code == "US" end test "uses BunnyCDN's custom header for client IP address if present", %{ @@ -958,9 +958,9 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do |> put_req_header("b-forwarded-for", "216.160.83.56,9.9.9.9") |> post("/api/event", params) - pageview = get_event(site) + session = get_created_session(site) - assert pageview.country_code == "US" + assert session.country_code == "US" end test "prioritizes x-plausible-ip header over everything else", %{ @@ -981,9 +981,9 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do |> put_req_header("x-plausible-ip", "216.160.83.56") |> post("/api/event", params) - pageview = get_event(site) + session = get_created_session(site) - assert pageview.country_code == "US" + assert session.country_code == "US" end test "Uses the Forwarded header when cf-connecting-ip and x-forwarded-for are missing", %{ @@ -999,9 +999,9 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do |> put_req_header("forwarded", "by=0.0.0.0;for=216.160.83.56;host=somehost.com;proto=https") |> post("/api/event", params) - pageview = get_event(site) + session = get_created_session(site) - assert pageview.country_code == "US" + assert session.country_code == "US" end test "Forwarded header can parse ipv6", %{site: site} do @@ -1018,9 +1018,9 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do ) |> post("/api/event", params) - pageview = get_event(site) + session = get_created_session(site) - assert pageview.country_code == "JP" + assert session.country_code == "JP" end test "URL is decoded", %{conn: conn, site: site} do @@ -1051,10 +1051,11 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do |> post("/api/event", params) pageview = get_event(site) + session = get_created_session(site) assert pageview.pathname == "/opportunity" - assert pageview.referrer_source == "Facebook" - assert pageview.referrer == "facebook.com/page" + assert session.referrer_source == "Facebook" + assert session.referrer == "facebook.com/page" end test "records hash when in hash mode", %{conn: conn, site: site} do @@ -1102,10 +1103,11 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do |> post("/api/event", params) pageview = get_event(site) + session = get_created_session(site) assert pageview.hostname == "test.com" assert pageview.pathname == "/ﺝﻭﺎﺋﺯ-ﻮﻤﺳﺎﺒﻗﺎﺗ" - assert pageview.utm_source == "%balle%" + assert session.utm_source == "%balle%" end test "can use double quotes in query params", %{conn: conn, site: site} do @@ -1121,9 +1123,9 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do conn |> post("/api/event", params) - pageview = get_event(site) + session = get_created_session(site) - assert pageview.utm_source == "Something \"quoted\"" + assert session.utm_source == "Something \"quoted\"" end test "responds 400 when required fields are missing", %{conn: conn, site: site} do @@ -1377,6 +1379,17 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do ) end + defp get_created_session(site) do + Plausible.Session.WriteBuffer.flush() + + ClickhouseRepo.one( + from(s in Plausible.ClickhouseSessionV2, + where: s.site_id == ^site.id and s.sign == 1, + order_by: [desc: s.timestamp] + ) + ) + end + defp get_events(site) do Plausible.Event.WriteBuffer.flush() diff --git a/test/plausible_web/controllers/api/external_stats_controller/aggregate_test.exs b/test/plausible_web/controllers/api/external_stats_controller/aggregate_test.exs index 6c1ccc07b..c3239e785 100644 --- a/test/plausible_web/controllers/api/external_stats_controller/aggregate_test.exs +++ b/test/plausible_web/controllers/api/external_stats_controller/aggregate_test.exs @@ -429,7 +429,10 @@ defmodule PlausibleWeb.Api.ExternalStatsController.AggregateTest do populate_stats(site, [ build(:imported_visitors, date: ~D[2023-01-01]), build(:imported_sources, date: ~D[2023-01-01]), - build(:pageview, referrer_source: "Google", timestamp: ~N[2023-01-02 00:10:00]) + build(:pageview, + session_referrer_source: "Google", + timestamp: ~N[2023-01-02 00:10:00] + ) ]) conn = @@ -482,7 +485,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController.AggregateTest do test "can filter by source", %{conn: conn, site: site} do populate_stats(site, [ build(:pageview, - referrer_source: "Google", + session_referrer_source: "Google", user_id: @user_id, timestamp: ~N[2021-01-01 00:00:00] ), @@ -521,7 +524,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController.AggregateTest do timestamp: ~N[2021-01-01 00:25:00] ), build(:pageview, - referrer_source: "Google", + session_referrer_source: "Google", timestamp: ~N[2021-01-01 00:00:00] ) ]) @@ -546,7 +549,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController.AggregateTest do test "can filter by referrer", %{conn: conn, site: site} do populate_stats(site, [ build(:pageview, - referrer: "https://facebook.com", + session_referrer: "https://facebook.com", user_id: @user_id, timestamp: ~N[2021-01-01 00:00:00] ), @@ -576,9 +579,9 @@ defmodule PlausibleWeb.Api.ExternalStatsController.AggregateTest do test "wildcard referrer filter with special regex characters", %{conn: conn, site: site} do populate_stats(site, [ - build(:pageview, referrer: "https://a.com"), - build(:pageview, referrer: "https://a.com"), - build(:pageview, referrer: "https://ab.com") + build(:pageview, session_referrer: "https://a.com"), + build(:pageview, session_referrer: "https://a.com"), + build(:pageview, session_referrer: "https://ab.com") ]) conn = @@ -594,7 +597,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController.AggregateTest do test "can filter by utm_medium", %{conn: conn, site: site} do populate_stats(site, [ build(:pageview, - utm_medium: "social", + session_utm_medium: "social", user_id: @user_id, timestamp: ~N[2021-01-01 00:00:00] ), @@ -625,7 +628,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController.AggregateTest do test "can filter by utm_source", %{conn: conn, site: site} do populate_stats(site, [ build(:pageview, - utm_source: "Twitter", + session_utm_source: "Twitter", user_id: @user_id, timestamp: ~N[2021-01-01 00:00:00] ), @@ -656,7 +659,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController.AggregateTest do test "can filter by utm_campaign", %{conn: conn, site: site} do populate_stats(site, [ build(:pageview, - utm_campaign: "profile", + session_utm_campaign: "profile", user_id: @user_id, timestamp: ~N[2021-01-01 00:00:00] ), @@ -687,7 +690,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController.AggregateTest do test "can filter by device type", %{conn: conn, site: site} do populate_stats(site, [ build(:pageview, - screen_size: "Desktop", + session_screen_size: "Desktop", user_id: @user_id, timestamp: ~N[2021-01-01 00:00:00] ), @@ -718,7 +721,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController.AggregateTest do test "can filter by browser", %{conn: conn, site: site} do populate_stats(site, [ build(:pageview, - browser: "Chrome", + session_browser: "Chrome", user_id: @user_id, timestamp: ~N[2021-01-01 00:00:00] ), @@ -749,8 +752,8 @@ defmodule PlausibleWeb.Api.ExternalStatsController.AggregateTest do test "can filter by browser version", %{conn: conn, site: site} do populate_stats(site, [ build(:pageview, - browser: "Chrome", - browser_version: "56", + session_browser: "Chrome", + session_browser_version: "56", user_id: @user_id, timestamp: ~N[2021-01-01 00:00:00] ), @@ -781,7 +784,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController.AggregateTest do test "can filter by operating system", %{conn: conn, site: site} do populate_stats(site, [ build(:pageview, - operating_system: "Mac", + session_operating_system: "Mac", user_id: @user_id, timestamp: ~N[2021-01-01 00:00:00] ), @@ -812,7 +815,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController.AggregateTest do test "can filter by operating system version", %{conn: conn, site: site} do populate_stats(site, [ build(:pageview, - operating_system_version: "10.5", + session_operating_system_version: "10.5", user_id: @user_id, timestamp: ~N[2021-01-01 00:00:00] ), @@ -843,7 +846,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController.AggregateTest do test "can filter by country", %{conn: conn, site: site} do populate_stats(site, [ build(:pageview, - country_code: "EE", + session_country_code: "EE", user_id: @user_id, timestamp: ~N[2021-01-01 00:00:00] ), @@ -1135,7 +1138,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController.AggregateTest do populate_stats(site, [ build(:pageview, pathname: "/blogpost", - country_code: "EE", + session_country_code: "EE", user_id: @user_id, timestamp: ~N[2021-01-01 00:00:00] ), @@ -1269,19 +1272,16 @@ defmodule PlausibleWeb.Api.ExternalStatsController.AggregateTest do %{ site_id: site.id, session_id: 1000, - country_code: "EE", name: "pageview" }, %{ site_id: site.id, session_id: 1000, - country_code: "EE", name: "pageview" }, %{ site_id: site.id, session_id: 1000, - country_code: "EE", name: "pageview" } ]) @@ -1349,10 +1349,10 @@ defmodule PlausibleWeb.Api.ExternalStatsController.AggregateTest do } do populate_stats(site, [ build(:event, name: "Signup"), - build(:event, name: "Signup", browser: "Chrome"), - build(:event, name: "Signup", browser: "Firefox", user_id: 123), - build(:event, name: "Signup", browser: "Firefox", user_id: 123), - build(:pageview, browser: "Firefox"), + build(:event, name: "Signup", session_browser: "Chrome"), + build(:event, name: "Signup", session_browser: "Firefox", user_id: 123), + build(:event, name: "Signup", session_browser: "Firefox", user_id: 123), + build(:pageview, session_browser: "Firefox"), build(:pageview) ]) diff --git a/test/plausible_web/controllers/api/external_stats_controller/breakdown_test.exs b/test/plausible_web/controllers/api/external_stats_controller/breakdown_test.exs index 1d3b5265c..b1fac22f8 100644 --- a/test/plausible_web/controllers/api/external_stats_controller/breakdown_test.exs +++ b/test/plausible_web/controllers/api/external_stats_controller/breakdown_test.exs @@ -227,15 +227,15 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do test "breakdown by visit:source", %{conn: conn, site: site} do populate_stats(site, [ build(:pageview, - referrer_source: "Google", + session_referrer_source: "Google", timestamp: ~N[2021-01-01 00:00:00] ), build(:pageview, - referrer_source: "Google", + session_referrer_source: "Google", timestamp: ~N[2021-01-01 00:25:00] ), build(:pageview, - referrer_source: "", + session_referrer_source: "", timestamp: ~N[2021-01-01 00:00:00] ) ]) @@ -258,9 +258,9 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do test "breakdown by visit:country", %{conn: conn, site: site} do populate_stats(site, [ - build(:pageview, country_code: "EE", timestamp: ~N[2021-01-01 00:00:00]), - build(:pageview, country_code: "EE", timestamp: ~N[2021-01-01 00:25:00]), - build(:pageview, country_code: "US", timestamp: ~N[2021-01-01 00:00:00]) + build(:pageview, session_country_code: "EE", timestamp: ~N[2021-01-01 00:00:00]), + build(:pageview, session_country_code: "EE", timestamp: ~N[2021-01-01 00:25:00]), + build(:pageview, session_country_code: "US", timestamp: ~N[2021-01-01 00:00:00]) ]) conn = @@ -282,15 +282,15 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do test "breakdown by visit:referrer", %{conn: conn, site: site} do populate_stats(site, [ build(:pageview, - referrer: "https://ref.com", + session_referrer: "https://ref.com", timestamp: ~N[2021-01-01 00:00:00] ), build(:pageview, - referrer: "https://ref.com", + session_referrer: "https://ref.com", timestamp: ~N[2021-01-01 00:25:00] ), build(:pageview, - referrer: "", + session_referrer: "", timestamp: ~N[2021-01-01 00:00:00] ) ]) @@ -314,15 +314,15 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do test "breakdown by visit:utm_medium", %{conn: conn, site: site} do populate_stats(site, [ build(:pageview, - utm_medium: "Search", + session_utm_medium: "Search", timestamp: ~N[2021-01-01 00:00:00] ), build(:pageview, - utm_medium: "Search", + session_utm_medium: "Search", timestamp: ~N[2021-01-01 00:25:00] ), build(:pageview, - utm_medium: "", + session_utm_medium: "", timestamp: ~N[2021-01-01 00:00:00] ) ]) @@ -345,15 +345,15 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do test "breakdown by visit:utm_source", %{conn: conn, site: site} do populate_stats(site, [ build(:pageview, - utm_source: "Google", + session_utm_source: "Google", timestamp: ~N[2021-01-01 00:00:00] ), build(:pageview, - utm_source: "Google", + session_utm_source: "Google", timestamp: ~N[2021-01-01 00:25:00] ), build(:pageview, - utm_source: "", + session_utm_source: "", timestamp: ~N[2021-01-01 00:00:00] ) ]) @@ -376,15 +376,15 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do test "breakdown by visit:utm_campaign", %{conn: conn, site: site} do populate_stats(site, [ build(:pageview, - utm_campaign: "ads", + session_utm_campaign: "ads", timestamp: ~N[2021-01-01 00:00:00] ), build(:pageview, - utm_campaign: "ads", + session_utm_campaign: "ads", timestamp: ~N[2021-01-01 00:25:00] ), build(:pageview, - utm_campaign: "", + session_utm_campaign: "", timestamp: ~N[2021-01-01 00:00:00] ) ]) @@ -407,15 +407,15 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do test "breakdown by visit:utm_content", %{conn: conn, site: site} do populate_stats(site, [ build(:pageview, - utm_content: "Content1", + session_utm_content: "Content1", timestamp: ~N[2021-01-01 00:00:00] ), build(:pageview, - utm_content: "Content1", + session_utm_content: "Content1", timestamp: ~N[2021-01-01 00:25:00] ), build(:pageview, - utm_content: "", + session_utm_content: "", timestamp: ~N[2021-01-01 00:00:00] ) ]) @@ -438,15 +438,15 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do test "breakdown by visit:utm_term", %{conn: conn, site: site} do populate_stats(site, [ build(:pageview, - utm_term: "Term1", + session_utm_term: "Term1", timestamp: ~N[2021-01-01 00:00:00] ), build(:pageview, - utm_term: "Term1", + session_utm_term: "Term1", timestamp: ~N[2021-01-01 00:25:00] ), build(:pageview, - utm_term: "", + session_utm_term: "", timestamp: ~N[2021-01-01 00:00:00] ) ]) @@ -469,15 +469,15 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do test "breakdown by visit:device", %{conn: conn, site: site} do populate_stats(site, [ build(:pageview, - screen_size: "Desktop", + session_screen_size: "Desktop", timestamp: ~N[2021-01-01 00:00:00] ), build(:pageview, - screen_size: "Desktop", + session_screen_size: "Desktop", timestamp: ~N[2021-01-01 00:25:00] ), build(:pageview, - screen_size: "Mobile", + session_screen_size: "Mobile", timestamp: ~N[2021-01-01 00:00:00] ) ]) @@ -501,15 +501,15 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do test "breakdown by visit:os", %{conn: conn, site: site} do populate_stats(site, [ build(:pageview, - operating_system: "Mac", + session_operating_system: "Mac", timestamp: ~N[2021-01-01 00:00:00] ), build(:pageview, - operating_system: "Mac", + session_operating_system: "Mac", timestamp: ~N[2021-01-01 00:25:00] ), build(:pageview, - operating_system: "Windows", + session_operating_system: "Windows", timestamp: ~N[2021-01-01 00:00:00] ) ]) @@ -532,12 +532,18 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do test "breakdown by visit:os_version", %{conn: conn, site: site} do populate_stats(site, [ - build(:pageview, operating_system: "Mac", operating_system_version: "14"), - build(:pageview, operating_system: "Mac", operating_system_version: "14"), - build(:pageview, operating_system: "Mac", operating_system_version: "14"), - build(:pageview, operating_system_version: "14"), - build(:pageview, operating_system: "Windows", operating_system_version: "11"), - build(:pageview, operating_system: "Windows", operating_system_version: "11") + build(:pageview, session_operating_system: "Mac", session_operating_system_version: "14"), + build(:pageview, session_operating_system: "Mac", session_operating_system_version: "14"), + build(:pageview, session_operating_system: "Mac", session_operating_system_version: "14"), + build(:pageview, session_operating_system_version: "14"), + build(:pageview, + session_operating_system: "Windows", + session_operating_system_version: "11" + ), + build(:pageview, + session_operating_system: "Windows", + session_operating_system_version: "11" + ) ]) conn = @@ -558,9 +564,9 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do test "breakdown by visit:browser", %{conn: conn, site: site} do populate_stats(site, [ - build(:pageview, browser: "Firefox", timestamp: ~N[2021-01-01 00:00:00]), - build(:pageview, browser: "Firefox", timestamp: ~N[2021-01-01 00:25:00]), - build(:pageview, browser: "Safari", timestamp: ~N[2021-01-01 00:00:00]) + build(:pageview, session_browser: "Firefox", timestamp: ~N[2021-01-01 00:00:00]), + build(:pageview, session_browser: "Firefox", timestamp: ~N[2021-01-01 00:25:00]), + build(:pageview, session_browser: "Safari", timestamp: ~N[2021-01-01 00:00:00]) ]) conn = @@ -582,15 +588,15 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do test "breakdown by visit:browser_version", %{conn: conn, site: site} do populate_stats(site, [ build(:pageview, - browser_version: "56", + session_browser_version: "56", timestamp: ~N[2021-01-01 00:00:00] ), build(:pageview, - browser_version: "56", + session_browser_version: "56", timestamp: ~N[2021-01-01 00:25:00] ), build(:pageview, - browser_version: "57", + session_browser_version: "57", timestamp: ~N[2021-01-01 00:00:00] ) ]) @@ -800,30 +806,30 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do build(:event, name: "Signup", pathname: "/pageA", - browser: "Chrome", + session_browser: "Chrome", timestamp: ~N[2021-01-01 00:00:00] ), build(:event, name: "Signup", pathname: "/pageA", - browser: "Chrome", + session_browser: "Chrome", timestamp: ~N[2021-01-01 00:00:00] ), build(:event, name: "Signup", pathname: "/pageA", - browser: "Safari", + session_browser: "Safari", timestamp: ~N[2021-01-01 00:00:00] ), build(:event, name: "Signup", pathname: "/pageB", - browser: "Chrome", + session_browser: "Chrome", timestamp: ~N[2021-01-01 00:00:00] ), build(:pageview, pathname: "/pageA", - browser: "Chrome", + session_browser: "Chrome", timestamp: ~N[2021-01-01 00:25:00] ) ]) @@ -851,7 +857,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do } do populate_stats(site, [ build(:pageview, - referrer_source: "Google", + session_referrer_source: "Google", user_id: @user_id, timestamp: ~N[2021-01-01 00:00:00] ), @@ -861,7 +867,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do timestamp: ~N[2021-01-01 00:00:00] ), build(:pageview, - referrer_source: "Twitter", + session_referrer_source: "Twitter", timestamp: ~N[2021-01-01 00:25:00] ) ]) @@ -899,7 +905,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do ), build(:pageview, pathname: "/pageB", - referrer_source: "Twitter", + session_referrer_source: "Twitter", timestamp: ~N[2021-01-01 00:25:00] ) ]) @@ -1282,21 +1288,21 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do populate_stats(site, [ build(:pageview, pathname: "/ignore", - browser: "Chrome", + session_browser: "Chrome", timestamp: ~N[2021-01-01 00:00:00] ), build(:pageview, pathname: "/plausible.io", - browser: "Chrome", + session_browser: "Chrome", timestamp: ~N[2021-01-01 00:00:00] ), build(:pageview, pathname: "/plausible.io", - browser: "Chrome", + session_browser: "Chrome", timestamp: ~N[2021-01-01 00:25:00] ), build(:pageview, - browser: "Safari", + session_browser: "Safari", pathname: "/plausible.io", timestamp: ~N[2021-01-01 00:00:00] ) @@ -1326,10 +1332,10 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do populate_stats(site, [ build(:pageview, pathname: "/", - referrer_source: "Twitter", - utm_medium: "Twitter", - utm_source: "Twitter", - utm_campaign: "Twitter", + session_referrer_source: "Twitter", + session_utm_medium: "Twitter", + session_utm_source: "Twitter", + session_utm_campaign: "Twitter", user_id: @user_id ), build(:pageview, @@ -1338,17 +1344,17 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do ), build(:pageview, pathname: "/plausible.io", - referrer_source: "Google", - utm_medium: "Google", - utm_source: "Google", - utm_campaign: "Google" + session_referrer_source: "Google", + session_utm_medium: "Google", + session_utm_source: "Google", + session_utm_campaign: "Google" ), build(:pageview, pathname: "/plausible.io", - referrer_source: "Google", - utm_medium: "Google", - utm_source: "Google", - utm_campaign: "Google" + session_referrer_source: "Google", + session_utm_medium: "Google", + session_utm_source: "Google", + session_utm_campaign: "Google" ) ]) @@ -1376,31 +1382,31 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do } do populate_stats(site, [ build(:pageview, - browser: "Chrome", + session_browser: "Chrome", user_id: @user_id, pathname: "/ignore", timestamp: ~N[2021-01-01 00:00:00] ), build(:pageview, - browser: "Chrome", + session_browser: "Chrome", user_id: @user_id, pathname: "/plausible.io", timestamp: ~N[2021-01-01 00:01:00] ), build(:pageview, - browser: "Chrome", + session_browser: "Chrome", user_id: 456, pathname: "/important-page", timestamp: ~N[2021-01-01 00:00:00] ), build(:pageview, - browser: "Chrome", + session_browser: "Chrome", user_id: 456, pathname: "/", timestamp: ~N[2021-01-01 00:01:00] ), build(:pageview, - browser: "Chrome", + session_browser: "Chrome", pathname: "/plausible.io", timestamp: ~N[2021-01-01 00:01:00] ) @@ -1432,11 +1438,11 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do populate_stats(site, [ build(:pageview, - referrer_source: "Bing", + session_referrer_source: "Bing", timestamp: ~N[2021-01-01 00:00:00] ), build(:pageview, - referrer_source: "Google", + session_referrer_source: "Google", user_id: @user_id, timestamp: ~N[2021-01-01 00:00:00] ), @@ -1468,11 +1474,11 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do populate_stats(site, [ build(:pageview, - referrer_source: "Bing", + session_referrer_source: "Bing", timestamp: ~N[2021-01-01 00:00:00] ), build(:pageview, - referrer_source: "Google", + session_referrer_source: "Google", user_id: @user_id, timestamp: ~N[2021-01-01 00:00:00] ), @@ -1529,10 +1535,10 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do test "mixed multi-goal filter for breakdown by visit:country", %{conn: conn, site: site} do populate_stats(site, [ - build(:pageview, country_code: "EE", pathname: "/en/register"), - build(:event, country_code: "EE", name: "Signup", pathname: "/en/register"), - build(:pageview, country_code: "US", pathname: "/123/it/register"), - build(:pageview, country_code: "US", pathname: "/different") + build(:pageview, session_country_code: "EE", pathname: "/en/register"), + build(:event, session_country_code: "EE", name: "Signup", pathname: "/en/register"), + build(:pageview, session_country_code: "US", pathname: "/123/it/register"), + build(:pageview, session_country_code: "US", pathname: "/different") ]) insert(:goal, %{site: site, page_path: "/**register"}) @@ -1630,22 +1636,22 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do populate_stats(site, [ build(:pageview, pathname: "/ignore", - browser: "Firefox", + session_browser: "Firefox", timestamp: ~N[2021-01-01 00:00:00] ), build(:pageview, pathname: "/plausible.io", - browser: "Chrome", + session_browser: "Chrome", timestamp: ~N[2021-01-01 00:00:00] ), build(:pageview, pathname: "/plausible.io", - browser: "Safari", + session_browser: "Safari", timestamp: ~N[2021-01-01 00:00:00] ), build(:pageview, pathname: "/important-page", - browser: "Safari", + session_browser: "Safari", timestamp: ~N[2021-01-01 00:00:00] ) ]) @@ -1745,25 +1751,25 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do test "IN filter for event:props:*", %{conn: conn, site: site} do populate_stats(site, [ build(:pageview, - browser: "Chrome", + session_browser: "Chrome", "meta.key": ["browser"], "meta.value": ["Chrome"], timestamp: ~N[2021-01-01 00:00:00] ), build(:pageview, - browser: "Chrome", + session_browser: "Chrome", "meta.key": ["browser"], "meta.value": ["Chrome"], timestamp: ~N[2021-01-01 00:00:00] ), build(:pageview, - browser: "Safari", + session_browser: "Safari", "meta.key": ["browser"], "meta.value": ["Safari"], timestamp: ~N[2021-01-01 00:00:00] ), build(:pageview, - browser: "Firefox", + session_browser: "Firefox", "meta.key": ["browser"], "meta.value": ["Firefox"], timestamp: ~N[2021-01-01 00:00:00] @@ -1790,25 +1796,25 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do test "Multiple event:props:* filters", %{conn: conn, site: site} do populate_stats(site, [ build(:pageview, - browser: "Chrome", + session_browser: "Chrome", "meta.key": ["browser"], "meta.value": ["Chrome"], timestamp: ~N[2021-01-01 00:00:00] ), build(:pageview, - browser: "Chrome", + session_browser: "Chrome", "meta.key": ["browser", "prop"], "meta.value": ["Chrome", "xyz"], timestamp: ~N[2021-01-01 00:00:00] ), build(:pageview, - browser: "Safari", + session_browser: "Safari", "meta.key": ["browser", "prop"], "meta.value": ["Safari", "target_value"], timestamp: ~N[2021-01-01 00:00:00] ), build(:pageview, - browser: "Firefox", + session_browser: "Firefox", "meta.key": ["browser", "prop"], "meta.value": ["Firefox", "target_value"], timestamp: ~N[2021-01-01 00:00:00] @@ -1834,23 +1840,23 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do test "IN filter for event:props:* including (none) value", %{conn: conn, site: site} do populate_stats(site, [ build(:pageview, - browser: "Chrome", + session_browser: "Chrome", "meta.key": ["browser"], "meta.value": ["Chrome"], timestamp: ~N[2021-01-01 00:00:00] ), build(:pageview, - browser: "Chrome", + session_browser: "Chrome", "meta.key": ["browser"], "meta.value": ["Chrome"], timestamp: ~N[2021-01-01 00:00:00] ), build(:pageview, - browser: "Safari", + session_browser: "Safari", timestamp: ~N[2021-01-01 00:00:00] ), build(:pageview, - browser: "Firefox", + session_browser: "Firefox", "meta.key": ["browser"], "meta.value": ["Firefox"], timestamp: ~N[2021-01-01 00:00:00] @@ -1876,10 +1882,10 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do test "can use a is_not filter", %{conn: conn, site: site} do populate_stats(site, [ - build(:pageview, browser: "Chrome"), - build(:pageview, browser: "Safari"), - build(:pageview, browser: "Safari"), - build(:pageview, browser: "Edge") + build(:pageview, session_browser: "Chrome"), + build(:pageview, session_browser: "Safari"), + build(:pageview, session_browser: "Safari"), + build(:pageview, session_browser: "Edge") ]) conn = @@ -2172,12 +2178,12 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do site: site } do populate_stats(site, [ - build(:event, pathname: "/en/register"), + build(:event, pathname: "/en/register", name: "pageview"), build(:event, pathname: "/en/register", name: "Signup"), build(:event, pathname: "/en/register", name: "Signup"), build(:event, pathname: "/it/register", name: "Signup", user_id: 1), build(:event, pathname: "/it/register", name: "Signup", user_id: 1), - build(:event, pathname: "/it/register") + build(:event, pathname: "/it/register", name: "pageview") ]) insert(:goal, %{site: site, event_name: "Signup"}) @@ -2214,7 +2220,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do site: site } do populate_stats(site, [ - build(:event, pathname: "/en/register"), + build(:event, pathname: "/en/register", name: "pageview"), build(:event, pathname: "/en/register", name: "Signup") ]) @@ -2244,12 +2250,12 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do site: site } do populate_stats(site, [ - build(:event, screen_size: "Mobile"), - build(:event, screen_size: "Mobile", name: "AddToCart"), - build(:event, screen_size: "Mobile", name: "AddToCart"), - build(:event, screen_size: "Desktop", name: "AddToCart", user_id: 1), - build(:event, screen_size: "Desktop", name: "Purchase", user_id: 1), - build(:event, screen_size: "Desktop") + build(:event, session_screen_size: "Mobile", name: "pageview"), + build(:event, session_screen_size: "Mobile", name: "AddToCart"), + build(:event, session_screen_size: "Mobile", name: "AddToCart"), + build(:event, session_screen_size: "Desktop", name: "AddToCart", user_id: 1), + build(:event, session_screen_size: "Desktop", name: "Purchase", user_id: 1), + build(:event, session_screen_size: "Desktop", name: "pageview") ]) # Make sure that revenue goals are treated the same @@ -2289,8 +2295,8 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do site: site } do populate_stats(site, [ - build(:event, screen_size: "Mobile"), - build(:event, screen_size: "Mobile", name: "AddToCart") + build(:event, session_screen_size: "Mobile", name: "pageview"), + build(:event, session_screen_size: "Mobile", name: "AddToCart") ]) insert(:goal, %{site: site, event_name: "AddToCart"}) @@ -2319,13 +2325,13 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do conn: conn } do populate_stats(site, [ - build(:pageview, browser: "Firefox", browser_version: "110"), - build(:pageview, browser: "Firefox", browser_version: "110"), - build(:pageview, browser: "Chrome", browser_version: "110"), - build(:pageview, browser: "Chrome", browser_version: "110"), - build(:pageview, browser: "Avast Secure Browser", browser_version: "110"), - build(:pageview, browser: "Avast Secure Browser", browser_version: "110"), - build(:event, name: "Signup", browser: "Edge", browser_version: "110") + build(:pageview, session_browser: "Firefox", session_browser_version: "110"), + build(:pageview, session_browser: "Firefox", session_browser_version: "110"), + build(:pageview, session_browser: "Chrome", session_browser_version: "110"), + build(:pageview, session_browser: "Chrome", session_browser_version: "110"), + build(:pageview, session_browser: "Avast Secure Browser", session_browser_version: "110"), + build(:pageview, session_browser: "Avast Secure Browser", session_browser_version: "110"), + build(:event, name: "Signup", session_browser: "Edge", session_browser_version: "110") ]) insert(:goal, site: site, event_name: "Signup") @@ -2357,26 +2363,26 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do populate_stats(site, [ build(:pageview, user_id: 1, - referrer_source: "Google", + session_referrer_source: "Google", timestamp: ~N[2021-01-01 00:00:00] ), build(:event, name: "signup", user_id: 1, - referrer_source: "Google", + session_referrer_source: "Google", timestamp: ~N[2021-01-01 00:05:00] ), build(:pageview, user_id: 1, - referrer_source: "Google", + session_referrer_source: "Google", timestamp: ~N[2021-01-01 00:10:00] ), build(:pageview, - referrer_source: "Google", + session_referrer_source: "Google", timestamp: ~N[2021-01-01 00:25:00] ), build(:pageview, - referrer_source: "Twitter", + session_referrer_source: "Twitter", timestamp: ~N[2021-01-01 00:00:00] ) ]) @@ -2429,7 +2435,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do build(:pageview, user_id: 2, pathname: "/entry-page-2", - referrer_source: "Google", + session_referrer_source: "Google", timestamp: ~N[2021-01-01 00:05:00] ) ]) @@ -2463,28 +2469,28 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do name: "Purchase", "meta.key": ["package"], "meta.value": ["business"], - browser: "Chrome", + session_browser: "Chrome", timestamp: ~N[2021-01-01 00:00:00] ), build(:event, name: "Purchase", "meta.key": ["package"], "meta.value": ["business"], - browser: "Safari", + session_browser: "Safari", timestamp: ~N[2021-01-01 00:00:00] ), build(:event, name: "Purchase", "meta.key": ["package"], "meta.value": ["business"], - browser: "Safari", + session_browser: "Safari", timestamp: ~N[2021-01-01 00:25:00] ), build(:event, name: "Purchase", "meta.key": ["package"], "meta.value": ["personal"], - browser: "IE", + session_browser: "IE", timestamp: ~N[2021-01-01 00:25:00] ) ]) diff --git a/test/plausible_web/controllers/api/external_stats_controller/timeseries_test.exs b/test/plausible_web/controllers/api/external_stats_controller/timeseries_test.exs index bfbfb3726..709a30d42 100644 --- a/test/plausible_web/controllers/api/external_stats_controller/timeseries_test.exs +++ b/test/plausible_web/controllers/api/external_stats_controller/timeseries_test.exs @@ -661,7 +661,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController.TimeseriesTest do test "can filter by source", %{conn: conn, site: site} do populate_stats(site, [ build(:pageview, - referrer_source: "Google", + session_referrer_source: "Google", timestamp: ~N[2021-01-01 00:00:00] ), build(:pageview, timestamp: ~N[2021-01-01 00:00:00]) @@ -683,7 +683,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController.TimeseriesTest do populate_stats(site, [ build(:pageview, timestamp: ~N[2021-01-01 00:00:00]), build(:pageview, - referrer_source: "Google", + session_referrer_source: "Google", timestamp: ~N[2021-01-01 00:00:00] ) ]) @@ -703,7 +703,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController.TimeseriesTest do test "can filter by referrer", %{conn: conn, site: site} do populate_stats(site, [ build(:pageview, - referrer: "https://facebook.com", + session_referrer: "https://facebook.com", timestamp: ~N[2021-01-01 00:00:00] ), build(:pageview, timestamp: ~N[2021-01-01 00:00:00]) @@ -724,7 +724,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController.TimeseriesTest do test "can filter by utm_medium", %{conn: conn, site: site} do populate_stats(site, [ build(:pageview, - utm_medium: "social", + session_utm_medium: "social", timestamp: ~N[2021-01-01 00:00:00] ), build(:pageview, timestamp: ~N[2021-01-01 00:00:00]) @@ -745,7 +745,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController.TimeseriesTest do test "can filter by utm_source", %{conn: conn, site: site} do populate_stats(site, [ build(:pageview, - utm_source: "Twitter", + session_utm_source: "Twitter", timestamp: ~N[2021-01-01 00:00:00] ), build(:pageview, timestamp: ~N[2021-01-01 00:00:00]) @@ -766,7 +766,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController.TimeseriesTest do test "can filter by utm_campaign", %{conn: conn, site: site} do populate_stats(site, [ build(:pageview, - utm_campaign: "profile", + session_utm_campaign: "profile", timestamp: ~N[2021-01-01 00:00:00] ), build(:pageview, timestamp: ~N[2021-01-01 00:00:00]) @@ -787,7 +787,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController.TimeseriesTest do test "can filter by device type", %{conn: conn, site: site} do populate_stats(site, [ build(:pageview, - screen_size: "Desktop", + session_screen_size: "Desktop", timestamp: ~N[2021-01-01 00:00:00] ), build(:pageview, timestamp: ~N[2021-01-01 00:00:00]) @@ -808,13 +808,13 @@ defmodule PlausibleWeb.Api.ExternalStatsController.TimeseriesTest do test "can filter by browser", %{conn: conn, site: site} do populate_stats(site, [ build(:pageview, - browser: "Chrome", - browser_version: "56.1", + session_browser: "Chrome", + session_browser_version: "56.1", timestamp: ~N[2021-01-01 00:00:00] ), build(:pageview, - browser: "Chrome", - browser_version: "55", + session_browser: "Chrome", + session_browser_version: "55", timestamp: ~N[2021-01-01 00:00:00] ), build(:pageview, timestamp: ~N[2021-01-01 00:00:00]) @@ -835,18 +835,18 @@ defmodule PlausibleWeb.Api.ExternalStatsController.TimeseriesTest do test "can filter by operating system", %{conn: conn, site: site} do populate_stats(site, [ build(:pageview, - operating_system: "Mac", - operating_system_version: "10.5", + session_operating_system: "Mac", + session_operating_system_version: "10.5", timestamp: ~N[2021-01-01 00:00:00] ), build(:pageview, - operating_system: "Something else", - operating_system_version: "10.5", + session_operating_system: "Something else", + session_operating_system_version: "10.5", timestamp: ~N[2021-01-01 00:00:00] ), build(:pageview, - operating_system: "Mac", - operating_system_version: "10.4", + session_operating_system: "Mac", + session_operating_system_version: "10.4", timestamp: ~N[2021-01-01 00:00:00] ), build(:pageview, timestamp: ~N[2021-01-01 00:00:00]) @@ -868,14 +868,12 @@ defmodule PlausibleWeb.Api.ExternalStatsController.TimeseriesTest do populate_stats(site, [ build(:pageview, user_id: @user_id, - country_code: "EE", - operating_system_version: "10.5", + session_country_code: "EE", timestamp: ~N[2021-01-01 00:00:00] ), build(:pageview, user_id: @user_id, - country_code: "EE", - operating_system_version: "10.5", + session_country_code: "EE", timestamp: ~N[2021-01-01 00:15:00] ), build(:pageview, timestamp: ~N[2021-01-01 00:00:00]) diff --git a/test/plausible_web/controllers/api/stats_controller/browsers_test.exs b/test/plausible_web/controllers/api/stats_controller/browsers_test.exs index 7d4e94890..c5cfb7030 100644 --- a/test/plausible_web/controllers/api/stats_controller/browsers_test.exs +++ b/test/plausible_web/controllers/api/stats_controller/browsers_test.exs @@ -6,9 +6,9 @@ defmodule PlausibleWeb.Api.StatsController.BrowsersTest do test "returns top browsers by unique visitors", %{conn: conn, site: site} do populate_stats(site, [ - build(:pageview, browser: "Chrome"), - build(:pageview, browser: "Chrome"), - build(:pageview, browser: "Firefox") + build(:pageview, session_browser: "Chrome"), + build(:pageview, session_browser: "Chrome"), + build(:pageview, session_browser: "Firefox") ]) conn = get(conn, "/api/stats/#{site.domain}/browsers?period=day") @@ -26,21 +26,21 @@ defmodule PlausibleWeb.Api.StatsController.BrowsersTest do populate_stats(site, [ build(:pageview, user_id: 123, - browser: "Chrome" + session_browser: "Chrome" ), build(:pageview, user_id: 123, - browser: "Chrome", + session_browser: "Chrome", "meta.key": ["author"], "meta.value": ["John Doe"] ), build(:pageview, - browser: "Firefox", + session_browser: "Firefox", "meta.key": ["author"], "meta.value": ["other"] ), build(:pageview, - browser: "Safari" + session_browser: "Safari" ) ]) @@ -59,23 +59,23 @@ defmodule PlausibleWeb.Api.StatsController.BrowsersTest do populate_stats(site, [ build(:pageview, user_id: 123, - browser: "Chrome", + session_browser: "Chrome", "meta.key": ["author"], "meta.value": ["John Doe"] ), build(:pageview, user_id: 123, - browser: "Chrome", + session_browser: "Chrome", "meta.key": ["author"], "meta.value": ["John Doe"] ), build(:pageview, - browser: "Firefox", + session_browser: "Firefox", "meta.key": ["author"], "meta.value": ["other"] ), build(:pageview, - browser: "Safari" + session_browser: "Safari" ) ]) @@ -90,8 +90,8 @@ defmodule PlausibleWeb.Api.StatsController.BrowsersTest do test "calculates conversion_rate when filtering for goal", %{conn: conn, site: site} do populate_stats(site, [ - build(:pageview, user_id: 1, browser: "Chrome"), - build(:pageview, user_id: 2, browser: "Chrome"), + build(:pageview, user_id: 1, session_browser: "Chrome"), + build(:pageview, user_id: 2, session_browser: "Chrome"), build(:event, user_id: 1, name: "Signup") ]) @@ -111,7 +111,7 @@ defmodule PlausibleWeb.Api.StatsController.BrowsersTest do test "returns top browsers including imported data", %{conn: conn, site: site} do populate_stats(site, [ - build(:pageview, browser: "Chrome"), + build(:pageview, session_browser: "Chrome"), build(:imported_browsers, browser: "Chrome"), build(:imported_browsers, browser: "Firefox"), build(:imported_visitors, visitors: 2) @@ -156,7 +156,7 @@ defmodule PlausibleWeb.Api.StatsController.BrowsersTest do populate_stats(site, [ build(:pageview, user_id: 123, - browser: "" + session_browser: "" ) ]) @@ -176,16 +176,16 @@ defmodule PlausibleWeb.Api.StatsController.BrowsersTest do site: site } do populate_stats(site, [ - build(:event, browser: "Chrome", browser_version: "110", name: "Signup"), - build(:event, browser: "Chrome", browser_version: "110", name: "Signup"), - build(:pageview, browser: "Chrome", browser_version: "110"), - build(:pageview, browser: "Chrome", browser_version: "121"), - build(:pageview, browser: "Chrome", browser_version: "121"), - build(:event, browser: "Firefox", browser_version: "121", name: "Signup"), - build(:pageview, browser: "Firefox", browser_version: "110"), - build(:pageview, browser: "Firefox", browser_version: "110"), - build(:pageview, browser: "Firefox", browser_version: "110"), - build(:pageview, browser: "Firefox", browser_version: "110") + build(:event, session_browser: "Chrome", session_browser_version: "110", name: "Signup"), + build(:event, session_browser: "Chrome", session_browser_version: "110", name: "Signup"), + build(:pageview, session_browser: "Chrome", session_browser_version: "110"), + build(:pageview, session_browser: "Chrome", session_browser_version: "121"), + build(:pageview, session_browser: "Chrome", session_browser_version: "121"), + build(:event, session_browser: "Firefox", session_browser_version: "121", name: "Signup"), + build(:pageview, session_browser: "Firefox", session_browser_version: "110"), + build(:pageview, session_browser: "Firefox", session_browser_version: "110"), + build(:pageview, session_browser: "Firefox", session_browser_version: "110"), + build(:pageview, session_browser: "Firefox", session_browser_version: "110") ]) insert(:goal, site: site, event_name: "Signup") @@ -217,10 +217,10 @@ defmodule PlausibleWeb.Api.StatsController.BrowsersTest do test "returns top browser versions by unique visitors", %{conn: conn, site: site} do populate_stats(site, [ - build(:pageview, browser: "Chrome", browser_version: "78.0"), - build(:pageview, browser: "Chrome", browser_version: "78.0"), - build(:pageview, browser: "Chrome", browser_version: "77.0"), - build(:pageview, browser: "Firefox", browser_version: "88.0") + build(:pageview, session_browser: "Chrome", session_browser_version: "78.0"), + build(:pageview, session_browser: "Chrome", session_browser_version: "78.0"), + build(:pageview, session_browser: "Chrome", session_browser_version: "77.0"), + build(:pageview, session_browser: "Firefox", session_browser_version: "88.0") ]) filters = Jason.encode!(%{browser: "Chrome"}) @@ -239,7 +239,7 @@ defmodule PlausibleWeb.Api.StatsController.BrowsersTest do test "returns results for (not set)", %{conn: conn, site: site} do populate_stats(site, [ - build(:pageview, browser: "", browser_version: "") + build(:pageview, session_browser: "", session_browser_version: "") ]) filters = Jason.encode!(%{browser: "(not set)"}) diff --git a/test/plausible_web/controllers/api/stats_controller/cities_test.exs b/test/plausible_web/controllers/api/stats_controller/cities_test.exs index ac5fe1ccb..46194f421 100644 --- a/test/plausible_web/controllers/api/stats_controller/cities_test.exs +++ b/test/plausible_web/controllers/api/stats_controller/cities_test.exs @@ -4,11 +4,31 @@ defmodule PlausibleWeb.Api.StatsController.CitiesTest do describe "GET /api/stats/:domain/cities" do defp seed(%{site: site}) do populate_stats(site, [ - build(:pageview, country_code: "EE", subdivision1_code: "EE-37", city_geoname_id: 588_409), - build(:pageview, country_code: "EE", subdivision1_code: "EE-37", city_geoname_id: 588_409), - build(:pageview, country_code: "EE", subdivision1_code: "EE-37", city_geoname_id: 588_409), - build(:pageview, country_code: "EE", subdivision1_code: "EE-39", city_geoname_id: 591_632), - build(:pageview, country_code: "EE", subdivision1_code: "EE-39", city_geoname_id: 591_632) + build(:pageview, + session_country_code: "EE", + session_subdivision1_code: "EE-37", + session_city_geoname_id: 588_409 + ), + build(:pageview, + session_country_code: "EE", + session_subdivision1_code: "EE-37", + session_city_geoname_id: 588_409 + ), + build(:pageview, + session_country_code: "EE", + session_subdivision1_code: "EE-37", + session_city_geoname_id: 588_409 + ), + build(:pageview, + session_country_code: "EE", + session_subdivision1_code: "EE-39", + session_city_geoname_id: 591_632 + ), + build(:pageview, + session_country_code: "EE", + session_subdivision1_code: "EE-39", + session_city_geoname_id: 591_632 + ) ]) end diff --git a/test/plausible_web/controllers/api/stats_controller/countries_test.exs b/test/plausible_web/controllers/api/stats_controller/countries_test.exs index 245e3a1f7..2abe41c82 100644 --- a/test/plausible_web/controllers/api/stats_controller/countries_test.exs +++ b/test/plausible_web/controllers/api/stats_controller/countries_test.exs @@ -6,9 +6,9 @@ defmodule PlausibleWeb.Api.StatsController.CountriesTest do test "returns top countries by new visitors", %{conn: conn, site: site} do populate_stats(site, [ - build(:pageview, country_code: "EE"), - build(:pageview, country_code: "EE"), - build(:pageview, country_code: "GB"), + build(:pageview, session_country_code: "EE"), + build(:pageview, session_country_code: "EE"), + build(:pageview, session_country_code: "GB"), build(:imported_locations, country: "EE"), build(:imported_locations, country: "GB"), build(:imported_visitors, visitors: 2) @@ -59,7 +59,7 @@ defmodule PlausibleWeb.Api.StatsController.CountriesTest do test "ignores unknown country code ZZ", %{conn: conn, site: site} do populate_stats(site, [ - build(:pageview, country_code: "ZZ"), + build(:pageview, session_country_code: "ZZ"), build(:imported_locations, country: "ZZ") ]) @@ -72,16 +72,16 @@ defmodule PlausibleWeb.Api.StatsController.CountriesTest do populate_stats(site, [ build(:pageview, user_id: 1, - country_code: "EE" + session_country_code: "EE" ), build(:event, user_id: 1, name: "Signup"), build(:pageview, user_id: 2, - country_code: "EE" + session_country_code: "EE" ), build(:pageview, user_id: 3, - country_code: "GB" + session_country_code: "GB" ), build(:event, user_id: 3, name: "Signup") ]) @@ -119,21 +119,21 @@ defmodule PlausibleWeb.Api.StatsController.CountriesTest do populate_stats(site, [ build(:pageview, user_id: 123, - country_code: "EE" + session_country_code: "EE" ), build(:pageview, user_id: 123, - country_code: "EE", + session_country_code: "EE", "meta.key": ["author"], "meta.value": ["John Doe"] ), build(:pageview, - country_code: "GB", + session_country_code: "GB", "meta.key": ["author"], "meta.value": ["other"] ), build(:pageview, - country_code: "US" + session_country_code: "US" ) ]) @@ -159,23 +159,23 @@ defmodule PlausibleWeb.Api.StatsController.CountriesTest do populate_stats(site, [ build(:pageview, user_id: 123, - country_code: "EE", + session_country_code: "EE", "meta.key": ["author"], "meta.value": ["John Doe"] ), build(:pageview, user_id: 123, - country_code: "EE", + session_country_code: "EE", "meta.key": ["author"], "meta.value": ["John Doe"] ), build(:pageview, - country_code: "GB", + session_country_code: "GB", "meta.key": ["author"], "meta.value": ["other"] ), build(:pageview, - country_code: "GB" + session_country_code: "GB" ) ]) @@ -200,17 +200,17 @@ defmodule PlausibleWeb.Api.StatsController.CountriesTest do } do populate_stats(site, [ build(:pageview, - country_code: "EE", + session_country_code: "EE", "meta.key": ["author"], "meta.value": ["John Doe"] ), build(:pageview, - country_code: "GB", + session_country_code: "GB", "meta.key": ["logged_in"], "meta.value": ["true"] ), build(:pageview, - country_code: "GB" + session_country_code: "GB" ) ]) @@ -235,22 +235,22 @@ defmodule PlausibleWeb.Api.StatsController.CountriesTest do } do populate_stats(site, [ build(:pageview, - country_code: "EE", + session_country_code: "EE", "meta.key": ["author"], "meta.value": ["John Doe"] ), build(:pageview, - country_code: "EE", + session_country_code: "EE", "meta.key": ["author"], "meta.value": [""] ), build(:pageview, - country_code: "GB", + session_country_code: "GB", "meta.key": ["logged_in"], "meta.value": ["true"] ), build(:pageview, - country_code: "GB" + session_country_code: "GB" ) ]) @@ -271,9 +271,9 @@ defmodule PlausibleWeb.Api.StatsController.CountriesTest do test "when list is filtered by country returns one country only", %{conn: conn, site: site} do populate_stats(site, [ - build(:pageview, country_code: "EE"), - build(:pageview, country_code: "EE"), - build(:pageview, country_code: "GB") + build(:pageview, session_country_code: "EE"), + build(:pageview, session_country_code: "EE"), + build(:pageview, session_country_code: "GB") ]) filters = Jason.encode!(%{country: "GB"}) diff --git a/test/plausible_web/controllers/api/stats_controller/custom_prop_breakdown_test.exs b/test/plausible_web/controllers/api/stats_controller/custom_prop_breakdown_test.exs index 598ec5666..6548f49cd 100644 --- a/test/plausible_web/controllers/api/stats_controller/custom_prop_breakdown_test.exs +++ b/test/plausible_web/controllers/api/stats_controller/custom_prop_breakdown_test.exs @@ -688,14 +688,14 @@ defmodule PlausibleWeb.Api.StatsController.CustomPropBreakdownTest do test "Property breakdown with prop and goal filter", %{conn: conn, site: site} do populate_stats(site, [ - build(:pageview, user_id: 1, utm_campaign: "campaignA"), + build(:pageview, user_id: 1, session_utm_campaign: "campaignA"), build(:event, user_id: 1, name: "ButtonClick", "meta.key": ["variant"], "meta.value": ["A"] ), - build(:pageview, user_id: 2, utm_campaign: "campaignA"), + build(:pageview, user_id: 2, session_utm_campaign: "campaignA"), build(:event, user_id: 2, name: "ButtonClick", @@ -710,7 +710,7 @@ defmodule PlausibleWeb.Api.StatsController.CustomPropBreakdownTest do Jason.encode!(%{ goal: "ButtonClick", props: %{variant: "A"}, - utm_campaign: "campaignA" + session_utm_campaign: "campaignA" }) prop_key = "variant" @@ -733,15 +733,15 @@ defmodule PlausibleWeb.Api.StatsController.CustomPropBreakdownTest do test "Property breakdown with goal and source filter", %{conn: conn, site: site} do populate_stats(site, [ - build(:pageview, user_id: 1, referrer_source: "Google"), + build(:pageview, user_id: 1, session_referrer_source: "Google"), build(:event, user_id: 1, name: "ButtonClick", "meta.key": ["variant"], "meta.value": ["A"] ), - build(:pageview, user_id: 2, referrer_source: "Google"), - build(:pageview, user_id: 3, referrer_source: "ignore"), + build(:pageview, user_id: 2, session_referrer_source: "Google"), + build(:pageview, user_id: 3, session_referrer_source: "ignore"), build(:event, user_id: 3, name: "ButtonClick", @@ -962,7 +962,11 @@ defmodule PlausibleWeb.Api.StatsController.CustomPropBreakdownTest do populate_stats(site, [ build(:pageview, "meta.key": [prop_key], "meta.value": ["K2sna Kalle"]), - build(:pageview, browser: "Chrome", "meta.key": [prop_key], "meta.value": ["Sipsik"]) + build(:pageview, + session_browser: "Chrome", + "meta.key": [prop_key], + "meta.value": ["Sipsik"] + ) ]) filters = Jason.encode!(%{browser: "Chrome"}) diff --git a/test/plausible_web/controllers/api/stats_controller/funnels_test.exs b/test/plausible_web/controllers/api/stats_controller/funnels_test.exs index f73a3f728..dbb38d10f 100644 --- a/test/plausible_web/controllers/api/stats_controller/funnels_test.exs +++ b/test/plausible_web/controllers/api/stats_controller/funnels_test.exs @@ -106,15 +106,15 @@ defmodule PlausibleWeb.Api.StatsController.FunnelsTest do build(:pageview, pathname: "/blog/announcement", user_id: @other_user_id, - utm_medium: "social", - timestamp: ~N[2021-01-01 12:00:00] + timestamp: ~N[2021-01-01 12:00:00], + session_utm_medium: "social" ), build(:event, name: "Signup", user_id: @user_id), build(:event, name: "Signup", user_id: @other_user_id, - utm_medium: "social", - timestamp: ~N[2021-01-01 12:01:00] + timestamp: ~N[2021-01-01 12:01:00], + session_utm_medium: "social" ), build(:pageview, pathname: "/cart/add/product", user_id: @user_id), build(:event, name: "Purchase", user_id: @user_id) diff --git a/test/plausible_web/controllers/api/stats_controller/imported_test.exs b/test/plausible_web/controllers/api/stats_controller/imported_test.exs index 027db390c..d1a61a7d0 100644 --- a/test/plausible_web/controllers/api/stats_controller/imported_test.exs +++ b/test/plausible_web/controllers/api/stats_controller/imported_test.exs @@ -154,18 +154,18 @@ defmodule PlausibleWeb.Api.StatsController.ImportedTest do test "Sources are imported", %{conn: conn, site: site, import_id: import_id} do populate_stats(site, [ build(:pageview, - referrer_source: "Google", - referrer: "google.com", + session_referrer_source: "Google", + session_referrer: "google.com", timestamp: ~N[2021-01-01 00:00:00] ), build(:pageview, - referrer_source: "Google", - referrer: "google.com", + session_referrer_source: "Google", + session_referrer: "google.com", timestamp: ~N[2021-01-01 00:00:00] ), build(:pageview, - referrer_source: "DuckDuckGo", - referrer: "duckduckgo.com", + session_referrer_source: "DuckDuckGo", + session_referrer: "duckduckgo.com", timestamp: ~N[2021-01-01 00:00:00] ) ]) @@ -296,11 +296,11 @@ defmodule PlausibleWeb.Api.StatsController.ImportedTest do } do populate_stats(site, [ build(:pageview, - utm_medium: "social", + session_utm_medium: "social", timestamp: ~N[2021-01-01 00:00:00] ), build(:pageview, - utm_medium: "social", + session_utm_medium: "social", timestamp: ~N[2021-01-01 12:00:00] ) ]) @@ -367,8 +367,8 @@ defmodule PlausibleWeb.Api.StatsController.ImportedTest do import_id: import_id } do populate_stats(site, [ - build(:pageview, utm_campaign: "profile", timestamp: ~N[2021-01-01 00:00:00]), - build(:pageview, utm_campaign: "august", timestamp: ~N[2021-01-01 00:00:00]) + build(:pageview, session_utm_campaign: "profile", timestamp: ~N[2021-01-01 00:00:00]), + build(:pageview, session_utm_campaign: "august", timestamp: ~N[2021-01-01 00:00:00]) ]) import_data( @@ -455,9 +455,9 @@ defmodule PlausibleWeb.Api.StatsController.ImportedTest do import_id: import_id } do populate_stats(site, [ - build(:pageview, utm_term: "oat milk", timestamp: ~N[2021-01-01 00:00:00]), - build(:pageview, utm_term: "Sweden", timestamp: ~N[2021-01-01 00:00:00]), - build(:pageview, utm_term: "Sweden", timestamp: ~N[2021-01-01 00:00:00]) + build(:pageview, session_utm_term: "oat milk", timestamp: ~N[2021-01-01 00:00:00]), + build(:pageview, session_utm_term: "Sweden", timestamp: ~N[2021-01-01 00:00:00]), + build(:pageview, session_utm_term: "Sweden", timestamp: ~N[2021-01-01 00:00:00]) ]) import_data( @@ -544,8 +544,8 @@ defmodule PlausibleWeb.Api.StatsController.ImportedTest do import_id: import_id } do populate_stats(site, [ - build(:pageview, utm_content: "ad", timestamp: ~N[2021-01-01 00:00:00]), - build(:pageview, utm_content: "blog", timestamp: ~N[2021-01-01 00:00:00]) + build(:pageview, session_utm_content: "ad", timestamp: ~N[2021-01-01 00:00:00]), + build(:pageview, session_utm_content: "blog", timestamp: ~N[2021-01-01 00:00:00]) ]) import_data( @@ -817,15 +817,15 @@ defmodule PlausibleWeb.Api.StatsController.ImportedTest do } do populate_stats(site, [ build(:pageview, - country_code: "EE", + session_country_code: "EE", timestamp: ~N[2021-01-01 00:15:00] ), build(:pageview, - country_code: "EE", + session_country_code: "EE", timestamp: ~N[2021-01-01 00:15:00] ), build(:pageview, - country_code: "GB", + session_country_code: "GB", timestamp: ~N[2021-01-01 00:15:00] ) ]) @@ -890,15 +890,15 @@ defmodule PlausibleWeb.Api.StatsController.ImportedTest do } do populate_stats(site, import_id, [ build(:pageview, - country_code: "EE", + session_country_code: "EE", timestamp: ~N[2021-01-01 00:15:00] ), build(:pageview, - country_code: "EE", + session_country_code: "EE", timestamp: ~N[2021-01-01 00:15:00] ), build(:pageview, - country_code: "GB", + session_country_code: "GB", timestamp: ~N[2021-01-01 00:15:00] ), build(:imported_visitors, date: ~D[2021-01-01], visitors: 2) @@ -972,9 +972,9 @@ defmodule PlausibleWeb.Api.StatsController.ImportedTest do import_id: import_id } do populate_stats(site, import_id, [ - build(:pageview, screen_size: "Desktop", timestamp: ~N[2021-01-01 00:15:00]), - build(:pageview, screen_size: "Desktop", timestamp: ~N[2021-01-01 00:15:00]), - build(:pageview, screen_size: "Laptop", timestamp: ~N[2021-01-01 00:15:00]), + build(:pageview, session_screen_size: "Desktop", timestamp: ~N[2021-01-01 00:15:00]), + build(:pageview, session_screen_size: "Desktop", timestamp: ~N[2021-01-01 00:15:00]), + build(:pageview, session_screen_size: "Laptop", timestamp: ~N[2021-01-01 00:15:00]), build(:imported_visitors, date: ~D[2021-01-01], visitors: 2) ]) @@ -1023,8 +1023,8 @@ defmodule PlausibleWeb.Api.StatsController.ImportedTest do import_id: import_id } do populate_stats(site, import_id, [ - build(:pageview, browser: "Chrome", timestamp: ~N[2021-01-01 00:15:00]), - build(:pageview, browser: "Firefox", timestamp: ~N[2021-01-01 00:15:00]), + build(:pageview, session_browser: "Chrome", timestamp: ~N[2021-01-01 00:15:00]), + build(:pageview, session_browser: "Firefox", timestamp: ~N[2021-01-01 00:15:00]), build(:imported_visitors, visitors: 2, date: ~D[2021-01-01]) ]) @@ -1076,9 +1076,12 @@ defmodule PlausibleWeb.Api.StatsController.ImportedTest do import_id: import_id } do populate_stats(site, import_id, [ - build(:pageview, operating_system: "Mac", timestamp: ~N[2021-01-01 00:15:00]), - build(:pageview, operating_system: "Mac", timestamp: ~N[2021-01-01 00:15:00]), - build(:pageview, operating_system: "GNU/Linux", timestamp: ~N[2021-01-01 00:15:00]), + build(:pageview, session_operating_system: "Mac", timestamp: ~N[2021-01-01 00:15:00]), + build(:pageview, session_operating_system: "Mac", timestamp: ~N[2021-01-01 00:15:00]), + build(:pageview, + session_operating_system: "GNU/Linux", + timestamp: ~N[2021-01-01 00:15:00] + ), build(:imported_visitors, date: ~D[2021-01-01], visitors: 2) ]) diff --git a/test/plausible_web/controllers/api/stats_controller/operating_systems_test.exs b/test/plausible_web/controllers/api/stats_controller/operating_systems_test.exs index d21aa884f..45a7c7b78 100644 --- a/test/plausible_web/controllers/api/stats_controller/operating_systems_test.exs +++ b/test/plausible_web/controllers/api/stats_controller/operating_systems_test.exs @@ -6,9 +6,9 @@ defmodule PlausibleWeb.Api.StatsController.OperatingSystemsTest do test "returns operating systems by unique visitors", %{conn: conn, site: site} do populate_stats(site, [ - build(:pageview, operating_system: "Mac"), - build(:pageview, operating_system: "Mac"), - build(:pageview, operating_system: "Android") + build(:pageview, session_operating_system: "Mac"), + build(:pageview, session_operating_system: "Mac"), + build(:pageview, session_operating_system: "Android") ]) conn = get(conn, "/api/stats/#{site.domain}/operating-systems?period=day") @@ -22,10 +22,10 @@ defmodule PlausibleWeb.Api.StatsController.OperatingSystemsTest do test "returns (not set) when appropriate", %{conn: conn, site: site} do populate_stats(site, [ build(:pageview, - operating_system: "" + session_operating_system: "" ), build(:pageview, - operating_system: "Linux" + session_operating_system: "Linux" ) ]) @@ -48,8 +48,8 @@ defmodule PlausibleWeb.Api.StatsController.OperatingSystemsTest do test "calculates conversion_rate when filtering for goal", %{conn: conn, site: site} do populate_stats(site, [ - build(:pageview, user_id: 1, operating_system: "Mac"), - build(:pageview, user_id: 2, operating_system: "Mac"), + build(:pageview, user_id: 1, session_operating_system: "Mac"), + build(:pageview, user_id: 2, session_operating_system: "Mac"), build(:event, user_id: 1, name: "Signup") ]) @@ -75,21 +75,21 @@ defmodule PlausibleWeb.Api.StatsController.OperatingSystemsTest do populate_stats(site, [ build(:pageview, user_id: 123, - operating_system: "Mac" + session_operating_system: "Mac" ), build(:pageview, user_id: 123, - operating_system: "Mac", + session_operating_system: "Mac", "meta.key": ["author"], "meta.value": ["John Doe"] ), build(:pageview, - operating_system: "Windows", + session_operating_system: "Windows", "meta.key": ["author"], "meta.value": ["other"] ), build(:pageview, - operating_system: "Android" + session_operating_system: "Android" ) ]) @@ -110,23 +110,23 @@ defmodule PlausibleWeb.Api.StatsController.OperatingSystemsTest do populate_stats(site, [ build(:pageview, user_id: 123, - operating_system: "Windows", + session_operating_system: "Windows", "meta.key": ["author"], "meta.value": ["John Doe"] ), build(:pageview, user_id: 123, - operating_system: "Windows", + session_operating_system: "Windows", "meta.key": ["author"], "meta.value": ["John Doe"] ), build(:pageview, - operating_system: "Mac", + session_operating_system: "Mac", "meta.key": ["author"], "meta.value": ["other"] ), build(:pageview, - operating_system: "Android" + session_operating_system: "Android" ) ]) @@ -146,9 +146,9 @@ defmodule PlausibleWeb.Api.StatsController.OperatingSystemsTest do site: site } do populate_stats(site, [ - build(:pageview, operating_system: "Mac"), - build(:pageview, operating_system: "Mac"), - build(:pageview, operating_system: "Android"), + build(:pageview, session_operating_system: "Mac"), + build(:pageview, session_operating_system: "Mac"), + build(:pageview, session_operating_system: "Android"), build(:imported_operating_systems, operating_system: "Mac"), build(:imported_operating_systems, operating_system: "Android"), build(:imported_visitors, visitors: 2) @@ -172,8 +172,8 @@ defmodule PlausibleWeb.Api.StatsController.OperatingSystemsTest do test "imported data is ignored when filtering for goal", %{conn: conn, site: site} do populate_stats(site, [ - build(:pageview, user_id: 1, operating_system: "Mac"), - build(:pageview, user_id: 2, operating_system: "Mac"), + build(:pageview, user_id: 1, session_operating_system: "Mac"), + build(:pageview, user_id: 2, session_operating_system: "Mac"), build(:imported_operating_systems, operating_system: "Mac"), build(:event, user_id: 1, name: "Signup") ]) @@ -199,10 +199,22 @@ defmodule PlausibleWeb.Api.StatsController.OperatingSystemsTest do test "returns top OS versions by unique visitors", %{conn: conn, site: site} do populate_stats(site, [ - build(:pageview, operating_system: "Mac", operating_system_version: "10.15"), - build(:pageview, operating_system: "Mac", operating_system_version: "10.16"), - build(:pageview, operating_system: "Mac", operating_system_version: "10.16"), - build(:pageview, operating_system: "Android", operating_system_version: "4") + build(:pageview, + session_operating_system: "Mac", + session_operating_system_version: "10.15" + ), + build(:pageview, + session_operating_system: "Mac", + session_operating_system_version: "10.16" + ), + build(:pageview, + session_operating_system: "Mac", + session_operating_system_version: "10.16" + ), + build(:pageview, + session_operating_system: "Android", + session_operating_system_version: "4" + ) ]) filters = Jason.encode!(%{os: "Mac"}) diff --git a/test/plausible_web/controllers/api/stats_controller/regions_test.exs b/test/plausible_web/controllers/api/stats_controller/regions_test.exs index d70c8ccd5..f20630b07 100644 --- a/test/plausible_web/controllers/api/stats_controller/regions_test.exs +++ b/test/plausible_web/controllers/api/stats_controller/regions_test.exs @@ -4,11 +4,31 @@ defmodule PlausibleWeb.Api.StatsController.RegionsTest do describe "GET /api/stats/:domain/regions" do defp seed(%{site: site}) do populate_stats(site, [ - build(:pageview, country_code: "EE", subdivision1_code: "EE-37", city_geoname_id: 588_409), - build(:pageview, country_code: "EE", subdivision1_code: "EE-37", city_geoname_id: 588_409), - build(:pageview, country_code: "EE", subdivision1_code: "EE-37", city_geoname_id: 588_409), - build(:pageview, country_code: "EE", subdivision1_code: "EE-39", city_geoname_id: 591_632), - build(:pageview, country_code: "EE", subdivision1_code: "EE-39", city_geoname_id: 591_632) + build(:pageview, + session_country_code: "EE", + session_subdivision1_code: "EE-37", + session_city_geoname_id: 588_409 + ), + build(:pageview, + session_country_code: "EE", + session_subdivision1_code: "EE-37", + session_city_geoname_id: 588_409 + ), + build(:pageview, + session_country_code: "EE", + session_subdivision1_code: "EE-37", + session_city_geoname_id: 588_409 + ), + build(:pageview, + session_country_code: "EE", + session_subdivision1_code: "EE-39", + session_city_geoname_id: 591_632 + ), + build(:pageview, + session_country_code: "EE", + session_subdivision1_code: "EE-39", + session_city_geoname_id: 591_632 + ) ]) end diff --git a/test/plausible_web/controllers/api/stats_controller/screen_sizes_test.exs b/test/plausible_web/controllers/api/stats_controller/screen_sizes_test.exs index d1511914f..799f6d1e2 100644 --- a/test/plausible_web/controllers/api/stats_controller/screen_sizes_test.exs +++ b/test/plausible_web/controllers/api/stats_controller/screen_sizes_test.exs @@ -6,9 +6,9 @@ defmodule PlausibleWeb.Api.StatsController.ScreenSizesTest do test "returns screen sizes by new visitors", %{conn: conn, site: site} do populate_stats(site, [ - build(:pageview, screen_size: "Desktop"), - build(:pageview, screen_size: "Desktop"), - build(:pageview, screen_size: "Laptop") + build(:pageview, session_screen_size: "Desktop"), + build(:pageview, session_screen_size: "Desktop"), + build(:pageview, session_screen_size: "Laptop") ]) conn = get(conn, "/api/stats/#{site.domain}/screen-sizes?period=day") @@ -22,10 +22,10 @@ defmodule PlausibleWeb.Api.StatsController.ScreenSizesTest do test "returns (not set) when appropriate", %{conn: conn, site: site} do populate_stats(site, [ build(:pageview, - screen_size: "" + session_screen_size: "" ), build(:pageview, - screen_size: "Desktop" + session_screen_size: "Desktop" ) ]) @@ -53,21 +53,21 @@ defmodule PlausibleWeb.Api.StatsController.ScreenSizesTest do populate_stats(site, [ build(:pageview, user_id: 123, - screen_size: "Desktop" + session_screen_size: "Desktop" ), build(:pageview, user_id: 123, - screen_size: "Desktop", + session_screen_size: "Desktop", "meta.key": ["author"], "meta.value": ["John Doe"] ), build(:pageview, - screen_size: "Mobile", + session_screen_size: "Mobile", "meta.key": ["author"], "meta.value": ["other"] ), build(:pageview, - screen_size: "Tablet" + session_screen_size: "Tablet" ) ]) @@ -86,23 +86,23 @@ defmodule PlausibleWeb.Api.StatsController.ScreenSizesTest do populate_stats(site, [ build(:pageview, user_id: 123, - screen_size: "Desktop", + session_screen_size: "Desktop", "meta.key": ["author"], "meta.value": ["John Doe"] ), build(:pageview, user_id: 123, - screen_size: "Desktop", + session_screen_size: "Desktop", "meta.key": ["author"], "meta.value": ["John Doe"] ), build(:pageview, - screen_size: "Mobile", + session_screen_size: "Mobile", "meta.key": ["author"], "meta.value": ["other"] ), build(:pageview, - screen_size: "Tablet" + session_screen_size: "Tablet" ) ]) @@ -117,9 +117,9 @@ defmodule PlausibleWeb.Api.StatsController.ScreenSizesTest do test "returns screen sizes by new visitors with imported data", %{conn: conn, site: site} do populate_stats(site, [ - build(:pageview, screen_size: "Desktop"), - build(:pageview, screen_size: "Desktop"), - build(:pageview, screen_size: "Laptop") + build(:pageview, session_screen_size: "Desktop"), + build(:pageview, session_screen_size: "Desktop"), + build(:pageview, session_screen_size: "Laptop") ]) populate_stats(site, [ @@ -146,8 +146,8 @@ defmodule PlausibleWeb.Api.StatsController.ScreenSizesTest do test "calculates conversion_rate when filtering for goal", %{conn: conn, site: site} do populate_stats(site, [ - build(:pageview, user_id: 1, screen_size: "Desktop"), - build(:pageview, user_id: 2, screen_size: "Desktop"), + build(:pageview, user_id: 1, session_screen_size: "Desktop"), + build(:pageview, user_id: 2, session_screen_size: "Desktop"), build(:event, user_id: 1, name: "Signup") ]) @@ -167,11 +167,14 @@ defmodule PlausibleWeb.Api.StatsController.ScreenSizesTest do test "returns screen sizes with not_member filter type", %{conn: conn, site: site} do populate_stats(site, [ - build(:pageview, referrer_source: "Google", screen_size: "Desktop"), - build(:pageview, referrer_source: "Bad source", screen_size: "Desktop"), - build(:pageview, referrer_source: "Google", screen_size: "Desktop"), - build(:pageview, referrer_source: "Twitter", screen_size: "Mobile"), - build(:pageview, referrer_source: "Second bad source", screen_size: "Mobile") + build(:pageview, session_referrer_source: "Google", session_screen_size: "Desktop"), + build(:pageview, session_referrer_source: "Bad source", session_screen_size: "Desktop"), + build(:pageview, session_referrer_source: "Google", session_screen_size: "Desktop"), + build(:pageview, session_referrer_source: "Twitter", session_screen_size: "Mobile"), + build(:pageview, + session_referrer_source: "Second bad source", + session_screen_size: "Mobile" + ) ]) filters = Jason.encode!(%{"source" => "!Bad source|Second bad source"}) diff --git a/test/plausible_web/controllers/api/stats_controller/sources_test.exs b/test/plausible_web/controllers/api/stats_controller/sources_test.exs index c3735e946..438389922 100644 --- a/test/plausible_web/controllers/api/stats_controller/sources_test.exs +++ b/test/plausible_web/controllers/api/stats_controller/sources_test.exs @@ -9,24 +9,24 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest do test "returns top sources by unique user ids", %{conn: conn, site: site} do populate_stats(site, [ build(:pageview, - referrer_source: "Google", - referrer: "google.com" + session_referrer_source: "Google", + session_referrer: "google.com" ), build(:pageview, - referrer_source: "Google", - referrer: "google.com" + session_referrer_source: "Google", + session_referrer: "google.com" ), build(:pageview, - referrer_source: "Google", - referrer: "google.com" + session_referrer_source: "Google", + session_referrer: "google.com" ), build(:pageview, - referrer_source: "DuckDuckGo", - referrer: "duckduckgo.com" + session_referrer_source: "DuckDuckGo", + session_referrer: "duckduckgo.com" ), build(:pageview, - referrer_source: "DuckDuckGo", - referrer: "duckduckgo.com" + session_referrer_source: "DuckDuckGo", + session_referrer: "duckduckgo.com" ), build(:pageview) ]) @@ -43,8 +43,8 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest do test "returns top sources with :is filter on custom pageview props", %{conn: conn, site: site} do populate_stats(site, [ build(:pageview, - referrer_source: "DuckDuckGo", - referrer: "duckduckgo.com", + session_referrer_source: "DuckDuckGo", + session_referrer: "duckduckgo.com", user_id: 123, timestamp: ~N[2021-01-01 00:00:00] ), @@ -55,22 +55,22 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest do timestamp: ~N[2021-01-01 00:01:00] ), build(:pageview, - referrer_source: "Google", - referrer: "google.com", + session_referrer_source: "Google", + session_referrer: "google.com", "meta.key": ["author"], "meta.value": ["John Doe"], timestamp: ~N[2021-01-01 00:00:00] ), build(:pageview, - referrer_source: "Google", - referrer: "google.com", + session_referrer_source: "Google", + session_referrer: "google.com", "meta.key": ["author"], "meta.value": ["John Doe"], timestamp: ~N[2021-01-01 00:00:00] ), build(:pageview, - referrer_source: "Facebook", - referrer: "facebook.com", + session_referrer_source: "Facebook", + session_referrer: "facebook.com", timestamp: ~N[2021-01-01 00:00:00] ) ]) @@ -95,8 +95,8 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest do } do populate_stats(site, [ build(:pageview, - referrer_source: "DuckDuckGo", - referrer: "duckduckgo.com", + session_referrer_source: "DuckDuckGo", + session_referrer: "duckduckgo.com", "meta.key": ["author"], "meta.value": ["John Doe"], user_id: 123, @@ -109,20 +109,20 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest do timestamp: ~N[2021-01-01 00:01:00] ), build(:pageview, - referrer_source: "Google", - referrer: "google.com", + session_referrer_source: "Google", + session_referrer: "google.com", "meta.key": ["author"], "meta.value": ["other"], timestamp: ~N[2021-01-01 00:00:00] ), build(:pageview, - referrer_source: "Google", - referrer: "google.com", + session_referrer_source: "Google", + session_referrer: "google.com", timestamp: ~N[2021-01-01 00:00:00] ), build(:pageview, - referrer_source: "Facebook", - referrer: "facebook.com", + session_referrer_source: "Facebook", + session_referrer: "facebook.com", "meta.key": ["author"], "meta.value": ["John Doe"], timestamp: ~N[2021-01-01 00:00:00] @@ -149,8 +149,8 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest do } do populate_stats(site, [ build(:pageview, - referrer_source: "DuckDuckGo", - referrer: "duckduckgo.com", + session_referrer_source: "DuckDuckGo", + session_referrer: "duckduckgo.com", "meta.key": ["author"], "meta.value": ["John Doe"], user_id: 123, @@ -161,20 +161,20 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest do timestamp: ~N[2021-01-01 00:01:00] ), build(:pageview, - referrer_source: "Google", - referrer: "google.com", + session_referrer_source: "Google", + session_referrer: "google.com", "meta.key": ["author"], "meta.value": ["other"], timestamp: ~N[2021-01-01 00:00:00] ), build(:pageview, - referrer_source: "Facebook", - referrer: "facebook.com", + session_referrer_source: "Facebook", + session_referrer: "facebook.com", timestamp: ~N[2021-01-01 00:00:00] ), build(:pageview, - referrer_source: "Facebook", - referrer: "facebook.com", + session_referrer_source: "Facebook", + session_referrer: "facebook.com", timestamp: ~N[2021-01-01 00:00:00] ) ]) @@ -199,8 +199,8 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest do } do populate_stats(site, [ build(:pageview, - referrer_source: "DuckDuckGo", - referrer: "duckduckgo.com", + session_referrer_source: "DuckDuckGo", + session_referrer: "duckduckgo.com", "meta.key": ["logged_in"], "meta.value": ["true"], user_id: 123, @@ -213,22 +213,22 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest do timestamp: ~N[2021-01-01 00:01:00] ), build(:pageview, - referrer_source: "Google", - referrer: "google.com", + session_referrer_source: "Google", + session_referrer: "google.com", "meta.key": ["author"], "meta.value": ["other"], timestamp: ~N[2021-01-01 00:00:00] ), build(:pageview, - referrer_source: "Google", - referrer: "google.com", + session_referrer_source: "Google", + session_referrer: "google.com", "meta.key": ["author"], "meta.value": ["another"], timestamp: ~N[2021-01-01 00:00:00] ), build(:pageview, - referrer_source: "Facebook", - referrer: "facebook.com", + session_referrer_source: "Facebook", + session_referrer: "facebook.com", timestamp: ~N[2021-01-01 00:00:00] ) ]) @@ -249,9 +249,12 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest do test "returns top sources with imported data", %{conn: conn, site: site} do populate_stats(site, [ - build(:pageview, referrer_source: "Google", referrer: "google.com"), - build(:pageview, referrer_source: "Google", referrer: "google.com"), - build(:pageview, referrer_source: "DuckDuckGo", referrer: "duckduckgo.com") + build(:pageview, session_referrer_source: "Google", session_referrer: "google.com"), + build(:pageview, session_referrer_source: "Google", session_referrer: "google.com"), + build(:pageview, + session_referrer_source: "DuckDuckGo", + session_referrer: "duckduckgo.com" + ) ]) populate_stats(site, [ @@ -283,20 +286,20 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest do test "calculates bounce rate and visit duration for sources", %{conn: conn, site: site} do populate_stats(site, [ build(:pageview, - referrer_source: "Google", - referrer: "google.com", + session_referrer_source: "Google", + session_referrer: "google.com", user_id: @user_id, timestamp: ~N[2021-01-01 00:00:00] ), build(:pageview, - referrer_source: "Google", - referrer: "google.com", + session_referrer_source: "Google", + session_referrer: "google.com", user_id: @user_id, timestamp: ~N[2021-01-01 00:15:00] ), build(:pageview, - referrer_source: "DuckDuckGo", - referrer: "duckduckgo.com", + session_referrer_source: "DuckDuckGo", + session_referrer: "duckduckgo.com", timestamp: ~N[2021-01-01 00:00:00] ) ]) @@ -329,20 +332,20 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest do } do populate_stats(site, [ build(:pageview, - referrer_source: "Google", - referrer: "google.com", + session_referrer_source: "Google", + session_referrer: "google.com", user_id: @user_id, timestamp: ~N[2021-01-01 00:00:00] ), build(:pageview, - referrer_source: "Google", - referrer: "google.com", + session_referrer_source: "Google", + session_referrer: "google.com", user_id: @user_id, timestamp: ~N[2021-01-01 00:15:00] ), build(:pageview, - referrer_source: "DuckDuckGo", - referrer: "duckduckgo.com", + session_referrer_source: "DuckDuckGo", + session_referrer: "duckduckgo.com", timestamp: ~N[2021-01-01 00:00:00] ) ]) @@ -412,18 +415,18 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest do test "returns top sources in realtime report", %{conn: conn, site: site} do populate_stats(site, [ build(:pageview, - referrer_source: "Google", - referrer: "google.com", + session_referrer_source: "Google", + session_referrer: "google.com", timestamp: relative_time(minutes: -3) ), build(:pageview, - referrer_source: "Google", - referrer: "google.com", + session_referrer_source: "Google", + session_referrer: "google.com", timestamp: relative_time(minutes: -2) ), build(:pageview, - referrer_source: "DuckDuckGo", - referrer: "duckduckgo.com", + session_referrer_source: "DuckDuckGo", + session_referrer: "duckduckgo.com", timestamp: relative_time(minutes: -1) ) ]) @@ -439,16 +442,16 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest do test "can paginate the results", %{conn: conn, site: site} do populate_stats(site, [ build(:pageview, - referrer_source: "Google", - referrer: "google.com" + session_referrer_source: "Google", + session_referrer: "google.com" ), build(:pageview, - referrer_source: "Google", - referrer: "google.com" + session_referrer_source: "Google", + session_referrer: "google.com" ), build(:pageview, - referrer_source: "DuckDuckGo", - referrer: "duckduckgo.com" + session_referrer_source: "DuckDuckGo", + session_referrer: "duckduckgo.com" ), build(:imported_sources, source: "DuckDuckGo" @@ -470,10 +473,18 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest do test "shows sources for a page", %{conn: conn, site: site} do populate_stats(site, [ - build(:pageview, pathname: "/page1", referrer_source: "Google"), - build(:pageview, pathname: "/page1", referrer_source: "Google"), - build(:pageview, user_id: 1, pathname: "/page2", referrer_source: "DuckDuckGo"), - build(:pageview, user_id: 1, pathname: "/page1", referrer_source: "DuckDuckGo") + build(:pageview, pathname: "/page1", session_referrer_source: "Google"), + build(:pageview, pathname: "/page1", session_referrer_source: "Google"), + build(:pageview, + user_id: 1, + pathname: "/page2", + session_referrer_source: "DuckDuckGo" + ), + build(:pageview, + user_id: 1, + pathname: "/page1", + session_referrer_source: "DuckDuckGo" + ) ]) filters = Jason.encode!(%{"page" => "/page1"}) @@ -492,17 +503,17 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest do test "returns top utm_mediums by unique user ids", %{conn: conn, site: site} do populate_stats(site, [ build(:pageview, - utm_medium: "social", + session_utm_medium: "social", user_id: @user_id, timestamp: ~N[2021-01-01 00:00:00] ), build(:pageview, - utm_medium: "social", + session_utm_medium: "social", user_id: @user_id, timestamp: ~N[2021-01-01 00:15:00] ), build(:pageview, - utm_medium: "email", + session_utm_medium: "email", timestamp: ~N[2021-01-01 00:00:00] ) ]) @@ -572,17 +583,17 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest do test "filters out entries without utm_medium present", %{conn: conn, site: site} do populate_stats(site, [ build(:pageview, - utm_medium: "social", + session_utm_medium: "social", user_id: @user_id, timestamp: ~N[2021-01-01 00:00:00] ), build(:pageview, - utm_medium: "social", + session_utm_medium: "social", user_id: @user_id, timestamp: ~N[2021-01-01 00:15:00] ), build(:pageview, - utm_medium: "", + session_utm_medium: "", timestamp: ~N[2021-01-01 00:00:00] ) ]) @@ -644,21 +655,21 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest do test "returns top utm_campaigns by unique user ids", %{conn: conn, site: site} do populate_stats(site, [ build(:pageview, - utm_campaign: "profile", + session_utm_campaign: "profile", user_id: @user_id, timestamp: ~N[2021-01-01 00:00:00] ), build(:pageview, - utm_campaign: "profile", + session_utm_campaign: "profile", user_id: @user_id, timestamp: ~N[2021-01-01 00:15:00] ), build(:pageview, - utm_campaign: "august", + session_utm_campaign: "august", timestamp: ~N[2021-01-01 00:00:00] ), build(:pageview, - utm_campaign: "august", + session_utm_campaign: "august", timestamp: ~N[2021-01-01 00:00:00] ) ]) @@ -728,21 +739,21 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest do test "filters out entries without utm_campaign present", %{conn: conn, site: site} do populate_stats(site, [ build(:pageview, - utm_campaign: "profile", + session_utm_campaign: "profile", user_id: @user_id, timestamp: ~N[2021-01-01 00:00:00] ), build(:pageview, - utm_campaign: "profile", + session_utm_campaign: "profile", user_id: @user_id, timestamp: ~N[2021-01-01 00:15:00] ), build(:pageview, - utm_campaign: "", + session_utm_campaign: "", timestamp: ~N[2021-01-01 00:00:00] ), build(:pageview, - utm_campaign: "", + session_utm_campaign: "", timestamp: ~N[2021-01-01 00:00:00] ) ]) @@ -804,21 +815,21 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest do test "returns top utm_sources by unique user ids", %{conn: conn, site: site} do populate_stats(site, [ build(:pageview, - utm_source: "Twitter", + session_utm_source: "Twitter", user_id: @user_id, timestamp: ~N[2021-01-01 00:00:00] ), build(:pageview, - utm_source: "Twitter", + session_utm_source: "Twitter", user_id: @user_id, timestamp: ~N[2021-01-01 00:15:00] ), build(:pageview, - utm_source: "newsletter", + session_utm_source: "newsletter", timestamp: ~N[2021-01-01 00:00:00] ), build(:pageview, - utm_source: "newsletter", + session_utm_source: "newsletter", timestamp: ~N[2021-01-01 00:00:00] ) ]) @@ -852,21 +863,21 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest do test "returns top utm_terms by unique user ids", %{conn: conn, site: site} do populate_stats(site, [ build(:pageview, - utm_term: "oat milk", + session_utm_term: "oat milk", user_id: @user_id, timestamp: ~N[2021-01-01 00:00:00] ), build(:pageview, - utm_term: "oat milk", + session_utm_term: "oat milk", user_id: @user_id, timestamp: ~N[2021-01-01 00:15:00] ), build(:pageview, - utm_term: "Sweden", + session_utm_term: "Sweden", timestamp: ~N[2021-01-01 00:00:00] ), build(:pageview, - utm_term: "Sweden", + session_utm_term: "Sweden", timestamp: ~N[2021-01-01 00:00:00] ) ]) @@ -936,21 +947,21 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest do test "filters out entries without utm_term present", %{conn: conn, site: site} do populate_stats(site, [ build(:pageview, - utm_term: "oat milk", + session_utm_term: "oat milk", user_id: @user_id, timestamp: ~N[2021-01-01 00:00:00] ), build(:pageview, - utm_term: "oat milk", + session_utm_term: "oat milk", user_id: @user_id, timestamp: ~N[2021-01-01 00:15:00] ), build(:pageview, - utm_term: "", + session_utm_term: "", timestamp: ~N[2021-01-01 00:00:00] ), build(:pageview, - utm_term: "", + session_utm_term: "", timestamp: ~N[2021-01-01 00:00:00] ) ]) @@ -1012,21 +1023,21 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest do test "returns top utm_contents by unique user ids", %{conn: conn, site: site} do populate_stats(site, [ build(:pageview, - utm_content: "ad", + session_utm_content: "ad", user_id: @user_id, timestamp: ~N[2021-01-01 00:00:00] ), build(:pageview, - utm_content: "ad", + session_utm_content: "ad", user_id: @user_id, timestamp: ~N[2021-01-01 00:15:00] ), build(:pageview, - utm_content: "blog", + session_utm_content: "blog", timestamp: ~N[2021-01-01 00:00:00] ), build(:pageview, - utm_content: "blog", + session_utm_content: "blog", timestamp: ~N[2021-01-01 00:00:00] ) ]) @@ -1096,21 +1107,21 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest do test "filters out entries without utm_content present", %{conn: conn, site: site} do populate_stats(site, [ build(:pageview, - utm_content: "ad", + session_utm_content: "ad", user_id: @user_id, timestamp: ~N[2021-01-01 00:00:00] ), build(:pageview, - utm_content: "ad", + session_utm_content: "ad", user_id: @user_id, timestamp: ~N[2021-01-01 00:15:00] ), build(:pageview, - utm_content: "", + session_utm_content: "", timestamp: ~N[2021-01-01 00:00:00] ), build(:pageview, - utm_content: "", + session_utm_content: "", timestamp: ~N[2021-01-01 00:00:00] ) ]) @@ -1175,7 +1186,7 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest do } do populate_stats(site, [ build(:pageview, - referrer_source: "Twitter", + session_referrer_source: "Twitter", user_id: @user_id ), build(:event, @@ -1183,7 +1194,7 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest do user_id: @user_id ), build(:pageview, - referrer_source: "Twitter" + session_referrer_source: "Twitter" ) ]) @@ -1213,8 +1224,8 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest do test "returns top referrers with goal filter + :is prop filter", %{conn: conn, site: site} do populate_stats(site, [ build(:pageview, - referrer_source: "DuckDuckGo", - referrer: "duckduckgo.com", + session_referrer_source: "DuckDuckGo", + session_referrer: "duckduckgo.com", user_id: 123, timestamp: ~N[2021-01-01 00:00:00] ), @@ -1226,15 +1237,15 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest do timestamp: ~N[2021-01-01 00:01:00] ), build(:pageview, - referrer_source: "DuckDuckGo", - referrer: "duckduckgo.com", + session_referrer_source: "DuckDuckGo", + session_referrer: "duckduckgo.com", "meta.key": ["logged_in"], "meta.value": ["true"], timestamp: ~N[2021-01-01 00:00:00] ), build(:event, - referrer_source: "Facebook", - referrer: "facebook.com", + session_referrer_source: "Facebook", + session_referrer: "facebook.com", name: "Download", "meta.key": ["method", "logged_in"], "meta.value": ["HTTP", "false"], @@ -1263,8 +1274,8 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest do test "returns top referrers with goal filter + prop :is_not filter", %{conn: conn, site: site} do populate_stats(site, [ build(:pageview, - referrer_source: "DuckDuckGo", - referrer: "duckduckgo.com", + session_referrer_source: "DuckDuckGo", + session_referrer: "duckduckgo.com", user_id: 123, timestamp: ~N[2021-01-01 00:00:00] ), @@ -1277,16 +1288,16 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest do ), build(:event, name: "Download", - referrer_source: "Google", - referrer: "google.com", + session_referrer_source: "Google", + session_referrer: "google.com", "meta.key": ["method", "logged_in"], "meta.value": ["HTTP", "true"], timestamp: ~N[2021-01-01 00:00:00] ), build(:event, name: "Download", - referrer_source: "Google", - referrer: "google.com", + session_referrer_source: "Google", + session_referrer: "google.com", "meta.key": ["method", "logged_in"], "meta.value": ["HTTP", "false"], timestamp: ~N[2021-01-01 00:00:00] @@ -1323,7 +1334,7 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest do } do populate_stats(site, [ build(:pageview, - referrer_source: "Twitter", + session_referrer_source: "Twitter", user_id: @user_id ), build(:pageview, @@ -1331,7 +1342,7 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest do user_id: @user_id ), build(:pageview, - referrer_source: "Twitter" + session_referrer_source: "Twitter" ) ]) @@ -1360,20 +1371,20 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest do test "returns top referrers for a particular source", %{conn: conn, site: site} do populate_stats(site, [ build(:pageview, - referrer_source: "10words", - referrer: "10words.com" + session_referrer_source: "10words", + session_referrer: "10words.com" ), build(:pageview, - referrer_source: "10words", - referrer: "10words.com" + session_referrer_source: "10words", + session_referrer: "10words.com" ), build(:pageview, - referrer_source: "10words", - referrer: "10words.com/page1" + session_referrer_source: "10words", + session_referrer: "10words.com/page1" ), build(:pageview, - referrer_source: "ignored", - referrer: "ignored" + session_referrer_source: "ignored", + session_referrer: "ignored" ) ]) @@ -1392,25 +1403,25 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest do test "calculates bounce rate and visit duration for referrer urls", %{conn: conn, site: site} do populate_stats(site, [ build(:pageview, - referrer_source: "10words", - referrer: "10words.com", + session_referrer_source: "10words", + session_referrer: "10words.com", user_id: @user_id, timestamp: ~N[2021-01-01 00:00:00] ), build(:pageview, - referrer_source: "10words", - referrer: "10words.com", + session_referrer_source: "10words", + session_referrer: "10words.com", user_id: @user_id, timestamp: ~N[2021-01-01 00:15:00] ), build(:pageview, - referrer_source: "10words", - referrer: "10words.com", + session_referrer_source: "10words", + session_referrer: "10words.com", timestamp: ~N[2021-01-01 00:15:00] ), build(:pageview, - referrer_source: "ignored", - referrer: "ignored", + session_referrer_source: "ignored", + session_referrer: "ignored", timestamp: ~N[2021-01-01 00:00:00] ) ]) @@ -1436,16 +1447,16 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest do populate_stats(site, [ build(:pageview, - referrer_source: "DuckDuckGo", - referrer: "duckduckgo.com" + session_referrer_source: "DuckDuckGo", + session_referrer: "duckduckgo.com" ), build(:pageview, - referrer_source: "Google", - referrer: "google.com" + session_referrer_source: "Google", + session_referrer: "google.com" ), build(:pageview, - referrer_source: "Google", - referrer: "google.com" + session_referrer_source: "Google", + session_referrer: "google.com" ) ]) @@ -1464,16 +1475,16 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest do } do populate_stats(site, [ build(:pageview, - referrer_source: "DuckDuckGo", - referrer: "duckduckgo.com" + session_referrer_source: "DuckDuckGo", + session_referrer: "duckduckgo.com" ), build(:pageview, - referrer_source: "Google", - referrer: "google.com" + session_referrer_source: "Google", + session_referrer: "google.com" ), build(:pageview, - referrer_source: "Google", - referrer: "google.com" + session_referrer_source: "Google", + session_referrer: "google.com" ) ]) @@ -1493,12 +1504,12 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest do test "returns top referring urls for a custom goal", %{conn: conn, site: site} do populate_stats(site, [ build(:pageview, - referrer_source: "10words", - referrer: "10words.com" + session_referrer_source: "10words", + session_referrer: "10words.com" ), build(:pageview, - referrer_source: "10words", - referrer: "10words.com", + session_referrer_source: "10words", + session_referrer: "10words.com", user_id: @user_id ), build(:event, @@ -1531,12 +1542,12 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest do test "returns top referring urls for a pageview goal", %{conn: conn, site: site} do populate_stats(site, [ build(:pageview, - referrer_source: "10words", - referrer: "10words.com" + session_referrer_source: "10words", + session_referrer: "10words.com" ), build(:pageview, - referrer_source: "10words", - referrer: "10words.com", + session_referrer_source: "10words", + session_referrer: "10words.com", user_id: @user_id ), build(:pageview, diff --git a/test/plausible_web/controllers/api/stats_controller/suggestions_test.exs b/test/plausible_web/controllers/api/stats_controller/suggestions_test.exs index 92e6ad908..8d4c86a5e 100644 --- a/test/plausible_web/controllers/api/stats_controller/suggestions_test.exs +++ b/test/plausible_web/controllers/api/stats_controller/suggestions_test.exs @@ -57,9 +57,12 @@ defmodule PlausibleWeb.Api.StatsController.SuggestionsTest do test "returns suggestions for sources", %{conn: conn, site: site} do populate_stats(site, [ - build(:pageview, timestamp: ~N[2019-01-01 23:00:00], referrer_source: "Bing"), - build(:pageview, timestamp: ~N[2019-01-01 23:00:00], referrer_source: "Bing"), - build(:pageview, timestamp: ~N[2019-01-01 23:00:00], referrer_source: "10words") + build(:pageview, timestamp: ~N[2019-01-01 23:00:00], session_referrer_source: "Bing"), + build(:pageview, timestamp: ~N[2019-01-01 23:00:00], session_referrer_source: "Bing"), + build(:pageview, + timestamp: ~N[2019-01-01 23:00:00], + session_referrer_source: "10words" + ) ]) conn = @@ -73,7 +76,11 @@ defmodule PlausibleWeb.Api.StatsController.SuggestionsTest do test "returns suggestions for countries", %{conn: conn, site: site} do populate_stats(site, [ - build(:pageview, timestamp: ~N[2019-01-01 23:00:01], pathname: "/", country_code: "US") + build(:pageview, + timestamp: ~N[2019-01-01 23:00:01], + pathname: "/", + session_country_code: "US" + ) ]) conn = @@ -89,8 +96,8 @@ defmodule PlausibleWeb.Api.StatsController.SuggestionsTest do {:ok, [site: site]} = create_new_site(%{user: user}) populate_stats(site, [ - build(:pageview, country_code: "EE", subdivision1_code: "EE-37"), - build(:pageview, country_code: "EE", subdivision1_code: "EE-39") + build(:pageview, session_country_code: "EE", session_subdivision1_code: "EE-37"), + build(:pageview, session_country_code: "EE", session_subdivision1_code: "EE-39") ]) conn = @@ -106,8 +113,16 @@ defmodule PlausibleWeb.Api.StatsController.SuggestionsTest do {:ok, [site: site]} = create_new_site(%{user: user}) populate_stats(site, [ - build(:pageview, country_code: "EE", subdivision1_code: "EE-37", city_geoname_id: 588_409), - build(:pageview, country_code: "EE", subdivision1_code: "EE-39", city_geoname_id: 591_632) + build(:pageview, + session_country_code: "EE", + session_subdivision1_code: "EE-37", + session_city_geoname_id: 588_409 + ), + build(:pageview, + session_country_code: "EE", + session_subdivision1_code: "EE-39", + session_city_geoname_id: 591_632 + ) ]) conn = @@ -134,7 +149,7 @@ defmodule PlausibleWeb.Api.StatsController.SuggestionsTest do build(:pageview, timestamp: ~N[2019-01-01 23:00:00], pathname: "/", - screen_size: "Desktop" + session_screen_size: "Desktop" ) ]) @@ -146,7 +161,11 @@ defmodule PlausibleWeb.Api.StatsController.SuggestionsTest do test "returns suggestions for browsers", %{conn: conn, site: site} do populate_stats(site, [ - build(:pageview, timestamp: ~N[2019-01-01 23:00:00], pathname: "/", browser: "Chrome") + build(:pageview, + timestamp: ~N[2019-01-01 23:00:00], + pathname: "/", + session_browser: "Chrome" + ) ]) conn = @@ -161,8 +180,8 @@ defmodule PlausibleWeb.Api.StatsController.SuggestionsTest do populate_stats(site, [ build(:pageview, timestamp: ~N[2019-01-01 00:00:00], - browser: "Chrome", - browser_version: "78.0" + session_browser: "Chrome", + session_browser_version: "78.0" ) ]) @@ -177,7 +196,7 @@ defmodule PlausibleWeb.Api.StatsController.SuggestionsTest do test "returns suggestions for OS", %{conn: conn, site: site} do populate_stats(site, [ - build(:pageview, timestamp: ~N[2019-01-01 00:00:00], operating_system: "Mac") + build(:pageview, timestamp: ~N[2019-01-01 00:00:00], session_operating_system: "Mac") ]) conn = get(conn, "/api/stats/#{site.domain}/suggestions/os?period=month&date=2019-01-01") @@ -191,8 +210,8 @@ defmodule PlausibleWeb.Api.StatsController.SuggestionsTest do populate_stats(site, [ build(:pageview, timestamp: ~N[2019-01-01 00:00:00], - operating_system: "Mac", - operating_system_version: "10.15" + session_operating_system: "Mac", + session_operating_system_version: "10.15" ) ]) @@ -222,7 +241,7 @@ defmodule PlausibleWeb.Api.StatsController.SuggestionsTest do build(:pageview, timestamp: ~N[2019-01-01 23:00:00], pathname: "/", - referrer: "10words.com/page1" + session_referrer: "10words.com/page1" ) ]) diff --git a/test/plausible_web/controllers/api/stats_controller/top_stats_test.exs b/test/plausible_web/controllers/api/stats_controller/top_stats_test.exs index 380a932ee..8d344083e 100644 --- a/test/plausible_web/controllers/api/stats_controller/top_stats_test.exs +++ b/test/plausible_web/controllers/api/stats_controller/top_stats_test.exs @@ -511,9 +511,9 @@ defmodule PlausibleWeb.Api.StatsController.TopStatsTest do test "returns only visitors from a country based on alpha2 code", %{conn: conn, site: site} do populate_stats(site, [ - build(:pageview, country_code: "US"), - build(:pageview, country_code: "US"), - build(:pageview, country_code: "EE") + build(:pageview, session_country_code: "US"), + build(:pageview, session_country_code: "US"), + build(:pageview, session_country_code: "EE") ]) filters = Jason.encode!(%{country: "US"}) @@ -571,9 +571,9 @@ defmodule PlausibleWeb.Api.StatsController.TopStatsTest do test "returns only visitors with specific screen size", %{conn: conn, site: site} do populate_stats(site, [ - build(:pageview, screen_size: "Desktop"), - build(:pageview, screen_size: "Desktop"), - build(:pageview, screen_size: "Mobile") + build(:pageview, session_screen_size: "Desktop"), + build(:pageview, session_screen_size: "Desktop"), + build(:pageview, session_screen_size: "Mobile") ]) filters = Jason.encode!(%{screen: "Desktop"}) @@ -591,9 +591,9 @@ defmodule PlausibleWeb.Api.StatsController.TopStatsTest do test "returns only visitors with specific browser", %{conn: conn, site: site} do populate_stats(site, [ - build(:pageview, browser: "Chrome"), - build(:pageview, browser: "Chrome"), - build(:pageview, browser: "Safari") + build(:pageview, session_browser: "Chrome"), + build(:pageview, session_browser: "Chrome"), + build(:pageview, session_browser: "Safari") ]) filters = Jason.encode!(%{browser: "Chrome"}) @@ -611,9 +611,9 @@ defmodule PlausibleWeb.Api.StatsController.TopStatsTest do test "returns only visitors with specific operating system", %{conn: conn, site: site} do populate_stats(site, [ - build(:pageview, operating_system: "Mac"), - build(:pageview, operating_system: "Mac"), - build(:pageview, operating_system: "Windows") + build(:pageview, session_operating_system: "Mac"), + build(:pageview, session_operating_system: "Mac"), + build(:pageview, session_operating_system: "Windows") ]) filters = Jason.encode!(%{os: "Mac"}) @@ -633,17 +633,17 @@ defmodule PlausibleWeb.Api.StatsController.TopStatsTest do populate_stats(site, [ build(:pageview, user_id: @user_id, - referrer_source: "Google", + session_referrer_source: "Google", timestamp: ~N[2021-01-01 00:00:00] ), build(:pageview, user_id: @user_id, - referrer_source: "Google", + session_referrer_source: "Google", timestamp: ~N[2021-01-01 00:05:00] ), build(:pageview, user_id: @user_id, - referrer_source: "Google", + session_referrer_source: "Google", timestamp: ~N[2021-01-01 05:00:00] ), build(:pageview, timestamp: ~N[2021-01-01 00:10:00]) diff --git a/test/plausible_web/controllers/stats_controller_test.exs b/test/plausible_web/controllers/stats_controller_test.exs index 963368d57..2b1859c9f 100644 --- a/test/plausible_web/controllers/stats_controller_test.exs +++ b/test/plausible_web/controllers/stats_controller_test.exs @@ -292,12 +292,18 @@ defmodule PlausibleWeb.StatsControllerTest do test "exports operating system versions", %{conn: conn, site: site} do populate_stats(site, [ - build(:pageview, operating_system: "Mac", operating_system_version: "14"), - build(:pageview, operating_system: "Mac", operating_system_version: "14"), - build(:pageview, operating_system: "Mac", operating_system_version: "14"), - build(:pageview, operating_system: "Ubuntu", operating_system_version: "20.04"), - build(:pageview, operating_system: "Ubuntu", operating_system_version: "20.04"), - build(:pageview, operating_system: "Mac", operating_system_version: "13") + build(:pageview, session_operating_system: "Mac", session_operating_system_version: "14"), + build(:pageview, session_operating_system: "Mac", session_operating_system_version: "14"), + build(:pageview, session_operating_system: "Mac", session_operating_system_version: "14"), + build(:pageview, + session_operating_system: "Ubuntu", + session_operating_system_version: "20.04" + ), + build(:pageview, + session_operating_system: "Ubuntu", + session_operating_system_version: "20.04" + ), + build(:pageview, session_operating_system: "Mac", session_operating_system_version: "13") ]) conn = get(conn, "/#{site.domain}/export") @@ -412,56 +418,56 @@ defmodule PlausibleWeb.StatsControllerTest do defp populate_exported_stats(site) do populate_stats(site, [ build(:pageview, - country_code: "EE", - subdivision1_code: "EE-37", - city_geoname_id: 588_409, + user_id: 123, pathname: "/", timestamp: Timex.shift(~N[2021-10-20 12:00:00], minutes: -1) |> NaiveDateTime.truncate(:second), - referrer_source: "Google", - user_id: 123 + session_country_code: "EE", + session_subdivision1_code: "EE-37", + session_city_geoname_id: 588_409, + session_referrer_source: "Google" ), build(:pageview, - country_code: "EE", - subdivision1_code: "EE-37", - city_geoname_id: 588_409, + user_id: 123, pathname: "/some-other-page", timestamp: Timex.shift(~N[2021-10-20 12:00:00], minutes: -2) |> NaiveDateTime.truncate(:second), - referrer_source: "Google", - user_id: 123 + session_country_code: "EE", + session_subdivision1_code: "EE-37", + session_city_geoname_id: 588_409, + session_referrer_source: "Google" ), build(:pageview, pathname: "/", - utm_medium: "search", - utm_campaign: "ads", - utm_source: "google", - utm_content: "content", - utm_term: "term", timestamp: Timex.shift(~N[2021-10-20 12:00:00], days: -1) |> NaiveDateTime.truncate(:second), - browser: "Firefox", - browser_version: "120", - operating_system: "Mac", - operating_system_version: "14" + session_utm_medium: "search", + session_utm_campaign: "ads", + session_utm_source: "google", + session_utm_content: "content", + session_utm_term: "term", + session_browser: "Firefox", + session_browser_version: "120", + session_operating_system: "Mac", + session_operating_system_version: "14" ), build(:pageview, timestamp: Timex.shift(~N[2021-10-20 12:00:00], months: -1) |> NaiveDateTime.truncate(:second), - country_code: "EE", - browser: "Firefox", - browser_version: "120", - operating_system: "Mac", - operating_system_version: "14" + session_country_code: "EE", + session_browser: "Firefox", + session_browser_version: "120", + session_operating_system: "Mac", + session_operating_system_version: "14" ), build(:pageview, timestamp: Timex.shift(~N[2021-10-20 12:00:00], months: -5) |> NaiveDateTime.truncate(:second), - utm_campaign: "ads", - country_code: "EE", - referrer_source: "Google", - browser: "FirefoxNoVersion", - operating_system: "MacNoVersion" + session_utm_campaign: "ads", + session_country_code: "EE", + session_referrer_source: "Google", + session_browser: "FirefoxNoVersion", + session_operating_system: "MacNoVersion" ), build(:event, timestamp: @@ -519,24 +525,36 @@ defmodule PlausibleWeb.StatsControllerTest do site: site } do populate_stats(site, [ - build(:pageview, operating_system: "Mac", operating_system_version: "14"), - build(:event, name: "Signup", operating_system: "Mac", operating_system_version: "14"), - build(:event, name: "Signup", operating_system: "Mac", operating_system_version: "14"), - build(:event, name: "Signup", operating_system: "Mac", operating_system_version: "14"), + build(:pageview, session_operating_system: "Mac", session_operating_system_version: "14"), + build(:event, + name: "Signup", + session_operating_system: "Mac", + session_operating_system_version: "14" + ), + build(:event, + name: "Signup", + session_operating_system: "Mac", + session_operating_system_version: "14" + ), + build(:event, + name: "Signup", + session_operating_system: "Mac", + session_operating_system_version: "14" + ), build(:event, name: "Signup", - operating_system: "Ubuntu", - operating_system_version: "20.04" + session_operating_system: "Ubuntu", + session_operating_system_version: "20.04" ), build(:event, name: "Signup", - operating_system: "Ubuntu", - operating_system_version: "20.04" + session_operating_system: "Ubuntu", + session_operating_system_version: "20.04" ), build(:event, name: "Signup", - operating_system: "Lubuntu", - operating_system_version: "20.04" + session_operating_system: "Lubuntu", + session_operating_system_version: "20.04" ) ]) diff --git a/test/support/factory.ex b/test/support/factory.ex index afe3182df..090ca55d3 100644 --- a/test/support/factory.ex +++ b/test/support/factory.ex @@ -83,24 +83,20 @@ defmodule Plausible.Factory do end def pageview_factory do - struct!( - event_factory(), - %{ - name: "pageview" - } - ) + Map.put(event_factory(), :name, "pageview") end def event_factory do hostname = sequence(:domain, &"example-#{&1}.com") - %Plausible.ClickhouseEventV2{ + %{ hostname: hostname, site_id: Enum.random(1000..10_000), pathname: "/", timestamp: NaiveDateTime.utc_now() |> NaiveDateTime.truncate(:second), user_id: SipHash.hash!(hash_key(), Ecto.UUID.generate()), - session_id: SipHash.hash!(hash_key(), Ecto.UUID.generate()) + session_id: SipHash.hash!(hash_key(), Ecto.UUID.generate()), + _factory_event: true } end diff --git a/test/support/test_utils.ex b/test/support/test_utils.ex index ac21fc29c..1890fc57b 100644 --- a/test/support/test_utils.ex +++ b/test/support/test_utils.ex @@ -82,9 +82,8 @@ defmodule Plausible.TestUtils do |> Map.put(:site_id, pageview.site.id) Factory.build(:pageview, pageview) - |> Map.from_struct() - |> Map.delete(:__meta__) |> update_in([:timestamp], &to_naive_truncate/1) + |> Map.delete(:_factory_event) end) Plausible.IngestRepo.insert_all(Plausible.ClickhouseEventV2, pageviews) @@ -94,9 +93,8 @@ defmodule Plausible.TestUtils do events = Enum.map(events, fn event -> Factory.build(:event, event) - |> Map.from_struct() - |> Map.delete(:__meta__) |> update_in([:timestamp], &to_naive_truncate/1) + |> Map.delete(:_factory_event) end) Plausible.IngestRepo.insert_all(Plausible.ClickhouseEventV2, events) @@ -150,7 +148,7 @@ defmodule Plausible.TestUtils do event = Map.put(event, :site_id, site.id) case event do - %Plausible.ClickhouseEventV2{} -> + %{_factory_event: true} -> event imported_event -> @@ -181,7 +179,7 @@ defmodule Plausible.TestUtils do end) |> Enum.split_with(fn event -> case event do - %Plausible.ClickhouseEventV2{} -> + %{_factory_event: true} -> true _ -> @@ -196,13 +194,17 @@ defmodule Plausible.TestUtils do defp populate_native_stats(events) do sessions = Enum.reduce(events, %{}, fn event, sessions -> - session_id = Plausible.Session.CacheStore.on_event(event, nil) + session_id = Plausible.Session.CacheStore.on_event(event, session_params(event), nil) Map.put(sessions, {event.site_id, event.user_id}, session_id) end) Enum.each(events, fn event -> - event = Map.put(event, :session_id, sessions[{event.site_id, event.user_id}]) - Plausible.Event.WriteBuffer.insert(event) + clickhouse_event = + %Plausible.ClickhouseEventV2{} + |> Map.merge(event) + |> Map.put(:session_id, sessions[{event.site_id, event.user_id}]) + + Plausible.Event.WriteBuffer.insert(clickhouse_event) end) Plausible.Session.WriteBuffer.flush() @@ -214,6 +216,24 @@ defmodule Plausible.TestUtils do |> Enum.map(fn {table, events} -> Plausible.Imported.Buffer.insert_all(table, events) end) end + defp session_params(event) do + event + |> Enum.reduce(%{}, fn {key, value}, acc -> + str_key = Atom.to_string(key) + + if String.starts_with?(str_key, "session_") and key != :session_id do + session_key = + str_key + |> String.trim("session_") + |> String.to_existing_atom() + + Map.put(acc, session_key, value) + else + acc + end + end) + end + def relative_time(shifts) do NaiveDateTime.utc_now() |> Timex.shift(shifts) diff --git a/test/workers/send_email_report_test.exs b/test/workers/send_email_report_test.exs index f5058854a..7e4bf79d4 100644 --- a/test/workers/send_email_report_test.exs +++ b/test/workers/send_email_report_test.exs @@ -79,9 +79,9 @@ defmodule Plausible.Workers.SendEmailReportTest do populate_stats(site, [ build(:pageview, - referrer_source: "Google", user_id: 123, - timestamp: Timex.shift(now, days: -7) + timestamp: Timex.shift(now, days: -7), + session_referrer_source: "Google" ), build(:pageview, user_id: 123, timestamp: Timex.shift(now, days: -7)), build(:pageview, timestamp: Timex.shift(now, days: -7))