diff --git a/lib/plausible/session/clickhouse_schema.ex b/lib/plausible/session/clickhouse_schema.ex index 348984319..3645fd0f9 100644 --- a/lib/plausible/session/clickhouse_schema.ex +++ b/lib/plausible/session/clickhouse_schema.ex @@ -18,8 +18,8 @@ defmodule Plausible.ClickhouseSession do field :events, :integer field :sign, :integer - # field :"entry.meta.key", {:array, :string}, default: [] - # field :"entry.meta.value", {:array, :string}, default: [] + field :"entry_meta.key", {:array, :string}, default: [] + field :"entry_meta.value", {:array, :string}, default: [] field :utm_medium, :string field :utm_source, :string diff --git a/lib/plausible/session/store.ex b/lib/plausible/session/store.ex index 4a3f4f905..25ab936bd 100644 --- a/lib/plausible/session/store.ex +++ b/lib/plausible/session/store.ex @@ -160,9 +160,9 @@ defmodule Plausible.Session.Store do browser: event.browser, browser_version: event.browser_version, timestamp: event.timestamp, - start: event.timestamp - # "entry.meta.key": Map.get(event, :"meta.key"), - # "entry.meta.value": Map.get(event, :"meta.value") + start: event.timestamp, + "entry_meta.key": Map.get(event, :"meta.key"), + "entry_meta.value": Map.get(event, :"meta.value") } end diff --git a/lib/plausible/stats/base.ex b/lib/plausible/stats/base.ex index 0f65815d5..c9b70223b 100644 --- a/lib/plausible/stats/base.ex +++ b/lib/plausible/stats/base.ex @@ -184,13 +184,13 @@ defmodule Plausible.Stats.Base do {:is, "(none)"} -> from( s in sessions_q, - where: fragment("not has(?, ?)", field(s, :"entry.meta.key"), ^prop_name) + where: fragment("not has(?, ?)", field(s, :"entry_meta.key"), ^prop_name) ) {:is, value} -> from( s in sessions_q, - inner_lateral_join: meta in "entry.meta", + inner_lateral_join: meta in "entry_meta", as: :meta, where: meta.key == ^prop_name and meta.value == ^value ) @@ -198,17 +198,17 @@ defmodule Plausible.Stats.Base do {:is_not, "(none)"} -> from( s in sessions_q, - where: fragment("has(?, ?)", field(s, :"entry.meta.key"), ^prop_name) + where: fragment("has(?, ?)", field(s, :"entry_meta.key"), ^prop_name) ) {:is_not, value} -> from( s in sessions_q, - left_lateral_join: meta in "entry.meta", + left_lateral_join: meta in "entry_meta", as: :meta, where: (meta.key == ^prop_name and meta.value != ^value) or - fragment("not has(?, ?)", field(s, :"entry.meta.key"), ^prop_name) + fragment("not has(?, ?)", field(s, :"entry_meta.key"), ^prop_name) ) _ -> diff --git a/priv/clickhouse_repo/migrations/20220422075510_add_entry_props.exs b/priv/clickhouse_repo/migrations/20220422075510_add_entry_props.exs new file mode 100644 index 000000000..a2907a70a --- /dev/null +++ b/priv/clickhouse_repo/migrations/20220422075510_add_entry_props.exs @@ -0,0 +1,9 @@ +defmodule Plausible.ClickhouseRepo.Migrations.AddEntryProps do + use Ecto.Migration + + def change do + alter table(:sessions) do + add(:entry_meta, {:nested, {{:key, :string}, {:value, :string}}}) + end + end +end diff --git a/test/plausible/site/admin_test.exs b/test/plausible/site/admin_test.exs index f5cd92538..2d4b1541f 100644 --- a/test/plausible/site/admin_test.exs +++ b/test/plausible/site/admin_test.exs @@ -66,7 +66,7 @@ defmodule Plausible.SiteAdminTest do actual = SiteAdmin.session_transfer_query("from.com", "to.com") expected = - "INSERT INTO sessions (browser, browser_version, city_geoname_id, country_code, domain, duration, entry_page, events, exit_page, hostname, is_bounce, operating_system, operating_system_version, pageviews, referrer, referrer_source, screen_size, session_id, sign, start, subdivision1_code, subdivision2_code, timestamp, transferred_from, user_id, utm_campaign, utm_content, utm_medium, utm_source, utm_term) SELECT browser, browser_version, city_geoname_id, country_code, 'to.com' as domain, duration, entry_page, events, exit_page, hostname, is_bounce, operating_system, operating_system_version, pageviews, referrer, referrer_source, screen_size, session_id, sign, start, subdivision1_code, subdivision2_code, timestamp, 'from.com' as transferred_from, user_id, utm_campaign, utm_content, utm_medium, utm_source, utm_term FROM (SELECT * FROM sessions WHERE domain='from.com')" + "INSERT INTO sessions (browser, browser_version, city_geoname_id, country_code, domain, duration, entry_meta.key, entry_meta.value, entry_page, events, exit_page, hostname, is_bounce, operating_system, operating_system_version, pageviews, referrer, referrer_source, screen_size, session_id, sign, start, subdivision1_code, subdivision2_code, timestamp, transferred_from, user_id, utm_campaign, utm_content, utm_medium, utm_source, utm_term) SELECT browser, browser_version, city_geoname_id, country_code, 'to.com' as domain, duration, entry_meta.key, entry_meta.value, entry_page, events, exit_page, hostname, is_bounce, operating_system, operating_system_version, pageviews, referrer, referrer_source, screen_size, session_id, sign, start, subdivision1_code, subdivision2_code, timestamp, 'from.com' as transferred_from, user_id, utm_campaign, utm_content, utm_medium, utm_source, utm_term FROM (SELECT * FROM sessions WHERE domain='from.com')" assert actual == expected end diff --git a/test/plausible_web/controllers/api/stats_controller/sources_test.exs b/test/plausible_web/controllers/api/stats_controller/sources_test.exs index e44fbe8b5..0ece9bca9 100644 --- a/test/plausible_web/controllers/api/stats_controller/sources_test.exs +++ b/test/plausible_web/controllers/api/stats_controller/sources_test.exs @@ -30,183 +30,183 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest do ] end - # test "returns top sources with :is filter on custom pageview props", %{conn: conn, site: site} do - # populate_stats(site, [ - # build(:pageview, - # referrer_source: "DuckDuckGo", - # referrer: "duckduckgo.com", - # user_id: 123, - # timestamp: ~N[2021-01-01 00:00:00] - # ), - # build(:pageview, - # "meta.key": ["author"], - # "meta.value": ["John Doe"], - # user_id: 123, - # timestamp: ~N[2021-01-01 00:01:00] - # ), - # build(:pageview, - # referrer_source: "Google", - # referrer: "google.com", - # "meta.key": ["author"], - # "meta.value": ["John Doe"], - # timestamp: ~N[2021-01-01 00:00:00] - # ), - # build(:pageview, - # referrer_source: "Facebook", - # referrer: "facebook.com", - # timestamp: ~N[2021-01-01 00:00:00] - # ) - # ]) - - # filters = Jason.encode!(%{props: %{"author" => "John Doe"}}) - - # conn = - # get( - # conn, - # "/api/stats/#{site.domain}/sources?period=day&date=2021-01-01&filters=#{filters}" - # ) - - # assert json_response(conn, 200) == [ - # %{"name" => "Google", "visitors" => 1} - # ] - # end - - # test "returns top sources with :is_not filter on custom pageview props", %{ - # conn: conn, - # site: site - # } do - # populate_stats(site, [ - # build(:pageview, - # referrer_source: "DuckDuckGo", - # referrer: "duckduckgo.com", - # "meta.key": ["author"], - # "meta.value": ["John Doe"], - # user_id: 123, - # timestamp: ~N[2021-01-01 00:00:00] - # ), - # build(:pageview, - # "meta.key": ["author"], - # "meta.value": ["other"], - # user_id: 123, - # timestamp: ~N[2021-01-01 00:01:00] - # ), - # build(:pageview, - # referrer_source: "Google", - # referrer: "google.com", - # "meta.key": ["author"], - # "meta.value": ["other"], - # timestamp: ~N[2021-01-01 00:00:00] - # ), - # build(:pageview, - # referrer_source: "Facebook", - # referrer: "facebook.com", - # timestamp: ~N[2021-01-01 00:00:00] - # ) - # ]) - - # filters = Jason.encode!(%{props: %{"author" => "!John Doe"}}) - - # conn = - # get( - # conn, - # "/api/stats/#{site.domain}/sources?period=day&date=2021-01-01&filters=#{filters}" - # ) - - # assert json_response(conn, 200) == [ - # %{"name" => "Facebook", "visitors" => 1}, - # %{"name" => "Google", "visitors" => 1} - # ] - # end - - # test "returns top sources with :is (none) filter on custom pageview props", %{ - # conn: conn, - # site: site - # } do - # populate_stats(site, [ - # build(:pageview, - # referrer_source: "DuckDuckGo", - # referrer: "duckduckgo.com", - # "meta.key": ["author"], - # "meta.value": ["John Doe"], - # user_id: 123, - # timestamp: ~N[2021-01-01 00:00:00] - # ), - # build(:pageview, - # user_id: 123, - # timestamp: ~N[2021-01-01 00:01:00] - # ), - # build(:pageview, - # referrer_source: "Google", - # referrer: "google.com", - # "meta.key": ["author"], - # "meta.value": ["other"], - # timestamp: ~N[2021-01-01 00:00:00] - # ), - # build(:pageview, - # referrer_source: "Facebook", - # referrer: "facebook.com", - # timestamp: ~N[2021-01-01 00:00:00] - # ) - # ]) - - # filters = Jason.encode!(%{props: %{"author" => "(none)"}}) - - # conn = - # get( - # conn, - # "/api/stats/#{site.domain}/sources?period=day&date=2021-01-01&filters=#{filters}" - # ) - - # assert json_response(conn, 200) == [ - # %{"name" => "Facebook", "visitors" => 1} - # ] - # end - - # test "returns top sources with :is_not (none) filter on custom pageview props", %{ - # conn: conn, - # site: site - # } do - # populate_stats(site, [ - # build(:pageview, - # referrer_source: "DuckDuckGo", - # referrer: "duckduckgo.com", - # "meta.key": ["logged_in"], - # "meta.value": ["true"], - # user_id: 123, - # timestamp: ~N[2021-01-01 00:00:00] - # ), - # build(:pageview, - # "meta.key": ["author"], - # "meta.value": ["other"], - # user_id: 123, - # timestamp: ~N[2021-01-01 00:01:00] - # ), - # build(:pageview, - # referrer_source: "Google", - # referrer: "google.com", - # "meta.key": ["author"], - # "meta.value": ["other"], - # timestamp: ~N[2021-01-01 00:00:00] - # ), - # build(:pageview, - # referrer_source: "Facebook", - # referrer: "facebook.com", - # timestamp: ~N[2021-01-01 00:00:00] - # ) - # ]) - - # filters = Jason.encode!(%{props: %{"author" => "!(none)"}}) - - # conn = - # get( - # conn, - # "/api/stats/#{site.domain}/sources?period=day&date=2021-01-01&filters=#{filters}" - # ) - - # assert json_response(conn, 200) == [ - # %{"name" => "Google", "visitors" => 1} - # ] - # end + test "returns top sources with :is filter on custom pageview props", %{conn: conn, site: site} do + populate_stats(site, [ + build(:pageview, + referrer_source: "DuckDuckGo", + referrer: "duckduckgo.com", + user_id: 123, + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:pageview, + "meta.key": ["author"], + "meta.value": ["John Doe"], + user_id: 123, + timestamp: ~N[2021-01-01 00:01:00] + ), + build(:pageview, + referrer_source: "Google", + referrer: "google.com", + "meta.key": ["author"], + "meta.value": ["John Doe"], + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:pageview, + referrer_source: "Facebook", + referrer: "facebook.com", + timestamp: ~N[2021-01-01 00:00:00] + ) + ]) + + filters = Jason.encode!(%{props: %{"author" => "John Doe"}}) + + conn = + get( + conn, + "/api/stats/#{site.domain}/sources?period=day&date=2021-01-01&filters=#{filters}" + ) + + assert json_response(conn, 200) == [ + %{"name" => "Google", "visitors" => 1} + ] + end + + test "returns top sources with :is_not filter on custom pageview props", %{ + conn: conn, + site: site + } do + populate_stats(site, [ + build(:pageview, + referrer_source: "DuckDuckGo", + referrer: "duckduckgo.com", + "meta.key": ["author"], + "meta.value": ["John Doe"], + user_id: 123, + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:pageview, + "meta.key": ["author"], + "meta.value": ["other"], + user_id: 123, + timestamp: ~N[2021-01-01 00:01:00] + ), + build(:pageview, + referrer_source: "Google", + referrer: "google.com", + "meta.key": ["author"], + "meta.value": ["other"], + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:pageview, + referrer_source: "Facebook", + referrer: "facebook.com", + timestamp: ~N[2021-01-01 00:00:00] + ) + ]) + + filters = Jason.encode!(%{props: %{"author" => "!John Doe"}}) + + conn = + get( + conn, + "/api/stats/#{site.domain}/sources?period=day&date=2021-01-01&filters=#{filters}" + ) + + assert json_response(conn, 200) == [ + %{"name" => "Facebook", "visitors" => 1}, + %{"name" => "Google", "visitors" => 1} + ] + end + + test "returns top sources with :is (none) filter on custom pageview props", %{ + conn: conn, + site: site + } do + populate_stats(site, [ + build(:pageview, + referrer_source: "DuckDuckGo", + referrer: "duckduckgo.com", + "meta.key": ["author"], + "meta.value": ["John Doe"], + user_id: 123, + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:pageview, + user_id: 123, + timestamp: ~N[2021-01-01 00:01:00] + ), + build(:pageview, + referrer_source: "Google", + referrer: "google.com", + "meta.key": ["author"], + "meta.value": ["other"], + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:pageview, + referrer_source: "Facebook", + referrer: "facebook.com", + timestamp: ~N[2021-01-01 00:00:00] + ) + ]) + + filters = Jason.encode!(%{props: %{"author" => "(none)"}}) + + conn = + get( + conn, + "/api/stats/#{site.domain}/sources?period=day&date=2021-01-01&filters=#{filters}" + ) + + assert json_response(conn, 200) == [ + %{"name" => "Facebook", "visitors" => 1} + ] + end + + test "returns top sources with :is_not (none) filter on custom pageview props", %{ + conn: conn, + site: site + } do + populate_stats(site, [ + build(:pageview, + referrer_source: "DuckDuckGo", + referrer: "duckduckgo.com", + "meta.key": ["logged_in"], + "meta.value": ["true"], + user_id: 123, + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:pageview, + "meta.key": ["author"], + "meta.value": ["other"], + user_id: 123, + timestamp: ~N[2021-01-01 00:01:00] + ), + build(:pageview, + referrer_source: "Google", + referrer: "google.com", + "meta.key": ["author"], + "meta.value": ["other"], + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:pageview, + referrer_source: "Facebook", + referrer: "facebook.com", + timestamp: ~N[2021-01-01 00:00:00] + ) + ]) + + filters = Jason.encode!(%{props: %{"author" => "!(none)"}}) + + conn = + get( + conn, + "/api/stats/#{site.domain}/sources?period=day&date=2021-01-01&filters=#{filters}" + ) + + assert json_response(conn, 200) == [ + %{"name" => "Google", "visitors" => 1} + ] + end test "returns top sources with imported data", %{conn: conn, site: site} do populate_stats(site, [