From 27f5ea0d30507c2b35be58c3a3269bef5e637468 Mon Sep 17 00:00:00 2001 From: Adam Rutkowski Date: Mon, 8 May 2023 12:19:53 +0200 Subject: [PATCH] Put private functions at the bottom --- .../controllers/api/stats_controller.ex | 806 +++++++++--------- 1 file changed, 401 insertions(+), 405 deletions(-) diff --git a/lib/plausible_web/controllers/api/stats_controller.ex b/lib/plausible_web/controllers/api/stats_controller.ex index bba59a5f2..5bab414db 100644 --- a/lib/plausible_web/controllers/api/stats_controller.ex +++ b/lib/plausible_web/controllers/api/stats_controller.ex @@ -144,52 +144,6 @@ defmodule PlausibleWeb.Api.StatsController do end end - defp plot_timeseries(timeseries, metric) do - Enum.map(timeseries, fn row -> row[metric] || 0 end) - end - - defp label_timeseries(main_result, nil) do - Enum.map(main_result, & &1.date) - end - - @blank_value "__blank__" - defp label_timeseries(main_result, comparison_result) do - blanks_to_fill = Enum.count(comparison_result) - Enum.count(main_result) - - if blanks_to_fill > 0 do - blanks = List.duplicate(@blank_value, blanks_to_fill) - Enum.map(main_result, & &1.date) ++ blanks - else - Enum.map(main_result, & &1.date) - end - end - - defp build_full_intervals(%{interval: "week", date_range: range}, labels) do - for label <- labels, into: %{} do - interval_start = Timex.beginning_of_week(label) - interval_end = Timex.end_of_week(label) - - within_interval? = Enum.member?(range, interval_start) && Enum.member?(range, interval_end) - - {label, within_interval?} - end - end - - defp build_full_intervals(%{interval: "month", date_range: range}, labels) do - for label <- labels, into: %{} do - interval_start = Timex.beginning_of_month(label) - interval_end = Timex.end_of_month(label) - - within_interval? = Enum.member?(range, interval_start) && Enum.member?(range, interval_end) - - {label, within_interval?} - end - end - - defp build_full_intervals(_query, _labels) do - nil - end - def top_stats(conn, params) do site = conn.assigns[:site] @@ -206,254 +160,19 @@ defmodule PlausibleWeb.Api.StatsController do {top_stats, sample_percent} = fetch_top_stats(site, query, comparison_query) - json(conn, %{ - top_stats: top_stats, - interval: query.interval, - sample_percent: sample_percent, - with_imported: with_imported?(query, comparison_query), - imported_source: site.imported_data && site.imported_data.source, - comparing_from: comparison_query && comparison_query.date_range.first, - comparing_to: comparison_query && comparison_query.date_range.last, - from: query.date_range.first, - to: query.date_range.last - }) - else - {:error, message} when is_binary(message) -> bad_request(conn, message) - end - end - - defp present_index_for(site, query, dates) do - case query.interval do - "hour" -> - current_date = - Timex.now(site.timezone) - |> Timex.format!("{YYYY}-{0M}-{0D} {h24}:00:00") - - Enum.find_index(dates, &(&1 == current_date)) - - "date" -> - current_date = - Timex.now(site.timezone) - |> Timex.to_date() - - Enum.find_index(dates, &(&1 == current_date)) - - "week" -> - current_date = - Timex.now(site.timezone) - |> Timex.to_date() - |> date_or_weekstart(query) - - Enum.find_index(dates, &(&1 == current_date)) - - "month" -> - current_date = - Timex.now(site.timezone) - |> Timex.to_date() - |> Timex.beginning_of_month() - - Enum.find_index(dates, &(&1 == current_date)) - - "minute" -> - current_date = - Timex.now(site.timezone) - |> Timex.format!("{YYYY}-{0M}-{0D} {h24}:{0m}:00") - - Enum.find_index(dates, &(&1 == current_date)) - end - end - - defp date_or_weekstart(date, query) do - weekstart = Timex.beginning_of_week(date) - - if Enum.member?(query.date_range, weekstart) do - weekstart - else - date - end - end - - defp fetch_top_stats( - site, - %Query{period: "realtime", filters: %{"event:goal" => _goal}} = query, - _comparison_query - ) do - query_30m = %Query{query | period: "30m"} - - %{ - visitors: %{value: unique_conversions}, - events: %{value: total_conversions} - } = Stats.aggregate(site, query_30m, [:visitors, :events]) - - stats = [ - %{ - name: "Current visitors", - value: Stats.current_visitors(site) - }, - %{ - name: "Unique conversions (last 30 min)", - value: unique_conversions - }, - %{ - name: "Total conversions (last 30 min)", - value: total_conversions - } - ] - - {stats, 100} - end - - defp fetch_top_stats(site, %Query{period: "realtime"} = query, _comparison_query) do - query_30m = %Query{query | period: "30m"} - - %{ - visitors: %{value: visitors}, - pageviews: %{value: pageviews} - } = Stats.aggregate(site, query_30m, [:visitors, :pageviews]) - - stats = [ - %{ - name: "Current visitors", - value: Stats.current_visitors(site) - }, - %{ - name: "Unique visitors (last 30 min)", - value: visitors - }, - %{ - name: "Pageviews (last 30 min)", - value: pageviews - } - ] - - {stats, 100} - end - - defp fetch_top_stats(site, %Query{filters: %{"event:goal" => _goal}} = query, comparison_query) do - totals_query = Query.remove_event_filters(query, [:goal, :props]) - - %{ - visitors: %{value: unique_visitors} - } = Stats.aggregate(site, totals_query, [:visitors]) - - %{ - visitors: %{value: converted_visitors}, - events: %{value: completions} - } = Stats.aggregate(site, query, [:visitors, :events]) - - {prev_unique_visitors, prev_converted_visitors, prev_completions} = - if comparison_query do - totals_comparison_query = Query.remove_event_filters(comparison_query, [:goal, :props]) - - %{ - visitors: %{value: prev_unique_visitors} - } = Stats.aggregate(site, totals_comparison_query, [:visitors]) - - %{ - visitors: %{value: prev_converted_visitors}, - events: %{value: prev_completions} - } = Stats.aggregate(site, comparison_query, [:visitors, :events]) - - {prev_unique_visitors, prev_converted_visitors, prev_completions} - else - {nil, nil, nil} - end - - conversion_rate = calculate_cr(unique_visitors, converted_visitors) - prev_conversion_rate = calculate_cr(prev_unique_visitors, prev_converted_visitors) - - build_item = fn name, value, comparison_value -> - if comparison_value do - change = percent_change(comparison_value, value) - %{name: name, value: value, comparison_value: comparison_value, change: change} - else - %{name: name, value: value} - end - end - - {[ - build_item.("Unique visitors", unique_visitors, prev_unique_visitors), - build_item.("Unique conversions", converted_visitors, prev_converted_visitors), - build_item.("Total conversions", completions, prev_completions), - build_item.("Conversion rate", conversion_rate, prev_conversion_rate) - ], 100} - end - - defp fetch_top_stats(site, query, comparison_query) do - metrics = - if query.filters["event:page"] do - [ - :visitors, - :visits, - :pageviews, - :bounce_rate, - :time_on_page, - :sample_percent - ] - else - [ - :visitors, - :visits, - :pageviews, - :views_per_visit, - :bounce_rate, - :visit_duration, - :sample_percent - ] - end - - current_results = Stats.aggregate(site, query, metrics) - prev_results = comparison_query && Stats.aggregate(site, comparison_query, metrics) - - stats = - [ - top_stats_entry(current_results, prev_results, "Unique visitors", :visitors), - top_stats_entry(current_results, prev_results, "Total visits", :visits), - top_stats_entry(current_results, prev_results, "Total pageviews", :pageviews), - top_stats_entry(current_results, prev_results, "Views per visit", :views_per_visit), - top_stats_entry(current_results, prev_results, "Bounce rate", :bounce_rate), - top_stats_entry(current_results, prev_results, "Visit duration", :visit_duration), - top_stats_entry(current_results, prev_results, "Time on page", :time_on_page) - ] - |> Enum.filter(& &1) - - {stats, current_results[:sample_percent][:value]} - end - - defp top_stats_entry(current_results, prev_results, name, key) do - if current_results[key] do - value = get_in(current_results, [key, :value]) - - if prev_results do - prev_value = get_in(prev_results, [key, :value]) - change = calculate_change(key, prev_value, value) - %{name: name, value: value, comparison_value: prev_value, change: change} - else - %{name: name, value: value} - end - end - end - - defp calculate_change(:bounce_rate, old_count, new_count) do - if old_count > 0, do: new_count - old_count - end - - defp calculate_change(_metric, old_count, new_count) do - percent_change(old_count, new_count) - end - - defp percent_change(nil, _new_count), do: nil - - defp percent_change(old_count, new_count) do - cond do - old_count == 0 and new_count > 0 -> - 100 - - old_count == 0 and new_count == 0 -> - 0 - - true -> - round((new_count - old_count) / old_count * 100) + json(conn, %{ + top_stats: top_stats, + interval: query.interval, + sample_percent: sample_percent, + with_imported: with_imported?(query, comparison_query), + imported_source: site.imported_data && site.imported_data.source, + comparing_from: comparison_query && comparison_query.date_range.first, + comparing_to: comparison_query && comparison_query.date_range.last, + from: query.date_range.first, + to: query.date_range.last + }) + else + {:error, message} when is_binary(message) -> bad_request(conn, message) end end @@ -1000,147 +719,378 @@ defmodule PlausibleWeb.Api.StatsController do json(conn, versions) end - def screen_sizes(conn, params) do - site = conn.assigns[:site] - query = Query.from(site, params) |> Filters.add_prefix() - pagination = parse_pagination(params) + def screen_sizes(conn, params) do + site = conn.assigns[:site] + query = Query.from(site, params) |> Filters.add_prefix() + pagination = parse_pagination(params) + + sizes = + Stats.breakdown(site, query, "visit:device", [:visitors], pagination) + |> maybe_add_cr(site, query, pagination, :device, "visit:device") + |> transform_keys(%{device: :name}) + |> maybe_add_percentages(query) + + if params["csv"] do + if Map.has_key?(query.filters, "event:goal") do + sizes + |> transform_keys(%{visitors: :conversions}) + |> to_csv([:name, :conversions, :conversion_rate]) + else + sizes |> to_csv([:name, :visitors]) + end + else + json(conn, sizes) + end + end + + def conversions(conn, params) do + site = conn.assigns[:site] + query = Query.from(site, params) |> Filters.add_prefix() + + query = + if query.period == "realtime" do + %Query{query | period: "30m"} + else + query + end + + total_q = Query.remove_event_filters(query, [:goal, :props]) + + %{visitors: %{value: total_visitors}} = Stats.aggregate(site, total_q, [:visitors]) + + conversions = + Stats.breakdown(site, query, "event:goal", [:visitors, :events], {100, 1}) + |> transform_keys(%{ + goal: :name, + visitors: :unique_conversions, + events: :total_conversions + }) + |> Enum.map(fn goal -> + goal + |> Map.put(:prop_names, CustomProps.props_for_goal(site, query)) + |> Map.put(:conversion_rate, calculate_cr(total_visitors, goal[:unique_conversions])) + end) + + if params["csv"] do + conversions |> to_csv([:name, :unique_conversions, :total_conversions]) + else + json(conn, conversions) + end + end + + def prop_breakdown(conn, params) do + site = conn.assigns[:site] + query = Query.from(site, params) |> Filters.add_prefix() + pagination = parse_pagination(params) + + total_q = Query.remove_event_filters(query, [:goal, :props]) + + %{:visitors => %{value: unique_visitors}} = Stats.aggregate(site, total_q, [:visitors]) + + prop_name = "event:props:" <> params["prop_name"] + + props = + Stats.breakdown(site, query, prop_name, [:visitors, :events], pagination) + |> transform_keys(%{ + params["prop_name"] => :name, + :events => :total_conversions, + :visitors => :unique_conversions + }) + |> Enum.map(fn prop -> + Map.put( + prop, + :conversion_rate, + calculate_cr(unique_visitors, prop[:unique_conversions]) + ) + end) + + if params["csv"] do + props + else + json(conn, props) + end + end + + def all_props_breakdown(conn, params) do + site = conn.assigns[:site] + query = Query.from(site, params) |> Filters.add_prefix() + + prop_names = Plausible.Stats.CustomProps.props_for_goal(site, query) + + values = + prop_names + |> Enum.map(fn prop -> + prop_breakdown(conn, Map.put(params, "prop_name", prop)) + |> Enum.map(&Map.put(&1, :prop, prop)) + end) + |> Enum.concat() + + to_csv(values, [:prop, :name, :unique_conversions, :total_conversions]) + end + + def current_visitors(conn, _) do + site = conn.assigns[:site] + json(conn, Stats.current_visitors(site)) + end + + def filter_suggestions(conn, params) do + site = conn.assigns[:site] + + query = + Query.from(site, params) + |> Filters.add_prefix() + + json( + conn, + Stats.filter_suggestions(site, query, params["filter_name"], params["q"]) + ) + end + + defp calculate_cr(nil, _converted_visitors), do: nil + + defp calculate_cr(unique_visitors, converted_visitors) do + if unique_visitors > 0, + do: Float.round(converted_visitors / unique_visitors * 100, 1), + else: 0.0 + end + + defp present_index_for(site, query, dates) do + case query.interval do + "hour" -> + current_date = + Timex.now(site.timezone) + |> Timex.format!("{YYYY}-{0M}-{0D} {h24}:00:00") + + Enum.find_index(dates, &(&1 == current_date)) + + "date" -> + current_date = + Timex.now(site.timezone) + |> Timex.to_date() + + Enum.find_index(dates, &(&1 == current_date)) + + "week" -> + current_date = + Timex.now(site.timezone) + |> Timex.to_date() + |> date_or_weekstart(query) + + Enum.find_index(dates, &(&1 == current_date)) + + "month" -> + current_date = + Timex.now(site.timezone) + |> Timex.to_date() + |> Timex.beginning_of_month() + + Enum.find_index(dates, &(&1 == current_date)) + + "minute" -> + current_date = + Timex.now(site.timezone) + |> Timex.format!("{YYYY}-{0M}-{0D} {h24}:{0m}:00") + + Enum.find_index(dates, &(&1 == current_date)) + end + end + + defp date_or_weekstart(date, query) do + weekstart = Timex.beginning_of_week(date) + + if Enum.member?(query.date_range, weekstart) do + weekstart + else + date + end + end + + defp fetch_top_stats( + site, + %Query{period: "realtime", filters: %{"event:goal" => _goal}} = query, + _comparison_query + ) do + query_30m = %Query{query | period: "30m"} + + %{ + visitors: %{value: unique_conversions}, + events: %{value: total_conversions} + } = Stats.aggregate(site, query_30m, [:visitors, :events]) + + stats = [ + %{ + name: "Current visitors", + value: Stats.current_visitors(site) + }, + %{ + name: "Unique conversions (last 30 min)", + value: unique_conversions + }, + %{ + name: "Total conversions (last 30 min)", + value: total_conversions + } + ] + + {stats, 100} + end + + defp fetch_top_stats(site, %Query{period: "realtime"} = query, _comparison_query) do + query_30m = %Query{query | period: "30m"} + + %{ + visitors: %{value: visitors}, + pageviews: %{value: pageviews} + } = Stats.aggregate(site, query_30m, [:visitors, :pageviews]) + + stats = [ + %{ + name: "Current visitors", + value: Stats.current_visitors(site) + }, + %{ + name: "Unique visitors (last 30 min)", + value: visitors + }, + %{ + name: "Pageviews (last 30 min)", + value: pageviews + } + ] + + {stats, 100} + end + + defp fetch_top_stats(site, %Query{filters: %{"event:goal" => _goal}} = query, comparison_query) do + totals_query = Query.remove_event_filters(query, [:goal, :props]) - sizes = - Stats.breakdown(site, query, "visit:device", [:visitors], pagination) - |> maybe_add_cr(site, query, pagination, :device, "visit:device") - |> transform_keys(%{device: :name}) - |> maybe_add_percentages(query) + %{ + visitors: %{value: unique_visitors} + } = Stats.aggregate(site, totals_query, [:visitors]) - if params["csv"] do - if Map.has_key?(query.filters, "event:goal") do - sizes - |> transform_keys(%{visitors: :conversions}) - |> to_csv([:name, :conversions, :conversion_rate]) - else - sizes |> to_csv([:name, :visitors]) - end - else - json(conn, sizes) - end - end + %{ + visitors: %{value: converted_visitors}, + events: %{value: completions} + } = Stats.aggregate(site, query, [:visitors, :events]) - defp calculate_cr(nil, _converted_visitors), do: nil + {prev_unique_visitors, prev_converted_visitors, prev_completions} = + if comparison_query do + totals_comparison_query = Query.remove_event_filters(comparison_query, [:goal, :props]) - defp calculate_cr(unique_visitors, converted_visitors) do - if unique_visitors > 0, - do: Float.round(converted_visitors / unique_visitors * 100, 1), - else: 0.0 - end + %{ + visitors: %{value: prev_unique_visitors} + } = Stats.aggregate(site, totals_comparison_query, [:visitors]) - def conversions(conn, params) do - site = conn.assigns[:site] - query = Query.from(site, params) |> Filters.add_prefix() + %{ + visitors: %{value: prev_converted_visitors}, + events: %{value: prev_completions} + } = Stats.aggregate(site, comparison_query, [:visitors, :events]) - query = - if query.period == "realtime" do - %Query{query | period: "30m"} + {prev_unique_visitors, prev_converted_visitors, prev_completions} else - query + {nil, nil, nil} end - total_q = Query.remove_event_filters(query, [:goal, :props]) - - %{visitors: %{value: total_visitors}} = Stats.aggregate(site, total_q, [:visitors]) - - conversions = - Stats.breakdown(site, query, "event:goal", [:visitors, :events], {100, 1}) - |> transform_keys(%{ - goal: :name, - visitors: :unique_conversions, - events: :total_conversions - }) - |> Enum.map(fn goal -> - goal - |> Map.put(:prop_names, CustomProps.props_for_goal(site, query)) - |> Map.put(:conversion_rate, calculate_cr(total_visitors, goal[:unique_conversions])) - end) + conversion_rate = calculate_cr(unique_visitors, converted_visitors) + prev_conversion_rate = calculate_cr(prev_unique_visitors, prev_converted_visitors) - if params["csv"] do - conversions |> to_csv([:name, :unique_conversions, :total_conversions]) - else - json(conn, conversions) + build_item = fn name, value, comparison_value -> + if comparison_value do + change = percent_change(comparison_value, value) + %{name: name, value: value, comparison_value: comparison_value, change: change} + else + %{name: name, value: value} + end end + + {[ + build_item.("Unique visitors", unique_visitors, prev_unique_visitors), + build_item.("Unique conversions", converted_visitors, prev_converted_visitors), + build_item.("Total conversions", completions, prev_completions), + build_item.("Conversion rate", conversion_rate, prev_conversion_rate) + ], 100} end - def prop_breakdown(conn, params) do - site = conn.assigns[:site] - query = Query.from(site, params) |> Filters.add_prefix() - pagination = parse_pagination(params) + defp fetch_top_stats(site, query, comparison_query) do + metrics = + if query.filters["event:page"] do + [ + :visitors, + :visits, + :pageviews, + :bounce_rate, + :time_on_page, + :sample_percent + ] + else + [ + :visitors, + :visits, + :pageviews, + :views_per_visit, + :bounce_rate, + :visit_duration, + :sample_percent + ] + end - total_q = Query.remove_event_filters(query, [:goal, :props]) + current_results = Stats.aggregate(site, query, metrics) + prev_results = comparison_query && Stats.aggregate(site, comparison_query, metrics) - %{:visitors => %{value: unique_visitors}} = Stats.aggregate(site, total_q, [:visitors]) + stats = + [ + top_stats_entry(current_results, prev_results, "Unique visitors", :visitors), + top_stats_entry(current_results, prev_results, "Total visits", :visits), + top_stats_entry(current_results, prev_results, "Total pageviews", :pageviews), + top_stats_entry(current_results, prev_results, "Views per visit", :views_per_visit), + top_stats_entry(current_results, prev_results, "Bounce rate", :bounce_rate), + top_stats_entry(current_results, prev_results, "Visit duration", :visit_duration), + top_stats_entry(current_results, prev_results, "Time on page", :time_on_page) + ] + |> Enum.filter(& &1) - prop_name = "event:props:" <> params["prop_name"] + {stats, current_results[:sample_percent][:value]} + end - props = - Stats.breakdown(site, query, prop_name, [:visitors, :events], pagination) - |> transform_keys(%{ - params["prop_name"] => :name, - :events => :total_conversions, - :visitors => :unique_conversions - }) - |> Enum.map(fn prop -> - Map.put( - prop, - :conversion_rate, - calculate_cr(unique_visitors, prop[:unique_conversions]) - ) - end) + defp top_stats_entry(current_results, prev_results, name, key) do + if current_results[key] do + value = get_in(current_results, [key, :value]) - if params["csv"] do - props - else - json(conn, props) + if prev_results do + prev_value = get_in(prev_results, [key, :value]) + change = calculate_change(key, prev_value, value) + %{name: name, value: value, comparison_value: prev_value, change: change} + else + %{name: name, value: value} + end end end - def all_props_breakdown(conn, params) do - site = conn.assigns[:site] - query = Query.from(site, params) |> Filters.add_prefix() - - prop_names = Plausible.Stats.CustomProps.props_for_goal(site, query) - - values = - prop_names - |> Enum.map(fn prop -> - prop_breakdown(conn, Map.put(params, "prop_name", prop)) - |> Enum.map(&Map.put(&1, :prop, prop)) - end) - |> Enum.concat() - - to_csv(values, [:prop, :name, :unique_conversions, :total_conversions]) + defp calculate_change(:bounce_rate, old_count, new_count) do + if old_count > 0, do: new_count - old_count end - def current_visitors(conn, _) do - site = conn.assigns[:site] - json(conn, Stats.current_visitors(site)) + defp calculate_change(_metric, old_count, new_count) do + percent_change(old_count, new_count) end - defp google_api(), do: Application.fetch_env!(:plausible, :google_api) - - def handle_errors(conn, %{kind: kind, reason: reason}) do - json(conn, %{error: Exception.format_banner(kind, reason)}) - end + defp percent_change(nil, _new_count), do: nil - def filter_suggestions(conn, params) do - site = conn.assigns[:site] + defp percent_change(old_count, new_count) do + cond do + old_count == 0 and new_count > 0 -> + 100 - query = - Query.from(site, params) - |> Filters.add_prefix() + old_count == 0 and new_count == 0 -> + 0 - json( - conn, - Stats.filter_suggestions(site, query, params["filter_name"], params["q"]) - ) + true -> + round((new_count - old_count) / old_count * 100) + end end + defp google_api(), do: Application.fetch_env!(:plausible, :google_api) + defp transform_keys(results, keys_to_replace) do Enum.map(results, fn map -> Enum.map(map, fn {key, val} -> @@ -1315,4 +1265,50 @@ defmodule PlausibleWeb.Api.StatsController do true -> false end end + + defp plot_timeseries(timeseries, metric) do + Enum.map(timeseries, fn row -> row[metric] || 0 end) + end + + defp label_timeseries(main_result, nil) do + Enum.map(main_result, & &1.date) + end + + @blank_value "__blank__" + defp label_timeseries(main_result, comparison_result) do + blanks_to_fill = Enum.count(comparison_result) - Enum.count(main_result) + + if blanks_to_fill > 0 do + blanks = List.duplicate(@blank_value, blanks_to_fill) + Enum.map(main_result, & &1.date) ++ blanks + else + Enum.map(main_result, & &1.date) + end + end + + defp build_full_intervals(%{interval: "week", date_range: range}, labels) do + for label <- labels, into: %{} do + interval_start = Timex.beginning_of_week(label) + interval_end = Timex.end_of_week(label) + + within_interval? = Enum.member?(range, interval_start) && Enum.member?(range, interval_end) + + {label, within_interval?} + end + end + + defp build_full_intervals(%{interval: "month", date_range: range}, labels) do + for label <- labels, into: %{} do + interval_start = Timex.beginning_of_month(label) + interval_end = Timex.end_of_month(label) + + within_interval? = Enum.member?(range, interval_start) && Enum.member?(range, interval_end) + + {label, within_interval?} + end + end + + defp build_full_intervals(_query, _labels) do + nil + end end