diff --git a/config/config.exs b/config/config.exs index 8e9a9bae0..0b8cf2bd0 100644 --- a/config/config.exs +++ b/config/config.exs @@ -9,11 +9,11 @@ config :plausible, environment: System.get_env("ENVIRONMENT", "dev") config :plausible, :clickhouse, - hostname: System.get_env("CLICKHOUSE_DATABASE_HOST", "localhost"), - database: System.get_env("CLICKHOUSE_DATABASE_NAME", "plausible_test"), - username: System.get_env("CLICKHOUSE_DATABASE_USER"), - password: System.get_env("CLICKHOUSE_DATABASE_PASSWORD"), - pool_size: 10 + hostname: System.get_env("CLICKHOUSE_DATABASE_HOST", "localhost"), + database: System.get_env("CLICKHOUSE_DATABASE_NAME", "plausible_test"), + username: System.get_env("CLICKHOUSE_DATABASE_USER"), + password: System.get_env("CLICKHOUSE_DATABASE_PASSWORD"), + pool_size: 10 # Configures the endpoint config :plausible, PlausibleWeb.Endpoint, @@ -64,14 +64,16 @@ config :plausible, config :plausible, # 30 minutes - session_timeout: 1000 * 60 * 30, # 30 minutes + # 30 minutes + session_timeout: 1000 * 60 * 30, session_length_minutes: 30 config :plausible, :paddle, vendor_id: "49430", vendor_auth_code: System.get_env("PADDLE_VENDOR_AUTH_CODE") -config :plausible, Plausible.Repo, +config :plausible, + Plausible.Repo, pool_size: String.to_integer(System.get_env("DATABASE_POOL_SIZE", "10")), timeout: 300_000, connect_timeout: 300_000, @@ -83,19 +85,24 @@ config :plausible, Plausible.Repo, ), ssl: String.to_existing_atom(System.get_env("DATABASE_TLS_ENABLED", "false")) - -crontab = if String.to_existing_atom(System.get_env("CRON_ENABLED", "false")) do - [ - {"0 * * * *", Plausible.Workers.SendSiteSetupEmails}, # hourly - {"0 * * * *", Plausible.Workers.SendEmailReports}, # hourly - {"0 0 * * *", Plausible.Workers.FetchTweets}, - {"0 12 * * *", Plausible.Workers.SendTrialNotifications}, # Daily at midday - {"0 12 * * *", Plausible.Workers.SendCheckStatsEmails}, # Daily at midday - {"*/10 * * * *", Plausible.Workers.ProvisionSslCertificates}, # Every 10 minutes - ] -else - false -end +crontab = + if String.to_existing_atom(System.get_env("CRON_ENABLED", "false")) do + [ + # hourly + {"0 * * * *", Plausible.Workers.SendSiteSetupEmails}, + #  hourly + {"0 * * * *", Plausible.Workers.SendEmailReports}, + {"0 0 * * *", Plausible.Workers.FetchTweets}, + # Daily at midday + {"0 12 * * *", Plausible.Workers.SendTrialNotifications}, + # Daily at midday + {"0 12 * * *", Plausible.Workers.SendCheckStatsEmails}, + # Every 10 minutes + {"*/10 * * * *", Plausible.Workers.ProvisionSslCertificates} + ] + else + false + end config :plausible, Oban, repo: Plausible.Repo, @@ -139,8 +146,8 @@ case mailer_adapter do auth: :always "Bamboo.LocalAdapter" -> - config :plausible, Plausible.Mailer, - adapter: :"Elixir.#{mailer_adapter}" + config :plausible, Plausible.Mailer, adapter: :"Elixir.#{mailer_adapter}" + _ -> raise "Unknown mailer_adapter; expected SMTPAdapter or PostmarkAdapter" end diff --git a/config/releases.exs b/config/releases.exs index 34a99b33d..e31bb7782 100644 --- a/config/releases.exs +++ b/config/releases.exs @@ -93,11 +93,11 @@ config :plausible, :google, config :plausible, :slack, webhook: slack_hook_url config :plausible, :clickhouse, - hostname: ck_host, - database: ck_db, - username: ck_db_user, - password: ck_db_pwd, - pool_size: ck_db_pool + hostname: ck_host, + database: ck_db, + username: ck_db_user, + password: ck_db_pwd, + pool_size: ck_db_pool case mailer_adapter do "Bamboo.PostmarkAdapter" -> @@ -136,12 +136,18 @@ config :plausible, :custom_domain_server, ip: custom_domain_server_ip crontab = [ - {"0 * * * *", Plausible.Workers.SendSiteSetupEmails}, # hourly - {"0 * * * *", Plausible.Workers.SendEmailReports}, # hourly - {"0 0 * * *", Plausible.Workers.FetchTweets}, # Daily at midnight - {"0 12 * * *", Plausible.Workers.SendTrialNotifications}, # Daily at midday - {"0 12 * * *", Plausible.Workers.SendCheckStatsEmails}, # Daily at midday - {"*/10 * * * *", Plausible.Workers.ProvisionSslCertificates}, # Every 10 minutes + # hourly + {"0 * * * *", Plausible.Workers.SendSiteSetupEmails}, + #  hourly + {"0 * * * *", Plausible.Workers.SendEmailReports}, + # Daily at midnight + {"0 0 * * *", Plausible.Workers.FetchTweets}, + # Daily at midday + {"0 12 * * *", Plausible.Workers.SendTrialNotifications}, + # Daily at midday + {"0 12 * * *", Plausible.Workers.SendCheckStatsEmails}, + # Every 10 minutes + {"*/10 * * * *", Plausible.Workers.ProvisionSslCertificates} ] config :plausible, Oban, @@ -154,6 +160,6 @@ config :plausible, Oban, site_setup_emails: 1, trial_notification_emails: 1 ], - crontab: if cron_enabled, do: crontab, else: false + crontab: if(cron_enabled, do: crontab, else: false) config :logger, level: :warn diff --git a/config/test.exs b/config/test.exs index 8bf4d7310..f2a42bffc 100644 --- a/config/test.exs +++ b/config/test.exs @@ -16,10 +16,10 @@ config :bcrypt_elixir, :log_rounds, 4 config :plausible, Plausible.Repo, url: - System.get_env( - "DATABASE_URL", - "postgres://postgres:postgres@127.0.0.1:5432/plausible_test=default" - ), + System.get_env( + "DATABASE_URL", + "postgres://postgres:postgres@127.0.0.1:5432/plausible_test=default" + ), pool: Ecto.Adapters.SQL.Sandbox config :plausible, Plausible.Mailer, adapter: Bamboo.TestAdapter @@ -30,7 +30,6 @@ config :plausible, paddle_api: Plausible.PaddleApi.Mock, google_api: Plausible.Google.Api.Mock - config :junit_formatter, report_file: "report.xml", report_dir: File.cwd!(), @@ -39,4 +38,4 @@ config :junit_formatter, include_filename?: true config :plausible, - session_timeout: 0 + session_timeout: 0 diff --git a/lib/mix/tasks/check_overuse.ex b/lib/mix/tasks/check_overuse.ex index e7e8f5f0a..0bb56a53c 100644 --- a/lib/mix/tasks/check_overuse.ex +++ b/lib/mix/tasks/check_overuse.ex @@ -12,17 +12,20 @@ defmodule Mix.Tasks.CheckOveruse do end def execute(_args \\ []) do - active_users = Repo.all( - from u in Plausible.Auth.User, - join: s in Plausible.Billing.Subscription, on: s.user_id == u.id, - where: s.status == "active", - select: {u, s} - ) + active_users = + Repo.all( + from u in Plausible.Auth.User, + join: s in Plausible.Billing.Subscription, + on: s.user_id == u.id, + where: s.status == "active", + select: {u, s} + ) for {user, subscription} <- active_users do IO.puts("Checking #{user.email}...") usage = Plausible.Billing.usage(user) allowance = Plausible.Billing.Plans.allowance(subscription) + if usage > allowance do IO.puts("Overuse: #{user.email}") IO.puts("Usage: #{usage}") diff --git a/lib/mix/tasks/create_free_subscription.ex b/lib/mix/tasks/create_free_subscription.ex index be503d78a..be0c7a993 100644 --- a/lib/mix/tasks/create_free_subscription.ex +++ b/lib/mix/tasks/create_free_subscription.ex @@ -17,7 +17,7 @@ defmodule Mix.Tasks.CreateFreeSubscription do user = Repo.get(Plausible.Auth.User, user_id) Subscription.free(%{user_id: user_id}) - |> Repo.insert! + |> Repo.insert!() IO.puts("Created a free subscription for user: #{user.name}") end diff --git a/lib/plausible/application.ex b/lib/plausible/application.ex index 4c49a5fef..bae4a9751 100644 --- a/lib/plausible/application.ex +++ b/lib/plausible/application.ex @@ -5,12 +5,15 @@ defmodule Plausible.Application do def start(_type, _args) do clickhouse_config = Application.get_env(:plausible, :clickhouse) + children = [ Plausible.Repo, PlausibleWeb.Endpoint, Plausible.Event.WriteBuffer, Plausible.Session.WriteBuffer, - Clickhousex.child_spec(Keyword.merge([scheme: :http, port: 8123, name: :clickhouse], clickhouse_config)), + Clickhousex.child_spec( + Keyword.merge([scheme: :http, port: 8123, name: :clickhouse], clickhouse_config) + ), Plausible.Session.Store, {Oban, Application.get_env(:plausible, Oban)} ] diff --git a/lib/plausible/auth/auth.ex b/lib/plausible/auth/auth.ex index e60e2439f..aba2c0715 100644 --- a/lib/plausible/auth/auth.ex +++ b/lib/plausible/auth/auth.ex @@ -6,7 +6,7 @@ defmodule Plausible.Auth do def create_user(name, email) do %Auth.User{} |> Auth.User.new(%{name: name, email: email}) - |> Repo.insert + |> Repo.insert() end def find_user_by(opts) do @@ -14,15 +14,17 @@ defmodule Plausible.Auth do end def user_completed_setup?(user) do - domains = Repo.all( - from u in Plausible.Auth.User, - where: u.id == ^user.id, - join: sm in Plausible.Site.Membership, - on: sm.user_id == u.id, - join: s in Plausible.Site, - on: s.id == sm.site_id, - select: s.domain - ) + domains = + Repo.all( + from u in Plausible.Auth.User, + where: u.id == ^user.id, + join: sm in Plausible.Site.Membership, + on: sm.user_id == u.id, + join: s in Plausible.Site, + on: s.id == sm.site_id, + select: s.domain + ) + Stats.has_pageviews?(domains) end end diff --git a/lib/plausible/auth/token.ex b/lib/plausible/auth/token.ex index f3215ec6c..3720a339c 100644 --- a/lib/plausible/auth/token.ex +++ b/lib/plausible/auth/token.ex @@ -7,7 +7,9 @@ defmodule Plausible.Auth.Token do end def verify_password_reset(token) do - Phoenix.Token.verify(PlausibleWeb.Endpoint, "password-reset", token, max_age: @one_hour_in_seconds) + Phoenix.Token.verify(PlausibleWeb.Endpoint, "password-reset", token, + max_age: @one_hour_in_seconds + ) end def sign_activation(name, email) do diff --git a/lib/plausible/billing/billing.ex b/lib/plausible/billing/billing.ex index b1c39f4cc..028917ef4 100644 --- a/lib/plausible/billing/billing.ex +++ b/lib/plausible/billing/billing.ex @@ -8,12 +8,13 @@ defmodule Plausible.Billing do end def subscription_created(params) do - params = if present?(params["passthrough"]) do - params - else - user = Repo.get_by(Plausible.Auth.User, email: params["email"]) - Map.put(params, "passthrough", user && user.id) - end + params = + if present?(params["passthrough"]) do + params + else + user = Repo.get_by(Plausible.Auth.User, email: params["email"]) + Map.put(params, "passthrough", user && user.id) + end changeset = Subscription.changeset(%Subscription{}, format_subscription(params)) @@ -31,9 +32,10 @@ defmodule Plausible.Billing do subscription = Repo.get_by(Subscription, paddle_subscription_id: params["subscription_id"]) if subscription do - changeset = Subscription.changeset(subscription, %{ - status: params["status"] - }) + changeset = + Subscription.changeset(subscription, %{ + status: params["status"] + }) Repo.update(changeset) else @@ -46,12 +48,15 @@ defmodule Plausible.Billing do if subscription do {:ok, api_subscription} = @paddle_api.get_subscription(subscription.paddle_subscription_id) - amount = :erlang.float_to_binary(api_subscription["next_payment"]["amount"] / 1, decimals: 2) - changeset = Subscription.changeset(subscription, %{ - next_bill_amount: amount, - next_bill_date: api_subscription["next_payment"]["date"] - }) + amount = + :erlang.float_to_binary(api_subscription["next_payment"]["amount"] / 1, decimals: 2) + + changeset = + Subscription.changeset(subscription, %{ + next_bill_amount: amount, + next_bill_date: api_subscription["next_payment"]["date"] + }) Repo.update(changeset) else @@ -62,9 +67,10 @@ defmodule Plausible.Billing do def change_plan(user, new_plan_id) do subscription = active_subscription_for(user.id) - res = @paddle_api.update_subscription(subscription.paddle_subscription_id, %{ - plan_id: new_plan_id - }) + res = + @paddle_api.update_subscription(subscription.paddle_subscription_id, %{ + plan_id: new_plan_id + }) case res do {:ok, response} -> @@ -73,9 +79,12 @@ defmodule Plausible.Billing do Subscription.changeset(subscription, %{ paddle_plan_id: Integer.to_string(response["plan_id"]), next_bill_amount: amount, - next_bill_date: response["next_payment"]["date"], - }) |> Repo.update - e -> e + next_bill_date: response["next_payment"]["date"] + }) + |> Repo.update() + + e -> + e end end @@ -99,6 +108,7 @@ defmodule Plausible.Billing do def usage(user) do user = Repo.preload(user, :sites) + Enum.reduce(user.sites, 0, fn site, total -> total + site_usage(site) end) @@ -126,5 +136,4 @@ defmodule Plausible.Billing do defp present?(""), do: false defp present?(nil), do: false defp present?(_), do: true - end diff --git a/lib/plausible/billing/paddle_api.ex b/lib/plausible/billing/paddle_api.ex index 38c39a38e..ab3e083f9 100644 --- a/lib/plausible/billing/paddle_api.ex +++ b/lib/plausible/billing/paddle_api.ex @@ -34,12 +34,13 @@ defmodule Plausible.Billing.PaddleApi do def update_subscription(paddle_subscription_id, params) do config = get_config() - params = Map.merge(params, %{ - vendor_id: config[:vendor_id], - vendor_auth_code: config[:vendor_auth_code], - subscription_id: paddle_subscription_id, - quantity: 1 - }) + params = + Map.merge(params, %{ + vendor_id: config[:vendor_id], + vendor_auth_code: config[:vendor_auth_code], + subscription_id: paddle_subscription_id, + quantity: 1 + }) {:ok, response} = HTTPoison.post(@update_endpoint, Poison.encode!(params), @headers) body = Poison.decode!(response.body) @@ -53,6 +54,7 @@ defmodule Plausible.Billing.PaddleApi do def get_subscription(paddle_subscription_id) do config = get_config() + params = %{ vendor_id: config[:vendor_id], vendor_auth_code: config[:vendor_auth_code], diff --git a/lib/plausible/billing/plans.ex b/lib/plausible/billing/plans.ex index 9e10dd385..205a47e89 100644 --- a/lib/plausible/billing/plans.ex +++ b/lib/plausible/billing/plans.ex @@ -3,13 +3,13 @@ defmodule Plausible.Billing.Plans do monthly: %{ "10k": %{product_id: "558018", due_now: "$6"}, "100k": %{product_id: "558745", due_now: "$12"}, - "1m": %{product_id: "558746", due_now: "$36"}, + "1m": %{product_id: "558746", due_now: "$36"} }, yearly: %{ "10k": %{product_id: "572810", due_now: "$48"}, "100k": %{product_id: "590752", due_now: "$96"}, "1m": %{product_id: "590753", due_now: "$288"} - }, + } } def plans do @@ -19,12 +19,12 @@ defmodule Plausible.Billing.Plans do def allowance(subscription) do allowed_volume = %{ "free_10k" => 10_000, - "558018" => 10_000, - "572810" => 10_000, - "558745" => 100_000, - "590752" => 100_000, - "558746" => 1_000_000, - "590753" => 1_000_000, + "558018" => 10_000, + "572810" => 10_000, + "558745" => 100_000, + "590752" => 100_000, + "558746" => 1_000_000, + "590753" => 1_000_000 } allowed_volume[subscription.paddle_plan_id] diff --git a/lib/plausible/billing/subscription.ex b/lib/plausible/billing/subscription.ex index d2d8e0567..d8ca66a96 100644 --- a/lib/plausible/billing/subscription.ex +++ b/lib/plausible/billing/subscription.ex @@ -2,7 +2,16 @@ defmodule Plausible.Billing.Subscription do use Ecto.Schema import Ecto.Changeset - @required_fields [:paddle_subscription_id, :paddle_plan_id, :update_url, :cancel_url, :status, :next_bill_amount, :next_bill_date, :user_id] + @required_fields [ + :paddle_subscription_id, + :paddle_plan_id, + :update_url, + :cancel_url, + :status, + :next_bill_amount, + :next_bill_date, + :user_id + ] @valid_statuses ["active", "past_due", "deleted", "paused"] schema "subscriptions" do diff --git a/lib/plausible/clickhouse.ex b/lib/plausible/clickhouse.ex index added79cc..68820eec5 100644 --- a/lib/plausible/clickhouse.ex +++ b/lib/plausible/clickhouse.ex @@ -3,6 +3,7 @@ defmodule Plausible.Clickhouse do {q, params} = Ecto.Adapters.SQL.to_sql(:all, Plausible.Repo, query) q = String.replace(q, ~r/\$[0-9]+/, "?") res = Clickhousex.query!(:clickhouse, q, params, log: {Plausible.Clickhouse, :log, []}) + Enum.map(res.rows, fn row -> Enum.zip(res.columns, row) |> Enum.into(%{}) @@ -10,27 +11,71 @@ defmodule Plausible.Clickhouse do end def insert_events(events) do - insert = """ - INSERT INTO events (name, timestamp, domain, user_id, session_id, hostname, pathname, referrer, referrer_source, initial_referrer, initial_referrer_source, country_code, screen_size, browser, operating_system) - VALUES - """ <> String.duplicate(" (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?),", length(events)) + insert = + """ + INSERT INTO events (name, timestamp, domain, user_id, session_id, hostname, pathname, referrer, referrer_source, initial_referrer, initial_referrer_source, country_code, screen_size, browser, operating_system) + VALUES + """ <> String.duplicate(" (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?),", length(events)) - args = Enum.reduce(events, [], fn event, acc -> - [escape_quote(event.name), event.timestamp, event.domain, event.user_id, event.session_id, event.hostname, escape_quote(event.pathname), escape_quote(event.referrer || ""), escape_quote(event.referrer_source || ""), escape_quote(event.initial_referrer || ""), escape_quote(event.initial_referrer_source || ""), event.country_code || "", event.screen_size || "", event.browser || "", event.operating_system || ""] ++ acc - end) + args = + Enum.reduce(events, [], fn event, acc -> + [ + escape_quote(event.name), + event.timestamp, + event.domain, + event.user_id, + event.session_id, + event.hostname, + escape_quote(event.pathname), + escape_quote(event.referrer || ""), + escape_quote(event.referrer_source || ""), + escape_quote(event.initial_referrer || ""), + escape_quote(event.initial_referrer_source || ""), + event.country_code || "", + event.screen_size || "", + event.browser || "", + event.operating_system || "" + ] ++ acc + end) Clickhousex.query(:clickhouse, insert, args, log: {Plausible.Clickhouse, :log, []}) end def insert_sessions(sessions) do - insert = """ - INSERT INTO sessions (sign, session_id, domain, user_id, timestamp, hostname, start, is_bounce, entry_page, exit_page, events, pageviews, duration, referrer, referrer_source, country_code, screen_size, browser, operating_system) - VALUES - """ <> String.duplicate(" (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?),", Enum.count(sessions)) + insert = + """ + INSERT INTO sessions (sign, session_id, domain, user_id, timestamp, hostname, start, is_bounce, entry_page, exit_page, events, pageviews, duration, referrer, referrer_source, country_code, screen_size, browser, operating_system) + VALUES + """ <> + String.duplicate( + " (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?),", + Enum.count(sessions) + ) - args = Enum.reduce(sessions, [], fn session, acc -> - [session.sign, session.session_id, session.domain, session.user_id, session.timestamp, session.hostname, session.start, session.is_bounce && 1 || 0, escape_quote(session.entry_page), escape_quote(session.exit_page), session.events, session.pageviews, session.duration, escape_quote(session.referrer || ""), escape_quote(session.referrer_source || ""), session.country_code || "", session.screen_size || "", session.browser || "", session.operating_system || ""] ++ acc - end) + args = + Enum.reduce(sessions, [], fn session, acc -> + [ + session.sign, + session.session_id, + session.domain, + session.user_id, + session.timestamp, + session.hostname, + session.start, + (session.is_bounce && 1) || 0, + escape_quote(session.entry_page), + escape_quote(session.exit_page), + session.events, + session.pageviews, + session.duration, + escape_quote(session.referrer || ""), + escape_quote(session.referrer_source || ""), + session.country_code || "", + session.screen_size || "", + session.browser || "", + session.operating_system || "" + ] ++ acc + end) Clickhousex.query(:clickhouse, insert, args, log: {Plausible.Clickhouse, :log, []}) end @@ -40,18 +85,20 @@ defmodule Plausible.Clickhouse do def log(query) do require Logger + case query.result do {:ok, _q, _res} -> timing = System.convert_time_unit(query.connection_time, :native, :millisecond) Logger.info("Clickhouse query OK db=#{timing}ms") + e -> Logger.error("Clickhouse query ERROR") - Logger.error(inspect e) + Logger.error(inspect(e)) end Logger.debug(fn -> statement = String.replace(query.query.statement, "\n", " ") - "#{statement} #{inspect query.params}" + "#{statement} #{inspect(query.params)}" end) end end diff --git a/lib/plausible/event/clickhouse_schema.ex b/lib/plausible/event/clickhouse_schema.ex index ed19fde63..a8bb84f1b 100644 --- a/lib/plausible/event/clickhouse_schema.ex +++ b/lib/plausible/event/clickhouse_schema.ex @@ -25,7 +25,21 @@ defmodule Plausible.ClickhouseEvent do def changeset(pageview, attrs) do pageview - |> cast(attrs, [:name, :domain, :hostname, :pathname, :user_id, :operating_system, :browser, :referrer, :referrer_source, :initial_referrer, :initial_referrer_source, :country_code, :screen_size]) + |> cast(attrs, [ + :name, + :domain, + :hostname, + :pathname, + :user_id, + :operating_system, + :browser, + :referrer, + :referrer_source, + :initial_referrer, + :initial_referrer_source, + :country_code, + :screen_size + ]) |> validate_required([:name, :domain, :hostname, :pathname, :user_id]) end end diff --git a/lib/plausible/event/write_buffer.ex b/lib/plausible/event/write_buffer.ex index 0a61aeb27..2811c6a0b 100644 --- a/lib/plausible/event/write_buffer.ex +++ b/lib/plausible/event/write_buffer.ex @@ -26,7 +26,7 @@ defmodule Plausible.Event.WriteBuffer do end def handle_cast({:insert, event}, %{buffer: buffer} = state) do - new_buffer = [ event | buffer ] + new_buffer = [event | buffer] if length(new_buffer) >= @max_buffer_size do Logger.info("Buffer full, flushing to disk") @@ -59,7 +59,9 @@ defmodule Plausible.Event.WriteBuffer do defp do_flush(buffer) do case buffer do - [] -> nil + [] -> + nil + events -> Logger.info("Flushing #{length(events)} events") Clickhouse.insert_events(events) diff --git a/lib/plausible/goals.ex b/lib/plausible/goals.ex index 3f44ce2e8..202ee815d 100644 --- a/lib/plausible/goals.ex +++ b/lib/plausible/goals.ex @@ -5,17 +5,17 @@ defmodule Plausible.Goals do def create(site, params) do params = Map.merge(params, %{"domain" => site.domain}) - Goal.changeset(%Goal{}, params) |> Repo.insert + Goal.changeset(%Goal{}, params) |> Repo.insert() end def for_site(domain) do Repo.all( from g in Goal, - where: g.domain == ^domain + where: g.domain == ^domain ) end def delete(id) do - Repo.one(from g in Goal, where: g.id == ^id) |> Repo.delete! + Repo.one(from g in Goal, where: g.id == ^id) |> Repo.delete!() end end diff --git a/lib/plausible/google/api.ex b/lib/plausible/google/api.ex index 417813ab3..2bcb50282 100644 --- a/lib/plausible/google/api.ex +++ b/lib/plausible/google/api.ex @@ -6,18 +6,34 @@ defmodule Plausible.Google.Api do if Application.get_env(:plausible, :environment) == "test" do "" else - "https://accounts.google.com/o/oauth2/v2/auth?client_id=#{client_id()}&redirect_uri=#{redirect_uri()}&prompt=consent&response_type=code&access_type=offline&scope=#{@scope}&state=#{site_id}" + "https://accounts.google.com/o/oauth2/v2/auth?client_id=#{client_id()}&redirect_uri=#{ + redirect_uri() + }&prompt=consent&response_type=code&access_type=offline&scope=#{@scope}&state=#{site_id}" end end def fetch_access_token(code) do - res = HTTPoison.post!("https://www.googleapis.com/oauth2/v4/token", "client_id=#{client_id()}&client_secret=#{client_secret()}&code=#{code}&grant_type=authorization_code&redirect_uri=#{redirect_uri()}", ["Content-Type": "application/x-www-form-urlencoded"]) + res = + HTTPoison.post!( + "https://www.googleapis.com/oauth2/v4/token", + "client_id=#{client_id()}&client_secret=#{client_secret()}&code=#{code}&grant_type=authorization_code&redirect_uri=#{ + redirect_uri() + }", + "Content-Type": "application/x-www-form-urlencoded" + ) + Jason.decode!(res.body) end def fetch_verified_properties(auth) do auth = refresh_if_needed(auth) - res = HTTPoison.get!("https://www.googleapis.com/webmasters/v3/sites",["Content-Type": "application/json", "Authorization": "Bearer #{auth.access_token}"]) + + res = + HTTPoison.get!("https://www.googleapis.com/webmasters/v3/sites", + "Content-Type": "application/json", + Authorization: "Bearer #{auth.access_token}" + ) + Jason.decode!(res.body) |> Map.get("siteEntry", []) |> Enum.filter(fn site -> site["permissionLevel"] in @verified_permission_levels end) @@ -29,27 +45,37 @@ defmodule Plausible.Google.Api do auth = refresh_if_needed(auth) property = URI.encode_www_form(auth.property) - res = HTTPoison.post!("https://www.googleapis.com/webmasters/v3/sites/#{property}/searchAnalytics/query", Jason.encode!(%{ - startDate: Date.to_iso8601(query.date_range.first), - endDate: Date.to_iso8601(query.date_range.last), - dimensions: ["query"], - rowLimit: 20 - }), ["Content-Type": "application/json", "Authorization": "Bearer #{auth.access_token}"]) + res = + HTTPoison.post!( + "https://www.googleapis.com/webmasters/v3/sites/#{property}/searchAnalytics/query", + Jason.encode!(%{ + startDate: Date.to_iso8601(query.date_range.first), + endDate: Date.to_iso8601(query.date_range.last), + dimensions: ["query"], + rowLimit: 20 + }), + "Content-Type": "application/json", + Authorization: "Bearer #{auth.access_token}" + ) case res.status_code do 200 -> - terms = (Jason.decode!(res.body)["rows"] || []) - |> Enum.filter(fn row -> row["clicks"] > 0 end) - |> Enum.map(fn row -> %{name: row["keys"], count: round(row["clicks"])} end) + terms = + (Jason.decode!(res.body)["rows"] || []) + |> Enum.filter(fn row -> row["clicks"] > 0 end) + |> Enum.map(fn row -> %{name: row["keys"], count: round(row["clicks"])} end) {:ok, terms} + 401 -> Sentry.capture_message("Error fetching Google queries", extra: Jason.decode!(res.body)) {:error, :invalid_credentials} + 403 -> Sentry.capture_message("Error fetching Google queries", extra: Jason.decode!(res.body)) msg = Jason.decode!(res.body)["error"]["message"] {:error, msg} + _ -> Sentry.capture_message("Error fetching Google queries", extra: Jason.decode!(res.body)) {:error, :unknown} @@ -65,14 +91,22 @@ defmodule Plausible.Google.Api do end defp refresh_token(auth) do - res = HTTPoison.post!("https://www.googleapis.com/oauth2/v4/token", "client_id=#{client_id()}&client_secret=#{client_secret()}&refresh_token=#{auth.refresh_token}&grant_type=refresh_token&redirect_uri=#{redirect_uri()}", ["Content-Type": "application/x-www-form-urlencoded"]) - body = Jason.decode!(res.body) + res = + HTTPoison.post!( + "https://www.googleapis.com/oauth2/v4/token", + "client_id=#{client_id()}&client_secret=#{client_secret()}&refresh_token=#{ + auth.refresh_token + }&grant_type=refresh_token&redirect_uri=#{redirect_uri()}", + "Content-Type": "application/x-www-form-urlencoded" + ) + body = Jason.decode!(res.body) Plausible.Site.GoogleAuth.changeset(auth, %{ access_token: body["access_token"], - expires: NaiveDateTime.utc_now() |> NaiveDateTime.add(body["expires_in"]), - }) |> Plausible.Repo.update! + expires: NaiveDateTime.utc_now() |> NaiveDateTime.add(body["expires_in"]) + }) + |> Plausible.Repo.update!() end defp client_id() do @@ -86,5 +120,4 @@ defmodule Plausible.Google.Api do defp redirect_uri() do PlausibleWeb.Endpoint.clean_url() <> "/auth/google/callback" end - end diff --git a/lib/plausible/session/clickhouse_schema.ex b/lib/plausible/session/clickhouse_schema.ex index 9e2c7c203..9affd799e 100644 --- a/lib/plausible/session/clickhouse_schema.ex +++ b/lib/plausible/session/clickhouse_schema.ex @@ -33,7 +33,22 @@ defmodule Plausible.ClickhouseSession do def changeset(session, attrs) do session - |> cast(attrs, [:hostname, :domain, :entry_page, :exit_page, :referrer, :fingerprint, :start, :length, :is_bounce, :operating_system, :browser, :referrer_source, :country_code, :screen_size]) + |> cast(attrs, [ + :hostname, + :domain, + :entry_page, + :exit_page, + :referrer, + :fingerprint, + :start, + :length, + :is_bounce, + :operating_system, + :browser, + :referrer_source, + :country_code, + :screen_size + ]) |> validate_required([:hostname, :domain, :fingerprint, :is_bounce, :start]) end end diff --git a/lib/plausible/session/store.ex b/lib/plausible/session/store.ex index 6bd020f1b..248a1b74c 100644 --- a/lib/plausible/session/store.ex +++ b/lib/plausible/session/store.ex @@ -6,7 +6,8 @@ defmodule Plausible.Session.Store do require Logger @session_length_seconds Application.get_env(:plausible, :session_length_minutes) * 60 - @forget_session_after @session_length_seconds * 2 # Remember session for longer in case of upstream latency + # Remember session for longer in case of upstream latency + @forget_session_after @session_length_seconds * 2 @garbage_collect_interval_milliseconds 60 * 1000 def start_link(opts) do @@ -16,26 +17,28 @@ defmodule Plausible.Session.Store do def init(_opts) do timer = Process.send_after(self(), :garbage_collect, @garbage_collect_interval_milliseconds) - latest_sessions = from( - s in "sessions", - where: s.timestamp >= fragment("now() - INTERVAL ? SECOND", @forget_session_after), - group_by: s.session_id, - select: %{session_id: s.session_id, timestamp: max(s.timestamp)} - ) - - sessions = try do - Plausible.Clickhouse.all( - from s in Plausible.ClickhouseSession, - join: ls in subquery(latest_sessions), - on: s.session_id == ls.session_id and s.timestamp == ls.timestamp, - order_by: s.timestamp + latest_sessions = + from( + s in "sessions", + where: s.timestamp >= fragment("now() - INTERVAL ? SECOND", @forget_session_after), + group_by: s.session_id, + select: %{session_id: s.session_id, timestamp: max(s.timestamp)} ) - |> Enum.map(fn s -> Map.new(s, fn {k, v} -> {String.to_atom(k), v} end) end) - |> Enum.map(fn s -> {s[:user_id], struct(Plausible.ClickhouseSession, s)} end) - |> Enum.into(%{}) - rescue - _e -> %{} - end + + sessions = + try do + Plausible.Clickhouse.all( + from s in Plausible.ClickhouseSession, + join: ls in subquery(latest_sessions), + on: s.session_id == ls.session_id and s.timestamp == ls.timestamp, + order_by: s.timestamp + ) + |> Enum.map(fn s -> Map.new(s, fn {k, v} -> {String.to_atom(k), v} end) end) + |> Enum.map(fn s -> {s[:user_id], struct(Plausible.ClickhouseSession, s)} end) + |> Enum.into(%{}) + rescue + _e -> %{} + end {:ok, %{timer: timer, sessions: sessions}} end @@ -48,23 +51,26 @@ defmodule Plausible.Session.Store do found_session = sessions[event.user_id] active = is_active?(found_session, event) - updated_sessions = cond do - found_session && active -> - new_session = update_session(found_session, event) - WriteBuffer.insert([%{new_session | sign: 1}, %{found_session | sign: -1}]) - Map.put(sessions, event.user_id, new_session) - found_session && !active -> - new_session = new_session_from_event(event) - WriteBuffer.insert([new_session]) - Map.put(sessions, event.user_id, new_session) - true -> - new_session = new_session_from_event(event) - WriteBuffer.insert([new_session]) - Map.put(sessions, event.user_id, new_session) - end + updated_sessions = + cond do + found_session && active -> + new_session = update_session(found_session, event) + WriteBuffer.insert([%{new_session | sign: 1}, %{found_session | sign: -1}]) + Map.put(sessions, event.user_id, new_session) + + found_session && !active -> + new_session = new_session_from_event(event) + WriteBuffer.insert([new_session]) + Map.put(sessions, event.user_id, new_session) + + true -> + new_session = new_session_from_event(event) + WriteBuffer.insert([new_session]) + Map.put(sessions, event.user_id, new_session) + end session_id = updated_sessions[event.user_id].session_id - {:reply, session_id, %{ state | sessions: updated_sessions }} + {:reply, session_id, %{state | sessions: updated_sessions}} end defp is_active?(session, event) do @@ -72,13 +78,15 @@ defmodule Plausible.Session.Store do end defp update_session(session, event) do - %{ session | - timestamp: event.timestamp, - exit_page: event.pathname, - is_bounce: false, - duration: Timex.diff(event.timestamp, session.start, :second), - pageviews: (if event.name == "pageview", do: session.pageviews + 1, else: session.pageviews), - events: session.events + 1 + %{ + session + | timestamp: event.timestamp, + exit_page: event.pathname, + is_bounce: false, + duration: Timex.diff(event.timestamp, session.start, :second), + pageviews: + if(event.name == "pageview", do: session.pageviews + 1, else: session.pageviews), + events: session.events + 1 } end @@ -93,7 +101,7 @@ defmodule Plausible.Session.Store do exit_page: event.pathname, is_bounce: true, duration: 0, - pageviews: (if event.name == "pageview", do: 1, else: 0), + pageviews: if(event.name == "pageview", do: 1, else: 0), events: 1, referrer: event.referrer, referrer_source: event.referrer_source, @@ -109,16 +117,21 @@ defmodule Plausible.Session.Store do Logger.debug("Session store collecting garbage") now = Timex.now() - new_sessions = Enum.reduce(state[:sessions], %{}, fn {key, session}, acc -> - if Timex.diff(now, session.timestamp, :second) <= @forget_session_after do - Map.put(acc, key, session) - else - acc # forget the session - end - end) + + new_sessions = + Enum.reduce(state[:sessions], %{}, fn {key, session}, acc -> + if Timex.diff(now, session.timestamp, :second) <= @forget_session_after do + Map.put(acc, key, session) + else + # forget the session + acc + end + end) Process.cancel_timer(state[:timer]) - new_timer = Process.send_after(self(), :garbage_collect, @garbage_collect_interval_milliseconds) + + new_timer = + Process.send_after(self(), :garbage_collect, @garbage_collect_interval_milliseconds) Logger.debug(fn -> n_old = Enum.count(state[:sessions]) diff --git a/lib/plausible/session/write_buffer.ex b/lib/plausible/session/write_buffer.ex index 3975feaef..a0736aba8 100644 --- a/lib/plausible/session/write_buffer.ex +++ b/lib/plausible/session/write_buffer.ex @@ -46,7 +46,9 @@ defmodule Plausible.Session.WriteBuffer do defp flush(buffer) do case buffer do - [] -> nil + [] -> + nil + sessions -> Logger.info("Flushing #{length(sessions)} sessions") Plausible.Clickhouse.insert_sessions(sessions) diff --git a/lib/plausible/site/custom_domain.ex b/lib/plausible/site/custom_domain.ex index a7db64fce..f590f52e2 100644 --- a/lib/plausible/site/custom_domain.ex +++ b/lib/plausible/site/custom_domain.ex @@ -1,6 +1,7 @@ defmodule Plausible.Site.CustomDomain do use Ecto.Schema import Ecto.Changeset + @domain_name_regex ~r/(?:[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?\.)+[a-z0-9][a-z0-9-]{0,61}[a-z0-9]/ schema "custom_domains" do diff --git a/lib/plausible/site/schema.ex b/lib/plausible/site/schema.ex index cdbcdc466..b438cf4a1 100644 --- a/lib/plausible/site/schema.ex +++ b/lib/plausible/site/schema.ex @@ -35,13 +35,14 @@ defmodule Plausible.Site do end defp clean_domain(changeset) do - clean_domain = (get_field(changeset, :domain) || "") - |> String.trim - |> String.replace_leading("http://", "") - |> String.replace_leading("https://", "") - |> String.replace_leading("www.", "") - |> String.replace_trailing("/", "") - |> String.downcase() + clean_domain = + (get_field(changeset, :domain) || "") + |> String.trim() + |> String.replace_leading("http://", "") + |> String.replace_leading("https://", "") + |> String.replace_leading("www.", "") + |> String.replace_trailing("/", "") + |> String.downcase() change(changeset, %{ domain: clean_domain diff --git a/lib/plausible/site/shared_link.ex b/lib/plausible/site/shared_link.ex index c8985aad5..ff5d9ca7c 100644 --- a/lib/plausible/site/shared_link.ex +++ b/lib/plausible/site/shared_link.ex @@ -21,7 +21,9 @@ defmodule Plausible.Site.SharedLink do defp hash_password(link) do case link.changes[:password] do - nil -> link + nil -> + link + password -> hash = Plausible.Auth.Password.hash(password) change(link, password_hash: hash) diff --git a/lib/plausible/sites.ex b/lib/plausible/sites.ex index 22470428b..66ed33da2 100644 --- a/lib/plausible/sites.ex +++ b/lib/plausible/sites.ex @@ -5,24 +5,25 @@ defmodule Plausible.Sites do def get_for_user!(user_id, domain) do Repo.one!( from s in Plausible.Site, - join: sm in Plausible.Site.Membership, on: sm.site_id == s.id, - where: sm.user_id == ^user_id, - where: s.domain == ^domain, - select: s + join: sm in Plausible.Site.Membership, + on: sm.site_id == s.id, + where: sm.user_id == ^user_id, + where: s.domain == ^domain, + select: s ) end def has_goals?(site) do Repo.exists?( from g in Plausible.Goal, - where: g.domain == ^site.domain + where: g.domain == ^site.domain ) end def is_owner?(user_id, site) do Repo.exists?( from sm in Plausible.Site.Membership, - where: sm.user_id == ^user_id and sm.site_id == ^site.id + where: sm.user_id == ^user_id and sm.site_id == ^site.id ) end @@ -30,6 +31,7 @@ defmodule Plausible.Sites do CustomDomain.changeset(%CustomDomain{}, %{ site_id: site.id, domain: custom_domain - }) |> Repo.insert + }) + |> Repo.insert() end end diff --git a/lib/plausible/slack.ex b/lib/plausible/slack.ex index cbb603830..968bcd097 100644 --- a/lib/plausible/slack.ex +++ b/lib/plausible/slack.ex @@ -7,6 +7,7 @@ defmodule Plausible.Slack do case @app_env do "prod" -> HTTPoison.post!(webhook_url(), Poison.encode!(%{text: text})) + _ -> Logger.debug(text) end diff --git a/lib/plausible/stats/clickhouse.ex b/lib/plausible/stats/clickhouse.ex index 80e7b4669..b39f79197 100644 --- a/lib/plausible/stats/clickhouse.ex +++ b/lib/plausible/stats/clickhouse.ex @@ -10,46 +10,58 @@ defmodule Plausible.Stats.Clickhouse do cond do old_pageviews == 0 and pageviews > 0 -> {100, 100} + old_pageviews == 0 and pageviews == 0 -> {0, 0} + true -> { round((pageviews - old_pageviews) / old_pageviews * 100), - round((visitors - old_visitors) / old_visitors * 100), + round((visitors - old_visitors) / old_visitors * 100) } - end end def calculate_plot(site, %Query{step_type: "month"} = query) do - steps = Enum.map((query.steps - 1)..0, fn shift -> - Timex.now(site.timezone) - |> Timex.beginning_of_month - |> Timex.shift(months: -shift) - |> DateTime.to_date - end) - - groups = Clickhouse.all( - from e in base_query(site, %{query | filters: %{}}), - select: {fragment("toStartOfMonth(toTimeZone(?, ?)) as month", e.timestamp, ^site.timezone), fragment("uniq(?) as visitors", e.user_id)}, - group_by: fragment("month"), - order_by: fragment("month") - ) - |> Enum.map(fn row -> {row["month"], row["visitors"]} end) - |> Enum.into(%{}) + steps = + Enum.map((query.steps - 1)..0, fn shift -> + Timex.now(site.timezone) + |> Timex.beginning_of_month() + |> Timex.shift(months: -shift) + |> DateTime.to_date() + end) - compare_groups = if query.filters["goal"] do + groups = Clickhouse.all( - from e in base_query(site, query), - select: {fragment("toStartOfMonth(toTimeZone(?, ?)) as month", e.timestamp, ^site.timezone), fragment("uniq(?) as visitors", e.user_id)}, - group_by: fragment("month"), - order_by: fragment("month") + from e in base_query(site, %{query | filters: %{}}), + select: + {fragment("toStartOfMonth(toTimeZone(?, ?)) as month", e.timestamp, ^site.timezone), + fragment("uniq(?) as visitors", e.user_id)}, + group_by: fragment("month"), + order_by: fragment("month") ) |> Enum.map(fn row -> {row["month"], row["visitors"]} end) |> Enum.into(%{}) - end - present_index = Enum.find_index(steps, fn step -> step == Timex.now(site.timezone) |> Timex.to_date |> Timex.beginning_of_month end) + compare_groups = + if query.filters["goal"] do + Clickhouse.all( + from e in base_query(site, query), + select: + {fragment("toStartOfMonth(toTimeZone(?, ?)) as month", e.timestamp, ^site.timezone), + fragment("uniq(?) as visitors", e.user_id)}, + group_by: fragment("month"), + order_by: fragment("month") + ) + |> Enum.map(fn row -> {row["month"], row["visitors"]} end) + |> Enum.into(%{}) + end + + present_index = + Enum.find_index(steps, fn step -> + step == Timex.now(site.timezone) |> Timex.to_date() |> Timex.beginning_of_month() + end) + plot = Enum.map(steps, fn step -> groups[step] || 0 end) compare_plot = compare_groups && Enum.map(steps, fn step -> compare_groups[step] || 0 end) labels = Enum.map(steps, fn step -> Timex.format!(step, "{ISOdate}") end) @@ -60,27 +72,35 @@ defmodule Plausible.Stats.Clickhouse do def calculate_plot(site, %Query{step_type: "date"} = query) do steps = Enum.into(query.date_range, []) - groups = Clickhouse.all( - from e in base_query(site, %{ query | filters: %{} }), - select: {fragment("toDate(toTimeZone(?, ?)) as day", e.timestamp, ^site.timezone), fragment("uniq(?) as visitors", e.user_id)}, - group_by: fragment("day"), - order_by: fragment("day") - ) - |> Enum.map(fn row -> {row["day"], row["visitors"]} end) - |> Enum.into(%{}) - - compare_groups = if query.filters["goal"] do + groups = Clickhouse.all( - from e in base_query(site, query), - select: {fragment("toDate(toTimeZone(?, ?)) as day", e.timestamp, ^site.timezone), fragment("uniq(?) as visitors", e.user_id)}, - group_by: fragment("day"), - order_by: fragment("day") + from e in base_query(site, %{query | filters: %{}}), + select: + {fragment("toDate(toTimeZone(?, ?)) as day", e.timestamp, ^site.timezone), + fragment("uniq(?) as visitors", e.user_id)}, + group_by: fragment("day"), + order_by: fragment("day") ) |> Enum.map(fn row -> {row["day"], row["visitors"]} end) |> Enum.into(%{}) - end - present_index = Enum.find_index(steps, fn step -> step == Timex.now(site.timezone) |> Timex.to_date end) + compare_groups = + if query.filters["goal"] do + Clickhouse.all( + from e in base_query(site, query), + select: + {fragment("toDate(toTimeZone(?, ?)) as day", e.timestamp, ^site.timezone), + fragment("uniq(?) as visitors", e.user_id)}, + group_by: fragment("day"), + order_by: fragment("day") + ) + |> Enum.map(fn row -> {row["day"], row["visitors"]} end) + |> Enum.into(%{}) + end + + present_index = + Enum.find_index(steps, fn step -> step == Timex.now(site.timezone) |> Timex.to_date() end) + steps_to_show = if present_index, do: present_index + 1, else: Enum.count(steps) plot = Enum.map(steps, fn step -> groups[step] || 0 end) |> Enum.take(steps_to_show) compare_plot = compare_groups && Enum.map(steps, fn step -> compare_groups[step] || 0 end) @@ -92,31 +112,44 @@ defmodule Plausible.Stats.Clickhouse do def calculate_plot(site, %Query{step_type: "hour"} = query) do steps = 0..23 - groups = Clickhouse.all( - from e in base_query(site, %{query | filters: %{}}), - select: {fragment("toHour(toTimeZone(?, ?)) as hour", e.timestamp, ^site.timezone), fragment("uniq(?) as visitors", e.user_id)}, - group_by: fragment("hour"), - order_by: fragment("hour") - ) - |> Enum.map(fn row -> {row["hour"], row["visitors"]} end) - |> Enum.into(%{}) - - compare_groups = if query.filters["goal"] do + groups = Clickhouse.all( - from e in base_query(site, query), - select: {fragment("toHour(toTimeZone(?, ?)) as hour", e.timestamp, ^site.timezone), fragment("uniq(?) as visitors", e.user_id)}, - group_by: fragment("hour"), - order_by: fragment("hour") + from e in base_query(site, %{query | filters: %{}}), + select: + {fragment("toHour(toTimeZone(?, ?)) as hour", e.timestamp, ^site.timezone), + fragment("uniq(?) as visitors", e.user_id)}, + group_by: fragment("hour"), + order_by: fragment("hour") ) |> Enum.map(fn row -> {row["hour"], row["visitors"]} end) |> Enum.into(%{}) - end + + compare_groups = + if query.filters["goal"] do + Clickhouse.all( + from e in base_query(site, query), + select: + {fragment("toHour(toTimeZone(?, ?)) as hour", e.timestamp, ^site.timezone), + fragment("uniq(?) as visitors", e.user_id)}, + group_by: fragment("hour"), + order_by: fragment("hour") + ) + |> Enum.map(fn row -> {row["hour"], row["visitors"]} end) + |> Enum.into(%{}) + end now = Timex.now(site.timezone) is_today = Timex.to_date(now) == query.date_range.first present_index = is_today && Enum.find_index(steps, fn step -> step == now.hour end) steps_to_show = if present_index, do: present_index + 1, else: Enum.count(steps) - labels = Enum.map(steps, fn step -> Timex.to_datetime(query.date_range.first) |> Timex.shift(hours: step) |> NaiveDateTime.to_iso8601 end) + + labels = + Enum.map(steps, fn step -> + Timex.to_datetime(query.date_range.first) + |> Timex.shift(hours: step) + |> NaiveDateTime.to_iso8601() + end) + plot = Enum.map(steps, fn step -> groups[step] || 0 end) |> Enum.take(steps_to_show) compare_plot = compare_groups && Enum.map(steps, fn step -> compare_groups[step] || 0 end) {plot, compare_plot, labels, present_index} @@ -125,66 +158,83 @@ defmodule Plausible.Stats.Clickhouse do def bounce_rate(site, query) do {first_datetime, last_datetime} = date_range_utc_boundaries(query.date_range, site.timezone) - [res] = Clickhouse.all( - from s in "sessions", - select: {fragment("round(sum(is_bounce * sign) / sum(sign) * 100) as bounce_rate")}, - where: s.domain == ^site.domain, - where: s.start >= ^first_datetime and s.start < ^last_datetime - ) + [res] = + Clickhouse.all( + from s in "sessions", + select: {fragment("round(sum(is_bounce * sign) / sum(sign) * 100) as bounce_rate")}, + where: s.domain == ^site.domain, + where: s.start >= ^first_datetime and s.start < ^last_datetime + ) + res["bounce_rate"] || 0 end def pageviews_and_visitors(site, query) do - [res] = Clickhouse.all( - from e in base_query(site, query), - select: {fragment("count(*) as pageviews"), fragment("uniq(user_id) as visitors")} - ) + [res] = + Clickhouse.all( + from e in base_query(site, query), + select: {fragment("count(*) as pageviews"), fragment("uniq(user_id) as visitors")} + ) + {res["pageviews"], res["visitors"]} end def unique_visitors(site, query) do - [res] = Clickhouse.all( - from e in base_query(site, query), - select: fragment("uniq(user_id) as visitors") - ) + [res] = + Clickhouse.all( + from e in base_query(site, query), + select: fragment("uniq(user_id) as visitors") + ) + res["visitors"] end def top_referrers_for_goal(site, query, limit \\ 5) do - converted_sessions = from( - from e in base_query(site, query), - select: %{session_id: e.session_id} - ) + converted_sessions = + from( + from e in base_query(site, query), + select: %{session_id: e.session_id} + ) Plausible.Clickhouse.all( from s in Plausible.ClickhouseSession, - join: cs in subquery(converted_sessions), - on: s.session_id == cs.session_id, - select: {fragment("? as name", s.referrer_source), fragment("any(?) as url", s.referrer), fragment("uniq(user_id) as count")}, - where: s.referrer_source != "", - group_by: s.referrer_source, - order_by: [desc: fragment("count")], - limit: ^limit - ) |> Enum.map(fn ref -> + join: cs in subquery(converted_sessions), + on: s.session_id == cs.session_id, + select: + {fragment("? as name", s.referrer_source), fragment("any(?) as url", s.referrer), + fragment("uniq(user_id) as count")}, + where: s.referrer_source != "", + group_by: s.referrer_source, + order_by: [desc: fragment("count")], + limit: ^limit + ) + |> Enum.map(fn ref -> Map.update(ref, "url", nil, fn url -> url && URI.parse("http://" <> url).host end) end) end def top_referrers(site, query, limit \\ 5, include \\ []) do - referrers = Clickhouse.all( - from e in base_query(site, query), - select: {fragment("? as name", e.referrer_source), fragment("any(?) as url", e.referrer), fragment("uniq(user_id) as count")}, - group_by: e.referrer_source, - where: e.referrer_source != "", - order_by: [desc: fragment("count")], - limit: ^limit - ) |> Enum.map(fn ref -> - Map.update(ref, "url", nil, fn url -> url && URI.parse("http://" <> url).host end) - end) + referrers = + Clickhouse.all( + from e in base_query(site, query), + select: + {fragment("? as name", e.referrer_source), fragment("any(?) as url", e.referrer), + fragment("uniq(user_id) as count")}, + group_by: e.referrer_source, + where: e.referrer_source != "", + order_by: [desc: fragment("count")], + limit: ^limit + ) + |> Enum.map(fn ref -> + Map.update(ref, "url", nil, fn url -> url && URI.parse("http://" <> url).host end) + end) if "bounce_rate" in include do bounce_rates = bounce_rates_by_referrer_source(site, query) - Enum.map(referrers, fn referrer -> Map.put(referrer, "bounce_rate", bounce_rates[referrer["name"]]) end) + + Enum.map(referrers, fn referrer -> + Map.put(referrer, "bounce_rate", bounce_rates[referrer["name"]]) + end) else referrers end @@ -195,67 +245,81 @@ defmodule Plausible.Stats.Clickhouse do Clickhouse.all( from s in "sessions", - select: {s.referrer_source, fragment("count(*) as total"), fragment("round(sum(is_bounce * sign) / sum(sign) * 100) as bounce_rate")}, - where: s.domain == ^site.domain, - where: s.start >= ^first_datetime and s.start < ^last_datetime, - where: s.referrer_source != "", - group_by: s.referrer_source, - order_by: [desc: fragment("total")], - limit: 100 - ) |> Enum.map(fn row -> {row["referrer_source"], row["bounce_rate"]} end) + select: + {s.referrer_source, fragment("count(*) as total"), + fragment("round(sum(is_bounce * sign) / sum(sign) * 100) as bounce_rate")}, + where: s.domain == ^site.domain, + where: s.start >= ^first_datetime and s.start < ^last_datetime, + where: s.referrer_source != "", + group_by: s.referrer_source, + order_by: [desc: fragment("total")], + limit: 100 + ) + |> Enum.map(fn row -> {row["referrer_source"], row["bounce_rate"]} end) |> Enum.into(%{}) end def visitors_from_referrer(site, query, referrer) do - [res] = Clickhouse.all( - from e in base_query(site, query), - select: fragment("uniq(user_id) as visitors"), - where: e.referrer_source == ^referrer - ) + [res] = + Clickhouse.all( + from e in base_query(site, query), + select: fragment("uniq(user_id) as visitors"), + where: e.referrer_source == ^referrer + ) + res["visitors"] end def conversions_from_referrer(site, query, referrer) do - converted_sessions = from( - from e in base_query(site, query), - select: %{session_id: e.session_id} - ) + converted_sessions = + from( + from e in base_query(site, query), + select: %{session_id: e.session_id} + ) - [res] = Plausible.Clickhouse.all( - from s in Plausible.ClickhouseSession, - join: cs in subquery(converted_sessions), - on: s.session_id == cs.session_id, - where: s.referrer_source == ^referrer, - select: fragment("uniq(user_id) as visitors") - ) + [res] = + Plausible.Clickhouse.all( + from s in Plausible.ClickhouseSession, + join: cs in subquery(converted_sessions), + on: s.session_id == cs.session_id, + where: s.referrer_source == ^referrer, + select: fragment("uniq(user_id) as visitors") + ) res["visitors"] end def referrer_drilldown(site, query, referrer, include \\ []) do - referring_urls = Clickhouse.all( - from e in base_query(site, query), - select: {fragment("? as name", e.referrer), fragment("uniq(user_id) as count")}, - group_by: e.referrer, - where: e.referrer_source == ^referrer, - order_by: [desc: fragment("count")], - limit: 100 - ) + referring_urls = + Clickhouse.all( + from e in base_query(site, query), + select: {fragment("? as name", e.referrer), fragment("uniq(user_id) as count")}, + group_by: e.referrer, + where: e.referrer_source == ^referrer, + order_by: [desc: fragment("count")], + limit: 100 + ) - referring_urls = if "bounce_rate" in include do - bounce_rates = bounce_rates_by_referring_url(site, query) - Enum.map(referring_urls, fn url -> Map.put(url, "bounce_rate", bounce_rates[url["name"]]) end) - else - referring_urls - end + referring_urls = + if "bounce_rate" in include do + bounce_rates = bounce_rates_by_referring_url(site, query) + + Enum.map(referring_urls, fn url -> + Map.put(url, "bounce_rate", bounce_rates[url["name"]]) + end) + else + referring_urls + end if referrer == "Twitter" do - urls = Enum.map(referring_urls, &(&1["name"])) + urls = Enum.map(referring_urls, & &1["name"]) - tweets = Repo.all( - from t in Plausible.Twitter.Tweet, - where: t.link in ^urls - ) |> Enum.group_by(&(&1.link)) + tweets = + Repo.all( + from t in Plausible.Twitter.Tweet, + where: t.link in ^urls + ) + |> Enum.group_by(& &1.link) Enum.map(referring_urls, fn url -> Map.put(url, "tweets", tweets[url["name"]]) @@ -266,20 +330,21 @@ defmodule Plausible.Stats.Clickhouse do end def referrer_drilldown_for_goal(site, query, referrer) do - converted_sessions = from( - from e in base_query(site, query), - select: %{session_id: e.session_id} - ) + converted_sessions = + from( + from e in base_query(site, query), + select: %{session_id: e.session_id} + ) Plausible.Clickhouse.all( from s in Plausible.ClickhouseSession, - join: cs in subquery(converted_sessions), - on: s.session_id == cs.session_id, - select: {fragment("? as name", s.referrer), fragment("uniq(user_id) as count")}, - where: s.referrer_source == ^referrer, - group_by: s.referrer, - order_by: [desc: fragment("count")], - limit: 100 + join: cs in subquery(converted_sessions), + on: s.session_id == cs.session_id, + select: {fragment("? as name", s.referrer), fragment("uniq(user_id) as count")}, + where: s.referrer_source == ^referrer, + group_by: s.referrer, + order_by: [desc: fragment("count")], + limit: 100 ) end @@ -288,25 +353,29 @@ defmodule Plausible.Stats.Clickhouse do Clickhouse.all( from s in "sessions", - select: {s.referrer, fragment("count(*) as total"), fragment("round(sum(is_bounce * sign) / sum(sign) * 100) as bounce_rate")}, - where: s.domain == ^site.domain, - where: s.start >= ^first_datetime and s.start < ^last_datetime, - where: s.referrer != "", - group_by: s.referrer, - order_by: [desc: fragment("total")], - limit: 100 - ) |> Enum.map(fn row -> {row["referrer"], row["bounce_rate"]} end) + select: + {s.referrer, fragment("count(*) as total"), + fragment("round(sum(is_bounce * sign) / sum(sign) * 100) as bounce_rate")}, + where: s.domain == ^site.domain, + where: s.start >= ^first_datetime and s.start < ^last_datetime, + where: s.referrer != "", + group_by: s.referrer, + order_by: [desc: fragment("total")], + limit: 100 + ) + |> Enum.map(fn row -> {row["referrer"], row["bounce_rate"]} end) |> Enum.into(%{}) end def top_pages(site, query, limit \\ 5, include \\ []) do - pages = Clickhouse.all( - from e in base_query(site, query), - select: {fragment("? as name", e.pathname), fragment("count(?) as count", e.pathname)}, - group_by: e.pathname, - order_by: [desc: fragment("count")], - limit: ^limit - ) + pages = + Clickhouse.all( + from e in base_query(site, query), + select: {fragment("? as name", e.pathname), fragment("count(?) as count", e.pathname)}, + group_by: e.pathname, + order_by: [desc: fragment("count")], + limit: ^limit + ) if "bounce_rate" in include do bounce_rates = bounce_rates_by_page_url(site, query) @@ -321,18 +390,22 @@ defmodule Plausible.Stats.Clickhouse do Clickhouse.all( from s in "sessions", - select: {s.entry_page, fragment("count(*) as total"), fragment("round(sum(is_bounce * sign) / sum(sign) * 100) as bounce_rate")}, - where: s.domain == ^site.domain, - where: s.start >= ^first_datetime and s.start < ^last_datetime, - group_by: s.entry_page, - order_by: [desc: fragment("total")], - limit: 100 - ) |> Enum.map(fn row -> {row["entry_page"], row["bounce_rate"]} end) + select: + {s.entry_page, fragment("count(*) as total"), + fragment("round(sum(is_bounce * sign) / sum(sign) * 100) as bounce_rate")}, + where: s.domain == ^site.domain, + where: s.start >= ^first_datetime and s.start < ^last_datetime, + group_by: s.entry_page, + order_by: [desc: fragment("total")], + limit: 100 + ) + |> Enum.map(fn row -> {row["entry_page"], row["bounce_rate"]} end) |> Enum.into(%{}) end defp add_percentages(stat_list) do total = Enum.reduce(stat_list, 0, fn %{"count" => count}, total -> total + count end) + Enum.map(stat_list, fn stat -> Map.put(stat, "percentage", round(stat["count"] / total * 100)) end) @@ -343,9 +416,9 @@ defmodule Plausible.Stats.Clickhouse do def top_screen_sizes(site, query) do Clickhouse.all( from e in base_query(site, query), - select: {fragment("? as name", e.screen_size), fragment("uniq(user_id) as count")}, - group_by: e.screen_size, - where: e.screen_size != "" + select: {fragment("? as name", e.screen_size), fragment("uniq(user_id) as count")}, + group_by: e.screen_size, + where: e.screen_size != "" ) |> Enum.sort(fn %{"name" => screen_size1}, %{"name" => screen_size2} -> index1 = Enum.find_index(@available_screen_sizes, fn s -> s == screen_size1 end) @@ -358,13 +431,14 @@ defmodule Plausible.Stats.Clickhouse do def countries(site, query) do Clickhouse.all( from e in base_query(site, query), - select: {fragment("? as name", e.country_code), fragment("uniq(user_id) as count")}, - group_by: e.country_code, - where: e.country_code != "\0\0", - order_by: [desc: fragment("count")] + select: {fragment("? as name", e.country_code), fragment("uniq(user_id) as count")}, + group_by: e.country_code, + where: e.country_code != "\0\0", + order_by: [desc: fragment("count")] ) |> Enum.map(fn stat -> two_letter_code = stat["name"] + stat |> Map.put("name", Plausible.Stats.CountryName.to_alpha3(two_letter_code)) |> Map.put("full_country_name", Plausible.Stats.CountryName.from_iso3166(two_letter_code)) @@ -375,10 +449,10 @@ defmodule Plausible.Stats.Clickhouse do def browsers(site, query, limit \\ 5) do Clickhouse.all( from e in base_query(site, query), - select: {fragment("? as name", e.browser), fragment("uniq(user_id) as count")}, - group_by: e.browser, - where: e.browser != "", - order_by: [desc: fragment("count")] + select: {fragment("? as name", e.browser), fragment("uniq(user_id) as count")}, + group_by: e.browser, + where: e.browser != "", + order_by: [desc: fragment("count")] ) |> add_percentages |> Enum.take(limit) @@ -387,72 +461,81 @@ defmodule Plausible.Stats.Clickhouse do def operating_systems(site, query, limit \\ 5) do Clickhouse.all( from e in base_query(site, query), - select: {fragment("? as name", e.operating_system), fragment("uniq(user_id) as count")}, - group_by: e.operating_system, - where: e.operating_system != "", - order_by: [desc: fragment("count")] + select: {fragment("? as name", e.operating_system), fragment("uniq(user_id) as count")}, + group_by: e.operating_system, + where: e.operating_system != "", + order_by: [desc: fragment("count")] ) |> add_percentages |> Enum.take(limit) end def current_visitors(site) do - [res] = Clickhouse.all( - from e in "events", - select: fragment("uniq(user_id) as visitors"), - where: e.timestamp >= fragment("now() - INTERVAL 5 MINUTE"), - where: e.domain == ^site.domain - ) + [res] = + Clickhouse.all( + from e in "events", + select: fragment("uniq(user_id) as visitors"), + where: e.timestamp >= fragment("now() - INTERVAL 5 MINUTE"), + where: e.domain == ^site.domain + ) res["visitors"] end def has_pageviews?([]), do: false + def has_pageviews?(domains) when is_list(domains) do - res = Clickhouse.all( - from e in "events", - select: e.timestamp, - where: fragment("? IN tuple(?)", e.domain, ^domains), - limit: 1 - ) + res = + Clickhouse.all( + from e in "events", + select: e.timestamp, + where: fragment("? IN tuple(?)", e.domain, ^domains), + limit: 1 + ) + !Enum.empty?(res) end def has_pageviews?(site) do - res = Clickhouse.all( - from e in "events", - select: e.timestamp, - where: e.domain == ^site.domain, - limit: 1 - ) + res = + Clickhouse.all( + from e in "events", + select: e.timestamp, + where: e.domain == ^site.domain, + limit: 1 + ) + !Enum.empty?(res) end def goal_conversions(site, %Query{filters: %{"goal" => goal}} = query) when is_binary(goal) do Clickhouse.all( from e in base_query(site, query), - select: {e.name, fragment("uniq(user_id) as count")}, - group_by: e.name, - order_by: [desc: fragment("count")] - ) |> Enum.map(fn row -> %{"name" => goal, "count" => row["count"]} end) + select: {e.name, fragment("uniq(user_id) as count")}, + group_by: e.name, + order_by: [desc: fragment("count")] + ) + |> Enum.map(fn row -> %{"name" => goal, "count" => row["count"]} end) end def goal_conversions(site, query) do goals = Repo.all(from g in Plausible.Goal, where: g.domain == ^site.domain) - fetch_pageview_goals(goals, site, query) - ++ fetch_event_goals(goals, site, query) + + (fetch_pageview_goals(goals, site, query) ++ + fetch_event_goals(goals, site, query)) |> sort_conversions() end defp fetch_event_goals(goals, site, query) do - events = Enum.map(goals, fn goal -> goal.event_name end) - |> Enum.filter(&(&1)) + events = + Enum.map(goals, fn goal -> goal.event_name end) + |> Enum.filter(& &1) if Enum.count(events) > 0 do Clickhouse.all( from e in base_query(site, query, events), - select: {e.name, fragment("uniq(user_id) as count")}, - group_by: e.name + select: {e.name, fragment("uniq(user_id) as count")}, + group_by: e.name ) else [] @@ -460,15 +543,18 @@ defmodule Plausible.Stats.Clickhouse do end defp fetch_pageview_goals(goals, site, query) do - pages = Enum.map(goals, fn goal -> goal.page_path end) - |> Enum.filter(&(&1)) + pages = + Enum.map(goals, fn goal -> goal.page_path end) + |> Enum.filter(& &1) if Enum.count(pages) > 0 do Clickhouse.all( from e in base_query(site, query), - select: {fragment("concat('Visit ', ?) as name", e.pathname), fragment("uniq(user_id) as count")}, - where: fragment("? IN tuple(?)", e.pathname, ^pages), - group_by: e.pathname + select: + {fragment("concat('Visit ', ?) as name", e.pathname), + fragment("uniq(user_id) as count")}, + where: fragment("? IN tuple(?)", e.pathname, ^pages), + group_by: e.pathname ) else [] @@ -483,16 +569,18 @@ defmodule Plausible.Stats.Clickhouse do {first_datetime, last_datetime} = date_range_utc_boundaries(query.date_range, site.timezone) {goal_event, path} = event_name_for_goal(query) - q = from(e in "events", - where: e.domain == ^site.domain, - where: e.timestamp >= ^first_datetime and e.timestamp < ^last_datetime - ) + q = + from(e in "events", + where: e.domain == ^site.domain, + where: e.timestamp >= ^first_datetime and e.timestamp < ^last_datetime + ) - q = if path do - from(e in q, where: e.pathname == ^path) - else - q - end + q = + if path do + from(e in q, where: e.pathname == ^path) + else + q + end if goal_event do from(e in q, where: e.name == ^goal_event) @@ -503,12 +591,16 @@ defmodule Plausible.Stats.Clickhouse do defp date_range_utc_boundaries(date_range, timezone) do {:ok, first} = NaiveDateTime.new(date_range.first, ~T[00:00:00]) - first_datetime = Timex.to_datetime(first, timezone) - |> Timex.Timezone.convert("UTC") + + first_datetime = + Timex.to_datetime(first, timezone) + |> Timex.Timezone.convert("UTC") {:ok, last} = NaiveDateTime.new(date_range.last |> Timex.shift(days: 1), ~T[00:00:00]) - last_datetime = Timex.to_datetime(last, timezone) - |> Timex.Timezone.convert("UTC") + + last_datetime = + Timex.to_datetime(last, timezone) + |> Timex.Timezone.convert("UTC") {first_datetime, last_datetime} end @@ -517,8 +609,10 @@ defmodule Plausible.Stats.Clickhouse do case query.filters["goal"] do "Visit " <> page -> {"pageview", page} + goal when is_binary(goal) -> {goal, nil} + _ -> {nil, nil} end diff --git a/lib/plausible/stats/countries.ex b/lib/plausible/stats/countries.ex index af0a33eae..adb2ca563 100644 --- a/lib/plausible/stats/countries.ex +++ b/lib/plausible/stats/countries.ex @@ -491,7 +491,7 @@ defmodule Plausible.Stats.CountryName do "CU" => "CUB", "TF" => "ATF", "EH" => "ESH" - } + } def to_alpha3(code) do Map.get(@alpha3_codes, code, code) diff --git a/lib/plausible/stats/query.ex b/lib/plausible/stats/query.ex index cc3537344..018993693 100644 --- a/lib/plausible/stats/query.ex +++ b/lib/plausible/stats/query.ex @@ -1,5 +1,5 @@ defmodule Plausible.Stats.Query do - defstruct [date_range: nil, step_type: nil, period: nil, steps: nil, filters: %{}] + defstruct date_range: nil, step_type: nil, period: nil, steps: nil, filters: %{} def shift_back(%__MODULE__{period: "day"} = query) do new_date = query.date_range.first |> Timex.shift(days: -1) @@ -72,7 +72,7 @@ defmodule Plausible.Stats.Query do end def from(_tz, %{"period" => "month", "date" => date} = params) do - start_date = Date.from_iso8601!(date) |> Timex.beginning_of_month + start_date = Date.from_iso8601!(date) |> Timex.beginning_of_month() end_date = Timex.end_of_month(start_date) %__MODULE__{ @@ -85,8 +85,9 @@ defmodule Plausible.Stats.Query do end def from(tz, %{"period" => "6mo"} = params) do - start_date = Timex.shift(today(tz), months: -5) - |> Timex.beginning_of_month() + start_date = + Timex.shift(today(tz), months: -5) + |> Timex.beginning_of_month() %__MODULE__{ period: "6mo", @@ -98,8 +99,9 @@ defmodule Plausible.Stats.Query do end def from(tz, %{"period" => "12mo"} = params) do - start_date = Timex.shift(today(tz), months: -11) - |> Timex.beginning_of_month() + start_date = + Timex.shift(today(tz), months: -11) + |> Timex.beginning_of_month() %__MODULE__{ period: "12mo", @@ -127,7 +129,7 @@ defmodule Plausible.Stats.Query do end defp today(tz) do - Timex.now(tz) |> Timex.to_date + Timex.now(tz) |> Timex.to_date() end defp parse_filters(params) do @@ -136,4 +138,3 @@ defmodule Plausible.Stats.Query do end end end - diff --git a/lib/plausible/timezones.ex b/lib/plausible/timezones.ex index e49cba3ae..816e31b96 100644 --- a/lib/plausible/timezones.ex +++ b/lib/plausible/timezones.ex @@ -1,89 +1,120 @@ defmodule Plausible.Timezones do - @moduledoc "https://stackoverflow.com/a/52265733" + @moduledoc "https://stackoverflow.com/a/52265733" @options [ [key: "(GMT-12:00) International Date Line West", value: "Etc/GMT+12", offset: "720"], - [key: "(GMT-11:00) Midway Island, Samoa",value: "Pacific/Midway", offset: "660"], - [key: "(GMT-10:00) Hawaii",value: "Pacific/Honolulu", offset: "600"], - [key: "(GMT-09:00) Alaska",value: "US/Alaska", offset: "540"], - [key: "(GMT-08:00) Pacific Time (US & Canada)",value: "America/Los_Angeles", offset: "480"], - [key: "(GMT-08:00) Tijuana, Baja California",value: "America/Tijuana"], - [key: "(GMT-07:00) Arizona",value: "US/Arizona"], - [key: "(GMT-07:00) Chihuahua, La Paz, Mazatlan",value: "America/Chihuahua"], - [key: "(GMT-07:00) Mountain Time (US & Canada)",value: "US/Mountain", offset: "420"], - [key: "(GMT-06:00) Central America",value: "America/Managua"], - [key: "(GMT-06:00) Central Time (US & Canada)",value: "US/Central", offset: "360"], - [key: "(GMT-06:00) Guadalajara, Mexico City, Monterrey",value: "America/Mexico_City"], - [key: "(GMT-06:00) Saskatchewan",value: "Canada/Saskatchewan"], - [key: "(GMT-05:00) Bogota, Lima, Quito, Rio Branco",value: "America/Bogota"], - [key: "(GMT-05:00) Eastern Time (US & Canada)",value: "US/Eastern", offset: "300"], - [key: "(GMT-05:00) Indiana (East)",value: "US/East-Indiana"], - [key: "(GMT-04:00) Atlantic Time (Canada)",value: "Canada/Atlantic", offset: "240"], - [key: "(GMT-04:00) Caracas, La Paz",value: "America/Caracas"], - [key: "(GMT-04:00) Manaus",value: "America/Manaus"], - [key: "(GMT-04:00) Santiago",value: "America/Santiago"], - [key: "(GMT-03:30) Newfoundland",value: "Canada/Newfoundland"], - [key: "(GMT-03:00) Brasilia",value: "America/Sao_Paulo", offset: "180"], - [key: "(GMT-03:00) Buenos Aires, Georgetown",value: "America/Argentina/Buenos_Aires"], - [key: "(GMT-03:00) Greenland",value: "America/Godthab"], - [key: "(GMT-03:00) Montevideo",value: "America/Montevideo"], - [key: "(GMT-02:00) Mid-Atlantic",value: "America/Noronha", offset: "120"], - [key: "(GMT-01:00) Cape Verde Is.",value: "Atlantic/Cape_Verde", offset: "60"], - [key: "(GMT-01:00) Azores",value: "Atlantic/Azores"], - [key: "(GMT+00:00) Casablanca, Monrovia, Reykjavik",value: "Africa/Casablanca"], - [key: "(GMT+00:00) Greenwich Mean Time : Dublin, Edinburgh, Lisbon, London",value: "Etc/Greenwich", offset: "0"], - [key: "(GMT+01:00) Amsterdam, Berlin, Bern, Rome, Stockholm, Vienna",value: "Europe/Amsterdam", offset: "-60"], - [key: "(GMT+01:00) Belgrade, Bratislava, Budapest, Ljubljana, Prague",value: "Europe/Belgrade"], - [key: "(GMT+01:00) Brussels, Copenhagen, Madrid, Paris",value: "Europe/Brussels"], - [key: "(GMT+01:00) Sarajevo, Skopje, Warsaw, Zagreb",value: "Europe/Sarajevo"], - [key: "(GMT+01:00) West Central Africa",value: "Africa/Lagos"], - [key: "(GMT+02:00) Amman",value: "Asia/Amman"], - [key: "(GMT+02:00) Athens, Bucharest, Istanbul",value: "Europe/Athens"], - [key: "(GMT+02:00) Beirut",value: "Asia/Beirut"], - [key: "(GMT+02:00) Cairo",value: "Africa/Cairo"], - [key: "(GMT+02:00) Harare, Pretoria",value: "Africa/Harare"], - [key: "(GMT+02:00) Helsinki, Kyiv, Riga, Sofia, Tallinn, Vilnius",value: "Europe/Helsinki", offset: "-120"], - [key: "(GMT+02:00) Jerusalem",value: "Asia/Jerusalem"], - [key: "(GMT+02:00) Minsk",value: "Europe/Minsk"], - [key: "(GMT+02:00) Windhoek",value: "Africa/Windhoek"], - [key: "(GMT+03:00) Kuwait, Riyadh, Baghdad",value: "Asia/Kuwait"], - [key: "(GMT+03:00) Moscow, St. Petersburg, Volgograd",value: "Europe/Moscow", offset: "-180"], - [key: "(GMT+03:00) Nairobi",value: "Africa/Nairobi"], - [key: "(GMT+03:00) Tbilisi",value: "Asia/Tbilisi"], - [key: "(GMT+03:30) Tehran",value: "Asia/Tehran", offset: "-210"], - [key: "(GMT+04:00) Abu Dhabi, Muscat",value: "Asia/Muscat", offset: "-240"], - [key: "(GMT+04:00) Baku",value: "Asia/Baku"], - [key: "(GMT+04:00) Yerevan",value: "Asia/Yerevan"], - [key: "(GMT+04:30) Kabul",value: "Asia/Kabul", offset: "-270"], - [key: "(GMT+05:00) Yekaterinburg",value: "Asia/Yekaterinburg"], - [key: "(GMT+05:00) Islamabad, Karachi, Tashkent",value: "Asia/Karachi", offset: "-300"], - [key: "(GMT+05:30) Chennai, Kolkata, Mumbai, New Delhi",value: "Asia/Calcutta", offset: "-330"], - [key: "(GMT+05:30) Sri Jayawardenapura",value: "Asia/Calcutta"], - [key: "(GMT+05:45) Kathmandu",value: "Asia/Katmandu", offset: "-345"], - [key: "(GMT+06:00) Almaty, Novosibirsk",value: "Asia/Almaty", offset: "-360"], - [key: "(GMT+06:00) Astana, Dhaka",value: "Asia/Dhaka"], - [key: "(GMT+06:30) Yangon (Rangoon)",value: "Asia/Rangoon", offset: "-390"], - [key: "(GMT+07:00) Bangkok, Hanoi, Jakarta",value: "Asia/Bangkok", offset: "-420"], - [key: "(GMT+07:00) Krasnoyarsk",value: "Asia/Krasnoyarsk"], - [key: "(GMT+08:00) Beijing, Chongqing, Hong Kong, Urumqi",value: "Asia/Hong_Kong", offset: "-480"], - [key: "(GMT+08:00) Kuala Lumpur, Singapore",value: "Asia/Kuala_Lumpur"], - [key: "(GMT+08:00) Irkutsk, Ulaan Bataar",value: "Asia/Irkutsk"], - [key: "(GMT+08:00) Perth",value: "Australia/Perth"], - [key: "(GMT+08:00) Taipei",value: "Asia/Taipei"], - [key: "(GMT+09:00) Osaka, Sapporo, Tokyo",value: "Asia/Tokyo", offset: "-540"], - [key: "(GMT+09:00) Seoul",value: "Asia/Seoul"], - [key: "(GMT+09:00) Yakutsk",value: "Asia/Yakutsk"], - [key: "(GMT+09:30) Adelaide",value: "Australia/Adelaide", offset: "-570"], - [key: "(GMT+09:30) Darwin",value: "Australia/Darwin"], - [key: "(GMT+10:00) Brisbane",value: "Australia/Brisbane", offset: "-600"], - [key: "(GMT+10:00) Canberra, Melbourne, Sydney",value: "Australia/Canberra"], - [key: "(GMT+10:00) Hobart",value: "Australia/Hobart"], - [key: "(GMT+10:00) Guam, Port Moresby",value: "Pacific/Guam"], - [key: "(GMT+10:00) Vladivostok",value: "Asia/Vladivostok"], - [key: "(GMT+11:00) Magadan, Solomon Is., New Caledonia",value: "Asia/Magadan", offset: "-660"], - [key: "(GMT+12:00) Auckland, Wellington",value: "Pacific/Auckland", offset: "-720"], - [key: "(GMT+12:00) Fiji, Kamchatka, Marshall Is.",value: "Pacific/Fiji"], - [key: "(GMT+13:00) Nuku'alofa",value: "Pacific/Tongatapu", offset: "-780"], + [key: "(GMT-11:00) Midway Island, Samoa", value: "Pacific/Midway", offset: "660"], + [key: "(GMT-10:00) Hawaii", value: "Pacific/Honolulu", offset: "600"], + [key: "(GMT-09:00) Alaska", value: "US/Alaska", offset: "540"], + [key: "(GMT-08:00) Pacific Time (US & Canada)", value: "America/Los_Angeles", offset: "480"], + [key: "(GMT-08:00) Tijuana, Baja California", value: "America/Tijuana"], + [key: "(GMT-07:00) Arizona", value: "US/Arizona"], + [key: "(GMT-07:00) Chihuahua, La Paz, Mazatlan", value: "America/Chihuahua"], + [key: "(GMT-07:00) Mountain Time (US & Canada)", value: "US/Mountain", offset: "420"], + [key: "(GMT-06:00) Central America", value: "America/Managua"], + [key: "(GMT-06:00) Central Time (US & Canada)", value: "US/Central", offset: "360"], + [key: "(GMT-06:00) Guadalajara, Mexico City, Monterrey", value: "America/Mexico_City"], + [key: "(GMT-06:00) Saskatchewan", value: "Canada/Saskatchewan"], + [key: "(GMT-05:00) Bogota, Lima, Quito, Rio Branco", value: "America/Bogota"], + [key: "(GMT-05:00) Eastern Time (US & Canada)", value: "US/Eastern", offset: "300"], + [key: "(GMT-05:00) Indiana (East)", value: "US/East-Indiana"], + [key: "(GMT-04:00) Atlantic Time (Canada)", value: "Canada/Atlantic", offset: "240"], + [key: "(GMT-04:00) Caracas, La Paz", value: "America/Caracas"], + [key: "(GMT-04:00) Manaus", value: "America/Manaus"], + [key: "(GMT-04:00) Santiago", value: "America/Santiago"], + [key: "(GMT-03:30) Newfoundland", value: "Canada/Newfoundland"], + [key: "(GMT-03:00) Brasilia", value: "America/Sao_Paulo", offset: "180"], + [key: "(GMT-03:00) Buenos Aires, Georgetown", value: "America/Argentina/Buenos_Aires"], + [key: "(GMT-03:00) Greenland", value: "America/Godthab"], + [key: "(GMT-03:00) Montevideo", value: "America/Montevideo"], + [key: "(GMT-02:00) Mid-Atlantic", value: "America/Noronha", offset: "120"], + [key: "(GMT-01:00) Cape Verde Is.", value: "Atlantic/Cape_Verde", offset: "60"], + [key: "(GMT-01:00) Azores", value: "Atlantic/Azores"], + [key: "(GMT+00:00) Casablanca, Monrovia, Reykjavik", value: "Africa/Casablanca"], + [ + key: "(GMT+00:00) Greenwich Mean Time : Dublin, Edinburgh, Lisbon, London", + value: "Etc/Greenwich", + offset: "0" + ], + [ + key: "(GMT+01:00) Amsterdam, Berlin, Bern, Rome, Stockholm, Vienna", + value: "Europe/Amsterdam", + offset: "-60" + ], + [ + key: "(GMT+01:00) Belgrade, Bratislava, Budapest, Ljubljana, Prague", + value: "Europe/Belgrade" + ], + [key: "(GMT+01:00) Brussels, Copenhagen, Madrid, Paris", value: "Europe/Brussels"], + [key: "(GMT+01:00) Sarajevo, Skopje, Warsaw, Zagreb", value: "Europe/Sarajevo"], + [key: "(GMT+01:00) West Central Africa", value: "Africa/Lagos"], + [key: "(GMT+02:00) Amman", value: "Asia/Amman"], + [key: "(GMT+02:00) Athens, Bucharest, Istanbul", value: "Europe/Athens"], + [key: "(GMT+02:00) Beirut", value: "Asia/Beirut"], + [key: "(GMT+02:00) Cairo", value: "Africa/Cairo"], + [key: "(GMT+02:00) Harare, Pretoria", value: "Africa/Harare"], + [ + key: "(GMT+02:00) Helsinki, Kyiv, Riga, Sofia, Tallinn, Vilnius", + value: "Europe/Helsinki", + offset: "-120" + ], + [key: "(GMT+02:00) Jerusalem", value: "Asia/Jerusalem"], + [key: "(GMT+02:00) Minsk", value: "Europe/Minsk"], + [key: "(GMT+02:00) Windhoek", value: "Africa/Windhoek"], + [key: "(GMT+03:00) Kuwait, Riyadh, Baghdad", value: "Asia/Kuwait"], + [ + key: "(GMT+03:00) Moscow, St. Petersburg, Volgograd", + value: "Europe/Moscow", + offset: "-180" + ], + [key: "(GMT+03:00) Nairobi", value: "Africa/Nairobi"], + [key: "(GMT+03:00) Tbilisi", value: "Asia/Tbilisi"], + [key: "(GMT+03:30) Tehran", value: "Asia/Tehran", offset: "-210"], + [key: "(GMT+04:00) Abu Dhabi, Muscat", value: "Asia/Muscat", offset: "-240"], + [key: "(GMT+04:00) Baku", value: "Asia/Baku"], + [key: "(GMT+04:00) Yerevan", value: "Asia/Yerevan"], + [key: "(GMT+04:30) Kabul", value: "Asia/Kabul", offset: "-270"], + [key: "(GMT+05:00) Yekaterinburg", value: "Asia/Yekaterinburg"], + [key: "(GMT+05:00) Islamabad, Karachi, Tashkent", value: "Asia/Karachi", offset: "-300"], + [ + key: "(GMT+05:30) Chennai, Kolkata, Mumbai, New Delhi", + value: "Asia/Calcutta", + offset: "-330" + ], + [key: "(GMT+05:30) Sri Jayawardenapura", value: "Asia/Calcutta"], + [key: "(GMT+05:45) Kathmandu", value: "Asia/Katmandu", offset: "-345"], + [key: "(GMT+06:00) Almaty, Novosibirsk", value: "Asia/Almaty", offset: "-360"], + [key: "(GMT+06:00) Astana, Dhaka", value: "Asia/Dhaka"], + [key: "(GMT+06:30) Yangon (Rangoon)", value: "Asia/Rangoon", offset: "-390"], + [key: "(GMT+07:00) Bangkok, Hanoi, Jakarta", value: "Asia/Bangkok", offset: "-420"], + [key: "(GMT+07:00) Krasnoyarsk", value: "Asia/Krasnoyarsk"], + [ + key: "(GMT+08:00) Beijing, Chongqing, Hong Kong, Urumqi", + value: "Asia/Hong_Kong", + offset: "-480" + ], + [key: "(GMT+08:00) Kuala Lumpur, Singapore", value: "Asia/Kuala_Lumpur"], + [key: "(GMT+08:00) Irkutsk, Ulaan Bataar", value: "Asia/Irkutsk"], + [key: "(GMT+08:00) Perth", value: "Australia/Perth"], + [key: "(GMT+08:00) Taipei", value: "Asia/Taipei"], + [key: "(GMT+09:00) Osaka, Sapporo, Tokyo", value: "Asia/Tokyo", offset: "-540"], + [key: "(GMT+09:00) Seoul", value: "Asia/Seoul"], + [key: "(GMT+09:00) Yakutsk", value: "Asia/Yakutsk"], + [key: "(GMT+09:30) Adelaide", value: "Australia/Adelaide", offset: "-570"], + [key: "(GMT+09:30) Darwin", value: "Australia/Darwin"], + [key: "(GMT+10:00) Brisbane", value: "Australia/Brisbane", offset: "-600"], + [key: "(GMT+10:00) Canberra, Melbourne, Sydney", value: "Australia/Canberra"], + [key: "(GMT+10:00) Hobart", value: "Australia/Hobart"], + [key: "(GMT+10:00) Guam, Port Moresby", value: "Pacific/Guam"], + [key: "(GMT+10:00) Vladivostok", value: "Asia/Vladivostok"], + [ + key: "(GMT+11:00) Magadan, Solomon Is., New Caledonia", + value: "Asia/Magadan", + offset: "-660" + ], + [key: "(GMT+12:00) Auckland, Wellington", value: "Pacific/Auckland", offset: "-720"], + [key: "(GMT+12:00) Fiji, Kamchatka, Marshall Is.", value: "Pacific/Fiji"], + [key: "(GMT+13:00) Nuku'alofa", value: "Pacific/Tongatapu", offset: "-780"] ] def options() do diff --git a/lib/plausible/twitter/api.ex b/lib/plausible/twitter/api.ex index af0a69d08..bdb13211e 100644 --- a/lib/plausible/twitter/api.ex +++ b/lib/plausible/twitter/api.ex @@ -1,9 +1,18 @@ defmodule Plausible.Twitter.Api do def search(link) do params = [{"count", 5}, {"tweet_mode", "extended"}, {"q", "https://#{link} -filter:retweets"}] - params = OAuther.sign("get", "https://api.twitter.com/1.1/search/tweets.json", params, oauth_credentials()) - uri = "https://api.twitter.com/1.1/search/tweets.json?" <> URI.encode_query(params) + + params = + OAuther.sign( + "get", + "https://api.twitter.com/1.1/search/tweets.json", + params, + oauth_credentials() + ) + + uri = "https://api.twitter.com/1.1/search/tweets.json?" <> URI.encode_query(params) response = HTTPoison.get!(uri) + Jason.decode!(response.body) |> Map.get("statuses") end diff --git a/lib/plausible/twitter/tweet.ex b/lib/plausible/twitter/tweet.ex index 8570768aa..0e08fc87f 100644 --- a/lib/plausible/twitter/tweet.ex +++ b/lib/plausible/twitter/tweet.ex @@ -2,7 +2,15 @@ defmodule Plausible.Twitter.Tweet do use Ecto.Schema import Ecto.Changeset - @required_fields [:link, :tweet_id, :author_handle, :author_name, :author_image, :text, :created] + @required_fields [ + :link, + :tweet_id, + :author_handle, + :author_name, + :author_image, + :text, + :created + ] @derive {Jason.Encoder, only: @required_fields} schema "tweets" do diff --git a/lib/plausible_release.ex b/lib/plausible_release.ex index a018c67c2..59fef555a 100644 --- a/lib/plausible_release.ex +++ b/lib/plausible_release.ex @@ -106,73 +106,72 @@ defmodule Plausible.Release do for repo <- repos() do :ok = ensure_repo_created(repo) end + do_create_ch_db() end - defp do_create_ch_db() do - db_to_create = Keyword.get(Application.get_env(:plausible, :clickhouse),:database) + db_to_create = Keyword.get(Application.get_env(:plausible, :clickhouse), :database) IO.puts("create #{inspect(db_to_create)} clickhouse database/tables if it doesn't exist") Clickhousex.query(:clickhouse, "CREATE DATABASE IF NOT EXISTS #{db_to_create}", []) - tb_events = """ - CREATE TABLE IF NOT EXISTS #{db_to_create}.events ( - timestamp DateTime, - name String, - domain String, - user_id UInt64, - session_id UInt64, - hostname String, - pathname String, - referrer String, - referrer_source String, - initial_referrer String, - initial_referrer_source String, - country_code LowCardinality(FixedString(2)), - screen_size LowCardinality(String), - operating_system LowCardinality(String), - browser LowCardinality(String) - ) ENGINE = MergeTree() - PARTITION BY toYYYYMM(timestamp) - ORDER BY (name, domain, user_id, timestamp) - SETTINGS index_granularity = 8192 - """ + tb_events = """ + CREATE TABLE IF NOT EXISTS #{db_to_create}.events ( + timestamp DateTime, + name String, + domain String, + user_id UInt64, + session_id UInt64, + hostname String, + pathname String, + referrer String, + referrer_source String, + initial_referrer String, + initial_referrer_source String, + country_code LowCardinality(FixedString(2)), + screen_size LowCardinality(String), + operating_system LowCardinality(String), + browser LowCardinality(String) + ) ENGINE = MergeTree() + PARTITION BY toYYYYMM(timestamp) + ORDER BY (name, domain, user_id, timestamp) + SETTINGS index_granularity = 8192 + """ Clickhousex.query(:clickhouse, tb_events, []) tb_sessions = """ - CREATE TABLE IF NOT EXISTS #{db_to_create}.sessions ( - session_id UInt64, - sign Int8, - domain String, - user_id UInt64, - hostname String, - timestamp DateTime, - start DateTime, - is_bounce UInt8, - entry_page String, - exit_page String, - pageviews Int32, - events Int32, - duration UInt32, - referrer String, - referrer_source String, - country_code LowCardinality(FixedString(2)), - screen_size LowCardinality(String), - operating_system LowCardinality(String), - browser LowCardinality(String) - ) ENGINE = CollapsingMergeTree(sign) - PARTITION BY toYYYYMM(start) - ORDER BY (domain, user_id, session_id, start) - SETTINGS index_granularity = 8192 - """ - - Clickhousex.query(:clickhouse, tb_sessions, []) + CREATE TABLE IF NOT EXISTS #{db_to_create}.sessions ( + session_id UInt64, + sign Int8, + domain String, + user_id UInt64, + hostname String, + timestamp DateTime, + start DateTime, + is_bounce UInt8, + entry_page String, + exit_page String, + pageviews Int32, + events Int32, + duration UInt32, + referrer String, + referrer_source String, + country_code LowCardinality(FixedString(2)), + screen_size LowCardinality(String), + operating_system LowCardinality(String), + browser LowCardinality(String) + ) ENGINE = CollapsingMergeTree(sign) + PARTITION BY toYYYYMM(start) + ORDER BY (domain, user_id, session_id, start) + SETTINGS index_granularity = 8192 + """ + + Clickhousex.query(:clickhouse, tb_sessions, []) end - defp ensure_repo_created(repo) do IO.puts("create #{inspect(repo)} database if it doesn't exist") @@ -202,23 +201,22 @@ defmodule Plausible.Release do # Start apps necessary for executing migrations Enum.each(@start_apps, &Application.ensure_all_started/1) - # Start the Repo(s) for myapp IO.puts("Starting repos..") Enum.each(repos(), & &1.start_link(pool_size: 2)) - end defp prepare_clickhouse do Application.ensure_all_started(:db_connection) Application.ensure_all_started(:hackney) - Clickhousex.start_link([ + + Clickhousex.start_link( scheme: :http, port: 8123, name: :clickhouse, database: "default", - hostname: Keyword.get(Application.get_env(:plausible,:clickhouse),:hostname) - ]) + hostname: Keyword.get(Application.get_env(:plausible, :clickhouse), :hostname) + ) end defp seeds_path(repo), do: priv_path_for(repo, "seeds.exs") diff --git a/lib/plausible_web/controllers/api/external_controller.ex b/lib/plausible_web/controllers/api/external_controller.ex index a84321f8c..b035e2caa 100644 --- a/lib/plausible_web/controllers/api/external_controller.ex +++ b/lib/plausible_web/controllers/api/external_controller.ex @@ -8,9 +8,14 @@ defmodule PlausibleWeb.Api.ExternalController do case create_event(conn, params) do {:ok, _} -> conn |> send_resp(202, "") + {:error, changeset} -> request = Sentry.Plug.build_request_interface_data(conn, []) - Sentry.capture_message("Error processing event", extra: %{errors: inspect(changeset.errors), params: params, request: request}) + + Sentry.capture_message("Error processing event", + extra: %{errors: inspect(changeset.errors), params: params, request: request} + ) + Logger.info("Error processing event: #{inspect(changeset)}") conn |> send_resp(400, "") end @@ -24,14 +29,16 @@ defmodule PlausibleWeb.Api.ExternalController do defp create_event(conn, params) do uri = params["url"] && URI.parse(params["url"]) - country_code = Plug.Conn.get_req_header(conn, "x-country") |> List.first - user_agent = Plug.Conn.get_req_header(conn, "user-agent") |> List.first + country_code = Plug.Conn.get_req_header(conn, "x-country") |> List.first() + user_agent = Plug.Conn.get_req_header(conn, "user-agent") |> List.first() + if UAInspector.bot?(user_agent) do {:ok, nil} else - ua = if user_agent do - UAInspector.Parser.parse(user_agent) - end + ua = + if user_agent do + UAInspector.Parser.parse(user_agent) + end ref = parse_referrer(uri, params["referrer"]) initial_ref = parse_referrer(uri, params["initial_referrer"]) @@ -54,6 +61,7 @@ defmodule PlausibleWeb.Api.ExternalController do } changeset = Plausible.ClickhouseEvent.changeset(%Plausible.ClickhouseEvent{}, event_attrs) + if changeset.valid? do event = struct(Plausible.ClickhouseEvent, event_attrs) session_id = Plausible.Session.Store.on_event(event) @@ -67,6 +75,7 @@ defmodule PlausibleWeb.Api.ExternalController do end defp parse_referrer(_, nil), do: nil + defp parse_referrer(uri, referrer_str) do referrer_uri = URI.parse(referrer_str) @@ -76,21 +85,26 @@ defmodule PlausibleWeb.Api.ExternalController do end defp generate_user_id(conn, params) do - hash_key = Keyword.fetch!(Application.get_env(:plausible, PlausibleWeb.Endpoint), :secret_key_base) |> binary_part(0, 16) + hash_key = + Keyword.fetch!(Application.get_env(:plausible, PlausibleWeb.Endpoint), :secret_key_base) + |> binary_part(0, 16) + user_agent = List.first(Plug.Conn.get_req_header(conn, "user-agent")) || "" - ip_address = List.first(Plug.Conn.get_req_header(conn, "x-bb-ip")) || "" # Netlify sets this header as the remote client IP + # Netlify sets this header as the remote client IP + ip_address = List.first(Plug.Conn.get_req_header(conn, "x-bb-ip")) || "" domain = strip_www(params["domain"]) || "" SipHash.hash!(hash_key, user_agent <> ip_address <> domain) end - defp calculate_screen_size(nil) , do: nil + defp calculate_screen_size(nil), do: nil defp calculate_screen_size(width) when width < 576, do: "Mobile" defp calculate_screen_size(width) when width < 992, do: "Tablet" defp calculate_screen_size(width) when width < 1440, do: "Laptop" defp calculate_screen_size(width) when width >= 1440, do: "Desktop" defp clean_referrer(nil), do: nil + defp clean_referrer(ref) do uri = URI.parse(ref.referer) @@ -107,6 +121,7 @@ defmodule PlausibleWeb.Api.ExternalController do end defp strip_www(nil), do: nil + defp strip_www(hostname) do String.replace_prefix(hostname, "www.", "") end @@ -130,10 +145,12 @@ defmodule PlausibleWeb.Api.ExternalController do end defp referrer_source(nil), do: nil + defp referrer_source(ref) do case ref.source do :unknown -> clean_uri(ref.referer) + source -> source end @@ -141,6 +158,7 @@ defmodule PlausibleWeb.Api.ExternalController do defp clean_uri(uri) do uri = URI.parse(String.trim(uri)) + if uri.scheme in ["http", "https"] do String.replace_leading(uri.host, "www.", "") end diff --git a/lib/plausible_web/controllers/api/paddle_controller.ex b/lib/plausible_web/controllers/api/paddle_controller.ex index 665ec6893..aa2d8442c 100644 --- a/lib/plausible_web/controllers/api/paddle_controller.ex +++ b/lib/plausible_web/controllers/api/paddle_controller.ex @@ -33,10 +33,12 @@ defmodule PlausibleWeb.Api.PaddleController do def verify_signature(conn, _opts) do signature = Base.decode64!(conn.params["p_signature"]) - msg = Map.delete(conn.params, "p_signature") - |> Enum.map(fn {key, val} -> {key, "#{val}"} end) - |> List.keysort(0) - |> PhpSerializer.serialize() + + msg = + Map.delete(conn.params, "p_signature") + |> Enum.map(fn {key, val} -> {key, "#{val}"} end) + |> List.keysort(0) + |> PhpSerializer.serialize() [key_entry] = :public_key.pem_decode(@paddle_key) public_key = :public_key.pem_entry_decode(key_entry) @@ -50,10 +52,12 @@ defmodule PlausibleWeb.Api.PaddleController do def verified_signature?(params) do signature = Base.decode64!(params["p_signature"]) - msg = Map.delete(params, "p_signature") - |> Enum.map(fn {key, val} -> {key, "#{val}"} end) - |> List.keysort(0) - |> PhpSerializer.serialize() + + msg = + Map.delete(params, "p_signature") + |> Enum.map(fn {key, val} -> {key, "#{val}"} end) + |> List.keysort(0) + |> PhpSerializer.serialize() [key_entry] = :public_key.pem_decode(@paddle_key) public_key = :public_key.pem_entry_decode(key_entry) @@ -66,7 +70,11 @@ defmodule PlausibleWeb.Api.PaddleController do defp webhook_response({:error, changeset}, conn, params) do request = Sentry.Plug.build_request_interface_data(conn, []) - Sentry.capture_message("Error processing Paddle webhook", extra: %{errors: inspect(changeset.errors), params: params, request: request}) + + Sentry.capture_message("Error processing Paddle webhook", + extra: %{errors: inspect(changeset.errors), params: params, request: request} + ) + Logger.error("Error processing Paddle webhook: #{inspect(changeset)}") conn |> send_resp(400, "") |> halt diff --git a/lib/plausible_web/controllers/api/stats_controller.ex b/lib/plausible_web/controllers/api/stats_controller.ex index 9aebbbfca..6e7ac3850 100644 --- a/lib/plausible_web/controllers/api/stats_controller.ex +++ b/lib/plausible_web/controllers/api/stats_controller.ex @@ -19,7 +19,7 @@ defmodule PlausibleWeb.Api.StatsController do labels: labels, present_index: present_index, top_stats: top_stats, - interval: query.step_type, + interval: query.step_type }) end @@ -29,13 +29,33 @@ defmodule PlausibleWeb.Api.StatsController do prev_total_visitors = Stats.unique_visitors(site, %{prev_query | filters: %{}}) converted_visitors = Stats.unique_visitors(site, query) prev_converted_visitors = Stats.unique_visitors(site, prev_query) - conversion_rate = if total_visitors > 0, do: Float.round(converted_visitors / total_visitors * 100, 1), else: 0.0 - prev_conversion_rate = if prev_total_visitors > 0, do: Float.round(prev_converted_visitors / prev_total_visitors * 100, 1), else: 0.0 + + conversion_rate = + if total_visitors > 0, + do: Float.round(converted_visitors / total_visitors * 100, 1), + else: 0.0 + + prev_conversion_rate = + if prev_total_visitors > 0, + do: Float.round(prev_converted_visitors / prev_total_visitors * 100, 1), + else: 0.0 [ - %{name: "Total visitors", count: total_visitors, change: percent_change(prev_total_visitors, total_visitors)}, - %{name: "Converted visitors", count: converted_visitors, change: percent_change(prev_converted_visitors, converted_visitors)}, - %{name: "Conversion rate", percentage: conversion_rate, change: percent_change(prev_conversion_rate, conversion_rate)}, + %{ + name: "Total visitors", + count: total_visitors, + change: percent_change(prev_total_visitors, total_visitors) + }, + %{ + name: "Converted visitors", + count: converted_visitors, + change: percent_change(prev_converted_visitors, converted_visitors) + }, + %{ + name: "Conversion rate", + percentage: conversion_rate, + change: percent_change(prev_conversion_rate, conversion_rate) + } ] end @@ -48,9 +68,17 @@ defmodule PlausibleWeb.Api.StatsController do change_bounce_rate = if prev_bounce_rate > 0, do: bounce_rate - prev_bounce_rate [ - %{name: "Unique visitors", count: visitors, change: percent_change(prev_visitors, visitors)}, - %{name: "Total pageviews", count: pageviews, change: percent_change(prev_pageviews, pageviews)}, - %{name: "Bounce rate", percentage: bounce_rate, change: change_bounce_rate}, + %{ + name: "Unique visitors", + count: visitors, + change: percent_change(prev_visitors, visitors) + }, + %{ + name: "Total pageviews", + count: pageviews, + change: percent_change(prev_pageviews, pageviews) + }, + %{name: "Bounce rate", percentage: bounce_rate, change: change_bounce_rate} ] end @@ -58,8 +86,10 @@ defmodule PlausibleWeb.Api.StatsController do cond do old_count == 0 and new_count > 0 -> 100 + old_count == 0 and new_count == 0 -> 0 + true -> round((new_count - old_count) / old_count * 100) end @@ -87,9 +117,10 @@ defmodule PlausibleWeb.Api.StatsController do site = conn.assigns[:site] |> Repo.preload(:google_auth) query = Query.from(site.timezone, params) - search_terms = if site.google_auth && site.google_auth.property && !query.filters["goal"] do - @google_api.fetch_stats(site.google_auth, query) - end + search_terms = + if site.google_auth && site.google_auth.property && !query.filters["goal"] do + @google_api.fetch_stats(site.google_auth, query) + end case search_terms do nil -> @@ -97,9 +128,11 @@ defmodule PlausibleWeb.Api.StatsController do user_id = get_session(conn, :current_user_id) is_owner = user_id && Plausible.Sites.is_owner?(user_id, site) json(conn, %{not_configured: true, is_owner: is_owner, total_visitors: total_visitors}) + {:ok, terms} -> total_visitors = Stats.visitors_from_referrer(site, query, "Google") json(conn, %{search_terms: terms, total_visitors: total_visitors}) + {:error, e} -> put_status(conn, 500) |> json(%{error: e}) diff --git a/lib/plausible_web/controllers/auth_controller.ex b/lib/plausible_web/controllers/auth_controller.ex index ea3e0be0c..99bf4dda9 100644 --- a/lib/plausible_web/controllers/auth_controller.ex +++ b/lib/plausible_web/controllers/auth_controller.ex @@ -4,12 +4,19 @@ defmodule PlausibleWeb.AuthController do alias Plausible.Auth require Logger - plug PlausibleWeb.RequireLoggedOutPlug when action in [:register_form, :register, :login_form, :login] - plug PlausibleWeb.RequireAccountPlug when action in [:user_settings, :save_settings, :delete_me, :password_form, :set_password] + plug PlausibleWeb.RequireLoggedOutPlug + when action in [:register_form, :register, :login_form, :login] + + plug PlausibleWeb.RequireAccountPlug + when action in [:user_settings, :save_settings, :delete_me, :password_form, :set_password] def register_form(conn, _params) do changeset = Plausible.Auth.User.changeset(%Plausible.Auth.User{}) - render(conn, "register_form.html", changeset: changeset, layout: {PlausibleWeb.LayoutView, "focus.html"}) + + render(conn, "register_form.html", + changeset: changeset, + layout: {PlausibleWeb.LayoutView, "focus.html"} + ) end def register(conn, %{"user" => params}) do @@ -22,9 +29,18 @@ defmodule PlausibleWeb.AuthController do Logger.info(url) email_template = PlausibleWeb.Email.activation_email(user, url) Plausible.Mailer.send_email(email_template) - conn |> render("register_success.html", email: user.email, layout: {PlausibleWeb.LayoutView, "focus.html"}) + + conn + |> render("register_success.html", + email: user.email, + layout: {PlausibleWeb.LayoutView, "focus.html"} + ) + {:error, changeset} -> - render(conn, "register_form.html", changeset: changeset, layout: {PlausibleWeb.LayoutView, "focus.html"}) + render(conn, "register_form.html", + changeset: changeset, + layout: {PlausibleWeb.LayoutView, "focus.html"} + ) end end @@ -40,22 +56,30 @@ defmodule PlausibleWeb.AuthController do |> put_session(:current_user_id, user.id) |> put_resp_cookie("logged_in", "true", http_only: false) |> redirect(to: "/password") + {:error, changeset} -> send_resp(conn, 400, inspect(changeset.errors)) end + {:error, :expired} -> render_error(conn, 401, "Your token has expired. Please request another activation link.") + {:error, _} -> render_error(conn, 400, "Your token is invalid. Please request another activation link.") end end def password_reset_request_form(conn, _) do - render(conn, "password_reset_request_form.html", layout: {PlausibleWeb.LayoutView, "focus.html"}) + render(conn, "password_reset_request_form.html", + layout: {PlausibleWeb.LayoutView, "focus.html"} + ) end def password_reset_request(conn, %{"email" => ""}) do - render(conn, "password_reset_request_form.html", error: "Please enter an email address", layout: {PlausibleWeb.LayoutView, "focus.html"}) + render(conn, "password_reset_request_form.html", + error: "Please enter an email address", + layout: {PlausibleWeb.LayoutView, "focus.html"} + ) end def password_reset_request(conn, %{"email" => email}) do @@ -67,20 +91,40 @@ defmodule PlausibleWeb.AuthController do Logger.debug("PASSWORD RESET LINK: " <> url) email_template = PlausibleWeb.Email.password_reset_email(email, url) Plausible.Mailer.deliver_now(email_template) - render(conn, "password_reset_request_success.html", email: email, layout: {PlausibleWeb.LayoutView, "focus.html"}) + + render(conn, "password_reset_request_success.html", + email: email, + layout: {PlausibleWeb.LayoutView, "focus.html"} + ) else - render(conn, "password_reset_request_success.html", email: email, layout: {PlausibleWeb.LayoutView, "focus.html"}) + render(conn, "password_reset_request_success.html", + email: email, + layout: {PlausibleWeb.LayoutView, "focus.html"} + ) end end def password_reset_form(conn, %{"token" => token}) do case Auth.Token.verify_password_reset(token) do {:ok, _} -> - render(conn, "password_reset_form.html", token: token, layout: {PlausibleWeb.LayoutView, "focus.html"}) + render(conn, "password_reset_form.html", + token: token, + layout: {PlausibleWeb.LayoutView, "focus.html"} + ) + {:error, :expired} -> - render_error(conn, 401, "Your token has expired. Please request another password reset link.") + render_error( + conn, + 401, + "Your token has expired. Please request another password reset link." + ) + {:error, _} -> - render_error(conn, 401, "Your token is invalid. Please request another password reset link.") + render_error( + conn, + 401, + "Your token is invalid. Please request another password reset link." + ) end end @@ -89,6 +133,7 @@ defmodule PlausibleWeb.AuthController do {:ok, %{email: email}} -> user = Repo.get_by(Auth.User, email: email) changeset = Auth.User.set_password(user, pw) + case Repo.update(changeset) do {:ok, _updated} -> conn @@ -97,23 +142,39 @@ defmodule PlausibleWeb.AuthController do |> put_session(:current_user_id, nil) |> delete_resp_cookie("logged_in") |> redirect(to: "/login") + {:error, changeset} -> - render(conn, "password_reset_form.html", changeset: changeset, token: token, layout: {PlausibleWeb.LayoutView, "focus.html"}) + render(conn, "password_reset_form.html", + changeset: changeset, + token: token, + layout: {PlausibleWeb.LayoutView, "focus.html"} + ) end + {:error, :expired} -> - render_error(conn, 401, "Your token has expired. Please request another password reset link.") + render_error( + conn, + 401, + "Your token has expired. Please request another password reset link." + ) + {:error, _} -> - render_error(conn, 401, "Your token is invalid. Please request another password reset link.") + render_error( + conn, + 401, + "Your token is invalid. Please request another password reset link." + ) end end def login(conn, %{"email" => email, "password" => password}) do alias Plausible.Auth.Password - user = Repo.one( - from u in Plausible.Auth.User, - where: u.email == ^email - ) + user = + Repo.one( + from u in Plausible.Auth.User, + where: u.email == ^email + ) if user do if Password.match?(password, user.password_hash || "") do @@ -125,11 +186,20 @@ defmodule PlausibleWeb.AuthController do |> put_session(:login_dest, nil) |> redirect(to: login_dest) else - conn |> render("login_form.html", error: "Wrong email or password. Please try again.", layout: {PlausibleWeb.LayoutView, "focus.html"}) + conn + |> render("login_form.html", + error: "Wrong email or password. Please try again.", + layout: {PlausibleWeb.LayoutView, "focus.html"} + ) end else Password.dummy_calculation() - conn |> render("login_form.html", error: "Wrong email or password. Please try again.", layout: {PlausibleWeb.LayoutView, "focus.html"}) + + conn + |> render("login_form.html", + error: "Wrong email or password. Please try again.", + layout: {PlausibleWeb.LayoutView, "focus.html"} + ) end end @@ -138,7 +208,10 @@ defmodule PlausibleWeb.AuthController do end def password_form(conn, _params) do - render(conn, "password_form.html", layout: {PlausibleWeb.LayoutView, "focus.html"}, skip_plausible_tracking: true) + render(conn, "password_form.html", + layout: {PlausibleWeb.LayoutView, "focus.html"}, + skip_plausible_tracking: true + ) end def set_password(conn, %{"password" => pw}) do @@ -147,8 +220,12 @@ defmodule PlausibleWeb.AuthController do case Repo.update(changeset) do {:ok, _user} -> redirect(conn, to: "/sites/new") + {:error, changeset} -> - render(conn, "password_form.html", changeset: changeset, layout: {PlausibleWeb.LayoutView, "focus.html"}) + render(conn, "password_form.html", + changeset: changeset, + layout: {PlausibleWeb.LayoutView, "focus.html"} + ) end end @@ -160,20 +237,23 @@ defmodule PlausibleWeb.AuthController do def save_settings(conn, %{"user" => user_params}) do changes = Auth.User.changeset(conn.assigns[:current_user], user_params) + case Repo.update(changes) do {:ok, _user} -> conn |> put_flash(:success, "Account settings saved succesfully") |> redirect(to: "/settings") + {:error, changeset} -> render(conn, "user_settings.html", changeset: changeset) end end def delete_me(conn, params) do - user = conn.assigns[:current_user] - |> Repo.preload(:sites) - |> Repo.preload(:subscription) + user = + conn.assigns[:current_user] + |> Repo.preload(:sites) + |> Repo.preload(:subscription) for site_membership <- user.site_memberships do Repo.delete!(site_membership) @@ -200,7 +280,7 @@ defmodule PlausibleWeb.AuthController do res = Plausible.Google.Api.fetch_access_token(code) id_token = res["id_token"] [_, body, _] = String.split(id_token, ".") - id = body |> Base.decode64!(padding: false) |> Jason.decode! + id = body |> Base.decode64!(padding: false) |> Jason.decode!() Plausible.Site.GoogleAuth.changeset(%Plausible.Site.GoogleAuth{}, %{ email: id["email"], @@ -209,7 +289,8 @@ defmodule PlausibleWeb.AuthController do expires: NaiveDateTime.utc_now() |> NaiveDateTime.add(res["expires_in"]), user_id: conn.assigns[:current_user].id, site_id: site_id - }) |> Repo.insert! + }) + |> Repo.insert!() site = Repo.get(Plausible.Site, site_id) diff --git a/lib/plausible_web/controllers/billing_controller.ex b/lib/plausible_web/controllers/billing_controller.ex index 7fd51481e..634781e25 100644 --- a/lib/plausible_web/controllers/billing_controller.ex +++ b/lib/plausible_web/controllers/billing_controller.ex @@ -14,7 +14,10 @@ defmodule PlausibleWeb.BillingController do subscription = Billing.active_subscription_for(conn.assigns[:current_user].id) if subscription do - render(conn, "change_plan.html", subscription: subscription, layout: {PlausibleWeb.LayoutView, "focus.html"}) + render(conn, "change_plan.html", + subscription: subscription, + layout: {PlausibleWeb.LayoutView, "focus.html"} + ) else redirect(conn, to: "/billing/upgrade") end @@ -25,7 +28,12 @@ defmodule PlausibleWeb.BillingController do if subscription do {:ok, preview_info} = Billing.change_plan_preview(subscription, new_plan_id) - render(conn, "change_plan_preview.html", subscription: subscription, preview_info: preview_info, layout: {PlausibleWeb.LayoutView, "focus.html"}) + + render(conn, "change_plan_preview.html", + subscription: subscription, + preview_info: preview_info, + layout: {PlausibleWeb.LayoutView, "focus.html"} + ) else redirect(conn, to: "/billing/upgrade") end @@ -37,10 +45,21 @@ defmodule PlausibleWeb.BillingController do conn |> put_flash(:success, "Plan changed successfully") |> redirect(to: "/settings") + {:error, e} -> - Sentry.capture_message("Error changing plans", extra: %{errors: inspect(e), new_plan_id: new_plan_id, user_id: conn.assigns[:current_user].id}) + Sentry.capture_message("Error changing plans", + extra: %{ + errors: inspect(e), + new_plan_id: new_plan_id, + user_id: conn.assigns[:current_user].id + } + ) + conn - |> put_flash(:error, "Something went wrong. Please try again or contact support at support@plausible.io") + |> put_flash( + :error, + "Something went wrong. Please try again or contact support at support@plausible.io" + ) |> redirect(to: "/settings") end end @@ -49,7 +68,12 @@ defmodule PlausibleWeb.BillingController do usage = Plausible.Billing.usage(conn.assigns[:current_user]) today = Timex.today() - render(conn, "upgrade.html", usage: usage, today: today, user: conn.assigns[:current_user], layout: {PlausibleWeb.LayoutView, "focus.html"}) + render(conn, "upgrade.html", + usage: usage, + today: today, + user: conn.assigns[:current_user], + layout: {PlausibleWeb.LayoutView, "focus.html"} + ) end def success(conn, _params) do diff --git a/lib/plausible_web/controllers/page_controller.ex b/lib/plausible_web/controllers/page_controller.ex index 8d7235b07..7f7167856 100644 --- a/lib/plausible_web/controllers/page_controller.ex +++ b/lib/plausible_web/controllers/page_controller.ex @@ -9,17 +9,17 @@ defmodule PlausibleWeb.PageController do {"Reddit", 8}, {"Google", 6}, {"DuckDuckGo", 4}, - {"Bing", 2}, + {"Bing", 2} ] - @demo_pages [ - {"/", 140}, - {"/plausible.io", 63}, - {"/blog", 41}, - {"/register", 13}, - {"/login", 7}, - {"/blog/google-analytics-and-privacy", 4}, - {"/blog/launching-plausible", 2} + @demo_pages [ + {"/", 140}, + {"/plausible.io", 63}, + {"/blog", 41}, + {"/register", 13}, + {"/login", 7}, + {"/blog/google-analytics-and-privacy", 4}, + {"/blog/launching-plausible", 2} ] def index(conn, _params) do @@ -27,7 +27,11 @@ defmodule PlausibleWeb.PageController do user = conn.assigns[:current_user] |> Repo.preload(:sites) render(conn, "sites.html", sites: user.sites) else - render(conn, "index.html", demo_referrers: @demo_referrers, demo_pages: @demo_pages, landing_nav: true) + render(conn, "index.html", + demo_referrers: @demo_referrers, + demo_pages: @demo_pages, + landing_nav: true + ) end end @@ -39,7 +43,7 @@ defmodule PlausibleWeb.PageController do claims = %{ id: user.id, email: user.email, - name: user.name, + name: user.name } signer = Joken.Signer.create("HS256", "4d1d2ae6-4595-4d0b-b98a-8ca5b1f2095a") @@ -59,7 +63,11 @@ defmodule PlausibleWeb.PageController do def roadmap(conn, _params) do if conn.assigns[:current_user] do token = sign_token!(conn.assigns[:current_user]) - redirect(conn, external: "https://plausible.nolt.io/sso/#{token}?returnUrl=https://plausible.nolt.io/roadmap") + + redirect(conn, + external: + "https://plausible.nolt.io/sso/#{token}?returnUrl=https://plausible.nolt.io/roadmap" + ) else redirect(conn, external: "https://plausible.nolt.io/roadmap") end diff --git a/lib/plausible_web/controllers/site_controller.ex b/lib/plausible_web/controllers/site_controller.ex index cceb564f1..7a173b834 100644 --- a/lib/plausible_web/controllers/site_controller.ex +++ b/lib/plausible_web/controllers/site_controller.ex @@ -22,17 +22,23 @@ defmodule PlausibleWeb.SiteController do case insert_site(user.id, site_params) do {:ok, %{site: site}} -> Plausible.Slack.notify("#{user.name} created #{site.domain}") + conn |> put_session(site.domain <> "_offer_email_report", true) |> redirect(to: "/#{URI.encode_www_form(site.domain)}/snippet") + {:error, :site, changeset, _} -> - render(conn, "new.html", changeset: changeset, layout: {PlausibleWeb.LayoutView, "focus.html"}) + render(conn, "new.html", + changeset: changeset, + layout: {PlausibleWeb.LayoutView, "focus.html"} + ) end end def add_snippet(conn, %{"website" => website}) do - site = Sites.get_for_user!(conn.assigns[:current_user].id, website) - |> Repo.preload(:custom_domain) + site = + Sites.get_for_user!(conn.assigns[:current_user].id, website) + |> Repo.preload(:custom_domain) conn |> assign(:skip_plausible_tracking, true) @@ -60,6 +66,7 @@ defmodule PlausibleWeb.SiteController do conn |> put_flash(:success, "Goal created succesfully") |> redirect(to: "/#{URI.encode_www_form(site.domain)}/settings") + {:error, :goal, changeset, _} -> conn |> assign(:skip_plausible_tracking, true) @@ -80,13 +87,15 @@ defmodule PlausibleWeb.SiteController do end def settings(conn, %{"website" => website}) do - site = Sites.get_for_user!(conn.assigns[:current_user].id, website) - |> Repo.preload(:google_auth) - |> Repo.preload(:custom_domain) + site = + Sites.get_for_user!(conn.assigns[:current_user].id, website) + |> Repo.preload(:google_auth) + |> Repo.preload(:custom_domain) - search_console_domains = if site.google_auth do - Plausible.Google.Api.fetch_verified_properties(site.google_auth) - end + search_console_domains = + if site.google_auth do + Plausible.Google.Api.fetch_verified_properties(site.google_auth) + end weekly_report = Repo.get_by(Plausible.Site.WeeklyReport, site_id: site.id) monthly_report = Repo.get_by(Plausible.Site.MonthlyReport, site_id: site.id) @@ -107,11 +116,12 @@ defmodule PlausibleWeb.SiteController do end def update_google_auth(conn, %{"website" => website, "google_auth" => attrs}) do - site = Sites.get_for_user!(conn.assigns[:current_user].id, website) - |> Repo.preload(:google_auth) + site = + Sites.get_for_user!(conn.assigns[:current_user].id, website) + |> Repo.preload(:google_auth) Plausible.Site.GoogleAuth.set_property(site.google_auth, attrs) - |> Repo.update! + |> Repo.update!() conn |> put_flash(:success, "Google integration saved succesfully") @@ -121,7 +131,7 @@ defmodule PlausibleWeb.SiteController do def update_settings(conn, %{"website" => website, "site" => site_params}) do site = Sites.get_for_user!(conn.assigns[:current_user].id, website) changeset = site |> Plausible.Site.changeset(site_params) - res = changeset |> Repo.update + res = changeset |> Repo.update() case res do {:ok, site} -> @@ -131,6 +141,7 @@ defmodule PlausibleWeb.SiteController do |> put_session(site_session_key, nil) |> put_flash(:success, "Site settings saved succesfully") |> redirect(to: "/#{URI.encode_www_form(site.domain)}/settings") + {:error, changeset} -> render("settings.html", site: site, changeset: changeset) end @@ -146,6 +157,7 @@ defmodule PlausibleWeb.SiteController do if site.google_auth do Repo.delete!(site.google_auth) end + Repo.delete!(site) conn @@ -154,9 +166,10 @@ defmodule PlausibleWeb.SiteController do end def make_public(conn, %{"website" => website}) do - site = Sites.get_for_user!(conn.assigns[:current_user].id, website) - |> Plausible.Site.make_public - |> Repo.update! + site = + Sites.get_for_user!(conn.assigns[:current_user].id, website) + |> Plausible.Site.make_public() + |> Repo.update!() conn |> put_flash(:success, "Congrats! Stats for #{site.domain} are now public.") @@ -164,9 +177,10 @@ defmodule PlausibleWeb.SiteController do end def make_private(conn, %{"website" => website}) do - site = Sites.get_for_user!(conn.assigns[:current_user].id, website) - |> Plausible.Site.make_private - |> Repo.update! + site = + Sites.get_for_user!(conn.assigns[:current_user].id, website) + |> Plausible.Site.make_private() + |> Repo.update!() conn |> put_flash(:success, "Stats for #{site.domain} are now private.") @@ -180,7 +194,7 @@ defmodule PlausibleWeb.SiteController do site_id: site.id, recipients: [conn.assigns[:current_user].email] }) - |> Repo.insert! + |> Repo.insert!() conn |> put_flash(:success, "Success! You will receive an email report every Monday going forward") @@ -201,7 +215,7 @@ defmodule PlausibleWeb.SiteController do Repo.get_by(Plausible.Site.WeeklyReport, site_id: site.id) |> Plausible.Site.WeeklyReport.add_recipient(recipient) - |> Repo.update! + |> Repo.update!() conn |> put_flash(:success, "Succesfully added #{recipient} as a recipient for the weekly report") @@ -213,10 +227,13 @@ defmodule PlausibleWeb.SiteController do Repo.get_by(Plausible.Site.WeeklyReport, site_id: site.id) |> Plausible.Site.WeeklyReport.remove_recipient(recipient) - |> Repo.update! + |> Repo.update!() conn - |> put_flash(:success, "Succesfully removed #{recipient} as a recipient for the weekly report") + |> put_flash( + :success, + "Succesfully removed #{recipient} as a recipient for the weekly report" + ) |> redirect(to: "/#{URI.encode_www_form(site.domain)}/settings#email-reports") end @@ -227,7 +244,7 @@ defmodule PlausibleWeb.SiteController do site_id: site.id, recipients: [conn.assigns[:current_user].email] }) - |> Repo.insert! + |> Repo.insert!() conn |> put_flash(:success, "Success! You will receive an email report every month going forward") @@ -248,7 +265,7 @@ defmodule PlausibleWeb.SiteController do Repo.get_by(Plausible.Site.MonthlyReport, site_id: site.id) |> Plausible.Site.MonthlyReport.add_recipient(recipient) - |> Repo.update! + |> Repo.update!() conn |> put_flash(:success, "Succesfully added #{recipient} as a recipient for the monthly report") @@ -260,10 +277,13 @@ defmodule PlausibleWeb.SiteController do Repo.get_by(Plausible.Site.MonthlyReport, site_id: site.id) |> Plausible.Site.MonthlyReport.remove_recipient(recipient) - |> Repo.update! + |> Repo.update!() conn - |> put_flash(:success, "Succesfully removed #{recipient} as a recipient for the monthly report") + |> put_flash( + :success, + "Succesfully removed #{recipient} as a recipient for the monthly report" + ) |> redirect(to: "/#{URI.encode_www_form(site.domain)}/settings#email-reports") end @@ -273,30 +293,45 @@ defmodule PlausibleWeb.SiteController do conn |> assign(:skip_plausible_tracking, true) - |> render("new_shared_link.html", site: site, changeset: changeset, layout: {PlausibleWeb.LayoutView, "focus.html"}) + |> render("new_shared_link.html", + site: site, + changeset: changeset, + layout: {PlausibleWeb.LayoutView, "focus.html"} + ) end def create_shared_link(conn, %{"website" => website, "shared_link" => link}) do site = Sites.get_for_user!(conn.assigns[:current_user].id, website) - changes = Plausible.Site.SharedLink.changeset(%Plausible.Site.SharedLink{ - site_id: site.id, - slug: Nanoid.generate() - }, link) + + changes = + Plausible.Site.SharedLink.changeset( + %Plausible.Site.SharedLink{ + site_id: site.id, + slug: Nanoid.generate() + }, + link + ) case Repo.insert(changes) do {:ok, _created} -> redirect(conn, to: "/#{URI.encode_www_form(site.domain)}/settings#visibility") + {:error, changeset} -> conn |> assign(:skip_plausible_tracking, true) - |> render("new_shared_link.html", site: site, changeset: changeset, layout: {PlausibleWeb.LayoutView, "focus.html"}) + |> render("new_shared_link.html", + site: site, + changeset: changeset, + layout: {PlausibleWeb.LayoutView, "focus.html"} + ) end end def delete_shared_link(conn, %{"website" => website, "slug" => slug}) do site = Sites.get_for_user!(conn.assigns[:current_user].id, website) + Repo.get_by(Plausible.Site.SharedLink, slug: slug) - |> Repo.delete! + |> Repo.delete!() redirect(conn, to: "/#{URI.encode_www_form(site.domain)}/settings#visibility") end @@ -307,25 +342,37 @@ defmodule PlausibleWeb.SiteController do conn |> assign(:skip_plausible_tracking, true) - |> render("new_custom_domain.html", site: site, changeset: changeset, layout: {PlausibleWeb.LayoutView, "focus.html"}) + |> render("new_custom_domain.html", + site: site, + changeset: changeset, + layout: {PlausibleWeb.LayoutView, "focus.html"} + ) end def custom_domain_dns_setup(conn, %{"website" => website}) do - site = Sites.get_for_user!(conn.assigns[:current_user].id, website) - |> Repo.preload(:custom_domain) + site = + Sites.get_for_user!(conn.assigns[:current_user].id, website) + |> Repo.preload(:custom_domain) conn |> assign(:skip_plausible_tracking, true) - |> render("custom_domain_dns_setup.html", site: site, layout: {PlausibleWeb.LayoutView, "focus.html"}) + |> render("custom_domain_dns_setup.html", + site: site, + layout: {PlausibleWeb.LayoutView, "focus.html"} + ) end def custom_domain_snippet(conn, %{"website" => website}) do - site = Sites.get_for_user!(conn.assigns[:current_user].id, website) - |> Repo.preload(:custom_domain) + site = + Sites.get_for_user!(conn.assigns[:current_user].id, website) + |> Repo.preload(:custom_domain) conn |> assign(:skip_plausible_tracking, true) - |> render("custom_domain_snippet.html", site: site, layout: {PlausibleWeb.LayoutView, "focus.html"}) + |> render("custom_domain_snippet.html", + site: site, + layout: {PlausibleWeb.LayoutView, "focus.html"} + ) end def add_custom_domain(conn, %{"website" => website, "custom_domain" => domain}) do @@ -334,10 +381,15 @@ defmodule PlausibleWeb.SiteController do case Sites.add_custom_domain(site, domain["domain"]) do {:ok, _custom_domain} -> redirect(conn, to: "/sites/#{URI.encode_www_form(site.domain)}/custom-domains/dns-setup") + {:error, changeset} -> conn |> assign(:skip_plausible_tracking, true) - |> render("new_custom_domain.html", site: site, changeset: changeset, layout: {PlausibleWeb.LayoutView, "focus.html"}) + |> render("new_custom_domain.html", + site: site, + changeset: changeset, + layout: {PlausibleWeb.LayoutView, "focus.html"} + ) end end @@ -346,14 +398,15 @@ defmodule PlausibleWeb.SiteController do Ecto.Multi.new() |> Ecto.Multi.insert(:site, site_changeset) - |> Ecto.Multi.run(:site_membership, fn repo, %{site: site} -> - membership_changeset = Plausible.Site.Membership.changeset(%Plausible.Site.Membership{}, %{ - site_id: site.id, - user_id: user_id - }) + |> Ecto.Multi.run(:site_membership, fn repo, %{site: site} -> + membership_changeset = + Plausible.Site.Membership.changeset(%Plausible.Site.Membership{}, %{ + site_id: site.id, + user_id: user_id + }) + repo.insert(membership_changeset) end) - |> Repo.transaction + |> Repo.transaction() end - end diff --git a/lib/plausible_web/controllers/stats_controller.ex b/lib/plausible_web/controllers/stats_controller.ex index 719169767..87ec20944 100644 --- a/lib/plausible_web/controllers/stats_controller.ex +++ b/lib/plausible_web/controllers/stats_controller.ex @@ -45,30 +45,39 @@ defmodule PlausibleWeb.StatsController do query = Query.from(site.timezone, conn.params) {plot, _, labels, _} = Stats.calculate_plot(site, query) - csv_content = Enum.zip(labels, plot) - |> Enum.map(fn {k, v} -> [k, v] end) - |> (fn data -> [["Date", "Visitors"] | data] end).() - |> CSV.encode - |> Enum.into([]) - |> Enum.join() - filename = "Visitors #{domain} #{Timex.format!(query.date_range.first, "{ISOdate} ")} to #{Timex.format!(query.date_range.last, "{ISOdate} ")}.csv" + csv_content = + Enum.zip(labels, plot) + |> Enum.map(fn {k, v} -> [k, v] end) + |> (fn data -> [["Date", "Visitors"] | data] end).() + |> CSV.encode() + |> Enum.into([]) + |> Enum.join() + + filename = + "Visitors #{domain} #{Timex.format!(query.date_range.first, "{ISOdate} ")} to #{ + Timex.format!(query.date_range.last, "{ISOdate} ") + }.csv" conn |> put_resp_content_type("text/csv") |> put_resp_header("content-disposition", "attachment; filename=\"#{filename}\"") - |> send_resp(200, csv_content) + |> send_resp(200, csv_content) end def shared_link(conn, %{"slug" => slug}) do - shared_link = Repo.get_by(Plausible.Site.SharedLink, slug: slug) - |> Repo.preload(:site) + shared_link = + Repo.get_by(Plausible.Site.SharedLink, slug: slug) + |> Repo.preload(:site) if shared_link do if shared_link.password_hash do conn |> assign(:skip_plausible_tracking, true) - |> render("shared_link_password.html", link: shared_link, layout: {PlausibleWeb.LayoutView, "focus.html"}) + |> render("shared_link_password.html", + link: shared_link, + layout: {PlausibleWeb.LayoutView, "focus.html"} + ) else shared_link_auth_success(conn, shared_link) end @@ -78,8 +87,9 @@ defmodule PlausibleWeb.StatsController do end def authenticate_shared_link(conn, %{"slug" => slug, "password" => password}) do - shared_link = Repo.get_by(Plausible.Site.SharedLink, slug: slug) - |> Repo.preload(:site) + shared_link = + Repo.get_by(Plausible.Site.SharedLink, slug: slug) + |> Repo.preload(:site) if shared_link do if Plausible.Auth.Password.match?(password, shared_link.password_hash) do @@ -87,7 +97,11 @@ defmodule PlausibleWeb.StatsController do else conn |> assign(:skip_plausible_tracking, true) - |> render("shared_link_password.html", link: shared_link, error: "Incorrect password. Please try again.", layout: {PlausibleWeb.LayoutView, "focus.html"}) + |> render("shared_link_password.html", + link: shared_link, + error: "Incorrect password. Please try again.", + layout: {PlausibleWeb.LayoutView, "focus.html"} + ) end else render_error(conn, 404) diff --git a/lib/plausible_web/controllers/unsubscribe_controller.ex b/lib/plausible_web/controllers/unsubscribe_controller.ex index 32cb694da..f8eff5fee 100644 --- a/lib/plausible_web/controllers/unsubscribe_controller.ex +++ b/lib/plausible_web/controllers/unsubscribe_controller.ex @@ -8,11 +8,15 @@ defmodule PlausibleWeb.UnsubscribeController do Repo.get_by(WeeklyReport, site_id: site.id) |> WeeklyReport.remove_recipient(email) - |> Repo.update! + |> Repo.update!() conn |> assign(:skip_plausible_tracking, true) - |> render("success.html", interval: "weekly", site: website, layout: {PlausibleWeb.LayoutView, "focus.html"}) + |> render("success.html", + interval: "weekly", + site: website, + layout: {PlausibleWeb.LayoutView, "focus.html"} + ) end def monthly_report(conn, %{"website" => website, "email" => email}) do @@ -20,10 +24,14 @@ defmodule PlausibleWeb.UnsubscribeController do Repo.get_by(MonthlyReport, site_id: site.id) |> MonthlyReport.remove_recipient(email) - |> Repo.update! + |> Repo.update!() conn |> assign(:skip_plausible_tracking, true) - |> render("success.html", interval: "monthly", site: website, layout: {PlausibleWeb.LayoutView, "focus.html"}) + |> render("success.html", + interval: "monthly", + site: website, + layout: {PlausibleWeb.LayoutView, "focus.html"} + ) end end diff --git a/lib/plausible_web/endpoint.ex b/lib/plausible_web/endpoint.ex index 29c07e81f..ea82c2a14 100644 --- a/lib/plausible_web/endpoint.ex +++ b/lib/plausible_web/endpoint.ex @@ -35,15 +35,16 @@ defmodule PlausibleWeb.Endpoint do store: :cookie, key: "_plausible_key", signing_salt: "3IL0ob4k", - max_age: 60*60*24*365*5, # 5 years, this is super long but the SlidingSessionTimeout will log people out if they don't return for 2 weeks + # 5 years, this is super long but the SlidingSessionTimeout will log people out if they don't return for 2 weeks + max_age: 60 * 60 * 24 * 365 * 5, extra: "SameSite=Lax" - plug CORSPlug plug PlausibleWeb.Router def clean_url() do - url = PlausibleWeb.Endpoint.url + url = PlausibleWeb.Endpoint.url() + case Application.get_env(:plausible, :environment) do # do not truncate the port in case of dev or test environment env when env in ["dev", "test"] -> url diff --git a/lib/plausible_web/plugs/auth_plug.ex b/lib/plausible_web/plugs/auth_plug.ex index 608993d36..bf4ba1178 100644 --- a/lib/plausible_web/plugs/auth_plug.ex +++ b/lib/plausible_web/plugs/auth_plug.ex @@ -8,10 +8,14 @@ defmodule PlausibleWeb.AuthPlug do def call(conn, _opts) do case get_session(conn, :current_user_id) do - nil -> conn + nil -> + conn + id -> - user = Repo.get_by(Plausible.Auth.User, id: id) - |> Repo.preload(:subscription) + user = + Repo.get_by(Plausible.Auth.User, id: id) + |> Repo.preload(:subscription) + if user do Sentry.Context.set_user_context(%{id: user.id, name: user.name}) assign(conn, :current_user, user) diff --git a/lib/plausible_web/plugs/authorize_stats_plug.ex b/lib/plausible_web/plugs/authorize_stats_plug.ex index c8a6e7223..21f45dc8c 100644 --- a/lib/plausible_web/plugs/authorize_stats_plug.ex +++ b/lib/plausible_web/plugs/authorize_stats_plug.ex @@ -16,9 +16,10 @@ defmodule PlausibleWeb.AuthorizeStatsPlug do shared_link_key = "shared_link_auth_" <> site.domain shared_link_auth = get_session(conn, shared_link_key) - can_access = site.public || - (user_id && Plausible.Sites.is_owner?(user_id, site)) || - (shared_link_auth && shared_link_auth[:valid_until] > DateTime.to_unix(Timex.now())) + can_access = + site.public || + (user_id && Plausible.Sites.is_owner?(user_id, site)) || + (shared_link_auth && shared_link_auth[:valid_until] > DateTime.to_unix(Timex.now())) if !can_access do PlausibleWeb.ControllerHelpers.render_error(conn, 404) |> halt @@ -26,6 +27,5 @@ defmodule PlausibleWeb.AuthorizeStatsPlug do assign(conn, :site, site) end end - end end diff --git a/lib/plausible_web/plugs/last_seen_plug.ex b/lib/plausible_web/plugs/last_seen_plug.ex index be6f1c917..5a935d62f 100644 --- a/lib/plausible_web/plugs/last_seen_plug.ex +++ b/lib/plausible_web/plugs/last_seen_plug.ex @@ -13,11 +13,13 @@ defmodule PlausibleWeb.LastSeenPlug do user = conn.assigns[:current_user] cond do - user && last_seen && last_seen < (unix_now() - @one_hour) -> + user && last_seen && last_seen < unix_now() - @one_hour -> persist_last_seen(user) put_session(conn, :last_seen, unix_now()) + user && !last_seen -> put_session(conn, :last_seen, unix_now()) + true -> conn end @@ -26,10 +28,10 @@ defmodule PlausibleWeb.LastSeenPlug do defp persist_last_seen(user) do q = from(u in Plausible.Auth.User, where: u.id == ^user.id) - Repo.update_all(q, [set: [last_seen: DateTime.utc_now()]]) + Repo.update_all(q, set: [last_seen: DateTime.utc_now()]) end defp unix_now do - DateTime.utc_now() |> DateTime.to_unix + DateTime.utc_now() |> DateTime.to_unix() end end diff --git a/lib/plausible_web/plugs/require_account.ex b/lib/plausible_web/plugs/require_account.ex index de80e24ea..9df89095a 100644 --- a/lib/plausible_web/plugs/require_account.ex +++ b/lib/plausible_web/plugs/require_account.ex @@ -11,6 +11,7 @@ defmodule PlausibleWeb.RequireAccountPlug do Plug.Conn.put_session(conn, :login_dest, conn.request_path) |> Phoenix.Controller.redirect(to: "/login") |> halt + _email -> conn end diff --git a/lib/plausible_web/plugs/require_logged_out.ex b/lib/plausible_web/plugs/require_logged_out.ex index b8d233854..d25ee1b53 100644 --- a/lib/plausible_web/plugs/require_logged_out.ex +++ b/lib/plausible_web/plugs/require_logged_out.ex @@ -12,6 +12,7 @@ defmodule PlausibleWeb.RequireLoggedOutPlug do |> put_resp_cookie("logged_in", "true", http_only: false) |> Phoenix.Controller.redirect(to: "/sites") |> halt + :else -> conn end diff --git a/lib/plausible_web/plugs/session_timeout_plug.ex b/lib/plausible_web/plugs/session_timeout_plug.ex index 1cf673428..efa9fa054 100644 --- a/lib/plausible_web/plugs/session_timeout_plug.ex +++ b/lib/plausible_web/plugs/session_timeout_plug.ex @@ -14,15 +14,21 @@ defmodule PlausibleWeb.SessionTimeoutPlug do conn |> configure_session(drop: true) |> delete_resp_cookie("logged_in") + user_id -> - put_session(conn, :session_timeout_at, new_session_timeout_at(opts[:timeout_after_seconds])) + put_session( + conn, + :session_timeout_at, + new_session_timeout_at(opts[:timeout_after_seconds]) + ) + true -> conn end end defp now do - DateTime.utc_now() |> DateTime.to_unix + DateTime.utc_now() |> DateTime.to_unix() end defp new_session_timeout_at(timeout_after_seconds) do diff --git a/lib/plausible_web/router.ex b/lib/plausible_web/router.ex index 04cc59a9e..9e5af3846 100644 --- a/lib/plausible_web/router.ex +++ b/lib/plausible_web/router.ex @@ -110,11 +110,21 @@ defmodule PlausibleWeb.Router do post "/sites/:website/weekly-report/enable", SiteController, :enable_weekly_report post "/sites/:website/weekly-report/disable", SiteController, :disable_weekly_report post "/sites/:website/weekly-report/recipients", SiteController, :add_weekly_report_recipient - delete "/sites/:website/weekly-report/recipients/:recipient", SiteController, :remove_weekly_report_recipient + + delete "/sites/:website/weekly-report/recipients/:recipient", + SiteController, + :remove_weekly_report_recipient + post "/sites/:website/monthly-report/enable", SiteController, :enable_monthly_report post "/sites/:website/monthly-report/disable", SiteController, :disable_monthly_report - post "/sites/:website/monthly-report/recipients", SiteController, :add_monthly_report_recipient - delete "/sites/:website/monthly-report/recipients/:recipient", SiteController, :remove_monthly_report_recipient + + post "/sites/:website/monthly-report/recipients", + SiteController, + :add_monthly_report_recipient + + delete "/sites/:website/monthly-report/recipients/:recipient", + SiteController, + :remove_monthly_report_recipient get "/sites/:website/shared-links/new", SiteController, :new_shared_link post "/sites/:website/shared-links", SiteController, :create_shared_link @@ -128,7 +138,6 @@ defmodule PlausibleWeb.Router do get "/sites/:website/weekly-report/unsubscribe", UnsubscribeController, :weekly_report get "/sites/:website/monthly-report/unsubscribe", UnsubscribeController, :monthly_report - get "/:website/snippet", SiteController, :add_snippet get "/:website/settings", SiteController, :settings get "/:website/goals", SiteController, :goals diff --git a/lib/plausible_web/views/billing_view.ex b/lib/plausible_web/views/billing_view.ex index 9346899fd..54650c413 100644 --- a/lib/plausible_web/views/billing_view.ex +++ b/lib/plausible_web/views/billing_view.ex @@ -22,10 +22,13 @@ defmodule PlausibleWeb.BillingView do cond do usage < 9000 -> "10k / mo" + usage < 90_000 -> "100k / mo" + usage < 900_000 -> "1m / mo" + true -> "custom" end diff --git a/lib/plausible_web/views/email_view.ex b/lib/plausible_web/views/email_view.ex index c4074c657..5548506b7 100644 --- a/lib/plausible_web/views/email_view.ex +++ b/lib/plausible_web/views/email_view.ex @@ -15,7 +15,7 @@ defmodule PlausibleWeb.EmailView do def user_salutation(user) do if user.name do - String.split(user.name) |> List.first + String.split(user.name) |> List.first() else "" end @@ -25,13 +25,15 @@ defmodule PlausibleWeb.EmailView do cond do usage < 9_000 -> "Personal" + usage < 90_000 -> "Startup" + usage < 900_000 -> "Business" - true -> - throw "Huge account" + true -> + throw("Huge account") end end @@ -39,13 +41,15 @@ defmodule PlausibleWeb.EmailView do cond do usage < 9_000 -> "$6/mo" + usage < 90_000 -> "$12/mo" + usage < 900_000 -> "$36/mo" - true -> - throw "Huge account" + true -> + throw("Huge account") end end end diff --git a/lib/plausible_web/views/error_view.ex b/lib/plausible_web/views/error_view.ex index 3ad0ce866..441a1129d 100644 --- a/lib/plausible_web/views/error_view.ex +++ b/lib/plausible_web/views/error_view.ex @@ -9,28 +9,46 @@ defmodule PlausibleWeb.ErrorView do end def render("404.html", assigns) do - render("error.html", Map.merge(%{ - layout: false, - status: 404, - message: "Oops! There's nothing here" - }, assigns)) + render( + "error.html", + Map.merge( + %{ + layout: false, + status: 404, + message: "Oops! There's nothing here" + }, + assigns + ) + ) end def render("500.html", assigns) do - render("error.html", Map.merge(%{ - layout: false, - status: 500, - message: "Oops! Looks like we're having server issues" - }, assigns)) + render( + "error.html", + Map.merge( + %{ + layout: false, + status: 500, + message: "Oops! Looks like we're having server issues" + }, + assigns + ) + ) end def template_not_found(template, assigns) do status = String.trim_trailing(template, ".html") - render("error.html", Map.merge(%{ - layout: false, - status: status, - message: Phoenix.Controller.status_message_from_template(template) - }, assigns)) + render( + "error.html", + Map.merge( + %{ + layout: false, + status: status, + message: Phoenix.Controller.status_message_from_template(template) + }, + assigns + ) + ) end end diff --git a/lib/plausible_web/views/layout_view.ex b/lib/plausible_web/views/layout_view.ex index 4ec2b068a..2a5c20c13 100644 --- a/lib/plausible_web/views/layout_view.ex +++ b/lib/plausible_web/views/layout_view.ex @@ -25,10 +25,13 @@ defmodule PlausibleWeb.LayoutView do case Plausible.Billing.trial_days_left(user) do days when days > 1 -> "#{days} trial days left" + days when days == 1 -> "Trial ends tomorrow" + days when days == 0 -> "Trial ends today" + days when days < 0 -> "Trial over, upgrade now" end diff --git a/lib/plausible_web/views/stats_view.ex b/lib/plausible_web/views/stats_view.ex index 96f7e7132..1edef558b 100644 --- a/lib/plausible_web/views/stats_view.ex +++ b/lib/plausible_web/views/stats_view.ex @@ -17,20 +17,24 @@ defmodule PlausibleWeb.StatsView do cond do n >= 1_000 && n < 1_000_000 -> thousands = trunc(n / 100) / 10 + if thousands == trunc(thousands) || n >= 100_000 do "#{trunc(thousands)}k" else "#{thousands}k" end + n >= 1_000_000 && n < 100_000_000 -> millions = trunc(n / 100_000) / 10 + if millions == trunc(millions) do "#{trunc(millions)}m" else "#{millions}m" end - true -> - Integer.to_string(n) + + true -> + Integer.to_string(n) end end @@ -41,10 +45,12 @@ defmodule PlausibleWeb.StatsView do end defp bar_width(count, all) do - max = Enum.max_by(all, fn - {_, count} -> count - {_, count, _} -> count - end) |> elem(1) + max = + Enum.max_by(all, fn + {_, count} -> count + {_, count, _} -> count + end) + |> elem(1) count / max * 100 end diff --git a/lib/workers/fetch_tweets.ex b/lib/workers/fetch_tweets.ex index 6a63e85a6..1471d1c79 100644 --- a/lib/workers/fetch_tweets.ex +++ b/lib/workers/fetch_tweets.ex @@ -6,21 +6,26 @@ defmodule Plausible.Workers.FetchTweets do @impl Oban.Worker def perform(_args, _job, twitter_api \\ Plausible.Twitter.Api) do - new_links = Clickhouse.all( - from e in Plausible.ClickhouseEvent, - where: e.timestamp > fragment("(now() - INTERVAL 6 day)") and e.timestamp < fragment("(now() - INTERVAL 5 day)"), - or_where: e.timestamp > fragment("(now() - INTERVAL 1 day)"), - where: e.referrer_source == "Twitter", - where: e.referrer not in ["t.co", "t.co/"], - distinct: true, - select: e.referrer - ) |> Enum.map(fn event -> event["referrer"] end) + new_links = + Clickhouse.all( + from e in Plausible.ClickhouseEvent, + where: + e.timestamp > fragment("(now() - INTERVAL 6 day)") and + e.timestamp < fragment("(now() - INTERVAL 5 day)"), + or_where: e.timestamp > fragment("(now() - INTERVAL 1 day)"), + where: e.referrer_source == "Twitter", + where: e.referrer not in ["t.co", "t.co/"], + distinct: true, + select: e.referrer + ) + |> Enum.map(fn event -> event["referrer"] end) for link <- new_links do results = twitter_api.search(link) for tweet <- results do - {:ok, created} = Timex.parse(tweet["created_at"], "{WDshort} {Mshort} {D} {ISOtime} {Z} {YYYY}") + {:ok, created} = + Timex.parse(tweet["created_at"], "{WDshort} {Mshort} {D} {ISOtime} {Z} {YYYY}") Tweet.changeset(%Tweet{}, %{ link: link, @@ -30,17 +35,20 @@ defmodule Plausible.Workers.FetchTweets do author_image: tweet["user"]["profile_image_url_https"], text: html_body(tweet), created: created - }) |> Repo.insert!(on_conflict: :nothing) + }) + |> Repo.insert!(on_conflict: :nothing) end end + :ok end def html_body(tweet) do - body = Enum.reduce(tweet["entities"]["urls"], tweet["full_text"], fn url, text -> - html = "#{url["display_url"]}" - String.replace(text, url["url"], html) - end) + body = + Enum.reduce(tweet["entities"]["urls"], tweet["full_text"], fn url, text -> + html = "#{url["display_url"]}" + String.replace(text, url["url"], html) + end) Enum.reduce(tweet["entities"]["user_mentions"], body, fn mention, text -> link = "https://twitter.com/#{mention["screen_name"]}" diff --git a/lib/workers/provision_ssl_certificates.ex b/lib/workers/provision_ssl_certificates.ex index dc4391b2f..7ca0b3345 100644 --- a/lib/workers/provision_ssl_certificates.ex +++ b/lib/workers/provision_ssl_certificates.ex @@ -6,34 +6,41 @@ defmodule Plausible.Workers.ProvisionSslCertificates do def perform(_args, _job, ssh \\ SSHEx) do config = get_config() - {:ok, conn} = ssh.connect( - ip: to_charlist(config[:ip]), - user: to_charlist(config[:user]), - password: to_charlist(config[:password]) - ) + {:ok, conn} = + ssh.connect( + ip: to_charlist(config[:ip]), + user: to_charlist(config[:user]), + password: to_charlist(config[:password]) + ) - recent_custom_domains = Repo.all( - from cd in Plausible.Site.CustomDomain, - where: cd.updated_at > fragment("now() - '3 days'::interval"), - where: not cd.has_ssl_certificate - ) + recent_custom_domains = + Repo.all( + from cd in Plausible.Site.CustomDomain, + where: cd.updated_at > fragment("now() - '3 days'::interval"), + where: not cd.has_ssl_certificate + ) for domain <- recent_custom_domains do {:ok, res, code} = ssh.run(conn, 'sudo certbot certonly --nginx -n -d \"#{domain.domain}\"') report_result({res, code}, domain) end + :ok end defp report_result({_, 0}, domain) do - Ecto.Changeset.change(domain, has_ssl_certificate: true) |> Repo.update! + Ecto.Changeset.change(domain, has_ssl_certificate: true) |> Repo.update!() Plausible.Slack.notify("Obtained SSL cert for #{domain.domain}") :ok end defp report_result({error_msg, error_code}, domain) do - Sentry.capture_message("Error obtaining SSL certificate", extra: %{error_msg: error_msg, error_code: error_code, domain: domain.domain}) - :ok # Failing to obtain is expected, not a failure for the job queue + Sentry.capture_message("Error obtaining SSL certificate", + extra: %{error_msg: error_msg, error_code: error_code, domain: domain.domain} + ) + + # Failing to obtain is expected, not a failure for the job queue + :ok end defp get_config() do diff --git a/lib/workers/send_check_stats_emails.ex b/lib/workers/send_check_stats_emails.ex index b44974219..b1577452b 100644 --- a/lib/workers/send_check_stats_emails.ex +++ b/lib/workers/send_check_stats_emails.ex @@ -6,12 +6,13 @@ defmodule Plausible.Workers.SendCheckStatsEmails do def perform(_args, _job) do q = from(u in Plausible.Auth.User, - left_join: ce in "check_stats_emails", on: ce.user_id == u.id, + left_join: ce in "check_stats_emails", + on: ce.user_id == u.id, where: is_nil(ce.id), where: - u.inserted_at > fragment("(now() at time zone 'utc') - '14 days'::interval") and - u.inserted_at < fragment("(now() at time zone 'utc') - '7 days'::interval") and - u.last_seen < fragment("(now() at time zone 'utc') - '7 days'::interval"), + u.inserted_at > fragment("(now() at time zone 'utc') - '14 days'::interval") and + u.inserted_at < fragment("(now() at time zone 'utc') - '7 days'::interval") and + u.last_seen < fragment("(now() at time zone 'utc') - '7 days'::interval"), preload: [sites: :weekly_report] ) @@ -30,9 +31,11 @@ defmodule Plausible.Workers.SendCheckStatsEmails do PlausibleWeb.Email.check_stats_email(user) |> Plausible.Mailer.send_email() - Repo.insert_all("check_stats_emails", [%{ - user_id: user.id, - timestamp: NaiveDateTime.utc_now() - }]) + Repo.insert_all("check_stats_emails", [ + %{ + user_id: user.id, + timestamp: NaiveDateTime.utc_now() + } + ]) end end diff --git a/lib/workers/send_email_reports.ex b/lib/workers/send_email_reports.ex index 2c23a6692..9c5067d12 100644 --- a/lib/workers/send_email_reports.ex +++ b/lib/workers/send_email_reports.ex @@ -6,12 +6,15 @@ defmodule Plausible.Workers.SendEmailReports do alias Plausible.Stats.Clickhouse, as: Stats @impl Oban.Worker - @doc""" + @doc """ The email report should be sent on Monday at 9am according to the timezone of the site. This job runs every hour to be able to send it with hourly precision. """ def perform(args, _job) do - current_time = if args["current_time"], do: Timex.parse!(args["current_time"], "{ISO:Extended}"), else: Timex.now() + current_time = + if args["current_time"], + do: Timex.parse!(args["current_time"], "{ISO:Extended}"), + else: Timex.now() send_weekly_emails(current_time) send_monthly_emails(current_time) @@ -20,22 +23,36 @@ defmodule Plausible.Workers.SendEmailReports do end defp send_weekly_emails(job_start) do - sites = Repo.all( - from s in Plausible.Site, - join: wr in Plausible.Site.WeeklyReport, on: wr.site_id == s.id, - left_join: se in "sent_weekly_reports", on: se.site_id == s.id and se.year == fragment("EXTRACT(isoyear from (? at time zone ?))", ^job_start, s.timezone) and se.week == fragment("EXTRACT(week from (? at time zone ?))", ^job_start, s.timezone), - where: is_nil(se), # We haven't sent a report for this site on this week - where: fragment("EXTRACT(dow from (? at time zone ?))", ^job_start, s.timezone) == 1, # It's monday in the local timezone - where: fragment("EXTRACT(hour from (? at time zone ?))", ^job_start, s.timezone) >= 9, # It's after 9am - preload: [weekly_report: wr] - ) + sites = + Repo.all( + from s in Plausible.Site, + join: wr in Plausible.Site.WeeklyReport, + on: wr.site_id == s.id, + left_join: se in "sent_weekly_reports", + on: + se.site_id == s.id and + se.year == + fragment("EXTRACT(isoyear from (? at time zone ?))", ^job_start, s.timezone) and + se.week == fragment("EXTRACT(week from (? at time zone ?))", ^job_start, s.timezone), + # We haven't sent a report for this site on this week + where: is_nil(se), + # It's monday in the local timezone + where: fragment("EXTRACT(dow from (? at time zone ?))", ^job_start, s.timezone) == 1, + # It's after 9am + where: fragment("EXTRACT(hour from (? at time zone ?))", ^job_start, s.timezone) >= 9, + preload: [weekly_report: wr] + ) for site <- sites do query = Query.from(site.timezone, %{"period" => "7d"}) for email <- site.weekly_report.recipients do Logger.info("Sending weekly report for #{URI.encode_www_form(site.domain)} to #{email}") - unsubscribe_link = PlausibleWeb.Endpoint.url() <> "/sites/#{URI.encode_www_form(site.domain)}/weekly-report/unsubscribe?email=#{email}" + + unsubscribe_link = + PlausibleWeb.Endpoint.url() <> + "/sites/#{URI.encode_www_form(site.domain)}/weekly-report/unsubscribe?email=#{email}" + send_report(email, site, "Weekly", unsubscribe_link, query) end @@ -44,23 +61,46 @@ defmodule Plausible.Workers.SendEmailReports do end defp send_monthly_emails(job_start) do - sites = Repo.all( - from s in Plausible.Site, - join: mr in Plausible.Site.MonthlyReport, on: mr.site_id == s.id, - left_join: se in "sent_monthly_reports", on: se.site_id == s.id and se.year == fragment("EXTRACT(year from (? at time zone ?))", ^job_start, s.timezone) and se.month == fragment("EXTRACT(month from (? at time zone ?))", ^job_start, s.timezone), - where: is_nil(se), # We haven't sent a report for this site this month - where: fragment("EXTRACT(day from (? at time zone ?))", ^job_start, s.timezone) == 1, # It's the 1st of the month in the local timezone - where: fragment("EXTRACT(hour from (? at time zone ?))", ^job_start, s.timezone) >= 9, # It's after 9am - preload: [monthly_report: mr] - ) + sites = + Repo.all( + from s in Plausible.Site, + join: mr in Plausible.Site.MonthlyReport, + on: mr.site_id == s.id, + left_join: se in "sent_monthly_reports", + on: + se.site_id == s.id and + se.year == fragment("EXTRACT(year from (? at time zone ?))", ^job_start, s.timezone) and + se.month == + fragment("EXTRACT(month from (? at time zone ?))", ^job_start, s.timezone), + # We haven't sent a report for this site this month + where: is_nil(se), + # It's the 1st of the month in the local timezone + where: fragment("EXTRACT(day from (? at time zone ?))", ^job_start, s.timezone) == 1, + # It's after 9am + where: fragment("EXTRACT(hour from (? at time zone ?))", ^job_start, s.timezone) >= 9, + preload: [monthly_report: mr] + ) for site <- sites do - last_month = job_start |> Timex.Timezone.convert(site.timezone) |> Timex.shift(months: -1) |> Timex.beginning_of_month - query = Query.from(site.timezone, %{"period" => "month", "date" => Timex.format!(last_month, "{ISOdate}")}) + last_month = + job_start + |> Timex.Timezone.convert(site.timezone) + |> Timex.shift(months: -1) + |> Timex.beginning_of_month() + + query = + Query.from(site.timezone, %{ + "period" => "month", + "date" => Timex.format!(last_month, "{ISOdate}") + }) for email <- site.monthly_report.recipients do Logger.info("Sending monthly report for #{site.domain} to #{email}") - unsubscribe_link = PlausibleWeb.Endpoint.url() <> "/sites/#{URI.encode_www_form(site.domain)}/monthly-report/unsubscribe?email=#{email}" + + unsubscribe_link = + PlausibleWeb.Endpoint.url() <> + "/sites/#{URI.encode_www_form(site.domain)}/monthly-report/unsubscribe?email=#{email}" + send_report(email, site, Timex.format!(last_month, "{Mfull}"), unsubscribe_link, query) end @@ -70,7 +110,10 @@ defmodule Plausible.Workers.SendEmailReports do defp send_report(email, site, name, unsubscribe_link, query) do {pageviews, unique_visitors} = Stats.pageviews_and_visitors(site, query) - {change_pageviews, change_visitors} = Stats.compare_pageviews_and_visitors(site, query, {pageviews, unique_visitors}) + + {change_pageviews, change_visitors} = + Stats.compare_pageviews_and_visitors(site, query, {pageviews, unique_visitors}) + bounce_rate = Stats.bounce_rate(site, query) prev_bounce_rate = Stats.bounce_rate(site, Query.shift_back(query)) change_bounce_rate = if prev_bounce_rate > 0, do: bounce_rate - prev_bounce_rate @@ -97,24 +140,28 @@ defmodule Plausible.Workers.SendEmailReports do end defp weekly_report_sent(site, time) do - {year, week} = time |> DateTime.to_date |> Timex.iso_week - - Repo.insert_all("sent_weekly_reports", [%{ - site_id: site.id, - year: year, - week: week, - timestamp: Timex.now() - }]) + {year, week} = time |> DateTime.to_date() |> Timex.iso_week() + + Repo.insert_all("sent_weekly_reports", [ + %{ + site_id: site.id, + year: year, + week: week, + timestamp: Timex.now() + } + ]) end defp monthly_report_sent(site, time) do date = DateTime.to_date(time) - Repo.insert_all("sent_monthly_reports", [%{ - site_id: site.id, - year: date.year, - month: date.month, - timestamp: Timex.now() - }]) + Repo.insert_all("sent_monthly_reports", [ + %{ + site_id: site.id, + year: date.year, + month: date.month, + timestamp: Timex.now() + } + ]) end end diff --git a/lib/workers/send_site_setup_emails.ex b/lib/workers/send_site_setup_emails.ex index 171ec811d..03b2e2ce5 100644 --- a/lib/workers/send_site_setup_emails.ex +++ b/lib/workers/send_site_setup_emails.ex @@ -16,9 +16,12 @@ defmodule Plausible.Workers.SendSiteSetupEmails do defp send_create_site_emails() do q = from(s in Plausible.Auth.User, - left_join: se in "create_site_emails", on: se.user_id == s.id, + left_join: se in "create_site_emails", + on: se.user_id == s.id, where: is_nil(se.id), - where: s.inserted_at > fragment("(now() at time zone 'utc') - '72 hours'::interval") and s.inserted_at < fragment("(now() at time zone 'utc') - '48 hours'::interval"), + where: + s.inserted_at > fragment("(now() at time zone 'utc') - '72 hours'::interval") and + s.inserted_at < fragment("(now() at time zone 'utc') - '48 hours'::interval"), preload: :sites ) @@ -32,7 +35,8 @@ defmodule Plausible.Workers.SendSiteSetupEmails do defp send_setup_help_emails() do q = from(s in Plausible.Site, - left_join: se in "setup_help_emails", on: se.site_id == s.id, + left_join: se in "setup_help_emails", + on: se.site_id == s.id, where: is_nil(se.id), where: s.inserted_at > fragment("(now() at time zone 'utc') - '72 hours'::interval"), preload: :members @@ -53,7 +57,8 @@ defmodule Plausible.Workers.SendSiteSetupEmails do defp send_setup_success_emails() do q = from(s in Plausible.Site, - left_join: se in "setup_success_emails", on: se.site_id == s.id, + left_join: se in "setup_success_emails", + on: se.site_id == s.id, where: is_nil(se.id), where: s.inserted_at > fragment("(now() at time zone 'utc') - '72 hours'::interval"), preload: :members @@ -72,29 +77,35 @@ defmodule Plausible.Workers.SendSiteSetupEmails do PlausibleWeb.Email.create_site_email(user) |> Plausible.Mailer.send_email() - Repo.insert_all("create_site_emails", [%{ - user_id: user.id, - timestamp: NaiveDateTime.utc_now() - }]) + Repo.insert_all("create_site_emails", [ + %{ + user_id: user.id, + timestamp: NaiveDateTime.utc_now() + } + ]) end defp send_setup_success_email(user, site) do PlausibleWeb.Email.site_setup_success(user, site) |> Plausible.Mailer.send_email() - Repo.insert_all("setup_success_emails", [%{ - site_id: site.id, - timestamp: NaiveDateTime.utc_now() - }]) + Repo.insert_all("setup_success_emails", [ + %{ + site_id: site.id, + timestamp: NaiveDateTime.utc_now() + } + ]) end defp send_setup_help_email(user, site) do PlausibleWeb.Email.site_setup_help(user, site) |> Plausible.Mailer.send_email() - Repo.insert_all("setup_help_emails", [%{ - site_id: site.id, - timestamp: NaiveDateTime.utc_now() - }]) + Repo.insert_all("setup_help_emails", [ + %{ + site_id: site.id, + timestamp: NaiveDateTime.utc_now() + } + ]) end end diff --git a/lib/workers/send_trial_notifications.ex b/lib/workers/send_trial_notifications.ex index 13aec9f08..bc4b3c65d 100644 --- a/lib/workers/send_trial_notifications.ex +++ b/lib/workers/send_trial_notifications.ex @@ -5,31 +5,37 @@ defmodule Plausible.Workers.SendTrialNotifications do @impl Oban.Worker def perform(_args, _job) do - users = Repo.all( - from u in Plausible.Auth.User, - left_join: s in Plausible.Billing.Subscription, on: s.user_id == u.id, - where: is_nil(s.id), - order_by: u.inserted_at - ) + users = + Repo.all( + from u in Plausible.Auth.User, + left_join: s in Plausible.Billing.Subscription, + on: s.user_id == u.id, + where: is_nil(s.id), + order_by: u.inserted_at + ) for user <- users do case Timex.diff(user.trial_expiry_date, Timex.today(), :days) do - 7 -> + 7 -> if Plausible.Auth.user_completed_setup?(user) do send_one_week_reminder(user) end - 1 -> + + 1 -> if Plausible.Auth.user_completed_setup?(user) do send_tomorrow_reminder(user) end - 0 -> + + 0 -> if Plausible.Auth.user_completed_setup?(user) do send_today_reminder(user) end - -1 -> + + -1 -> if Plausible.Auth.user_completed_setup?(user) do send_over_reminder(user) end + _ -> nil end @@ -43,7 +49,7 @@ defmodule Plausible.Workers.SendTrialNotifications do |> Plausible.Mailer.send_email() end - defp send_tomorrow_reminder( user) do + defp send_tomorrow_reminder(user) do usage = Plausible.Billing.usage(user) PlausibleWeb.Email.trial_upgrade_email(user, "tomorrow", usage) diff --git a/priv/repo/migrations/20190127213938_add_tz_to_sites.exs b/priv/repo/migrations/20190127213938_add_tz_to_sites.exs index 570bb1b63..3649dfb09 100644 --- a/priv/repo/migrations/20190127213938_add_tz_to_sites.exs +++ b/priv/repo/migrations/20190127213938_add_tz_to_sites.exs @@ -9,7 +9,7 @@ defmodule Plausible.Repo.Migrations.AddTzToSites do flush() - Repo.update_all(Plausible.Site, [set: [timezone: "UTC"]]) + Repo.update_all(Plausible.Site, set: [timezone: "UTC"]) alter table(:sites) do modify :timezone, :string, null: false diff --git a/priv/repo/migrations/20190219130809_delete_intro_emails_when_user_is_deleted.exs b/priv/repo/migrations/20190219130809_delete_intro_emails_when_user_is_deleted.exs index 3490fc1ee..3fb741dd1 100644 --- a/priv/repo/migrations/20190219130809_delete_intro_emails_when_user_is_deleted.exs +++ b/priv/repo/migrations/20190219130809_delete_intro_emails_when_user_is_deleted.exs @@ -3,7 +3,9 @@ defmodule Plausible.Repo.Migrations.DeleteIntroEmailsWhenUserIsDeleted do def change do alter table(:intro_emails) do - modify :user_id, references(:users, on_delete: :delete_all), null: false, from: references(:users) + modify :user_id, references(:users, on_delete: :delete_all), + null: false, + from: references(:users) end end end diff --git a/priv/repo/migrations/20190424162903_delete_feedback_emails_when_user_is_deleted.exs b/priv/repo/migrations/20190424162903_delete_feedback_emails_when_user_is_deleted.exs index 681644237..88a0aea43 100644 --- a/priv/repo/migrations/20190424162903_delete_feedback_emails_when_user_is_deleted.exs +++ b/priv/repo/migrations/20190424162903_delete_feedback_emails_when_user_is_deleted.exs @@ -3,7 +3,9 @@ defmodule Plausible.Repo.Migrations.DeleteFeedbackEmailsWhenUserIsDeleted do def change do alter table(:feedback_emails) do - modify :user_id, references(:users, on_delete: :delete_all), null: false, from: references(:users) + modify :user_id, references(:users, on_delete: :delete_all), + null: false, + from: references(:users) end end end diff --git a/priv/repo/migrations/20190911102027_add_monthly_reports.exs b/priv/repo/migrations/20190911102027_add_monthly_reports.exs index 122b95e4f..57b95a61a 100644 --- a/priv/repo/migrations/20190911102027_add_monthly_reports.exs +++ b/priv/repo/migrations/20190911102027_add_monthly_reports.exs @@ -60,7 +60,6 @@ defmodule Plausible.Repo.Migrations.AddMonthlyReports do execute "ALTER SEQUENCE weekly_reports_id_seq RENAME TO email_settings_id_seq;" create unique_index(:email_settings, :site_id) - drop constraint(:sent_weekly_reports, "sent_weekly_reports_site_id_fkey") drop constraint(:sent_weekly_reports, "sent_weekly_reports_pkey") diff --git a/priv/repo/migrations/20191010031425_add_property_to_google_auth.exs b/priv/repo/migrations/20191010031425_add_property_to_google_auth.exs index 9601d1cce..9c16c8bd2 100644 --- a/priv/repo/migrations/20191010031425_add_property_to_google_auth.exs +++ b/priv/repo/migrations/20191010031425_add_property_to_google_auth.exs @@ -12,8 +12,9 @@ defmodule Plausible.Repo.Migrations.AddPropertyToGoogleAuth do for auth <- Repo.all(Plausible.Site.GoogleAuth) do auth = Repo.preload(auth, :site) property = "https://#{auth.site.domain}" + Plausible.Site.GoogleAuth.set_property(auth, %{property: property}) - |> Repo.update! + |> Repo.update!() end end end diff --git a/priv/repo/migrations/20200324132431_make_cookie_fields_non_required.exs b/priv/repo/migrations/20200324132431_make_cookie_fields_non_required.exs index 2a3a4dbdf..1ac09de77 100644 --- a/priv/repo/migrations/20200324132431_make_cookie_fields_non_required.exs +++ b/priv/repo/migrations/20200324132431_make_cookie_fields_non_required.exs @@ -9,6 +9,5 @@ defmodule Plausible.Repo.Migrations.MakeCookieFieldsNonRequired do end def down do - end end diff --git a/test/plausible/billing/billing_test.exs b/test/plausible/billing/billing_test.exs index 41074a52c..cbaa7dd6d 100644 --- a/test/plausible/billing/billing_test.exs +++ b/test/plausible/billing/billing_test.exs @@ -76,10 +76,10 @@ defmodule Plausible.BillingTest do @subscription_id "subscription-123" @plan_id "plan-123" - describe "subscription_created" do test "creates a subscription" do user = insert(:user) + Billing.subscription_created(%{ "alert_name" => "subscription_created", "subscription_id" => @subscription_id, @@ -160,11 +160,12 @@ defmodule Plausible.BillingTest do end test "ignores if the subscription cannot be found" do - res = Billing.subscription_cancelled(%{ - "alert_name" => "subscription_cancelled", - "subscription_id" => "some_nonexistent_id", - "status" => "deleted" - }) + res = + Billing.subscription_cancelled(%{ + "alert_name" => "subscription_cancelled", + "subscription_id" => "some_nonexistent_id", + "status" => "deleted" + }) assert res == {:ok, nil} end @@ -186,12 +187,13 @@ defmodule Plausible.BillingTest do end test "ignores if the subscription cannot be found" do - res = Billing.subscription_payment_succeeded(%{ - "alert_name" => "subscription_payment_succeeded", - "subscription_id" => "nonexistent_subscription_id", - "next_bill_date" => Timex.shift(Timex.today(), days: 30), - "unit_price" => "12.00" - }) + res = + Billing.subscription_payment_succeeded(%{ + "alert_name" => "subscription_payment_succeeded", + "subscription_id" => "nonexistent_subscription_id", + "next_bill_date" => Timex.shift(Timex.today(), days: 30), + "unit_price" => "12.00" + }) assert res == {:ok, nil} end @@ -200,7 +202,7 @@ defmodule Plausible.BillingTest do describe "change_plan" do test "sets the next bill amount and date" do user = insert(:user) - insert(:subscription, user: user) + insert(:subscription, user: user) Billing.change_plan(user, "123123") diff --git a/test/plausible/stats/query_test.exs b/test/plausible/stats/query_test.exs index 300515adb..955c2e502 100644 --- a/test/plausible/stats/query_test.exs +++ b/test/plausible/stats/query_test.exs @@ -31,7 +31,9 @@ defmodule Plausible.Stats.QueryTest do test "parses 6 month format" do q = Query.from(@tz, %{"period" => "6mo"}) - assert q.date_range.first == Timex.shift(Timex.today(), months: -5) |> Timex.beginning_of_month() + assert q.date_range.first == + Timex.shift(Timex.today(), months: -5) |> Timex.beginning_of_month() + assert q.date_range.last == Timex.today() assert q.step_type == "month" end @@ -39,7 +41,9 @@ defmodule Plausible.Stats.QueryTest do test "parses 12 month format" do q = Query.from(@tz, %{"period" => "12mo"}) - assert q.date_range.first == Timex.shift(Timex.today(), months: -11) |> Timex.beginning_of_month() + assert q.date_range.first == + Timex.shift(Timex.today(), months: -11) |> Timex.beginning_of_month() + assert q.date_range.last == Timex.today() assert q.step_type == "month" end diff --git a/test/plausible_web/controllers/api/external_controller_test.exs b/test/plausible_web/controllers/api/external_controller_test.exs index bf1a07800..ef8b95609 100644 --- a/test/plausible_web/controllers/api/external_controller_test.exs +++ b/test/plausible_web/controllers/api/external_controller_test.exs @@ -5,12 +5,14 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do defp get_event(domain) do Plausible.Event.WriteBuffer.flush() - events = Plausible.Clickhouse.all( - from e in Plausible.ClickhouseEvent, - where: e.domain ==^domain, - order_by: [desc: e.timestamp], - limit: 1 - ) + events = + Plausible.Clickhouse.all( + from e in Plausible.ClickhouseEvent, + where: e.domain == ^domain, + order_by: [desc: e.timestamp], + limit: 1 + ) + List.first(events) end @@ -27,11 +29,12 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do screen_width: 1440 } - conn = conn - |> put_req_header("content-type", "text/plain") - |> put_req_header("user-agent", @user_agent) - |> put_req_header("x-country", @country_code) - |> post("/api/event", Jason.encode!(params)) + conn = + conn + |> put_req_header("content-type", "text/plain") + |> put_req_header("user-agent", @user_agent) + |> put_req_header("x-country", @country_code) + |> post("/api/event", Jason.encode!(params)) pageview = get_event("external-controller-test-1.com") @@ -112,10 +115,11 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do domain: "external-controller-test-6.com" } - conn = conn - |> put_req_header("content-type", "text/plain") - |> put_req_header("user-agent", @user_agent) - |> post("/api/event", Jason.encode!(params)) + conn = + conn + |> put_req_header("content-type", "text/plain") + |> put_req_header("user-agent", @user_agent) + |> post("/api/event", Jason.encode!(params)) pageview = get_event("external-controller-test-6.com") @@ -132,10 +136,11 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do domain: "external-controller-test-7.com" } - conn = conn - |> put_req_header("content-type", "text/plain") - |> put_req_header("user-agent", @user_agent) - |> post("/api/event", Jason.encode!(params)) + conn = + conn + |> put_req_header("content-type", "text/plain") + |> put_req_header("user-agent", @user_agent) + |> post("/api/event", Jason.encode!(params)) pageview = get_event("external-controller-test-7.com") @@ -151,10 +156,11 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do domain: "external-controller-test-8.com" } - conn = conn - |> put_req_header("content-type", "text/plain") - |> put_req_header("user-agent", @user_agent) - |> post("/api/event", Jason.encode!(params)) + conn = + conn + |> put_req_header("content-type", "text/plain") + |> put_req_header("user-agent", @user_agent) + |> post("/api/event", Jason.encode!(params)) pageview = get_event("external-controller-test-8.com") @@ -171,10 +177,11 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do domain: "external-controller-test-9.com" } - conn = conn - |> put_req_header("content-type", "text/plain") - |> put_req_header("user-agent", @user_agent) - |> post("/api/event", Jason.encode!(params)) + conn = + conn + |> put_req_header("content-type", "text/plain") + |> put_req_header("user-agent", @user_agent) + |> post("/api/event", Jason.encode!(params)) pageview = get_event("external-controller-test-9.com") @@ -190,10 +197,11 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do domain: "external-controller-test-10.com" } - conn = conn - |> put_req_header("content-type", "text/plain") - |> put_req_header("user-agent", @user_agent) - |> post("/api/event", Jason.encode!(params)) + conn = + conn + |> put_req_header("content-type", "text/plain") + |> put_req_header("user-agent", @user_agent) + |> post("/api/event", Jason.encode!(params)) pageview = get_event("external-controller-test-10.com") @@ -209,10 +217,11 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do domain: "external-controller-test-11.com" } - conn = conn - |> put_req_header("content-type", "text/plain") - |> put_req_header("user-agent", @user_agent) - |> post("/api/event", Jason.encode!(params)) + conn = + conn + |> put_req_header("content-type", "text/plain") + |> put_req_header("user-agent", @user_agent) + |> post("/api/event", Jason.encode!(params)) pageview = get_event("external-controller-test-11.com") @@ -263,10 +272,11 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do domain: "external-controller-test-14.com" } - conn = conn - |> put_req_header("content-type", "text/plain") - |> put_req_header("user-agent", @user_agent) - |> post("/api/event", Jason.encode!(params)) + conn = + conn + |> put_req_header("content-type", "text/plain") + |> put_req_header("user-agent", @user_agent) + |> post("/api/event", Jason.encode!(params)) pageview = get_event("external-controller-test-14.com") @@ -282,17 +292,17 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do domain: "external-controller-test-15.com" } - conn = conn - |> put_req_header("content-type", "text/plain") - |> put_req_header("user-agent", @user_agent) - |> post("/api/event", Jason.encode!(params)) + conn = + conn + |> put_req_header("content-type", "text/plain") + |> put_req_header("user-agent", @user_agent) + |> post("/api/event", Jason.encode!(params)) pageview = get_event("external-controller-test-15.com") assert response(conn, 202) == "" assert pageview["referrer_source"] == "" end - end test "screen size is calculated from screen_width", %{conn: conn} do @@ -303,10 +313,11 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do domain: "external-controller-test-16.com" } - conn = conn - |> put_req_header("content-type", "text/plain") - |> put_req_header("user-agent", @user_agent) - |> post("/api/event", Jason.encode!(params)) + conn = + conn + |> put_req_header("content-type", "text/plain") + |> put_req_header("user-agent", @user_agent) + |> post("/api/event", Jason.encode!(params)) pageview = get_event("external-controller-test-16.com") @@ -321,10 +332,11 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do domain: "external-controller-test-17.com" } - conn = conn - |> put_req_header("content-type", "text/plain") - |> put_req_header("user-agent", @user_agent) - |> post("/api/event", Jason.encode!(params)) + conn = + conn + |> put_req_header("content-type", "text/plain") + |> put_req_header("user-agent", @user_agent) + |> post("/api/event", Jason.encode!(params)) pageview = get_event("external-controller-test-17.com") @@ -339,10 +351,11 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do domain: "external-controller-test-18.com" } - conn = conn - |> put_req_header("content-type", "text/plain") - |> put_req_header("user-agent", @user_agent) - |> post("/api/event", Jason.encode!(params)) + conn = + conn + |> put_req_header("content-type", "text/plain") + |> put_req_header("user-agent", @user_agent) + |> post("/api/event", Jason.encode!(params)) event = get_event("external-controller-test-18.com") @@ -353,10 +366,11 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do test "responds 400 when required fields are missing", %{conn: conn} do params = %{} - conn = conn - |> put_req_header("content-type", "text/plain") - |> put_req_header("user-agent", @user_agent) - |> post("/api/event", Jason.encode!(params)) + conn = + conn + |> put_req_header("content-type", "text/plain") + |> put_req_header("user-agent", @user_agent) + |> post("/api/event", Jason.encode!(params)) assert response(conn, 400) == "" end diff --git a/test/plausible_web/controllers/api/paddle_controller_test.exs b/test/plausible_web/controllers/api/paddle_controller_test.exs index ad4ca8db9..2eb63e9b0 100644 --- a/test/plausible_web/controllers/api/paddle_controller_test.exs +++ b/test/plausible_web/controllers/api/paddle_controller_test.exs @@ -4,23 +4,26 @@ defmodule PlausibleWeb.Api.PaddleControllerTest do @body %{ "alert_id" => "16173800", - "alert_name"=> "subscription_created", - "cancel_url"=> "https://checkout.paddle.com/subscription/cancel?user=1032746&subscription=1869424&hash=eyJpdiI6Ik5XSnhkT1k2RSticStZcEVYZ2FEeTVZTHVSaTNpcFM4XC9aMFJNSjh6TFNrPSIsInZhbHVlIjoiWTMzd3dENGQySTc0UjVPM0U3dzVDUWlBMEQ3dGVzM2lyODRPczBVcjdBRXlncmQzRUVmQjhnZVhpTzJIRjFtNW41MmZOVmJNVUJ4empmeXgyUno1Z3c9PSIsIm1hYyI6ImFjNDMyNDIxNmNmMWNiNmViMmFlNGVkMzQ3ZjYyYTQ5ZWI2YTEyMDQ4YjFhNTEyMjAxNzNlNjEwOWU4OTVhOWMifQ%3D%3D", - "checkout_id"=> "38111668-chre6449b9c3cc8-f473e48a86", - "currency"=> "USD", - "email"=> "josh@joshuae.com", - "event_time"=> "2019-08-20 21:44:54", - "marketing_consent"=> 0, - "next_bill_date"=> "2019-09-20", - "passthrough"=> "235", - "quantity"=> "1", - "status"=> "active", - "subscription_id"=> "1869424", - "subscription_plan_id"=> "558018", - "unit_price"=> "6.00", - "update_url"=> "https://checkout.paddle.com/subscription/update?user=1032746&subscription=1869424&hash=eyJpdiI6IkNUS2VZQlRxcFA5MVlEXC9Oa1ZwRDBNaGN4VVwvaU1RU2srWXc0bU1tQndyTT0iLCJ2YWx1ZSI6ImtTeU1ESkxWcEVrTDFmKzkyZ0FaSFo2Q0VNK3A2XC9NdEU4S2tGVFE2blJicGxBQzZ1XC9mMG1PcUo3MWV2OGY4YURjT1UxY2hpeHh5SFhlMmhXaFpoalE9PSIsIm1hYyI6ImZkNGU5MTg3YzQxZWYxNDJjNDkyMWFkYmZhZjIyMGQ3ZGI2YTVmMTcxZGViY2VkNzI0ZjNlMDRkZTgwNTEwMzUifQ%3D%3D", - "user_id"=> "1032746", - "p_signature"=> "qfqKA3dI9d60uie9IORcvkHYV+rd1UaCu/f5kh4miTkeIQNimgusQG8pS1OHobCvN/OktwKCjFcbIwoa4nakOOWGroHJ8FjLJHBK4g1uI37Bp6l73dNl8mB4dNGW1M+atkz7ag6pETRIdEKCmC5tV9afN5CvbcqRV1lsj/x2fAsjAe/sQkmAP1jbDXOMEuHqkWssSB7Q+NGHHLHuNQ67m7YFBnZSgYzLeLMEApkZClJn0j6MokUVjW37ISn5eA5FlUbT7s6Kph54roRzLIpYvC+ff/n6ae2Iu1OsORxRBg4Uv8dqqjqBKXlv84/OB80U89yMIbRw/pbHD6+zF4FxgNV7nk2bjgK2V6h55AOuhJHHUMb4XX9R8i8iG1FOlNJaTwbhkIkvQF3q7nEItKCqizn+l4tFQ9MUcrjw8jytDznbOnSlmNhtcDVlnvXNDaSPkEA7AyR6c+BiZV/Y6I3y8sr8h/F/cBM3OPTwfdKK34jyWW4LRn15nSxq2kjH3SyLPEpTJUMdcRGAgBZc06E4lENU2x22E/JKG5BRi1aDs5OFQtrjYi2hOTI0dyPF3OLNeZcCgBCKBmKq5XIf1T0RPFWAWtKkzXhl/QH+4feNATb9/i6k5xKeUJf0ltWzsI5x84kvsC/m05hn/AuBDmcZGkVnDLXrqttR+zDXY6P1euE=" + "alert_name" => "subscription_created", + "cancel_url" => + "https://checkout.paddle.com/subscription/cancel?user=1032746&subscription=1869424&hash=eyJpdiI6Ik5XSnhkT1k2RSticStZcEVYZ2FEeTVZTHVSaTNpcFM4XC9aMFJNSjh6TFNrPSIsInZhbHVlIjoiWTMzd3dENGQySTc0UjVPM0U3dzVDUWlBMEQ3dGVzM2lyODRPczBVcjdBRXlncmQzRUVmQjhnZVhpTzJIRjFtNW41MmZOVmJNVUJ4empmeXgyUno1Z3c9PSIsIm1hYyI6ImFjNDMyNDIxNmNmMWNiNmViMmFlNGVkMzQ3ZjYyYTQ5ZWI2YTEyMDQ4YjFhNTEyMjAxNzNlNjEwOWU4OTVhOWMifQ%3D%3D", + "checkout_id" => "38111668-chre6449b9c3cc8-f473e48a86", + "currency" => "USD", + "email" => "josh@joshuae.com", + "event_time" => "2019-08-20 21:44:54", + "marketing_consent" => 0, + "next_bill_date" => "2019-09-20", + "passthrough" => "235", + "quantity" => "1", + "status" => "active", + "subscription_id" => "1869424", + "subscription_plan_id" => "558018", + "unit_price" => "6.00", + "update_url" => + "https://checkout.paddle.com/subscription/update?user=1032746&subscription=1869424&hash=eyJpdiI6IkNUS2VZQlRxcFA5MVlEXC9Oa1ZwRDBNaGN4VVwvaU1RU2srWXc0bU1tQndyTT0iLCJ2YWx1ZSI6ImtTeU1ESkxWcEVrTDFmKzkyZ0FaSFo2Q0VNK3A2XC9NdEU4S2tGVFE2blJicGxBQzZ1XC9mMG1PcUo3MWV2OGY4YURjT1UxY2hpeHh5SFhlMmhXaFpoalE9PSIsIm1hYyI6ImZkNGU5MTg3YzQxZWYxNDJjNDkyMWFkYmZhZjIyMGQ3ZGI2YTVmMTcxZGViY2VkNzI0ZjNlMDRkZTgwNTEwMzUifQ%3D%3D", + "user_id" => "1032746", + "p_signature" => + "qfqKA3dI9d60uie9IORcvkHYV+rd1UaCu/f5kh4miTkeIQNimgusQG8pS1OHobCvN/OktwKCjFcbIwoa4nakOOWGroHJ8FjLJHBK4g1uI37Bp6l73dNl8mB4dNGW1M+atkz7ag6pETRIdEKCmC5tV9afN5CvbcqRV1lsj/x2fAsjAe/sQkmAP1jbDXOMEuHqkWssSB7Q+NGHHLHuNQ67m7YFBnZSgYzLeLMEApkZClJn0j6MokUVjW37ISn5eA5FlUbT7s6Kph54roRzLIpYvC+ff/n6ae2Iu1OsORxRBg4Uv8dqqjqBKXlv84/OB80U89yMIbRw/pbHD6+zF4FxgNV7nk2bjgK2V6h55AOuhJHHUMb4XX9R8i8iG1FOlNJaTwbhkIkvQF3q7nEItKCqizn+l4tFQ9MUcrjw8jytDznbOnSlmNhtcDVlnvXNDaSPkEA7AyR6c+BiZV/Y6I3y8sr8h/F/cBM3OPTwfdKK34jyWW4LRn15nSxq2kjH3SyLPEpTJUMdcRGAgBZc06E4lENU2x22E/JKG5BRi1aDs5OFQtrjYi2hOTI0dyPF3OLNeZcCgBCKBmKq5XIf1T0RPFWAWtKkzXhl/QH+4feNATb9/i6k5xKeUJf0ltWzsI5x84kvsC/m05hn/AuBDmcZGkVnDLXrqttR+zDXY6P1euE=" } describe "webhook verification" do @@ -37,5 +40,4 @@ defmodule PlausibleWeb.Api.PaddleControllerTest do assert conn.status == 400 end end - end diff --git a/test/plausible_web/controllers/api/stats_controller/browsers_test.exs b/test/plausible_web/controllers/api/stats_controller/browsers_test.exs index 5aeff5e3a..8bbbbe445 100644 --- a/test/plausible_web/controllers/api/stats_controller/browsers_test.exs +++ b/test/plausible_web/controllers/api/stats_controller/browsers_test.exs @@ -9,9 +9,9 @@ defmodule PlausibleWeb.Api.StatsController.BrowsersTest do conn = get(conn, "/api/stats/#{site.domain}/browsers?period=day&date=2019-01-01") assert json_response(conn, 200) == [ - %{"name" => "Chrome", "count" => 2, "percentage" => 67}, - %{"name" => "Firefox", "count" => 1, "percentage" => 33}, - ] + %{"name" => "Chrome", "count" => 2, "percentage" => 67}, + %{"name" => "Firefox", "count" => 1, "percentage" => 33} + ] end end end diff --git a/test/plausible_web/controllers/api/stats_controller/conversions_test.exs b/test/plausible_web/controllers/api/stats_controller/conversions_test.exs index bdfd2f187..e0547023a 100644 --- a/test/plausible_web/controllers/api/stats_controller/conversions_test.exs +++ b/test/plausible_web/controllers/api/stats_controller/conversions_test.exs @@ -12,9 +12,9 @@ defmodule PlausibleWeb.Api.StatsController.ConversionsTest do conn = get(conn, "/api/stats/#{site.domain}/conversions?period=day&date=2019-01-01") assert json_response(conn, 200) == [ - %{"name" => "Signup", "count" => 3}, - %{"name" => "Visit /register", "count" => 2} - ] + %{"name" => "Signup", "count" => 3}, + %{"name" => "Visit /register", "count" => 2} + ] end end @@ -26,11 +26,16 @@ defmodule PlausibleWeb.Api.StatsController.ConversionsTest do insert(:goal, %{domain: site.domain, event_name: "Signup"}) filters = Jason.encode!(%{goal: "Signup"}) - conn = get(conn, "/api/stats/#{site.domain}/conversions?period=day&date=2019-01-01&filters=#{filters}") + + conn = + get( + conn, + "/api/stats/#{site.domain}/conversions?period=day&date=2019-01-01&filters=#{filters}" + ) assert json_response(conn, 200) == [ - %{"name" => "Signup", "count" => 3}, - ] + %{"name" => "Signup", "count" => 3} + ] end end end diff --git a/test/plausible_web/controllers/api/stats_controller/countries_test.exs b/test/plausible_web/controllers/api/stats_controller/countries_test.exs index f56021d84..fd885dfa6 100644 --- a/test/plausible_web/controllers/api/stats_controller/countries_test.exs +++ b/test/plausible_web/controllers/api/stats_controller/countries_test.exs @@ -9,9 +9,19 @@ defmodule PlausibleWeb.Api.StatsController.CountriesTest do conn = get(conn, "/api/stats/#{site.domain}/countries?period=day&date=2019-01-01") assert json_response(conn, 200) == [ - %{"name" => "EST", "full_country_name" => "Estonia", "count" => 2, "percentage" => 67}, - %{"name" => "GBR", "full_country_name" => "United Kingdom", "count" => 1, "percentage" => 33}, - ] + %{ + "name" => "EST", + "full_country_name" => "Estonia", + "count" => 2, + "percentage" => 67 + }, + %{ + "name" => "GBR", + "full_country_name" => "United Kingdom", + "count" => 1, + "percentage" => 33 + } + ] end end end diff --git a/test/plausible_web/controllers/api/stats_controller/main_graph_test.exs b/test/plausible_web/controllers/api/stats_controller/main_graph_test.exs index b66cb3b8b..2a7dfcb86 100644 --- a/test/plausible_web/controllers/api/stats_controller/main_graph_test.exs +++ b/test/plausible_web/controllers/api/stats_controller/main_graph_test.exs @@ -17,7 +17,8 @@ defmodule PlausibleWeb.Api.StatsController.MainGraphTest do end test "displays hourly stats in configured timezone", %{conn: conn, user: user} do - site = insert(:site, domain: "tz-test.com", members: [user], timezone: "CET") # UTC+1 + # UTC+1 + site = insert(:site, domain: "tz-test.com", members: [user], timezone: "CET") conn = get(conn, "/api/stats/#{site.domain}/main-graph?period=day&date=2019-01-01") @@ -25,7 +26,8 @@ defmodule PlausibleWeb.Api.StatsController.MainGraphTest do zeroes = Stream.repeatedly(fn -> 0 end) |> Stream.take(22) |> Enum.into([]) - assert plot == [0, 1] ++ zeroes # Expecting pageview to show at 1am CET + # Expecting pageview to show at 1am CET + assert plot == [0, 1] ++ zeroes end test "displays visitors for a month", %{conn: conn, site: site} do @@ -90,13 +92,17 @@ defmodule PlausibleWeb.Api.StatsController.MainGraphTest do end end - describe "GET /api/stats/main-graph - filtered for goal" do setup [:create_user, :log_in, :create_site] test "returns total unique visitors", %{conn: conn, site: site} do filters = Jason.encode!(%{goal: "Signup"}) - conn = get(conn, "/api/stats/#{site.domain}/main-graph?period=day&date=2019-01-01&filters=#{filters}") + + conn = + get( + conn, + "/api/stats/#{site.domain}/main-graph?period=day&date=2019-01-01&filters=#{filters}" + ) res = json_response(conn, 200) assert %{"name" => "Total visitors", "count" => 6, "change" => 100} in res["top_stats"] @@ -104,7 +110,12 @@ defmodule PlausibleWeb.Api.StatsController.MainGraphTest do test "returns converted visitors", %{conn: conn, site: site} do filters = Jason.encode!(%{goal: "Signup"}) - conn = get(conn, "/api/stats/#{site.domain}/main-graph?period=month&date=2019-01-01&filters=#{filters}") + + conn = + get( + conn, + "/api/stats/#{site.domain}/main-graph?period=month&date=2019-01-01&filters=#{filters}" + ) res = json_response(conn, 200) assert %{"name" => "Converted visitors", "count" => 3, "change" => 100} in res["top_stats"] @@ -112,10 +123,18 @@ defmodule PlausibleWeb.Api.StatsController.MainGraphTest do test "returns conversion rate", %{conn: conn, site: site} do filters = Jason.encode!(%{goal: "Signup"}) - conn = get(conn, "/api/stats/#{site.domain}/main-graph?period=day&date=2019-01-01&filters=#{filters}") + + conn = + get( + conn, + "/api/stats/#{site.domain}/main-graph?period=day&date=2019-01-01&filters=#{filters}" + ) res = json_response(conn, 200) - assert %{"name" => "Conversion rate", "percentage" => 50.0, "change" => 100} in res["top_stats"] + + assert %{"name" => "Conversion rate", "percentage" => 50.0, "change" => 100} in res[ + "top_stats" + ] end end end diff --git a/test/plausible_web/controllers/api/stats_controller/operating_systems_test.exs b/test/plausible_web/controllers/api/stats_controller/operating_systems_test.exs index 3259891a4..b842cb469 100644 --- a/test/plausible_web/controllers/api/stats_controller/operating_systems_test.exs +++ b/test/plausible_web/controllers/api/stats_controller/operating_systems_test.exs @@ -9,9 +9,9 @@ defmodule PlausibleWeb.Api.StatsController.OperatingSystemsTest do conn = get(conn, "/api/stats/#{site.domain}/operating-systems?period=day&date=2019-01-01") assert json_response(conn, 200) == [ - %{"name" => "Mac", "count" => 2, "percentage" => 67}, - %{"name" => "Android", "count" => 1, "percentage" => 33}, - ] + %{"name" => "Mac", "count" => 2, "percentage" => 67}, + %{"name" => "Android", "count" => 1, "percentage" => 33} + ] end end end diff --git a/test/plausible_web/controllers/api/stats_controller/pages_test.exs b/test/plausible_web/controllers/api/stats_controller/pages_test.exs index 107327867..5cb918505 100644 --- a/test/plausible_web/controllers/api/stats_controller/pages_test.exs +++ b/test/plausible_web/controllers/api/stats_controller/pages_test.exs @@ -9,22 +9,26 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do conn = get(conn, "/api/stats/#{site.domain}/pages?period=day&date=2019-01-01") assert json_response(conn, 200) == [ - %{"count" => 2, "name" => "/"}, - %{"count" => 2, "name" => "/register"}, - %{"count" => 1, "name" => "/contact"}, - %{"count" => 1, "name" => "/irrelevant"} - ] + %{"count" => 2, "name" => "/"}, + %{"count" => 2, "name" => "/register"}, + %{"count" => 1, "name" => "/contact"}, + %{"count" => 1, "name" => "/irrelevant"} + ] end test "calculates bounce rate for pages", %{conn: conn, site: site} do - conn = get(conn, "/api/stats/#{site.domain}/pages?period=day&date=2019-01-01&include=bounce_rate") + conn = + get( + conn, + "/api/stats/#{site.domain}/pages?period=day&date=2019-01-01&include=bounce_rate" + ) assert json_response(conn, 200) == [ - %{"count" => 2, "name" => "/", "bounce_rate" => 50.0}, - %{"bounce_rate" => nil, "count" => 2, "name" => "/register"}, - %{"bounce_rate" => nil, "count" => 1, "name" => "/contact"}, - %{"bounce_rate" => nil, "count" => 1, "name" => "/irrelevant"} - ] + %{"count" => 2, "name" => "/", "bounce_rate" => 50.0}, + %{"bounce_rate" => nil, "count" => 2, "name" => "/register"}, + %{"bounce_rate" => nil, "count" => 1, "name" => "/contact"}, + %{"bounce_rate" => nil, "count" => 1, "name" => "/irrelevant"} + ] end end end diff --git a/test/plausible_web/controllers/api/stats_controller/referrers_test.exs b/test/plausible_web/controllers/api/stats_controller/referrers_test.exs index bdb2c8ebf..c17578f09 100644 --- a/test/plausible_web/controllers/api/stats_controller/referrers_test.exs +++ b/test/plausible_web/controllers/api/stats_controller/referrers_test.exs @@ -9,18 +9,27 @@ defmodule PlausibleWeb.Api.StatsController.ReferrersTest do conn = get(conn, "/api/stats/#{site.domain}/referrers?period=day&date=2019-01-01") assert json_response(conn, 200) == [ - %{"name" => "10words", "count" => 2, "url" => "10words.com"}, - %{"name" => "Bing", "count" => 1, "url" => ""}, - ] + %{"name" => "10words", "count" => 2, "url" => "10words.com"}, + %{"name" => "Bing", "count" => 1, "url" => ""} + ] end test "calculates bounce rate for referrers", %{conn: conn, site: site} do - conn = get(conn, "/api/stats/#{site.domain}/referrers?period=day&date=2019-01-01&include=bounce_rate") + conn = + get( + conn, + "/api/stats/#{site.domain}/referrers?period=day&date=2019-01-01&include=bounce_rate" + ) assert json_response(conn, 200) == [ - %{"name" => "10words", "count" => 2, "bounce_rate" => 50.0, "url" => "10words.com"}, - %{"name" => "Bing", "count" => 1, "bounce_rate" => nil, "url" => ""}, - ] + %{ + "name" => "10words", + "count" => 2, + "bounce_rate" => 50.0, + "url" => "10words.com" + }, + %{"name" => "Bing", "count" => 1, "bounce_rate" => nil, "url" => ""} + ] end end @@ -29,20 +38,30 @@ defmodule PlausibleWeb.Api.StatsController.ReferrersTest do test "returns top referrers for a custom goal", %{conn: conn, site: site} do filters = Jason.encode!(%{goal: "Signup"}) - conn = get(conn, "/api/stats/#{site.domain}/goal/referrers?period=day&date=2019-01-01&filters=#{filters}") + + conn = + get( + conn, + "/api/stats/#{site.domain}/goal/referrers?period=day&date=2019-01-01&filters=#{filters}" + ) assert json_response(conn, 200) == [ - %{"name" => "10words", "count" => 2, "url" => "10words.com"}, - ] + %{"name" => "10words", "count" => 2, "url" => "10words.com"} + ] end test "returns top referrers for a pageview goal", %{conn: conn, site: site} do filters = Jason.encode!(%{goal: "Visit /register"}) - conn = get(conn, "/api/stats/#{site.domain}/goal/referrers?period=day&date=2019-01-01&filters=#{filters}") + + conn = + get( + conn, + "/api/stats/#{site.domain}/goal/referrers?period=day&date=2019-01-01&filters=#{filters}" + ) assert json_response(conn, 200) == [ - %{"name" => "10words", "count" => 2, "url" => "10words.com"}, - ] + %{"name" => "10words", "count" => 2, "url" => "10words.com"} + ] end end @@ -53,35 +72,39 @@ defmodule PlausibleWeb.Api.StatsController.ReferrersTest do conn = get(conn, "/api/stats/#{site.domain}/referrers/10words?period=day&date=2019-01-01") assert json_response(conn, 200) == %{ - "total_visitors" => 2, - "referrers" => [ - %{"name" => "10words.com/page2", "count" => 1}, - %{"name" => "10words.com/page1", "count" => 1}, - ] - } + "total_visitors" => 2, + "referrers" => [ + %{"name" => "10words.com/page2", "count" => 1}, + %{"name" => "10words.com/page1", "count" => 1} + ] + } end test "calculates bounce rate for referrer urls", %{conn: conn, site: site} do - conn = get(conn, "/api/stats/#{site.domain}/referrers/10words?period=day&date=2019-01-01&include=bounce_rate") + conn = + get( + conn, + "/api/stats/#{site.domain}/referrers/10words?period=day&date=2019-01-01&include=bounce_rate" + ) assert json_response(conn, 200) == %{ - "total_visitors" => 2, - "referrers" => [ - %{"name" => "10words.com/page2", "count" => 1, "bounce_rate" => nil}, - %{"name" => "10words.com/page1", "count" => 1, "bounce_rate" => 50.0}, - ] - } + "total_visitors" => 2, + "referrers" => [ + %{"name" => "10words.com/page2", "count" => 1, "bounce_rate" => nil}, + %{"name" => "10words.com/page1", "count" => 1, "bounce_rate" => 50.0} + ] + } end test "gets keywords from Google", %{conn: conn, user: user, site: site} do - insert(:google_auth, user: user, user: user,site: site, property: "sc-domain:example.com") + insert(:google_auth, user: user, user: user, site: site, property: "sc-domain:example.com") conn = get(conn, "/api/stats/#{site.domain}/referrers/Google?period=day&date=2019-02-01") {:ok, terms} = Plausible.Google.Api.Mock.fetch_stats(nil, nil) assert json_response(conn, 200) == %{ - "total_visitors" => 2, - "search_terms" => terms - } + "total_visitors" => 2, + "search_terms" => terms + } end test "enriches twitter referrers with tweets if available", %{conn: conn, site: site} do @@ -92,7 +115,13 @@ defmodule PlausibleWeb.Api.StatsController.ReferrersTest do res = json_response(conn, 200) assert res["total_visitors"] == 3 assert [tweet1, tweet2] = res["referrers"] - assert %{"name" => "t.co/some-link", "count" => 2, "tweets" => [%{"text" => "important tweet"}]} = tweet1 + + assert %{ + "name" => "t.co/some-link", + "count" => 2, + "tweets" => [%{"text" => "important tweet"}] + } = tweet1 + assert %{"name" => "t.co/nonexistent-link", "count" => 1, "tweets" => nil} = tweet2 end end @@ -102,26 +131,40 @@ defmodule PlausibleWeb.Api.StatsController.ReferrersTest do test "returns top referring urls for a custom goal", %{conn: conn, site: site} do filters = Jason.encode!(%{goal: "Signup"}) - conn = get(conn, "/api/stats/#{site.domain}/goal/referrers/10words?period=day&date=2019-01-01&filters=#{filters}") + + conn = + get( + conn, + "/api/stats/#{site.domain}/goal/referrers/10words?period=day&date=2019-01-01&filters=#{ + filters + }" + ) assert json_response(conn, 200) == %{ - "total_visitors" => 2, - "referrers" => [ - %{"name" => "10words.com/page1", "count" => 2} - ] - } + "total_visitors" => 2, + "referrers" => [ + %{"name" => "10words.com/page1", "count" => 2} + ] + } end test "returns top referring urls for a pageview goal", %{conn: conn, site: site} do filters = Jason.encode!(%{goal: "Visit /register"}) - conn = get(conn, "/api/stats/#{site.domain}/goal/referrers/10words?period=day&date=2019-01-01&filters=#{filters}") + + conn = + get( + conn, + "/api/stats/#{site.domain}/goal/referrers/10words?period=day&date=2019-01-01&filters=#{ + filters + }" + ) assert json_response(conn, 200) == %{ - "total_visitors" => 2, - "referrers" => [ - %{"name" => "10words.com/page1", "count" => 2}, - ] - } + "total_visitors" => 2, + "referrers" => [ + %{"name" => "10words.com/page1", "count" => 2} + ] + } end end end diff --git a/test/plausible_web/controllers/api/stats_controller/screen_sizes_test.exs b/test/plausible_web/controllers/api/stats_controller/screen_sizes_test.exs index 24b979818..6058265a3 100644 --- a/test/plausible_web/controllers/api/stats_controller/screen_sizes_test.exs +++ b/test/plausible_web/controllers/api/stats_controller/screen_sizes_test.exs @@ -9,9 +9,9 @@ defmodule PlausibleWeb.Api.StatsController.ScreenSizesTest do conn = get(conn, "/api/stats/#{site.domain}/screen-sizes?period=day&date=2019-01-01") assert json_response(conn, 200) == [ - %{"name" => "Desktop", "count" => 2, "percentage" => 67}, - %{"name" => "Mobile", "count" => 1, "percentage" => 33}, - ] + %{"name" => "Desktop", "count" => 2, "percentage" => 67}, + %{"name" => "Mobile", "count" => 1, "percentage" => 33} + ] end end end diff --git a/test/plausible_web/controllers/auth_controller_test.exs b/test/plausible_web/controllers/auth_controller_test.exs index 9f880eb3e..de2d3f0fe 100644 --- a/test/plausible_web/controllers/auth_controller_test.exs +++ b/test/plausible_web/controllers/auth_controller_test.exs @@ -11,19 +11,24 @@ defmodule PlausibleWeb.AuthControllerTest do end test "registering sends an activation link", %{conn: conn} do - post(conn, "/register", user: %{ - name: "Jane Doe", - email: "user@example.com" - }) + post(conn, "/register", + user: %{ + name: "Jane Doe", + email: "user@example.com" + } + ) assert_email_delivered_with(subject: "Activate your Plausible free trial") end test "user sees success page after registering", %{conn: conn} do - conn = post(conn, "/register", user: %{ - name: "Jane Doe", - email: "user@example.com" - }) + conn = + post(conn, "/register", + user: %{ + name: "Jane Doe", + email: "user@example.com" + } + ) assert html_response(conn, 200) =~ "Success!" end @@ -79,7 +84,6 @@ defmodule PlausibleWeb.AuthControllerTest do end test "email does not exist - renders login form again", %{conn: conn} do - conn = post(conn, "/login", email: "user@example.com", password: "password") assert get_session(conn, :current_user_id) == nil @@ -159,7 +163,7 @@ defmodule PlausibleWeb.AuthControllerTest do setup [:create_user, :log_in] test "updates user record", %{conn: conn, user: user} do - put(conn, "/settings", %{"user" => %{"name" => "New name"}}) + put(conn, "/settings", %{"user" => %{"name" => "New name"}}) user = Plausible.Repo.get(Plausible.Auth.User, user.id) assert user.name == "New name" @@ -171,15 +175,19 @@ defmodule PlausibleWeb.AuthControllerTest do use Plausible.Repo test "deletes the user", %{conn: conn, user: user, site: site} do - Repo.insert_all("intro_emails", [%{ - user_id: user.id, - timestamp: NaiveDateTime.utc_now() - }]) - - Repo.insert_all("feedback_emails", [%{ - user_id: user.id, - timestamp: NaiveDateTime.utc_now() - }]) + Repo.insert_all("intro_emails", [ + %{ + user_id: user.id, + timestamp: NaiveDateTime.utc_now() + } + ]) + + Repo.insert_all("feedback_emails", [ + %{ + user_id: user.id, + timestamp: NaiveDateTime.utc_now() + } + ]) insert(:google_auth, site: site, user: user) insert(:subscription, user: user, status: "deleted") diff --git a/test/plausible_web/controllers/site_controller_test.exs b/test/plausible_web/controllers/site_controller_test.exs index a20d55a4e..c3366e381 100644 --- a/test/plausible_web/controllers/site_controller_test.exs +++ b/test/plausible_web/controllers/site_controller_test.exs @@ -16,35 +16,38 @@ defmodule PlausibleWeb.SiteControllerTest do setup [:create_user, :log_in] test "creates the site with valid params", %{conn: conn} do - conn = post(conn, "/sites", %{ - "site" => %{ - "domain" => "example.com", - "timezone" => "Europe/London" - } - }) + conn = + post(conn, "/sites", %{ + "site" => %{ + "domain" => "example.com", + "timezone" => "Europe/London" + } + }) assert redirected_to(conn) == "/example.com/snippet" assert Repo.exists?(Plausible.Site, domain: "example.com") end test "cleans up the url", %{conn: conn} do - conn = post(conn, "/sites", %{ - "site" => %{ - "domain" => "https://www.Example.com/", - "timezone" => "Europe/London" - } - }) + conn = + post(conn, "/sites", %{ + "site" => %{ + "domain" => "https://www.Example.com/", + "timezone" => "Europe/London" + } + }) assert redirected_to(conn) == "/example.com/snippet" assert Repo.exists?(Plausible.Site, domain: "example.com") end test "renders form again when domain is missing", %{conn: conn} do - conn = post(conn, "/sites", %{ - "site" => %{ - "timezone" => "Europe/London" - } - }) + conn = + post(conn, "/sites", %{ + "site" => %{ + "timezone" => "Europe/London" + } + }) assert html_response(conn, 200) =~ "can't be blank" end @@ -52,12 +55,13 @@ defmodule PlausibleWeb.SiteControllerTest do test "renders form again when it is a duplicate domain", %{conn: conn} do insert(:site, domain: "example.com") - conn = post(conn, "/sites", %{ - "site" => %{ - "domain" => "example.com", - "timezone" => "Europe/London" - } - }) + conn = + post(conn, "/sites", %{ + "site" => %{ + "domain" => "example.com", + "timezone" => "Europe/London" + } + }) assert html_response(conn, 200) =~ "has already been taken" end @@ -200,7 +204,11 @@ defmodule PlausibleWeb.SiteControllerTest do describe "POST /sites/:website/weekly-report/enable" do setup [:create_user, :log_in, :create_site] - test "creates a weekly report record with the user email", %{conn: conn, site: site, user: user} do + test "creates a weekly report record with the user email", %{ + conn: conn, + site: site, + user: user + } do post(conn, "/sites/#{site.domain}/weekly-report/enable") report = Repo.get_by(Plausible.Site.WeeklyReport, site_id: site.id) @@ -249,7 +257,11 @@ defmodule PlausibleWeb.SiteControllerTest do describe "POST /sites/:website/monthly-report/enable" do setup [:create_user, :log_in, :create_site] - test "creates a monthly report record with the user email", %{conn: conn, site: site, user: user} do + test "creates a monthly report record with the user email", %{ + conn: conn, + site: site, + user: user + } do post(conn, "/sites/#{site.domain}/monthly-report/enable") report = Repo.get_by(Plausible.Site.MonthlyReport, site_id: site.id) @@ -356,7 +368,10 @@ defmodule PlausibleWeb.SiteControllerTest do setup [:create_user, :log_in, :create_site] test "creates a custom domain", %{conn: conn, site: site} do - conn = post(conn, "/sites/#{site.domain}/custom-domains", %{"custom_domain" => %{"domain" => "plausible.example.com"}}) + conn = + post(conn, "/sites/#{site.domain}/custom-domains", %{ + "custom_domain" => %{"domain" => "plausible.example.com"} + }) domain = Repo.one(Plausible.Site.CustomDomain) @@ -365,14 +380,18 @@ defmodule PlausibleWeb.SiteControllerTest do end test "validates presence of domain name", %{conn: conn, site: site} do - conn = post(conn, "/sites/#{site.domain}/custom-domains", %{"custom_domain" => %{"domain" => ""}}) + conn = + post(conn, "/sites/#{site.domain}/custom-domains", %{"custom_domain" => %{"domain" => ""}}) refute Repo.one(Plausible.Site.CustomDomain) assert html_response(conn, 200) =~ "Setup custom domain" end test "validates format of domain name", %{conn: conn, site: site} do - conn = post(conn, "/sites/#{site.domain}/custom-domains", %{"custom_domain" => %{"domain" => "ASD?/not-domain"}}) + conn = + post(conn, "/sites/#{site.domain}/custom-domains", %{ + "custom_domain" => %{"domain" => "ASD?/not-domain"} + }) refute Repo.one(Plausible.Site.CustomDomain) assert html_response(conn, 200) =~ "Setup custom domain" diff --git a/test/plausible_web/controllers/stats_controller_test.exs b/test/plausible_web/controllers/stats_controller_test.exs index 3c232ddec..18d470014 100644 --- a/test/plausible_web/controllers/stats_controller_test.exs +++ b/test/plausible_web/controllers/stats_controller_test.exs @@ -46,13 +46,17 @@ defmodule PlausibleWeb.StatsControllerTest do describe "GET /share/:slug" do test "prompts a password for a password-protected link", %{conn: conn} do site = insert(:site) - link = insert(:shared_link, site: site, password_hash: Plausible.Auth.Password.hash("password")) + + link = + insert(:shared_link, site: site, password_hash: Plausible.Auth.Password.hash("password")) conn = get(conn, "/share/#{link.slug}") assert response(conn, 200) =~ "Enter password" end - test "logs anonymous user in straight away if the link is not password-protected", %{conn: conn} do + test "logs anonymous user in straight away if the link is not password-protected", %{ + conn: conn + } do site = insert(:site, domain: "test-site.com") link = insert(:shared_link, site: site) @@ -67,7 +71,9 @@ defmodule PlausibleWeb.StatsControllerTest do describe "POST /share/:slug/authenticate" do test "logs anonymous user in with correct password", %{conn: conn} do site = insert(:site, domain: "test-site.com") - link = insert(:shared_link, site: site, password_hash: Plausible.Auth.Password.hash("password")) + + link = + insert(:shared_link, site: site, password_hash: Plausible.Auth.Password.hash("password")) conn = post(conn, "/share/#{link.slug}/authenticate", %{password: "password"}) assert redirected_to(conn, 302) == "/#{site.domain}" @@ -78,7 +84,9 @@ defmodule PlausibleWeb.StatsControllerTest do test "shows form again with wrong password", %{conn: conn} do site = insert(:site, domain: "test-site.com") - link = insert(:shared_link, site: site, password_hash: Plausible.Auth.Password.hash("password")) + + link = + insert(:shared_link, site: site, password_hash: Plausible.Auth.Password.hash("password")) conn = post(conn, "/share/#{link.slug}/authenticate", %{password: "WRONG!"}) assert html_response(conn, 200) =~ "Enter password" diff --git a/test/plausible_web/controllers/unsubscribe_controller_test.exs b/test/plausible_web/controllers/unsubscribe_controller_test.exs index 828e462be..20eb77979 100644 --- a/test/plausible_web/controllers/unsubscribe_controller_test.exs +++ b/test/plausible_web/controllers/unsubscribe_controller_test.exs @@ -7,7 +7,8 @@ defmodule PlausibleWeb.UnsubscribeControllerTest do site = insert(:site) insert(:weekly_report, site: site, recipients: ["recipient@email.com"]) - conn = get(conn, "/sites/#{site.domain}/weekly-report/unsubscribe?email=recipient@email.com") + conn = + get(conn, "/sites/#{site.domain}/weekly-report/unsubscribe?email=recipient@email.com") assert html_response(conn, 200) =~ "Unsubscribe successful" @@ -21,7 +22,8 @@ defmodule PlausibleWeb.UnsubscribeControllerTest do site = insert(:site) insert(:monthly_report, site: site, recipients: ["recipient@email.com"]) - conn = get(conn, "/sites/#{site.domain}/monthly-report/unsubscribe?email=recipient@email.com") + conn = + get(conn, "/sites/#{site.domain}/monthly-report/unsubscribe?email=recipient@email.com") assert html_response(conn, 200) =~ "Unsubscribe successful" diff --git a/test/plausible_web/plugs/session_timeout_plug_test.exs b/test/plausible_web/plugs/session_timeout_plug_test.exs index a78e9c649..7859a9021 100644 --- a/test/plausible_web/plugs/session_timeout_plug_test.exs +++ b/test/plausible_web/plugs/session_timeout_plug_test.exs @@ -5,27 +5,30 @@ defmodule PlausibleWeb.SessionTimeoutPlugTest do @opts %{timeout_after_seconds: 10} test "does nothing if user is not logged in" do - conn = conn(:get, "/") - |> init_test_session(%{}) - |> SessionTimeoutPlug.call(@opts) + conn = + conn(:get, "/") + |> init_test_session(%{}) + |> SessionTimeoutPlug.call(@opts) refute get_session(conn, :session_timeout_at) end test "sets session timeout if user is logged in" do - conn = conn(:get, "/") - |> init_test_session(%{current_user_id: 1}) - |> SessionTimeoutPlug.call(@opts) + conn = + conn(:get, "/") + |> init_test_session(%{current_user_id: 1}) + |> SessionTimeoutPlug.call(@opts) timeout = get_session(conn, :session_timeout_at) - now = DateTime.utc_now() |> DateTime.to_unix + now = DateTime.utc_now() |> DateTime.to_unix() assert timeout > now end test "logs user out if timeout passed" do - conn = conn(:get, "/") - |> init_test_session(%{current_user_id: 1, session_timeout_at: 1}) - |> SessionTimeoutPlug.call(@opts) + conn = + conn(:get, "/") + |> init_test_session(%{current_user_id: 1, session_timeout_at: 1}) + |> SessionTimeoutPlug.call(@opts) assert conn.private[:plug_session_info] == :drop end diff --git a/test/support/clickhouse_setup.ex b/test/support/clickhouse_setup.ex index 2e0c156fe..c02bf0b42 100644 --- a/test/support/clickhouse_setup.ex +++ b/test/support/clickhouse_setup.ex @@ -7,6 +7,7 @@ defmodule Plausible.Test.ClickhouseSetup do def create_events() do drop = "DROP TABLE events" + create = """ CREATE TABLE events ( timestamp DateTime, @@ -30,12 +31,13 @@ defmodule Plausible.Test.ClickhouseSetup do SETTINGS index_granularity = 8192 """ - Clickhousex.query(:clickhouse, drop, [],log: {Plausible.Clickhouse, :log, []}) - Clickhousex.query(:clickhouse, create, [],log: {Plausible.Clickhouse, :log, []}) + Clickhousex.query(:clickhouse, drop, [], log: {Plausible.Clickhouse, :log, []}) + Clickhousex.query(:clickhouse, create, [], log: {Plausible.Clickhouse, :log, []}) end def create_sessions() do drop = "DROP TABLE sessions" + create = """ CREATE TABLE sessions ( session_id UInt64, @@ -63,8 +65,8 @@ defmodule Plausible.Test.ClickhouseSetup do SETTINGS index_granularity = 8192 """ - Clickhousex.query(:clickhouse, drop, [],log: {Plausible.Clickhouse, :log, []}) - Clickhousex.query(:clickhouse, create, [],log: {Plausible.Clickhouse, :log, []}) + Clickhousex.query(:clickhouse, drop, [], log: {Plausible.Clickhouse, :log, []}) + Clickhousex.query(:clickhouse, create, [], log: {Plausible.Clickhouse, :log, []}) end @conversion_1_session_id 123 @@ -72,40 +74,163 @@ defmodule Plausible.Test.ClickhouseSetup do def load_fixtures() do Plausible.TestUtils.create_events([ - %{name: "pageview", domain: "test-site.com", pathname: "/", country_code: "EE", browser: "Chrome", operating_system: "Mac", screen_size: "Desktop", referrer_source: "10words", referrer: "10words.com/page1", timestamp: ~N[2019-01-01 00:00:00]}, - %{name: "pageview", domain: "test-site.com", pathname: "/", country_code: "EE", browser: "Chrome", operating_system: "Mac", screen_size: "Desktop", referrer_source: "10words", referrer: "10words.com/page2", timestamp: ~N[2019-01-01 00:00:00]}, - %{name: "pageview", domain: "test-site.com", pathname: "/contact", country_code: "GB", browser: "Firefox", operating_system: "Android", screen_size: "Mobile", referrer_source: "Bing", timestamp: ~N[2019-01-01 00:00:00]}, - + %{ + name: "pageview", + domain: "test-site.com", + pathname: "/", + country_code: "EE", + browser: "Chrome", + operating_system: "Mac", + screen_size: "Desktop", + referrer_source: "10words", + referrer: "10words.com/page1", + timestamp: ~N[2019-01-01 00:00:00] + }, + %{ + name: "pageview", + domain: "test-site.com", + pathname: "/", + country_code: "EE", + browser: "Chrome", + operating_system: "Mac", + screen_size: "Desktop", + referrer_source: "10words", + referrer: "10words.com/page2", + timestamp: ~N[2019-01-01 00:00:00] + }, + %{ + name: "pageview", + domain: "test-site.com", + pathname: "/contact", + country_code: "GB", + browser: "Firefox", + operating_system: "Android", + screen_size: "Mobile", + referrer_source: "Bing", + timestamp: ~N[2019-01-01 00:00:00] + }, %{name: "pageview", domain: "test-site.com", timestamp: ~N[2019-01-31 00:00:00]}, - - %{name: "Signup", domain: "test-site.com", session_id: @conversion_1_session_id, timestamp: ~N[2019-01-01 01:00:00]}, - %{name: "Signup", domain: "test-site.com", session_id: @conversion_1_session_id, timestamp: ~N[2019-01-01 02:00:00]}, - %{name: "Signup", domain: "test-site.com", session_id: @conversion_2_session_id, timestamp: ~N[2019-01-01 02:00:00]}, - - %{name: "pageview", pathname: "/register", domain: "test-site.com", session_id: @conversion_1_session_id, timestamp: ~N[2019-01-01 23:00:00]}, - %{name: "pageview", pathname: "/register", domain: "test-site.com", session_id: @conversion_2_session_id, timestamp: ~N[2019-01-01 23:00:00]}, - %{name: "pageview", pathname: "/irrelevant", domain: "test-site.com", session_id: @conversion_1_session_id, timestamp: ~N[2019-01-01 23:00:00]}, - - %{name: "pageview", domain: "test-site.com", referrer_source: "Google", timestamp: ~N[2019-02-01 01:00:00]}, - %{name: "pageview", domain: "test-site.com", referrer_source: "Google", timestamp: ~N[2019-02-01 02:00:00]}, - - %{name: "pageview", domain: "test-site.com", referrer: "t.co/some-link", referrer_source: "Twitter", timestamp: ~N[2019-03-01 01:00:00]}, - %{name: "pageview", domain: "test-site.com", referrer: "t.co/some-link", referrer_source: "Twitter", timestamp: ~N[2019-03-01 01:00:00]}, - %{name: "pageview", domain: "test-site.com", referrer: "t.co/nonexistent-link", referrer_source: "Twitter", timestamp: ~N[2019-03-01 02:00:00]}, - + %{ + name: "Signup", + domain: "test-site.com", + session_id: @conversion_1_session_id, + timestamp: ~N[2019-01-01 01:00:00] + }, + %{ + name: "Signup", + domain: "test-site.com", + session_id: @conversion_1_session_id, + timestamp: ~N[2019-01-01 02:00:00] + }, + %{ + name: "Signup", + domain: "test-site.com", + session_id: @conversion_2_session_id, + timestamp: ~N[2019-01-01 02:00:00] + }, + %{ + name: "pageview", + pathname: "/register", + domain: "test-site.com", + session_id: @conversion_1_session_id, + timestamp: ~N[2019-01-01 23:00:00] + }, + %{ + name: "pageview", + pathname: "/register", + domain: "test-site.com", + session_id: @conversion_2_session_id, + timestamp: ~N[2019-01-01 23:00:00] + }, + %{ + name: "pageview", + pathname: "/irrelevant", + domain: "test-site.com", + session_id: @conversion_1_session_id, + timestamp: ~N[2019-01-01 23:00:00] + }, + %{ + name: "pageview", + domain: "test-site.com", + referrer_source: "Google", + timestamp: ~N[2019-02-01 01:00:00] + }, + %{ + name: "pageview", + domain: "test-site.com", + referrer_source: "Google", + timestamp: ~N[2019-02-01 02:00:00] + }, + %{ + name: "pageview", + domain: "test-site.com", + referrer: "t.co/some-link", + referrer_source: "Twitter", + timestamp: ~N[2019-03-01 01:00:00] + }, + %{ + name: "pageview", + domain: "test-site.com", + referrer: "t.co/some-link", + referrer_source: "Twitter", + timestamp: ~N[2019-03-01 01:00:00] + }, + %{ + name: "pageview", + domain: "test-site.com", + referrer: "t.co/nonexistent-link", + referrer_source: "Twitter", + timestamp: ~N[2019-03-01 02:00:00] + }, %{name: "pageview", domain: "test-site.com"}, - %{name: "pageview", domain: "test-site.com", timestamp: Timex.now() |> Timex.shift(minutes: -3)}, - %{name: "pageview", domain: "test-site.com", timestamp: Timex.now() |> Timex.shift(minutes: -6)}, - + %{ + name: "pageview", + domain: "test-site.com", + timestamp: Timex.now() |> Timex.shift(minutes: -3) + }, + %{ + name: "pageview", + domain: "test-site.com", + timestamp: Timex.now() |> Timex.shift(minutes: -6) + }, %{name: "pageview", domain: "tz-test.com", timestamp: ~N[2019-01-01 00:00:00]}, %{name: "pageview", domain: "public-site.io"}, - %{name: "pageview", domain: "fetch-tweets-test.com", referrer: "t.co/a-link", referrer_source: "Twitter"}, - %{name: "pageview", domain: "fetch-tweets-test.com", referrer: "t.co/b-link", referrer_source: "Twitter", timestamp: Timex.now() |> Timex.shift(days: -5)} + %{ + name: "pageview", + domain: "fetch-tweets-test.com", + referrer: "t.co/a-link", + referrer_source: "Twitter" + }, + %{ + name: "pageview", + domain: "fetch-tweets-test.com", + referrer: "t.co/b-link", + referrer_source: "Twitter", + timestamp: Timex.now() |> Timex.shift(days: -5) + } ]) Plausible.TestUtils.create_sessions([ - %{domain: "test-site.com", entry_page: "/", exit_page: "/", referrer_source: "10words", referrer: "10words.com/page1", session_id: @conversion_1_session_id, is_bounce: true, start: ~N[2019-01-01 02:00:00]}, - %{domain: "test-site.com", entry_page: "/", exit_page: "/", referrer_source: "10words", referrer: "10words.com/page1", session_id: @conversion_2_session_id, is_bounce: false, start: ~N[2019-01-01 02:00:00]} + %{ + domain: "test-site.com", + entry_page: "/", + exit_page: "/", + referrer_source: "10words", + referrer: "10words.com/page1", + session_id: @conversion_1_session_id, + is_bounce: true, + start: ~N[2019-01-01 02:00:00] + }, + %{ + domain: "test-site.com", + entry_page: "/", + exit_page: "/", + referrer_source: "10words", + referrer: "10words.com/page1", + session_id: @conversion_2_session_id, + is_bounce: false, + start: ~N[2019-01-01 02:00:00] + } ]) end end diff --git a/test/support/factory.ex b/test/support/factory.ex index 9e62fd310..c32b96596 100644 --- a/test/support/factory.ex +++ b/test/support/factory.ex @@ -1,6 +1,11 @@ defmodule Plausible.Factory do use ExMachina.Ecto, repo: Plausible.Repo - @hash_key Keyword.fetch!(Application.get_env(:plausible, PlausibleWeb.Endpoint), :secret_key_base) |> binary_part(0, 16) + + @hash_key Keyword.fetch!( + Application.get_env(:plausible, PlausibleWeb.Endpoint), + :secret_key_base + ) + |> binary_part(0, 16) def user_factory(attrs) do pw = Map.get(attrs, :password, "password") @@ -20,7 +25,7 @@ defmodule Plausible.Factory do %Plausible.Site{ domain: domain, - timezone: "UTC", + timezone: "UTC" } end diff --git a/test/support/google_api_mock.ex b/test/support/google_api_mock.ex index 5060242a1..2e4e922f8 100644 --- a/test/support/google_api_mock.ex +++ b/test/support/google_api_mock.ex @@ -1,8 +1,9 @@ defmodule Plausible.Google.Api.Mock do def fetch_stats(_auth, _query) do - {:ok, [ - %{"name" => "simple web analytics", "count" => 6}, - %{"name" => "open-source analytics", "count" => 2}, - ]} + {:ok, + [ + %{"name" => "simple web analytics", "count" => 6}, + %{"name" => "open-source analytics", "count" => 2} + ]} end end diff --git a/test/support/paddle_api_mock.ex b/test/support/paddle_api_mock.ex index 3789438fd..8201d0d85 100644 --- a/test/support/paddle_api_mock.ex +++ b/test/support/paddle_api_mock.ex @@ -1,22 +1,24 @@ defmodule Plausible.PaddleApi.Mock do def get_subscription(_) do - {:ok, %{ - "next_payment" => %{ - "date" => "2019-07-10", - "amount" => 6 - } - }} + {:ok, + %{ + "next_payment" => %{ + "date" => "2019-07-10", + "amount" => 6 + } + }} end def update_subscription(_, %{plan_id: new_plan_id}) do new_plan_id = String.to_integer(new_plan_id) - {:ok, %{ - "plan_id" => new_plan_id, - "next_payment" => %{ - "date" => "2019-07-10", - "amount" => 6 - } - }} + {:ok, + %{ + "plan_id" => new_plan_id, + "next_payment" => %{ + "date" => "2019-07-10", + "amount" => 6 + } + }} end end diff --git a/test/support/test_utils.ex b/test/support/test_utils.ex index a155258c5..25538ab14 100644 --- a/test/support/test_utils.ex +++ b/test/support/test_utils.ex @@ -12,26 +12,30 @@ defmodule Plausible.TestUtils do end def create_pageviews(pageviews) do - Enum.map(pageviews, fn pageview -> + Enum.map(pageviews, fn pageview -> Factory.build(:pageview, pageview) - end) |> Plausible.Clickhouse.insert_events + end) + |> Plausible.Clickhouse.insert_events() end def create_events(events) do - Enum.map(events, fn event -> + Enum.map(events, fn event -> Factory.build(:event, event) - end) |> Plausible.Clickhouse.insert_events + end) + |> Plausible.Clickhouse.insert_events() end def create_sessions(sessions) do - Enum.map(sessions, fn session -> + Enum.map(sessions, fn session -> Factory.build(:ch_session, session) - end) |> Plausible.Clickhouse.insert_sessions + end) + |> Plausible.Clickhouse.insert_sessions() end def log_in(%{user: user, conn: conn}) do - conn = init_session(conn) - |> Plug.Conn.put_session(:current_user_id, user.id) + conn = + init_session(conn) + |> Plug.Conn.put_session(:current_user_id, user.id) {:ok, conn: conn} end @@ -48,7 +52,7 @@ defmodule Plausible.TestUtils do ) conn - |> Plug.Session.call(opts) - |> Plug.Conn.fetch_session() + |> Plug.Session.call(opts) + |> Plug.Conn.fetch_session() end end diff --git a/test/workers/fetch_tweets_test.exs b/test/workers/fetch_tweets_test.exs index 496ec21a0..7d3c3fb38 100644 --- a/test/workers/fetch_tweets_test.exs +++ b/test/workers/fetch_tweets_test.exs @@ -4,14 +4,14 @@ defmodule Plausible.Workers.FetchTweetsTest do import Double test "fetches Twitter referrals from the last day" do - twitter_mock = stub(Plausible.Twitter.Api, :search, fn(_link) -> [] end) + twitter_mock = stub(Plausible.Twitter.Api, :search, fn _link -> [] end) FetchTweets.perform(nil, nil, twitter_mock) assert_receive({Plausible.Twitter.Api, :search, ["t.co/a-link"]}) end test "fetches Twitter referrals from 5-6 days ago" do - twitter_mock = stub(Plausible.Twitter.Api, :search, fn(_link) -> [] end) + twitter_mock = stub(Plausible.Twitter.Api, :search, fn _link -> [] end) FetchTweets.perform(nil, nil, twitter_mock) assert_receive({Plausible.Twitter.Api, :search, ["t.co/b-link"]}) @@ -33,13 +33,15 @@ defmodule Plausible.Workers.FetchTweetsTest do } } - twitter_mock = stub(Plausible.Twitter.Api, :search, fn - ("t.co/a-link") -> [tweet] - (_link) -> [] - end) + twitter_mock = + stub(Plausible.Twitter.Api, :search, fn + "t.co/a-link" -> [tweet] + _link -> [] + end) + FetchTweets.perform(nil, nil, twitter_mock) - [found_tweet] = Repo.all(from t in Plausible.Twitter.Tweet) + [found_tweet] = Repo.all(from(t in Plausible.Twitter.Tweet)) assert found_tweet.tweet_id == "the_tweet_id" assert found_tweet.text == "a Tweet body" assert found_tweet.author_handle == "twitter_author" @@ -54,13 +56,16 @@ defmodule Plausible.Workers.FetchTweetsTest do "full_text" => "asd https://t.co/somelink", "entities" => %{ "user_mentions" => [], - "urls" => [%{ - "display_url" => "plausible.io", - "indices" => [4, 17], - "url" => "https://t.co/somelink" - }] + "urls" => [ + %{ + "display_url" => "plausible.io", + "indices" => [4, 17], + "url" => "https://t.co/somelink" + } + ] } } + body = FetchTweets.html_body(tweet) assert body == "asd plausible.io" @@ -70,17 +75,19 @@ defmodule Plausible.Workers.FetchTweetsTest do tweet = %{ "full_text" => "asd @hello", "entities" => %{ - "user_mentions" => [%{ - "screen_name" => "hello", - "id_str" => "123123" - }], + "user_mentions" => [ + %{ + "screen_name" => "hello", + "id_str" => "123123" + } + ], "urls" => [] } } + body = FetchTweets.html_body(tweet) assert body == "asd @hello" end end - end diff --git a/test/workers/provision_ssl_certificates_test.exs b/test/workers/provision_ssl_certificates_test.exs index 5f4a99692..a3e0ce812 100644 --- a/test/workers/provision_ssl_certificates_test.exs +++ b/test/workers/provision_ssl_certificates_test.exs @@ -7,8 +7,10 @@ defmodule Plausible.Workers.SslCertificatesTest do site = insert(:site) insert(:custom_domain, site: site, domain: "custom-site.com") - ssh_stub = stub(SSHEx, :connect, fn(_cmd) -> {:ok, nil} end) - |> stub(:run, fn(_conn, _cmd) -> {:ok, "", 0} end) + ssh_stub = + stub(SSHEx, :connect, fn _cmd -> {:ok, nil} end) + |> stub(:run, fn _conn, _cmd -> {:ok, "", 0} end) + ProvisionSslCertificates.perform(nil, nil, ssh_stub) assert_receive({SSHEx, :run, [nil, 'sudo certbot certonly --nginx -n -d "custom-site.com"']}) @@ -18,8 +20,10 @@ defmodule Plausible.Workers.SslCertificatesTest do site = insert(:site) insert(:custom_domain, site: site, domain: "custom-site.com") - ssh_stub = stub(SSHEx, :connect, fn(_cmd) -> {:ok, nil} end) - |> stub(:run, fn(_conn, _cmd) -> {:ok, "", 0} end) + ssh_stub = + stub(SSHEx, :connect, fn _cmd -> {:ok, nil} end) + |> stub(:run, fn _conn, _cmd -> {:ok, "", 0} end) + ProvisionSslCertificates.perform(nil, nil, ssh_stub) domain = Repo.get_by(Plausible.Site.CustomDomain, site_id: site.id) @@ -30,8 +34,10 @@ defmodule Plausible.Workers.SslCertificatesTest do site = insert(:site) insert(:custom_domain, site: site, domain: "custom-site.com") - ssh_stub = stub(SSHEx, :connect, fn(_cmd) -> {:ok, nil} end) - |> stub(:run, fn(_conn, _cmd) -> {:ok, "", 1} end) + ssh_stub = + stub(SSHEx, :connect, fn _cmd -> {:ok, nil} end) + |> stub(:run, fn _conn, _cmd -> {:ok, "", 1} end) + ProvisionSslCertificates.perform(nil, nil, ssh_stub) domain = Repo.get_by(Plausible.Site.CustomDomain, site_id: site.id) diff --git a/test/workers/send_email_reports_test.exs b/test/workers/send_email_reports_test.exs index 40dfc6cf3..62bfb17e3 100644 --- a/test/workers/send_email_reports_test.exs +++ b/test/workers/send_email_reports_test.exs @@ -12,17 +12,22 @@ defmodule Plausible.Workers.EmailReportsTest do test "sends weekly report on Monday 9am local timezone" do site = insert(:site, timezone: "US/Eastern") insert(:weekly_report, site: site, recipients: ["user@email.com"]) - time = Timex.now() |> Timex.beginning_of_week |> Timex.shift(hours: 14) # 2pm UTC is 10am EST + # 2pm UTC is 10am EST + time = Timex.now() |> Timex.beginning_of_week() |> Timex.shift(hours: 14) perform(%{"current_time" => time}) - assert_email_delivered_with(subject: "Weekly report for #{site.domain}", to: [nil: "user@email.com"]) + assert_email_delivered_with( + subject: "Weekly report for #{site.domain}", + to: [nil: "user@email.com"] + ) end test "does not send a report on Monday before 9am in local timezone" do site = insert(:site, timezone: "US/Eastern") insert(:weekly_report, site: site, recipients: ["user@email.com"]) - time = Timex.now() |> Timex.beginning_of_week |> Timex.shift(hours: 12) # 12pm UTC is 8am EST + # 12pm UTC is 8am EST + time = Timex.now() |> Timex.beginning_of_week() |> Timex.shift(hours: 12) perform(%{"current_time" => time}) @@ -32,7 +37,7 @@ defmodule Plausible.Workers.EmailReportsTest do test "does not send a report on Tuesday" do site = insert(:site) insert(:weekly_report, site: site, recipients: ["user@email.com"]) - time = Timex.now() |> Timex.beginning_of_week |> Timex.shift(days: 1, hours: 10) + time = Timex.now() |> Timex.beginning_of_week() |> Timex.shift(days: 1, hours: 10) perform(%{"current_time" => time}) @@ -42,11 +47,14 @@ defmodule Plausible.Workers.EmailReportsTest do test "does not send the same report multiple times on the same week" do site = insert(:site) insert(:weekly_report, site: site, recipients: ["user@email.com"]) - time = Timex.now() |> Timex.beginning_of_week |> Timex.shift(hours: 10) + time = Timex.now() |> Timex.beginning_of_week() |> Timex.shift(hours: 10) perform(%{"current_time" => time}) - assert_email_delivered_with(subject: "Weekly report for #{site.domain}", to: [nil: "user@email.com"]) + assert_email_delivered_with( + subject: "Weekly report for #{site.domain}", + to: [nil: "user@email.com"] + ) perform(%{"current_time" => time}) assert_no_emails_delivered() @@ -61,13 +69,17 @@ defmodule Plausible.Workers.EmailReportsTest do perform(%{"current_time" => time}) - assert_email_delivered_with(subject: "March report for #{site.domain}", to: [nil: "user@email.com"]) + assert_email_delivered_with( + subject: "March report for #{site.domain}", + to: [nil: "user@email.com"] + ) end test "does not send a report on the 1st of the month before 9am in local timezone" do site = insert(:site, timezone: "US/Eastern") insert(:monthly_report, site: site, recipients: ["user@email.com"]) - time = Timex.now() |> Timex.beginning_of_month |> Timex.shift(hours: 12) # 12pm UTC is 8am EST + # 12pm UTC is 8am EST + time = Timex.now() |> Timex.beginning_of_month() |> Timex.shift(hours: 12) perform(%{"current_time" => time}) @@ -77,7 +89,7 @@ defmodule Plausible.Workers.EmailReportsTest do test "does not send a report on the 2nd of the month" do site = insert(:site) insert(:monthly_report, site: site, recipients: ["user@email.com"]) - time = Timex.now() |> Timex.beginning_of_month |> Timex.shift(days: 1, hours: 10) + time = Timex.now() |> Timex.beginning_of_month() |> Timex.shift(days: 1, hours: 10) perform(%{"current_time" => time}) @@ -91,7 +103,10 @@ defmodule Plausible.Workers.EmailReportsTest do perform(%{"current_time" => time}) - assert_email_delivered_with(subject: "January report for #{site.domain}", to: [nil: "user@email.com"]) + assert_email_delivered_with( + subject: "January report for #{site.domain}", + to: [nil: "user@email.com"] + ) perform(%{"current_time" => time}) assert_no_emails_delivered() diff --git a/test/workers/send_site_setup_emails_test.exs b/test/workers/send_site_setup_emails_test.exs index f66a62afd..cdbadd613 100644 --- a/test/workers/send_site_setup_emails_test.exs +++ b/test/workers/send_site_setup_emails_test.exs @@ -44,7 +44,7 @@ defmodule Plausible.Workers.SendSiteSetupEmailsTest do describe "when user has managed to set up their site" do test "sends the setup completed email as soon as possible" do user = insert(:user) - insert(:site, members: [user], domain: "test-site.com") + insert(:site, members: [user], domain: "test-site.com") perform() diff --git a/test/workers/send_trial_notifications_test.exs b/test/workers/send_trial_notifications_test.exs index 80b5dbe19..619bd27b9 100644 --- a/test/workers/send_trial_notifications_test.exs +++ b/test/workers/send_trial_notifications_test.exs @@ -8,10 +8,10 @@ defmodule Plausible.Workers.SendTrialNotificationsTest do end test "does not send a notification if user didn't set up their site" do - insert(:user, inserted_at: Timex.now |> Timex.shift(days: -14)) - insert(:user, inserted_at: Timex.now |> Timex.shift(days: -29)) - insert(:user, inserted_at: Timex.now |> Timex.shift(days: -30)) - insert(:user, inserted_at: Timex.now |> Timex.shift(days: -31)) + insert(:user, inserted_at: Timex.now() |> Timex.shift(days: -14)) + insert(:user, inserted_at: Timex.now() |> Timex.shift(days: -29)) + insert(:user, inserted_at: Timex.now() |> Timex.shift(days: -30)) + insert(:user, inserted_at: Timex.now() |> Timex.shift(days: -31)) perform() @@ -20,7 +20,7 @@ defmodule Plausible.Workers.SendTrialNotificationsTest do describe "with site and pageviews" do test "sends a reminder 7 days before trial ends (16 days after user signed up)" do - user = insert(:user, trial_expiry_date: Timex.now |> Timex.shift(days: 7)) + user = insert(:user, trial_expiry_date: Timex.now() |> Timex.shift(days: 7)) insert(:site, domain: "test-site.com", members: [user]) perform() @@ -29,7 +29,7 @@ defmodule Plausible.Workers.SendTrialNotificationsTest do end test "sends an upgrade email the day before the trial ends" do - user = insert(:user, trial_expiry_date: Timex.now |> Timex.shift(days: 1)) + user = insert(:user, trial_expiry_date: Timex.now() |> Timex.shift(days: 1)) insert(:site, domain: "test-site.com", members: [user]) perform() @@ -56,7 +56,7 @@ defmodule Plausible.Workers.SendTrialNotificationsTest do end test "does not send a notification if user has a subscription" do - user = insert(:user, trial_expiry_date: Timex.now |> Timex.shift(days: 7)) + user = insert(:user, trial_expiry_date: Timex.now() |> Timex.shift(days: 7)) insert(:site, domain: "test-site.com", members: [user]) insert(:subscription, user: user)