parent
6eef32a8ff
commit
5a3072ca21
@ -0,0 +1,349 @@
|
||||
defmodule Plausible.Exports do
|
||||
@moduledoc """
|
||||
Contains functions to export data for events and sessions as Zip archives.
|
||||
"""
|
||||
|
||||
require Plausible
|
||||
import Ecto.Query
|
||||
|
||||
@doc """
|
||||
Builds Ecto queries to export data from `events_v2` and `sessions_v2`
|
||||
tables into the format of `imported_*` tables for a website.
|
||||
"""
|
||||
@spec export_queries(pos_integer, extname: String.t(), date_range: Date.Range.t()) ::
|
||||
%{String.t() => Ecto.Query.t()}
|
||||
def export_queries(site_id, opts \\ []) do
|
||||
extname = opts[:extname] || ".csv"
|
||||
date_range = opts[:date_range]
|
||||
|
||||
filename = fn table ->
|
||||
name =
|
||||
if date_range do
|
||||
first_date = Timex.format!(date_range.first, "{YYYY}{0M}{0D}")
|
||||
last_date = Timex.format!(date_range.last, "{YYYY}{0M}{0D}")
|
||||
"#{table}_#{first_date}_#{last_date}"
|
||||
else
|
||||
table
|
||||
end
|
||||
|
||||
name <> extname
|
||||
end
|
||||
|
||||
%{
|
||||
filename.("imported_visitors") => export_visitors_q(site_id),
|
||||
filename.("imported_sources") => export_sources_q(site_id),
|
||||
# NOTE: this query can result in `MEMORY_LIMIT_EXCEEDED` error
|
||||
filename.("imported_pages") => export_pages_q(site_id),
|
||||
filename.("imported_entry_pages") => export_entry_pages_q(site_id),
|
||||
filename.("imported_exit_pages") => export_exit_pages_q(site_id),
|
||||
filename.("imported_locations") => export_locations_q(site_id),
|
||||
filename.("imported_devices") => export_devices_q(site_id),
|
||||
filename.("imported_browsers") => export_browsers_q(site_id),
|
||||
filename.("imported_operating_systems") => export_operating_systems_q(site_id)
|
||||
}
|
||||
end
|
||||
|
||||
Plausible.on_full_build do
|
||||
defp sampled(table) do
|
||||
Plausible.Stats.Sampling.add_query_hint(from(table))
|
||||
end
|
||||
else
|
||||
defp sampled(table) do
|
||||
table
|
||||
end
|
||||
end
|
||||
|
||||
defmacrop date(timestamp) do
|
||||
quote do
|
||||
selected_as(fragment("toDate(?)", unquote(timestamp)), :date)
|
||||
end
|
||||
end
|
||||
|
||||
defmacrop visit_duration(t) do
|
||||
quote do
|
||||
selected_as(
|
||||
fragment("greatest(sum(?*?),0)", unquote(t).sign, unquote(t).duration),
|
||||
:visit_duration
|
||||
)
|
||||
end
|
||||
end
|
||||
|
||||
defmacrop visitors(t) do
|
||||
quote do
|
||||
selected_as(
|
||||
fragment("toUInt64(round(uniq(?)*any(_sample_factor)))", unquote(t).user_id),
|
||||
:visitors
|
||||
)
|
||||
end
|
||||
end
|
||||
|
||||
defmacrop visits(t) do
|
||||
quote do
|
||||
selected_as(sum(unquote(t).sign), :visits)
|
||||
end
|
||||
end
|
||||
|
||||
defmacrop bounces(t) do
|
||||
quote do
|
||||
selected_as(
|
||||
fragment("greatest(sum(?*?),0)", unquote(t).sign, unquote(t).is_bounce),
|
||||
:bounces
|
||||
)
|
||||
end
|
||||
end
|
||||
|
||||
@spec export_visitors_q(pos_integer) :: Ecto.Query.t()
|
||||
def export_visitors_q(site_id) do
|
||||
visitors_sessions_q =
|
||||
from s in sampled("sessions_v2"),
|
||||
where: s.site_id == ^site_id,
|
||||
group_by: selected_as(:date),
|
||||
select: %{
|
||||
date: date(s.start),
|
||||
bounces: bounces(s),
|
||||
visits: visits(s),
|
||||
visit_duration: visit_duration(s)
|
||||
# NOTE: can we use just sessions_v2 table in this query? sum(pageviews) and visitors(s)?
|
||||
# visitors: visitors(s)
|
||||
}
|
||||
|
||||
visitors_events_q =
|
||||
from e in sampled("events_v2"),
|
||||
where: e.site_id == ^site_id,
|
||||
group_by: selected_as(:date),
|
||||
select: %{
|
||||
date: date(e.timestamp),
|
||||
visitors: visitors(e),
|
||||
pageviews:
|
||||
selected_as(
|
||||
fragment("toUInt64(round(countIf(?='pageview')*any(_sample_factor)))", e.name),
|
||||
:pageviews
|
||||
)
|
||||
}
|
||||
|
||||
visitors_q =
|
||||
"e"
|
||||
|> with_cte("e", as: ^visitors_events_q)
|
||||
|> with_cte("s", as: ^visitors_sessions_q)
|
||||
|
||||
from e in visitors_q,
|
||||
full_join: s in "s",
|
||||
on: e.date == s.date,
|
||||
order_by: selected_as(:date),
|
||||
select: [
|
||||
selected_as(fragment("greatest(?,?)", s.date, e.date), :date),
|
||||
e.visitors,
|
||||
e.pageviews,
|
||||
s.bounces,
|
||||
s.visits,
|
||||
s.visit_duration
|
||||
]
|
||||
end
|
||||
|
||||
@spec export_sources_q(pos_integer) :: Ecto.Query.t()
|
||||
def export_sources_q(site_id) do
|
||||
from s in sampled("sessions_v2"),
|
||||
where: s.site_id == ^site_id,
|
||||
group_by: [
|
||||
selected_as(:date),
|
||||
selected_as(:source),
|
||||
s.utm_medium,
|
||||
s.utm_campaign,
|
||||
s.utm_content,
|
||||
s.utm_term
|
||||
],
|
||||
order_by: selected_as(:date),
|
||||
select: [
|
||||
date(s.start),
|
||||
selected_as(s.referrer_source, :source),
|
||||
s.utm_medium,
|
||||
s.utm_campaign,
|
||||
s.utm_content,
|
||||
s.utm_term,
|
||||
visitors(s),
|
||||
visits(s),
|
||||
visit_duration(s),
|
||||
bounces(s)
|
||||
]
|
||||
end
|
||||
|
||||
@spec export_pages_q(pos_integer) :: Ecto.Query.t()
|
||||
def export_pages_q(site_id) do
|
||||
window_q =
|
||||
from e in sampled("events_v2"),
|
||||
where: e.site_id == ^site_id,
|
||||
select: %{
|
||||
timestamp: e.timestamp,
|
||||
next_timestamp:
|
||||
over(fragment("leadInFrame(?)", e.timestamp),
|
||||
partition_by: e.session_id,
|
||||
order_by: e.timestamp,
|
||||
frame: fragment("ROWS BETWEEN CURRENT ROW AND 1 FOLLOWING")
|
||||
),
|
||||
pathname: e.pathname,
|
||||
hostname: e.hostname,
|
||||
name: e.name,
|
||||
user_id: e.user_id,
|
||||
_sample_factor: fragment("_sample_factor")
|
||||
}
|
||||
|
||||
from e in subquery(window_q),
|
||||
group_by: [selected_as(:date), e.pathname],
|
||||
order_by: selected_as(:date),
|
||||
select: [
|
||||
date(e.timestamp),
|
||||
selected_as(fragment("any(?)", e.hostname), :hostname),
|
||||
selected_as(e.pathname, :page),
|
||||
visitors(e),
|
||||
selected_as(
|
||||
fragment("toUInt64(round(countIf(?='pageview')*any(_sample_factor)))", e.name),
|
||||
:pageviews
|
||||
),
|
||||
# NOTE: are exits pageviews or any events?
|
||||
selected_as(
|
||||
fragment("toUInt64(round(countIf(?=0)*any(_sample_factor)))", e.next_timestamp),
|
||||
:exits
|
||||
),
|
||||
selected_as(
|
||||
fragment("sum(greatest(?,0))", e.next_timestamp - e.timestamp),
|
||||
:time_on_page
|
||||
)
|
||||
]
|
||||
end
|
||||
|
||||
@spec export_entry_pages_q(pos_integer) :: Ecto.Query.t()
|
||||
def export_entry_pages_q(site_id) do
|
||||
from s in sampled("sessions_v2"),
|
||||
where: s.site_id == ^site_id,
|
||||
group_by: [selected_as(:date), s.entry_page],
|
||||
order_by: selected_as(:date),
|
||||
select: [
|
||||
date(s.start),
|
||||
s.entry_page,
|
||||
visitors(s),
|
||||
selected_as(
|
||||
fragment("toUInt64(round(sum(?)*any(_sample_factor)))", s.sign),
|
||||
:entrances
|
||||
),
|
||||
visit_duration(s),
|
||||
bounces(s)
|
||||
]
|
||||
end
|
||||
|
||||
@spec export_exit_pages_q(pos_integer) :: Ecto.Query.t()
|
||||
def export_exit_pages_q(site_id) do
|
||||
from s in sampled("sessions_v2"),
|
||||
where: s.site_id == ^site_id,
|
||||
group_by: [selected_as(:date), s.exit_page],
|
||||
order_by: selected_as(:date),
|
||||
select: [
|
||||
date(s.start),
|
||||
s.exit_page,
|
||||
visitors(s),
|
||||
selected_as(
|
||||
fragment("toUInt64(round(sum(?)*any(_sample_factor)))", s.sign),
|
||||
:exits
|
||||
)
|
||||
]
|
||||
end
|
||||
|
||||
@spec export_locations_q(pos_integer) :: Ecto.Query.t()
|
||||
def export_locations_q(site_id) do
|
||||
from s in sampled("sessions_v2"),
|
||||
where: s.site_id == ^site_id,
|
||||
where: s.city_geoname_id != 0 and s.country_code != "\0\0" and s.country_code != "ZZ",
|
||||
group_by: [selected_as(:date), s.country_code, selected_as(:region), s.city_geoname_id],
|
||||
order_by: selected_as(:date),
|
||||
select: [
|
||||
date(s.start),
|
||||
selected_as(s.country_code, :country),
|
||||
selected_as(s.subdivision1_code, :region),
|
||||
selected_as(s.city_geoname_id, :city),
|
||||
visitors(s),
|
||||
visits(s),
|
||||
visit_duration(s),
|
||||
bounces(s)
|
||||
]
|
||||
end
|
||||
|
||||
@spec export_devices_q(pos_integer) :: Ecto.Query.t()
|
||||
def export_devices_q(site_id) do
|
||||
from s in sampled("sessions_v2"),
|
||||
where: s.site_id == ^site_id,
|
||||
group_by: [selected_as(:date), s.screen_size],
|
||||
order_by: selected_as(:date),
|
||||
select: [
|
||||
date(s.start),
|
||||
selected_as(s.screen_size, :device),
|
||||
visitors(s),
|
||||
visits(s),
|
||||
visit_duration(s),
|
||||
bounces(s)
|
||||
]
|
||||
end
|
||||
|
||||
@spec export_browsers_q(pos_integer) :: Ecto.Query.t()
|
||||
def export_browsers_q(site_id) do
|
||||
from s in sampled("sessions_v2"),
|
||||
where: s.site_id == ^site_id,
|
||||
group_by: [selected_as(:date), s.browser],
|
||||
order_by: selected_as(:date),
|
||||
select: [
|
||||
date(s.start),
|
||||
s.browser,
|
||||
visitors(s),
|
||||
visits(s),
|
||||
visit_duration(s),
|
||||
bounces(s)
|
||||
]
|
||||
end
|
||||
|
||||
@spec export_operating_systems_q(pos_integer) :: Ecto.Query.t()
|
||||
def export_operating_systems_q(site_id) do
|
||||
from s in sampled("sessions_v2"),
|
||||
where: s.site_id == ^site_id,
|
||||
group_by: [selected_as(:date), s.operating_system],
|
||||
order_by: selected_as(:date),
|
||||
select: [
|
||||
date(s.start),
|
||||
s.operating_system,
|
||||
visitors(s),
|
||||
visits(s),
|
||||
visit_duration(s),
|
||||
bounces(s)
|
||||
]
|
||||
end
|
||||
|
||||
@doc """
|
||||
Creates a streamable Zip archive from the provided (named) Ecto queries.
|
||||
|
||||
Example usage:
|
||||
|
||||
{:ok, pool} = Ch.start_link(pool_size: 1)
|
||||
|
||||
DBConnection.run(pool, fn conn ->
|
||||
conn
|
||||
|> stream_archive(export_queries(_site_id = 1), format: "CSVWithNames")
|
||||
|> Stream.into(File.stream!("export.zip"))
|
||||
|> Stream.run()
|
||||
end)
|
||||
|
||||
"""
|
||||
@spec stream_archive(DBConnection.t(), %{String.t() => Ecto.Query.t()}, [Ch.query_option()]) ::
|
||||
Enumerable.t()
|
||||
def stream_archive(conn, named_queries, opts \\ []) do
|
||||
entries =
|
||||
Enum.map(named_queries, fn {name, query} ->
|
||||
{sql, params} = Plausible.ClickhouseRepo.to_sql(:all, query)
|
||||
|
||||
datastream =
|
||||
conn
|
||||
|> Ch.stream(sql, params, opts)
|
||||
|> Stream.map(fn %Ch.Result{data: data} -> data end)
|
||||
|
||||
Zstream.entry(name, datastream, coder: Zstream.Coder.Stored)
|
||||
end)
|
||||
|
||||
Zstream.zip(entries)
|
||||
end
|
||||
end
|
||||
@ -0,0 +1,83 @@
|
||||
defmodule Plausible.Workers.ExportCSV do
|
||||
@moduledoc """
|
||||
Worker for running CSV export jobs.
|
||||
"""
|
||||
|
||||
use Oban.Worker,
|
||||
queue: :s3_csv_export,
|
||||
max_attempts: 3,
|
||||
unique: [fields: [:args], keys: [:s3_bucket, :s3_path], period: 60]
|
||||
|
||||
@impl true
|
||||
def perform(job) do
|
||||
%Oban.Job{
|
||||
args:
|
||||
%{
|
||||
"site_id" => site_id,
|
||||
"email_to" => email,
|
||||
"s3_bucket" => s3_bucket,
|
||||
"s3_path" => s3_path
|
||||
} = args
|
||||
} = job
|
||||
|
||||
{:ok, ch} =
|
||||
Plausible.ClickhouseRepo.config()
|
||||
|> Keyword.replace!(:pool_size, 1)
|
||||
|> Ch.start_link()
|
||||
|
||||
# NOTE: should we use site.timezone?
|
||||
# %Ch.Result{rows: [[min_date, max_date]]} =
|
||||
# Ch.query!(
|
||||
# ch,
|
||||
# "SELECT toDate(min(timestamp)), toDate(max(timestamp)) FROM events_v2 WHERE site_id={site_id:UInt64}",
|
||||
# %{"site_id" => site_id}
|
||||
# )
|
||||
|
||||
download_url =
|
||||
DBConnection.run(
|
||||
ch,
|
||||
fn conn ->
|
||||
conn
|
||||
|> Plausible.Exports.stream_archive(
|
||||
# date_range: Date.range(min_date, max_date)
|
||||
Plausible.Exports.export_queries(site_id, extname: ".csv"),
|
||||
format: "CSVWithNames"
|
||||
)
|
||||
|> Plausible.S3.export_upload_multipart(s3_bucket, s3_path, s3_config_overrides(args))
|
||||
end,
|
||||
timeout: :infinity
|
||||
)
|
||||
|
||||
# NOTE: replace with proper Plausible.Email template
|
||||
Plausible.Mailer.deliver_now!(
|
||||
Bamboo.Email.new_email(
|
||||
from: "plausible@email.com",
|
||||
to: email,
|
||||
subject: "EXPORT SUCCESS",
|
||||
text_body: """
|
||||
download it from #{download_url}! hurry up! you have 24 hours!"
|
||||
""",
|
||||
html_body: """
|
||||
download it from <a href="#{download_url}">here</a>! hurry up! you have 24 hours!
|
||||
"""
|
||||
)
|
||||
)
|
||||
|
||||
:ok
|
||||
end
|
||||
|
||||
# right now custom config is used in tests only (to access the minio container)
|
||||
# ideally it would be passed via the s3 url
|
||||
# but ExAws.S3.upload is hard to make work with s3 urls
|
||||
if Mix.env() in [:test, :small_test] do
|
||||
defp s3_config_overrides(args) do
|
||||
if config_overrides = args["s3_config_overrides"] do
|
||||
Enum.map(config_overrides, fn {k, v} -> {String.to_existing_atom(k), v} end)
|
||||
else
|
||||
[]
|
||||
end
|
||||
end
|
||||
else
|
||||
defp s3_config_overrides(_args), do: []
|
||||
end
|
||||
end
|
||||
@ -0,0 +1,141 @@
|
||||
defmodule Plausible.ExportsTest do
|
||||
use Plausible.DataCase, async: true
|
||||
|
||||
# for e2e export->import tests please see Plausible.Imported.CSVImporterTest
|
||||
|
||||
describe "export_queries/2" do
|
||||
test "returns named ecto queries" do
|
||||
queries = Plausible.Exports.export_queries(_site_id = 1)
|
||||
assert queries |> Map.values() |> Enum.all?(&match?(%Ecto.Query{}, &1))
|
||||
|
||||
assert Map.keys(queries) == [
|
||||
"imported_browsers.csv",
|
||||
"imported_devices.csv",
|
||||
"imported_entry_pages.csv",
|
||||
"imported_exit_pages.csv",
|
||||
"imported_locations.csv",
|
||||
"imported_operating_systems.csv",
|
||||
"imported_pages.csv",
|
||||
"imported_sources.csv",
|
||||
"imported_visitors.csv"
|
||||
]
|
||||
end
|
||||
|
||||
test "with date range" do
|
||||
queries =
|
||||
Plausible.Exports.export_queries(_site_id = 1,
|
||||
date_range: Date.range(~D[2023-01-01], ~D[2024-03-12])
|
||||
)
|
||||
|
||||
assert Map.keys(queries) == [
|
||||
"imported_browsers_20230101_20240312.csv",
|
||||
"imported_devices_20230101_20240312.csv",
|
||||
"imported_entry_pages_20230101_20240312.csv",
|
||||
"imported_exit_pages_20230101_20240312.csv",
|
||||
"imported_locations_20230101_20240312.csv",
|
||||
"imported_operating_systems_20230101_20240312.csv",
|
||||
"imported_pages_20230101_20240312.csv",
|
||||
"imported_sources_20230101_20240312.csv",
|
||||
"imported_visitors_20230101_20240312.csv"
|
||||
]
|
||||
end
|
||||
|
||||
test "with custom extension" do
|
||||
queries =
|
||||
Plausible.Exports.export_queries(_site_id = 1,
|
||||
extname: ".ch"
|
||||
)
|
||||
|
||||
assert Map.keys(queries) == [
|
||||
"imported_browsers.ch",
|
||||
"imported_devices.ch",
|
||||
"imported_entry_pages.ch",
|
||||
"imported_exit_pages.ch",
|
||||
"imported_locations.ch",
|
||||
"imported_operating_systems.ch",
|
||||
"imported_pages.ch",
|
||||
"imported_sources.ch",
|
||||
"imported_visitors.ch"
|
||||
]
|
||||
end
|
||||
end
|
||||
|
||||
describe "stream_archive/3" do
|
||||
@describetag :tmp_dir
|
||||
|
||||
setup do
|
||||
config = Keyword.replace!(Plausible.ClickhouseRepo.config(), :pool_size, 1)
|
||||
{:ok, ch: start_supervised!({Ch, config})}
|
||||
end
|
||||
|
||||
test "creates zip archive", %{ch: ch, tmp_dir: tmp_dir} do
|
||||
queries = %{
|
||||
"1.csv" => from(n in "numbers", select: n.number, limit: 3),
|
||||
"2.csv" =>
|
||||
from(n in "numbers",
|
||||
select: [n.number, selected_as(n.number + n.number, :double)],
|
||||
limit: 3
|
||||
)
|
||||
}
|
||||
|
||||
DBConnection.run(ch, fn conn ->
|
||||
conn
|
||||
|> Plausible.Exports.stream_archive(queries, database: "system", format: "CSVWithNames")
|
||||
|> Stream.into(File.stream!(Path.join(tmp_dir, "numbers.zip")))
|
||||
|> Stream.run()
|
||||
end)
|
||||
|
||||
assert {:ok, files} =
|
||||
:zip.unzip(to_charlist(Path.join(tmp_dir, "numbers.zip")), cwd: tmp_dir)
|
||||
|
||||
assert Enum.map(files, &Path.basename/1) == ["1.csv", "2.csv"]
|
||||
|
||||
read_csv = fn file ->
|
||||
Enum.find(files, &(Path.basename(&1) == file))
|
||||
|> File.read!()
|
||||
|> NimbleCSV.RFC4180.parse_string(skip_headers: false)
|
||||
end
|
||||
|
||||
assert read_csv.("1.csv") ==
|
||||
NimbleCSV.RFC4180.parse_string(
|
||||
"""
|
||||
number
|
||||
0
|
||||
1
|
||||
2
|
||||
""",
|
||||
skip_headers: false
|
||||
)
|
||||
|
||||
assert read_csv.("2.csv") ==
|
||||
NimbleCSV.RFC4180.parse_string(
|
||||
"""
|
||||
number,double
|
||||
0,0
|
||||
1,2
|
||||
2,4
|
||||
""",
|
||||
skip_headers: false
|
||||
)
|
||||
end
|
||||
|
||||
test "stops on error", %{ch: ch, tmp_dir: tmp_dir} do
|
||||
queries = %{
|
||||
"1.csv" => from(n in "numbers", select: n.number, limit: 1000),
|
||||
"2.csv" => from(n in "no_such_table", select: n.number)
|
||||
}
|
||||
|
||||
assert_raise Ch.Error, ~r/UNKNOWN_TABLE/, fn ->
|
||||
DBConnection.run(ch, fn conn ->
|
||||
conn
|
||||
|> Plausible.Exports.stream_archive(queries, database: "system", format: "CSVWithNames")
|
||||
|> Stream.into(File.stream!(Path.join(tmp_dir, "failed.zip")))
|
||||
|> Stream.run()
|
||||
end)
|
||||
end
|
||||
|
||||
assert {:error, :einval} =
|
||||
:zip.unzip(to_charlist(Path.join(tmp_dir, "failed.zip")), cwd: tmp_dir)
|
||||
end
|
||||
end
|
||||
end
|
||||
Loading…
Reference in New Issue