Refine parallel application-side joins (#2564)

* Consolidate task timeouts, incrase to 15s

* Use Task.async_stream for parallel Clickhouse queries

* Propagate Opentelemetry context to child Task process
pull/2569/head
Uku Taht 4 years ago committed by GitHub
parent f949278d39
commit 51fc9c0945
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23

@ -10,4 +10,18 @@ defmodule Plausible.ClickhouseRepo do
import Ecto.Query, only: [from: 1, from: 2]
end
end
@task_timeout 60_000
def parallel_tasks(queries) do
ctx = OpenTelemetry.Ctx.get_current()
execute_with_tracing = fn fun ->
OpenTelemetry.Ctx.attach(ctx)
fun.()
end
Task.async_stream(queries, execute_with_tracing, max_concurrency: 3, timeout: @task_timeout)
|> Enum.to_list()
|> Keyword.values()
end
end

@ -8,20 +8,19 @@ defmodule Plausible.Stats.Aggregate do
def aggregate(site, query, metrics) do
event_metrics = Enum.filter(metrics, &(&1 in @event_metrics))
event_task = Task.async(fn -> aggregate_events(site, query, event_metrics) end)
event_task = fn -> aggregate_events(site, query, event_metrics) end
session_metrics = Enum.filter(metrics, &(&1 in @session_metrics))
session_task = Task.async(fn -> aggregate_sessions(site, query, session_metrics) end)
session_task = fn -> aggregate_sessions(site, query, session_metrics) end
time_on_page_task =
if :time_on_page in metrics do
Task.async(fn -> aggregate_time_on_page(site, query) end)
fn -> aggregate_time_on_page(site, query) end
else
Task.async(fn -> %{} end)
fn -> %{} end
end
Task.await(session_task, 10_000)
|> Map.merge(Task.await(event_task, 10_000))
|> Map.merge(Task.await(time_on_page_task, 10_000))
Plausible.ClickhouseRepo.parallel_tasks([session_task, event_task, time_on_page_task])
|> Enum.reduce(%{}, fn aggregate, task_result -> Map.merge(aggregate, task_result) end)
|> Enum.map(fn {metric, value} ->
{metric, %{value: round(value || 0)}}
end)

@ -13,13 +13,10 @@ defmodule Plausible.Stats.Timeseries do
session_metrics = Enum.filter(metrics, &(&1 in @session_metrics))
[event_result, session_result] =
Task.await_many(
[
Task.async(fn -> events_timeseries(site, query, event_metrics) end),
Task.async(fn -> sessions_timeseries(site, query, session_metrics) end)
],
10_000
)
Plausible.ClickhouseRepo.parallel_tasks([
fn -> events_timeseries(site, query, event_metrics) end,
fn -> sessions_timeseries(site, query, session_metrics) end
])
Enum.map(steps, fn step ->
empty_row(step, metrics)

@ -23,13 +23,10 @@ defmodule PlausibleWeb.Api.ExternalStatsController do
prev_query = Query.shift_back(query, site)
[prev_result, curr_result] =
Task.await_many(
[
Task.async(fn -> Plausible.Stats.aggregate(site, prev_query, metrics) end),
Task.async(fn -> Plausible.Stats.aggregate(site, query, metrics) end)
],
10_000
)
Plausible.ClickhouseRepo.parallel_tasks([
fn -> Plausible.Stats.aggregate(site, prev_query, metrics) end,
fn -> Plausible.Stats.aggregate(site, query, metrics) end
])
Enum.map(curr_result, fn {metric, %{value: current_val}} ->
%{value: prev_val} = prev_result[metric]

@ -111,30 +111,33 @@ defmodule PlausibleWeb.StatsController do
params = Map.merge(params, %{"limit" => "300", "csv" => "True", "detailed" => "True"})
limited_params = Map.merge(params, %{"limit" => "100"})
csvs = [
{'sources.csv', fn -> Api.StatsController.sources(conn, params) end},
{'utm_mediums.csv', fn -> Api.StatsController.utm_mediums(conn, params) end},
{'utm_sources.csv', fn -> Api.StatsController.utm_sources(conn, params) end},
{'utm_campaigns.csv', fn -> Api.StatsController.utm_campaigns(conn, params) end},
{'utm_contents.csv', fn -> Api.StatsController.utm_contents(conn, params) end},
{'utm_terms.csv', fn -> Api.StatsController.utm_terms(conn, params) end},
{'pages.csv', fn -> Api.StatsController.pages(conn, limited_params) end},
{'entry_pages.csv', fn -> Api.StatsController.entry_pages(conn, params) end},
{'exit_pages.csv', fn -> Api.StatsController.exit_pages(conn, limited_params) end},
{'countries.csv', fn -> Api.StatsController.countries(conn, params) end},
{'regions.csv', fn -> Api.StatsController.regions(conn, params) end},
{'cities.csv', fn -> Api.StatsController.cities(conn, params) end},
{'browsers.csv', fn -> Api.StatsController.browsers(conn, params) end},
{'operating_systems.csv', fn -> Api.StatsController.operating_systems(conn, params) end},
{'devices.csv', fn -> Api.StatsController.screen_sizes(conn, params) end},
{'conversions.csv', fn -> Api.StatsController.conversions(conn, params) end},
{'prop_breakdown.csv', fn -> Api.StatsController.all_props_breakdown(conn, params) end}
]
csvs = %{
'sources.csv' => fn -> Api.StatsController.sources(conn, params) end,
'utm_mediums.csv' => fn -> Api.StatsController.utm_mediums(conn, params) end,
'utm_sources.csv' => fn -> Api.StatsController.utm_sources(conn, params) end,
'utm_campaigns.csv' => fn -> Api.StatsController.utm_campaigns(conn, params) end,
'utm_contents.csv' => fn -> Api.StatsController.utm_contents(conn, params) end,
'utm_terms.csv' => fn -> Api.StatsController.utm_terms(conn, params) end,
'pages.csv' => fn -> Api.StatsController.pages(conn, limited_params) end,
'entry_pages.csv' => fn -> Api.StatsController.entry_pages(conn, params) end,
'exit_pages.csv' => fn -> Api.StatsController.exit_pages(conn, limited_params) end,
'countries.csv' => fn -> Api.StatsController.countries(conn, params) end,
'regions.csv' => fn -> Api.StatsController.regions(conn, params) end,
'cities.csv' => fn -> Api.StatsController.cities(conn, params) end,
'browsers.csv' => fn -> Api.StatsController.browsers(conn, params) end,
'operating_systems.csv' => fn -> Api.StatsController.operating_systems(conn, params) end,
'devices.csv' => fn -> Api.StatsController.screen_sizes(conn, params) end,
'conversions.csv' => fn -> Api.StatsController.conversions(conn, params) end,
'prop_breakdown.csv' => fn -> Api.StatsController.all_props_breakdown(conn, params) end
}
csv_values =
Map.values(csvs)
|> Plausible.ClickhouseRepo.parallel_tasks()
csvs =
csvs
|> Enum.map(fn {file, task} -> {file, Task.async(task)} end)
|> Enum.map(fn {file, task} -> {file, Task.await(task)} end)
Map.keys(csvs)
|> Enum.zip(csv_values)
csvs = [{'visitors.csv', visitors} | csvs]

Loading…
Cancel
Save