CSV exports (UI) (#3875)

* ui

* fix redirect link

* improve make minio

* use implicit button form for csv export

* add exports_bucket helper

* read S3_EXPORTS_BUCKET

* supply s3_bucket in export_csv job args

* make plausible_minio use unprotected port

* move s3_csv_export queue to base queues

* Update lib/plausible_web/controllers/site_controller.ex

Co-authored-by: Adrian Gruntkowski <adrian.gruntkowski@gmail.com>

---------

Co-authored-by: Adrian Gruntkowski <adrian.gruntkowski@gmail.com>
pull/3904/head r2024.03.18-396
ruslandoga 3 years ago committed by GitHub
parent d3586a81e3
commit 5e74b1cf74
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194

@ -38,7 +38,10 @@ postgres-stop: ## Stop and remove the postgres container
docker stop plausible_db && docker rm plausible_db docker stop plausible_db && docker rm plausible_db
minio: ## Start a transient container with a recent version of minio (s3) minio: ## Start a transient container with a recent version of minio (s3)
docker run -d --rm -p 6000:6000 -p 6001:6001 --name plausible_minio minio/minio server /data --address ":6000" --console-address ":6001" docker run -d --rm -p 10000:10000 -p 10001:10001 --name plausible_minio minio/minio server /data --address ":10000" --console-address ":10001"
while ! docker exec plausible_minio mc alias set local http://localhost:10000 minioadmin minioadmin; do sleep 1; done
docker exec plausible_minio mc mb local/dev-exports
docker exec plausible_minio mc mb local/dev-imports
minio-stop: ## Stop and remove the minio container minio-stop: ## Stop and remove the minio container
docker stop plausible_minio docker stop plausible_minio

@ -25,4 +25,5 @@ S3_DISABLED=false
S3_ACCESS_KEY_ID=minioadmin S3_ACCESS_KEY_ID=minioadmin
S3_SECRET_ACCESS_KEY=minioadmin S3_SECRET_ACCESS_KEY=minioadmin
S3_REGION=us-east-1 S3_REGION=us-east-1
S3_ENDPOINT=http://localhost:6000 S3_ENDPOINT=http://localhost:10000
S3_EXPORTS_BUCKET=dev-exports

@ -20,4 +20,5 @@ S3_DISABLED=false
S3_ACCESS_KEY_ID=minioadmin S3_ACCESS_KEY_ID=minioadmin
S3_SECRET_ACCESS_KEY=minioadmin S3_SECRET_ACCESS_KEY=minioadmin
S3_REGION=us-east-1 S3_REGION=us-east-1
S3_ENDPOINT=http://localhost:6000 S3_ENDPOINT=http://localhost:10000
S3_EXPORTS_BUCKET=test-exports

@ -547,15 +547,16 @@ base_queues = [
clean_invitations: 1, clean_invitations: 1,
analytics_imports: 1, analytics_imports: 1,
domain_change_transition: 1, domain_change_transition: 1,
check_accept_traffic_until: 1 check_accept_traffic_until: 1,
# NOTE: maybe move s3_csv_export to cloud_queues?
s3_csv_export: 1
] ]
cloud_queues = [ cloud_queues = [
trial_notification_emails: 1, trial_notification_emails: 1,
check_usage: 1, check_usage: 1,
notify_annual_renewal: 1, notify_annual_renewal: 1,
lock_sites: 1, lock_sites: 1
s3_csv_export: 1
] ]
queues = if(is_selfhost, do: base_queues, else: base_queues ++ cloud_queues) queues = if(is_selfhost, do: base_queues, else: base_queues ++ cloud_queues)
@ -740,6 +741,10 @@ unless s3_disabled? do
%{ %{
name: "S3_ENDPOINT", name: "S3_ENDPOINT",
example: "https://<ACCOUNT_ID>.r2.cloudflarestorage.com" example: "https://<ACCOUNT_ID>.r2.cloudflarestorage.com"
},
%{
name: "S3_EXPORTS_BUCKET",
example: "my-csv-exports-bucket"
} }
] ]
@ -774,4 +779,6 @@ unless s3_disabled? do
scheme: s3_scheme <> "://", scheme: s3_scheme <> "://",
host: s3_host, host: s3_host,
port: s3_port port: s3_port
config :plausible, Plausible.S3, exports_bucket: s3_env_value.("S3_EXPORTS_BUCKET")
end end

@ -3,6 +3,24 @@ defmodule Plausible.S3 do
Helper functions for S3 exports/imports. Helper functions for S3 exports/imports.
""" """
@doc """
Returns the pre-configured S3 bucket for CSV exports.
config :plausible, Plausible.S3,
exports_bucket: System.fetch_env!("S3_EXPORTS_BUCKET")
Example:
iex> exports_bucket()
"test-exports"
"""
@spec exports_bucket :: String.t()
def exports_bucket, do: config(:exports_bucket)
defp config, do: Application.fetch_env!(:plausible, __MODULE__)
defp config(key), do: Keyword.fetch!(config(), key)
@doc """ @doc """
Chunks and uploads Zip archive to the provided S3 destination. Chunks and uploads Zip archive to the provided S3 destination.

@ -708,6 +708,23 @@ defmodule PlausibleWeb.SiteController do
|> redirect(external: Routes.site_path(conn, :settings_integrations, site.domain)) |> redirect(external: Routes.site_path(conn, :settings_integrations, site.domain))
end end
def export(conn, _params) do
%{site: site, current_user: user} = conn.assigns
Oban.insert!(
Plausible.Workers.ExportCSV.new(%{
"site_id" => site.id,
"email_to" => user.email,
"s3_bucket" => Plausible.S3.exports_bucket(),
"s3_path" => "Plausible-#{site.id}.zip"
})
)
conn
|> put_flash(:success, "SCHEDULED. WAIT FOR MAIL")
|> redirect(to: Routes.site_path(conn, :settings_imports_exports, site.domain))
end
def change_domain(conn, _params) do def change_domain(conn, _params) do
changeset = Plausible.Site.update_changeset(conn.assigns.site) changeset = Plausible.Site.update_changeset(conn.assigns.site)

@ -397,6 +397,7 @@ defmodule PlausibleWeb.Router do
delete "/:website/settings/forget-imported", SiteController, :forget_imported delete "/:website/settings/forget-imported", SiteController, :forget_imported
delete "/:website/settings/forget-import/:import_id", SiteController, :forget_import delete "/:website/settings/forget-import/:import_id", SiteController, :forget_import
post "/:website/settings/export", SiteController, :export
get "/:domain/export", StatsController, :csv_export get "/:domain/export", StatsController, :csv_export
get "/:domain/*path", StatsController, :stats get "/:domain/*path", StatsController, :stats

@ -56,8 +56,12 @@
</header> </header>
<div class="mt-4"> <div class="mt-4">
<PlausibleWeb.Components.Generic.button_link disabled href=""> <PlausibleWeb.Components.Generic.button
data-method="post"
data-to={"/#{URI.encode_www_form(@site.domain)}/settings/export"}
data-csrf={Plug.CSRFProtection.get_csrf_token()}
>
Export to CSV Export to CSV
</PlausibleWeb.Components.Generic.button_link> </PlausibleWeb.Components.Generic.button>
</div> </div>
</div> </div>

@ -197,7 +197,8 @@ defmodule Plausible.ConfigTest do
{"S3_ACCESS_KEY_ID", nil}, {"S3_ACCESS_KEY_ID", nil},
{"S3_SECRET_ACCESS_KEY", nil}, {"S3_SECRET_ACCESS_KEY", nil},
{"S3_REGION", nil}, {"S3_REGION", nil},
{"S3_ENDPOINT", nil} {"S3_ENDPOINT", nil},
{"S3_EXPORTS_BUCKET", nil}
] ]
result = result =
@ -210,12 +211,13 @@ defmodule Plausible.ConfigTest do
assert %ArgumentError{} = result assert %ArgumentError{} = result
assert Exception.message(result) == """ assert Exception.message(result) == """
Missing S3 configuration. Please set S3_ACCESS_KEY_ID, S3_SECRET_ACCESS_KEY, S3_REGION, S3_ENDPOINT environment variable(s): Missing S3 configuration. Please set S3_ACCESS_KEY_ID, S3_SECRET_ACCESS_KEY, S3_REGION, S3_ENDPOINT, S3_EXPORTS_BUCKET environment variable(s):
\tS3_ACCESS_KEY_ID=AKIAIOSFODNN7EXAMPLE \tS3_ACCESS_KEY_ID=AKIAIOSFODNN7EXAMPLE
\tS3_SECRET_ACCESS_KEY=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY \tS3_SECRET_ACCESS_KEY=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY
\tS3_REGION=us-east-1 \tS3_REGION=us-east-1
\tS3_ENDPOINT=https://<ACCOUNT_ID>.r2.cloudflarestorage.com \tS3_ENDPOINT=https://<ACCOUNT_ID>.r2.cloudflarestorage.com
\tS3_EXPORTS_BUCKET=my-csv-exports-bucket
""" """
end end
@ -224,7 +226,8 @@ defmodule Plausible.ConfigTest do
{"S3_ACCESS_KEY_ID", "AKIAIOSFODNN7EXAMPLE"}, {"S3_ACCESS_KEY_ID", "AKIAIOSFODNN7EXAMPLE"},
{"S3_SECRET_ACCESS_KEY", nil}, {"S3_SECRET_ACCESS_KEY", nil},
{"S3_REGION", "eu-north-1"}, {"S3_REGION", "eu-north-1"},
{"S3_ENDPOINT", nil} {"S3_ENDPOINT", nil},
{"S3_EXPORTS_BUCKET", "my-exports"}
] ]
result = result =
@ -249,7 +252,8 @@ defmodule Plausible.ConfigTest do
{"S3_ACCESS_KEY_ID", "minioadmin"}, {"S3_ACCESS_KEY_ID", "minioadmin"},
{"S3_SECRET_ACCESS_KEY", "minioadmin"}, {"S3_SECRET_ACCESS_KEY", "minioadmin"},
{"S3_REGION", "us-east-1"}, {"S3_REGION", "us-east-1"},
{"S3_ENDPOINT", "http://localhost:6000"} {"S3_ENDPOINT", "http://localhost:6000"},
{"S3_EXPORTS_BUCKET", "my-exports"}
] ]
config = runtime_config(env) config = runtime_config(env)
@ -261,6 +265,10 @@ defmodule Plausible.ConfigTest do
region: "us-east-1", region: "us-east-1",
s3: [scheme: "http://", host: "localhost", port: 6000] s3: [scheme: "http://", host: "localhost", port: 6000]
] ]
assert get_in(runtime_config(env), [:plausible, Plausible.S3]) == [
exports_bucket: "my-exports"
]
end end
end end

Loading…
Cancel
Save