From 5e74b1cf745e449d6862a11314346d91a381bbaa Mon Sep 17 00:00:00 2001 From: ruslandoga Date: Mon, 18 Mar 2024 15:52:57 +0800 Subject: [PATCH] CSV exports (UI) (#3875) * ui * fix redirect link * improve make minio * use implicit button form for csv export * add exports_bucket helper * read S3_EXPORTS_BUCKET * supply s3_bucket in export_csv job args * make plausible_minio use unprotected port * move s3_csv_export queue to base queues * Update lib/plausible_web/controllers/site_controller.ex Co-authored-by: Adrian Gruntkowski --------- Co-authored-by: Adrian Gruntkowski --- Makefile | 5 ++++- config/.env.dev | 3 ++- config/.env.test | 3 ++- config/runtime.exs | 13 ++++++++++--- lib/plausible/s3.ex | 18 ++++++++++++++++++ .../controllers/site_controller.ex | 17 +++++++++++++++++ lib/plausible_web/router.ex | 1 + .../site/settings_imports_exports.html.heex | 8 ++++++-- test/plausible/config_test.exs | 16 ++++++++++++---- 9 files changed, 72 insertions(+), 12 deletions(-) diff --git a/Makefile b/Makefile index c2c7dc722..ea4d92bbb 100644 --- a/Makefile +++ b/Makefile @@ -38,7 +38,10 @@ postgres-stop: ## Stop and remove the postgres container docker stop plausible_db && docker rm plausible_db minio: ## Start a transient container with a recent version of minio (s3) - docker run -d --rm -p 6000:6000 -p 6001:6001 --name plausible_minio minio/minio server /data --address ":6000" --console-address ":6001" + docker run -d --rm -p 10000:10000 -p 10001:10001 --name plausible_minio minio/minio server /data --address ":10000" --console-address ":10001" + while ! docker exec plausible_minio mc alias set local http://localhost:10000 minioadmin minioadmin; do sleep 1; done + docker exec plausible_minio mc mb local/dev-exports + docker exec plausible_minio mc mb local/dev-imports minio-stop: ## Stop and remove the minio container docker stop plausible_minio diff --git a/config/.env.dev b/config/.env.dev index 22f3c85bd..32ab90a45 100644 --- a/config/.env.dev +++ b/config/.env.dev @@ -25,4 +25,5 @@ S3_DISABLED=false S3_ACCESS_KEY_ID=minioadmin S3_SECRET_ACCESS_KEY=minioadmin S3_REGION=us-east-1 -S3_ENDPOINT=http://localhost:6000 +S3_ENDPOINT=http://localhost:10000 +S3_EXPORTS_BUCKET=dev-exports diff --git a/config/.env.test b/config/.env.test index 56092d916..d41cde0eb 100644 --- a/config/.env.test +++ b/config/.env.test @@ -20,4 +20,5 @@ S3_DISABLED=false S3_ACCESS_KEY_ID=minioadmin S3_SECRET_ACCESS_KEY=minioadmin S3_REGION=us-east-1 -S3_ENDPOINT=http://localhost:6000 +S3_ENDPOINT=http://localhost:10000 +S3_EXPORTS_BUCKET=test-exports diff --git a/config/runtime.exs b/config/runtime.exs index 2d056318d..83807cf1b 100644 --- a/config/runtime.exs +++ b/config/runtime.exs @@ -547,15 +547,16 @@ base_queues = [ clean_invitations: 1, analytics_imports: 1, domain_change_transition: 1, - check_accept_traffic_until: 1 + check_accept_traffic_until: 1, + # NOTE: maybe move s3_csv_export to cloud_queues? + s3_csv_export: 1 ] cloud_queues = [ trial_notification_emails: 1, check_usage: 1, notify_annual_renewal: 1, - lock_sites: 1, - s3_csv_export: 1 + lock_sites: 1 ] queues = if(is_selfhost, do: base_queues, else: base_queues ++ cloud_queues) @@ -740,6 +741,10 @@ unless s3_disabled? do %{ name: "S3_ENDPOINT", example: "https://.r2.cloudflarestorage.com" + }, + %{ + name: "S3_EXPORTS_BUCKET", + example: "my-csv-exports-bucket" } ] @@ -774,4 +779,6 @@ unless s3_disabled? do scheme: s3_scheme <> "://", host: s3_host, port: s3_port + + config :plausible, Plausible.S3, exports_bucket: s3_env_value.("S3_EXPORTS_BUCKET") end diff --git a/lib/plausible/s3.ex b/lib/plausible/s3.ex index 1d27a8f13..9086e002e 100644 --- a/lib/plausible/s3.ex +++ b/lib/plausible/s3.ex @@ -3,6 +3,24 @@ defmodule Plausible.S3 do Helper functions for S3 exports/imports. """ + @doc """ + Returns the pre-configured S3 bucket for CSV exports. + + config :plausible, Plausible.S3, + exports_bucket: System.fetch_env!("S3_EXPORTS_BUCKET") + + Example: + + iex> exports_bucket() + "test-exports" + + """ + @spec exports_bucket :: String.t() + def exports_bucket, do: config(:exports_bucket) + + defp config, do: Application.fetch_env!(:plausible, __MODULE__) + defp config(key), do: Keyword.fetch!(config(), key) + @doc """ Chunks and uploads Zip archive to the provided S3 destination. diff --git a/lib/plausible_web/controllers/site_controller.ex b/lib/plausible_web/controllers/site_controller.ex index aa38531cc..436c391c7 100644 --- a/lib/plausible_web/controllers/site_controller.ex +++ b/lib/plausible_web/controllers/site_controller.ex @@ -708,6 +708,23 @@ defmodule PlausibleWeb.SiteController do |> redirect(external: Routes.site_path(conn, :settings_integrations, site.domain)) end + def export(conn, _params) do + %{site: site, current_user: user} = conn.assigns + + Oban.insert!( + Plausible.Workers.ExportCSV.new(%{ + "site_id" => site.id, + "email_to" => user.email, + "s3_bucket" => Plausible.S3.exports_bucket(), + "s3_path" => "Plausible-#{site.id}.zip" + }) + ) + + conn + |> put_flash(:success, "SCHEDULED. WAIT FOR MAIL") + |> redirect(to: Routes.site_path(conn, :settings_imports_exports, site.domain)) + end + def change_domain(conn, _params) do changeset = Plausible.Site.update_changeset(conn.assigns.site) diff --git a/lib/plausible_web/router.ex b/lib/plausible_web/router.ex index 162dee3ca..d7c0ff414 100644 --- a/lib/plausible_web/router.ex +++ b/lib/plausible_web/router.ex @@ -397,6 +397,7 @@ defmodule PlausibleWeb.Router do delete "/:website/settings/forget-imported", SiteController, :forget_imported delete "/:website/settings/forget-import/:import_id", SiteController, :forget_import + post "/:website/settings/export", SiteController, :export get "/:domain/export", StatsController, :csv_export get "/:domain/*path", StatsController, :stats diff --git a/lib/plausible_web/templates/site/settings_imports_exports.html.heex b/lib/plausible_web/templates/site/settings_imports_exports.html.heex index b1b31f877..f3dd977b8 100644 --- a/lib/plausible_web/templates/site/settings_imports_exports.html.heex +++ b/lib/plausible_web/templates/site/settings_imports_exports.html.heex @@ -56,8 +56,12 @@
- + Export to CSV - +
diff --git a/test/plausible/config_test.exs b/test/plausible/config_test.exs index 28e6cad4c..5ba0005bf 100644 --- a/test/plausible/config_test.exs +++ b/test/plausible/config_test.exs @@ -197,7 +197,8 @@ defmodule Plausible.ConfigTest do {"S3_ACCESS_KEY_ID", nil}, {"S3_SECRET_ACCESS_KEY", nil}, {"S3_REGION", nil}, - {"S3_ENDPOINT", nil} + {"S3_ENDPOINT", nil}, + {"S3_EXPORTS_BUCKET", nil} ] result = @@ -210,12 +211,13 @@ defmodule Plausible.ConfigTest do assert %ArgumentError{} = result assert Exception.message(result) == """ - Missing S3 configuration. Please set S3_ACCESS_KEY_ID, S3_SECRET_ACCESS_KEY, S3_REGION, S3_ENDPOINT environment variable(s): + Missing S3 configuration. Please set S3_ACCESS_KEY_ID, S3_SECRET_ACCESS_KEY, S3_REGION, S3_ENDPOINT, S3_EXPORTS_BUCKET environment variable(s): \tS3_ACCESS_KEY_ID=AKIAIOSFODNN7EXAMPLE \tS3_SECRET_ACCESS_KEY=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY \tS3_REGION=us-east-1 \tS3_ENDPOINT=https://.r2.cloudflarestorage.com + \tS3_EXPORTS_BUCKET=my-csv-exports-bucket """ end @@ -224,7 +226,8 @@ defmodule Plausible.ConfigTest do {"S3_ACCESS_KEY_ID", "AKIAIOSFODNN7EXAMPLE"}, {"S3_SECRET_ACCESS_KEY", nil}, {"S3_REGION", "eu-north-1"}, - {"S3_ENDPOINT", nil} + {"S3_ENDPOINT", nil}, + {"S3_EXPORTS_BUCKET", "my-exports"} ] result = @@ -249,7 +252,8 @@ defmodule Plausible.ConfigTest do {"S3_ACCESS_KEY_ID", "minioadmin"}, {"S3_SECRET_ACCESS_KEY", "minioadmin"}, {"S3_REGION", "us-east-1"}, - {"S3_ENDPOINT", "http://localhost:6000"} + {"S3_ENDPOINT", "http://localhost:6000"}, + {"S3_EXPORTS_BUCKET", "my-exports"} ] config = runtime_config(env) @@ -261,6 +265,10 @@ defmodule Plausible.ConfigTest do region: "us-east-1", s3: [scheme: "http://", host: "localhost", port: 6000] ] + + assert get_in(runtime_config(env), [:plausible, Plausible.S3]) == [ + exports_bucket: "my-exports" + ] end end