You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
analytics/lib/plausible/s3.ex

159 lines
5.0 KiB
Elixir

defmodule Plausible.S3 do
@moduledoc """
Helper functions for S3 exports/imports.
"""
@doc """
Returns the pre-configured S3 bucket for CSV exports.
config :plausible, Plausible.S3,
exports_bucket: System.fetch_env!("S3_EXPORTS_BUCKET")
Example:
iex> exports_bucket()
"test-exports"
"""
@spec exports_bucket :: String.t()
def exports_bucket, do: config(:exports_bucket)
@doc """
Returns the pre-configured S3 bucket for CSV imports.
config :plausible, Plausible.S3,
imports_bucket: System.fetch_env!("S3_IMPORTS_BUCKET")
Example:
iex> imports_bucket()
"test-imports"
"""
@spec imports_bucket :: String.t()
def imports_bucket, do: config(:imports_bucket)
defp config, do: Application.fetch_env!(:plausible, __MODULE__)
defp config(key), do: Keyword.fetch!(config(), key)
@doc """
Presigns an upload for an imported file.
In the current implementation the bucket always goes into the path component.
Example:
iex> %{
...> s3_url: "http://localhost:10000/test-imports/123/imported_browsers.csv",
...> presigned_url: "http://localhost:10000/test-imports/123/imported_browsers.csv?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=minioadmin" <> _
...> } = import_presign_upload(_site_id = 123, _filename = "imported_browsers.csv")
"""
def import_presign_upload(site_id, filename) do
config = ExAws.Config.new(:s3)
s3_path = Path.join(to_string(site_id), filename)
bucket = imports_bucket()
{:ok, presigned_url} = ExAws.S3.presigned_url(config, :put, bucket, s3_path)
%{s3_url: extract_s3_url(presigned_url), presigned_url: presigned_url}
end
# to make ClickHouse see MinIO in dev and test envs we replace
# the host in the S3 URL with whatever's set in S3_CLICKHOUSE_HOST env var
if Mix.env() in [:dev, :test, :ce_dev, :ce_test] do
defp extract_s3_url(presigned_url) do
[s3_url, _] = String.split(presigned_url, "?")
if ch_host = System.get_env("S3_CLICKHOUSE_HOST") do
URI.to_string(%URI{URI.parse(s3_url) | host: ch_host})
else
s3_url
end
end
else
defp extract_s3_url(presigned_url) do
[s3_url, _] = String.split(presigned_url, "?")
s3_url
end
end
@doc """
Chunks and uploads Zip archive to the provided S3 destination.
Returns a presigned URL to download the exported Zip archive from S3.
The URL expires in 24 hours.
In the current implementation the bucket always goes into the path component.
"""
@spec export_upload_multipart(Enumerable.t(), String.t(), Path.t(), String.t(), keyword) ::
:uri_string.uri_string()
def export_upload_multipart(stream, s3_bucket, s3_path, filename, config_overrides \\ []) do
config = ExAws.Config.new(:s3)
encoded_filename = URI.encode(filename)
disposition = ~s[attachment; filename="#{encoded_filename}"]
disposition =
if encoded_filename != filename do
disposition <> "; filename*=utf-8''#{encoded_filename}"
else
disposition
end
# 5 MiB is the smallest chunk size AWS S3 supports
chunk_into_parts(stream, 5 * 1024 * 1024)
|> ExAws.S3.upload(s3_bucket, s3_path,
content_disposition: disposition,
content_type: "application/zip"
)
|> ExAws.request!(config_overrides)
{:ok, download_url} =
ExAws.S3.presigned_url(config, :get, s3_bucket, s3_path, expires_in: _24hr = 86_400)
download_url
end
defp chunk_into_parts(stream, min_part_size) do
Stream.chunk_while(
stream,
_acc = %{buffer_size: 0, buffer: [], min_part_size: min_part_size},
_chunk_fun = &buffer_until_big_enough/2,
_after_fun = &flush_leftovers/1
)
end
defp buffer_until_big_enough(data, acc) do
%{buffer_size: prev_buffer_size, buffer: prev_buffer, min_part_size: min_part_size} = acc
new_buffer_size = prev_buffer_size + IO.iodata_length(data)
new_buffer = [prev_buffer | data]
if new_buffer_size > min_part_size do
# NOTE: PR to make ExAws.Operation.ExAws.Operation.S3.put_content_length_header/3 accept iodata
{:cont, IO.iodata_to_binary(new_buffer), %{acc | buffer_size: 0, buffer: []}}
else
{:cont, %{acc | buffer_size: new_buffer_size, buffer: new_buffer}}
end
end
defp flush_leftovers(acc) do
# NOTE: PR to make ExAws.Operation.ExAws.Operation.S3.put_content_length_header/3 accept iodata
{:cont, IO.iodata_to_binary(acc.buffer), %{acc | buffer_size: 0, buffer: []}}
end
@doc """
Returns `access_key_id` and `secret_access_key` to be used by ClickHouse during imports from S3.
Example:
iex> import_clickhouse_credentials()
%{access_key_id: "minioadmin", secret_access_key: "minioadmin"}
"""
@spec import_clickhouse_credentials ::
%{access_key_id: String.t(), secret_access_key: String.t()}
def import_clickhouse_credentials do
%{access_key_id: access_key_id, secret_access_key: secret_access_key} = ExAws.Config.new(:s3)
%{access_key_id: access_key_id, secret_access_key: secret_access_key}
end
end