From 12ba2c1cc8249c87481bf68925495d95bec6f7f0 Mon Sep 17 00:00:00 2001 From: ruslandoga <67764432+ruslandoga@users.noreply.github.com> Date: Thu, 4 Apr 2024 21:46:55 +0800 Subject: [PATCH] refactor input columns --- lib/plausible/imported/csv_importer.ex | 44 ++++++++++++++------------ 1 file changed, 23 insertions(+), 21 deletions(-) diff --git a/lib/plausible/imported/csv_importer.ex b/lib/plausible/imported/csv_importer.ex index bb3070ea7..4de8f7a4d 100644 --- a/lib/plausible/imported/csv_importer.ex +++ b/lib/plausible/imported/csv_importer.ex @@ -71,20 +71,12 @@ defmodule Plausible.Imported.CSVImporter do {table, _, _} = parse_filename!(filename) s3_structure = input_structure!(table) - - # NOTE: refactor - s3_structure_cols_expr = - s3_structure - |> String.split(",", trim: true) - |> Enum.map_join(", ", fn kv -> - [col, _type] = String.split(kv) - col - end) + s3_columns = input_columns!(table) statement = """ - INSERT INTO {table:Identifier}(site_id, #{s3_structure_cols_expr}, import_id) \ - SELECT {site_id:UInt64} AS site_id, *, {import_id:UInt64} AS import_id \ + INSERT INTO {table:Identifier}(site_id,import_id,#{s3_columns}) \ + SELECT {site_id:UInt64}, {import_id:UInt64}, * \ FROM s3({s3_url:String},{s3_access_key_id:String},{s3_secret_access_key:String},{s3_format:String},{s3_structure:String}) \ WHERE date >= {start_date:Date} AND date <= {end_date:Date}\ """ @@ -115,16 +107,6 @@ defmodule Plausible.Imported.CSVImporter do end_date: end_date } = site_import - # NOTE: s3_structure_cols_expr - statement = - """ - INSERT INTO {table:Identifier} \ - SELECT {site_id:UInt64} AS site_id, *, {import_id:UInt64} AS import_id \ - FROM input({input_structure:String}) \ - WHERE date >= {start_date:Date} AND date <= {end_date:Date} \ - FORMAT CSVWithNames\ - """ - DBConnection.run( ch, fn conn -> @@ -133,6 +115,16 @@ defmodule Plausible.Imported.CSVImporter do {table, _, _} = parse_filename!(filename) input_structure = input_structure!(table) + input_columns = input_columns!(table) + + statement = + """ + INSERT INTO {table:Identifier}(site_id,import_id,#{input_columns}) \ + SELECT {site_id:UInt64}, {import_id:UInt64}, * \ + FROM input({input_structure:String}) \ + WHERE date >= {start_date:Date} AND date <= {end_date:Date} \ + FORMAT CSVWithNames\ + """ params = %{ "table" => table, @@ -249,6 +241,16 @@ defmodule Plausible.Imported.CSVImporter do for {table, input_structure} <- input_structures do defp input_structure!(unquote(table)), do: unquote(input_structure) + input_columns = + input_structure + |> String.split(",", trim: true) + |> Enum.map_join(",", fn kv -> + [col, _type] = String.split(kv) + String.trim(col) + end) + + defp input_columns!(unquote(table)), do: unquote(input_columns) + def parse_filename!( <> ) do