From 2ba4988c95cdb311e4b90fb6e39f1ba147741cc5 Mon Sep 17 00:00:00 2001 From: ruslandoga Date: Wed, 3 Apr 2024 17:10:49 +0800 Subject: [PATCH] specify insert columns in import from s3 (#3968) Co-authored-by: RobertJoonas <56999674+RobertJoonas@users.noreply.github.com> --- lib/plausible/imported/csv_importer.ex | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/lib/plausible/imported/csv_importer.ex b/lib/plausible/imported/csv_importer.ex index 6376d851d..c87300099 100644 --- a/lib/plausible/imported/csv_importer.ex +++ b/lib/plausible/imported/csv_importer.ex @@ -43,9 +43,17 @@ defmodule Plausible.Imported.CSVImporter do {table, _, _} = parse_filename!(filename) s3_structure = input_structure!(table) + s3_structure_cols_expr = + s3_structure + |> String.split(",", trim: true) + |> Enum.map_join(", ", fn kv -> + [col, _type] = String.split(kv) + col + end) + statement = """ - INSERT INTO {table:Identifier} \ + INSERT INTO {table:Identifier}(site_id, #{s3_structure_cols_expr}, import_id) \ SELECT {site_id:UInt64} AS site_id, *, {import_id:UInt64} AS import_id \ FROM s3({s3_url:String},{s3_access_key_id:String},{s3_secret_access_key:String},{s3_format:String},{s3_structure:String}) \ WHERE date >= {start_date:Date} AND date <= {end_date:Date}\