From ce55dc4ae9e2a11ea2ce10e4788de46df5c1f16e Mon Sep 17 00:00:00 2001 From: Uku Taht Date: Tue, 26 Apr 2022 13:57:32 +0300 Subject: [PATCH] Retries Google Analytics API request if it returns empty rows (#1858) * Retries Google Analytics API request if it returns empty rows * Remove HTTPoison.Retry completely --- lib/plausible/google/api.ex | 47 +++++++---- mix.exs | 1 - mix.lock | 12 --- test/plausible/google/api_test.exs | 126 +++++++++++++++++++++++++++++ 4 files changed, 158 insertions(+), 28 deletions(-) create mode 100644 test/plausible/google/api_test.exs diff --git a/lib/plausible/google/api.ex b/lib/plausible/google/api.ex index d1d74ba8c..5b81ba899 100644 --- a/lib/plausible/google/api.ex +++ b/lib/plausible/google/api.ex @@ -2,7 +2,6 @@ defmodule Plausible.Google.Api do alias Plausible.Imported use Timex require Logger - require HTTPoison.Retry @scope URI.encode_www_form( "https://www.googleapis.com/auth/webmasters.readonly email https://www.googleapis.com/auth/analytics.readonly" @@ -323,33 +322,51 @@ defmodule Plausible.Google.Api do end end - defp fetch_and_persist(site, request) do + @max_attempts 5 + def fetch_and_persist(site, request, opts \\ []) do report_request = build_import_report_request(request) + http_client = Keyword.get(opts, :http_client, HTTPoison) + attempt = Keyword.get(opts, :attempt, 1) + sleep_time = Keyword.get(opts, :sleep_time, 1000) - {:ok, res} = - HTTPoison.post( + res = + http_client.post( "https://analyticsreporting.googleapis.com/v4/reports:batchGet", Jason.encode!(%{reportRequests: [report_request]}), [Authorization: "Bearer #{request.access_token}"], timeout: 30_000, recv_timeout: 30_000 ) - |> HTTPoison.Retry.autoretry(max_attempts: 5, wait: 5_000) - if res.status_code == 200 do - report = List.first(Jason.decode!(res.body)["reports"]) - data = get_in(report, ["data", "rows"]) + with {:ok, %HTTPoison.Response{status_code: 200, body: raw_body}} <- res, + {:ok, body} <- Jason.decode(raw_body), + report <- List.first(body["reports"]), + {:ok, data} <- get_non_empty_rows(report) do Imported.from_google_analytics(data, site.id, request.dataset) - case report["nextPageToken"] do - nil -> - :ok - - token -> - fetch_and_persist(site, %{request | page_token: token}) + if report["nextPageToken"] do + fetch_and_persist(site, %{request | page_token: report["nextPageToken"]}) + else + :ok end else - {:error, Jason.decode!(res.body)["error"]["message"]} + error -> + context_key = "request:#{attempt}" + Sentry.Context.set_extra_context(%{context_key => error}) + + if attempt >= @max_attempts do + raise "Google API request failed too many times" + else + Process.sleep(sleep_time) + fetch_and_persist(site, request, Keyword.merge(opts, attempt: attempt + 1)) + end + end + end + + defp get_non_empty_rows(report) do + case get_in(report, ["data", "rows"]) do + [] -> {:error, :empty_response_rows} + rows -> {:ok, rows} end end diff --git a/mix.exs b/mix.exs index 9b2547926..849060f23 100644 --- a/mix.exs +++ b/mix.exs @@ -74,7 +74,6 @@ defmodule Plausible.MixProject do {:bamboo_smtp, "~> 4.1"}, {:sentry, "~> 8.0"}, {:httpoison, "~> 1.4"}, - {:httpoison_retry, git: "https://github.com/ukutaht/httpoison_retry.git", tag: "master"}, {:ex_machina, "~> 2.3", only: :test}, {:excoveralls, "~> 0.10", only: :test}, {:double, "~> 0.8.0", only: :test}, diff --git a/mix.lock b/mix.lock index 180ad0400..76f600c36 100644 --- a/mix.lock +++ b/mix.lock @@ -1,5 +1,4 @@ %{ - "acceptor_pool": {:hex, :acceptor_pool, "1.0.0", "43c20d2acae35f0c2bcd64f9d2bde267e459f0f3fd23dab26485bf518c281b21", [:rebar3], [], "hexpm", "0cbcd83fdc8b9ad2eee2067ef8b91a14858a5883cb7cd800e6fcd5803e158788"}, "bamboo": {:hex, :bamboo, "2.2.0", "f10a406d2b7f5123eb1f02edfa043c259db04b47ab956041f279eaac776ef5ce", [:mix], [{:hackney, ">= 1.15.2", [hex: :hackney, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:mime, "~> 1.4", [hex: :mime, repo: "hexpm", optional: false]}, {:plug, "~> 1.0", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "8c3b14ba7d2f40cb4be04128ed1e2aff06d91d9413d38bafb4afccffa3ade4fc"}, "bamboo_phoenix": {:hex, :bamboo_phoenix, "1.0.0", "f3cc591ffb163ed0bf935d256f1f4645cd870cf436545601215745fb9cc9953f", [:mix], [{:bamboo, ">= 2.0.0", [hex: :bamboo, repo: "hexpm", optional: false]}, {:phoenix, ">= 1.3.0", [hex: :phoenix, repo: "hexpm", optional: false]}], "hexpm", "6db88fbb26019c84a47994bb2bd879c0887c29ce6c559bc6385fd54eb8b37dee"}, "bamboo_postmark": {:git, "https://github.com/pablo-co/bamboo_postmark.git", "c8b8cbf6103fc38a8ad467a3ab0eb6ebe44a84e6", [tag: "master"]}, @@ -8,7 +7,6 @@ "bunt": {:hex, :bunt, "0.2.0", "951c6e801e8b1d2cbe58ebbd3e616a869061ddadcc4863d0a2182541acae9a38", [:mix], [], "hexpm", "7af5c7e09fe1d40f76c8e4f9dd2be7cebd83909f31fee7cd0e9eadc567da8353"}, "cachex": {:hex, :cachex, "3.4.0", "868b2959ea4aeb328c6b60ff66c8d5123c083466ad3c33d3d8b5f142e13101fb", [:mix], [{:eternal, "~> 1.2", [hex: :eternal, repo: "hexpm", optional: false]}, {:jumper, "~> 1.0", [hex: :jumper, repo: "hexpm", optional: false]}, {:sleeplocks, "~> 1.1", [hex: :sleeplocks, repo: "hexpm", optional: false]}, {:unsafe, "~> 1.0", [hex: :unsafe, repo: "hexpm", optional: false]}], "hexpm", "370123b1ab4fba4d2965fb18f87fd758325709787c8c5fce35b3fe80645ccbe5"}, "certifi": {:hex, :certifi, "2.9.0", "6f2a475689dd47f19fb74334859d460a2dc4e3252a3324bd2111b8f0429e7e21", [:rebar3], [], "hexpm", "266da46bdb06d6c6d35fde799bcb28d36d985d424ad7c08b5bb48f5b5cdd4641"}, - "chatterbox": {:hex, :ts_chatterbox, "0.11.0", "b8f372c706023eb0de5bf2976764edb27c70fe67052c88c1f6a66b3a5626847f", [:rebar3], [{:hpack, "~>0.2.3", [hex: :hpack_erl, repo: "hexpm", optional: false]}], "hexpm", "722fe2bad52913ab7e87d849fc6370375f0c961ffb2f0b5e6d647c9170c382a6"}, "clickhouse_ecto": {:git, "https://github.com/plausible/clickhouse_ecto.git", "7bc94cce111d3e9dbd8534fe96bd5195181826a2", []}, "clickhousex": {:git, "https://github.com/plausible/clickhousex", "e99010dc672391d89c0fd01052df4366b9fc69a6", []}, "combination": {:hex, :combination, "0.0.3", "746aedca63d833293ec6e835aa1f34974868829b1486b1e1cb0685f0b2ae1f41", [:mix], [], "hexpm", "72b099f463df42ef7dc6371d250c7070b57b6c5902853f69deb894f79eda18ca"}, @@ -21,10 +19,8 @@ "cowlib": {:hex, :cowlib, "2.11.0", "0b9ff9c346629256c42ebe1eeb769a83c6cb771a6ee5960bd110ab0b9b872063", [:make, :rebar3], [], "hexpm", "2b3e9da0b21c4565751a6d4901c20d1b4cc25cbb7fd50d91d2ab6dd287bc86a9"}, "credo": {:hex, :credo, "1.6.4", "ddd474afb6e8c240313f3a7b0d025cc3213f0d171879429bf8535d7021d9ad78", [:mix], [{:bunt, "~> 0.2.0", [hex: :bunt, repo: "hexpm", optional: false]}, {:file_system, "~> 0.2.8", [hex: :file_system, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "c28f910b61e1ff829bffa056ef7293a8db50e87f2c57a9b5c3f57eee124536b7"}, "csv": {:hex, :csv, "2.4.1", "50e32749953b6bf9818dbfed81cf1190e38cdf24f95891303108087486c5925e", [:mix], [{:parallel_stream, "~> 1.0.4", [hex: :parallel_stream, repo: "hexpm", optional: false]}], "hexpm", "54508938ac67e27966b10ef49606e3ad5995d665d7fc2688efb3eab1307c9079"}, - "ctx": {:hex, :ctx, "0.6.0", "8ff88b70e6400c4df90142e7f130625b82086077a45364a78d208ed3ed53c7fe", [:rebar3], [], "hexpm", "a14ed2d1b67723dbebbe423b28d7615eb0bdcba6ff28f2d1f1b0a7e1d4aa5fc2"}, "db_connection": {:hex, :db_connection, "2.2.2", "3bbca41b199e1598245b716248964926303b5d4609ff065125ce98bcd368939e", [:mix], [{:connection, "~> 1.0.2", [hex: :connection, repo: "hexpm", optional: false]}], "hexpm", "642af240d8a8affb93b4ba5a6fcd2bbcbdc327e1a524b825d383711536f8070c"}, "decimal": {:hex, :decimal, "2.0.0", "a78296e617b0f5dd4c6caf57c714431347912ffb1d0842e998e9792b5642d697", [:mix], [], "hexpm", "34666e9c55dea81013e77d9d87370fe6cb6291d1ef32f46a1600230b1d44f577"}, - "decorator": {:hex, :decorator, "1.4.0", "a57ac32c823ea7e4e67f5af56412d12b33274661bb7640ec7fc882f8d23ac419", [:mix], [], "hexpm", "0a07cedd9083da875c7418dea95b78361197cf2bf3211d743f6f7ce39656597f"}, "dialyxir": {:hex, :dialyxir, "1.1.0", "c5aab0d6e71e5522e77beff7ba9e08f8e02bad90dfbeffae60eaf0cb47e29488", [:mix], [{:erlex, ">= 0.2.6", [hex: :erlex, repo: "hexpm", optional: false]}], "hexpm", "07ea8e49c45f15264ebe6d5b93799d4dd56a44036cf42d0ad9c960bc266c0b9a"}, "double": {:hex, :double, "0.8.2", "8e1cfcccdaef76c18846bc08e555555a2a699b806fa207b6468572a60513cc6a", [:mix], [], "hexpm", "90287642b2ec86125e0457aaba2ab0e80f7d7050cc80a0cef733e59bd70aa67c"}, "ecto": {:hex, :ecto, "3.7.2", "44c034f88e1980754983cc4400585970b4206841f6f3780967a65a9150ef09a8", [:mix], [{:decimal, "~> 1.6 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "a600da5772d1c31abbf06f3e4a1ffb150e74ed3e2aa92ff3cee95901657a874e"}, @@ -35,25 +31,19 @@ "erlex": {:hex, :erlex, "0.2.6", "c7987d15e899c7a2f34f5420d2a2ea0d659682c06ac607572df55a43753aa12e", [:mix], [], "hexpm", "2ed2e25711feb44d52b17d2780eabf998452f6efda104877a3881c2f8c0c0c75"}, "eternal": {:hex, :eternal, "1.2.2", "d1641c86368de99375b98d183042dd6c2b234262b8d08dfd72b9eeaafc2a1abd", [:mix], [], "hexpm", "2c9fe32b9c3726703ba5e1d43a1d255a4f3f2d8f8f9bc19f094c7cb1a7a9e782"}, "ex_machina": {:hex, :ex_machina, "2.7.0", "b792cc3127fd0680fecdb6299235b4727a4944a09ff0fa904cc639272cd92dc7", [:mix], [{:ecto, "~> 2.2 or ~> 3.0", [hex: :ecto, repo: "hexpm", optional: true]}, {:ecto_sql, "~> 3.0", [hex: :ecto_sql, repo: "hexpm", optional: true]}], "hexpm", "419aa7a39bde11894c87a615c4ecaa52d8f107bbdd81d810465186f783245bf8"}, - "exconstructor": {:hex, :exconstructor, "1.2.6", "246473024fad510329ea569d02eead44ca35c413f582a94752e503929b97afe7", [:mix], [], "hexpm", "52142eaf77d3783f4b88003e33d41eed83605d4b32116bfb6e8198e981d10c8c"}, "excoveralls": {:hex, :excoveralls, "0.14.4", "295498f1ae47bdc6dce59af9a585c381e1aefc63298d48172efaaa90c3d251db", [:mix], [{:hackney, "~> 1.16", [hex: :hackney, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "e3ab02f2df4c1c7a519728a6f0a747e71d7d6e846020aae338173619217931c1"}, "file_system": {:hex, :file_system, "0.2.10", "fb082005a9cd1711c05b5248710f8826b02d7d1784e7c3451f9c1231d4fc162d", [:mix], [], "hexpm", "41195edbfb562a593726eda3b3e8b103a309b733ad25f3d642ba49696bf715dc"}, "floki": {:hex, :floki, "0.32.1", "dfe3b8db3b793939c264e6f785bca01753d17318d144bd44b407fb3493acaa87", [:mix], [{:html_entities, "~> 0.5.0", [hex: :html_entities, repo: "hexpm", optional: false]}], "hexpm", "d4b91c713e4a784a3f7b1e3cc016eefc619f6b1c3898464222867cafd3c681a3"}, "fun_with_flags": {:hex, :fun_with_flags, "1.8.1", "5999dd64d2e97646554244baa9b1da8ca33d656eed37307238b2dda8c17dba45", [:mix], [{:ecto_sql, "~> 3.0", [hex: :ecto_sql, repo: "hexpm", optional: true]}, {:phoenix_pubsub, "~> 2.0", [hex: :phoenix_pubsub, repo: "hexpm", optional: true]}, {:redix, "~> 1.0", [hex: :redix, repo: "hexpm", optional: true]}], "hexpm", "02222ddffccbb1fa339f6ee3f14fac88ba44121b167c72bce93c2390013b3adc"}, "fun_with_flags_ui": {:hex, :fun_with_flags_ui, "0.8.0", "70587e344ba2035516a639e7bd8cb2ce8d54ee6c5f5dfd3e4e6f6a776e14ac1d", [:mix], [{:cowboy, ">= 2.0.0", [hex: :cowboy, repo: "hexpm", optional: true]}, {:fun_with_flags, "~> 1.8", [hex: :fun_with_flags, repo: "hexpm", optional: false]}, {:plug, "~> 1.12", [hex: :plug, repo: "hexpm", optional: false]}, {:plug_cowboy, ">= 2.0.0", [hex: :plug_cowboy, repo: "hexpm", optional: true]}], "hexpm", "95e1e5afae77e259a4a43f930f3da97ff3c388a72d4622f7848cb7ee34de6338"}, - "gen_retry": {:hex, :gen_retry, "1.4.0", "682fadcb7ebf629f8dc4d84b4e62e03dce1baa526141133a19a8e7cee5ef7e1b", [:mix], [{:exconstructor, "~> 1.0", [hex: :exconstructor, repo: "hexpm", optional: false]}], "hexpm", "c15311afe0770f5fc58f4391b2ff3362fd25f1e107d02ff63592d36f30eaaae4"}, "gen_smtp": {:hex, :gen_smtp, "1.1.1", "bf9303c31735100631b1d708d629e4c65944319d1143b5c9952054f4a1311d85", [:rebar3], [{:hut, "1.3.0", [hex: :hut, repo: "hexpm", optional: false]}, {:ranch, ">= 1.7.0", [hex: :ranch, repo: "hexpm", optional: false]}], "hexpm", "51bc50cc017efd4a4248cbc39ea30fb60efa7d4a49688986fafad84434ff9ab7"}, "geolix": {:hex, :geolix, "1.1.0", "8b0fe847fef486d9e8b7c21eae6cbc2d998fb249e61d3f4f136f8016b9c1c833", [:mix], [{:poolboy, "~> 1.0", [hex: :poolboy, repo: "hexpm", optional: false]}], "hexpm", "980854f2aef30c288dc79e86c5267806d704c4525fde1b75de9a92f67fb16300"}, "geolix_adapter_mmdb2": {:hex, :geolix_adapter_mmdb2, "0.5.0", "5912723d9538ecddc6b29b1d8041b917b735a78fd3c122bfea8c44aa782e3369", [:mix], [{:geolix, "~> 1.1", [hex: :geolix, repo: "hexpm", optional: false]}, {:mmdb2_decoder, "~> 3.0", [hex: :mmdb2_decoder, repo: "hexpm", optional: false]}], "hexpm", "cb1485b6a0a2d3e541949207428a245718dbf1258453a0df0e5fdd925bcecd3e"}, "gettext": {:hex, :gettext, "0.19.1", "564953fd21f29358e68b91634799d9d26989f8d039d7512622efb3c3b1c97892", [:mix], [], "hexpm", "10c656c0912b8299adba9b061c06947511e3f109ab0d18b44a866a4498e77222"}, - "gproc": {:hex, :gproc, "0.8.0", "cea02c578589c61e5341fce149ea36ccef236cc2ecac8691fba408e7ea77ec2f", [:rebar3], [], "hexpm", "580adafa56463b75263ef5a5df4c86af321f68694e7786cb057fd805d1e2a7de"}, - "grpcbox": {:hex, :grpcbox, "0.14.0", "3eb321bcd2275baf8b54cf381feb7b0559a50c02544de28fda039c7f2f9d1a7a", [:rebar3], [{:acceptor_pool, "~>1.0.0", [hex: :acceptor_pool, repo: "hexpm", optional: false]}, {:chatterbox, "~>0.11.0", [hex: :ts_chatterbox, repo: "hexpm", optional: false]}, {:ctx, "~>0.6.0", [hex: :ctx, repo: "hexpm", optional: false]}, {:gproc, "~>0.8.0", [hex: :gproc, repo: "hexpm", optional: false]}], "hexpm", "e24159b7b6d3f9869bbe528845c0125fed2259366ba908fd04a1f45fe81d0660"}, "hackney": {:hex, :hackney, "1.18.1", "f48bf88f521f2a229fc7bae88cf4f85adc9cd9bcf23b5dc8eb6a1788c662c4f6", [:rebar3], [{:certifi, "~>2.9.0", [hex: :certifi, repo: "hexpm", optional: false]}, {:idna, "~>6.1.0", [hex: :idna, repo: "hexpm", optional: false]}, {:metrics, "~>1.0.0", [hex: :metrics, repo: "hexpm", optional: false]}, {:mimerl, "~>1.1", [hex: :mimerl, repo: "hexpm", optional: false]}, {:parse_trans, "3.3.1", [hex: :parse_trans, repo: "hexpm", optional: false]}, {:ssl_verify_fun, "~>1.1.0", [hex: :ssl_verify_fun, repo: "hexpm", optional: false]}, {:unicode_util_compat, "~>0.7.0", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm", "a4ecdaff44297e9b5894ae499e9a070ea1888c84afdd1fd9b7b2bc384950128e"}, "hammer": {:hex, :hammer, "6.0.0", "72ec6fff10e9d63856968988a22ee04c4d6d5248071ddccfbda50aa6c455c1d7", [:mix], [{:poolboy, "~> 1.5", [hex: :poolboy, repo: "hexpm", optional: false]}], "hexpm", "d8e1ec2e534c4aae508b906759e077c3c1eb3e2b9425235d4b7bbab0b016210a"}, - "hpack": {:hex, :hpack_erl, "0.2.3", "17670f83ff984ae6cd74b1c456edde906d27ff013740ee4d9efaa4f1bf999633", [:rebar3], [], "hexpm", "06f580167c4b8b8a6429040df36cc93bba6d571faeaec1b28816523379cbb23a"}, "html_entities": {:hex, :html_entities, "0.5.2", "9e47e70598da7de2a9ff6af8758399251db6dbb7eebe2b013f2bbd2515895c3c", [:mix], [], "hexpm", "c53ba390403485615623b9531e97696f076ed415e8d8058b1dbaa28181f4fdcc"}, "httpoison": {:hex, :httpoison, "1.8.1", "df030d96de89dad2e9983f92b0c506a642d4b1f4a819c96ff77d12796189c63e", [:mix], [{:hackney, "~> 1.17", [hex: :hackney, repo: "hexpm", optional: false]}], "hexpm", "35156a6d678d6d516b9229e208942c405cf21232edd632327ecfaf4fd03e79e0"}, - "httpoison_retry": {:git, "https://github.com/ukutaht/httpoison_retry.git", "fa723ce6c79fec9002768be7aca3f4ce0b1c59e9", [tag: "master"]}, "hut": {:hex, :hut, "1.3.0", "71f2f054e657c03f959cf1acc43f436ea87580696528ca2a55c8afb1b06c85e7", [:"erlang.mk", :rebar, :rebar3], [], "hexpm", "7e15d28555d8a1f2b5a3a931ec120af0753e4853a4c66053db354f35bf9ab563"}, "idna": {:hex, :idna, "6.1.1", "8a63070e9f7d0c62eb9d9fcb360a7de382448200fbbd1b106cc96d3d8099df8d", [:rebar3], [{:unicode_util_compat, "~>0.7.0", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm", "92376eb7894412ed19ac475e4a86f7b413c1b9fbb5bd16dccd57934157944cea"}, "jason": {:hex, :jason, "1.3.0", "fa6b82a934feb176263ad2df0dbd91bf633d4a46ebfdffea0c8ae82953714946", [:mix], [{:decimal, "~> 1.0 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "53fc1f51255390e0ec7e50f9cb41e751c260d065dcba2bf0d08dc51a4002c2ac"}, @@ -80,7 +70,6 @@ "plug": {:hex, :plug, "1.13.4", "addb6e125347226e3b11489e23d22a60f7ab74786befb86c14f94fb5f23ca9a4", [:mix], [{:mime, "~> 1.0 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:plug_crypto, "~> 1.1.1 or ~> 1.2", [hex: :plug_crypto, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4.3 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "06114c1f2a334212fe3ae567dbb3b1d29fd492c1a09783d52f3d489c1a6f4cf2"}, "plug_cowboy": {:hex, :plug_cowboy, "2.5.2", "62894ccd601cf9597e2c23911ff12798a8a18d237e9739f58a6b04e4988899fe", [:mix], [{:cowboy, "~> 2.7", [hex: :cowboy, repo: "hexpm", optional: false]}, {:cowboy_telemetry, "~> 0.3", [hex: :cowboy_telemetry, repo: "hexpm", optional: false]}, {:plug, "~> 1.7", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "ea6e87f774c8608d60c8d34022a7d073bd7680a0a013f049fc62bf35efea1044"}, "plug_crypto": {:hex, :plug_crypto, "1.2.2", "05654514ac717ff3a1843204b424477d9e60c143406aa94daf2274fdd280794d", [:mix], [], "hexpm", "87631c7ad914a5a445f0a3809f99b079113ae4ed4b867348dd9eec288cecb6db"}, - "poison": {:hex, :poison, "3.1.0", "d9eb636610e096f86f25d9a46f35a9facac35609a7591b3be3326e99a0484665", [:mix], [], "hexpm", "fec8660eb7733ee4117b85f55799fd3833eb769a6df71ccf8903e8dc5447cfce"}, "poolboy": {:hex, :poolboy, "1.5.2", "392b007a1693a64540cead79830443abf5762f5d30cf50bc95cb2c1aaafa006b", [:rebar3], [], "hexpm", "dad79704ce5440f3d5a3681c8590b9dc25d1a561e8f5a9c995281012860901e3"}, "postgrex": {:hex, :postgrex, "0.16.2", "0f83198d0e73a36e8d716b90f45f3bde75b5eebf4ade4f43fa1f88c90a812f74", [:mix], [{:connection, "~> 1.1", [hex: :connection, repo: "hexpm", optional: false]}, {:db_connection, "~> 2.1", [hex: :db_connection, repo: "hexpm", optional: false]}, {:decimal, "~> 1.5 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}], "hexpm", "a9ea589754d9d4d076121090662b7afe155b374897a6550eb288f11d755acfa0"}, "public_suffix": {:git, "https://github.com/axelson/publicsuffix-elixir", "89372422ab8b433de508519ef474e39699fd11ca", []}, @@ -92,7 +81,6 @@ "sleeplocks": {:hex, :sleeplocks, "1.1.1", "3d462a0639a6ef36cc75d6038b7393ae537ab394641beb59830a1b8271faeed3", [:rebar3], [], "hexpm", "84ee37aeff4d0d92b290fff986d6a95ac5eedf9b383fadfd1d88e9b84a1c02e1"}, "ssl_verify_fun": {:hex, :ssl_verify_fun, "1.1.6", "cf344f5692c82d2cd7554f5ec8fd961548d4fd09e7d22f5b62482e5aeaebd4b0", [:make, :mix, :rebar3], [], "hexpm", "bdb0d2471f453c88ff3908e7686f86f9be327d065cc1ec16fa4540197ea04680"}, "telemetry": {:hex, :telemetry, "0.4.3", "a06428a514bdbc63293cd9a6263aad00ddeb66f608163bdec7c8995784080818", [:rebar3], [], "hexpm", "eb72b8365ffda5bed68a620d1da88525e326cb82a75ee61354fc24b844768041"}, - "telemetry_registry": {:hex, :telemetry_registry, "0.3.0", "6768f151ea53fc0fbca70dbff5b20a8d663ee4e0c0b2ae589590e08658e76f1e", [:mix, :rebar3], [{:telemetry, "~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "492e2adbc609f3e79ece7f29fec363a97a2c484ac78a83098535d6564781e917"}, "timex": {:hex, :timex, "3.7.7", "3ed093cae596a410759104d878ad7b38e78b7c2151c6190340835515d4a46b8a", [:mix], [{:combine, "~> 0.10", [hex: :combine, repo: "hexpm", optional: false]}, {:gettext, "~> 0.10", [hex: :gettext, repo: "hexpm", optional: false]}, {:tzdata, "~> 1.0", [hex: :tzdata, repo: "hexpm", optional: false]}], "hexpm", "0ec4b09f25fe311321f9fc04144a7e3affe48eb29481d7a5583849b6c4dfa0a7"}, "tzdata": {:hex, :tzdata, "1.1.1", "20c8043476dfda8504952d00adac41c6eda23912278add38edc140ae0c5bcc46", [:mix], [{:hackney, "~> 1.17", [hex: :hackney, repo: "hexpm", optional: false]}], "hexpm", "a69cec8352eafcd2e198dea28a34113b60fdc6cb57eb5ad65c10292a6ba89787"}, "ua_inspector": {:hex, :ua_inspector, "3.0.1", "649e68e31296a7bc39204e84b13a14c245b61df43fde0dd91ef784a98ecb04d4", [:mix], [{:hackney, "~> 1.0", [hex: :hackney, repo: "hexpm", optional: false]}, {:yamerl, "~> 0.7", [hex: :yamerl, repo: "hexpm", optional: false]}], "hexpm", "b510ce5a5fd70a37bf5eab8008a25fc57388ea240b95bb4d98fc892648f9b916"}, diff --git a/test/plausible/google/api_test.exs b/test/plausible/google/api_test.exs new file mode 100644 index 000000000..3838b4ef9 --- /dev/null +++ b/test/plausible/google/api_test.exs @@ -0,0 +1,126 @@ +defmodule Plausible.Google.ApiTest do + use Plausible.DataCase, async: true + alias Plausible.Google.Api + import Plausible.TestUtils + import Double + + @ok_response Jason.encode!(%{ + "reports" => [ + %{ + "data" => %{ + "rows" => [ + %{ + "dimensions" => ["20220101"], + "metrics" => [%{"values" => ["1", "1", "1", "1", "1"]}] + } + ] + } + } + ] + }) + + @empty_response Jason.encode!(%{ + "reports" => [%{"data" => %{"rows" => []}}] + }) + + describe "fetch_and_persist/4" do + setup [:create_user, :create_new_site] + + test "will fetch and persist import data from Google Analytics", %{site: site} do + httpoison = + HTTPoison + |> stub(:post, fn _url, _body, _headers, _opts -> + {:ok, %HTTPoison.Response{status_code: 200, body: @ok_response}} + end) + + request = %{ + dataset: "imported_visitors", + view_id: "123", + date_range: Date.range(~D[2022-01-01], ~D[2022-02-01]), + dimensions: ["ga:date"], + metrics: ["ga:users"], + access_token: "fake-token", + page_token: nil + } + + Api.fetch_and_persist(site, request, http_client: httpoison, sleep_time: 0) + + assert imported_visitor_count(site) == 1 + end + + test "retries HTTP request up to 5 times before raising the last error", %{site: site} do + httpoison = + HTTPoison + |> stub(:post, fn _url, _body, _headers, _opts -> + {:error, %HTTPoison.Error{reason: :nxdomain}} + end) + |> stub(:post, fn _url, _body, _headers, _opts -> + {:error, %HTTPoison.Error{reason: :timeout}} + end) + |> stub(:post, fn _url, _body, _headers, _opts -> + {:error, %HTTPoison.Error{reason: :closed}} + end) + |> stub(:post, fn _url, _body, _headers, _opts -> + {:ok, %HTTPoison.Response{status_code: 503}} + end) + |> stub(:post, fn _url, _body, _headers, _opts -> + {:ok, %HTTPoison.Response{status_code: 502}} + end) + + request = %{ + view_id: "123", + date_range: Date.range(~D[2022-01-01], ~D[2022-02-01]), + dimensions: ["ga:date"], + metrics: ["ga:users"], + access_token: "fake-token", + page_token: nil + } + + assert_raise RuntimeError, "Google API request failed too many times", fn -> + Api.fetch_and_persist(site, request, http_client: httpoison, sleep_time: 0) + end + + assert_receive({HTTPoison, :post, [_, _, _, _]}) + assert_receive({HTTPoison, :post, [_, _, _, _]}) + assert_receive({HTTPoison, :post, [_, _, _, _]}) + assert_receive({HTTPoison, :post, [_, _, _, _]}) + assert_receive({HTTPoison, :post, [_, _, _, _]}) + end + + test "retries HTTP request if the rows are empty", %{site: site} do + httpoison = + HTTPoison + |> stub(:post, fn _url, _body, _headers, _opts -> + {:ok, %HTTPoison.Response{status_code: 200, body: @empty_response}} + end) + |> stub(:post, fn _url, _body, _headers, _opts -> + {:ok, %HTTPoison.Response{status_code: 200, body: @ok_response}} + end) + + request = %{ + dataset: "imported_visitors", + view_id: "123", + date_range: Date.range(~D[2022-01-01], ~D[2022-02-01]), + dimensions: ["ga:date"], + metrics: ["ga:users"], + access_token: "fake-token", + page_token: nil + } + + Api.fetch_and_persist(site, request, http_client: httpoison, sleep_time: 0) + + assert_receive({HTTPoison, :post, [_, _, _, _]}) + assert_receive({HTTPoison, :post, [_, _, _, _]}) + + assert imported_visitor_count(site) == 1 + end + end + + defp imported_visitor_count(site) do + Plausible.ClickhouseRepo.one( + from iv in "imported_visitors", + where: iv.site_id == ^site.id, + select: sum(iv.visitors) + ) + end +end