From 3bedf9281c69b125445b96b87f643dacdfb6403a Mon Sep 17 00:00:00 2001 From: Vini Brasil Date: Thu, 17 Nov 2022 21:46:42 -0300 Subject: [PATCH] Seed database with pageviews (#2449) * Seed database with pageviews This commit adds basic support for database seeding useful for testing, especially dashboard changes, like intervals. It creates two years of pageviews with random timestamps. There is lot of room for improvement, such as adding sources, entry pages, geolocation, devices, custom events, but this already helps us with testing. * Update CONTRIBUTING.md file --- CONTRIBUTING.md | 22 ++++++++++++++++------ mix.exs | 5 ++--- priv/repo/seeds.exs | 28 ++++++++++++++++++++++++++++ test/support/factory.ex | 8 ++++---- 4 files changed, 50 insertions(+), 13 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index bcb4de2c8..5996d6b6c 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -8,21 +8,31 @@ The easiest way to get up and running is to [install](https://docs.docker.com/ge Make sure Docker, Elixir, Erlang and Node.js are all installed on your development machine. The [`.tool-versions`](https://github.com/plausible/analytics/blob/master/.tool-versions) file is available to use with [asdf](https://github.com/asdf-vm/asdf) or similar tools. -### Start the environment: +### Start the environment 1. Run both `make postgres` and `make clickhouse`. 2. You can set up everything with `make install`, alternatively run each command separately: 1. Run `mix deps.get`. This will download the required Elixir dependencies. 2. Run `mix ecto.create`. This will create the required databases in both Postgres and Clickhouse. 3. Run `mix ecto.migrate` to build the database schema. - 4. Run `npm ci --prefix assets` to install the required client-side dependencies. - 5. Run `npm ci --prefix tracker` to install the required tracker dependencies. - 6. Run `npm run deploy --prefix tracker` to generate tracker files in `priv/tracker/js` - 7. Run `mix download_country_database` to fetch geolocation database + 4. Run `mix run priv/repo/seeds.exs` to seed the database. Check the [Seeds](#Seeds) section for more. + 5. Run `npm ci --prefix assets` to install the required client-side dependencies. + 6. Run `npm ci --prefix tracker` to install the required tracker dependencies. + 7. Run `npm run deploy --prefix tracker` to generate tracker files in `priv/tracker/js` + 8. Run `mix download_country_database` to fetch geolocation database 3. Run `make server` or `mix phx.server` to start the Phoenix server. 4. The system is now available on `localhost:8000`. -### Creating an account +### Seeds + +You can optionally seed your database to automatically create an account and a site with stats: + +1. Run `mix run priv/repo/seeds.exs` to seed the database. +2. Start the server with `make server` and navigate to `http://localhost:8000/login`. +3. Log in with the following e-mail and password combination: `user@plausible.test` and `plausible`. +4. You should now have a `dummy.site` site with generated stats. + +Alternatively, you can manually create a new account: 1. Navigate to `http://localhost:8000/register` and fill in the form. 2. Fill in the rest of the forms and for the domain use `dummy.site` diff --git a/mix.exs b/mix.exs index a78b93987..ec13b6efd 100644 --- a/mix.exs +++ b/mix.exs @@ -47,7 +47,7 @@ defmodule Plausible.MixProject do end # Specifies which paths to compile per environment. - defp elixirc_paths(:test), do: ["lib", "test/support"] + defp elixirc_paths(env) when env in [:test, :dev], do: ["lib", "test/support"] defp elixirc_paths(_), do: ["lib"] # Specifies your project dependencies. @@ -71,9 +71,8 @@ defmodule Plausible.MixProject do {:dialyxir, "~> 1.0", only: [:dev, :test], runtime: false}, {:double, "~> 0.8.0", only: :test}, {:ecto_sql, "~> 3.0"}, - {:elixir_uuid, "~> 1.2", only: :test}, {:envy, "~> 1.1.1"}, - {:ex_machina, "~> 2.3", only: :test}, + {:ex_machina, "~> 2.3", only: [:dev, :test]}, {:excoveralls, "~> 0.10", only: :test}, {:exvcr, "~> 0.11", only: :test}, {:finch, "~> 0.12.0", override: true}, diff --git a/priv/repo/seeds.exs b/priv/repo/seeds.exs index b4931d33b..612dde53f 100644 --- a/priv/repo/seeds.exs +++ b/priv/repo/seeds.exs @@ -9,3 +9,31 @@ # # We recommend using the bang functions (`insert!`, `update!` # and so on) as they will fail if something goes wrong. + +user = Plausible.Factory.insert(:user, email: "user@plausible.test", password: "plausible") + +site = Plausible.Factory.insert(:site, domain: "dummy.site") + +membership = Plausible.Factory.insert(:site_membership, user: user, site: site, role: :owner) + +put_random_time = fn date -> + random_time = Time.new!(:rand.uniform(23), :rand.uniform(59), 0) + + date + |> NaiveDateTime.new!(random_time) + |> NaiveDateTime.truncate(:second) +end + +Enum.flat_map(-720..0, fn day_index -> + number_of_events = :rand.uniform(500) + date = Date.add(Date.utc_today(), day_index) + + attrs = [ + domain: site.domain, + hostname: site.domain, + timestamp: fn -> put_random_time.(date) end + ] + + Plausible.Factory.build_list(number_of_events, :pageview, attrs) +end) +|> Plausible.TestUtils.populate_stats() diff --git a/test/support/factory.ex b/test/support/factory.ex index 30ca26806..9797fb2fb 100644 --- a/test/support/factory.ex +++ b/test/support/factory.ex @@ -39,8 +39,8 @@ defmodule Plausible.Factory do %Plausible.ClickhouseSession{ sign: 1, - session_id: SipHash.hash!(hash_key(), UUID.uuid4()), - user_id: SipHash.hash!(hash_key(), UUID.uuid4()), + session_id: SipHash.hash!(hash_key(), Ecto.UUID.generate()), + user_id: SipHash.hash!(hash_key(), Ecto.UUID.generate()), hostname: hostname, domain: hostname, referrer: "", @@ -83,8 +83,8 @@ defmodule Plausible.Factory do domain: hostname, pathname: "/", timestamp: NaiveDateTime.utc_now() |> NaiveDateTime.truncate(:second), - user_id: SipHash.hash!(hash_key(), UUID.uuid4()), - session_id: SipHash.hash!(hash_key(), UUID.uuid4()), + user_id: SipHash.hash!(hash_key(), Ecto.UUID.generate()), + session_id: SipHash.hash!(hash_key(), Ecto.UUID.generate()), referrer: "", referrer_source: "", utm_medium: "",