You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
analytics/lib/plausible/site.ex

251 lines
7.2 KiB
Elixir

defmodule Plausible.Site do
@moduledoc """
Site schema
"""
use Ecto.Schema
import Ecto.Changeset
alias Plausible.Auth.User
alias Plausible.Site.GoogleAuth
@type t() :: %__MODULE__{}
@derive {Jason.Encoder, only: [:domain, :timezone]}
schema "sites" do
field :domain, :string
field :timezone, :string, default: "Etc/UTC"
field :public, :boolean
field :locked, :boolean
field :stats_start_date, :date
field :native_stats_start_at, :naive_datetime
field :allowed_event_props, {:array, :string}
Funnel site settings (#3039) * Update formatter config * Install LiveView JS integration & hooks * Temporarily update endpoint/session config * Optionally allow preloading funnels for goals * Site controller * Implement funnel settings lib/plausible_web/live/funnel_settings/combo_box.ex - restored from: 054de6e2 Fix the tab/blur bug again 20da4c89 Rename InputPicker to ComboBox lib/plausible_web/live/funnel_settings/form.ex - restored from: 9bedda3b Remove potential FIXME 20da4c89 Rename InputPicker to ComboBox 028036ad Review comments aea4ebc4 Access Funnel min/max steps via the __using__/1 macro 0dde27fd Remove inspect call eed588a7 Start testing the funnel editor 0e95228b Extract funnel settings test module 7b16ace5 Leverage aplinejs to deal with the tyranny 8dc6a3e7 wip cf228630 wip 30a43fd1 wip 89f10ecb wip 950a18d9 Dirty funnel save 298a6a53 wip 7690d50f wip 639c6238 fixup aa59adeb wip ff75c00b wip lib/plausible_web/live/funnel_settings/list.ex - restored from: 4eae122c Fix data-confirm attr interpolation 51f0397d Implement deleting funnels 1f6fe25d Add number of steps to funnels list 298a6a53 wip ff75c00b wip test/plausible_web/live/funnel_settings/funnel_settings/combo_box_test.exs - restored from: 20da4c89 Rename InputPicker to ComboBox test/plausible_web/live/funnel_settings/funnel_settings_test.exs - restored from: 34822ff4 Bootstrap InputPicker tests lib/plausible_web/live/funnel_settings.ex - restored from: 028036ad Review comments acd9c4f2 Prepare ephemeral funnel definitions so that users can test funnels 51f0397d Implement deleting funnels 0e95228b Extract funnel settings test module 8dc6a3e7 wip 89f10ecb wip 950a18d9 Dirty funnel save 298a6a53 wip aa59adeb wip ff75c00b wip test/plausible_web/controllers/error_report_controller_test.exs - restored from: 34822ff4 Bootstrap InputPicker tests test/support/html.ex - restored from: 0a53979d Improve InputPicker tests - include AlpineJS assertions 34822ff4 Bootstrap InputPicker tests lib/plausible_web/views/layout_view.ex - restored from: b490403b !ifxup lib/plausible_web/templates/site/settings_funnels.html.eex - restored from: 51f0397d Implement deleting funnels ea1315f3 Test funnels list in settings 7b16ace5 Leverage aplinejs to deal with the tyranny ff75c00b wip 4da25c35 Fixup lib/plausible_web/templates/layout/app.html.eex - restored from: ff75c00b wip * Add funnel settings route * Warn about funnels deletion when deleting goals lib/plausible_web/templates/site/settings_goals.html.eex - restored from: fdd9bcd0 Fixup f1e6364d Merge remote-tracking branch 'origin/master' into funnels-rebase 9d0b7c6d Fix markup error 4a4ddbdc Optionally preload funnels for goals and stub funnel-goal deletion ebdc4333 Extend the prompt in case of funnel-goal deletion 639c6238 fixup aa59adeb wip * Split new JS LiveView additions * Put funnels behind a feature flag * Integrate dashboard feature toggle * Update signing salt for live view * Update moduledocs * Update live reloader config * Use Phoenix.HTML.Safe for goal names * Workaround to get flashes working in embedded liveview * Keep feature toggles idempotent, rename property to setting We'll still retain the ability to flip bools on a lower level. * Update moduledocs * Make live flash disappear after 5s * Tailwind: purge .heex files too * Update docs link * Add live components to tailwind purge config * Update another flaky test Ref f0bdf872 cc @vinibrsl * Fix combobox input length w/ WebKit * Intoduce generic notice component * Revert "Fix combobox input length w/ WebKit" This reverts commit 3c653a6d85d5000167631e10ef45a93c13b41ed1. * Fix combobox input length on webkit * Make whole combobox item clickable, not only text * Fix glitch moving Save button on activation * Tweak dark mode * Show funnel form without waiting for funnel name input * Tweak dark mode * Include static Phoenix components in tailwind purge * Tune funnels form into a liveview of its own This is so that ComboBoxes can publish their selections and unavailable choices can be propagated to other siblings. * Push less data over websocket * Undo Lsp/formatter race condition * Fixup typespecs * Bust CI cache
3 years ago
field :conversions_enabled, :boolean, default: true
field :props_enabled, :boolean, default: true
field :funnels_enabled, :boolean, default: true
field :ingest_rate_limit_scale_seconds, :integer, default: 60
# default is set via changeset/2
field :ingest_rate_limit_threshold, :integer
field :domain_changed_from, :string
field :domain_changed_at, :naive_datetime
embeds_one :imported_data, Plausible.Site.ImportedData, on_replace: :update
many_to_many :members, User, join_through: Plausible.Site.Membership
has_many :memberships, Plausible.Site.Membership
has_many :invitations, Plausible.Auth.Invitation
has_many :goals, Plausible.Goal, preload_order: [desc: :id]
has_many :revenue_goals, Plausible.Goal, where: [currency: {:not, nil}]
has_one :google_auth, GoogleAuth
has_one :weekly_report, Plausible.Site.WeeklyReport
has_one :monthly_report, Plausible.Site.MonthlyReport
has_one :spike_notification, Plausible.Site.SpikeNotification
has_one :ownership, Plausible.Site.Membership, where: [role: :owner]
has_one :owner, through: [:ownership, :user]
# If `from_cache?` is set, the struct might be incomplete - see `Plausible.Site.Cache`.
# Use `Plausible.Repo.reload!(cached_site)` to pre-fill missing fields if
# strictly necessary.
field :from_cache?, :boolean, virtual: true, default: false
# Used in the context of paginated sites list to order in relation to
Implement pinned sites (#3469) * Revert "Remove site pins for now" This reverts commit 5eccf4eaf62508432d53c7557f79650ced0e6516. * Implement basic site pin schema level logic within user specific preferences * Add vertical ellipsis menu markup * Implement basic changesets for user preferences * Implement pin toggling * Try to fix pin sorting * Implement pin toggling in LV * Adjust moduledocs for new schema(s) * Remove unnecessary `distinct` from query * Use `button` for pin/unpin action * Generalize preference setting * Rename schema and fields for clarity * Rename `list_type` -> `entry_type` * Safeguard setting options * Test `set_option/4` and `toggle_pin/2` * Add test for listing pinned sites via `Sites.list` * Disallow pinning sites outside page explicitly * Test pinning in LV * Test conditional rendering of site settings in /sites * Remove unnecessary TODO comment * Safeguard `Sites.set_option/4` against invalid user/site combo * Handle pinned sites in dashboard site picker * Clear flashes upon (un)pinning sites * Update CHANGELOG * Prevent blinking of hamburger menu items on first paint * Highlight hamburger handle on hover in /sites * Start showing hotkeys in site picker again * Sort pinned sites in the order they were pinned * Update sites list order immediately after pin/unpin toggle * Refactor and split `Sites.list/3`, extracting `Sites.list_with_invitations/3` * Cap number of pinned sites at 9 per user * First pass on visual indication of site cards (dis)appearing * Apply ellipsis gradient+shadow on card hover * Fix responsive padding of site cards * Sort by invitations first, pinned sites second and then the rest * Revert "Apply ellipsis gradient+shadow on card hover" This reverts commit 0608796612639030ccbb12df639709f78edc1434. * Apply more subtle hover effect on the ellipsis menu * Make error and success flash LV boxes use separate component containers * Promote `pinned_at` in table migration to a column * Switch logic to using `pinned_at` as a standard schema field * Refactor `Sites.list*` getting rid of subquery (h/t @ukutaht) * Remove migration which is already merged upstream --------- Co-authored-by: Adam Rutkowski <hq@mtod.org>
3 years ago
# user's membership state. Currently it can be either "invitation",
# "pinned_site" or "site", where invitations are first.
field :entry_type, :string, virtual: true
field :pinned_at, :naive_datetime, virtual: true
Implement new sites view (#3463) * Implement complete basics of LV sites * Reimplement everything in LV except pagination * Implement basic search capability * PoC: plot visitors on sites index * Add rudimentary clipped gradient in minicharts * Fix clipping gradient, define once * Format * Add moduledoc to visitors component * Move paginator helpers to the top core namespace * Fix typespec of `Plausible.Sites.list` * Split sites component into subcomponents * Add function to uniformly calculate 24h intervals and visitor totals across multiple sites. * Integrate batch 24h interval query with plots on sites view * Don't confuse heex compiler with alpine @ shorthands * Make linear gradient svg definition truly invisible * Implement basic pagination * Extract `site_stats` from site and invitation cards * Improve pagination * Tweak css * Improve filtering on pagination and make WSS fail graceful * Test `last_24h_visitors_hourly_intervals/2` * Replace /sites with LV implementation * Add debounce to search filter * Fix typespecs * Fix styling * Fix mini graph scaling factor calculation * Fix search consuming itself * Minimal tweaks to the plots * Fixup * Remove magic numbers from the plot * Create `site_pins` table * Add `SitePin` schema * Implement listing invitations, sites and pins in a single query * Add FIXME note * Remove site pins for now * Add tests for `Plausible.Sites.list/3` * Add a couple more tests to sites dead view * Remove unnecessary FIXME * Add LV tests for Sites * Calculate and display 24h visitors change * Render the change in bold * Add clarfying comment on virtual field in `Site` schema * Remove unnecessary function from Invitations API * Remove unused list opt from type definition in `Sites` * Improve joins in list query slightly * Add comment on manually computing sites list total * Start searching from a singly character in domain field * Add typespec to `last_24h_visitors_hourly_intervals` * Extend moduledoc in visitors component * Simplify loading sites in LV * Simplify assigns in LV * Add missing group for shadow under site card * Make invitation modal render * Make HTML in sites LV semantically correct * Remove autofocus and focus search on `/` * Remove shadow from search input * Make search cancel on escape * Fix tests relying on outdated HTML structure * Make visitor chart color scheme consistent with dashboard chart * Update styling of trend labels * Fix empty state and improve search blur/focus handling * Use live navigation for pagination * Implement spinner on load from search * Remove unused `Plausible.Stats.Clickhouse.last_24h_visitors/1` * Calculate uniques correctly across hour boundaries * Swap inlined svg for Heroicons component in invitation modal * Add order by to base query in 24h hourly intervals * Revert "Add order by to base query in 24h hourly intervals" This reverts commit a6be5e302662b3a90f68da0e3fdfa6ff271c168e. * Query clickhouse 24h visitors only on second mount * Remove redundant sign from percentage change when negative * Switch to offset-based pagination - offset seems easier to deal with for when actions on paginated list will be performed such as site pinning; tracking cursor data makes some entries disappear in edge cases. The data set is still fairly small and static, even for large customers. - we're removing Phoenix.Pagination as it doesn't really fir any use case, and it was only used to limit the number of sites in the site picker - site picker is now limited to 9 sites (future: pinned sites will be prioritized there) - no need to re-query for total count any more - BTW, the old /sites template was removed * Refine the plot queries; Tests pass snapshot * Add PromEx plugin for LiveView * Fix tiny plot cut-off at the top --------- Co-authored-by: Adam Rutkowski <hq@mtod.org>
3 years ago
Add multiple imports per site (#3724) * Clean up references to no longer active `google_analytics_imports` Oban queue * Stub CSV importer * Add SiteImport schema * Rename `Plausible.Imported` module file to match module name * Add `import_id` column to `Imported.*` CH schemas * Implement Importer behavior and manage imports state using new entities * Implement importer callbacks and maintain site.imported_data for UA * Keep imports in sync when forgetting all imports * Scope imported data queries to completed import IDs * Mark newly imported data with respective import ID * Clean up Importer implementation a bit * Test querying legacy and new imported data * Send Oban notifications on import worker failure too * Fix checking for forgettable imports and remove redundant function * Fix UA integration test * Change site import source to atom enum and add source label * Add typespecs and reduce repetition in `Plausible.Imported` * Improve documentation and typespecs * Add test for purging particular import * Switch email notification templates depending on import source * Document running import synchronously * Fix UA importer args parsing and ensure it's covered by tests * Clear `site.stats_start_date` on complete import to force recalculation * Test Oban notifications (h/t @ruslandoga) * Purge stats on import failure right away to reduce a chance of leaving debris behind * Fix typos Co-authored-by: hq1 <hq@mtod.org> * Fix another typo * Refactor fetching earliest import and earliest stats start date * Use `Date.after?` instead of `Timex.after?` * Cache import data in site virtual fields and limit queried imports to 5 * Ensure always current `stats_start_date` is used * Work around broken typespec in Timex * Make `SiteController.forget_imported` action idempotent * Discard irrecoverably failed import tasks * Use macros for site import statuses There's also a fix ensuring only complete imports are considered where relevant - couldn't isolate it as it was in a common hunk * Use `import_id` as worker job uniqueness criterion * Do not load imported stats data in plugins API context --------- Co-authored-by: hq1 <hq@mtod.org>
3 years ago
# Used for caching imports data for the duration of the whole request
# to avoid multiple identical fetches. Populated by plugs putting
# `site` in `assigns`.
field :import_data_loaded, :boolean, default: false, virtual: true
field :earliest_import_start_date, :date, virtual: true
field :latest_import_end_date, :date, virtual: true
Add multiple imports per site (#3724) * Clean up references to no longer active `google_analytics_imports` Oban queue * Stub CSV importer * Add SiteImport schema * Rename `Plausible.Imported` module file to match module name * Add `import_id` column to `Imported.*` CH schemas * Implement Importer behavior and manage imports state using new entities * Implement importer callbacks and maintain site.imported_data for UA * Keep imports in sync when forgetting all imports * Scope imported data queries to completed import IDs * Mark newly imported data with respective import ID * Clean up Importer implementation a bit * Test querying legacy and new imported data * Send Oban notifications on import worker failure too * Fix checking for forgettable imports and remove redundant function * Fix UA integration test * Change site import source to atom enum and add source label * Add typespecs and reduce repetition in `Plausible.Imported` * Improve documentation and typespecs * Add test for purging particular import * Switch email notification templates depending on import source * Document running import synchronously * Fix UA importer args parsing and ensure it's covered by tests * Clear `site.stats_start_date` on complete import to force recalculation * Test Oban notifications (h/t @ruslandoga) * Purge stats on import failure right away to reduce a chance of leaving debris behind * Fix typos Co-authored-by: hq1 <hq@mtod.org> * Fix another typo * Refactor fetching earliest import and earliest stats start date * Use `Date.after?` instead of `Timex.after?` * Cache import data in site virtual fields and limit queried imports to 5 * Ensure always current `stats_start_date` is used * Work around broken typespec in Timex * Make `SiteController.forget_imported` action idempotent * Discard irrecoverably failed import tasks * Use macros for site import statuses There's also a fix ensuring only complete imports are considered where relevant - couldn't isolate it as it was in a common hunk * Use `import_id` as worker job uniqueness criterion * Do not load imported stats data in plugins API context --------- Co-authored-by: hq1 <hq@mtod.org>
3 years ago
field :complete_import_ids, {:array, :integer}, default: [], virtual: true
timestamps()
end
def new(params), do: changeset(%__MODULE__{}, params)
@domain_unique_error """
This domain cannot be registered. Perhaps one of your colleagues registered it? If that's not the case, please contact support@plausible.io
"""
def changeset(site, attrs \\ %{}) do
site
|> cast(attrs, [:domain, :timezone])
|> clean_domain()
|> validate_required([:domain, :timezone])
|> validate_timezone()
|> validate_domain_format()
|> validate_domain_reserved_characters()
|> unique_constraint(:domain,
message: @domain_unique_error
)
|> unique_constraint(:domain,
name: "domain_change_disallowed",
message: @domain_unique_error
)
|> put_change(
:ingest_rate_limit_threshold,
Application.get_env(:plausible, __MODULE__)[:default_ingest_threshold]
)
end
def update_changeset(site, attrs \\ %{}, opts \\ []) do
at =
opts
|> Keyword.get(:at, NaiveDateTime.utc_now())
|> NaiveDateTime.truncate(:second)
site
|> changeset(attrs)
|> handle_domain_change(at)
end
def crm_changeset(site, attrs) do
site
|> cast(attrs, [
:timezone,
:public,
:stats_start_date,
:ingest_rate_limit_threshold,
:ingest_rate_limit_scale_seconds
])
|> validate_required([:timezone, :public])
|> validate_number(:ingest_rate_limit_scale_seconds,
greater_than_or_equal_to: 1,
message: "must be at least 1 second"
)
|> validate_number(:ingest_rate_limit_threshold,
greater_than_or_equal_to: 0,
message: "must be empty, zero or positive"
)
end
def tz_offset(site, utc_now \\ DateTime.utc_now()) do
case DateTime.shift_zone(utc_now, site.timezone) do
{:ok, datetime} ->
datetime.utc_offset + datetime.std_offset
res ->
Sentry.capture_message("Unable to determine timezone offset for",
extra: %{site: site, result: res}
)
0
end
end
def make_public(site) do
change(site, public: true)
end
def make_private(site) do
change(site, public: false)
end
def set_stats_start_date(site, val) do
change(site, stats_start_date: val)
end
def set_native_stats_start_at(site, val) do
change(site, native_stats_start_at: val)
end
def start_import(site, start_date, end_date, imported_source, status \\ "importing") do
change(site,
imported_data: %{
start_date: start_date,
end_date: end_date,
source: imported_source,
status: status
}
)
end
def import_success(site) do
change(site,
imported_data: %{status: "ok"}
)
end
def import_failure(site) do
change(site, imported_data: %{status: "error"})
end
def remove_imported_data(site) do
change(site, imported_data: nil)
end
defp clean_domain(changeset) do
clean_domain =
(get_field(changeset, :domain) || "")
|> String.downcase()
|> String.trim()
|> String.replace_leading("http://", "")
|> String.replace_leading("https://", "")
|> String.trim("/")
|> String.replace_leading("www.", "")
change(changeset, %{domain: clean_domain})
end
# https://tools.ietf.org/html/rfc3986#section-2.2
@uri_reserved_chars ~w(: ? # [ ] @ ! $ & ' \( \) * + , ; =)
defp validate_domain_reserved_characters(changeset) do
domain = get_field(changeset, :domain) || ""
if String.contains?(domain, @uri_reserved_chars) do
add_error(
changeset,
:domain,
"must not contain URI reserved characters #{@uri_reserved_chars}"
)
else
changeset
end
end
defp validate_domain_format(changeset) do
validate_format(changeset, :domain, ~r/^[-\.\\\/:\p{L}\d]*$/u,
message: "only letters, numbers, slashes and period allowed"
)
end
defp handle_domain_change(changeset, at) do
new_domain = get_change(changeset, :domain)
if new_domain do
changeset
|> put_change(:domain_changed_from, changeset.data.domain)
|> put_change(:domain_changed_at, at)
|> unique_constraint(:domain,
name: "domain_change_disallowed",
message: @domain_unique_error
)
|> unique_constraint(:domain_changed_from,
message: @domain_unique_error
)
else
changeset
end
end
defp validate_timezone(changeset) do
tz = get_field(changeset, :timezone)
if Timex.is_valid_timezone?(tz) do
changeset
else
add_error(changeset, :timezone, "is invalid")
end
end
end
defimpl FunWithFlags.Actor, for: Plausible.Site do
def id(%{domain: domain}) do
"site:#{domain}"
end
end