Introduce initial referrer and initial referrer source (#32)

* Introduce initial referrer and initial referrer source

* Add test
pull/33/head
Uku Taht 7 years ago committed by GitHub
parent 4861248c3f
commit ed3110a6ea
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23

@ -47,20 +47,29 @@
return window.location.protocol + '//' + window.location.hostname + window.location.pathname + window.location.search;
}
function getSourceFromQueryParam() {
const result = window.location.search.match(/[?&](ref|source|utm_source)=([^?&]+)/);
return result ? result[2] : null
}
function getUserData() {
var userData = JSON.parse(getCookie('plausible_user'))
if (userData) {
userData.new_visitor = false
userData.user_agent = decodeURIComponent(userData.user_agent)
userData.referrer = decodeURIComponent(userData.referrer)
if (userData.referrer) {
userData.initial_referrer = decodeURIComponent(userData.referrer)
} else {
userData.initial_referrer = decodeURIComponent(userData.initial_referrer)
userData.initial_source = decodeURIComponent(userData.initial_source)
}
return userData
} else {
return {
uid: pseudoUUIDv4(),
new_visitor: true,
user_agent: window.navigator.userAgent,
referrer: window.document.referrer,
initial_referrer: window.document.referrer,
initial_source: getSourceFromQueryParam(),
screen_width: window.innerWidth
}
}
@ -70,7 +79,8 @@
setCookie('plausible_user', JSON.stringify({
uid: payload.uid,
user_agent: encodeURIComponent(payload.user_agent),
referrer: encodeURIComponent(payload.referrer),
initial_referrer: encodeURIComponent(payload.initial_referrer),
initial_source: encodeURIComponent(payload.initial_source),
screen_width: payload.screen_width
}))
}
@ -84,6 +94,9 @@
payload.name = eventName
payload.url = getUrl()
payload.domain = CONFIG['domain']
payload.referrer = window.document.referrer
payload.user_agent = window.navigator.userAgent
payload.source = getSourceFromQueryParam()
var request = new XMLHttpRequest();
request.open('POST', plausibleHost + '/api/event', true);

@ -13,6 +13,8 @@ defmodule Plausible.Event do
field :referrer, :string
field :referrer_source, :string
field :initial_referrer, :string
field :initial_referrer_source, :string
field :country_code, :string
field :screen_size, :string
field :operating_system, :string
@ -23,7 +25,7 @@ defmodule Plausible.Event do
def changeset(pageview, attrs) do
pageview
|> cast(attrs, [:name, :domain, :hostname, :pathname, :referrer, :new_visitor, :user_id, :fingerprint, :operating_system, :browser, :referrer_source, :country_code, :screen_size])
|> cast(attrs, [:name, :domain, :hostname, :pathname, :new_visitor, :user_id, :fingerprint, :operating_system, :browser, :referrer, :referrer_source, :initial_referrer, :initial_referrer_source, :country_code, :screen_size])
|> validate_required([:name, :domain, :hostname, :pathname, :new_visitor, :user_id, :fingerprint])
end
end

@ -127,7 +127,6 @@ defmodule Plausible.Stats do
sessions_query = from(s in Plausible.Session,
where: s.domain == ^site.domain,
where: s.new_visitor,
where: s.start >= ^first_datetime and s.start < ^last_datetime
)
total_sessions = Repo.one( from s in sessions_query, select: count(s))
@ -139,16 +138,6 @@ defmodule Plausible.Stats do
end
end
def session_length(site, query) do
{first_datetime, last_datetime} = date_range_utc_boundaries(query.date_range, site.timezone)
Repo.one(from s in Plausible.Session,
where: s.domain == ^site.domain,
where: s.start >= ^first_datetime and s.start < ^last_datetime,
select: coalesce(avg(s.length), 0)
) |> Decimal.round |> Decimal.to_integer
end
def pageviews_and_visitors(site, query) do
Repo.one(from(
e in base_query(site, query),
@ -165,9 +154,9 @@ defmodule Plausible.Stats do
def top_referrers_for_goal(site, query, limit \\ 5) do
Repo.all(from e in base_query(site, query),
select: %{name: e.referrer_source, count: count(e.user_id, :distinct)},
group_by: e.referrer_source,
where: not is_nil(e.referrer_source),
select: %{name: e.initial_referrer_source, count: count(e.user_id, :distinct)},
group_by: e.initial_referrer_source,
where: not is_nil(e.initial_referrer_source),
order_by: [desc: 2],
limit: ^limit
)
@ -175,10 +164,9 @@ defmodule Plausible.Stats do
def top_referrers(site, query, limit \\ 5, include \\ []) do
referrers = Repo.all(from e in base_query(site, query),
select: %{name: e.referrer_source, count: count(e)},
select: %{name: e.referrer_source, count: count(e.user_id, :distinct)},
group_by: e.referrer_source,
where: not is_nil(e.referrer_source),
where: e.new_visitor,
order_by: [desc: 2],
limit: ^limit
)
@ -200,7 +188,6 @@ defmodule Plausible.Stats do
total_sessions_by_referrer = Repo.all(
from s in Plausible.Session,
where: s.domain == ^site.domain,
where: s.new_visitor,
where: s.start >= ^first_datetime and s.start < ^last_datetime,
where: s.referrer_source in ^referrers,
group_by: s.referrer_source,
@ -210,7 +197,6 @@ defmodule Plausible.Stats do
bounced_sessions_by_referrer = Repo.all(
from s in Plausible.Session,
where: s.domain == ^site.domain,
where: s.new_visitor,
where: s.start >= ^first_datetime and s.start < ^last_datetime,
where: s.is_bounce,
where: s.referrer_source in ^referrers,
@ -243,17 +229,16 @@ defmodule Plausible.Stats do
Repo.one(
from e in base_query(site, query),
select: count(e.user_id, :distinct),
where: e.referrer_source == ^referrer
where: e.initial_referrer_source == ^referrer
)
end
def referrer_drilldown(site, query, referrer, include \\ []) do
referring_urls = Repo.all(
from e in base_query(site, query),
select: %{name: e.referrer, count: count(e)},
select: %{name: e.referrer, count: count(e.user_id, :distinct)},
group_by: e.referrer,
where: e.referrer_source == ^referrer,
where: e.new_visitor,
order_by: [desc: 2],
limit: 100
)
@ -287,9 +272,9 @@ defmodule Plausible.Stats do
def referrer_drilldown_for_goal(site, query, referrer) do
Repo.all(
from e in base_query(site, query),
select: %{name: e.referrer, count: count(e.user_id, :distinct)},
group_by: e.referrer,
where: e.referrer_source == ^referrer,
select: %{name: e.initial_referrer, count: count(e.user_id, :distinct)},
group_by: e.initial_referrer,
where: e.initial_referrer_source == ^referrer,
order_by: [desc: 2],
limit: 100
)
@ -301,7 +286,6 @@ defmodule Plausible.Stats do
total_sessions_by_url = Repo.all(
from s in Plausible.Session,
where: s.domain == ^site.domain,
where: s.new_visitor,
where: s.start >= ^first_datetime and s.start < ^last_datetime,
where: s.referrer in ^referring_urls,
group_by: s.referrer,
@ -311,7 +295,6 @@ defmodule Plausible.Stats do
bounced_sessions_by_url = Repo.all(
from s in Plausible.Session,
where: s.domain == ^site.domain,
where: s.new_visitor,
where: s.start >= ^first_datetime and s.start < ^last_datetime,
where: s.is_bounce,
where: s.referrer in ^referring_urls,
@ -356,7 +339,6 @@ defmodule Plausible.Stats do
total_sessions_by_url = Repo.all(
from s in Plausible.Session,
where: s.domain == ^site.domain,
where: s.new_visitor,
where: s.start >= ^first_datetime and s.start < ^last_datetime,
where: s.entry_page in ^page_urls,
group_by: s.entry_page,
@ -366,7 +348,6 @@ defmodule Plausible.Stats do
bounced_sessions_by_url = Repo.all(
from s in Plausible.Session,
where: s.domain == ^site.domain,
where: s.new_visitor,
where: s.start >= ^first_datetime and s.start < ^last_datetime,
where: s.is_bounce,
where: s.entry_page in ^page_urls,

@ -14,7 +14,7 @@ defmodule PlausibleWeb.Api.ExternalController do
{:error, changeset} ->
request = Sentry.Plug.build_request_interface_data(conn, [])
Sentry.capture_message("Error processing event", extra: %{errors: inspect(changeset.errors), params: params, request: request})
Logger.error("Error processing event: #{inspect(changeset)}")
Logger.info("Error processing event: #{inspect(changeset)}")
conn |> send_resp(400, "")
end
end
@ -42,10 +42,8 @@ defmodule PlausibleWeb.Api.ExternalController do
UAInspector.Parser.parse(user_agent)
end
ref = params["referrer"]
ref = if ref && strip_www(URI.parse(ref).host) !== strip_www(uri.host) && URI.parse(ref).host !== "localhost" do
RefInspector.parse(ref)
end
ref = parse_referrer(uri, params["referrer"])
initial_ref = parse_referrer(uri, params["initial_referrer"])
event_attrs = %{
name: params["name"],
@ -58,8 +56,10 @@ defmodule PlausibleWeb.Api.ExternalController do
fingerprint: calculate_fingerprint(conn, params),
operating_system: ua && os_name(ua),
browser: ua && browser_name(ua),
referrer_source: ref && referrer_source(uri, ref),
referrer: ref && clean_referrer(params["referrer"]),
referrer_source: params["source"] || referrer_source(ref),
referrer: clean_referrer(ref),
initial_referrer_source: params["initial_source"] || referrer_source(initial_ref),
initial_referrer: clean_referrer(initial_ref),
screen_size: calculate_screen_size(params["screen_width"])
}
@ -68,6 +68,15 @@ defmodule PlausibleWeb.Api.ExternalController do
end
end
defp parse_referrer(_, nil), do: nil
defp parse_referrer(uri, referrer_str) do
referrer_uri = URI.parse(referrer_str)
if strip_www(referrer_uri.host) !== strip_www(uri.host) && referrer_uri.host !== "localhost" do
RefInspector.parse(referrer_str)
end
end
defp calculate_fingerprint(conn, params) do
user_agent = List.first(Plug.Conn.get_req_header(conn, "user-agent")) || ""
ip_address = List.first(Plug.Conn.get_req_header(conn, "cf-connecting-ip")) || ""
@ -84,10 +93,9 @@ defmodule PlausibleWeb.Api.ExternalController do
defp calculate_screen_size(width) when width < 1440, do: "Laptop"
defp calculate_screen_size(width) when width >= 1440, do: "Desktop"
defp clean_referrer(referrer) do
uri = if referrer do
URI.parse(String.trim_trailing(referrer, "/"))
end
defp clean_referrer(nil), do: nil
defp clean_referrer(ref) do
uri = URI.parse(String.trim_trailing(ref.referer, "/"))
if uri && uri.scheme in ["http", "https"] do
host = String.replace_prefix(uri.host, "www.", "")
@ -123,10 +131,11 @@ defmodule PlausibleWeb.Api.ExternalController do
end
end
defp referrer_source(uri, ref) do
defp referrer_source(nil), do: nil
defp referrer_source(ref) do
case ref.source do
:unknown ->
query_param_source(uri) || clean_uri(ref.referer)
clean_uri(ref.referer)
source ->
source
end
@ -138,17 +147,4 @@ defmodule PlausibleWeb.Api.ExternalController do
String.replace_leading(uri.host, "www.", "")
end
end
@source_query_params ["ref", "utm_source", "source"]
defp query_param_source(uri) do
if uri && uri.query do
Enum.find_value(URI.query_decoder(uri.query), fn {key, val} ->
if Enum.member?(@source_query_params, key) do
val
end
end)
end
end
end

@ -0,0 +1,13 @@
defmodule Plausible.Repo.Migrations.AddInitialSourceAndReferrerToEvents do
use Ecto.Migration
def change do
alter table(:events) do
add :initial_referrer, :text
add :initial_referrer_source, :text
end
execute "UPDATE events SET initial_referrer=referrer, initial_referrer_source=referrer_source"
execute "UPDATE events SET referrer=null, referrer_source=null WHERE new_visitor=false"
end
end

@ -141,6 +141,7 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do
name: "pageview",
url: "http://gigride.live/",
referrer: "https://facebook.com",
initial_referrer: "https://facebook.com",
new_visitor: false,
uid: UUID.uuid4()
}
@ -155,6 +156,7 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do
assert response(conn, 202) == ""
assert pageview.referrer_source == "Facebook"
assert pageview.initial_referrer_source == "Facebook"
end
test "strips trailing slash from referrer", %{conn: conn} do
@ -162,6 +164,7 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do
name: "pageview",
url: "http://gigride.live/",
referrer: "https://facebook.com/page/",
initial_referrer: "https://facebook.com/page/",
new_visitor: false,
uid: UUID.uuid4()
}
@ -176,7 +179,9 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do
assert response(conn, 202) == ""
assert pageview.referrer == "facebook.com/page"
assert pageview.initial_referrer == "facebook.com/page"
assert pageview.referrer_source == "Facebook"
assert pageview.initial_referrer_source == "Facebook"
end
test "ignores when referrer is internal", %{conn: conn} do
@ -184,6 +189,7 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do
name: "pageview",
url: "http://gigride.live/",
referrer: "https://gigride.live",
initial_referrer: "https://gigride.live",
new_visitor: false,
uid: UUID.uuid4()
}
@ -198,6 +204,7 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do
assert response(conn, 202) == ""
assert pageview.referrer_source == nil
assert pageview.initial_referrer_source == nil
end
test "ignores localhost referrer", %{conn: conn} do
@ -205,6 +212,7 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do
name: "pageview",
url: "http://gigride.live/",
referrer: "http://localhost:4000/",
initial_referrer: "http://localhost:4000/",
new_visitor: true,
uid: UUID.uuid4()
}
@ -219,6 +227,7 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do
assert response(conn, 202) == ""
assert pageview.referrer_source == nil
assert pageview.initial_referrer_source == nil
end
test "parses subdomain referrer", %{conn: conn} do
@ -226,6 +235,7 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do
name: "pageview",
url: "http://gigride.live/",
referrer: "https://blog.gigride.live",
initial_referrer: "https://blog.gigride.live",
new_visitor: false,
uid: UUID.uuid4()
}
@ -240,6 +250,7 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do
assert response(conn, 202) == ""
assert pageview.referrer_source == "blog.gigride.live"
assert pageview.initial_referrer_source == "blog.gigride.live"
end
test "referrer is cleaned", %{conn: conn} do
@ -247,6 +258,7 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do
name: "pageview",
url: "http://www.example.com/",
referrer: "https://www.indiehackers.com/page?query=param#hash",
initial_referrer: "https://www.indiehackers.com/page?query=param#hash",
uid: UUID.uuid4(),
new_visitor: true
}
@ -259,32 +271,16 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do
finalize_session(pageview.user_id)
assert pageview.referrer == "indiehackers.com/page"
assert pageview.initial_referrer == "indiehackers.com/page"
end
test "?ref= query param controls the referrer source", %{conn: conn} do
test "source param controls the referrer source", %{conn: conn} do
params = %{
name: "pageview",
url: "http://www.example.com/?wat=wet&ref=traffic-source",
referrer: "https://www.indiehackers.com/page",
uid: UUID.uuid4(),
new_visitor: true
}
conn
|> put_req_header("content-type", "text/plain")
|> post("/api/event", Jason.encode!(params))
pageview = Repo.one(Plausible.Event)
finalize_session(pageview.user_id)
assert pageview.referrer_source == "traffic-source"
end
test "?utm_source= query param controls the referrer source", %{conn: conn} do
params = %{
name: "pageview",
url: "http://www.example.com/?wat=wet&utm_source=traffic-source",
referrer: "https://www.indiehackers.com/page",
url: "http://www.example.com/",
referrer: "https://betalist.com/my-produxct",
source: "betalist",
initial_source: "betalist",
uid: UUID.uuid4(),
new_visitor: true
}
@ -296,7 +292,8 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do
pageview = Repo.one(Plausible.Event)
finalize_session(pageview.user_id)
assert pageview.referrer_source == "traffic-source"
assert pageview.referrer_source == "betalist"
assert pageview.initial_referrer_source == "betalist"
end
test "if it's an :unknown referrer, just the domain is used", %{conn: conn} do
@ -403,4 +400,18 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do
assert response(conn, 202) == ""
assert event.name == "custom event"
end
test "responds 400 when required fields are missing", %{conn: conn} do
params = %{
name: "pageview",
url: "http://gigride.live/",
}
conn = conn
|> put_req_header("content-type", "text/plain")
|> put_req_header("user-agent", @user_agent)
|> post("/api/event", Jason.encode!(params))
assert response(conn, 400) == ""
end
end

@ -5,11 +5,11 @@ defmodule PlausibleWeb.Api.StatsController.ReferrersTest do
describe "GET /api/stats/:domain/referrers" do
setup [:create_user, :log_in, :create_site]
test "returns top referrer sources by new visitors", %{conn: conn, site: site} do
insert(:pageview, domain: site.domain, referrer_source: "Google", new_visitor: true, timestamp: ~N[2019-01-01 01:00:00])
insert(:pageview, domain: site.domain, referrer_source: "Google", new_visitor: false, timestamp: ~N[2019-01-01 02:00:00])
insert(:pageview, domain: site.domain, referrer_source: "Google", new_visitor: true, timestamp: ~N[2019-01-01 02:00:00])
insert(:pageview, domain: site.domain, referrer_source: "Bing", new_visitor: true, timestamp: ~N[2019-01-01 02:00:00])
test "returns top referrer sources by user ids", %{conn: conn, site: site} do
pageview1 = insert(:pageview, domain: site.domain, referrer_source: "Google", timestamp: ~N[2019-01-01 01:00:00])
insert(:pageview, domain: site.domain, referrer_source: "Google", user_id: pageview1.user_id, timestamp: ~N[2019-01-01 02:00:00])
insert(:pageview, domain: site.domain, referrer_source: "Google", timestamp: ~N[2019-01-01 02:00:00])
insert(:pageview, domain: site.domain, referrer_source: "Bing", timestamp: ~N[2019-01-01 02:00:00])
conn = get(conn, "/api/stats/#{site.domain}/referrers?period=day&date=2019-01-01")
@ -40,9 +40,9 @@ defmodule PlausibleWeb.Api.StatsController.ReferrersTest do
setup [:create_user, :log_in, :create_site]
test "returns top referrers for a custom goal", %{conn: conn, site: site} do
insert(:event, name: "Signup", domain: site.domain, referrer_source: "Google", timestamp: ~N[2019-01-01 01:00:00])
insert(:event, name: "Signup", domain: site.domain, referrer_source: "Google", timestamp: ~N[2019-01-01 02:00:00])
insert(:pageview, domain: site.domain, referrer_source: "Google", timestamp: ~N[2019-01-01 02:00:00])
insert(:event, name: "Signup", domain: site.domain, initial_referrer_source: "Google", timestamp: ~N[2019-01-01 01:00:00])
insert(:event, name: "Signup", domain: site.domain, initial_referrer_source: "Google", timestamp: ~N[2019-01-01 02:00:00])
insert(:pageview, domain: site.domain, initial_referrer_source: "Google", timestamp: ~N[2019-01-01 02:00:00])
filters = Jason.encode!(%{goal: "Signup"})
conn = get(conn, "/api/stats/#{site.domain}/goal/referrers?period=day&date=2019-01-01&filters=#{filters}")
@ -53,9 +53,9 @@ defmodule PlausibleWeb.Api.StatsController.ReferrersTest do
end
test "returns top referrers for a pageview goal", %{conn: conn, site: site} do
insert(:pageview, pathname: "/register", domain: site.domain, referrer_source: "Google", timestamp: ~N[2019-01-01 01:00:00])
insert(:pageview, pathname: "/register", domain: site.domain, referrer_source: "Google", timestamp: ~N[2019-01-01 01:00:00])
insert(:pageview, pathname: "/irrelevant", domain: site.domain, referrer_source: "Google", timestamp: ~N[2019-01-01 02:00:00])
insert(:pageview, pathname: "/register", domain: site.domain, initial_referrer_source: "Google", timestamp: ~N[2019-01-01 01:00:00])
insert(:pageview, pathname: "/register", domain: site.domain, initial_referrer_source: "Google", timestamp: ~N[2019-01-01 01:00:00])
insert(:pageview, pathname: "/irrelevant", domain: site.domain, initial_referrer_source: "Google", timestamp: ~N[2019-01-01 02:00:00])
filters = Jason.encode!(%{goal: "Visit /register"})
conn = get(conn, "/api/stats/#{site.domain}/goal/referrers?period=day&date=2019-01-01&filters=#{filters}")
@ -159,9 +159,9 @@ defmodule PlausibleWeb.Api.StatsController.ReferrersTest do
setup [:create_user, :log_in, :create_site]
test "returns top referring urls for a custom goal", %{conn: conn, site: site} do
insert(:event, name: "Signup", domain: site.domain, referrer_source: "Twitter", referrer: "a", timestamp: ~N[2019-01-01 01:00:00])
insert(:event, name: "Signup", domain: site.domain, referrer_source: "Twitter", referrer: "a", timestamp: ~N[2019-01-01 02:00:00])
insert(:event, name: "Signup", domain: site.domain, referrer_source: "Twitter", referrer: "b", timestamp: ~N[2019-01-01 02:00:00])
insert(:event, name: "Signup", domain: site.domain, initial_referrer_source: "Twitter", initial_referrer: "a", timestamp: ~N[2019-01-01 01:00:00])
insert(:event, name: "Signup", domain: site.domain, initial_referrer_source: "Twitter", initial_referrer: "a", timestamp: ~N[2019-01-01 02:00:00])
insert(:event, name: "Signup", domain: site.domain, initial_referrer_source: "Twitter", initial_referrer: "b", timestamp: ~N[2019-01-01 02:00:00])
filters = Jason.encode!(%{goal: "Signup"})
conn = get(conn, "/api/stats/#{site.domain}/goal/referrers/Twitter?period=day&date=2019-01-01&filters=#{filters}")
@ -176,16 +176,20 @@ defmodule PlausibleWeb.Api.StatsController.ReferrersTest do
end
test "returns top referring urls for a pageview goal", %{conn: conn, site: site} do
insert(:pageview, pathname: "/register", domain: site.domain, referrer_source: "Google", timestamp: ~N[2019-01-01 01:00:00])
insert(:pageview, pathname: "/register", domain: site.domain, referrer_source: "Google", timestamp: ~N[2019-01-01 01:00:00])
insert(:pageview, pathname: "/irrelevant", domain: site.domain, referrer_source: "Google", timestamp: ~N[2019-01-01 02:00:00])
insert(:pageview, pathname: "/register", domain: site.domain, initial_referrer_source: "Twitter", initial_referrer: "a", timestamp: ~N[2019-01-01 01:00:00])
insert(:pageview, pathname: "/register", domain: site.domain, initial_referrer_source: "Twitter", initial_referrer: "b", timestamp: ~N[2019-01-01 01:00:00])
insert(:pageview, pathname: "/irrelevant", domain: site.domain, initial_referrer_source: "Twitter", initial_referrer: "c", timestamp: ~N[2019-01-01 02:00:00])
filters = Jason.encode!(%{goal: "Visit /register"})
conn = get(conn, "/api/stats/#{site.domain}/goal/referrers?period=day&date=2019-01-01&filters=#{filters}")
conn = get(conn, "/api/stats/#{site.domain}/goal/referrers/Twitter?period=day&date=2019-01-01&filters=#{filters}")
assert json_response(conn, 200) == [
%{"name" => "Google", "count" => 2},
]
assert json_response(conn, 200) == %{
"total_visitors" => 2,
"referrers" => [
%{"name" => "a", "count" => 1},
%{"name" => "b", "count" => 1}
]
}
end
end
end

Loading…
Cancel
Save