diff --git a/assets/css/app.css b/assets/css/app.css
index a14e269b8..c3d92b837 100644
--- a/assets/css/app.css
+++ b/assets/css/app.css
@@ -222,3 +222,21 @@ a {
.table-striped tbody tr:nth-child(odd) {
background-color: #f1f5f8;
}
+
+.twitter-icon {
+ width: 1.25em;
+ height: 1.25em;
+ display: inline-block;
+ background-repeat: no-repeat;
+ background-size: contain;
+ vertical-align: text-bottom;
+ background-image: url(data:image/svg+xml;charset=utf-8,%3Csvg%20xmlns%3D%22http%3A%2F%2Fwww.w3.org%2F2000%2Fsvg%22%20viewBox%3D%220%200%2072%2072%22%3E%3Cpath%20fill%3D%22none%22%20d%3D%22M0%200h72v72H0z%22%2F%3E%3Cpath%20class%3D%22icon%22%20fill%3D%22%231da1f2%22%20d%3D%22M68.812%2015.14c-2.348%201.04-4.87%201.744-7.52%202.06%202.704-1.62%204.78-4.186%205.757-7.243-2.53%201.5-5.33%202.592-8.314%203.176C56.35%2010.59%2052.948%209%2049.182%209c-7.23%200-13.092%205.86-13.092%2013.093%200%201.026.118%202.02.338%202.98C25.543%2024.527%2015.9%2019.318%209.44%2011.396c-1.125%201.936-1.77%204.184-1.77%206.58%200%204.543%202.312%208.552%205.824%2010.9-2.146-.07-4.165-.658-5.93-1.64-.002.056-.002.11-.002.163%200%206.345%204.513%2011.638%2010.504%2012.84-1.1.298-2.256.457-3.45.457-.845%200-1.666-.078-2.464-.23%201.667%205.2%206.5%208.985%2012.23%209.09-4.482%203.51-10.13%205.605-16.26%205.605-1.055%200-2.096-.06-3.122-.184%205.794%203.717%2012.676%205.882%2020.067%205.882%2024.083%200%2037.25-19.95%2037.25-37.25%200-.565-.013-1.133-.038-1.693%202.558-1.847%204.778-4.15%206.532-6.774z%22%2F%3E%3C%2Fsvg%3E);
+}
+
+.tweet-text a {
+ @apply text-blue;
+}
+
+.tweet-text a:hover {
+ text-decoration: underline;
+}
diff --git a/assets/js/dashboard/stats/modals/referrer-drilldown.js b/assets/js/dashboard/stats/modals/referrer-drilldown.js
index 8ddb06851..81fc17287 100644
--- a/assets/js/dashboard/stats/modals/referrer-drilldown.js
+++ b/assets/js/dashboard/stats/modals/referrer-drilldown.js
@@ -1,5 +1,6 @@
import React from "react";
import { Link, withRouter } from 'react-router-dom'
+import TweetEmbed from 'react-tweet-embed'
import Modal from './modal'
import * as api from '../../api'
@@ -34,18 +35,69 @@ class ReferrerDrilldownModal extends React.Component {
}
}
- renderReferrer(referrer) {
+ renderReferrerName(name) {
+ if (name) {
+ return {name}
+ } else {
+ return '(no referrer)'
+ }
+ }
+
+ renderTweet(tweet, index) {
+ const authorUrl = `https://twitter.com/${tweet.author_handle}`
+ const tweetUrl = `${authorUrl}/status/${tweet.tweet_id}`
+ const border = index === 0 ? '' : ' pt-4 border-t border-grey-light'
+
return (
-
- |
- { referrer.name }
- |
- {numberFormatter(referrer.count)} |
- {this.showBounceRate() && {this.formatBounceRate(referrer)} | }
-
+
+
+
+
+
+ {tweet.created}
+
+
)
}
+ renderReferrer(referrer) {
+ if (false && referrer.tweets) {
+ return (
+
+ |
+ { this.renderReferrerName(referrer.name) }
+
+ appears in {referrer.tweets.length} tweets
+
+
+
+ { referrer.tweets.map(this.renderTweet) }
+
+ |
+ {numberFormatter(referrer.count)} |
+ {this.showBounceRate() && {this.formatBounceRate(referrer)} | }
+
+ )
+ } else {
+ return (
+
+ |
+ { this.renderReferrerName(referrer.name) }
+ |
+ {numberFormatter(referrer.count)} |
+ {this.showBounceRate() && {this.formatBounceRate(referrer)} | }
+
+ )
+ }
+ }
+
renderGoalText() {
if (this.state.query.filters.goal) {
return (
diff --git a/assets/package-lock.json b/assets/package-lock.json
index de73f6249..1b38f921a 100644
--- a/assets/package-lock.json
+++ b/assets/package-lock.json
@@ -8033,6 +8033,11 @@
"tiny-warning": "^1.0.0"
}
},
+ "react-tweet-embed": {
+ "version": "1.2.2",
+ "resolved": "https://registry.npmjs.org/react-tweet-embed/-/react-tweet-embed-1.2.2.tgz",
+ "integrity": "sha512-Y932BlSaJsDUsKDucC2opzzd+uhc0YNhrlTa/4Beb2be1od+AjLGo6Fhuo2wPT0D+fF4VTXOyoZyA8Yc88RdYA=="
+ },
"read-file-stdin": {
"version": "0.2.1",
"resolved": "https://registry.npmjs.org/read-file-stdin/-/read-file-stdin-0.2.1.tgz",
diff --git a/assets/package.json b/assets/package.json
index 3aabe4c1c..792a616dc 100644
--- a/assets/package.json
+++ b/assets/package.json
@@ -13,6 +13,7 @@
"react": "^16.11.0",
"react-dom": "^16.11.0",
"react-router-dom": "^5.1.2",
+ "react-tweet-embed": "^1.2.2",
"url-search-params-polyfill": "^7.0.0"
},
"devDependencies": {
diff --git a/lib/mix/tasks/fetch_tweets.ex b/lib/mix/tasks/fetch_tweets.ex
new file mode 100644
index 000000000..42e678dea
--- /dev/null
+++ b/lib/mix/tasks/fetch_tweets.ex
@@ -0,0 +1,62 @@
+defmodule Mix.Tasks.FetchTweets do
+ use Plausible.Repo
+ alias Plausible.Twitter.Tweet
+ @oauth_credentials Application.get_env(:plausible, :twitter, %{}) |> OAuther.credentials()
+
+ def run(_args) do
+ Application.ensure_all_started(:plausible)
+ execute()
+ end
+
+ def execute() do
+ new_links = Repo.all(
+ from e in Plausible.Event,
+ where: e.timestamp > fragment("(now() - '6 days'::interval)") and e.timestamp < fragment("(now() - '5 days'::interval)"),
+ or_where: e.timestamp > fragment("(now() - '1 days'::interval)"),
+ where: e.referrer_source == "Twitter",
+ where: e.referrer not in ["t.co", "t.co/"],
+ distinct: true,
+ select: e.referrer
+ )
+
+ for link <- new_links do
+ results = search(link)
+
+ for tweet <- results do
+ {:ok, created} = Timex.parse(tweet["created_at"], "{WDshort} {Mshort} {D} {ISOtime} {Z} {YYYY}")
+
+ Tweet.changeset(%Tweet{}, %{
+ link: link,
+ tweet_id: tweet["id_str"],
+ author_handle: tweet["user"]["screen_name"],
+ author_name: tweet["user"]["name"],
+ author_image: tweet["user"]["profile_image_url"],
+ text: html_body(tweet),
+ created: created
+ }) |> Repo.insert!(on_conflict: :nothing)
+ end
+ end
+ end
+
+ def html_body(tweet) do
+ body = Enum.reduce(tweet["entities"]["urls"], tweet["full_text"], fn url, text ->
+ html = "#{url["display_url"]}"
+ String.replace(text, url["url"], html)
+ end)
+
+ Enum.reduce(tweet["entities"]["user_mentions"], body, fn mention, text ->
+ link = "https://twitter.com/#{mention["screen_name"]}"
+ html = "@#{mention["screen_name"]}"
+ String.replace(text, "@" <> mention["screen_name"], html)
+ end)
+ end
+
+ defp search(link) do
+ params = [{"count", 5}, {"tweet_mode", "extended"}, {"q", "https://#{link} -filter:retweets"}]
+ params = OAuther.sign("get", "https://api.twitter.com/1.1/search/tweets.json", params, @oauth_credentials)
+ uri = "https://api.twitter.com/1.1/search/tweets.json?" <> URI.encode_query(params)
+ response = HTTPoison.get!(uri)
+ Jason.decode!(response.body)
+ |> Map.get("statuses")
+ end
+end
diff --git a/lib/plausible/stats/stats.ex b/lib/plausible/stats/stats.ex
index 66cc0b684..b17c6696c 100644
--- a/lib/plausible/stats/stats.ex
+++ b/lib/plausible/stats/stats.ex
@@ -228,7 +228,8 @@ defmodule Plausible.Stats do
end
def referrer_drilldown(site, query, referrer, include \\ []) do
- referring_urls = Repo.all(from e in base_query(site, query),
+ referring_urls = Repo.all(
+ from e in base_query(site, query),
select: %{name: e.referrer, count: count(e.user_id, :distinct)},
group_by: e.referrer,
where: e.referrer_source == ^referrer,
@@ -236,7 +237,7 @@ defmodule Plausible.Stats do
limit: 100
)
- if "bounce_rate" in include do
+ referring_urls = if "bounce_rate" in include do
bounce_rates = bounce_rates_by_referring_url(site, query, Enum.map(referring_urls, fn ref -> ref[:name] end))
Enum.map(referring_urls, fn url ->
@@ -245,6 +246,24 @@ defmodule Plausible.Stats do
else
referring_urls
end
+
+ if referrer == "Twitter" do
+ urls = Enum.map(referring_urls, &(&1[:name]))
+
+ tweets = Repo.all(
+ from t in Plausible.Twitter.Tweet,
+ where: t.link in ^urls
+ ) |> Enum.reduce(%{}, fn tweet, acc ->
+ Map.update(acc, tweet.link, [tweet], &([tweet | &1]))
+ end)
+ |> IO.inspect
+
+ Enum.map(referring_urls, fn url ->
+ Map.put(url, :tweets, tweets[url[:name]])
+ end)
+ else
+ referring_urls
+ end
end
defp bounce_rates_by_referring_url(site, query, referring_urls) do
diff --git a/lib/plausible/twitter/tweet.ex b/lib/plausible/twitter/tweet.ex
new file mode 100644
index 000000000..8570768aa
--- /dev/null
+++ b/lib/plausible/twitter/tweet.ex
@@ -0,0 +1,26 @@
+defmodule Plausible.Twitter.Tweet do
+ use Ecto.Schema
+ import Ecto.Changeset
+
+ @required_fields [:link, :tweet_id, :author_handle, :author_name, :author_image, :text, :created]
+
+ @derive {Jason.Encoder, only: @required_fields}
+ schema "tweets" do
+ field :link, :string
+
+ field :tweet_id, :string
+ field :author_handle, :string
+ field :author_name, :string
+ field :author_image, :string
+ field :text, :string
+ field :created, :naive_datetime, null: false
+
+ timestamps()
+ end
+
+ def changeset(tweet, attrs) do
+ tweet
+ |> cast(attrs, @required_fields)
+ |> validate_required(@required_fields)
+ end
+end
diff --git a/mix.exs b/mix.exs
index 2d5486b2d..f2889af83 100644
--- a/mix.exs
+++ b/mix.exs
@@ -59,7 +59,8 @@ defmodule Plausible.MixProject do
{:excoveralls, "~> 0.10", only: :test},
{:joken, "~> 2.0"},
{:php_serializer, "~> 0.9.0"},
- {:csv, "~> 2.3"}
+ {:csv, "~> 2.3"},
+ {:oauther, "~> 1.1"}
]
end
diff --git a/mix.lock b/mix.lock
index 22b89cbee..0d06ce4ee 100644
--- a/mix.lock
+++ b/mix.lock
@@ -20,6 +20,7 @@
"elixir_uuid": {:hex, :elixir_uuid, "1.2.0", "ff26e938f95830b1db152cb6e594d711c10c02c6391236900ddd070a6b01271d", [:mix], [], "hexpm"},
"ex_machina": {:hex, :ex_machina, "2.3.0", "92a5ad0a8b10ea6314b876a99c8c9e3f25f4dde71a2a835845b136b9adaf199a", [:mix], [{:ecto, "~> 2.2 or ~> 3.0", [hex: :ecto, repo: "hexpm", optional: true]}, {:ecto_sql, "~> 3.0", [hex: :ecto_sql, repo: "hexpm", optional: true]}], "hexpm"},
"excoveralls": {:hex, :excoveralls, "0.12.0", "50e17a1b116fdb7facc2fe127a94db246169f38d7627b391376a0bc418413ce1", [:mix], [{:hackney, "~> 1.0", [hex: :hackney, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm"},
+ "extwitter": {:hex, :extwitter, "0.11.0", "9472e19f1711bc60bc7efa594353164532475d7c47ea9f1bb66d4faa889b079e", [:mix], [{:oauther, "~> 1.1", [hex: :oauther, repo: "hexpm", optional: false]}, {:poison, "~> 3.0", [hex: :poison, repo: "hexpm", optional: false]}], "hexpm"},
"file_system": {:hex, :file_system, "0.2.6", "fd4dc3af89b9ab1dc8ccbcc214a0e60c41f34be251d9307920748a14bf41f1d3", [:mix], [], "hexpm"},
"gettext": {:hex, :gettext, "0.17.1", "8baab33482df4907b3eae22f719da492cee3981a26e649b9c2be1c0192616962", [:mix], [], "hexpm"},
"hackney": {:hex, :hackney, "1.15.2", "07e33c794f8f8964ee86cebec1a8ed88db5070e52e904b8f12209773c1036085", [:rebar3], [{:certifi, "2.5.1", [hex: :certifi, repo: "hexpm", optional: false]}, {:idna, "6.0.0", [hex: :idna, repo: "hexpm", optional: false]}, {:metrics, "1.0.1", [hex: :metrics, repo: "hexpm", optional: false]}, {:mimerl, "~>1.1", [hex: :mimerl, repo: "hexpm", optional: false]}, {:ssl_verify_fun, "1.1.5", [hex: :ssl_verify_fun, repo: "hexpm", optional: false]}], "hexpm"},
@@ -31,6 +32,7 @@
"metrics": {:hex, :metrics, "1.0.1", "25f094dea2cda98213cecc3aeff09e940299d950904393b2a29d191c346a8486", [:rebar3], [], "hexpm"},
"mime": {:hex, :mime, "1.3.1", "30ce04ab3175b6ad0bdce0035cba77bba68b813d523d1aac73d9781b4d193cf8", [:mix], [], "hexpm"},
"mimerl": {:hex, :mimerl, "1.2.0", "67e2d3f571088d5cfd3e550c383094b47159f3eee8ffa08e64106cdf5e981be3", [:rebar3], [], "hexpm"},
+ "oauther": {:hex, :oauther, "1.1.1", "7d8b16167bb587ecbcddd3f8792beb9ec3e7b65c1f8ebd86b8dd25318d535752", [:mix], [], "hexpm"},
"parallel_stream": {:hex, :parallel_stream, "1.0.6", "b967be2b23f0f6787fab7ed681b4c45a215a81481fb62b01a5b750fa8f30f76c", [:mix], [], "hexpm"},
"parse_trans": {:hex, :parse_trans, "3.3.0", "09765507a3c7590a784615cfd421d101aec25098d50b89d7aa1d66646bc571c1", [:rebar3], [], "hexpm"},
"phoenix": {:hex, :phoenix, "1.4.0", "56fe9a809e0e735f3e3b9b31c1b749d4b436e466d8da627b8d82f90eaae714d2", [:mix], [{:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:phoenix_pubsub, "~> 1.1", [hex: :phoenix_pubsub, repo: "hexpm", optional: false]}, {:plug, "~> 1.7", [hex: :plug, repo: "hexpm", optional: false]}, {:plug_cowboy, "~> 1.0 or ~> 2.0", [hex: :plug_cowboy, repo: "hexpm", optional: true]}], "hexpm"},
diff --git a/priv/repo/migrations/20200114131538_add_tweets.exs b/priv/repo/migrations/20200114131538_add_tweets.exs
new file mode 100644
index 000000000..c9cbce9fe
--- /dev/null
+++ b/priv/repo/migrations/20200114131538_add_tweets.exs
@@ -0,0 +1,20 @@
+defmodule Plausible.Repo.Migrations.AddTweets do
+ use Ecto.Migration
+
+ def change do
+ create table(:tweets) do
+ add :tweet_id, :text, null: false
+ add :text, :text, null: false
+ add :author_handle, :text, null: false
+ add :author_name, :text, null: false
+ add :author_image, :text, null: false
+ add :created, :naive_datetime, null: false
+ add :link, :string, null: false
+
+ timestamps()
+ end
+
+ create index(:tweets, :link)
+ create unique_index(:tweets, [:link, :tweet_id])
+ end
+end
diff --git a/test/mix/tasks/fetch_tweets_test.exs b/test/mix/tasks/fetch_tweets_test.exs
new file mode 100644
index 000000000..dfbe39947
--- /dev/null
+++ b/test/mix/tasks/fetch_tweets_test.exs
@@ -0,0 +1,40 @@
+defmodule Mix.Tasks.FetchTweetsTest do
+ use Plausible.DataCase
+ alias Mix.Tasks.FetchTweets
+
+ describe "processing tweet entities" do
+ test "inlines links to the body" do
+ tweet = %{
+ "full_text" => "asd https://t.co/somelink",
+ "entities" => %{
+ "user_mentions" => [],
+ "urls" => [%{
+ "display_url" => "plausible.io",
+ "indices" => [4, 17],
+ "url" => "https://t.co/somelink"
+ }]
+ }
+ }
+ body = FetchTweets.html_body(tweet)
+
+ assert body == "asd plausible.io"
+ end
+
+ test "inlines user mentions to the body" do
+ tweet = %{
+ "full_text" => "asd @hello",
+ "entities" => %{
+ "user_mentions" => [%{
+ "screen_name" => "hello",
+ "id_str" => "123123"
+ }],
+ "urls" => []
+ }
+ }
+ body = FetchTweets.html_body(tweet)
+
+ assert body == "asd @hello"
+ end
+ end
+
+end