diff --git a/config/runtime.exs b/config/runtime.exs
index 39a596837..7bc22a736 100644
--- a/config/runtime.exs
+++ b/config/runtime.exs
@@ -89,10 +89,6 @@ paddle_auth_code = get_var_from_path_or_env(config_dir, "PADDLE_VENDOR_AUTH_CODE
google_cid = get_var_from_path_or_env(config_dir, "GOOGLE_CLIENT_ID")
google_secret = get_var_from_path_or_env(config_dir, "GOOGLE_CLIENT_SECRET")
slack_hook_url = get_var_from_path_or_env(config_dir, "SLACK_WEBHOOK")
-twitter_consumer_key = get_var_from_path_or_env(config_dir, "TWITTER_CONSUMER_KEY")
-twitter_consumer_secret = get_var_from_path_or_env(config_dir, "TWITTER_CONSUMER_SECRET")
-twitter_token = get_var_from_path_or_env(config_dir, "TWITTER_ACCESS_TOKEN")
-twitter_token_secret = get_var_from_path_or_env(config_dir, "TWITTER_ACCESS_TOKEN_SECRET")
postmark_api_key = get_var_from_path_or_env(config_dir, "POSTMARK_API_KEY")
cron_enabled =
@@ -271,12 +267,6 @@ case mailer_adapter do
raise "Unknown mailer_adapter; expected SMTPAdapter or PostmarkAdapter"
end
-config :plausible, :twitter,
- consumer_key: twitter_consumer_key,
- consumer_secret: twitter_consumer_secret,
- token: twitter_token,
- token_secret: twitter_token_secret
-
config :plausible, :custom_domain_server,
user: custom_domain_server_user,
password: custom_domain_server_password,
@@ -296,8 +286,6 @@ if config_env() == :prod && !disable_cron do
{"0 * * * *", Plausible.Workers.ScheduleEmailReports},
# hourly
{"0 * * * *", Plausible.Workers.SendSiteSetupEmails},
- # Daily at midnight
- {"0 0 * * *", Plausible.Workers.FetchTweets},
# Daily at midday
{"0 12 * * *", Plausible.Workers.SendCheckStatsEmails},
# Every 15 minutes
@@ -326,7 +314,6 @@ if config_env() == :prod && !disable_cron do
schedule_email_reports: 1,
send_email_reports: 1,
spike_notifications: 1,
- fetch_tweets: 1,
check_stats_emails: 1,
site_setup_emails: 1,
clean_email_verification_codes: 1,
diff --git a/lib/plausible/stats/clickhouse.ex b/lib/plausible/stats/clickhouse.ex
index 8a44d9bda..71ac9d3be 100644
--- a/lib/plausible/stats/clickhouse.ex
+++ b/lib/plausible/stats/clickhouse.ex
@@ -474,29 +474,11 @@ defmodule Plausible.Stats.Clickhouse do
)
end
- referring_urls =
- ClickhouseRepo.all(q)
- |> Enum.map(fn ref ->
- url = if ref[:name] !== "", do: URI.parse("http://" <> ref[:name]).host
- Map.put(ref, :url, url)
- end)
-
- if referrer == "Twitter" do
- urls = Enum.map(referring_urls, & &1[:name])
-
- tweets =
- Repo.all(
- from t in Plausible.Twitter.Tweet,
- where: t.link in ^urls
- )
- |> Enum.group_by(& &1.link)
-
- Enum.map(referring_urls, fn url ->
- Map.put(url, :tweets, tweets[url[:name]])
- end)
- else
- referring_urls
- end
+ ClickhouseRepo.all(q)
+ |> Enum.map(fn ref ->
+ url = if ref[:name] !== "", do: URI.parse("http://" <> ref[:name]).host
+ Map.put(ref, :url, url)
+ end)
end
def referrer_drilldown_for_goal(site, query, referrer) do
diff --git a/lib/plausible/twitter/api.ex b/lib/plausible/twitter/api.ex
deleted file mode 100644
index bdb13211e..000000000
--- a/lib/plausible/twitter/api.ex
+++ /dev/null
@@ -1,24 +0,0 @@
-defmodule Plausible.Twitter.Api do
- def search(link) do
- params = [{"count", 5}, {"tweet_mode", "extended"}, {"q", "https://#{link} -filter:retweets"}]
-
- params =
- OAuther.sign(
- "get",
- "https://api.twitter.com/1.1/search/tweets.json",
- params,
- oauth_credentials()
- )
-
- uri = "https://api.twitter.com/1.1/search/tweets.json?" <> URI.encode_query(params)
- response = HTTPoison.get!(uri)
-
- Jason.decode!(response.body)
- |> Map.get("statuses")
- end
-
- defp oauth_credentials() do
- Application.get_env(:plausible, :twitter, %{})
- |> OAuther.credentials()
- end
-end
diff --git a/lib/plausible/twitter/tweet.ex b/lib/plausible/twitter/tweet.ex
deleted file mode 100644
index 0e08fc87f..000000000
--- a/lib/plausible/twitter/tweet.ex
+++ /dev/null
@@ -1,34 +0,0 @@
-defmodule Plausible.Twitter.Tweet do
- use Ecto.Schema
- import Ecto.Changeset
-
- @required_fields [
- :link,
- :tweet_id,
- :author_handle,
- :author_name,
- :author_image,
- :text,
- :created
- ]
-
- @derive {Jason.Encoder, only: @required_fields}
- schema "tweets" do
- field :link, :string
-
- field :tweet_id, :string
- field :author_handle, :string
- field :author_name, :string
- field :author_image, :string
- field :text, :string
- field :created, :naive_datetime, null: false
-
- timestamps()
- end
-
- def changeset(tweet, attrs) do
- tweet
- |> cast(attrs, @required_fields)
- |> validate_required(@required_fields)
- end
-end
diff --git a/lib/workers/fetch_tweets.ex b/lib/workers/fetch_tweets.ex
deleted file mode 100644
index 5b9ddbad1..000000000
--- a/lib/workers/fetch_tweets.ex
+++ /dev/null
@@ -1,57 +0,0 @@
-defmodule Plausible.Workers.FetchTweets do
- use Plausible.Repo
- alias Plausible.Twitter.Tweet
- use Oban.Worker, queue: :fetch_tweets
-
- @impl Oban.Worker
- def perform(_job, twitter_api \\ Plausible.Twitter.Api) do
- new_links =
- Plausible.ClickhouseRepo.all(
- from e in Plausible.ClickhouseEvent,
- where:
- e.timestamp > fragment("(now() - INTERVAL 6 day)") and
- e.timestamp < fragment("(now() - INTERVAL 5 day)"),
- or_where: e.timestamp > fragment("(now() - INTERVAL 1 day)"),
- where: e.referrer_source == "Twitter",
- where: e.referrer not in ["t.co", "t.co/"],
- distinct: true,
- select: e.referrer
- )
-
- for link <- new_links do
- results = twitter_api.search(link)
-
- for tweet <- results do
- {:ok, created} =
- Timex.parse(tweet["created_at"], "{WDshort} {Mshort} {D} {ISOtime} {Z} {YYYY}")
-
- Tweet.changeset(%Tweet{}, %{
- link: link,
- tweet_id: tweet["id_str"],
- author_handle: tweet["user"]["screen_name"],
- author_name: tweet["user"]["name"],
- author_image: tweet["user"]["profile_image_url_https"],
- text: html_body(tweet),
- created: created
- })
- |> Repo.insert!(on_conflict: :nothing)
- end
- end
-
- :ok
- end
-
- def html_body(tweet) do
- body =
- Enum.reduce(tweet["entities"]["urls"], tweet["full_text"], fn url, text ->
- html = "
#{url["display_url"]}"
- String.replace(text, url["url"], html)
- end)
-
- Enum.reduce(tweet["entities"]["user_mentions"], body, fn mention, text ->
- link = "https://twitter.com/#{mention["screen_name"]}"
- html = "
@#{mention["screen_name"]}"
- String.replace(text, "@" <> mention["screen_name"], html)
- end)
- end
-end
diff --git a/priv/repo/migrations/20211202094732_remove_tweets.exs b/priv/repo/migrations/20211202094732_remove_tweets.exs
new file mode 100644
index 000000000..b6ec36a18
--- /dev/null
+++ b/priv/repo/migrations/20211202094732_remove_tweets.exs
@@ -0,0 +1,7 @@
+defmodule Plausible.Repo.Migrations.RemoveTweets do
+ use Ecto.Migration
+
+ def change do
+ drop table(:tweets)
+ end
+end
diff --git a/test/support/factory.ex b/test/support/factory.ex
index 4ffbf7070..b51cb0a6f 100644
--- a/test/support/factory.ex
+++ b/test/support/factory.ex
@@ -142,17 +142,6 @@ defmodule Plausible.Factory do
}
end
- def tweet_factory do
- %Plausible.Twitter.Tweet{
- tweet_id: UUID.uuid4(),
- author_handle: "author-handle",
- author_name: "author-name",
- author_image: "pic.twitter.com/author.png",
- text: "tweet-text",
- created: Timex.now()
- }
- end
-
def weekly_report_factory do
%Plausible.Site.WeeklyReport{}
end
diff --git a/test/workers/fetch_tweets_test.exs b/test/workers/fetch_tweets_test.exs
deleted file mode 100644
index b9112c951..000000000
--- a/test/workers/fetch_tweets_test.exs
+++ /dev/null
@@ -1,93 +0,0 @@
-defmodule Plausible.Workers.FetchTweetsTest do
- use Plausible.DataCase
- alias Plausible.Workers.FetchTweets
- import Double
-
- test "fetches Twitter referrals from the last day" do
- twitter_mock = stub(Plausible.Twitter.Api, :search, fn _link -> [] end)
- FetchTweets.perform(nil, twitter_mock)
-
- assert_receive({Plausible.Twitter.Api, :search, ["t.co/a-link"]})
- end
-
- test "fetches Twitter referrals from 5-6 days ago" do
- twitter_mock = stub(Plausible.Twitter.Api, :search, fn _link -> [] end)
- FetchTweets.perform(nil, twitter_mock)
-
- assert_receive({Plausible.Twitter.Api, :search, ["t.co/b-link"]})
- end
-
- test "stores twitter results" do
- tweet = %{
- "full_text" => "a Tweet body",
- "id_str" => "the_tweet_id",
- "created_at" => "Mon May 06 20:01:29 +0000 2019",
- "user" => %{
- "screen_name" => "twitter_author",
- "name" => "Twitter Author",
- "profile_image_url_https" => "https://image.com"
- },
- "entities" => %{
- "user_mentions" => [],
- "urls" => []
- }
- }
-
- twitter_mock =
- stub(Plausible.Twitter.Api, :search, fn
- "t.co/a-link" -> [tweet]
- _link -> []
- end)
-
- FetchTweets.perform(nil, twitter_mock)
-
- [found_tweet] = Repo.all(from(t in Plausible.Twitter.Tweet))
- assert found_tweet.tweet_id == "the_tweet_id"
- assert found_tweet.text == "a Tweet body"
- assert found_tweet.author_handle == "twitter_author"
- assert found_tweet.author_name == "Twitter Author"
- assert found_tweet.author_image == "https://image.com"
- assert found_tweet.created == ~N[2019-05-06 20:01:29]
- end
-
- describe "processing tweet entities" do
- test "inlines links to the body" do
- tweet = %{
- "full_text" => "asd https://t.co/somelink",
- "entities" => %{
- "user_mentions" => [],
- "urls" => [
- %{
- "display_url" => "plausible.io",
- "indices" => [4, 17],
- "url" => "https://t.co/somelink"
- }
- ]
- }
- }
-
- body = FetchTweets.html_body(tweet)
-
- assert body == "asd
plausible.io"
- end
-
- test "inlines user mentions to the body" do
- tweet = %{
- "full_text" => "asd @hello",
- "entities" => %{
- "user_mentions" => [
- %{
- "screen_name" => "hello",
- "id_str" => "123123"
- }
- ],
- "urls" => []
- }
- }
-
- body = FetchTweets.html_body(tweet)
-
- assert body == "asd
@hello"
- end
- end
-end