Remove Twitter stuff
parent
e7e36b994d
commit
4d0bc61ffd
@ -1,24 +0,0 @@
|
||||
defmodule Plausible.Twitter.Api do
|
||||
def search(link) do
|
||||
params = [{"count", 5}, {"tweet_mode", "extended"}, {"q", "https://#{link} -filter:retweets"}]
|
||||
|
||||
params =
|
||||
OAuther.sign(
|
||||
"get",
|
||||
"https://api.twitter.com/1.1/search/tweets.json",
|
||||
params,
|
||||
oauth_credentials()
|
||||
)
|
||||
|
||||
uri = "https://api.twitter.com/1.1/search/tweets.json?" <> URI.encode_query(params)
|
||||
response = HTTPoison.get!(uri)
|
||||
|
||||
Jason.decode!(response.body)
|
||||
|> Map.get("statuses")
|
||||
end
|
||||
|
||||
defp oauth_credentials() do
|
||||
Application.get_env(:plausible, :twitter, %{})
|
||||
|> OAuther.credentials()
|
||||
end
|
||||
end
|
||||
@ -1,34 +0,0 @@
|
||||
defmodule Plausible.Twitter.Tweet do
|
||||
use Ecto.Schema
|
||||
import Ecto.Changeset
|
||||
|
||||
@required_fields [
|
||||
:link,
|
||||
:tweet_id,
|
||||
:author_handle,
|
||||
:author_name,
|
||||
:author_image,
|
||||
:text,
|
||||
:created
|
||||
]
|
||||
|
||||
@derive {Jason.Encoder, only: @required_fields}
|
||||
schema "tweets" do
|
||||
field :link, :string
|
||||
|
||||
field :tweet_id, :string
|
||||
field :author_handle, :string
|
||||
field :author_name, :string
|
||||
field :author_image, :string
|
||||
field :text, :string
|
||||
field :created, :naive_datetime, null: false
|
||||
|
||||
timestamps()
|
||||
end
|
||||
|
||||
def changeset(tweet, attrs) do
|
||||
tweet
|
||||
|> cast(attrs, @required_fields)
|
||||
|> validate_required(@required_fields)
|
||||
end
|
||||
end
|
||||
@ -1,57 +0,0 @@
|
||||
defmodule Plausible.Workers.FetchTweets do
|
||||
use Plausible.Repo
|
||||
alias Plausible.Twitter.Tweet
|
||||
use Oban.Worker, queue: :fetch_tweets
|
||||
|
||||
@impl Oban.Worker
|
||||
def perform(_job, twitter_api \\ Plausible.Twitter.Api) do
|
||||
new_links =
|
||||
Plausible.ClickhouseRepo.all(
|
||||
from e in Plausible.ClickhouseEvent,
|
||||
where:
|
||||
e.timestamp > fragment("(now() - INTERVAL 6 day)") and
|
||||
e.timestamp < fragment("(now() - INTERVAL 5 day)"),
|
||||
or_where: e.timestamp > fragment("(now() - INTERVAL 1 day)"),
|
||||
where: e.referrer_source == "Twitter",
|
||||
where: e.referrer not in ["t.co", "t.co/"],
|
||||
distinct: true,
|
||||
select: e.referrer
|
||||
)
|
||||
|
||||
for link <- new_links do
|
||||
results = twitter_api.search(link)
|
||||
|
||||
for tweet <- results do
|
||||
{:ok, created} =
|
||||
Timex.parse(tweet["created_at"], "{WDshort} {Mshort} {D} {ISOtime} {Z} {YYYY}")
|
||||
|
||||
Tweet.changeset(%Tweet{}, %{
|
||||
link: link,
|
||||
tweet_id: tweet["id_str"],
|
||||
author_handle: tweet["user"]["screen_name"],
|
||||
author_name: tweet["user"]["name"],
|
||||
author_image: tweet["user"]["profile_image_url_https"],
|
||||
text: html_body(tweet),
|
||||
created: created
|
||||
})
|
||||
|> Repo.insert!(on_conflict: :nothing)
|
||||
end
|
||||
end
|
||||
|
||||
:ok
|
||||
end
|
||||
|
||||
def html_body(tweet) do
|
||||
body =
|
||||
Enum.reduce(tweet["entities"]["urls"], tweet["full_text"], fn url, text ->
|
||||
html = "<a href=\"#{url["url"]}\" target=\"_blank\">#{url["display_url"]}</a>"
|
||||
String.replace(text, url["url"], html)
|
||||
end)
|
||||
|
||||
Enum.reduce(tweet["entities"]["user_mentions"], body, fn mention, text ->
|
||||
link = "https://twitter.com/#{mention["screen_name"]}"
|
||||
html = "<a href=\"#{link}\" target=\"_blank\">@#{mention["screen_name"]}</a>"
|
||||
String.replace(text, "@" <> mention["screen_name"], html)
|
||||
end)
|
||||
end
|
||||
end
|
||||
@ -0,0 +1,7 @@
|
||||
defmodule Plausible.Repo.Migrations.RemoveTweets do
|
||||
use Ecto.Migration
|
||||
|
||||
def change do
|
||||
drop table(:tweets)
|
||||
end
|
||||
end
|
||||
@ -1,93 +0,0 @@
|
||||
defmodule Plausible.Workers.FetchTweetsTest do
|
||||
use Plausible.DataCase
|
||||
alias Plausible.Workers.FetchTweets
|
||||
import Double
|
||||
|
||||
test "fetches Twitter referrals from the last day" do
|
||||
twitter_mock = stub(Plausible.Twitter.Api, :search, fn _link -> [] end)
|
||||
FetchTweets.perform(nil, twitter_mock)
|
||||
|
||||
assert_receive({Plausible.Twitter.Api, :search, ["t.co/a-link"]})
|
||||
end
|
||||
|
||||
test "fetches Twitter referrals from 5-6 days ago" do
|
||||
twitter_mock = stub(Plausible.Twitter.Api, :search, fn _link -> [] end)
|
||||
FetchTweets.perform(nil, twitter_mock)
|
||||
|
||||
assert_receive({Plausible.Twitter.Api, :search, ["t.co/b-link"]})
|
||||
end
|
||||
|
||||
test "stores twitter results" do
|
||||
tweet = %{
|
||||
"full_text" => "a Tweet body",
|
||||
"id_str" => "the_tweet_id",
|
||||
"created_at" => "Mon May 06 20:01:29 +0000 2019",
|
||||
"user" => %{
|
||||
"screen_name" => "twitter_author",
|
||||
"name" => "Twitter Author",
|
||||
"profile_image_url_https" => "https://image.com"
|
||||
},
|
||||
"entities" => %{
|
||||
"user_mentions" => [],
|
||||
"urls" => []
|
||||
}
|
||||
}
|
||||
|
||||
twitter_mock =
|
||||
stub(Plausible.Twitter.Api, :search, fn
|
||||
"t.co/a-link" -> [tweet]
|
||||
_link -> []
|
||||
end)
|
||||
|
||||
FetchTweets.perform(nil, twitter_mock)
|
||||
|
||||
[found_tweet] = Repo.all(from(t in Plausible.Twitter.Tweet))
|
||||
assert found_tweet.tweet_id == "the_tweet_id"
|
||||
assert found_tweet.text == "a Tweet body"
|
||||
assert found_tweet.author_handle == "twitter_author"
|
||||
assert found_tweet.author_name == "Twitter Author"
|
||||
assert found_tweet.author_image == "https://image.com"
|
||||
assert found_tweet.created == ~N[2019-05-06 20:01:29]
|
||||
end
|
||||
|
||||
describe "processing tweet entities" do
|
||||
test "inlines links to the body" do
|
||||
tweet = %{
|
||||
"full_text" => "asd https://t.co/somelink",
|
||||
"entities" => %{
|
||||
"user_mentions" => [],
|
||||
"urls" => [
|
||||
%{
|
||||
"display_url" => "plausible.io",
|
||||
"indices" => [4, 17],
|
||||
"url" => "https://t.co/somelink"
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
body = FetchTweets.html_body(tweet)
|
||||
|
||||
assert body == "asd <a href=\"https://t.co/somelink\" target=\"_blank\">plausible.io</a>"
|
||||
end
|
||||
|
||||
test "inlines user mentions to the body" do
|
||||
tweet = %{
|
||||
"full_text" => "asd @hello",
|
||||
"entities" => %{
|
||||
"user_mentions" => [
|
||||
%{
|
||||
"screen_name" => "hello",
|
||||
"id_str" => "123123"
|
||||
}
|
||||
],
|
||||
"urls" => []
|
||||
}
|
||||
}
|
||||
|
||||
body = FetchTweets.html_body(tweet)
|
||||
|
||||
assert body == "asd <a href=\"https://twitter.com/hello\" target=\"_blank\">@hello</a>"
|
||||
end
|
||||
end
|
||||
end
|
||||
Loading…
Reference in New Issue