Fetch and display tweets (#27)
parent
94a20fb0a2
commit
b02cb74181
@ -0,0 +1,62 @@
|
||||
defmodule Mix.Tasks.FetchTweets do
|
||||
use Plausible.Repo
|
||||
alias Plausible.Twitter.Tweet
|
||||
@oauth_credentials Application.get_env(:plausible, :twitter, %{}) |> OAuther.credentials()
|
||||
|
||||
def run(_args) do
|
||||
Application.ensure_all_started(:plausible)
|
||||
execute()
|
||||
end
|
||||
|
||||
def execute() do
|
||||
new_links = Repo.all(
|
||||
from e in Plausible.Event,
|
||||
where: e.timestamp > fragment("(now() - '6 days'::interval)") and e.timestamp < fragment("(now() - '5 days'::interval)"),
|
||||
or_where: e.timestamp > fragment("(now() - '1 days'::interval)"),
|
||||
where: e.referrer_source == "Twitter",
|
||||
where: e.referrer not in ["t.co", "t.co/"],
|
||||
distinct: true,
|
||||
select: e.referrer
|
||||
)
|
||||
|
||||
for link <- new_links do
|
||||
results = search(link)
|
||||
|
||||
for tweet <- results do
|
||||
{:ok, created} = Timex.parse(tweet["created_at"], "{WDshort} {Mshort} {D} {ISOtime} {Z} {YYYY}")
|
||||
|
||||
Tweet.changeset(%Tweet{}, %{
|
||||
link: link,
|
||||
tweet_id: tweet["id_str"],
|
||||
author_handle: tweet["user"]["screen_name"],
|
||||
author_name: tweet["user"]["name"],
|
||||
author_image: tweet["user"]["profile_image_url"],
|
||||
text: html_body(tweet),
|
||||
created: created
|
||||
}) |> Repo.insert!(on_conflict: :nothing)
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
def html_body(tweet) do
|
||||
body = Enum.reduce(tweet["entities"]["urls"], tweet["full_text"], fn url, text ->
|
||||
html = "<a href=\"#{url["url"]}\" target=\"_blank\">#{url["display_url"]}</a>"
|
||||
String.replace(text, url["url"], html)
|
||||
end)
|
||||
|
||||
Enum.reduce(tweet["entities"]["user_mentions"], body, fn mention, text ->
|
||||
link = "https://twitter.com/#{mention["screen_name"]}"
|
||||
html = "<a href=\"#{link}\" target=\"_blank\">@#{mention["screen_name"]}</a>"
|
||||
String.replace(text, "@" <> mention["screen_name"], html)
|
||||
end)
|
||||
end
|
||||
|
||||
defp search(link) do
|
||||
params = [{"count", 5}, {"tweet_mode", "extended"}, {"q", "https://#{link} -filter:retweets"}]
|
||||
params = OAuther.sign("get", "https://api.twitter.com/1.1/search/tweets.json", params, @oauth_credentials)
|
||||
uri = "https://api.twitter.com/1.1/search/tweets.json?" <> URI.encode_query(params)
|
||||
response = HTTPoison.get!(uri)
|
||||
Jason.decode!(response.body)
|
||||
|> Map.get("statuses")
|
||||
end
|
||||
end
|
||||
@ -0,0 +1,26 @@
|
||||
defmodule Plausible.Twitter.Tweet do
|
||||
use Ecto.Schema
|
||||
import Ecto.Changeset
|
||||
|
||||
@required_fields [:link, :tweet_id, :author_handle, :author_name, :author_image, :text, :created]
|
||||
|
||||
@derive {Jason.Encoder, only: @required_fields}
|
||||
schema "tweets" do
|
||||
field :link, :string
|
||||
|
||||
field :tweet_id, :string
|
||||
field :author_handle, :string
|
||||
field :author_name, :string
|
||||
field :author_image, :string
|
||||
field :text, :string
|
||||
field :created, :naive_datetime, null: false
|
||||
|
||||
timestamps()
|
||||
end
|
||||
|
||||
def changeset(tweet, attrs) do
|
||||
tweet
|
||||
|> cast(attrs, @required_fields)
|
||||
|> validate_required(@required_fields)
|
||||
end
|
||||
end
|
||||
@ -0,0 +1,20 @@
|
||||
defmodule Plausible.Repo.Migrations.AddTweets do
|
||||
use Ecto.Migration
|
||||
|
||||
def change do
|
||||
create table(:tweets) do
|
||||
add :tweet_id, :text, null: false
|
||||
add :text, :text, null: false
|
||||
add :author_handle, :text, null: false
|
||||
add :author_name, :text, null: false
|
||||
add :author_image, :text, null: false
|
||||
add :created, :naive_datetime, null: false
|
||||
add :link, :string, null: false
|
||||
|
||||
timestamps()
|
||||
end
|
||||
|
||||
create index(:tweets, :link)
|
||||
create unique_index(:tweets, [:link, :tweet_id])
|
||||
end
|
||||
end
|
||||
@ -0,0 +1,40 @@
|
||||
defmodule Mix.Tasks.FetchTweetsTest do
|
||||
use Plausible.DataCase
|
||||
alias Mix.Tasks.FetchTweets
|
||||
|
||||
describe "processing tweet entities" do
|
||||
test "inlines links to the body" do
|
||||
tweet = %{
|
||||
"full_text" => "asd https://t.co/somelink",
|
||||
"entities" => %{
|
||||
"user_mentions" => [],
|
||||
"urls" => [%{
|
||||
"display_url" => "plausible.io",
|
||||
"indices" => [4, 17],
|
||||
"url" => "https://t.co/somelink"
|
||||
}]
|
||||
}
|
||||
}
|
||||
body = FetchTweets.html_body(tweet)
|
||||
|
||||
assert body == "asd <a href=\"https://t.co/somelink\" target=\"_blank\">plausible.io</a>"
|
||||
end
|
||||
|
||||
test "inlines user mentions to the body" do
|
||||
tweet = %{
|
||||
"full_text" => "asd @hello",
|
||||
"entities" => %{
|
||||
"user_mentions" => [%{
|
||||
"screen_name" => "hello",
|
||||
"id_str" => "123123"
|
||||
}],
|
||||
"urls" => []
|
||||
}
|
||||
}
|
||||
body = FetchTweets.html_body(tweet)
|
||||
|
||||
assert body == "asd <a href=\"https://twitter.com/hello\" target=\"_blank\">@hello</a>"
|
||||
end
|
||||
end
|
||||
|
||||
end
|
||||
Loading…
Reference in New Issue