Remove Twitter stuff

pull/1505/head
Uku Taht 5 years ago
parent e7e36b994d
commit 4d0bc61ffd

@ -131,6 +131,7 @@ blockquote {
animation: pulse-ring 3s cubic-bezier(0.215, 0.61, 0.355, 1) infinite;
@apply bg-green-500;
}
.pulsating-circle::after {
content: '';
position: absolute;
@ -150,9 +151,11 @@ blockquote {
0% {
transform: scale(.33);
}
50% {
transform: scale(1);
}
40%, 100% {
opacity: 0;
}
@ -162,9 +165,11 @@ blockquote {
0% {
transform: scale(.8);
}
25% {
transform: scale(1);
}
50%, 100% {
transform: scale(.8);
}
@ -187,11 +192,13 @@ blockquote {
border: 8px solid transparent;
border-bottom-color: rgba(27,31,35,0.15);
}
.dropdown-content::before, .dropdown-content::after {
position: absolute;
display: inline-block;
content: "";
}
.dropdown-content::after {
top: -14px;
right: 9px;
@ -226,24 +233,6 @@ blockquote {
background-color: rgb(26, 32, 44);
}
.twitter-icon {
width: 1.25em;
height: 1.25em;
display: inline-block;
background-repeat: no-repeat;
background-size: contain;
vertical-align: text-bottom;
background-image: url(data:image/svg+xml;charset=utf-8,%3Csvg%20xmlns%3D%22http%3A%2F%2Fwww.w3.org%2F2000%2Fsvg%22%20viewBox%3D%220%200%2072%2072%22%3E%3Cpath%20fill%3D%22none%22%20d%3D%22M0%200h72v72H0z%22%2F%3E%3Cpath%20class%3D%22icon%22%20fill%3D%22%231da1f2%22%20d%3D%22M68.812%2015.14c-2.348%201.04-4.87%201.744-7.52%202.06%202.704-1.62%204.78-4.186%205.757-7.243-2.53%201.5-5.33%202.592-8.314%203.176C56.35%2010.59%2052.948%209%2049.182%209c-7.23%200-13.092%205.86-13.092%2013.093%200%201.026.118%202.02.338%202.98C25.543%2024.527%2015.9%2019.318%209.44%2011.396c-1.125%201.936-1.77%204.184-1.77%206.58%200%204.543%202.312%208.552%205.824%2010.9-2.146-.07-4.165-.658-5.93-1.64-.002.056-.002.11-.002.163%200%206.345%204.513%2011.638%2010.504%2012.84-1.1.298-2.256.457-3.45.457-.845%200-1.666-.078-2.464-.23%201.667%205.2%206.5%208.985%2012.23%209.09-4.482%203.51-10.13%205.605-16.26%205.605-1.055%200-2.096-.06-3.122-.184%205.794%203.717%2012.676%205.882%2020.067%205.882%2024.083%200%2037.25-19.95%2037.25-37.25%200-.565-.013-1.133-.038-1.693%202.558-1.847%204.778-4.15%206.532-6.774z%22%2F%3E%3C%2Fsvg%3E);
}
.tweet-text a {
@apply text-blue-500;
}
.tweet-text a:hover {
text-decoration: underline;
}
.stats-item {
min-height: 436px;
}
@ -258,7 +247,7 @@ blockquote {
height: 27.25rem;
}
.stats-item__header {
.stats-item-header {
height: inherit;
}
}
@ -274,6 +263,7 @@ blockquote {
.fade-enter {
opacity: 0;
}
.fade-enter-active {
opacity: 1;
transition: opacity 100ms ease-in;
@ -335,4 +325,4 @@ iframe[hidden] {
text-decoration-color: #6366F1; /* tailwind's indigo-500 */
}
}
}

@ -44,11 +44,6 @@ export function formatDayShort(date) {
return `${date.getDate()} ${formatMonth(date).substring(0, 3)}`;
}
export function formatFullDate(date) {
const shortDate = formatMonth(date).substring(0, 3)
return `${shortDate} ${date.getDate()}, ${date.getFullYear()}`;
}
export function parseUTCDate(dateString) {
var date = new Date(dateString);
return new Date(date.getTime() + date.getTimezoneOffset() * 60000);

@ -184,7 +184,7 @@ export default class Devices extends React.Component {
className="stats-item flex flex-col mt-6 stats-item--has-header w-full"
>
<div
className="stats-item__header flex flex-col flex-grow relative p-4 bg-white rounded shadow-xl dark:bg-gray-825"
className="stats-item-header flex flex-col flex-grow relative p-4 bg-white rounded shadow-xl dark:bg-gray-825"
>
<div className="flex justify-between w-full">
<h3 className="font-bold dark:text-gray-100">Devices</h3>

@ -137,7 +137,7 @@ export default class Locations extends React.Component {
className="stats-item flex flex-col w-full mt-6 stats-item--has-header"
>
<div
className="stats-item__header flex flex-col flex-grow bg-white dark:bg-gray-825 shadow-xl rounded p-4 relative"
className="stats-item-header flex flex-col flex-grow bg-white dark:bg-gray-825 shadow-xl rounded p-4 relative"
>
<div className="w-full flex justify-between">
<h3 className="font-bold dark:text-gray-100">

@ -5,7 +5,6 @@ import Modal from './modal'
import * as api from '../../api'
import numberFormatter, {durationFormatter} from '../../number-formatter'
import {parseQuery, toHuman} from '../../query'
import {formatFullDate} from '../../date'
class ReferrerDrilldownModal extends React.Component {
constructor(props) {
@ -84,68 +83,21 @@ class ReferrerDrilldownModal extends React.Component {
)
}
renderTweet(tweet, index) {
const authorUrl = `https://twitter.com/${tweet.author_handle}`
const tweetUrl = `${authorUrl}/status/${tweet.tweet_id}`
const border = index === 0 ? '' : ' pt-4 border-t border-gray-300 dark:border-gray-500'
renderReferrer(referrer) {
return (
<div key={tweet.tweet_id}>
<div className={"flex items-center my-4" + border} >
<a className="flex items-center group" href={authorUrl} target="_blank" rel="noreferrer">
<img className="rounded-full w-8" src={tweet.author_image} />
<div className="ml-2 leading-tight">
<div className="font-bold group-hover:text-blue-500">{tweet.author_name}</div>
<div className="text-xs text-gray-500 dark:text-gray-400">@{tweet.author_handle}</div>
</div>
</a>
<a className="ml-auto twitter-icon" href={tweetUrl} target="_blank" rel="noreferrer"></a>
</div>
<div className="my-2 cursor-text tweet-text whitespace-pre-wrap" dangerouslySetInnerHTML={{__html: tweet.text}}>
</div>
<div className="text-xs text-gray-700 dark:text-gray-300 font-medium">
{formatFullDate(new Date(tweet.created))}
</div>
</div>
<tr className="text-sm dark:text-gray-200" key={referrer.name}>
<td className="p-2">
{ this.renderReferrerName(referrer) }
</td>
{this.showConversionRate() && <td className="p-2 w-32 font-medium" align="right">{numberFormatter(referrer.total_visitors)}</td> }
<td className="p-2 w-32 font-medium" align="right">{numberFormatter(referrer.visitors)}</td>
{this.showExtra() && <td className="p-2 w-32 font-medium" align="right">{this.formatBounceRate(referrer)}</td> }
{this.showExtra() && <td className="p-2 w-32 font-medium" align="right">{this.formatDuration(referrer)}</td> }
{this.showConversionRate() && <td className="p-2 w-32 font-medium" align="right">{referrer.conversion_rate}%</td> }
</tr>
)
}
renderReferrer(referrer) {
if (referrer.tweets) {
return (
<tr className="text-sm dark:text-gray-200" key={referrer.name}>
<td className="p-2">
{ this.renderReferrerName(referrer) }
<span className="text-gray-500 ml-2 text-xs">
appears in {referrer.tweets.length} tweets
</span>
<div className="my-4 pl-4 border-l-2 border-gray-300 dark:border-gray-500">
{ referrer.tweets.map(this.renderTweet) }
</div>
</td>
{this.showConversionRate() && <td className="p-2 w-32 font-medium" align="right">{numberFormatter(referrer.total_visitors)}</td> }
<td className="p-2 w-32 font-medium" align="right" valign="top">{numberFormatter(referrer.visitors)}</td>
{this.showExtra() && <td className="p-2 w-32 font-medium" align="right" valign="top">{this.formatBounceRate(referrer)}</td> }
{this.showExtra() && <td className="p-2 w-32 font-medium" align="right" valign="top">{this.formatDuration(referrer)}</td> }
{this.showConversionRate() && <td className="p-2 w-32 font-medium" align="right">{referrer.conversion_rate}%</td> }
</tr>
)
} else {
return (
<tr className="text-sm dark:text-gray-200" key={referrer.name}>
<td className="p-2">
{ this.renderReferrerName(referrer) }
</td>
{this.showConversionRate() && <td className="p-2 w-32 font-medium" align="right">{numberFormatter(referrer.total_visitors)}</td> }
<td className="p-2 w-32 font-medium" align="right">{numberFormatter(referrer.visitors)}</td>
{this.showExtra() && <td className="p-2 w-32 font-medium" align="right">{this.formatBounceRate(referrer)}</td> }
{this.showExtra() && <td className="p-2 w-32 font-medium" align="right">{this.formatDuration(referrer)}</td> }
{this.showConversionRate() && <td className="p-2 w-32 font-medium" align="right">{referrer.conversion_rate}%</td> }
</tr>
)
}
}
renderGoalText() {
if (this.state.query.filters.goal) {
return (

@ -70,7 +70,7 @@ export default class Pages extends React.Component {
className="stats-item flex flex-col w-full mt-6 stats-item--has-header"
>
<div
className="stats-item__header flex flex-col flex-grow bg-white dark:bg-gray-825 shadow-xl rounded p-4 relative"
className="stats-item-header flex flex-col flex-grow bg-white dark:bg-gray-825 shadow-xl rounded p-4 relative"
>
{/* Header Container */}
<div className="w-full flex justify-between">

@ -89,10 +89,6 @@ paddle_auth_code = get_var_from_path_or_env(config_dir, "PADDLE_VENDOR_AUTH_CODE
google_cid = get_var_from_path_or_env(config_dir, "GOOGLE_CLIENT_ID")
google_secret = get_var_from_path_or_env(config_dir, "GOOGLE_CLIENT_SECRET")
slack_hook_url = get_var_from_path_or_env(config_dir, "SLACK_WEBHOOK")
twitter_consumer_key = get_var_from_path_or_env(config_dir, "TWITTER_CONSUMER_KEY")
twitter_consumer_secret = get_var_from_path_or_env(config_dir, "TWITTER_CONSUMER_SECRET")
twitter_token = get_var_from_path_or_env(config_dir, "TWITTER_ACCESS_TOKEN")
twitter_token_secret = get_var_from_path_or_env(config_dir, "TWITTER_ACCESS_TOKEN_SECRET")
postmark_api_key = get_var_from_path_or_env(config_dir, "POSTMARK_API_KEY")
cron_enabled =
@ -271,12 +267,6 @@ case mailer_adapter do
raise "Unknown mailer_adapter; expected SMTPAdapter or PostmarkAdapter"
end
config :plausible, :twitter,
consumer_key: twitter_consumer_key,
consumer_secret: twitter_consumer_secret,
token: twitter_token,
token_secret: twitter_token_secret
config :plausible, :custom_domain_server,
user: custom_domain_server_user,
password: custom_domain_server_password,
@ -296,8 +286,6 @@ if config_env() == :prod && !disable_cron do
{"0 * * * *", Plausible.Workers.ScheduleEmailReports},
# hourly
{"0 * * * *", Plausible.Workers.SendSiteSetupEmails},
# Daily at midnight
{"0 0 * * *", Plausible.Workers.FetchTweets},
# Daily at midday
{"0 12 * * *", Plausible.Workers.SendCheckStatsEmails},
# Every 15 minutes
@ -326,7 +314,6 @@ if config_env() == :prod && !disable_cron do
schedule_email_reports: 1,
send_email_reports: 1,
spike_notifications: 1,
fetch_tweets: 1,
check_stats_emails: 1,
site_setup_emails: 1,
clean_email_verification_codes: 1,

@ -474,29 +474,11 @@ defmodule Plausible.Stats.Clickhouse do
)
end
referring_urls =
ClickhouseRepo.all(q)
|> Enum.map(fn ref ->
url = if ref[:name] !== "", do: URI.parse("http://" <> ref[:name]).host
Map.put(ref, :url, url)
end)
if referrer == "Twitter" do
urls = Enum.map(referring_urls, & &1[:name])
tweets =
Repo.all(
from t in Plausible.Twitter.Tweet,
where: t.link in ^urls
)
|> Enum.group_by(& &1.link)
Enum.map(referring_urls, fn url ->
Map.put(url, :tweets, tweets[url[:name]])
end)
else
referring_urls
end
ClickhouseRepo.all(q)
|> Enum.map(fn ref ->
url = if ref[:name] !== "", do: URI.parse("http://" <> ref[:name]).host
Map.put(ref, :url, url)
end)
end
def referrer_drilldown_for_goal(site, query, referrer) do

@ -1,24 +0,0 @@
defmodule Plausible.Twitter.Api do
def search(link) do
params = [{"count", 5}, {"tweet_mode", "extended"}, {"q", "https://#{link} -filter:retweets"}]
params =
OAuther.sign(
"get",
"https://api.twitter.com/1.1/search/tweets.json",
params,
oauth_credentials()
)
uri = "https://api.twitter.com/1.1/search/tweets.json?" <> URI.encode_query(params)
response = HTTPoison.get!(uri)
Jason.decode!(response.body)
|> Map.get("statuses")
end
defp oauth_credentials() do
Application.get_env(:plausible, :twitter, %{})
|> OAuther.credentials()
end
end

@ -1,34 +0,0 @@
defmodule Plausible.Twitter.Tweet do
use Ecto.Schema
import Ecto.Changeset
@required_fields [
:link,
:tweet_id,
:author_handle,
:author_name,
:author_image,
:text,
:created
]
@derive {Jason.Encoder, only: @required_fields}
schema "tweets" do
field :link, :string
field :tweet_id, :string
field :author_handle, :string
field :author_name, :string
field :author_image, :string
field :text, :string
field :created, :naive_datetime, null: false
timestamps()
end
def changeset(tweet, attrs) do
tweet
|> cast(attrs, @required_fields)
|> validate_required(@required_fields)
end
end

@ -1,57 +0,0 @@
defmodule Plausible.Workers.FetchTweets do
use Plausible.Repo
alias Plausible.Twitter.Tweet
use Oban.Worker, queue: :fetch_tweets
@impl Oban.Worker
def perform(_job, twitter_api \\ Plausible.Twitter.Api) do
new_links =
Plausible.ClickhouseRepo.all(
from e in Plausible.ClickhouseEvent,
where:
e.timestamp > fragment("(now() - INTERVAL 6 day)") and
e.timestamp < fragment("(now() - INTERVAL 5 day)"),
or_where: e.timestamp > fragment("(now() - INTERVAL 1 day)"),
where: e.referrer_source == "Twitter",
where: e.referrer not in ["t.co", "t.co/"],
distinct: true,
select: e.referrer
)
for link <- new_links do
results = twitter_api.search(link)
for tweet <- results do
{:ok, created} =
Timex.parse(tweet["created_at"], "{WDshort} {Mshort} {D} {ISOtime} {Z} {YYYY}")
Tweet.changeset(%Tweet{}, %{
link: link,
tweet_id: tweet["id_str"],
author_handle: tweet["user"]["screen_name"],
author_name: tweet["user"]["name"],
author_image: tweet["user"]["profile_image_url_https"],
text: html_body(tweet),
created: created
})
|> Repo.insert!(on_conflict: :nothing)
end
end
:ok
end
def html_body(tweet) do
body =
Enum.reduce(tweet["entities"]["urls"], tweet["full_text"], fn url, text ->
html = "<a href=\"#{url["url"]}\" target=\"_blank\">#{url["display_url"]}</a>"
String.replace(text, url["url"], html)
end)
Enum.reduce(tweet["entities"]["user_mentions"], body, fn mention, text ->
link = "https://twitter.com/#{mention["screen_name"]}"
html = "<a href=\"#{link}\" target=\"_blank\">@#{mention["screen_name"]}</a>"
String.replace(text, "@" <> mention["screen_name"], html)
end)
end
end

@ -0,0 +1,7 @@
defmodule Plausible.Repo.Migrations.RemoveTweets do
use Ecto.Migration
def change do
drop table(:tweets)
end
end

@ -142,17 +142,6 @@ defmodule Plausible.Factory do
}
end
def tweet_factory do
%Plausible.Twitter.Tweet{
tweet_id: UUID.uuid4(),
author_handle: "author-handle",
author_name: "author-name",
author_image: "pic.twitter.com/author.png",
text: "tweet-text",
created: Timex.now()
}
end
def weekly_report_factory do
%Plausible.Site.WeeklyReport{}
end

@ -1,93 +0,0 @@
defmodule Plausible.Workers.FetchTweetsTest do
use Plausible.DataCase
alias Plausible.Workers.FetchTweets
import Double
test "fetches Twitter referrals from the last day" do
twitter_mock = stub(Plausible.Twitter.Api, :search, fn _link -> [] end)
FetchTweets.perform(nil, twitter_mock)
assert_receive({Plausible.Twitter.Api, :search, ["t.co/a-link"]})
end
test "fetches Twitter referrals from 5-6 days ago" do
twitter_mock = stub(Plausible.Twitter.Api, :search, fn _link -> [] end)
FetchTweets.perform(nil, twitter_mock)
assert_receive({Plausible.Twitter.Api, :search, ["t.co/b-link"]})
end
test "stores twitter results" do
tweet = %{
"full_text" => "a Tweet body",
"id_str" => "the_tweet_id",
"created_at" => "Mon May 06 20:01:29 +0000 2019",
"user" => %{
"screen_name" => "twitter_author",
"name" => "Twitter Author",
"profile_image_url_https" => "https://image.com"
},
"entities" => %{
"user_mentions" => [],
"urls" => []
}
}
twitter_mock =
stub(Plausible.Twitter.Api, :search, fn
"t.co/a-link" -> [tweet]
_link -> []
end)
FetchTweets.perform(nil, twitter_mock)
[found_tweet] = Repo.all(from(t in Plausible.Twitter.Tweet))
assert found_tweet.tweet_id == "the_tweet_id"
assert found_tweet.text == "a Tweet body"
assert found_tweet.author_handle == "twitter_author"
assert found_tweet.author_name == "Twitter Author"
assert found_tweet.author_image == "https://image.com"
assert found_tweet.created == ~N[2019-05-06 20:01:29]
end
describe "processing tweet entities" do
test "inlines links to the body" do
tweet = %{
"full_text" => "asd https://t.co/somelink",
"entities" => %{
"user_mentions" => [],
"urls" => [
%{
"display_url" => "plausible.io",
"indices" => [4, 17],
"url" => "https://t.co/somelink"
}
]
}
}
body = FetchTweets.html_body(tweet)
assert body == "asd <a href=\"https://t.co/somelink\" target=\"_blank\">plausible.io</a>"
end
test "inlines user mentions to the body" do
tweet = %{
"full_text" => "asd @hello",
"entities" => %{
"user_mentions" => [
%{
"screen_name" => "hello",
"id_str" => "123123"
}
],
"urls" => []
}
}
body = FetchTweets.html_body(tweet)
assert body == "asd <a href=\"https://twitter.com/hello\" target=\"_blank\">@hello</a>"
end
end
end
Loading…
Cancel
Save