diff --git a/lib/lsg_irc/link_plugin.ex b/lib/lsg_irc/link_plugin.ex index 3d657ad..aaf6c6f 100644 --- a/lib/lsg_irc/link_plugin.ex +++ b/lib/lsg_irc/link_plugin.ex @@ -1,266 +1,271 @@ defmodule LSG.IRC.LinkPlugin do @moduledoc """ # Link Previewer An extensible link previewer for IRC. To extend the supported sites, create a new handler implementing the callbacks. See `link_plugin/` directory for examples. The first in list handler that returns true to the `match/2` callback will be used, and if the handler returns `:error` or crashes, will fallback to the default preview. Unsupported websites will use the default link preview method, which is for html document the title, otherwise it'll use the mimetype and size. ## Configuration: ``` config :lsg, LSG.IRC.LinkPlugin, handlers: [ LSG.IRC.LinkPlugin.Youtube: [ invidious: true ], LSG.IRC.LinkPlugin.Twitter: [], LSG.IRC.LinkPlugin.Imgur: [], ] ``` """ @ircdoc """ # Link preview Previews links (just post a link!). Announces real URL after redirections and provides extended support for YouTube, Twitter and Imgur. """ def short_irc_doc, do: false def irc_doc, do: @ircdoc require Logger def start_link() do GenServer.start_link(__MODULE__, [], name: __MODULE__) end @callback match(uri :: URI.t, options :: Keyword.t) :: {true, params :: Map.t} | false @callback expand(uri :: URI.t, params :: Map.t, options :: Keyword.t) :: {:ok, lines :: [] | String.t} | :error @callback post_match(uri :: URI.t, content_type :: binary, headers :: [], opts :: Keyword.t) :: {:body | :file, params :: Map.t} | false @callback post_expand(uri :: URI.t, body :: binary() | Path.t, params :: Map.t, options :: Keyword.t) :: {:ok, lines :: [] | String.t} | :error @optional_callbacks [expand: 3, post_expand: 4] defstruct [:client] def init([]) do {:ok, _} = Registry.register(IRC.PubSub, "messages", [plugin: __MODULE__]) #{:ok, _} = Registry.register(IRC.PubSub, "messages:telegram", [plugin: __MODULE__]) Logger.info("Link handler started") {:ok, %__MODULE__{}} end def handle_info({:irc, :text, message = %{text: text}}, state) do String.split(text) |> Enum.map(fn(word) -> if String.starts_with?(word, "http://") || String.starts_with?(word, "https://") do uri = URI.parse(word) if uri.scheme && uri.host do spawn(fn() -> :timer.kill_after(:timer.seconds(30)) case expand_link([uri]) do {:ok, uris, text} -> text = case uris do [uri] -> text [luri | _] -> if luri.host == uri.host && luri.path == luri.path do text else ["-> #{URI.to_string(luri)}", text] end end if is_list(text) do for line <- text, do: message.replyfun.(line) else message.replyfun.(text) end _ -> nil end end) end end end) {:noreply, state} end def handle_info(msg, state) do {:noreply, state} end def terminate(_reason, state) do :ok end # 1. Match the first valid handler # 2. Try to run the handler # 3. If :error or crash, default link. # If :skip, nothing # 4. ? # Over five redirections: cancel. def expand_link(acc = [_, _, _, _, _ | _]) do {:ok, acc, "link redirects more than five times"} end def expand_link(acc=[uri | _]) do + Logger.debug("link: expanding: #{inspect uri}") handlers = Keyword.get(Application.get_env(:lsg, __MODULE__, [handlers: []]), :handlers) handler = Enum.reduce_while(handlers, nil, fn({module, opts}, acc) -> + Logger.debug("link: attempt expanding: #{inspect module} for #{inspect uri}") module = Module.concat([module]) case module.match(uri, opts) do {true, params} -> {:halt, {module, params, opts}} false -> {:cont, acc} end end) run_expand(acc, handler) end def run_expand(acc, nil) do expand_default(acc) end def run_expand(acc=[uri|_], {module, params, opts}) do + Logger.debug("link: expanding #{inspect uri} with #{inspect module}") case module.expand(uri, params, opts) do {:ok, data} -> {:ok, acc, data} :error -> expand_default(acc) :skip -> nil end rescue e -> - Logger.error(inspect(e)) + Logger.error("link: rescued #{inspect uri} with #{inspect module}: #{inspect e}") + Logger.error(Exception.format(:error, e, __STACKTRACE__)) expand_default(acc) catch e, b -> - Logger.error(inspect({b})) + Logger.error("link: catched #{inspect uri} with #{inspect module}: #{inspect {e, b}}") expand_default(acc) end defp get(url, headers \\ [], options \\ []) do get_req(url, :hackney.get(url, headers, <<>>, options)) end defp get_req(_, {:error, reason}) do {:error, reason} end defp get_req(url, {:ok, 200, headers, client}) do headers = Enum.reduce(headers, %{}, fn({key, value}, acc) -> Map.put(acc, String.downcase(key), value) end) content_type = Map.get(headers, "content-type", "application/octect-stream") length = Map.get(headers, "content-length", "0") {length, _} = Integer.parse(length) handlers = Keyword.get(Application.get_env(:lsg, __MODULE__, [handlers: []]), :handlers) handler = Enum.reduce_while(handlers, false, fn({module, opts}, acc) -> module = Module.concat([module]) try do case module.post_match(url, content_type, headers, opts) do {mode, params} when mode in [:body, :file] -> {:halt, {module, params, opts, mode}} false -> {:cont, acc} end rescue e -> Logger.error(inspect(e)) {:cont, false} catch e, b -> Logger.error(inspect({b})) {:cont, false} end end) cond do handler != false and length <= 30_000_000 -> case get_body(url, 30_000_000, client, handler, <<>>) do {:ok, _} = ok -> ok :error -> {:ok, "file: #{content_type}, size: #{human_size(length)}"} end #String.starts_with?(content_type, "text/html") && length <= 30_000_000 -> # get_body(url, 30_000_000, client, <<>>) true -> :hackney.close(client) {:ok, "file: #{content_type}, size: #{human_size(length)}"} end end defp get_req(_, {:ok, redirect, headers, client}) when redirect in 300..399 do headers = Enum.reduce(headers, %{}, fn({key, value}, acc) -> Map.put(acc, String.downcase(key), value) end) location = Map.get(headers, "location") :hackney.close(client) {:redirect, location} end defp get_req(_, {:ok, status, headers, client}) do :hackney.close(client) {:error, status, headers} end defp get_body(url, len, client, {handler, params, opts, mode} = h, acc) when len >= byte_size(acc) do case :hackney.stream_body(client) do {:ok, data} -> get_body(url, len, client, h, << acc::binary, data::binary >>) :done -> body = case mode do :body -> acc :file -> {:ok, tmpfile} = Plug.Upload.random_file("linkplugin") File.write!(tmpfile, acc) tmpfile end handler.post_expand(url, body, params, opts) {:error, reason} -> {:ok, "failed to fetch body: #{inspect reason}"} end end defp get_body(_, len, client, h, _acc) do :hackney.close(client) IO.inspect(h) {:ok, "Error: file over 30"} end def expand_default(acc = [uri = %URI{scheme: scheme} | _]) when scheme in ["http", "https"] do + Logger.debug("link: expanding #{uri} with default") headers = [{"user-agent", "DmzBot (like TwitterBot)"}] options = [follow_redirect: false, max_body_length: 30_000_000] case get(URI.to_string(uri), headers, options) do {:ok, text} -> {:ok, acc, text} {:redirect, link} -> new_uri = URI.parse(link) #new_uri = %URI{new_uri | scheme: scheme, authority: uri.authority, host: uri.host, port: uri.port} expand_link([new_uri | acc]) {:error, status, _headers} -> text = Plug.Conn.Status.reason_phrase(status) {:ok, acc, "Error: HTTP #{text} (#{status})"} {:error, {:tls_alert, {:handshake_failure, err}}} -> {:ok, acc, "TLS Error: #{to_string(err)}"} {:error, reason} -> {:ok, acc, "Error: #{to_string(reason)}"} end end # Unsupported scheme, came from a redirect. def expand_default(acc = [uri | _]) do {:ok, [uri], "-> #{URI.to_string(uri)}"} end defp human_size(bytes) do bytes |> FileSize.new(:b) |> FileSize.scale() |> FileSize.format() end end diff --git a/lib/lsg_irc/link_plugin/twitter.ex b/lib/lsg_irc/link_plugin/twitter.ex index e462384..41e10ab 100644 --- a/lib/lsg_irc/link_plugin/twitter.ex +++ b/lib/lsg_irc/link_plugin/twitter.ex @@ -1,105 +1,158 @@ defmodule LSG.IRC.LinkPlugin.Twitter do @behaviour LSG.IRC.LinkPlugin @moduledoc """ # Twitter Link Preview Configuration: needs an API key and auth tokens: ``` config :extwitter, :oauth, [ consumer_key: "zzzzz", consumer_secret: "xxxxxxx", access_token: "yyyyyy", access_token_secret: "ssshhhhhh" ] ``` options: * `expand_quoted`: Add the quoted tweet instead of its URL. Default: true. """ def match(uri = %URI{host: twitter, path: path}, _opts) when twitter in ["twitter.com", "m.twitter.com", "mobile.twitter.com"] do case String.split(path, "/", parts: 4) do ["", _username, "status", status_id] -> {status_id, _} = Integer.parse(status_id) {true, %{status_id: status_id}} _ -> false end end def match(_, _), do: false @impl true def post_match(_, _, _, _), do: false def expand(_uri, %{status_id: status_id}, opts) do expand_tweet(ExTwitter.show(status_id, tweet_mode: "extended"), opts) end defp expand_tweet(nil, _opts) do :error end + defp link_tweet(tweet_or_screen_id_tuple, opts, force_twitter_com \\ false) + + defp link_tweet({screen_name, id}, opts, force_twitter_com) do + path = "/#{screen_name}/status/#{id}" + nitter = Keyword.get(opts, :nitter) + host = if !force_twitter_com && nitter, do: nitter, else: "twitter.com" + "https://#{host}/#{screen_name}/status/#{id}" + end + + defp link_tweet(tweet, opts, force_twitter_com) do + link_tweet({tweet.user.screen_name, tweet.id}, opts, force_twitter_com) + end + defp expand_tweet(tweet, opts) do - text = expand_twitter_text(tweet) - text = if tweet.quoted_status do - quote_url = "https://twitter.com/#{tweet.quoted_status.user.screen_name}/status/#{tweet.quoted_status.id}" - String.replace(text, quote_url, "") - else - text - end + head = format_tweet_header(tweet, opts) + # Format tweet text + text = expand_twitter_text(tweet, opts) + text = if tweet.quoted_status do + quote_url = link_tweet(tweet.quoted_status, opts, true) + String.replace(text, quote_url, "") + else + text + end text = IRC.splitlong(text) - {:ok, at} = Timex.parse(tweet.created_at, "%a %b %e %H:%M:%S %z %Y", :strftime) - {:ok, format} = Timex.format(at, "{relative}", :relative) - - replyto = if tweet.in_reply_to_status_id do - replyurl = "https://twitter.com/#{tweet.in_reply_to_screen_name}/status/#{tweet.in_reply_to_status_id}" - if tweet.in_reply_to_screen_name == tweet.user.screen_name do - "— continued from #{replyurl}" - else - "— replying to #{replyurl}" - end - else - "" + reply_to = if tweet.in_reply_to_status_id do + reply_url = link_tweet({tweet.in_reply_to_screen_name, tweet.in_reply_to_status_id}, opts) + text = if tweet.in_reply_to_screen_name == tweet.user.screen_name, do: "continued from", else: "replying to" + <<3, 15, " ↪ ", text::binary, " ", reply_url::binary, 3>> end quoted = if tweet.quoted_status do - quote_url = "https://twitter.com/#{tweet.quoted_status.user.screen_name}/status/#{tweet.quoted_status.id}" - full_text = expand_twitter_text(tweet.quoted_status) - |> IRC.splitlong_with_prefix(">") - ["> #{tweet.quoted_status.user.name} (@#{tweet.quoted_status.user.screen_name}): #{quote_url}"] ++ full_text + full_text = tweet.quoted_status + |> expand_twitter_text(opts) + |> IRC.splitlong_with_prefix(">") + + head = format_tweet_header(tweet.quoted_status, opts, details: false, prefix: "↓ quoting") + + [head | full_text] else [] end - foot = "— #{format} - #{tweet.retweet_count} retweets - #{tweet.favorite_count} likes" + #<<2, "#{tweet.user.name} (@#{tweet.user.screen_name})", 2, " ", 3, 61, "#{foot} #{nitter_link}", 3>>, reply_to] ++ text ++ quoted - text = ["#{tweet.user.name} (@#{tweet.user.screen_name}):", replyto] ++ text ++ quoted ++ [foot] + text = [head, reply_to | text] ++ quoted + |> Enum.filter(& &1) {:ok, text} end - defp expand_twitter_text(tweet) do + defp expand_twitter_text(tweet, _opts) do text = Enum.reduce(tweet.entities.urls, tweet.full_text, fn(entity, text) -> String.replace(text, entity.url, entity.expanded_url) end) extended = tweet.extended_entities || %{media: []} text = Enum.reduce(extended.media, text, fn(entity, text) -> url = Enum.filter(extended.media, fn(e) -> entity.url == e.url end) |> Enum.map(fn(e) -> cond do e.type == "video" -> e.expanded_url true -> e.media_url_https end end) |> Enum.join(" ") String.replace(text, entity.url, url) end) + |> HtmlEntities.decode() + end + + defp format_tweet_header(tweet, opts, format_opts \\ []) do + prefix = Keyword.get(format_opts, :prefix, nil) + details = Keyword.get(format_opts, :details, true) + + padded_prefix = if prefix, do: "#{prefix} ", else: "" + author = <> + + link = link_tweet(tweet, opts) + + {:ok, at} = Timex.parse(tweet.created_at, "%a %b %e %H:%M:%S %z %Y", :strftime) + {:ok, formatted_time} = Timex.format(at, "{relative}", :relative) + + nsfw = if tweet.possibly_sensitive, do: <<3, 52, "NSFW", 3>> + + rts = if tweet.retweet_count && tweet.retweet_count > 0, do: "#{tweet.retweet_count} RT" + likes = if tweet.favorite_count && tweet.favorite_count > 0, do: "#{tweet.favorite_count} ❤︎" + qrts = if tweet.quote_count && tweet.quote_count > 0, do: "#{tweet.quote_count} QRT" + replies = if tweet.reply_count && tweet.reply_count > 0, do: "#{tweet.reply_count} Reps" + + dmcad = if tweet.withheld_copyright, do: <<3, 52, "DMCA", 3>> + withheld_local = if tweet.withheld_in_countries && length(tweet.withheld_in_countries) > 0 do + "Withheld in #{length(tweet.withheld_in_countries)} countries" + end + + verified = if tweet.user.verified, do: <<3, 51, "✔", 3>> + + meta = if details do + [verified, nsfw, formatted_time, dmcad, withheld_local, rts, qrts, likes, replies] + else + [verified, nsfw, formatted_time, dmcad, withheld_local] + end + + meta = meta + |> Enum.filter(& &1) + |> Enum.join(" - ") + + meta = <<3, 15, meta::binary, " → #{link}", 3>> + + <> end end diff --git a/lib/lsg_irc/link_plugin/youtube.ex b/lib/lsg_irc/link_plugin/youtube.ex index 6a16332..f38eca3 100644 --- a/lib/lsg_irc/link_plugin/youtube.ex +++ b/lib/lsg_irc/link_plugin/youtube.ex @@ -1,72 +1,72 @@ defmodule LSG.IRC.LinkPlugin.YouTube do @behaviour LSG.IRC.LinkPlugin @moduledoc """ # YouTube link preview needs an API key: ``` config :lsg, :youtube, api_key: "xxxxxxxxxxxxx" ``` options: - * `invidious`: Add a link to invidious. Default: "yewtu.be". + * `invidious`: Add a link to invidious. """ @impl true def match(uri = %URI{host: yt, path: "/watch", query: "v="<>video_id}, _opts) when yt in ["youtube.com", "www.youtube.com"] do {true, %{video_id: video_id}} end def match(%URI{host: "youtu.be", path: "/"<>video_id}, _opts) do {true, %{video_id: video_id}} end def match(_, _), do: false @impl true def post_match(_, _, _, _), do: false @impl true def expand(uri, %{video_id: video_id}, opts) do key = Application.get_env(:lsg, :youtube)[:api_key] params = %{ "part" => "snippet,contentDetails,statistics", "id" => video_id, "key" => key } headers = [] options = [params: params] case HTTPoison.get("https://www.googleapis.com/youtube/v3/videos", [], options) do {:ok, %HTTPoison.Response{status_code: 200, body: body}} -> case Jason.decode(body) do {:ok, json} -> item = List.first(json["items"]) if item do snippet = item["snippet"] duration = item["contentDetails"]["duration"] |> String.replace("PT", "") |> String.downcase date = snippet["publishedAt"] |> DateTime.from_iso8601() |> elem(1) |> Timex.format("{relative}", :relative) |> elem(1) - line = if host = Keyword.get(opts, :invidious, "yewtu.be") do + line = if host = Keyword.get(opts, :invidious) do ["-> https://#{host}/watch?v=#{video_id}"] else [] end {:ok, line ++ ["#{snippet["title"]}", "— #{duration} — uploaded by #{snippet["channelTitle"]} — #{date}" <> " — #{item["statistics"]["viewCount"]} views, #{item["statistics"]["likeCount"]} likes"]} else :error end _ -> :error end end end end