| Line | Hits | Source |
|---|---|---|
| 0 | defmodule Briefly do | |
| 1 | @moduledoc """ | |
| 2 | High-level integration point for the web portion of the project. | |
| 3 | """ | |
| 4 | require Logger | |
| 5 | ||
| 6 | alias Briefly.{FeedParser, Storage, ParallelRunner, Config, HttpClient} | |
| 7 | alias Briefly.Models.{Problem, Item} | |
| 8 | ||
| 9 | def refresh(opts \\ []) do | |
| 10 | 5 | with {:ok, config} <- Config.load_config(opts) do |
| 11 | 4 | results = |
| 12 | config | |
| 13 | 8 | |> Enum.map(fn %Config{url: url} -> url end) |
| 14 | |> ParallelRunner.load_all(fn url -> | |
| 15 | 8 | with {:ok, stream} <- HttpClient.stream_get(url, opts) do |
| 16 | 8 | FeedParser.parse_stream(stream) |
| 17 | end | |
| 18 | end) | |
| 19 | ||
| 20 | 4 | {items, problems} = |
| 21 | Enum.reduce(config, {[], []}, fn config, {all_items, all_problems} -> | |
| 22 | 8 | case Map.get(results, config.url) do |
| 23 | nil -> | |
| 24 | # coveralls-ignore-next-line | |
| 25 | problem = Problem.from_feed(config.url, "not processed") | |
| 26 | {all_items, [problem | all_problems]} | |
| 27 | ||
| 28 | {:ok, items, problems} -> | |
| 29 | 7 | items = Enum.map(items, &update_item(&1, config)) |
| 30 | 7 | problems = Enum.map(problems, &Problem.add_url(&1, config.url)) |
| 31 | {all_items ++ items, all_problems ++ problems} | |
| 32 | ||
| 33 | {:error, reason} -> | |
| 34 | 1 | problem = Problem.from_feed(config.url, reason) |
| 35 | {all_items, [problem | all_problems]} | |
| 36 | end | |
| 37 | end) | |
| 38 | ||
| 39 | 4 | Logger.info("DID complete feed refresh") |
| 40 | 4 | Storage.replace(Enum.reverse(items), Enum.reverse(problems)) |
| 41 | else | |
| 42 | {:error, reason} -> | |
| 43 | 1 | Logger.error("DID fail to read config during feed refresh") |
| 44 | 1 | Storage.replace([], [Problem.from_config(reason)]) |
| 45 | end | |
| 46 | end | |
| 47 | ||
| 48 | defp update_item(item, config) do | |
| 49 | item | |
| 50 | 7 | |> Item.add_group(config.group) |
| 51 | 7 | |> Item.maybe_override_feed(config.feed) |
| 52 | end | |
| 53 | ||
| 54 | defdelegate list_problems, to: Storage, as: :problems | |
| 55 | defdelegate last_updated, to: Storage | |
| 56 | ||
| 57 | @doc """ | |
| 58 | Retunrs feed items from **up to** `days_ago`. | |
| 59 | It always uses the _beginning of the day_. If `days_ago` is `0`, only items from **today** | |
| 60 | are retunred. If it is `1`, items from today **and yesterday** are returned. If its `2`, | |
| 61 | items of the last three days are returned. | |
| 62 | ||
| 63 | **Raises** If the given TimeZone is not supported. | |
| 64 | """ | |
| 65 | def list_items(opts) do | |
| 66 | 17 | opts = Keyword.validate!(opts, [:days_ago, :now]) |
| 67 | ||
| 68 | opts | |
| 69 | |> now!() | |
| 70 | |> Timex.beginning_of_day() | |
| 71 | |> Timex.shift(days: -Keyword.get(opts, :days_ago)) | |
| 72 | 17 | |> Storage.items() |
| 73 | end | |
| 74 | ||
| 75 | defp now!(opts) do | |
| 76 | # NOTE: Timex automatically installs its full Timezone Database | |
| 77 | 17 | case Keyword.fetch(opts, :now) do |
| 78 | 3 | {:ok, now} -> now |
| 79 | 14 | :error -> user_timezone() |> DateTime.now!() |
| 80 | end | |
| 81 | end | |
| 82 | ||
| 83 | def user_timezone do | |
| 84 | :briefly | |
| 85 | |> Application.fetch_env!(__MODULE__) | |
| 86 | 118 | |> Keyword.fetch!(:timezone) |
| 87 | end | |
| 88 | end |
| Line | Hits | Source |
|---|---|---|
| 0 | defmodule Briefly.Application do | |
| 1 | @moduledoc false | |
| 2 | ||
| 3 | use Application | |
| 4 | ||
| 5 | @impl true | |
| 6 | def start(_type, _args) do | |
| 7 | 2 | with :ok <- validate_timezone_config() do |
| 8 | 1 | children = [ |
| 9 | Briefly.Storage, | |
| 10 | {Task.Supervisor, name: Briefly.TaskSupervisor}, | |
| 11 | {Phoenix.PubSub, name: Briefly.PubSub}, | |
| 12 | BrieflyWeb.Endpoint, | |
| 13 | Briefly.CronScheduler | |
| 14 | ] | |
| 15 | ||
| 16 | 1 | opts = [strategy: :one_for_one, name: Briefly.Supervisor] |
| 17 | 1 | Supervisor.start_link(children, opts) |
| 18 | end | |
| 19 | end | |
| 20 | ||
| 21 | defp validate_timezone_config do | |
| 22 | 2 | timezone = Briefly.user_timezone() |
| 23 | ||
| 24 | 2 | case Timex.is_valid_timezone?(timezone) do |
| 25 | 1 | true -> :ok |
| 26 | 1 | false -> {:error, "Configured timezone '#{timezone}' is invalid"} |
| 27 | end | |
| 28 | end | |
| 29 | ||
| 30 | # Tell Phoenix to update the endpoint configuration | |
| 31 | # whenever the application is updated. | |
| 32 | @impl true | |
| 33 | 0 | def config_change(changed, _new, removed) do |
| 34 | 0 | BrieflyWeb.Endpoint.config_change(changed, removed) |
| 35 | :ok | |
| 36 | end | |
| 37 | end |
| Line | Hits | Source |
|---|---|---|
| 0 | defmodule Briefly.Config do | |
| 1 | @moduledoc """ | |
| 2 | Loads user configuration from the specified YAML file. | |
| 3 | The configuration currently contains: | |
| 4 | ||
| 5 | - The list of feeds to load | |
| 6 | - (Optional) group name for each feed | |
| 7 | """ | |
| 8 | alias YamlElixir, as: YAML | |
| 9 | ||
| 10 | require Logger | |
| 11 | ||
| 12 | defstruct url: nil, group: nil, feed: nil | |
| 13 | ||
| 14 | @type path_override :: {:path, binary()} | |
| 15 | @spec load_config([path_override()]) :: {:ok, [%__MODULE__{}]} | {:error, reason :: any()} | |
| 16 | def load_config(overrides \\ []) do | |
| 17 | 9 | file_path = file_path(overrides) |
| 18 | ||
| 19 | file_path | |
| 20 | |> YAML.read_from_file() | |
| 21 | 9 | |> case do |
| 22 | {:ok, yaml} -> | |
| 23 | 7 | parse_to(yaml) |
| 24 | ||
| 25 | {:error, reason} -> | |
| 26 | 2 | Logger.error("DID fail to load configuration", file_path: file_path, reason: reason) |
| 27 | {:error, reason} | |
| 28 | end | |
| 29 | end | |
| 30 | ||
| 31 | defp parse_to(%{"feeds" => feeds}) when is_list(feeds) do | |
| 32 | Enum.reduce(feeds, [], fn | |
| 33 | url, acc when is_binary(url) -> | |
| 34 | 3 | %__MODULE__{url: url} |> prepend(acc) |
| 35 | ||
| 36 | %{"url" => url} = map, acc -> | |
| 37 | %__MODULE__{url: url, group: Map.get(map, "group"), feed: Map.get(map, "feed")} | |
| 38 | 10 | |> prepend(acc) |
| 39 | ||
| 40 | entry, acc -> | |
| 41 | 1 | Logger.warning("DID skip malformed config entry", entry: entry) |
| 42 | 1 | acc |
| 43 | end) | |
| 44 | |> Enum.reverse() | |
| 45 | 6 | |> then(&{:ok, &1}) |
| 46 | end | |
| 47 | ||
| 48 | 1 | defp parse_to(_) do |
| 49 | 1 | Logger.error("DID fail to parse configuration, expected 'feeds' array at root") |
| 50 | {:error, :malformed} | |
| 51 | end | |
| 52 | ||
| 53 | 13 | defp prepend(element, list) do |
| 54 | [element | list] | |
| 55 | end | |
| 56 | ||
| 57 | defp file_path(overrides) do | |
| 58 | 9 | case Keyword.fetch(overrides, :path) do |
| 59 | {:ok, path} -> | |
| 60 | 8 | path |
| 61 | ||
| 62 | :error -> | |
| 63 | :briefly | |
| 64 | |> Application.fetch_env!(__MODULE__) | |
| 65 | 1 | |> Keyword.fetch!(:file_path) |
| 66 | end | |
| 67 | end | |
| 68 | end |
| Line | Hits | Source |
|---|---|---|
| 0 | defmodule Briefly.CronScheduler do | |
| 1 | @moduledoc """ | |
| 2 | Uses the `quantum` dependency to run tasks on a CRON-like schedule. | |
| 3 | The actual configuration is found in `config/runtime.ex` | |
| 4 | """ | |
| 5 | use Quantum, otp_app: :briefly | |
| 6 | end |
| Line | Hits | Source |
|---|---|---|
| 0 | defmodule Briefly.FeedParser do | |
| 1 | @moduledoc """ | |
| 2 | A "smart" parser for XML based feeds like RSS and Atom. | |
| 3 | Will inspect the XML and pick the specific format accordingly. | |
| 4 | ||
| 5 | This is implemented as a pull-parser, so it can be very resource friendly | |
| 6 | for even large feeds. | |
| 7 | The parser works on a best-effort basis, meaning it returns all items it | |
| 8 | was able to parse and any errors it encountered for others. | |
| 9 | """ | |
| 10 | @behaviour Saxy.Handler | |
| 11 | ||
| 12 | alias Briefly.FeedParser.{Atom, RSS} | |
| 13 | ||
| 14 | defstruct mod: nil, | |
| 15 | current_element: nil, | |
| 16 | current_type: nil, | |
| 17 | partial: %{}, | |
| 18 | item_index: -1, | |
| 19 | feed_title: nil, | |
| 20 | items: [], | |
| 21 | problems: [] | |
| 22 | ||
| 23 | @type item :: %Briefly.Models.Item{} | |
| 24 | @type problem :: {index :: pos_integer(), reason :: any()} | |
| 25 | @spec parse_stream(Stream.t()) :: {:ok, [item], [problem]} | {:error, %Saxy.ParseError{}} | |
| 26 | def parse_stream(stream) do | |
| 27 | 23 | with {:ok, state} <- Saxy.parse_stream(stream, __MODULE__, nil) do |
| 28 | 22 | {:ok, Enum.reverse(state.items), Enum.reverse(state.problems)} |
| 29 | end | |
| 30 | end | |
| 31 | ||
| 32 | 7 | def handle_event(:start_element, {"rss", _attr}, nil) do |
| 33 | {:ok, %__MODULE__{mod: RSS}} | |
| 34 | end | |
| 35 | ||
| 36 | 15 | def handle_event(:start_element, {"feed", _attr}, nil) do |
| 37 | # TODO do we _need_ to match on the `xmlns`? | |
| 38 | {:ok, %__MODULE__{mod: Atom, current_type: :feed, current_element: nil}} | |
| 39 | end | |
| 40 | ||
| 41 | def handle_event(event, data, %__MODULE__{mod: module} = state) do | |
| 42 | 2366 | module.handle_event(event, data, state) |
| 43 | end | |
| 44 | ||
| 45 | # NOTE: Ingnore events _before_ we know which type of feed it is. | |
| 46 | 23 | def handle_event(_event, _data, state), do: {:ok, state} |
| 47 | end |
| Line | Hits | Source |
|---|---|---|
| 0 | defmodule Briefly.FeedParser.Atom do | |
| 1 | @moduledoc """ | |
| 2 | Atom specific pull-parser implementation, used by `Briefly.FeedParser`. | |
| 3 | ||
| 4 | Implements a small subset of the Atom standard, as defined here: | |
| 5 | https://validator.w3.org/feed/docs/atom.html | |
| 6 | """ | |
| 7 | alias Briefly.Models.Problem | |
| 8 | alias Briefly.FeedParser, as: State | |
| 9 | alias Briefly.Models.Item | |
| 10 | ||
| 11 | @item_elements ~w(title published updated) | |
| 12 | ||
| 13 | 15 | def handle_event( |
| 14 | :start_element, | |
| 15 | {"title", _}, | |
| 16 | %State{current_type: :feed, current_element: nil} = state | |
| 17 | ) do | |
| 18 | {:ok, %{state | current_element: "title"}} | |
| 19 | end | |
| 20 | ||
| 21 | 15 | def handle_event( |
| 22 | :characters, | |
| 23 | chars, | |
| 24 | %State{current_type: :feed, current_element: "title"} = state | |
| 25 | ) do | |
| 26 | {:ok, %{state | current_element: nil, feed_title: String.trim(chars)}} | |
| 27 | end | |
| 28 | ||
| 29 | 27 | def handle_event(:start_element, {"entry", _}, %State{current_element: nil} = state) do |
| 30 | 27 | {:ok, %{state | current_type: :item, item_index: state.item_index + 1, partial: %{}}} |
| 31 | end | |
| 32 | ||
| 33 | def handle_event( | |
| 34 | :start_element, | |
| 35 | {"link", attributes}, | |
| 36 | %State{current_type: :item, current_element: nil, partial: map} = state | |
| 37 | ) do | |
| 38 | 38 | case fetch_attribute(attributes, "href") do |
| 39 | nil -> | |
| 40 | # No link information available, don't set value | |
| 41 | 1 | problem = Problem.from_item(state.item_index, "link missing") |
| 42 | 1 | {:ok, %{state | problems: [problem | state.problems]}} |
| 43 | ||
| 44 | link when is_binary(link) -> | |
| 45 | 37 | rel = fetch_attribute(attributes, "rel") |
| 46 | 37 | type = fetch_attribute(attributes, "type") |
| 47 | 37 | entry = {link, rel, type} |
| 48 | 37 | map = Map.update(map, "link", [entry], &[entry | &1]) |
| 49 | {:ok, %{state | partial: map}} | |
| 50 | end | |
| 51 | end | |
| 52 | ||
| 53 | 66 | def handle_event( |
| 54 | :start_element, | |
| 55 | {element, _}, | |
| 56 | %State{current_type: :item, current_element: nil} = state | |
| 57 | ) | |
| 58 | when element in @item_elements do | |
| 59 | {:ok, %{state | current_element: element}} | |
| 60 | end | |
| 61 | ||
| 62 | 66 | def handle_event( |
| 63 | :characters, | |
| 64 | chars, | |
| 65 | %State{current_type: :item, current_element: element, partial: map} = state | |
| 66 | ) | |
| 67 | when is_binary(element) do | |
| 68 | {:ok, %{state | current_element: nil, partial: Map.put(map, element, String.trim(chars))}} | |
| 69 | end | |
| 70 | ||
| 71 | def handle_event( | |
| 72 | :end_element, | |
| 73 | "entry", | |
| 74 | %State{current_type: :item, current_element: nil, partial: map} = state | |
| 75 | ) do | |
| 76 | 27 | with {:ok, title} <- Map.fetch(map, "title"), |
| 77 | 26 | {:ok, link_list} <- Map.fetch(map, "link"), |
| 78 | 24 | {:ok, link} <- pick_link(link_list), |
| 79 | 24 | {:ok, date} <- pick_date(map), |
| 80 | 23 | {:ok, date, _} <- DateTime.from_iso8601(date) do |
| 81 | 22 | item = %Item{ |
| 82 | 22 | feed: state.feed_title, |
| 83 | title: title, | |
| 84 | link: link, | |
| 85 | date: date | |
| 86 | } | |
| 87 | ||
| 88 | 22 | {:ok, %{state | items: [item | state.items]}} |
| 89 | else | |
| 90 | :error -> | |
| 91 | # Item didn't have required fields, ignore it | |
| 92 | 4 | problem = Problem.from_item(state.item_index, "missing required fields") |
| 93 | 4 | {:ok, %{state | problems: [problem | state.problems]}} |
| 94 | ||
| 95 | {:error, _reason} -> | |
| 96 | 1 | problem = Problem.from_item(state.item_index, "invalid date format") |
| 97 | 1 | {:ok, %{state | problems: [problem | state.problems]}} |
| 98 | end | |
| 99 | end | |
| 100 | ||
| 101 | # Ignore any other elements | |
| 102 | 782 | def handle_event(_event, _data, state), do: {:ok, state} |
| 103 | ||
| 104 | defp fetch_attribute(attributes, to_fetch) when is_list(attributes) do | |
| 105 | 112 | Enum.find_value(attributes, fn {name, val} -> |
| 106 | 194 | if name == to_fetch, do: val, else: false |
| 107 | end) | |
| 108 | end | |
| 109 | ||
| 110 | 22 | defp pick_date(%{"published" => date}), do: {:ok, date} |
| 111 | 1 | defp pick_date(%{"updated" => date}), do: {:ok, date} |
| 112 | 1 | defp pick_date(_), do: :error |
| 113 | ||
| 114 | # coveralls-ignore-next-line | |
| 115 | defp pick_link([]), do: :error | |
| 116 | 17 | defp pick_link([{link, _rel, _type}]), do: {:ok, link} |
| 117 | ||
| 118 | defp pick_link(links) when is_list(links) do | |
| 119 | links | |
| 120 | |> Enum.max_by(fn {_link, rel, type} -> | |
| 121 | 19 | weighted_score(rel: rel) + weighted_score(type: type) |
| 122 | end) | |
| 123 | 7 | |> then(fn {link, _rel, _type} -> {:ok, link} end) |
| 124 | end | |
| 125 | ||
| 126 | 19 | defp weighted_score(rel: rel), do: score_rel(rel) * 0.75 |
| 127 | 19 | defp weighted_score(type: type), do: score_type(type) * 0.25 |
| 128 | ||
| 129 | 12 | defp score_rel(rel) when rel in ["alternate", "", nil], do: 3 |
| 130 | 5 | defp score_rel("self"), do: 2 |
| 131 | 2 | defp score_rel(_rel), do: 1 |
| 132 | ||
| 133 | 8 | defp score_type("text/html"), do: 3 |
| 134 | 5 | defp score_type("text/" <> _rest), do: 2 |
| 135 | 6 | defp score_type(_type), do: 1 |
| 136 | end |
| Line | Hits | Source |
|---|---|---|
| 0 | defmodule Briefly.FeedParser.RSS do | |
| 1 | @moduledoc """ | |
| 2 | RSS specific pull-parser implementation, used by `Briefly.FeedParser`. | |
| 3 | ||
| 4 | Implements a small subset of the Atom standard, as defined here: | |
| 5 | https://www.rssboard.org/rss-specification | |
| 6 | """ | |
| 7 | alias Briefly.FeedParser, as: State | |
| 8 | alias Briefly.Models.{Item, Problem} | |
| 9 | ||
| 10 | @item_elements ~w(title link pubDate) | |
| 11 | ||
| 12 | 7 | def handle_event(:start_element, {"channel", _}, %State{current_type: nil} = state) do |
| 13 | {:ok, %{state | current_type: :feed, current_element: nil}} | |
| 14 | end | |
| 15 | ||
| 16 | 8 | def handle_event( |
| 17 | :start_element, | |
| 18 | {"title", _}, | |
| 19 | %State{current_type: :feed, current_element: nil} = state | |
| 20 | ) do | |
| 21 | {:ok, %{state | current_element: "title"}} | |
| 22 | end | |
| 23 | ||
| 24 | 8 | def handle_event( |
| 25 | :characters, | |
| 26 | chars, | |
| 27 | %State{current_type: :feed, current_element: "title"} = state | |
| 28 | ) do | |
| 29 | {:ok, %{state | current_element: nil, feed_title: String.trim(chars)}} | |
| 30 | end | |
| 31 | ||
| 32 | 46 | def handle_event(:start_element, {"item", _}, %State{current_element: nil} = state) do |
| 33 | 46 | {:ok, %{state | current_type: :item, item_index: state.item_index + 1, partial: %{}}} |
| 34 | end | |
| 35 | ||
| 36 | 135 | def handle_event( |
| 37 | :start_element, | |
| 38 | {element, _}, | |
| 39 | %State{current_type: :item, current_element: nil} = state | |
| 40 | ) | |
| 41 | when element in @item_elements do | |
| 42 | {:ok, %{state | current_element: element}} | |
| 43 | end | |
| 44 | ||
| 45 | 135 | def handle_event( |
| 46 | :characters, | |
| 47 | chars, | |
| 48 | %State{current_type: :item, current_element: element, partial: map} = state | |
| 49 | ) | |
| 50 | when is_binary(element) do | |
| 51 | {:ok, %{state | current_element: nil, partial: Map.put(map, element, String.trim(chars))}} | |
| 52 | end | |
| 53 | ||
| 54 | def handle_event( | |
| 55 | :end_element, | |
| 56 | "item", | |
| 57 | %State{current_type: :item, current_element: nil, partial: map} = state | |
| 58 | ) do | |
| 59 | 46 | with {:ok, title} <- Map.fetch(map, "title"), |
| 60 | 45 | {:ok, link} <- Map.fetch(map, "link"), |
| 61 | 44 | {:ok, published} <- Map.fetch(map, "pubDate"), |
| 62 | 43 | {:ok, date} <- published |> Timex.parse("{RFC1123}") do |
| 63 | 42 | item = %Item{feed: state.feed_title, title: title, link: link, date: date} |
| 64 | 42 | {:ok, %{state | items: [item | state.items]}} |
| 65 | else | |
| 66 | # Item didn't have required fields, ignore it | |
| 67 | :error -> | |
| 68 | 3 | problem = Problem.from_item(state.item_index, "missing required fields") |
| 69 | 3 | {:ok, %{state | problems: [problem | state.problems]}} |
| 70 | ||
| 71 | {:error, reason} -> | |
| 72 | 1 | problem = Problem.from_item(state.item_index, reason) |
| 73 | 1 | {:ok, %{state | problems: [problem | state.problems]}} |
| 74 | end | |
| 75 | end | |
| 76 | ||
| 77 | # Ignore any other elements | |
| 78 | 945 | def handle_event(_event, _data, state), do: {:ok, state} |
| 79 | end |
| Line | Hits | Source |
|---|---|---|
| 0 | defmodule Briefly.HttpClient do | |
| 1 | @moduledoc """ | |
| 2 | A `Req` based HTTP Client to request feeds over the internet. | |
| 3 | """ | |
| 4 | ||
| 5 | @accepted_opts ~w(into retry_log_level cache retry plug)a | |
| 6 | @default_opts [ | |
| 7 | into: :self, | |
| 8 | # retry is setup by default, doing max 4 requests with exponential back-off | |
| 9 | retry_log_level: :info, | |
| 10 | # this is the default, just making it explicit to document | |
| 11 | cache: false | |
| 12 | ] | |
| 13 | @type url :: binary() | |
| 14 | @type opts :: keyword() | |
| 15 | ||
| 16 | @spec stream_get(url, opts) :: {:ok, Enumerable.t()} | {:error, any()} | |
| 17 | def stream_get(url, opts \\ []) do | |
| 18 | 12 | case Req.get(url, request_opts(opts)) do |
| 19 | # NOTE: This is `Req.Response.Async` becasue of `into: :self`! | |
| 20 | 10 | {:ok, %Req.Response{status: status, body: body}} when status >= 200 and status < 299 -> |
| 21 | # Return the streaming body | |
| 22 | {:ok, body} | |
| 23 | ||
| 24 | 1 | {:ok, response} -> |
| 25 | {:error, response} | |
| 26 | ||
| 27 | 1 | {:error, exception} -> |
| 28 | {:error, exception} | |
| 29 | end | |
| 30 | end | |
| 31 | ||
| 32 | defp request_opts(overrides) do | |
| 33 | 12 | config = |
| 34 | Application.get_env(:briefly, __MODULE__, []) | |
| 35 | |> Keyword.get(:opts, []) | |
| 36 | ||
| 37 | @default_opts | |
| 38 | |> Keyword.merge(config) | |
| 39 | 12 | |> Keyword.merge(Keyword.take(overrides, @accepted_opts)) |
| 40 | end | |
| 41 | end |
| Line | Hits | Source |
|---|---|---|
| 0 | defmodule Briefly.Models.Feed do | |
| 1 | @moduledoc """ | |
| 2 | A feed information, loaded from the configuration. | |
| 3 | """ | |
| 4 | ||
| 5 | defstruct title: nil | |
| 6 | end |
| Line | Hits | Source |
|---|---|---|
| 0 | defmodule Briefly.Models.Item do | |
| 1 | @moduledoc """ | |
| 2 | A Feed Item, extracted from a Feed. | |
| 3 | """ | |
| 4 | ||
| 5 | defstruct feed: nil, title: nil, link: nil, date: nil, group: nil | |
| 6 | ||
| 7 | def add_group(%__MODULE__{} = state, group) do | |
| 8 | 7 | %__MODULE__{state | group: group} |
| 9 | end | |
| 10 | ||
| 11 | def maybe_override_feed(%__MODULE__{} = state, override) when is_binary(override) do | |
| 12 | 1 | %__MODULE__{state | feed: override} |
| 13 | end | |
| 14 | ||
| 15 | 6 | def maybe_override_feed(%__MODULE__{} = state, _), do: state |
| 16 | end |
| Line | Hits | Source |
|---|---|---|
| 0 | defmodule Briefly.Models.Problem do | |
| 1 | @moduledoc """ | |
| 2 | Describes a single problem encountered while loading/parsing items. | |
| 3 | """ | |
| 4 | # TODO "reason" is a bad name here. Whats better? | |
| 5 | defstruct url: nil, reason: nil, message: nil, metadata: %{} | |
| 6 | ||
| 7 | def from_error(%type{} = error) do | |
| 8 | 1 | %__MODULE__{ |
| 9 | reason: as_text(type), | |
| 10 | message: as_text(error), | |
| 11 | metadata: %{original: error} | |
| 12 | } | |
| 13 | end | |
| 14 | ||
| 15 | def from_item(feed_index, reason) do | |
| 16 | 10 | %__MODULE__{ |
| 17 | 10 | reason: "Item at index #{feed_index}", |
| 18 | message: as_text(reason), | |
| 19 | metadata: %{index: feed_index} | |
| 20 | } | |
| 21 | end | |
| 22 | ||
| 23 | def from_feed(url, reason) do | |
| 24 | 1 | %__MODULE__{ |
| 25 | reason: "Feed", | |
| 26 | message: as_text(reason), | |
| 27 | url: url | |
| 28 | } | |
| 29 | end | |
| 30 | ||
| 31 | def from_config(reason) do | |
| 32 | 1 | %__MODULE__{ |
| 33 | reason: "Config", | |
| 34 | message: as_text(reason) | |
| 35 | } | |
| 36 | end | |
| 37 | ||
| 38 | def add_url(%__MODULE__{} = problem, feed_url) do | |
| 39 | 1 | %__MODULE__{problem | url: feed_url} |
| 40 | end | |
| 41 | ||
| 42 | 10 | defp as_text(reason) when is_binary(reason), do: reason |
| 43 | 3 | defp as_text(reason) when is_exception(reason), do: Exception.message(reason) |
| 44 | 1 | defp as_text(reason) when is_atom(reason), do: to_string(reason) |
| 45 | # coveralls-ignore-next-line | |
| 46 | defp as_text(reason), do: inspect(reason) | |
| 47 | ||
| 48 | def message(%__MODULE__{url: url, reason: reason, message: message}) do | |
| 49 | 2 | "(#{url}) #{reason}: #{message}" |
| 50 | end | |
| 51 | end |
| Line | Hits | Source |
|---|---|---|
| 0 | defmodule Briefly.ParallelRunner do | |
| 1 | @moduledoc """ | |
| 2 | A multithreaded loader component to fetch multiple feeds. | |
| 3 | """ | |
| 4 | alias Briefly.FeedParser | |
| 5 | require Logger | |
| 6 | ||
| 7 | # NOTE: This is a hard upper-bound, enforced for task workflows. Ideally, tasks itself | |
| 8 | # enforce more grenular timeouts on their own. | |
| 9 | @default_opts [{:timeout, :timer.minutes(5)}] | |
| 10 | ||
| 11 | @type url :: binary() | |
| 12 | @spec load_all([url], (url -> any())) :: %{ | |
| 13 | url => {:ok, FeedParser.item(), FeedParser.problem()} | {:error, any()} | |
| 14 | } | |
| 15 | def load_all(urls, fun, opts \\ []) when is_list(urls) and is_function(fun, 1) do | |
| 16 | 7 | opts = Keyword.validate!(opts, @default_opts) |
| 17 | ||
| 18 | Briefly.TaskSupervisor | |
| 19 | |> Task.Supervisor.async_stream_nolink( | |
| 20 | urls, | |
| 21 | 14 | &do_work(&1, fun), |
| 22 | ordered: false, | |
| 23 | timeout: Keyword.fetch!(opts, :timeout), | |
| 24 | on_timeout: :kill_task, | |
| 25 | zip_input_on_exit: true | |
| 26 | ) | |
| 27 | 7 | |> Enum.reduce(%{}, fn |
| 28 | {:ok, {url, result}}, map -> | |
| 29 | 13 | Map.put(map, url, result) |
| 30 | ||
| 31 | {:exit, {url, reason}}, map -> | |
| 32 | 1 | Map.put(map, url, {:error, reason}) |
| 33 | end) | |
| 34 | end | |
| 35 | ||
| 36 | 14 | def do_work(url, fun) do |
| 37 | 14 | result = fun.(url) |
| 38 | {url, result} | |
| 39 | rescue | |
| 40 | 1 | error -> |
| 41 | 1 | Logger.error("RESCUED error in Task: #{Exception.format(:error, error)}") |
| 42 | {url, {:error, error}} | |
| 43 | end | |
| 44 | end |
| Line | Hits | Source |
|---|---|---|
| 0 | defmodule Briefly.Storage do | |
| 1 | @moduledoc """ | |
| 2 | An in-memory storage for `Briefly.Models.Item` parsed from | |
| 3 | feeds and any problems that occurred during the parsing. | |
| 4 | """ | |
| 5 | use GenServer | |
| 6 | ||
| 7 | alias Briefly.Models.Item | |
| 8 | ||
| 9 | defstruct items: [], problems: [], last_updated: nil | |
| 10 | ||
| 11 | # coveralls-ignore-start | |
| 12 | ##### CLIENT #### | |
| 13 | def start_link(opts), do: GenServer.start_link(__MODULE__, opts, name: __MODULE__) | |
| 14 | ||
| 15 | def replace(items, problems) when is_list(items) and is_list(problems), | |
| 16 | do: GenServer.cast(__MODULE__, {:replace, items, problems}) | |
| 17 | ||
| 18 | def items, do: GenServer.call(__MODULE__, :all_items) | |
| 19 | def items(%DateTime{} = cutoff), do: GenServer.call(__MODULE__, {:items, cutoff}) | |
| 20 | def problems, do: GenServer.call(__MODULE__, :problems) | |
| 21 | def last_updated, do: GenServer.call(__MODULE__, :last_updated) | |
| 22 | ||
| 23 | ##### SERVER #### | |
| 24 | def init(_opts), do: {:ok, %__MODULE__{}} | |
| 25 | # coveralls-ignore-stop | |
| 26 | ||
| 27 | 27 | def handle_cast({:replace, items, problems}, state) do |
| 28 | 27 | now = Briefly.user_timezone() |> DateTime.now!() |
| 29 | {:noreply, %{state | items: items, problems: problems, last_updated: now}} | |
| 30 | end | |
| 31 | ||
| 32 | def handle_call(:all_items, _from, %__MODULE__{items: items} = state) do | |
| 33 | 2 | {:reply, newest_first(items), state} |
| 34 | end | |
| 35 | ||
| 36 | def handle_call({:items, cutoff}, _from, %__MODULE__{items: items} = state) do | |
| 37 | items | |
| 38 | 110 | |> Enum.filter(fn %Item{date: released} -> DateTime.after?(released, cutoff) end) |
| 39 | |> newest_first() | |
| 40 | 23 | |> then(&{:reply, &1, state}) |
| 41 | end | |
| 42 | ||
| 43 | def handle_call(:problems, _from, %__MODULE__{problems: problems} = state) do | |
| 44 | 37 | {:reply, problems, state} |
| 45 | end | |
| 46 | ||
| 47 | def handle_call(:last_updated, _from, %__MODULE__{last_updated: last_updated} = state) do | |
| 48 | 15 | {:reply, last_updated, state} |
| 49 | end | |
| 50 | ||
| 51 | 25 | defp newest_first(items), do: Enum.sort_by(items, & &1.date, {:desc, DateTime}) |
| 52 | end |
| Line | Hits | Source |
|---|---|---|
| 0 | defmodule BrieflyWeb do | |
| 1 | @moduledoc """ | |
| 2 | The entrypoint for defining your web interface, such | |
| 3 | as controllers, components, channels, and so on. | |
| 4 | ||
| 5 | This can be used in your application as: | |
| 6 | ||
| 7 | use BrieflyWeb, :controller | |
| 8 | use BrieflyWeb, :html | |
| 9 | ||
| 10 | The definitions below will be executed for every controller, | |
| 11 | component, etc, so keep them short and clean, focused | |
| 12 | on imports, uses and aliases. | |
| 13 | ||
| 14 | Do NOT define functions inside the quoted expressions | |
| 15 | below. Instead, define additional modules and import | |
| 16 | those modules here. | |
| 17 | """ | |
| 18 | ||
| 19 | 3 | def static_paths, |
| 20 | do: ~w(assets fonts images webapp favicon.ico favicon.svg favicon-96x96.png robots.txt) | |
| 21 | ||
| 22 | def router do | |
| 23 | 0 | quote do |
| 24 | use Phoenix.Router, helpers: false | |
| 25 | ||
| 26 | # Import common connection and controller functions to use in pipelines | |
| 27 | import Plug.Conn | |
| 28 | import Phoenix.Controller | |
| 29 | end | |
| 30 | end | |
| 31 | ||
| 32 | def channel do | |
| 33 | 0 | quote do |
| 34 | use Phoenix.Channel | |
| 35 | end | |
| 36 | end | |
| 37 | ||
| 38 | def controller do | |
| 39 | 0 | quote do |
| 40 | use Phoenix.Controller, | |
| 41 | formats: [:html, :json], | |
| 42 | layouts: [html: BrieflyWeb.Layouts] | |
| 43 | ||
| 44 | import Plug.Conn | |
| 45 | import BrieflyWeb.Gettext | |
| 46 | ||
| 47 | unquote(verified_routes()) | |
| 48 | end | |
| 49 | end | |
| 50 | ||
| 51 | def html do | |
| 52 | 0 | quote do |
| 53 | # HTML escaping functionality | |
| 54 | import Phoenix.HTML | |
| 55 | import Phoenix.HTML.Form | |
| 56 | # Translation | |
| 57 | import BrieflyWeb.Gettext | |
| 58 | # Template helpers | |
| 59 | import Phoenix.Template, only: [embed_templates: 1] | |
| 60 | ||
| 61 | # Routes generation with the ~p sigil | |
| 62 | unquote(verified_routes()) | |
| 63 | ||
| 64 | # Import convenience functions from controllers | |
| 65 | import Phoenix.Controller, | |
| 66 | only: [get_csrf_token: 0, view_module: 1, view_template: 1] | |
| 67 | end | |
| 68 | end | |
| 69 | ||
| 70 | def verified_routes do | |
| 71 | 3 | quote do |
| 72 | use Phoenix.VerifiedRoutes, | |
| 73 | endpoint: BrieflyWeb.Endpoint, | |
| 74 | router: BrieflyWeb.Router, | |
| 75 | statics: BrieflyWeb.static_paths() | |
| 76 | end | |
| 77 | end | |
| 78 | ||
| 79 | @doc """ | |
| 80 | When used, dispatch to the appropriate controller/view/etc. | |
| 81 | """ | |
| 82 | defmacro __using__(which) when is_atom(which) do | |
| 83 | 3 | apply(__MODULE__, which, []) |
| 84 | end | |
| 85 | end |
| Line | Hits | Source |
|---|---|---|
| 0 | defmodule BrieflyWeb.Layouts do | |
| 1 | use BrieflyWeb, :html | |
| 2 | alias Timex.Format.DateTime.Formatters.Relative | |
| 3 | ||
| 4 | embed_templates "layouts/*" | |
| 5 | ||
| 6 | 14 | def days_ago do |
| 7 | [ | |
| 8 | {"today", "Today"}, | |
| 9 | {"yesterday", "Yesterday"}, | |
| 10 | {"3d", "Last 3 days"}, | |
| 11 | {"5d", "Last 5 days"}, | |
| 12 | {"7d", "Last week"} | |
| 13 | ] | |
| 14 | end | |
| 15 | ||
| 16 | def problem_label do | |
| 17 | Briefly.list_problems() | |
| 18 | |> length() | |
| 19 | 30 | |> case do |
| 20 | 24 | 0 -> nil |
| 21 | 3 | 1 -> "⚠️ 1 problem" |
| 22 | 3 | n -> "⚠️ #{n} problems" |
| 23 | end | |
| 24 | end | |
| 25 | ||
| 26 | def user_timezone do | |
| 27 | 14 | Briefly.user_timezone() |
| 28 | end | |
| 29 | ||
| 30 | def app_version do | |
| 31 | 14 | Application.spec(:briefly, :vsn) |
| 32 | end | |
| 33 | ||
| 34 | def last_updated do | |
| 35 | 14 | case Briefly.last_updated() do |
| 36 | 0 | nil -> "Not updated yet" |
| 37 | 14 | %DateTime{} = dt -> Relative.format!(dt, "{relative}") |
| 38 | end | |
| 39 | end | |
| 40 | end |
| Line | Hits | Source |
|---|---|---|
| 0 | defmodule BrieflyWeb.ErrorHTML do | |
| 1 | use BrieflyWeb, :html | |
| 2 | ||
| 3 | # The default is to render a plain text page based on | |
| 4 | # the template name. For example, "404.html" becomes | |
| 5 | # "Not Found". | |
| 6 | def render(template, _assigns) do | |
| 7 | 2 | Phoenix.Controller.status_message_from_template(template) |
| 8 | end | |
| 9 | end |
| Line | Hits | Source |
|---|---|---|
| 0 | defmodule BrieflyWeb.ErrorJSON do | |
| 1 | # If you want to customize a particular status code, | |
| 2 | # you may add your own clauses, such as: | |
| 3 | # | |
| 4 | # def render("500.json", _assigns) do | |
| 5 | # %{errors: %{detail: "Internal Server Error"}} | |
| 6 | # end | |
| 7 | ||
| 8 | # By default, Phoenix returns the status message from | |
| 9 | # the template name. For example, "404.json" becomes | |
| 10 | # "Not Found". | |
| 11 | def render(template, _assigns) do | |
| 12 | 2 | %{errors: %{detail: Phoenix.Controller.status_message_from_template(template)}} |
| 13 | end | |
| 14 | end |
| Line | Hits | Source |
|---|---|---|
| 0 | defmodule BrieflyWeb.PageController do | |
| 1 | use BrieflyWeb, :controller | |
| 2 | ||
| 3 | @doc "A simple health/readiness check Endpoint" | |
| 4 | def health(conn, _params) do | |
| 5 | conn | |
| 6 | |> put_resp_content_type("text/plain") | |
| 7 | 1 | |> resp(200, "OK") |
| 8 | end | |
| 9 | ||
| 10 | @doc "A configurable start view for the feed" | |
| 11 | def home(%Plug.Conn{} = conn, _params) do | |
| 12 | 2 | path_params = |
| 13 | :briefly | |
| 14 | |> Application.fetch_env!(__MODULE__) | |
| 15 | |> Keyword.fetch!(:home_action) | |
| 16 | 2 | |> then(&%{"days" => &1}) |
| 17 | ||
| 18 | # NOTE: Set `path_params` in conn also, so that the header marks the correct entry as active. | |
| 19 | %Plug.Conn{conn | path_params: path_params} | |
| 20 | # NOTE: Why not redirect? CURL does not follow redirects by default. We want to make | |
| 21 | # using clients other than browsers simple. | |
| 22 | 2 | |> feed(path_params) |
| 23 | end | |
| 24 | ||
| 25 | @doc "Lists any problems encountered while parsing the feeds" | |
| 26 | def problems(conn, _params) do | |
| 27 | 1 | render(conn, :problems, problems: Briefly.list_problems()) |
| 28 | end | |
| 29 | ||
| 30 | @doc "Refreshes the feeds and renders out the home page" | |
| 31 | def refresh(conn, params) do | |
| 32 | 1 | :ok = Briefly.refresh() |
| 33 | 1 | home(conn, params) |
| 34 | end | |
| 35 | ||
| 36 | @doc "Renders the items parsed from all configured feeds" | |
| 37 | def feed(conn, params) do | |
| 38 | params | |
| 39 | |> list_opts() | |
| 40 | |> Briefly.list_items() | |
| 41 | 60 | |> Enum.group_by(& &1.group) |
| 42 | |> Enum.sort() | |
| 43 | 13 | |> then(&render(conn, :feed, grouped_items: &1)) |
| 44 | end | |
| 45 | ||
| 46 | defp list_opts(params) do | |
| 47 | 13 | with {:ok, days} <- Map.fetch(params, "days"), |
| 48 | 13 | number when is_integer(number) <- parse_to_days(days) do |
| 49 | [days_ago: max(0, number)] | |
| 50 | else | |
| 51 | 1 | :error -> [days_ago: 0] |
| 52 | end | |
| 53 | end | |
| 54 | ||
| 55 | defp parse_to_days(path_param) do | |
| 56 | path_param | |
| 57 | |> String.trim() | |
| 58 | |> String.downcase() | |
| 59 | 13 | |> case do |
| 60 | 3 | "today" -> 0 |
| 61 | 3 | "yesterday" -> 1 |
| 62 | 7 | days -> with {days, _rest} <- Integer.parse(days), do: days |
| 63 | end | |
| 64 | end | |
| 65 | end |
| Line | Hits | Source |
|---|---|---|
| 0 | defmodule BrieflyWeb.PageHTML do | |
| 1 | use BrieflyWeb, :html | |
| 2 | ||
| 3 | embed_templates "page_html/*" | |
| 4 | ||
| 5 | def render_date(%DateTime{} = dt) do | |
| 6 | Briefly.user_timezone() | |
| 7 | 60 | |> then(&Timex.Timezone.convert(dt, &1)) |
| 8 | 60 | |> Calendar.strftime("%a, %d of %b at %H:%M") |
| 9 | end | |
| 10 | end |
| Line | Hits | Source |
|---|---|---|
| 0 | defmodule BrieflyWeb.Endpoint do | |
| 1 | use Phoenix.Endpoint, otp_app: :briefly | |
| 2 | ||
| 3 | @session_options [ | |
| 4 | store: :cookie, | |
| 5 | key: "_briefly_key", | |
| 6 | signing_salt: "jVjNJ8iH", | |
| 7 | same_site: "Lax" | |
| 8 | ] | |
| 9 | ||
| 10 | # Serve at "/" the static files from "priv/static" directory. | |
| 11 | # | |
| 12 | # You should set gzip to true if you are running phx.digest | |
| 13 | # when deploying your static files in production. | |
| 14 | plug Plug.Static, | |
| 15 | at: "/", | |
| 16 | from: :briefly, | |
| 17 | gzip: not code_reloading?, | |
| 18 | only: BrieflyWeb.static_paths(), | |
| 19 | raise_on_missing_only: code_reloading? | |
| 20 | ||
| 21 | # Code reloading can be explicitly enabled under the | |
| 22 | # :code_reloader configuration of your endpoint. | |
| 23 | if code_reloading? do | |
| 24 | socket "/phoenix/live_reload/socket", Phoenix.LiveReloader.Socket | |
| 25 | plug Phoenix.LiveReloader | |
| 26 | plug Phoenix.CodeReloader | |
| 27 | end | |
| 28 | ||
| 29 | plug Plug.RequestId | |
| 30 | ||
| 31 | plug Plug.Parsers, | |
| 32 | parsers: [:urlencoded, :multipart, :json], | |
| 33 | pass: ["*/*"], | |
| 34 | json_decoder: Phoenix.json_library() | |
| 35 | ||
| 36 | plug Plug.MethodOverride | |
| 37 | plug Plug.Head | |
| 38 | plug Plug.Session, @session_options | |
| 39 | plug BrieflyWeb.Router | |
| 40 | end |
| Line | Hits | Source |
|---|---|---|
| 0 | defmodule BrieflyWeb.Gettext do | |
| 1 | @moduledoc """ | |
| 2 | A module providing Internationalization with a gettext-based API. | |
| 3 | ||
| 4 | By using [Gettext](https://hexdocs.pm/gettext), | |
| 5 | your module gains a set of macros for translations, for example: | |
| 6 | ||
| 7 | import BrieflyWeb.Gettext | |
| 8 | ||
| 9 | # Simple translation | |
| 10 | gettext("Here is the string to translate") | |
| 11 | ||
| 12 | # Plural translation | |
| 13 | ngettext("Here is the string to translate", | |
| 14 | "Here are the strings to translate", | |
| 15 | 3) | |
| 16 | ||
| 17 | # Domain-based translation | |
| 18 | dgettext("errors", "Here is the error message to translate") | |
| 19 | ||
| 20 | See the [Gettext Docs](https://hexdocs.pm/gettext) for detailed usage. | |
| 21 | """ | |
| 22 | use Gettext.Backend, otp_app: :briefly | |
| 23 | end |
| Line | Hits | Source |
|---|---|---|
| 0 | defmodule BrieflyWeb.Router do | |
| 1 | use BrieflyWeb, :router | |
| 2 | ||
| 3 | 15 | pipeline :browser do |
| 4 | plug :accepts, ["html"] | |
| 5 | plug :put_root_layout, html: {BrieflyWeb.Layouts, :root} | |
| 6 | plug :put_secure_browser_headers | |
| 7 | end | |
| 8 | ||
| 9 | 0 | pipeline :api do |
| 10 | plug :accepts, ["json"] | |
| 11 | end | |
| 12 | ||
| 13 | scope "/", BrieflyWeb do | |
| 14 | pipe_through :browser | |
| 15 | ||
| 16 | 1 | get "/health", PageController, :health |
| 17 | ||
| 18 | 1 | get "/", PageController, :home |
| 19 | 11 | get "/since/:days", PageController, :feed |
| 20 | ||
| 21 | 1 | get "/problems", PageController, :problems |
| 22 | ||
| 23 | 1 | post "/refresh", PageController, :refresh |
| 24 | end | |
| 25 | end |