1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
|
defmodule Lithium.Util.PublicSuffix do
use GenServer
require Logger
@moduledoc """
A GenServer that loads the public suffix list from a file and provides
functionality to find the organisational domain of a given domain name.
"""
@public_suffix_list_path Application.app_dir(:lithium, "priv/public_suffix_list.dat")
@public_suffix_list_url "https://publicsuffix.org/list/public_suffix_list.dat"
@public_suffix_max_age div(:timer.hours(7 * 24), 1000) # 7 days in seconds
def start_link(_) do
GenServer.start_link(__MODULE__, :ok, name: __MODULE__)
end
@impl true
def init(:ok) do
if File.exists?(@public_suffix_list_path) and psl_fresh?() do
Logger.info("Loading public suffix list from disk cache.")
Process.send_after(self(), :update_public_suffix_list, psl_next_refresh() - :os.system_time(:seconds))
{:ok, load_public_suffix_list()}
else
Logger.info("Public suffix list file not found or not fresh, fetching from remote.")
case fetch_public_suffix_list() do
{:ok, list} ->
Process.send_after(self(), :update_public_suffix_list, @public_suffix_max_age)
{:ok, list}
{:error, reason} ->
Logger.error("Failed to fetch public suffix list: #{reason}")
{:stop, reason}
end
end
end
@impl true
def handle_info(:update_public_suffix_list, state) do
case fetch_public_suffix_list() do
{:ok, new_list} ->
Process.send_after(self(), :update_public_suffix_list, @public_suffix_max_age)
{:noreply, new_list}
{:error, reason} ->
Logger.error("Failed to update public suffix list: #{reason}")
{:noreply, state}
end
end
@impl true
def handle_call({:get_domain, domain}, _from, state) do
{:reply, get_od_for_domain(domain, state), state}
end
def load_public_suffix_list do
case File.read(@public_suffix_list_path) do
{:ok, content} ->
content
|> String.split("\n", trim: true)
|> Enum.reject(&String.starts_with?(&1, "//"))
|> Enum.map(&String.trim/1)
|> Enum.map(&String.downcase/1)
|> Enum.map(&String.replace(&1, "*.", ""))
{:error, reason} ->
Logger.error("Failed to load public suffix list: #{reason}")
[]
end
end
def fetch_public_suffix_list do
Logger.info("Fetching public suffix list from #{@public_suffix_list_url}")
request = :httpc.request(:get, {@public_suffix_list_url, []}, [], [])
case request do
{:ok, {{_, 200, _}, _, body}} ->
File.write(@public_suffix_list_path, body)
{:ok, load_public_suffix_list()}
{:error, reason} ->
{:error, reason}
end
end
defp get_od_for_domain(domain, public_suffix_list) do
# x.y.z.jb3.dev -> jb3.dev
# ["com", "jb3", "example"]
domain_parts = String.split(domain, ".") |> Enum.reverse()
# [["com", "public-suffix"], ...]
suffix_parts =
Enum.map(public_suffix_list, &String.split(&1, ".")) |> Enum.map(&Enum.reverse/1)
# Add domain parts together until it is no longer in the suffix_parts list
Enum.reduce_while(domain_parts, [], fn part, acc ->
new_acc = acc ++ [part]
if Enum.member?(suffix_parts, new_acc) do
{:cont, new_acc}
else
{:halt, new_acc}
end
end)
|> Enum.reverse()
|> Enum.join(".")
|> String.downcase()
|> String.trim()
end
def get_domain(domain) do
GenServer.call(__MODULE__, {:get_domain, domain})
end
defp psl_fresh?() do
case File.stat(@public_suffix_list_path, time: :posix) do
{:ok, stat} ->
now = :os.system_time(:seconds)
stat.mtime + @public_suffix_max_age > now
{:error, _} ->
false
end
end
defp psl_next_refresh() do
case File.stat(@public_suffix_list_path, time: :posix) do
{:ok, stat} ->
stat.mtime + @public_suffix_max_age
{:error, _} ->
nil
end
end
end
|
