All domains: sitemap.xml and robots.txt

1 files changed, 14 insertions, 0 deletions

diff --git a/templates/robots.txt b/templates/robots.txt
new file mode 100644
index 00000000..be2a512c
--- /dev/null
+++ b/templates/robots.txt
@@ -0,0 +1,14 @@
+{# robots.txt is a little picky about whitespace and newlines - that's why the below looks kind of bad.#}
+{% if rules is defined and rules %}
+{% for user_agent, disallowed in rules %}
+User-agent: {{ user_agent }}
+{% for rule in rules %}
+Disallow: {{ rule }}
+{% endfor %}
+{% endfor %}
+{% else %}
+User-agent: *
+Disallow:
+{% endif %}
+
+Sitemap: {{ sitemap_url }}