Replace `markdown2` with `markdown` and `python-frontmatter`.

This allows us to properly escape codeblocks within markdown, permalink
to headers on a page, and decouples getting metadata from a file and
getting generated HTML from the Markdown content.

8 files changed, 146 insertions, 41 deletions

diff --git a/Pipfile b/Pipfile
index 8af6e3ce..c2b87ab4 100644
--- a/Pipfile
+++ b/Pipfile
@@ -18,7 +18,8 @@ pyyaml = "~=5.1"
 pyuwsgi = {version = "~=2.0", sys_platform = "!='win32'"}
 sentry-sdk = "~=0.14"
 gitpython = "~=3.1.7"
-markdown2 = "~=2.3.9"
+markdown = "~=3.3.4"
+python-frontmatter = "~=1.0"
 
 [dev-packages]
 coverage = "~=5.0"
diff --git a/Pipfile.lock b/Pipfile.lock
index 3dfb7ec6..4f8bd0d3 100644
--- a/Pipfile.lock
+++ b/Pipfile.lock
@@ -1,7 +1,7 @@
 {
     "_meta": {
         "hash": {
-            "sha256": "893ab5cc0b64f7bb87c0304c9fb6eb24b1856c40286fbbdb1a2bf4a0d7a6d39f"
+            "sha256": "a338a377b64a5d25bf925646c97d932a2d1c783fc8f40d91b6a9ab8f30c2b14e"
         },
         "pipfile-spec": 6,
         "requires": {
@@ -118,6 +118,14 @@
             "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
             "version": "==2.10"
         },
+        "importlib-metadata": {
+            "hashes": [
+                "sha256:742add720a20d0467df2f444ae41704000f50e1234f46174b51f9c6031a1bd71",
+                "sha256:b74159469b464a99cb8cc3e21973e4d96e05d3024d337313fedb618a6e86e6f4"
+            ],
+            "markers": "python_version < '3.8'",
+            "version": "==3.7.3"
+        },
         "libsass": {
             "hashes": [
                 "sha256:1521d2a8d4b397c6ec90640a1f6b5529077035efc48ef1c2e53095544e713d1b",
@@ -136,13 +144,13 @@
             ],
             "version": "==0.20.1"
         },
-        "markdown2": {
+        "markdown": {
             "hashes": [
-                "sha256:85956d8119fa6378156fef65545d66705a842819d2e1b50379a2b9d2aaa17cf0",
-                "sha256:fef148e5fd68d4532286c3e2943e9d2c076a8ad781b0a70a9d599a0ffe91652d"
+                "sha256:31b5b491868dcc87d6c24b7e3d19a0d730d59d3e46f4eea6430a321bed387a49",
+                "sha256:96c3ba1261de2f7547b46a00ea8463832c921d3f9d6aba3f255a6f71386db20c"
             ],
             "index": "pypi",
-            "version": "==2.3.10"
+            "version": "==3.3.4"
         },
         "psycopg2-binary": {
             "hashes": [
@@ -185,6 +193,14 @@
             "index": "pypi",
             "version": "==2.8.6"
         },
+        "python-frontmatter": {
+            "hashes": [
+                "sha256:766ae75f1b301ffc5fe3494339147e0fd80bc3deff3d7590a93991978b579b08",
+                "sha256:e98152e977225ddafea6f01f40b4b0f1de175766322004c826ca99842d19a7cd"
+            ],
+            "index": "pypi",
+            "version": "==1.0.0"
+        },
         "pytz": {
             "hashes": [
                 "sha256:83a4a90894bf38e243cf052c8b58f381bfe9a7a483f6a9cab140bc7f702ac4da",
@@ -288,6 +304,15 @@
             "markers": "python_version >= '3.5'",
             "version": "==0.4.1"
         },
+        "typing-extensions": {
+            "hashes": [
+                "sha256:7cb407020f00f7bfc3cb3e7881628838e69d8f3fcab2f64742a5e76b2f841918",
+                "sha256:99d4073b617d30288f569d3f13d2bd7548c3a7e4c8de87db09a9d29bb3a4a60c",
+                "sha256:dafc7639cde7f1b6e1acc0f457842a83e722ccca8eef5270af2d74792619a89f"
+            ],
+            "markers": "python_version < '3.8'",
+            "version": "==3.7.4.3"
+        },
         "urllib3": {
             "hashes": [
                 "sha256:2f4da4594db7e1e110a944bb1b551fdf4e6c136ad42e4234131391e21eb5b0df",
@@ -303,6 +328,14 @@
             ],
             "index": "pypi",
             "version": "==5.2.0"
+        },
+        "zipp": {
+            "hashes": [
+                "sha256:3607921face881ba3e026887d8150cca609d517579abe052ac81fc5aeffdbd76",
+                "sha256:51cb66cc54621609dd593d1787f286ee42a5c0adbb4b29abea5a63edc3e03098"
+            ],
+            "markers": "python_version >= '3.6'",
+            "version": "==3.4.1"
         }
     },
     "develop": {
diff --git a/pydis_site/apps/content/resources/guides/pydis-guides/how-to-contribute-a-page.md b/pydis_site/apps/content/resources/guides/pydis-guides/how-to-contribute-a-page.md
index 8967e7da..12969ba2 100644
--- a/pydis_site/apps/content/resources/guides/pydis-guides/how-to-contribute-a-page.md
+++ b/pydis_site/apps/content/resources/guides/pydis-guides/how-to-contribute-a-page.md
@@ -69,18 +69,75 @@ You can learn more about Markdown metadata [here](https://github.com/trentm/pyth
 
 Apart from standard Markdown, certain additions are available:
 
+### Abbreviations
+HTML `<abbr>` tags can be used in markdown using this format:
+
+**Markdown:**
+```nohighlight
+This website is HTML generated from YAML and Markdown.
+
+*[HTML]: Hyper Text Markup Language
+*[YAML]: YAML Ain't Markup Language
+```
+
+**Output:**
+
+This website is <abbr title="Hyper Text Markup Language">HTML</abbr>
+generated from <abbr title="YAML Ain't Markup Language">YAML</abbr> and Markdown.
+
+---
+
+### Footnotes
+**Markdown:**
+```nohighlight
+This footnote[^1] links to the bottom[^custom_label] of the page[^3].
+
+[^1]: Footnote labels start with a caret `^`.
+[^3]: The footnote link is numbered based on the order of the labels.
+[^custom label]: Footnote labels can contain any text within square brackets.
+```
+
+**Output:**
+
+This footnote[^1] links to the bottom[^custom label] of the page[^3].
+
+[^1]: Footnote labels start with a caret `^`.
+[^3]: The footnote link is numbered based on the order of the labels.
+[^custom label]: Footnote labels can contain any text within square brackets.
+
+---
+
 ### Tables
 
+**Markdown:**
+```nohighlight
+| This is header | This is another header |
+| -------------- | ---------------------- |
+| An item        | Another item           |
+```
+
+**Output:**
+
 | This is header | This is another header |
 | -------------- | ---------------------- |
 | An item        | Another item           |
-| Big item       | Small item             |
 
+---
 
 ### Codeblock Syntax Highlighting
 Syntax highlighting is provided by `highlight.js`.
 To activate syntax highlighting, put the language directly after the starting backticks.
 
+**Markdown:**
+````nohighlight
+```python
+import os
+
+path = os.path.join("foo", "bar")
+```
+````
+
+**Output:**
 ```python
 import os
 
diff --git a/pydis_site/apps/content/utils.py b/pydis_site/apps/content/utils.py
index 0faf722c..11d2b792 100644
--- a/pydis_site/apps/content/utils.py
+++ b/pydis_site/apps/content/utils.py
@@ -1,9 +1,11 @@
 from pathlib import Path
-from typing import Dict, Union
+from typing import Dict, Tuple
 
+import frontmatter
+import markdown
 import yaml
 from django.http import Http404
-from markdown2 import markdown
+from markdown.extensions.toc import TocExtension
 
 
 def get_category(path: Path) -> Dict[str, str]:
@@ -31,29 +33,25 @@ def get_category_pages(path: Path) -> Dict[str, Dict]:
 
     for item in path.glob("*.md"):
         if item.is_file():
-            md = markdown(item.read_text(), extras=["metadata"])
-            pages[item.stem] = md.metadata
+            pages[item.stem] = frontmatter.load(item)
 
     return pages
 
 
-def get_page(path: Path) -> Dict[str, Union[str, Dict]]:
+def get_page(path: Path) -> Tuple[str, Dict]:
     """Get one specific page."""
     if not path.is_file():
         raise Http404("Page not found.")
 
-    html = markdown(
-        path.read_text(encoding="utf-8"),
-        extras=[
-            "metadata",
-            "fenced-code-blocks",
-            "highlightjs-lang",
-            "header-ids",
-            "strike",
-            "target-blank-links",
-            "tables",
-            "task_list"
+    metadata, content = frontmatter.parse(path.read_text(encoding="utf-8"))
+    html = markdown.markdown(
+        content,
+        extensions=[
+            "extra",
+            # Empty string for marker to disable text searching for [TOC]
+            # By using a metadata key instead, we save time on long markdown documents
+            TocExtension(title="Table of Contents:", permalink=True, marker="")
         ]
     )
 
-    return {"content": str(html), "metadata": html.metadata}
+    return str(html), metadata
diff --git a/pydis_site/apps/content/views/page_category.py b/pydis_site/apps/content/views/page_category.py
index 91aed7f0..4a2ed2d6 100644
--- a/pydis_site/apps/content/views/page_category.py
+++ b/pydis_site/apps/content/views/page_category.py
@@ -35,18 +35,19 @@ class PageOrCategoryView(TemplateView):
 
         if self.full_location.is_dir():
             context["categories"] = utils.get_categories(self.full_location)
+            context["pages"] = utils.get_category_pages(self.full_location)
+
             category = utils.get_category(self.full_location)
-            context["category_info"] = category
             context["page_title"] = category["name"]
             context["page_description"] = category["description"]
-            context["pages"] = utils.get_category_pages(self.full_location)
+
             context["path"] = f"{self.location}/"  # Add trailing slash here to simplify template
         elif self.full_location.with_suffix(".md").is_file():
-            page_result = utils.get_page(self.full_location.with_suffix(".md"))
-            context["page"] = page_result
-            context["page_title"] = page_result["metadata"]["title"]
-            context["page_description"] = page_result["metadata"]["description"]
-            context["relevant_links"] = page_result["metadata"].get("relevant_links", {})
+            page, metadata = utils.get_page(self.full_location.with_suffix(".md"))
+            context["page"] = page
+            context["page_title"] = metadata["title"]
+            context["page_description"] = metadata["description"]
+            context["relevant_links"] = metadata.get("relevant_links", {})
         else:
             raise Http404
 
diff --git a/pydis_site/static/css/content/page.css b/pydis_site/static/css/content/page.css
index f46d6b15..57d7472b 100644
--- a/pydis_site/static/css/content/page.css
+++ b/pydis_site/static/css/content/page.css
@@ -6,11 +6,26 @@ i.has-icon-padding {
     padding: 0 10px 25px 0;
 }
 
-pre {
-    /*
-     * Style it the same as the <code> tag, since highlight.js does not style
-     * backgrounds of <pre> tags but bulma does, resulting in a weird off-white
-     * border.
-     */
-    background-color: #282c34;
+/*
+ * Move padding padding from <pre> tag to hljs <code> tags so the padding
+ * space is colored the same as the background of hljs <code> blocks.
+ */
+.content pre {
+    padding: 0;
+}
+
+code.hljs {
+    padding: 1.75em 2em;
+}
+
+/*
+ * Show header permalink on hover.
+ */
+.headerlink {
+    display: none;
+    padding-left: 0.5em;
+}
+
+:is(h1, h2, h3, h4, h5, h6):hover > .headerlink {
+    display: inline;
 }
diff --git a/pydis_site/templates/content/base.html b/pydis_site/templates/content/base.html
index 1508dfb3..19eec5d4 100644
--- a/pydis_site/templates/content/base.html
+++ b/pydis_site/templates/content/base.html
@@ -3,7 +3,7 @@
 
 {% block title %}{{ page_title }}{% endblock %}
 {% block head %}
-<meta property="og:title" content="Python Discord - {{ page_title }}" />
+    <meta property="og:title" content="Python Discord - {{ page_title }}" />
     <meta property="og:type" content="website" />
     <meta property="og:description" content="{{ page_description }}" />
     <link rel="stylesheet" href="{% static "css/content/page.css" %}">
diff --git a/pydis_site/templates/content/page.html b/pydis_site/templates/content/page.html
index 5e820c26..06d74208 100644
--- a/pydis_site/templates/content/page.html
+++ b/pydis_site/templates/content/page.html
@@ -12,7 +12,7 @@
     {% if relevant_links|length > 0 %}
         <div class="columns is-variable is-8">
             <div class="column is-two-thirds">
-                {{ page.content|safe }}
+                {{ page|safe }}
             </div>
             <div class="column">
                 <div class="box">
@@ -26,6 +26,6 @@
             </div>
         </div>
     {% else %}
-        <div>{{ page.content|safe }}</div>
+        <div>{{ page|safe }}</div>
     {% endif %}
 {% endblock %}