diff options
author | 2022-12-09 22:27:08 -0800 | |
---|---|---|
committer | 2022-12-09 22:27:08 -0800 | |
commit | aeeec9d11307b75ed117dabf7c04698bd9186d77 (patch) | |
tree | a23267fa619b5f98187ec5e027d3701c6e34f231 /pydis_site/apps/content | |
parent | Appeased the requests from reviews. (diff) | |
parent | Merge pull request #811 from python-discord/dependabot/pip/flake8-bugbear-22.... (diff) |
Merge branch 'main' into 695-setting-different-statuses-on-your-bot
Diffstat (limited to 'pydis_site/apps/content')
37 files changed, 1942 insertions, 265 deletions
diff --git a/pydis_site/apps/content/apps.py b/pydis_site/apps/content/apps.py index 1e300a48..96019e1c 100644 --- a/pydis_site/apps/content/apps.py +++ b/pydis_site/apps/content/apps.py @@ -4,4 +4,4 @@ from django.apps import AppConfig class ContentConfig(AppConfig): """Django AppConfig for content app.""" - name = 'content' + name = 'pydis_site.apps.content' diff --git a/pydis_site/apps/content/migrations/0001_add_tags.py b/pydis_site/apps/content/migrations/0001_add_tags.py new file mode 100644 index 00000000..2c31e4c1 --- /dev/null +++ b/pydis_site/apps/content/migrations/0001_add_tags.py @@ -0,0 +1,35 @@ +# Generated by Django 4.0.6 on 2022-08-23 09:06 + +import django.db.models.deletion +from django.db import migrations, models + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + ] + + operations = [ + migrations.CreateModel( + name='Commit', + fields=[ + ('sha', models.CharField(help_text='The SHA hash of this commit.', max_length=40, primary_key=True, serialize=False)), + ('message', models.TextField(help_text='The commit message.')), + ('date', models.DateTimeField(help_text='The date and time the commit was created.')), + ('authors', models.TextField(help_text='The person(s) who created the commit. This is a serialized JSON object. Refer to the GitHub documentation on the commit endpoint (schema/commit.author & schema/commit.committer) for more info. https://docs.github.com/en/rest/commits/commits#get-a-commit')), + ], + ), + migrations.CreateModel( + name='Tag', + fields=[ + ('last_updated', models.DateTimeField(auto_now=True, help_text='The date and time this data was last fetched.')), + ('sha', models.CharField(help_text="The tag's hash, as calculated by GitHub.", max_length=40)), + ('name', models.CharField(help_text="The tag's name.", max_length=50, primary_key=True, serialize=False)), + ('group', models.CharField(help_text='The group the tag belongs to.', max_length=50, null=True)), + ('body', models.TextField(help_text='The content of the tag.')), + ('last_commit', models.ForeignKey(help_text='The commit this file was last touched in.', null=True, on_delete=django.db.models.deletion.CASCADE, to='content.commit')), + ], + ), + ] diff --git a/pydis_site/apps/content/migrations/__init__.py b/pydis_site/apps/content/migrations/__init__.py new file mode 100644 index 00000000..e69de29b --- /dev/null +++ b/pydis_site/apps/content/migrations/__init__.py diff --git a/pydis_site/apps/content/models/__init__.py b/pydis_site/apps/content/models/__init__.py new file mode 100644 index 00000000..60007e27 --- /dev/null +++ b/pydis_site/apps/content/models/__init__.py @@ -0,0 +1,3 @@ +from .tag import Commit, Tag + +__all__ = ["Commit", "Tag"] diff --git a/pydis_site/apps/content/models/tag.py b/pydis_site/apps/content/models/tag.py new file mode 100644 index 00000000..1a20d775 --- /dev/null +++ b/pydis_site/apps/content/models/tag.py @@ -0,0 +1,80 @@ +import collections.abc +import json + +from django.db import models + + +class Commit(models.Model): + """A git commit from the Python Discord Bot project.""" + + URL_BASE = "https://github.com/python-discord/bot/commit/" + + sha = models.CharField( + help_text="The SHA hash of this commit.", + primary_key=True, + max_length=40, + ) + message = models.TextField(help_text="The commit message.") + date = models.DateTimeField(help_text="The date and time the commit was created.") + authors = models.TextField(help_text=( + "The person(s) who created the commit. This is a serialized JSON object. " + "Refer to the GitHub documentation on the commit endpoint " + "(schema/commit.author & schema/commit.committer) for more info. " + "https://docs.github.com/en/rest/commits/commits#get-a-commit" + )) + + @property + def url(self) -> str: + """The URL to the commit on GitHub.""" + return self.URL_BASE + self.sha + + def lines(self) -> collections.abc.Iterable[str]: + """Return each line in the commit message.""" + for line in self.message.split("\n"): + yield line + + def format_authors(self) -> collections.abc.Iterable[str]: + """Return a nice representation of the author(s)' name and email.""" + for author in json.loads(self.authors): + yield f"{author['name']} <{author['email']}>" + + +class Tag(models.Model): + """A tag from the python-discord bot repository.""" + + URL_BASE = "https://github.com/python-discord/bot/tree/main/bot/resources/tags" + + last_updated = models.DateTimeField( + help_text="The date and time this data was last fetched.", + auto_now=True, + ) + sha = models.CharField( + help_text="The tag's hash, as calculated by GitHub.", + max_length=40, + ) + last_commit = models.ForeignKey( + Commit, + help_text="The commit this file was last touched in.", + null=True, + on_delete=models.CASCADE, + ) + name = models.CharField( + help_text="The tag's name.", + primary_key=True, + max_length=50, + ) + group = models.CharField( + help_text="The group the tag belongs to.", + null=True, + max_length=50, + ) + body = models.TextField(help_text="The content of the tag.") + + @property + def url(self) -> str: + """Get the URL of the tag on GitHub.""" + url = Tag.URL_BASE + if self.group: + url += f"/{self.group}" + url += f"/{self.name}.md" + return url diff --git a/pydis_site/apps/content/resources/guides/pydis-guides/asking-good-questions.md b/pydis_site/apps/content/resources/guides/pydis-guides/asking-good-questions.md index 971989a9..b08ba7c6 100644 --- a/pydis_site/apps/content/resources/guides/pydis-guides/asking-good-questions.md +++ b/pydis_site/apps/content/resources/guides/pydis-guides/asking-good-questions.md @@ -26,7 +26,7 @@ If none of the above steps help you or you're not sure how to do some of the abo # A Good Question -When you're ready to ask a question, there's a few things you should have to hand before forming a query. +When you're ready to ask a question, there are a few things you should have to hand before forming a query. * A code example that illustrates your problem * If possible, make this a minimal example rather than an entire application diff --git a/pydis_site/apps/content/resources/guides/pydis-guides/contributing.md b/pydis_site/apps/content/resources/guides/pydis-guides/contributing.md index 4013962c..2822d046 100644 --- a/pydis_site/apps/content/resources/guides/pydis-guides/contributing.md +++ b/pydis_site/apps/content/resources/guides/pydis-guides/contributing.md @@ -4,7 +4,7 @@ description: A guide to contributing to our open source projects. icon: fab fa-github --- -Our projects on Python Discord are open source and [available on Github](https://github.com/python-discord). If you would like to contribute, consider one of the following projects: +Our projects on Python Discord are open source and [available on GitHub](https://github.com/python-discord). If you would like to contribute, consider one of the following projects: <!-- Project cards --> <div class="columns is-multiline is-centered is-3 is-variable"> @@ -19,11 +19,7 @@ Our projects on Python Discord are open source and [available on Github](https:/ </div> <div class="card-content"> <div class="content"> - Our community-driven Discord bot. - </div> - <div class="tags has-addons"> - <span class="tag is-dark">Difficulty</span> - <span class="tag is-primary">Beginner</span> + Sir Lancebot has a collection of self-contained, for-fun features. If you're new to Discord bots or contributing, this is a great place to start! </div> </div> <div class="card-footer"> @@ -46,11 +42,7 @@ Our projects on Python Discord are open source and [available on Github](https:/ </div> <div class="card-content"> <div class="content"> - The community and moderation Discord bot. - </div> - <div class="tags has-addons"> - <span class="tag is-dark">Difficulty</span> - <span class="tag is-warning">Intermediate</span> + Called @Python on the server, this bot handles moderation tools, help channels, and other critical features for our community. </div> </div> <div class="card-footer"> @@ -73,11 +65,7 @@ Our projects on Python Discord are open source and [available on Github](https:/ </div> <div class="card-content"> <div class="content"> - The website, subdomains and API. - </div> - <div class="tags has-addons"> - <span class="tag is-dark">Difficulty</span> - <span class="tag is-danger">Advanced</span> + This website itself! This project is built with Django and includes our API, which is used by various services such as @Python. </div> </div> <div class="card-footer"> @@ -91,26 +79,64 @@ Our projects on Python Discord are open source and [available on Github](https:/ </div> </div> -If you don't understand anything or need clarification, feel free to ask any staff member with the **@PyDis Core Developers** role in the server. We're always happy to help! +# How do I start contributing? +Unsure of what contributing to open source projects involves? Have questions about how to use GitHub? Just need to know about our contribution etiquette? Completing these steps will have you ready to make your first contribution no matter your starting point. + +Feel free to skip any steps you're already familiar with, but please make sure not to miss the [Contributing Guidelines](#5-read-our-contributing-guidelines). + +If you are here looking for the answer to a specific question, check out the sub-articles in the top right of the page to see a list of our guides. + +**Note:** We use Git to keep track of changes to the files in our projects. Git allows you to make changes to your local code and then distribute those changes to the other people working on the project. You'll use Git in a couple steps of the contributing process. You can refer to this [**guide on using Git**](./working-with-git/). +{: .notification } + +### 1. Fork and clone the repo +GitHub is a website based on Git that stores project files in the cloud. We use GitHub as a central place for sending changes, reviewing others' changes, and communicating with each other. You'll need to create a copy under your own GitHub account, a.k.a. "fork" it. You'll make your changes to this copy, which can then later be merged into the Python Discord repository. + +*Note: Members of the Python Discord staff can create feature branches directly on the repo without forking it.* + +Check out our [**guide on forking a GitHub repo**](./forking-repository/). + +Now that you have your own fork you need to be able to make changes to the code. You can clone the repo to your local machine, commit changes to it there, then push those changes to GitHub. + +Check out our [**guide on cloning a GitHub repo**](./cloning-repository/). + +### 2. Set up the project +You have the source code on your local computer, now how do you actually run it? We have detailed guides on setting up the environment for each of our main projects: + +* [**Sir Lancebot**](./sir-lancebot/) + +* [**Python Bot**](./bot/) + +* [**Site**](./site/) + +### 3. Read our Contributing Guidelines +We have a few short rules that all contributors must follow. Make sure you read and follow them while working on our projects. + +[**Contributing Guidelines**](./contributing-guidelines/). + +As mentioned in the Contributing Guidelines, we have a simple style guide for our projects based on PEP 8. Give it a read to keep your code consistent with the rest of the codebase. + +[**Style Guide**](./style-guide/) + +### 4. Create an issue +The first step to any new contribution is an issue describing a problem with the current codebase or proposing a new feature. All the open issues are viewable on the GitHub repositories, for instance here is the [issues page for Sir Lancebot](https://github.com/python-discord/sir-lancebot/issues). If you have something that you want to implement open a new issue to present your idea. Otherwise, you can browse the unassigned issues and ask to be assigned to one that you're interested in, either in the comments on the issue or in the [`#dev-contrib`](https://discord.gg/2h3qBv8Xaa) channel on Discord. + +[**How to write a good issue**](./issues/) -### Useful Resources +Don't move forward until your issue is approved by a Core Developer. Issues are not guaranteed to be approved so your work may be wasted. +{: .notification .is-warning } -[Guidelines](./contributing-guidelines/) - General guidelines you should follow when contributing to our projects.<br> -[Style Guide](./style-guide/) - Information regarding the code styles you should follow when working on our projects.<br> -[Review Guide](../code-reviews-primer/) - A guide to get you started on doing code reviews. +### 5. Make changes +Now it is time to make the changes to fulfill your approved issue. You should create a new Git branch for your feature; that way you can keep your main branch up to date with ours and even work on multiple features at once in separate branches. -## Contributors Community -We are very happy to have many members in our community that contribute to [our open source projects](https://github.com/python-discord/). -Whether it's writing code, reviewing pull requests, or contributing graphics for our events, it’s great to see so many people being motivated to help out. -As a token of our appreciation, those who have made significant contributions to our projects will receive a special **@Contributors** role on our server that makes them stand out from other members. -That way, they can also serve as guides to others who are looking to start contributing to our open source projects or open source in general. +This is a good time to review [how to write good commit messages](./contributing-guidelines/commit-messages) if you haven't already. -#### Guidelines for the @Contributors Role +### 6. Open a pull request +After your issue has been approved and you've written your code and tested it, it's time to open a pull request. Pull requests are a feature in GitHub; you can think of them as asking the project maintainers to accept your changes. This gives other contributors a chance to review your code and make any needed changes before it's merged into the main branch of the project. -One question we get a lot is what the requirements for the **@Contributors** role are. -As it’s difficult to precisely quantify contributions, we’ve come up with the following guidelines for the role: +Check out our [**Pull Request Guide**](./pull-requests/) for help with opening a pull request and going through the review process. -- The member has made several significant contributions to our projects. -- The member has a positive influence in our contributors subcommunity. +Check out our [**Code Review Guide**](../code-reviews-primer/) to learn how to be a star reviewer. Reviewing PRs is a vital part of open source development, and we always need more reviewers! -The role will be assigned at the discretion of the Admin Team in consultation with the Core Developers Team. +### That's it! +Thank you for contributing to our community projects. If there's anything you don't understand or you just want to discuss with other contributors, come visit the [`#dev-contrib`](https://discord.gg/2h3qBv8Xaa) channel to ask questions. Keep an eye out for staff members with the **@PyDis Core Developers** role in the server; we're always happy to help! diff --git a/pydis_site/apps/content/resources/guides/pydis-guides/contributing/bot.md b/pydis_site/apps/content/resources/guides/pydis-guides/contributing/bot.md index 2aa10aa3..02316bca 100644 --- a/pydis_site/apps/content/resources/guides/pydis-guides/contributing/bot.md +++ b/pydis_site/apps/content/resources/guides/pydis-guides/contributing/bot.md @@ -5,34 +5,9 @@ icon: fab fa-github toc: 3 --- The purpose of this guide is to get you a running local version of [the Python bot](https://github.com/python-discord/bot). -This page will focus on the quickest steps one can take, with mentions of alternatives afterwards. - -### Clone The Repository -First things first, to run the bot's code and make changes to it, you need a local version of it (on your computer). +You should have already forked the repository and cloned it to your local machine. If not, check out our [detailed walkthrough](../#1-fork-and-clone-the-repo). -<div class="card"> - <button type="button" class="card-header collapsible"> - <span class="card-header-title subtitle is-6 my-2 ml-2">Getting started with Git and GitHub</span> - <span class="card-header-icon"> - <i class="fas fa-fw fa-angle-down title is-5" aria-hidden="true"></i> - </span> - </button> - <div class="collapsible-content collapsed"> - <div class="card-content"> - <p>If you don't have Git on your computer already, <a href="https://git-scm.com/downloads">install it</a>. You can additionally install a Git GUI such as <a href="https://www.gitkraken.com/download">GitKraken</a>, or the <a href="https://cli.github.com/manual/installation">GitHub CLI</a>.</p> - <p>To learn more about Git, you can look into <a href="../working-with-git">our guides</a>, as well as <a href="https://education.github.com/git-cheat-sheet-education.pdf">this cheatsheet</a>, <a href="https://learngitbranching.js.org">Learn Git Branching</a>, and otherwise any guide you can find on the internet. Once you got the basic idea though, the best way to learn Git is to use it.</p> - <p>Creating a copy of a repository under your own account is called a <em>fork</em>. This is where all your changes and commits will be pushed to, and from where your pull requests will originate from.</p> - <p><strong><a href="../forking-repository">Learn about forking a project</a></strong>.</p> - </div> - </div> -</div> -<br> - -You will need to create a fork of [the project](https://github.com/python-discord/bot), and clone the fork. -Once this is done, you will have completed the first step towards having a running version of the bot. - -#### Working on the Repository Directly -If you are a member of the organisation (a member of [this list](https://github.com/orgs/python-discord/people), or in our particular case, server staff), you can clone the project repository without creating a fork, and work on a feature branch instead. +This page will focus on the quickest steps one can take, with mentions of alternatives afterwards. --- @@ -113,6 +88,7 @@ urls: # Snekbox snekbox_eval_api: "http://localhost:8060/eval" + snekbox_311_eval_api: "http://localhost:8065/eval" ##### << Replace the following � characters with the channel IDs in your test server >> ##### # This assumes the template was used: https://discord.new/zmHtscpYN9E3 @@ -506,10 +482,14 @@ You are now almost ready to run the Python bot. The simplest way to do so is wit In your `config.yml` file: * Set `urls.site` to `"web:8000"`. -* If you wish to work with snekbox set `urls.snekbox_eval_api` to `"http://snekbox:8060/eval"`. +* If you wish to work with snekbox set the following: + * `urls.snekbox_eval_api` to `"http://snekbox:8060/eval"` + * `urls.snekbox_311_eval_api` to `"http://snekbox-311:8060/eval"`. Assuming you have Docker installed **and running**, enter the cloned repo in the command line and type `docker-compose up`. +If working with snekbox you can run `docker-compose --profile 3.10 up` to also start up a 3.10 snekbox container, in addition to the default 3.11 container! + After pulling the images and building the containers, your bot will start. Enter your server and type `!help` (or whatever prefix you chose instead of `!`). Your bot is now running, but this method makes debugging with an IDE a fairly involved process. For additional running methods, continue reading the following sections. @@ -519,12 +499,13 @@ The advantage of this method is that you can run the bot's code in your preferre * Append the following line to your `.env` file: `BOT_API_KEY=badbot13m0n8f570f942013fc818f234916ca531`. * In your `config.yml` file, set `urls.site` to `"localhost:8000"`. If you wish to keep using `web:8000`, then [COMPOSE_PROJECT_NAME](../docker/#compose-project-names) has to be set. -* To work with snekbox, set `urls.snekbox_eval_api` to `"http://localhost:8060/eval"` +* To work with snekbox, set `urls.snekbox_eval_api` to `"http://localhost:8060/eval"` and `urls.snekbox_311_eval_api` to `"http://localhost:8065/eval"` You will need to start the services separately, but if you got the previous section with Docker working, that's pretty simple: * `docker-compose up web` to start the site container. This is required. * `docker-compose up snekbox` to start the snekbox container. You only need this if you're planning on working on the snekbox cog. +* `docker-compose up snekbox-311` to start the snekbox 3.11 container. You only need this if you're planning on working on the snekbox cog. * `docker-compose up redis` to start the Redis container. You only need this if you're not using fakeredis. For more info refer to [Working with Redis](#optional-working-with-redis). You can start several services together: `docker-compose up web snekbox redis`. @@ -532,7 +513,7 @@ You can start several services together: `docker-compose up web snekbox redis`. ##### Setting Up a Development Environment The bot's code is Python code like any other. To run it locally, you will need the right version of Python with the necessary packages installed: -1. Make sure you have [Python 3.9](https://www.python.org/downloads/) installed. It helps if it is your system's default Python version. +1. Make sure you have [Python 3.10](https://www.python.org/downloads/) installed. It helps if it is your system's default Python version. 2. [Install Poetry](https://github.com/python-poetry/poetry#installation). 3. [Install the dependencies](../installing-project-dependencies). @@ -570,10 +551,7 @@ Now that you have everything setup, it is finally time to make changes to the bo #### Working with Git -If you have not yet [read the contributing guidelines](../contributing-guidelines), now is a good time. -Contributions that do not adhere to the guidelines may be rejected. - -Notably, version control of our projects is done using Git and Github. +Version control of our projects is done using Git and Github. It can be intimidating at first, so feel free to ask for any help in the server. [**Click here to see the basic Git workflow when contributing to one of our projects.**](../working-with-git/) @@ -664,4 +642,11 @@ The following is a list of all available environment variables used by the bot: | `METABASE_USERNAME` | When you wish to interact with Metabase | The username for a Metabase admin account. | `METABASE_PASSWORD` | When you wish to interact with Metabase | The password for a Metabase admin account. +--- + +# Next steps +Now that you have everything setup, it is finally time to make changes to the bot! If you have not yet read the [contributing guidelines](../contributing-guidelines.md), now is a good time. Contributions that do not adhere to the guidelines may be rejected. + +If you're not sure where to go from here, our [detailed walkthrough](../#2-set-up-the-project) is for you. + Have fun! diff --git a/pydis_site/apps/content/resources/guides/pydis-guides/contributing/commit-messages.md b/pydis_site/apps/content/resources/guides/pydis-guides/contributing/commit-messages.md new file mode 100644 index 00000000..ba476b65 --- /dev/null +++ b/pydis_site/apps/content/resources/guides/pydis-guides/contributing/commit-messages.md @@ -0,0 +1,15 @@ +--- +title: Writing Good Commit Messages +description: Information about logging in our projects. +--- + +A well-structured git log is key to a project's maintainability; it provides insight into when and *why* things were done for future maintainers of the project. + +Commits should be as narrow in scope as possible. +Commits that span hundreds of lines across multiple unrelated functions and/or files are very hard for maintainers to follow. +After about a week they'll probably be hard for you to follow, too. + +Please also avoid making minor commits for fixing typos or linting errors. +[Don’t forget to lint before you push!](https://soundcloud.com/lemonsaurusrex/lint-before-you-push) + +A more in-depth guide to writing great commit messages can be found in Chris Beam's [How to Write a Git Commit Message](https://chris.beams.io/posts/git-commit/). diff --git a/pydis_site/apps/content/resources/guides/pydis-guides/contributing/contributing-guidelines.md b/pydis_site/apps/content/resources/guides/pydis-guides/contributing/contributing-guidelines.md index de1777f2..d1e4250d 100644 --- a/pydis_site/apps/content/resources/guides/pydis-guides/contributing/contributing-guidelines.md +++ b/pydis_site/apps/content/resources/guides/pydis-guides/contributing/contributing-guidelines.md @@ -4,22 +4,15 @@ description: Guidelines to adhere to when contributing to our projects. --- Thank you for your interest in our projects! +This page contains the golden rules to follow when contributing. If you have questions about how to get started contributing, check out our [in-depth walkthrough](../../contributing/). -If you are interested in contributing, **this page contains the golden rules to follow when contributing.** -Supplemental information [can be found here](./supplemental-information/). -Do note that failing to comply with our guidelines may lead to a rejection of the contribution. - -If you are confused by any of these rules, feel free to ask us in the `#dev-contrib` channel in our [Discord server.](https://discord.gg/python) - -# The Golden Rules of Contributing - -1. **Lint before you push.** We have simple but strict style rules that are enforced through linting. -You must always lint your code before committing or pushing. -[Using tools](./supplemental-information/#linting-and-pre-commit) such as `flake8` and `pre-commit` can make this easier. -Make sure to follow our [style guide](../style-guide/) when contributing. +1. **Lint before you push.** +We have simple but strict style rules that are enforced through linting. +[Set up a pre-commit hook](../linting/) to lint your code when you commit it. +Not all of the style rules are enforced by linting, so make sure to read the [style guide](../style-guide/) as well. 2. **Make great commits.** Great commits should be atomic, with a commit message explaining what and why. -More on that can be found in [this section](./supplemental-information/#writing-good-commit-messages). +Check out [Writing Good Commit Messages](../commit-messages/) for details. 3. **Do not open a pull request if you aren't assigned to the issue.** If someone is already working on it, consider offering to collaborate with that person. 4. **Use assets licensed for public use.** @@ -28,4 +21,8 @@ Whenever the assets are images, audio or even code, they must have a license com We aim to foster a welcoming and friendly environment on our open source projects. We take violations of our Code of Conduct very seriously, and may respond with moderator action. -Welcome to our projects! +<br/> + +Failing to comply with our guidelines may lead to a rejection of the contribution. +If you have questions about any of the rules, feel free to ask us in the [`#dev-contrib`](https://discord.gg/2h3qBv8Xaa) channel in our [Discord server](https://discord.gg/python). +{: .notification .is-warning } diff --git a/pydis_site/apps/content/resources/guides/pydis-guides/contributing/contributing-guidelines/_info.yml b/pydis_site/apps/content/resources/guides/pydis-guides/contributing/contributing-guidelines/_info.yml deleted file mode 100644 index 80c8e772..00000000 --- a/pydis_site/apps/content/resources/guides/pydis-guides/contributing/contributing-guidelines/_info.yml +++ /dev/null @@ -1,2 +0,0 @@ -title: Contributing Guidelines -description: Guidelines to adhere to when contributing to our projects. diff --git a/pydis_site/apps/content/resources/guides/pydis-guides/contributing/contributing-guidelines/supplemental-information.md b/pydis_site/apps/content/resources/guides/pydis-guides/contributing/contributing-guidelines/supplemental-information.md deleted file mode 100644 index e64e4fc6..00000000 --- a/pydis_site/apps/content/resources/guides/pydis-guides/contributing/contributing-guidelines/supplemental-information.md +++ /dev/null @@ -1,99 +0,0 @@ ---- -title: Supplemental Information -description: Additional information related to our contributing guidelines. ---- - -This page contains additional information concerning a specific part of our development pipeline. - -## Writing Good Commit Messages - -A well-structured git log is key to a project's maintainability; it provides insight into when and *why* things were done for future maintainers of the project. - -Commits should be as narrow in scope as possible. -Commits that span hundreds of lines across multiple unrelated functions and/or files are very hard for maintainers to follow. -After about a week they'll probably be hard for you to follow, too. - -Please also avoid making minor commits for fixing typos or linting errors. -*[Don’t forget to lint before you push!](https://soundcloud.com/lemonsaurusrex/lint-before-you-push)* - -A more in-depth guide to writing great commit messages can be found in Chris Beam's *[How to Write a Git Commit Message](https://chris.beams.io/posts/git-commit/).* - -## Code Style - -All of our projects have a certain project-wide style that contributions should attempt to maintain consistency with. -During PR review, it's not unusual for style adjustments to be requested. - -[This page](../../style-guide/) will reference the differences between our projects and what is recommended by [PEP 8.](https://www.python.org/dev/peps/pep-0008/) - -## Linting and Pre-commit - -On most of our projects, we use `flake8` and `pre-commit` to ensure that the code style is consistent across the code base. - -Running `flake8` will warn you about any potential style errors in your contribution. -You must always check it **before pushing**. -Your commit will be rejected by the build server if it fails to lint. - -**Some style rules are not enforced by flake8. Make sure to read the [style guide](../../style-guide/).** - -`pre-commit` is a powerful tool that helps you automatically lint before you commit. -If the linter complains, the commit is aborted so that you can fix the linting errors before committing again. -That way, you never commit the problematic code in the first place! - -Please refer to the project-specific documentation to see how to setup and run those tools. -In most cases, you can install pre-commit using `poetry run task precommit`, and lint using `poetry run task lint`. - -## Type Hinting - -[PEP 484](https://www.python.org/dev/peps/pep-0484/) formally specifies type hints for Python functions, added to the Python Standard Library in version 3.5. -Type hints are recognized by most modern code editing tools and provide useful insight into both the input and output types of a function, preventing the user from having to go through the codebase to determine these types. - -For example: - -```python -import typing - -def foo(input_1: int, input_2: typing.Dict[str, str]) -> bool: - ... -``` - -This tells us that `foo` accepts an `int` and a `dict`, with `str` keys and values, and returns a `bool`. - -If the project is running Python 3.9 or above, you can use `dict` instead of `typing.Dict`. -See [PEP 585](https://www.python.org/dev/peps/pep-0585/) for more information. - -All function declarations should be type hinted in code contributed to the PyDis organization. - -## Logging - -Instead of using `print` statements for logging, we use the built-in [`logging`](https://docs.python.org/3/library/logging.html) module. -Here is an example usage: - -```python -import logging - -log = logging.getLogger(__name__) # Get a logger bound to the module name. -# This line is usually placed under the import statements at the top of the file. - -log.trace("This is a trace log.") -log.warning("BEEP! This is a warning.") -log.critical("It is about to go down!") -``` - -Print statements should be avoided when possible. -Our projects currently defines logging levels as follows, from lowest to highest severity: - -- **TRACE:** These events should be used to provide a *verbose* trace of every step of a complex process. This is essentially the `logging` equivalent of sprinkling `print` statements throughout the code. -- **Note:** This is a PyDis-implemented logging level. It may not be available on every project. -- **DEBUG:** These events should add context to what's happening in a development setup to make it easier to follow what's going while workig on a project. This is in the same vein as **TRACE** logging but at a much lower level of verbosity. -- **INFO:** These events are normal and don't need direct attention but are worth keeping track of in production, like checking which cogs were loaded during a start-up. -- **WARNING:** These events are out of the ordinary and should be fixed, but can cause a failure. -- **ERROR:** These events can cause a failure in a specific part of the application and require urgent attention. -- **CRITICAL:** These events can cause the whole application to fail and require immediate intervention. - -Any logging above the **INFO** level will trigger a [Sentry](https://sentry.io) issue and alert the Core Developer team. - -## Draft Pull Requests - -Github [provides a PR feature](https://github.blog/2019-02-14-introducing-draft-pull-requests/) that allows the PR author to mark it as a Draft when opening it. This provides both a visual and functional indicator that the contents of the PR are in a draft state and not yet ready for formal review. - -This feature should be utilized in place of the traditional method of prepending `[WIP]` to the PR title. diff --git a/pydis_site/apps/content/resources/guides/pydis-guides/contributing/linting.md b/pydis_site/apps/content/resources/guides/pydis-guides/contributing/linting.md new file mode 100644 index 00000000..f6f8a5f2 --- /dev/null +++ b/pydis_site/apps/content/resources/guides/pydis-guides/contributing/linting.md @@ -0,0 +1,14 @@ +--- +title: Linting +description: A guide for linting and setting up pre-commit. +--- + +Your commit will be rejected by the build server if it fails to lint. +On most of our projects, we use `flake8` and `pre-commit` to ensure that the code style is consistent across the code base. + +`pre-commit` is a powerful tool that helps you automatically lint before you commit. +If the linter complains, the commit is aborted so that you can fix the linting errors before committing again. +That way, you never commit the problematic code in the first place! + +Please refer to the project-specific documentation to see how to setup and run those tools. +In most cases, you can install pre-commit using `poetry run task precommit`, and lint using `poetry run task lint` in the console. diff --git a/pydis_site/apps/content/resources/guides/pydis-guides/contributing/logging.md b/pydis_site/apps/content/resources/guides/pydis-guides/contributing/logging.md new file mode 100644 index 00000000..1291a7a4 --- /dev/null +++ b/pydis_site/apps/content/resources/guides/pydis-guides/contributing/logging.md @@ -0,0 +1,31 @@ +--- +title: Logging +description: Information about logging in our projects. +--- + +Instead of using `print` statements for logging, we use the built-in [`logging`](https://docs.python.org/3/library/logging.html) module. +Here is an example usage: + +```python +import logging + +log = logging.getLogger(__name__) # Get a logger bound to the module name. +# This line is usually placed under the import statements at the top of the file. + +log.trace("This is a trace log.") +log.warning("BEEP! This is a warning.") +log.critical("It is about to go down!") +``` + +Print statements should be avoided when possible. +Our projects currently defines logging levels as follows, from lowest to highest severity: + +- **TRACE:** These events should be used to provide a *verbose* trace of every step of a complex process. This is essentially the `logging` equivalent of sprinkling `print` statements throughout the code. +- **Note:** This is a PyDis-implemented logging level. It may not be available on every project. +- **DEBUG:** These events should add context to what's happening in a development setup to make it easier to follow what's going while workig on a project. This is in the same vein as **TRACE** logging but at a much lower level of verbosity. +- **INFO:** These events are normal and don't need direct attention but are worth keeping track of in production, like checking which cogs were loaded during a start-up. +- **WARNING:** These events are out of the ordinary and should be fixed, but can cause a failure. +- **ERROR:** These events can cause a failure in a specific part of the application and require urgent attention. +- **CRITICAL:** These events can cause the whole application to fail and require immediate intervention. + +Any logging above the **INFO** level will trigger a [Sentry](https://sentry.io) issue and alert the Core Developer team. diff --git a/pydis_site/apps/content/resources/guides/pydis-guides/contributing/pull-requests.md b/pydis_site/apps/content/resources/guides/pydis-guides/contributing/pull-requests.md new file mode 100644 index 00000000..d193a455 --- /dev/null +++ b/pydis_site/apps/content/resources/guides/pydis-guides/contributing/pull-requests.md @@ -0,0 +1,40 @@ +--- +title: Pull Requests +description: A guide for opening pull requests. +--- + +As stated in our [Contributing Guidelines](../contributing-guidelines/), do not open a pull request if you aren't assigned to an approved issue. You can check out our [Issues Guide](../issues/) for help with opening an issue or getting assigned to an existing one. +{: .notification .is-warning } + +Before opening a pull request you should have: + +1. Committed your changes to your local repository +2. [Linted](../linting/) your code +3. Tested your changes +4. Pushed the branch to your fork of the project on GitHub + +## Opening a Pull Request + +Navigate to your fork on GitHub and make sure you're on the branch with your changes. Click on `Contribute` and then `Open pull request`: + + + +In the page that it opened, write an overview of the changes you made and why. This should explain how you resolved the issue that spawned this PR and highlight any differences from the proposed implementation. You should also [link the issue](https://docs.github.com/en/issues/tracking-your-work-with-issues/linking-a-pull-request-to-an-issue). + +At this stage you can also request reviews from individual contributors. If someone showed interest in the issue or has specific knowledge about it, they may be a good reviewer. It isn't necessary to request your reviewers; someone will review your PR either way. + +## The Review Process + +Before your changes are merged, your PR needs to be reviewed by other contributors. They will read the issue and your description of your PR, look at your code, test it, and then leave comments on the PR if they find any problems, possibly with suggested changes. Sometimes this can feel intrusive or insulting, but remember that the reviewers are there to help you make your code better. + +#### If the PR is already open, how do I make changes to it? + +A pull request is between a source branch and a target branch. Updating the source branch with new commits will automatically update the PR to include those commits; they'll even show up in the comment thread of the PR. Sometimes for small changes the reviewer will even write the suggested code themself, in which case you can simply accept them with the click of a button. + +If you truly disagree with a reviewer's suggestion, leave a reply in the thread explaining why or proposing an alternative change. Also feel free to ask questions if you want clarification about suggested changes or just want to discuss them further. + +## Draft Pull Requests + +GitHub [provides a PR feature](https://github.blog/2019-02-14-introducing-draft-pull-requests/) that allows the PR author to mark it as a draft when opening it. This provides both a visual and functional indicator that the contents of the PR are in a draft state and not yet ready for formal review. This is helpful when you want people to see the changes you're making before you're ready for the final pull request. + +This feature should be utilized in place of the traditional method of prepending `[WIP]` to the PR title. diff --git a/pydis_site/apps/content/resources/guides/pydis-guides/contributing/sir-lancebot.md b/pydis_site/apps/content/resources/guides/pydis-guides/contributing/sir-lancebot.md index e3cd8f0c..c9566d23 100644 --- a/pydis_site/apps/content/resources/guides/pydis-guides/contributing/sir-lancebot.md +++ b/pydis_site/apps/content/resources/guides/pydis-guides/contributing/sir-lancebot.md @@ -5,10 +5,11 @@ icon: fab fa-github toc: 1 --- -> Before contributing, please ensure you read the [contributing guidelines](../contributing-guidelines) in full. +You should have already forked the [`sir-lancebot`](https://github.com/python-discord/sir-lancebot) repository and cloned it to your local machine. If not, check out our [detailed walkthrough](../#1-fork-and-clone-the-repo). ---- -# Requirements +Remember to ensure that you have read the [contributing guidelines](../contributing-guidelines) in full before you start contributing. + +### Requirements - [Python 3.9](https://www.python.org/downloads/) - [Poetry](https://github.com/python-poetry/poetry#installation) - [Git](https://git-scm.com/downloads) @@ -16,10 +17,12 @@ toc: 1 - [MacOS Installer](https://git-scm.com/download/mac) or `brew install git` - [Linux](https://git-scm.com/download/linux) +--- + ## Using Gitpod Sir Lancebot can be edited and tested on Gitpod. Gitpod will automatically install the correct dependencies and Python version, so you can get straight to coding. -To do this, you will need a Gitpod account, which you can get [here](https://www.gitpod.io/#get-started), and a fork of Sir Lancebot. This guide covers forking the repository [here](#fork-the-project). +To do this, you will need a Gitpod account, which you can get [here](https://www.gitpod.io/#get-started), and a fork of Sir Lancebot. This guide covers forking the repository [here](../forking-repository). Afterwards, click on [this link](https://gitpod.io/#/github.com/python-discord/sir-lancebot) to spin up a new workspace for Sir Lancebot. Then run the following commands in the terminal after the existing tasks have finished running: ```sh @@ -41,19 +44,8 @@ The requirements for Docker are: * This is only a required step for linux. Docker comes bundled with docker-compose on Mac OS and Windows. --- - -# Fork the Project -You will need your own remote (online) copy of the project repository, known as a *fork*. - -- [**Learn how to create a fork of the repository here.**](../forking-repository) - -You will do all your work in the fork rather than directly in the main repository. - ---- - # Development Environment -1. Once you have your fork, you will need to [**clone the repository to your computer**](../cloning-repository). -2. After cloning, proceed to [**install the project's dependencies**](../installing-project-dependencies). (This is not required if using Docker) +If you aren't using Docker, you will need to [install the project's dependencies](../installing-project-dependencies) yourself. --- # Test Server and Bot Account @@ -120,14 +112,11 @@ After installing project dependencies use the poetry command `poetry run task st ```shell $ poetry run task start ``` - --- -# Working with Git -Now that you have everything setup, it is finally time to make changes to the bot! If you have not yet [read the contributing guidelines](https://github.com/python-discord/sir-lancebot/blob/main/CONTRIBUTING.md), now is a good time. Contributions that do not adhere to the guidelines may be rejected. - -Notably, version control of our projects is done using Git and Github. It can be intimidating at first, so feel free to ask for any help in the server. +# Next steps +Now that you have everything setup, it is finally time to make changes to the bot! If you have not yet read the [contributing guidelines](../contributing-guidelines.md), now is a good time. Contributions that do not adhere to the guidelines may be rejected. -[**Click here to see the basic Git workflow when contributing to one of our projects.**](../working-with-git/) +If you're not sure where to go from here, our [detailed walkthrough](../#2-set-up-the-project) is for you. Have fun! diff --git a/pydis_site/apps/content/resources/guides/pydis-guides/contributing/site.md b/pydis_site/apps/content/resources/guides/pydis-guides/contributing/site.md index f2c3bd95..9786698b 100644 --- a/pydis_site/apps/content/resources/guides/pydis-guides/contributing/site.md +++ b/pydis_site/apps/content/resources/guides/pydis-guides/contributing/site.md @@ -5,9 +5,11 @@ icon: fab fa-github toc: 1 --- -# Requirements +You should have already forked the [`site`](https://github.com/python-discord/site) repository and cloned it to your local machine. If not, check out our [detailed walkthrough](../#1-fork-and-clone-the-repo). -- [Python 3.9](https://www.python.org/downloads/) +### Requirements + +- [Python 3.10](https://www.python.org/downloads/) - [Poetry](https://python-poetry.org/docs/#installation) - `pip install poetry` - [Git](https://git-scm.com/downloads) @@ -27,22 +29,9 @@ Without Docker: - Note that if you wish, the webserver can run on the host and still use Docker for PostgreSQL. --- -# Fork the project - -You will need access to a copy of the git repository of your own that will allow you to edit the code and push your commits to. -Creating a copy of a repository under your own account is called a _fork_. - -- [Learn how to create a fork of the repository here.](../forking-repository/) - -This is where all your changes and commits will be pushed to, and from where your PRs will originate from. - -For any Core Developers, since you have write permissions already to the original repository, you can just create a feature branch to push your commits to instead. - ---- # Development environment -1. [Clone your fork to a local project directory](../cloning-repository/) -2. [Install the project's dependencies](../installing-project-dependencies/) +[Install the project's dependencies](../installing-project-dependencies/) ## Without Docker @@ -178,3 +167,12 @@ The website is configured through the following environment variables: - **`STATIC_ROOT`**: The root in which `python manage.py collectstatic` collects static files. Optional, defaults to `/app/staticfiles` for the standard Docker deployment. + +--- + +# Next steps +Now that you have everything setup, it is finally time to make changes to the site! If you have not yet read the [contributing guidelines](../contributing-guidelines.md), now is a good time. Contributions that do not adhere to the guidelines may be rejected. + +If you're not sure where to go from here, our [detailed walkthrough](../#2-set-up-the-project) is for you. + +Have fun! diff --git a/pydis_site/apps/content/resources/guides/pydis-guides/contributing/style-guide.md b/pydis_site/apps/content/resources/guides/pydis-guides/contributing/style-guide.md index f9962990..b26c467c 100644 --- a/pydis_site/apps/content/resources/guides/pydis-guides/contributing/style-guide.md +++ b/pydis_site/apps/content/resources/guides/pydis-guides/contributing/style-guide.md @@ -191,21 +191,14 @@ Present tense defines that the work being done is now, in the present, rather th **Use:** "Build an information embed."<br> **Don't use:** "Built an information embed." or "Will build an information embed." -# Type Annotations -Functions are required to have type annotations as per the style defined in [PEP 484](https://www.python.org/dev/peps/pep-0484/). +# Type Hinting +Functions are required to have type annotations as per the style defined in [PEP 484](https://www.python.org/dev/peps/pep-0484/). Type hints are recognized by most modern code editing tools and provide useful insight into both the input and output types of a function, preventing the user from having to go through the codebase to determine these types. -A function without annotations might look like: -```py -def divide(a, b): - """Divide the two given arguments.""" - return a / b -``` - -With annotations, the arguments and the function are annotated with their respective types: -```py -def divide(a: int, b: int) -> float: - """Divide the two given arguments.""" - return a / b +A function with type hints looks like: +```python +def foo(input_1: int, input_2: dict[str, int]) -> bool: + ... ``` +This tells us that `foo` accepts an `int` and a `dict`, with `str` keys and `int` values, and returns a `bool`. In previous examples, we have purposely omitted annotations to keep focus on the specific points they represent. diff --git a/pydis_site/apps/content/resources/guides/pydis-guides/contributing/working-with-git.md b/pydis_site/apps/content/resources/guides/pydis-guides/contributing/working-with-git.md index 26c89b56..59c57859 100644 --- a/pydis_site/apps/content/resources/guides/pydis-guides/contributing/working-with-git.md +++ b/pydis_site/apps/content/resources/guides/pydis-guides/contributing/working-with-git.md @@ -19,5 +19,7 @@ Below are links to regular workflows for working with Git using PyCharm or the C **Resources to learn Git** * [The Git Book](https://git-scm.com/book) -* [Corey Schafer's Youtube Tutorials](https://www.youtube.com/watch?v=HVsySz-h9r4&list=PL-osiE80TeTuRUfjRe54Eea17-YfnOOAx) -* [GitHub Git Resources Portal](https://try.github.io/) +* [Corey Schafer's YouTube tutorials](https://www.youtube.com/watch?v=HVsySz-h9r4&list=PL-osiE80TeTuRUfjRe54Eea17-YfnOOAx) +* [GitHub Git resources portal](https://try.github.io/) +* [Git cheatsheet](https://education.github.com/git-cheat-sheet-education.pdf) +* [Learn Git branching](https://learngitbranching.js.org) diff --git a/pydis_site/apps/content/resources/guides/pydis-guides/off-topic-etiquette.md b/pydis_site/apps/content/resources/guides/pydis-guides/off-topic-etiquette.md index f8031834..5e785cd9 100644 --- a/pydis_site/apps/content/resources/guides/pydis-guides/off-topic-etiquette.md +++ b/pydis_site/apps/content/resources/guides/pydis-guides/off-topic-etiquette.md @@ -5,7 +5,7 @@ icon: fab fa-discord --- ## Why do we need off-topic etiquette? -Everyone wants to have good conversations in our off-topic channels, but with tens of thousands of members, this might mean different things to different people. +Everyone wants to have good conversations in our off-topic channels, but with hundreds of thousands of members, this might mean different things to different people. To facilitate the best experience for everyone, here are some guidelines on conversation etiquette. ## Three things you shouldn't do diff --git a/pydis_site/apps/content/resources/guides/python-guides/discord-messages-with-colors.md b/pydis_site/apps/content/resources/guides/python-guides/discord-messages-with-colors.md new file mode 100644 index 00000000..62ff61f9 --- /dev/null +++ b/pydis_site/apps/content/resources/guides/python-guides/discord-messages-with-colors.md @@ -0,0 +1,68 @@ +--- +title: Discord Messages with Colors +description: A guide on how to add colors to your codeblocks on Discord +--- + +Discord is now slowly rolling out the ability to send colored text within code blocks. This is done using ANSI color codes which is also how you print colored text in your terminal. + +To send colored text in a code block you need to first specify the `ansi` language and use the prefixes similar to the one below: +```ansi +\u001b[{format};{color}m +``` +*`\u001b` is the unicode escape for ESCAPE/ESC, meant to be used in the source of your bot (see <http://www.unicode-symbol.com/u/001B.html>).* ***If you wish to send colored text without using your bot you need to copy the character from the website.*** + +After you've written this, you can now type the text you wish to color. If you want to reset the color back to normal, then you need to use the `\u001b[0m` prefix again. + +Here is the list of values you can use to replace `{format}`: + +* 0: Normal +* 1: **Bold** +* 4: <ins>Underline</ins> + +Here is the list of values you can use to replace `{color}`: + +*The following values will change the **text** color.* + +* 30: Gray +* 31: Red +* 32: Green +* 33: Yellow +* 34: Blue +* 35: Pink +* 36: Cyan +* 37: White + +*The following values will change the **text background** color.* + +* 40: Firefly dark blue +* 41: Orange +* 42: Marble blue +* 43: Greyish turquoise +* 44: Gray +* 45: Indigo +* 46: Light gray +* 47: White + +Let's take an example, I want a bold green colored text with the very dark blue background. +I simply use `\u001b[0;40m` (background color) and `\u001b[1;32m` (text color) as prefix. Note that the order is **important**, first you give the background color and then the text color. + +Alternatively you can also directly combine them into a single prefix like the following: `\u001b[1;40;32m` and you can also use multiple values. Something like `\u001b[1;40;4;32m` would underline the text, make it bold, make it green and have a dark blue background. + +Raw message: +````nohighlight +```ansi +\u001b[0;40m\u001b[1;32mThat's some cool formatted text right? +or +\u001b[1;40;32mThat's some cool formatted text right? +``` +```` + +Result: + + + +The way the colors look like on Discord is shown in the image below: + + + +Note: If the change as not been brought to you yet, or other users, then you can use other code blocks in the meantime to get colored text. See **[this gist](https://gist.github.com/matthewzring/9f7bbfd102003963f9be7dbcf7d40e51)**. diff --git a/pydis_site/apps/content/resources/guides/python-guides/discordpy_help_command.md b/pydis_site/apps/content/resources/guides/python-guides/discordpy_help_command.md new file mode 100644 index 00000000..4b475146 --- /dev/null +++ b/pydis_site/apps/content/resources/guides/python-guides/discordpy_help_command.md @@ -0,0 +1,66 @@ +--- +title: Custom Help Command +description: "Overwrite discord.py's help command to implement custom functionality" +--- + +First, a basic walkthrough can be found [here](https://gist.github.com/InterStella0/b78488fb28cadf279dfd3164b9f0cf96) by Stella#2000 on subclassing the HelpCommand. It will provide some foundational knowledge that is required before attempting a more customizable help command. + +## Custom Subclass of Help Command +If the types of classes of the HelpCommand do not fit your needs, you can subclass HelpCommand and use the class mehods to customize the output. Below is a simple demonstration using the following methods that can also be found on the documenation: + +- [filter_commands](https://discordpy.readthedocs.io/en/stable/ext/commands/api.html#discord.ext.commands.HelpCommand.filter_commands) + +- [send_group_help](https://discordpy.readthedocs.io/en/stable/ext/commands/api.html#discord.ext.commands.HelpCommand.send_bot_help) + +- [send_command_help](https://discordpy.readthedocs.io/en/stable/ext/commands/api.html#discord.ext.commands.HelpCommand.send_command_help) + +- [send_group_help](https://discordpy.readthedocs.io/en/stable/ext/commands/api.html#discord.ext.commands.HelpCommand.send_group_help) + +- [send_error_message](https://discordpy.readthedocs.io/en/stable/ext/commands/api.html#discord.ext.commands.HelpCommand.send_error_message) + +```python +class MyHelp(commands.HelpCommand): + + async def send_bot_help(self, mapping): + """ + This is triggered when !help is invoked. + + This example demonstrates how to list the commands that the member invoking the help command can run. + """ + filtered = await self.filter_commands(self.context.bot.commands, sort=True) # returns a list of command objects + names = [command.name for command in filtered] # iterating through the commands objects getting names + available_commands = "\n".join(names) # joining the list of names by a new line + embed = disnake.Embed(description=available_commands) + await self.context.send(embed=embed) + + async def send_command_help(self, command): + """This is triggered when !help <command> is invoked.""" + await self.context.send("This is the help page for a command") + + async def send_group_help(self, group): + """This is triggered when !help <group> is invoked.""" + await self.context.send("This is the help page for a group command") + + async def send_cog_help(self, cog): + """This is triggered when !help <cog> is invoked.""" + await self.context.send("This is the help page for a cog") + + async def send_error_message(self, error): + """If there is an error, send a embed containing the error.""" + channel = self.get_destination() # this defaults to the command context channel + await channel.send(error) + +bot.help_command = MyHelp() +``` + +You can handle when a user does not pass a command name when invoking the help command and make a fancy and customized embed; here a page that describes the bot and shows a list of commands is generally used. However if a command is passed in, you can display detailed information of the command. Below are references from the documentation below that can be utilised: + +- [Get the command object](https://discordpy.readthedocs.io/en/latest/ext/commands/api.html#discord.ext.commands.Bot.get_command) + +- [Get the command name](https://discordpy.readthedocs.io/en/latest/ext/commands/api.html#discord.ext.commands.Command.name) + +- [Get the command aliases](https://discordpy.readthedocs.io/en/latest/ext/commands/api.html#discord.ext.commands.Command.aliases) + +- [Get the command brief](https://discordpy.readthedocs.io/en/latest/ext/commands/api.html#discord.ext.commands.Command.brief) + +- [Get the command usage](https://discordpy.readthedocs.io/en/latest/ext/commands/api.html#discord.ext.commands.Command.usage) diff --git a/pydis_site/apps/content/resources/guides/python-guides/docker-hosting-guide.md b/pydis_site/apps/content/resources/guides/python-guides/docker-hosting-guide.md new file mode 100644 index 00000000..57d86e99 --- /dev/null +++ b/pydis_site/apps/content/resources/guides/python-guides/docker-hosting-guide.md @@ -0,0 +1,323 @@ +--- +title: How to host a bot with Docker and GitHub Actions on Ubuntu VPS +description: This guide shows how to host a bot with Docker and GitHub Actions on Ubuntu VPS +--- + +## Contents + +1. [You will learn](#you-will-learn) +2. [Introduction](#introduction) +3. [Installing Docker](#installing-docker) +4. [Creating Dockerfile](#creating-dockerfile) +5. [Building Image and Running Container](#building-image-and-running-container) +6. [Creating Volumes](#creating-volumes) +7. [Using GitHub Actions for full automation](#using-github-actions-for-full-automation) + +## You will learn how to + +- write Dockerfile +- build Docker image and run the container +- use Docker Compose +- make docker keep the files throughout the container's runs +- parse environment variables into container +- use GitHub Actions for automation +- set up self-hosted runner +- use runner secrets + +## Introduction + +Let's say you have got a nice discord bot written in python and you have a VPS to host it on. Now the only question is +how to run it 24/7. You might have been suggested to use *screen multiplexer*, but it has some disadvantages: + +1. Every time you update the bot you have to SSH to your server, attach to screen, shutdown the bot, run `git pull` and + run the bot again. You might have good extensions management that allows you to update the bot without restarting it, + but there are some other cons as well +2. If you update some dependencies, you have to update them manually +3. The bot doesn't run in an isolated environment, which is not good for security. + +But there's a nice and easy solution to these problems - **Docker**! Docker is a containerization utility that automates +some stuff like dependencies update and running the application in the background. So let's get started. + +## Installing Docker + +The best way to install Docker is to use +the [convenience script](https://docs.docker.com/engine/install/ubuntu/#install-using-the-convenience-script) provided +by Docker developers themselves. You just need 2 lines: + +```shell +$ curl -fsSL https://get.docker.com -o get-docker.sh +$ sudo sh get-docker.sh +``` + +## Creating Dockerfile + +To tell Docker what it has to do to run the application, we need to create a file named `Dockerfile` in our project's +root. + +1. First we need to specify the *base image*, which is the OS that the docker container will be running. Doing that will + make Docker install some apps we need to run our bot, for + example the Python interpreter + +```dockerfile +FROM python:3.10-bullseye +``` + +2. Next, we need to copy our Python project's external dependencies to some directory *inside the container*. Let's call + it `/app` + +```dockerfile +COPY requirements.txt /app/ +``` + +3. Now we need to set the directory as working and install the requirements + +```dockerfile +WORKDIR /app +RUN pip install -r requirements.txt +``` + +4. The only thing that is left to do is to copy the rest of project's files and run the main executable + +```dockerfile +COPY . . +CMD ["python3", "main.py"] +``` + +The final version of Dockerfile looks like this: + +```dockerfile +FROM python:3.10-bullseye +COPY requirements.txt /app/ +WORKDIR /app +RUN pip install -r requirements.txt +COPY . . +CMD ["python3", "main.py"] +``` + +## Building Image and Running Container + +Now update the project on your VPS, so we can run the bot with Docker. + +1. Build the image (dot at the end is very important) + +```shell +$ docker build -t mybot . +``` + +- the `-t` flag specifies a **tag** that will be assigned to the image. With it, we can easily run the image that the + tag was assigned to. +- the dot at the end is basically the path to search for Dockerfile. The dot means current directory (`./`) + +2. Run the container + +```shell +$ docker run -d --name mybot mybot:latest +``` + +- `-d` flag tells Docker to run the container in detached mode, meaning it will run the container in the background of + your terminal and not give us + any output from it. If we don't + provide it, the `run` will be giving us the output until the application exits. Discord bots aren't supposed to exit + after certain time, so we do need this flag +- `--name` assigns a name to the container. By default, container is identified by id that is not human-readable. To + conveniently refer to container when needed, + we can assign it a name +- `mybot:latest` means "latest version of `mybot` image" + +3. Read bot logs (keep in mind that this utility only allows to read STDERR) + +```shell +$ docker logs -f mybot +``` + +- `-f` flag tells the docker to keep reading logs as they appear in container and is called "follow mode". To exit + press `CTRL + C`. + +If everything went successfully, your bot will go online and will keep running! + +## Using Docker Compose + +Just 2 commands to run a container is cool, but we can shorten it down to just 1 simple command. For that, create +a `docker-compose.yml` file in project's root and fill it with the following contents: + +```yml +version: "3.8" +services: + main: + build: . + container_name: mybot +``` + +- `version` tells Docker what version of Compose to use. You may check all the + versions [here](https://docs.docker.com/compose/compose-file/compose-versioning/) +- `services` contains services to build and run. Read more about + services [here](https://docs.docker.com/compose/compose-file/#services-top-level-element) +- `main` is a service. We can call it whatever we would like to, not necessarily `main` +- `build: .` is a path to search for Dockerfile, just like `docker build` command's dot +- `container_name: mybot` is a container name to use for a bot, just like `docker run --name mybot` + +Update the project on VPS, remove the previous container with `docker rm -f mybot` and run this command + +```shell +docker compose up -d --build +``` + +Now the docker will automatically build the image for you and run the container. + +### Why docker-compose + +The main purpose of Compose is to run several services at once. Mostly we +don't need this in discord bots, however. +For us, it has the following benefits: + +- we can build and run the container with just one command +- if we need to parse some environment variables or volumes (more about them further in tutorial) our run command would + look like this + +```shell +$ docker run -d --name mybot -e TOKEN=... -e WEATHER_API_APIKEY=... -e SOME_USEFUL_ENVIRONMENT_VARIABLE=... --net=host -v /home/exenifix/bot-data/data:/app/data -v /home/exenifix/bot-data/images:/app/data/images +``` + +This is pretty long and unreadable. Compose allows us to transfer those flags into single config file and still +use just one short command to run the container. + +## Creating Volumes + +The files creating during container run are destroyed after its recreation. To prevent some files from getting +destroyed, we need to use *volumes* that basically save the files from directory inside of container somewhere on drive. + +1. Create a new directory somewhere and copy path to it + +```shell +$ mkdir mybot-data +$ echo $(pwd)/mybot-data +``` + +My path is `/home/exenifix/mybot-data`, yours is most likely **different**! + +2. In your project, store the files that need to be persistent in a separate directory (eg. `data`) +3. Add `volumes` to `docker-compose.yaml` so it looks like this: + +```yml +version: "3.8" +services: + main: + build: . + container_name: mybot + volumes: + - /home/exenifix/mybot-data:/app/data +``` + +The path before the colon `:` is the directory *on server's drive, outside of container*, and the second path is the +directory *inside of container*. +All the files saved in container in that directory will be saved on drive's directory as well and Docker will be +accessing them *from drive*. + +## Using GitHub Actions for full automation + +Now it's time to fully automate the process and make Docker update the bot automatically on every commit or release. For +that, we will use a **GitHub Actions workflow**, which basically runs some commands when we need to. You may read more +about them [here](https://docs.github.com/en/actions/using-workflows). + +### Create repository secret + +We will not have the ability to use `.env` files with the workflow, so it's better to store the environment variables +as **actions secrets**. Let's add your discord bot's token as a secret + +1. Head to your repository page -> Settings -> Secrets -> Actions +2. Press `New repository secret` +3. Give it a name like `TOKEN` and paste the token. + Now we will be able to access its value in workflow like `${{ secrets.TOKEN }}`. However, we also need to parse the + variable into container now. Edit `docker-compose` so it looks like this: + +```yml +version: "3.8" +services: + main: + build: . + container_name: mybot + volumes: + - /home/exenifix/mybot-data:/app/data + environment: + - TOKEN +``` + +### Setup self-hosted runner + +To run the workflow on our VPS, we will need to register it as *self-hosted runner*. + +1. Head to Settings -> Actions -> Runners +2. Press `New self-hosted runner` +3. Select runner image and architecture +4. Follow the instructions but don't run the runner +5. Instead, create a service + +```shell +$ sudo ./svc.sh install +$ sudo ./svc.sh start +``` + +Now we have registered our VPS as a self-hosted runner and we can run the workflow on it now. + +### Write a workflow + +Create a new file `.github/workflows/runner.yml` and paste the following content into it. Please pay attention to +the `branches` instruction. +The GitHub's standard main branch name is `main`, however it may be named `master` or something else if you edited its +name. Make sure to put +the correct branch name, otherwise it won't work. More about GitHub workflows +syntax [here](https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions) + +```yml +name: Docker Runner + +on: + push: + branches: [ main ] + +jobs: + run: + runs-on: self-hosted + environment: production + + steps: + - uses: actions/checkout@v3 + + - name: Run Container + run: docker compose up -d --build + env: + TOKEN: ${{ secrets.TOKEN }} + + - name: Cleanup Unused Images + run: docker image prune -f +``` + +Run `docker rm -f mybot` (it only needs to be done once) and push to GitHub. Now if you open `Actions` tab on your +repository, you should see a workflow running your bot. Congratulations! + +### Displaying logs in actions terminal + +There's a nice utility for reading docker container's logs and stopping upon meeting a certain phrase and it might be +useful for you as well. + +1. Install the utility on your VPS with + +```shell +$ pip install exendlr +``` + +2. Add a step to your workflow that would show the logs until it meets `"ready"` phrase. I recommend putting it before + the cleanup. + +```yml +- name: Display Logs + run: python3 -m exendlr mybot "ready" +``` + +Now you should see the logs of your bot until the stop phrase is met. + +**WARNING** +> The utility only reads from STDERR and redirects to STDERR, if you are using STDOUT for logs, it will not work and +> will be waiting for stop phrase forever. The utility automatically exits if bot's container is stopped (e.g. error +> occurred during starting) or if a log line contains a stop phrase. Make sure that your bot 100% displays a stop phrase +> when it's ready otherwise your workflow will get stuck. diff --git a/pydis_site/apps/content/resources/guides/python-guides/fix-ssl-certificate.md b/pydis_site/apps/content/resources/guides/python-guides/fix-ssl-certificate.md new file mode 100644 index 00000000..096e3a90 --- /dev/null +++ b/pydis_site/apps/content/resources/guides/python-guides/fix-ssl-certificate.md @@ -0,0 +1,23 @@ +--- +title: Fixing an SSL Certificate Verification Error +description: A guide on fixing verification of an SSL certificate. +--- + +We're fixing the error Python specifies as [ssl.SSLCertVerificationError](https://docs.python.org/3/library/ssl.html#ssl.SSLCertVerificationError). + +# How to fix SSL Certificate issue on Windows + +Firstly, try updating your OS, wouldn't hurt to try. + +Now, if you're still having an issue, you would need to download the certificate for the SSL. + +The SSL Certificate, Sectigo (cert vendor) provides a download link of an [SSL certificate](https://crt.sh/?id=2835394). You should find it in the bottom left corner, shown below: + +A picture where to find the certificate in the website is: + + +You have to setup the certificate yourself. To do that you can just click on it, or if that doesn't work, refer to [this link](https://portal.threatpulse.com/docs/sol/Solutions/ManagePolicy/SSL/ssl_chrome_cert_ta.htm) + +# How to fix SSL Certificate issue on Mac + +Navigate to your `Applications/Python 3.x/` folder and double-click the `Install Certificates.command` to fix this. diff --git a/pydis_site/apps/content/resources/guides/python-guides/keeping-tokens-safe.md b/pydis_site/apps/content/resources/guides/python-guides/keeping-tokens-safe.md new file mode 100644 index 00000000..9d523b4b --- /dev/null +++ b/pydis_site/apps/content/resources/guides/python-guides/keeping-tokens-safe.md @@ -0,0 +1,29 @@ +--- +title: Keeping Discord Bot Tokens Safe +description: How to keep your bot tokens safe and safety measures you can take. +--- +It's **very** important to keep a bot token safe, +primarily because anyone who has the bot token can do whatever they want with the bot -- +such as destroying servers your bot has been added to and getting your bot banned from the API. + +# How to Avoid Leaking your Token +To help prevent leaking your token, +you should ensure that you don't upload it to an open source program/website, +such as replit and github, as they show your code publicly. +The best practice for storing tokens is generally utilising .env files +([click here](https://vcokltfre.dev/tips/tokens/.) for more information on storing tokens safely). + +# What should I do if my token does get leaked? + +If for whatever reason your token gets leaked, you should immediately follow these steps: +- Go to the list of [Discord Bot Applications](https://discord.com/developers/applications) you have and select the bot application that had the token leaked. +- Select the Bot (1) tab on the left-hand side, next to a small image of a puzzle piece. After doing so you should see a small section named TOKEN (under your bot USERNAME and next to his avatar image) +- Press the Regenerate button to regenerate your bot token and invalidate the old one. + + + +Following these steps will create a new token for your bot, making it secure again and terminating any connections from the leaked token. +The old token will stop working though, so make sure to replace the old token with the new one in your code if you haven't already. + +# Summary +Make sure you keep your token secure by storing it safely, not sending it to anyone you don't trust, and regenerating your token if it does get leaked. diff --git a/pydis_site/apps/content/resources/guides/python-guides/proper-error-handling.md b/pydis_site/apps/content/resources/guides/python-guides/proper-error-handling.md new file mode 100644 index 00000000..74b0f59b --- /dev/null +++ b/pydis_site/apps/content/resources/guides/python-guides/proper-error-handling.md @@ -0,0 +1,70 @@ +--- +title: Proper error handling in discord.py +description: Are you not getting any errors? This might be why! +--- +If you're not recieving any errors in your console, even though you know you should be, try this: + +# With bot subclass: +```py +import discord +from discord.ext import commands + +import traceback +import sys + +class MyBot(commands.Bot): + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + async def on_command_error(self, ctx: commands.Context, error): + # Handle your errors here + if isinstance(error, commands.MemberNotFound): + await ctx.send("I could not find member '{error.argument}'. Please try again") + + elif isinstance(error, commands.MissingRequiredArgument): + await ctx.send(f"'{error.param.name}' is a required argument.") + else: + # All unhandled errors will print their original traceback + print(f'Ignoring exception in command {ctx.command}:', file=sys.stderr) + traceback.print_exception(type(error), error, error.__traceback__, file=sys.stderr) + +bot = MyBot(command_prefix="!", intents=discord.Intents.default()) + +bot.run("token") +``` + +# Without bot subclass +```py +import discord +from discord.ext import commands + +import traceback +import sys + +async def on_command_error(self, ctx: commands.Context, error): + # Handle your errors here + if isinstance(error, commands.MemberNotFound): + await ctx.send("I could not find member '{error.argument}'. Please try again") + + elif isinstance(error, commands.MissingRequiredArgument): + await ctx.send(f"'{error.param.name}' is a required argument.") + else: + # All unhandled errors will print their original traceback + print(f'Ignoring exception in command {ctx.command}:', file=sys.stderr) + traceback.print_exception(type(error), error, error.__traceback__, file=sys.stderr) + +bot = commands.Bot(command_prefix="!", intents=discord.Intents.default()) +bot.on_command_error = on_command_error + +bot.run("token") +``` + + +Make sure to import `traceback` and `sys`! + +------------------------------------------------------------------------------------------------------------- + +Useful Links: +- [FAQ](https://discordpy.readthedocs.io/en/latest/faq.html) +- [Simple Error Handling](https://gist.github.com/EvieePy/7822af90858ef65012ea500bcecf1612) diff --git a/pydis_site/apps/content/resources/guides/python-guides/vps-services.md b/pydis_site/apps/content/resources/guides/python-guides/vps-services.md index 0acd3e55..710fd914 100644 --- a/pydis_site/apps/content/resources/guides/python-guides/vps-services.md +++ b/pydis_site/apps/content/resources/guides/python-guides/vps-services.md @@ -1,9 +1,12 @@ --- -title: VPS Services -description: On different VPS services +title: VPS and Free Hosting Service for Discord bots +description: This article lists recommended VPS services and covers the disasdvantages of utilising a free hosting service to run a discord bot. +toc: 2 --- -If you need to run your bot 24/7 (with no downtime), you should consider using a virtual private server (VPS). This is a list of VPS services that are sufficient for running Discord bots. +## Recommended VPS services + +If you need to run your bot 24/7 (with no downtime), you should consider using a virtual private server (VPS). Here is a list of VPS services that are sufficient for running Discord bots. * Europe * [netcup](https://www.netcup.eu/) @@ -25,7 +28,31 @@ If you need to run your bot 24/7 (with no downtime), you should consider using a * [OVHcloud](https://www.ovhcloud.com/) * [Vultr](https://www.vultr.com/) ---- -# Free hosts -There are no reliable free options for VPS hosting. If you would rather not pay for a hosting service, you can consider self-hosting. -Any modern hardware should be sufficient for running a bot. An old computer with a few GB of ram could be suitable, or a Raspberry Pi. + +## Why not to use free hosting services for bots? +While these may seem like nice and free services, it has a lot more caveats than you may think. For example, the drawbacks of using common free hosting services to host a discord bot are discussed below. + +### Replit + +- The machines are super underpowered, resulting in your bot lagging a lot as it gets bigger. + +- You need to run a webserver alongside your bot to prevent it from being shut off. This uses extra machine power. + +- Repl.it uses an ephemeral file system. This means any file you saved through your bot will be overwritten when you next launch. + +- They use a shared IP for everything running on the service. +This one is important - if someone is running a user bot on their service and gets banned, everyone on that IP will be banned. Including you. + +### Heroku +- Bots are not what the platform is designed for. Heroku is designed to provide web servers (like Django, Flask, etc). This is why they give you a domain name and open a port on their local emulator. + +- Heroku's environment is heavily containerized, making it significantly underpowered for a standard use case. + +- Heroku's environment is volatile. In order to handle the insane amount of users trying to use it for their own applications, Heroku will dispose your environment every time your application dies unless you pay. + +- Heroku has minimal system dependency control. If any of your Python requirements need C bindings (such as PyNaCl + binding to libsodium, or lxml binding to libxml), they are unlikely to function properly, if at all, in a native + environment. As such, you often need to resort to adding third-party buildpacks to facilitate otherwise normal + CPython extension functionality. (This is the reason why voice doesn't work natively on heroku) + +- Heroku only offers a limited amount of time on their free programme for your applications. If you exceed this limit, which you probably will, they'll shut down your application until your free credit resets. diff --git a/pydis_site/apps/content/resources/guides/python-guides/why-not-json-as-database.md b/pydis_site/apps/content/resources/guides/python-guides/why-not-json-as-database.md new file mode 100644 index 00000000..ae34c2b4 --- /dev/null +++ b/pydis_site/apps/content/resources/guides/python-guides/why-not-json-as-database.md @@ -0,0 +1,28 @@ +--- +title: Why JSON is unsuitable as a database +description: The many reasons why you shouldn't use JSON as a database, and instead opt for SQL. +relevant_links: + Tips on Storing Data: https://tutorial.vcokltfre.dev/tips/storage/ +--- + +JSON, quite simply, is not a database. It's not designed to be a data storage format, +rather a wayof transmitting data over a network. It's also often used as a way of doing configuration files for programs. + +There is no redundancy built in to JSON. JSON is just a format, and Python has libraries for it +like json and ujson that let you load and dump it, sometimes to files, but that's all it does, write data to a file. +There is no sort of DBMS (Database Management System), which means no sort of sophistication in how the data is stored, +or built in ways to keep it safe and backed up, there's no built in encryption either - bear in mind +in larger applications encryption may be necessary for GDPR/relevant data protection regulations compliance. + +JSON, unlike relational databases, has no way to store relational data, +which is a very commonly needed way of storing data. +Relational data, as the name may suggest, is data that relates to other data. +For example if you have a table of users and a table of servers, the server table will probably have an owner field, +where you'd reference a user from the users table. (**This is only relevant for relational data**). + +JSON is primarily a KV (key-value) format, for example `{"a": "b"}` where `a` is the key and `b` is the value, +but what if you want to search not by that key but by a sub-key? Well, instead of being able to quickly use `var[key]`, +which in a Python dictionary has a constant return time (for more info look up hash tables), +you now have to iterate through every object in the dictionary and compare to find what you're looking for. +Most relational database systems, like MySQL, MariaDB, and PostgreSQL have ways of indexing secondary fields +apart from the primary key so that you can easily search by multiple attributes. diff --git a/pydis_site/apps/content/resources/server-info/roles.md b/pydis_site/apps/content/resources/server-info/roles.md index edc02066..409e037e 100644 --- a/pydis_site/apps/content/resources/server-info/roles.md +++ b/pydis_site/apps/content/resources/server-info/roles.md @@ -28,8 +28,12 @@ There are multiple requirements listed there for getting the role. This includes writing pull requests for open issues, and also for reviewing open pull requests (**we really need reviewers!**) **How to get it:** Contribute to the projects! -There is no minimum requirements, but the role is **not** assigned for every single contribution. -Read more about this in the [Guidelines for the Contributors Role](/pages/contributing/#guidelines-for-the-contributors-role) on the Contributing page. +It’s difficult to precisely quantify contributions, but we’ve come up with the following guidelines for the role: + +- The member has made several significant contributions to our projects. +- The member has a positive influence in our contributors subcommunity. + +The role will be assigned at the discretion of the Admin Team in consultation with the Core Developers Team. Check out our [walkthrough](/pages/contributing/) to get started contributing. --- diff --git a/pydis_site/apps/content/resources/tags/_info.yml b/pydis_site/apps/content/resources/tags/_info.yml new file mode 100644 index 00000000..054125ec --- /dev/null +++ b/pydis_site/apps/content/resources/tags/_info.yml @@ -0,0 +1,3 @@ +title: Tags +description: Useful snippets that are often used in the server. +icon: fas fa-tags diff --git a/pydis_site/apps/content/tests/test_utils.py b/pydis_site/apps/content/tests/test_utils.py index be5ea897..462818b5 100644 --- a/pydis_site/apps/content/tests/test_utils.py +++ b/pydis_site/apps/content/tests/test_utils.py @@ -1,12 +1,34 @@ +import datetime +import json +import tarfile +import tempfile +import textwrap from pathlib import Path +from unittest import mock +import httpx +import markdown from django.http import Http404 +from django.test import TestCase -from pydis_site.apps.content import utils +from pydis_site import settings +from pydis_site.apps.content import models, utils from pydis_site.apps.content.tests.helpers import ( BASE_PATH, MockPagesTestCase, PARSED_CATEGORY_INFO, PARSED_HTML, PARSED_METADATA ) +_time = datetime.datetime(2022, 10, 10, 10, 10, 10, tzinfo=datetime.timezone.utc) +_time_str = _time.strftime(settings.GITHUB_TIMESTAMP_FORMAT) +TEST_COMMIT_KWARGS = { + "sha": "123", + "message": "Hello world\n\nThis is a commit message", + "date": _time, + "authors": json.dumps([ + {"name": "Author 1", "email": "[email protected]", "date": _time_str}, + {"name": "Author 2", "email": "[email protected]", "date": _time_str}, + ]), +} + class GetCategoryTests(MockPagesTestCase): """Tests for the get_category function.""" @@ -96,3 +118,268 @@ class GetPageTests(MockPagesTestCase): def test_get_nonexistent_page_returns_404(self): with self.assertRaises(Http404): utils.get_page(Path(BASE_PATH, "invalid")) + + +class TagUtilsTests(TestCase): + """Tests for the tag-related utilities.""" + + def setUp(self) -> None: + super().setUp() + self.commit = models.Commit.objects.create(**TEST_COMMIT_KWARGS) + + @mock.patch.object(utils, "fetch_tags") + def test_static_fetch(self, fetch_mock: mock.Mock): + """Test that the static fetch function is only called at most once during static builds.""" + tags = [models.Tag(name="Name", body="body")] + fetch_mock.return_value = tags + result = utils.get_tags_static() + second_result = utils.get_tags_static() + + fetch_mock.assert_called_once() + self.assertEqual(tags, result) + self.assertEqual(tags, second_result) + + @mock.patch("httpx.Client.get") + def test_mocked_fetch(self, get_mock: mock.Mock): + """Test that proper data is returned from fetch, but with a mocked API response.""" + fake_request = httpx.Request("GET", "https://google.com") + + # Metadata requests + returns = [httpx.Response( + request=fake_request, + status_code=200, + json=[ + {"type": "file", "name": "first_tag.md", "sha": "123"}, + {"type": "file", "name": "second_tag.md", "sha": "456"}, + {"type": "dir", "name": "some_group", "sha": "789", "url": "/some_group"}, + ] + ), httpx.Response( + request=fake_request, + status_code=200, + json=[{"type": "file", "name": "grouped_tag.md", "sha": "789123"}] + )] + + # Main content request + bodies = ( + "This is the first tag!", + textwrap.dedent(""" + --- + frontmatter: empty + --- + This tag has frontmatter! + """), + "This is a grouped tag!", + ) + + # Generate a tar archive with a few tags + with tempfile.TemporaryDirectory() as tar_folder: + tar_folder = Path(tar_folder) + with tempfile.TemporaryDirectory() as folder: + folder = Path(folder) + (folder / "ignored_file.md").write_text("This is an ignored file.") + tags_folder = folder / "bot/resources/tags" + tags_folder.mkdir(parents=True) + + (tags_folder / "first_tag.md").write_text(bodies[0]) + (tags_folder / "second_tag.md").write_text(bodies[1]) + + group_folder = tags_folder / "some_group" + group_folder.mkdir() + (group_folder / "grouped_tag.md").write_text(bodies[2]) + + with tarfile.open(tar_folder / "temp.tar", "w") as file: + file.add(folder, recursive=True) + + body = (tar_folder / "temp.tar").read_bytes() + + returns.append(httpx.Response( + status_code=200, + content=body, + request=fake_request, + )) + + get_mock.side_effect = returns + result = utils.fetch_tags() + + def sort(_tag: models.Tag) -> str: + return _tag.name + + self.assertEqual(sorted([ + models.Tag(name="first_tag", body=bodies[0], sha="123"), + models.Tag(name="second_tag", body=bodies[1], sha="245"), + models.Tag(name="grouped_tag", body=bodies[2], group=group_folder.name, sha="789123"), + ], key=sort), sorted(result, key=sort)) + + def test_get_real_tag(self): + """Test that a single tag is returned if it exists.""" + tag = models.Tag.objects.create(name="real-tag", last_commit=self.commit) + result = utils.get_tag("real-tag") + + self.assertEqual(tag, result) + + def test_get_grouped_tag(self): + """Test fetching a tag from a group.""" + tag = models.Tag.objects.create( + name="real-tag", group="real-group", last_commit=self.commit + ) + result = utils.get_tag("real-group/real-tag") + + self.assertEqual(tag, result) + + def test_get_group(self): + """Test fetching a group of tags.""" + included = [ + models.Tag.objects.create(name="tag-1", group="real-group"), + models.Tag.objects.create(name="tag-2", group="real-group"), + models.Tag.objects.create(name="tag-3", group="real-group"), + ] + + models.Tag.objects.create(name="not-included-1") + models.Tag.objects.create(name="not-included-2", group="other-group") + + result = utils.get_tag("real-group") + self.assertListEqual(included, result) + + def test_get_tag_404(self): + """Test that an error is raised when we fetch a non-existing tag.""" + models.Tag.objects.create(name="real-tag") + with self.assertRaises(models.Tag.DoesNotExist): + utils.get_tag("fake") + + @mock.patch.object(utils, "get_tag_category") + def test_category_pages(self, get_mock: mock.Mock): + """Test that the category pages function calls the correct method for tags.""" + tag = models.Tag.objects.create(name="tag") + get_mock.return_value = tag + result = utils.get_category_pages(settings.CONTENT_PAGES_PATH / "tags") + self.assertEqual(tag, result) + get_mock.assert_called_once_with(collapse_groups=True) + + def test_get_category_root(self): + """Test that all tags are returned and formatted properly for the tag root page.""" + body = "normal body" + base = {"description": markdown.markdown(body), "icon": "fas fa-tag"} + + models.Tag.objects.create(name="tag-1", body=body), + models.Tag.objects.create(name="tag-2", body=body), + models.Tag.objects.create(name="tag-3", body=body), + + models.Tag.objects.create(name="tag-4", body=body, group="tag-group") + models.Tag.objects.create(name="tag-5", body=body, group="tag-group") + + result = utils.get_tag_category(collapse_groups=True) + + self.assertDictEqual({ + "tag-1": {**base, "title": "tag-1"}, + "tag-2": {**base, "title": "tag-2"}, + "tag-3": {**base, "title": "tag-3"}, + "tag-group": { + "title": "tag-group", + "description": "Contains the following tags: tag-4, tag-5", + "icon": "fas fa-tags" + } + }, result) + + def test_get_category_group(self): + """Test the function for a group root page.""" + body = "normal body" + base = {"description": markdown.markdown(body), "icon": "fas fa-tag"} + + included = [ + models.Tag.objects.create(name="tag-1", body=body, group="group"), + models.Tag.objects.create(name="tag-2", body=body, group="group"), + ] + models.Tag.objects.create(name="not-included", body=body) + + result = utils.get_tag_category(included, collapse_groups=False) + self.assertDictEqual({ + "tag-1": {**base, "title": "tag-1"}, + "tag-2": {**base, "title": "tag-2"}, + }, result) + + def test_tag_url(self): + """Test that tag URLs are generated correctly.""" + cases = [ + ({"name": "tag"}, f"{models.Tag.URL_BASE}/tag.md"), + ({"name": "grouped", "group": "abc"}, f"{models.Tag.URL_BASE}/abc/grouped.md"), + ] + + for options, url in cases: + tag = models.Tag(**options) + with self.subTest(tag=tag): + self.assertEqual(url, tag.url) + + @mock.patch("httpx.Client.get") + def test_get_tag_commit(self, get_mock: mock.Mock): + """Test the get commit function with a normal tag.""" + tag = models.Tag.objects.create(name="example") + + authors = json.loads(self.commit.authors) + + get_mock.return_value = httpx.Response( + request=httpx.Request("GET", "https://google.com"), + status_code=200, + json=[{ + "sha": self.commit.sha, + "commit": { + "message": self.commit.message, + "author": authors[0], + "committer": authors[1], + } + }] + ) + + result = utils.get_tag(tag.name) + self.assertEqual(tag, result) + + get_mock.assert_called_once() + call_params = get_mock.call_args[1]["params"] + + self.assertEqual({"path": "/bot/resources/tags/example.md"}, call_params) + self.assertEqual(self.commit, models.Tag.objects.get(name=tag.name).last_commit) + + @mock.patch("httpx.Client.get") + def test_get_group_tag_commit(self, get_mock: mock.Mock): + """Test the get commit function with a group tag.""" + tag = models.Tag.objects.create(name="example", group="group-name") + + authors = json.loads(self.commit.authors) + authors.pop() + self.commit.authors = json.dumps(authors) + self.commit.save() + + get_mock.return_value = httpx.Response( + request=httpx.Request("GET", "https://google.com"), + status_code=200, + json=[{ + "sha": self.commit.sha, + "commit": { + "message": self.commit.message, + "author": authors[0], + "committer": authors[0], + } + }] + ) + + utils.set_tag_commit(tag) + + get_mock.assert_called_once() + call_params = get_mock.call_args[1]["params"] + + self.assertEqual({"path": "/bot/resources/tags/group-name/example.md"}, call_params) + self.assertEqual(self.commit, models.Tag.objects.get(name=tag.name).last_commit) + + @mock.patch.object(utils, "set_tag_commit") + def test_exiting_commit(self, set_commit_mock: mock.Mock): + """Test that a commit is saved when the data has not changed.""" + tag = models.Tag.objects.create(name="tag-name", body="old body", last_commit=self.commit) + + # This is only applied to the object, not to the database + tag.last_commit = None + + utils.record_tags([tag]) + self.assertEqual(self.commit, tag.last_commit) + + result = utils.get_tag("tag-name") + self.assertEqual(tag, result) + set_commit_mock.assert_not_called() diff --git a/pydis_site/apps/content/tests/test_views.py b/pydis_site/apps/content/tests/test_views.py index eadad7e3..3ef9bcc4 100644 --- a/pydis_site/apps/content/tests/test_views.py +++ b/pydis_site/apps/content/tests/test_views.py @@ -1,12 +1,18 @@ +import textwrap from pathlib import Path from unittest import TestCase +import django.test +import markdown from django.http import Http404 from django.test import RequestFactory, SimpleTestCase, override_settings +from django.urls import reverse +from pydis_site.apps.content.models import Commit, Tag from pydis_site.apps.content.tests.helpers import ( BASE_PATH, MockPagesTestCase, PARSED_CATEGORY_INFO, PARSED_HTML, PARSED_METADATA ) +from pydis_site.apps.content.tests.test_utils import TEST_COMMIT_KWARGS from pydis_site.apps.content.views import PageOrCategoryView @@ -172,7 +178,7 @@ class PageOrCategoryViewTests(MockPagesTestCase, SimpleTestCase, TestCase): for item in context["breadcrumb_items"]: item["path"] = Path(item["path"]) - self.assertEquals( + self.assertEqual( context["breadcrumb_items"], [ {"name": PARSED_CATEGORY_INFO["title"], "path": Path(".")}, @@ -180,3 +186,217 @@ class PageOrCategoryViewTests(MockPagesTestCase, SimpleTestCase, TestCase): {"name": PARSED_CATEGORY_INFO["title"], "path": Path("category/subcategory")}, ] ) + + +class TagViewTests(django.test.TestCase): + """Tests for the TagView class.""" + + def setUp(self): + """Set test helpers, then set up fake filesystem.""" + super().setUp() + self.commit = Commit.objects.create(**TEST_COMMIT_KWARGS) + + def test_routing(self): + """Test that the correct template is returned for each route.""" + Tag.objects.create(name="example", last_commit=self.commit) + Tag.objects.create(name="grouped-tag", group="group-name", last_commit=self.commit) + + cases = [ + ("/pages/tags/example/", "content/tag.html"), + ("/pages/tags/group-name/", "content/listing.html"), + ("/pages/tags/group-name/grouped-tag/", "content/tag.html"), + ] + + for url, template in cases: + with self.subTest(url=url): + response = self.client.get(url) + self.assertEqual(200, response.status_code) + self.assertTemplateUsed(response, template) + + def test_valid_tag_returns_200(self): + """Test that a page is returned for a valid tag.""" + Tag.objects.create(name="example", body="This is the tag body.", last_commit=self.commit) + response = self.client.get("/pages/tags/example/") + self.assertEqual(200, response.status_code) + self.assertIn("This is the tag body", response.content.decode("utf-8")) + self.assertTemplateUsed(response, "content/tag.html") + + def test_invalid_tag_404(self): + """Test that a tag which doesn't exist raises a 404.""" + response = self.client.get("/pages/tags/non-existent/") + self.assertEqual(404, response.status_code) + + def test_context_tag(self): + """Test that the context contains the required data for a tag.""" + body = textwrap.dedent(""" + --- + unused: frontmatter + ---- + Tag content here. + """) + + tag = Tag.objects.create(name="example", body=body, last_commit=self.commit) + response = self.client.get("/pages/tags/example/") + expected = { + "page_title": "example", + "page": markdown.markdown("Tag content here."), + "tag": tag, + "breadcrumb_items": [ + {"name": "Pages", "path": "."}, + {"name": "Tags", "path": "tags"}, + ] + } + for key in expected: + self.assertEqual( + expected[key], response.context.get(key), f"context.{key} did not match" + ) + + def test_context_grouped_tag(self): + """ + Test the context for a tag in a group. + + The only difference between this and a regular tag are the breadcrumbs, + so only those are checked. + """ + Tag.objects.create( + name="example", body="Body text", group="group-name", last_commit=self.commit + ) + response = self.client.get("/pages/tags/group-name/example/") + self.assertListEqual([ + {"name": "Pages", "path": "."}, + {"name": "Tags", "path": "tags"}, + {"name": "group-name", "path": "tags/group-name"}, + ], response.context.get("breadcrumb_items")) + + def test_group_page(self): + """Test rendering of a group's root page.""" + Tag.objects.create(name="tag-1", body="Body 1", group="group-name", last_commit=self.commit) + Tag.objects.create(name="tag-2", body="Body 2", group="group-name", last_commit=self.commit) + Tag.objects.create(name="not-included", last_commit=self.commit) + + response = self.client.get("/pages/tags/group-name/") + content = response.content.decode("utf-8") + + self.assertInHTML("<div class='level-left'>group-name</div>", content) + self.assertInHTML( + f"<a class='level-item fab fa-github' href='{Tag.URL_BASE}/group-name'>", + content + ) + self.assertIn(">tag-1</span>", content) + self.assertIn(">tag-2</span>", content) + self.assertNotIn( + ">not-included</span>", + content, + "Tags not in this group shouldn't be rendered." + ) + + self.assertInHTML("<p>Body 1</p>", content) + + def test_markdown(self): + """Test that markdown content is rendered properly.""" + body = textwrap.dedent(""" + ```py + Hello world! + ``` + + **This text is in bold** + """) + + Tag.objects.create(name="example", body=body, last_commit=self.commit) + response = self.client.get("/pages/tags/example/") + content = response.content.decode("utf-8") + + self.assertInHTML('<code class="language-py">Hello world!</code>', content) + self.assertInHTML("<strong>This text is in bold</strong>", content) + + def test_embed(self): + """Test that an embed from the frontmatter is treated correctly.""" + body = textwrap.dedent(""" + --- + embed: + title: Embed title + image: + url: https://google.com + --- + Tag body. + """) + + Tag.objects.create(name="example", body=body, last_commit=self.commit) + response = self.client.get("/pages/tags/example/") + content = response.content.decode("utf-8") + + self.assertInHTML('<img alt="Embed title" src="https://google.com"/>', content) + self.assertInHTML("<p>Tag body.</p>", content) + + def test_embed_title(self): + """Test that the page title gets set to the embed title.""" + body = textwrap.dedent(""" + --- + embed: + title: Embed title + --- + """) + + Tag.objects.create(name="example", body=body, last_commit=self.commit) + response = self.client.get("/pages/tags/example/") + self.assertEqual( + "Embed title", + response.context.get("page_title"), + "The page title must match the embed title." + ) + + def test_hyperlinked_item(self): + """Test hyperlinking of tags works as intended.""" + filler_before, filler_after = "empty filler text\n\n", "more\nfiller" + body = filler_before + "`!tags return`" + filler_after + Tag.objects.create(name="example", body=body, last_commit=self.commit) + + other_url = reverse("content:tag", kwargs={"location": "return"}) + response = self.client.get("/pages/tags/example/") + self.assertEqual( + markdown.markdown(filler_before + f"[`!tags return`]({other_url})" + filler_after), + response.context.get("page") + ) + + def test_hyperlinked_group(self): + """Test hyperlinking with a group works as intended.""" + Tag.objects.create( + name="example", body="!tags group-name grouped-tag", last_commit=self.commit + ) + Tag.objects.create(name="grouped-tag", group="group-name") + + other_url = reverse("content:tag", kwargs={"location": "group-name/grouped-tag"}) + response = self.client.get("/pages/tags/example/") + self.assertEqual( + markdown.markdown(f"[!tags group-name grouped-tag]({other_url})"), + response.context.get("page") + ) + + def test_hyperlinked_extra_text(self): + """Test hyperlinking when a tag is followed by extra, unrelated text.""" + Tag.objects.create( + name="example", body="!tags other unrelated text", last_commit=self.commit + ) + Tag.objects.create(name="other") + + other_url = reverse("content:tag", kwargs={"location": "other"}) + response = self.client.get("/pages/tags/example/") + self.assertEqual( + markdown.markdown(f"[!tags other]({other_url}) unrelated text"), + response.context.get("page") + ) + + def test_tag_root_page(self): + """Test the root tag page which lists all tags.""" + Tag.objects.create(name="tag-1", last_commit=self.commit) + Tag.objects.create(name="tag-2", last_commit=self.commit) + Tag.objects.create(name="tag-3", last_commit=self.commit) + + response = self.client.get("/pages/tags/") + content = response.content.decode("utf-8") + + self.assertTemplateUsed(response, "content/listing.html") + self.assertInHTML('<div class="level-left">Tags</div>', content) + + for tag_number in range(1, 4): + self.assertIn(f"tag-{tag_number}</span>", content) diff --git a/pydis_site/apps/content/urls.py b/pydis_site/apps/content/urls.py index f8496095..a7695a27 100644 --- a/pydis_site/apps/content/urls.py +++ b/pydis_site/apps/content/urls.py @@ -3,7 +3,7 @@ from pathlib import Path from django_distill import distill_path -from . import views +from . import utils, views app_name = "content" @@ -29,15 +29,38 @@ def __get_all_files(root: Path, folder: typing.Optional[Path] = None) -> list[st return results -def get_all_pages() -> typing.Iterator[dict[str, str]]: +DISTILL_RETURN = typing.Iterator[dict[str, str]] + + +def get_all_pages() -> DISTILL_RETURN: """Yield a dict of all page categories.""" for location in __get_all_files(Path("pydis_site", "apps", "content", "resources")): yield {"location": location} +def get_all_tags() -> DISTILL_RETURN: + """Return all tag names and groups in static builds.""" + # We instantiate the set with None here to make filtering it out later easier + # whether it was added in the loop or not + groups = {None} + for tag in utils.get_tags_static(): + groups.add(tag.group) + yield {"location": (f"{tag.group}/" if tag.group else "") + tag.name} + + groups.remove(None) + for group in groups: + yield {"location": group} + + urlpatterns = [ distill_path("", views.PageOrCategoryView.as_view(), name='pages'), distill_path( + "tags/<path:location>/", + views.TagView.as_view(), + name="tag", + distill_func=get_all_tags + ), + distill_path( "<path:location>/", views.PageOrCategoryView.as_view(), name='page_category', diff --git a/pydis_site/apps/content/utils.py b/pydis_site/apps/content/utils.py index d3f270ff..c12893ef 100644 --- a/pydis_site/apps/content/utils.py +++ b/pydis_site/apps/content/utils.py @@ -1,14 +1,41 @@ +import datetime +import functools +import json +import tarfile +import tempfile +from io import BytesIO from pathlib import Path -from typing import Dict, Tuple import frontmatter +import httpx import markdown import yaml from django.http import Http404 +from django.utils import timezone from markdown.extensions.toc import TocExtension +from pydis_site import settings +from .models import Commit, Tag -def get_category(path: Path) -> Dict[str, str]: +TAG_CACHE_TTL = datetime.timedelta(hours=1) + + +def github_client(**kwargs) -> httpx.Client: + """Get a client to access the GitHub API with important settings pre-configured.""" + client = httpx.Client( + base_url=settings.GITHUB_API, + follow_redirects=True, + timeout=settings.TIMEOUT_PERIOD, + **kwargs + ) + if settings.GITHUB_TOKEN: # pragma: no cover + if not client.headers.get("Authorization"): + client.headers = {"Authorization": f"token {settings.GITHUB_TOKEN}"} + + return client + + +def get_category(path: Path) -> dict[str, str]: """Load category information by name from _info.yml.""" if not path.is_dir(): raise Http404("Category not found.") @@ -16,7 +43,7 @@ def get_category(path: Path) -> Dict[str, str]: return yaml.safe_load(path.joinpath("_info.yml").read_text(encoding="utf-8")) -def get_categories(path: Path) -> Dict[str, Dict]: +def get_categories(path: Path) -> dict[str, dict]: """Get information for all categories.""" categories = {} @@ -27,8 +54,253 @@ def get_categories(path: Path) -> Dict[str, Dict]: return categories -def get_category_pages(path: Path) -> Dict[str, Dict]: +def get_tags_static() -> list[Tag]: + """ + Fetch tag information in static builds. + + This also includes some fake tags to preview the tag groups feature. + This will return a cached value, so it should only be used for static builds. + """ + tags = fetch_tags() + for tag in tags[3:5]: # pragma: no cover + tag.group = "very-cool-group" + return tags + + +def fetch_tags() -> list[Tag]: + """ + Fetch tag data from the GitHub API. + + The entire repository is downloaded and extracted locally because + getting file content would require one request per file, and can get rate-limited. + """ + with github_client() as client: + # Grab metadata + metadata = client.get("/repos/python-discord/bot/contents/bot/resources") + metadata.raise_for_status() + + hashes = {} + for entry in metadata.json(): + if entry["type"] == "dir": + # Tag group + files = client.get(entry["url"]) + files.raise_for_status() + files = files.json() + else: + files = [entry] + + for file in files: + hashes[file["name"]] = file["sha"] + + # Download the files + tar_file = client.get("/repos/python-discord/bot/tarball") + tar_file.raise_for_status() + + tags = [] + with tempfile.TemporaryDirectory() as folder: + with tarfile.open(fileobj=BytesIO(tar_file.content)) as repo: + included = [] + for file in repo.getmembers(): + if "/bot/resources/tags" in file.path: + included.append(file) + repo.extractall(folder, included) + + for tag_file in Path(folder).rglob("*.md"): + name = tag_file.name + group = None + if tag_file.parent.name != "tags": + # Tags in sub-folders are considered part of a group + group = tag_file.parent.name + + tags.append(Tag( + name=name.removesuffix(".md"), + sha=hashes[name], + group=group, + body=tag_file.read_text(encoding="utf-8"), + last_commit=None, + )) + + return tags + + +def set_tag_commit(tag: Tag) -> None: + """Fetch commit information from the API, and save it for the tag.""" + if settings.STATIC_BUILD: # pragma: no cover + # Static builds request every page during build, which can ratelimit it. + # Instead, we return some fake data. + tag.last_commit = Commit( + sha="68da80efc00d9932a209d5cccd8d344cec0f09ea", + message="Initial Commit\n\nTHIS IS FAKE DEMO DATA", + date=datetime.datetime(2018, 2, 3, 12, 20, 26, tzinfo=datetime.timezone.utc), + authors=json.dumps([{"name": "Joseph", "email": "[email protected]"}]), + ) + return + + path = "/bot/resources/tags" + if tag.group: + path += f"/{tag.group}" + path += f"/{tag.name}.md" + + # Fetch and set the commit + with github_client() as client: + data = client.get("/repos/python-discord/bot/commits", params={"path": path}) + data.raise_for_status() + data = data.json()[0] + + commit = data["commit"] + author, committer = commit["author"], commit["committer"] + + date = datetime.datetime.strptime(committer["date"], settings.GITHUB_TIMESTAMP_FORMAT) + date = date.replace(tzinfo=datetime.timezone.utc) + + if author["email"] == committer["email"]: + authors = [author] + else: + authors = [author, committer] + + commit_obj, _ = Commit.objects.get_or_create( + sha=data["sha"], + message=commit["message"], + date=date, + authors=json.dumps(authors), + ) + tag.last_commit = commit_obj + tag.save() + + +def record_tags(tags: list[Tag]) -> None: + """Sync the database with an updated set of tags.""" + # Remove entries which no longer exist + Tag.objects.exclude(name__in=[tag.name for tag in tags]).delete() + + # Insert/update the tags + for new_tag in tags: + try: + old_tag = Tag.objects.get(name=new_tag.name) + except Tag.DoesNotExist: + # The tag is not in the database yet, + # pretend it's previous state is the current state + old_tag = new_tag + + if old_tag.sha == new_tag.sha and old_tag.last_commit is not None: + # We still have an up-to-date commit entry + new_tag.last_commit = old_tag.last_commit + + new_tag.save() + + # Drop old, unused commits + Commit.objects.filter(tag__isnull=True).delete() + + +def get_tags() -> list[Tag]: + """Return a list of all tags visible to the application, from the cache or API.""" + if settings.STATIC_BUILD: # pragma: no cover + last_update = None + else: + last_update = ( + Tag.objects.values_list("last_updated", flat=True) + .order_by("last_updated").first() + ) + + if last_update is None or timezone.now() >= (last_update + TAG_CACHE_TTL): + # Stale or empty cache + if settings.STATIC_BUILD: # pragma: no cover + tags = get_tags_static() + else: + tags = fetch_tags() + record_tags(tags) + + return tags + else: + # Get tags from database + return list(Tag.objects.all()) + + +def get_tag(path: str, *, skip_sync: bool = False) -> Tag | list[Tag]: + """ + Return a tag based on the search location. + + If certain tag data is out of sync (for instance a commit date is missing), + an extra request will be made to sync the information. + + The tag name and group must match. If only one argument is provided in the path, + it's assumed to either be a group name, or a no-group tag name. + + If it's a group name, a list of tags which belong to it is returned. + """ + path = path.split("/") + if len(path) == 2: + group, name = path + else: + name = path[0] + group = None + + matches = [] + for tag in get_tags(): + if tag.name == name and tag.group == group: + if tag.last_commit is None and not skip_sync: + set_tag_commit(tag) + return tag + elif tag.group == name and group is None: + matches.append(tag) + + if matches: + return matches + + raise Tag.DoesNotExist() + + +def get_tag_category(tags: list[Tag] | None = None, *, collapse_groups: bool) -> dict[str, dict]: + """ + Generate context data for `tags`, or all tags if None. + + If `tags` is None, `get_tag` is used to populate the data. + If `collapse_groups` is True, tags with parent groups are not included in the list, + and instead the parent itself is included as a single entry with it's sub-tags + in the description. + """ + if not tags: + tags = get_tags() + + data = [] + groups = {} + + # Create all the metadata for the tags + for tag in tags: + if tag.group is None or not collapse_groups: + content = frontmatter.parse(tag.body)[1] + data.append({ + "title": tag.name, + "description": markdown.markdown(content, extensions=["pymdownx.superfences"]), + "icon": "fas fa-tag", + }) + else: + if tag.group not in groups: + groups[tag.group] = { + "title": tag.group, + "description": [tag.name], + "icon": "fas fa-tags", + } + else: + groups[tag.group]["description"].append(tag.name) + + # Flatten group description into a single string + for group in groups.values(): + # If the following string is updated, make sure to update it in the frontend JS as well + group["description"] = "Contains the following tags: " + ", ".join(group["description"]) + data.append(group) + + # Sort the tags, and return them in the proper format + return {tag["title"]: tag for tag in sorted(data, key=lambda tag: tag["title"].casefold())} + + +def get_category_pages(path: Path) -> dict[str, dict]: """Get all page names and their metadata at a category path.""" + # Special handling for tags + if path == Path(__file__).parent / "resources/tags": + return get_tag_category(collapse_groups=True) + pages = {} for item in path.glob("*.md"): @@ -39,7 +311,7 @@ def get_category_pages(path: Path) -> Dict[str, Dict]: return pages -def get_page(path: Path) -> Tuple[str, Dict]: +def get_page(path: Path) -> tuple[str, dict]: """Get one specific page.""" if not path.is_file(): raise Http404("Page not found.") diff --git a/pydis_site/apps/content/views/__init__.py b/pydis_site/apps/content/views/__init__.py index 70ea1c7a..a969b1dc 100644 --- a/pydis_site/apps/content/views/__init__.py +++ b/pydis_site/apps/content/views/__init__.py @@ -1,3 +1,4 @@ from .page_category import PageOrCategoryView +from .tags import TagView -__all__ = ["PageOrCategoryView"] +__all__ = ["PageOrCategoryView", "TagView"] diff --git a/pydis_site/apps/content/views/page_category.py b/pydis_site/apps/content/views/page_category.py index 5af77aff..062c2bc1 100644 --- a/pydis_site/apps/content/views/page_category.py +++ b/pydis_site/apps/content/views/page_category.py @@ -1,18 +1,17 @@ -import typing as t from pathlib import Path import frontmatter from django.conf import settings -from django.http import Http404 +from django.http import Http404, HttpRequest, HttpResponse from django.views.generic import TemplateView -from pydis_site.apps.content import utils +from pydis_site.apps.content import models, utils class PageOrCategoryView(TemplateView): """Handles pages and page categories.""" - def dispatch(self, request: t.Any, *args, **kwargs) -> t.Any: + def dispatch(self, request: HttpRequest, *args, **kwargs) -> HttpResponse: """Conform URL path location to the filesystem path.""" self.location = Path(kwargs.get("location", "")) @@ -25,7 +24,7 @@ class PageOrCategoryView(TemplateView): return super().dispatch(request, *args, **kwargs) - def get_template_names(self) -> t.List[str]: + def get_template_names(self) -> list[str]: """Checks if the view uses the page template or listing template.""" if self.page_path.is_file(): template_name = "content/page.html" @@ -36,7 +35,7 @@ class PageOrCategoryView(TemplateView): return [template_name] - def get_context_data(self, **kwargs) -> t.Dict[str, t.Any]: + def get_context_data(self, **kwargs) -> dict[str, any]: """Assign proper context variables based on what resource user requests.""" context = super().get_context_data(**kwargs) @@ -73,7 +72,7 @@ class PageOrCategoryView(TemplateView): return context @staticmethod - def _get_page_context(path: Path) -> t.Dict[str, t.Any]: + def _get_page_context(path: Path) -> dict[str, any]: page, metadata = utils.get_page(path) return { "page": page, @@ -84,7 +83,7 @@ class PageOrCategoryView(TemplateView): } @staticmethod - def _get_category_context(path: Path) -> t.Dict[str, t.Any]: + def _get_category_context(path: Path) -> dict[str, any]: category = utils.get_category(path) return { "categories": utils.get_categories(path), @@ -92,4 +91,7 @@ class PageOrCategoryView(TemplateView): "page_title": category["title"], "page_description": category["description"], "icon": category.get("icon"), + "app_name": "content:page_category", + "is_tag_listing": "/resources/tags" in path.as_posix(), + "tag_url": models.Tag.URL_BASE, } diff --git a/pydis_site/apps/content/views/tags.py b/pydis_site/apps/content/views/tags.py new file mode 100644 index 00000000..4f4bb5a2 --- /dev/null +++ b/pydis_site/apps/content/views/tags.py @@ -0,0 +1,124 @@ +import re +import typing + +import frontmatter +import markdown +from django.conf import settings +from django.http import Http404 +from django.urls import reverse +from django.views.generic import TemplateView + +from pydis_site.apps.content import utils +from pydis_site.apps.content.models import Tag + +# The following regex tries to parse a tag command +# It'll read up to two words seperated by spaces +# If the command does not include a group, the tag name will be in the `first` group +# If there's a second word after the command, or if there's a tag group, extra logic +# is necessary to determine whether it's a tag with a group, or a tag with text after it +COMMAND_REGEX = re.compile(r"`*!tags? (?P<first>[\w-]+)(?P<second> [\w-]+)?`*") + + +class TagView(TemplateView): + """Handles tag pages.""" + + tag: typing.Union[Tag, list[Tag]] + is_group: bool + + def setup(self, *args, **kwargs) -> None: + """Look for a tag, and configure the view.""" + super().setup(*args, **kwargs) + + try: + self.tag = utils.get_tag(kwargs.get("location")) + self.is_group = isinstance(self.tag, list) + except Tag.DoesNotExist: + raise Http404 + + def get_template_names(self) -> list[str]: + """Either return the tag page template, or the listing.""" + if self.is_group: + template_name = "content/listing.html" + else: + template_name = "content/tag.html" + + return [template_name] + + def get_context_data(self, **kwargs) -> dict: + """Get the relevant context for this tag page or group.""" + context = super().get_context_data(**kwargs) + context["breadcrumb_items"] = [{ + "name": utils.get_category(settings.CONTENT_PAGES_PATH / location)["title"], + "path": location, + } for location in (".", "tags")] + + if self.is_group: + self._set_group_context(context, self.tag) + else: + self._set_tag_context(context, self.tag) + + return context + + @staticmethod + def _set_tag_context(context: dict[str, any], tag: Tag) -> None: + """Update the context with the information for a tag page.""" + context.update({ + "page_title": tag.name, + "tag": tag, + }) + + if tag.group: + # Add group names to the breadcrumbs + context["breadcrumb_items"].append({ + "name": tag.group, + "path": f"tags/{tag.group}", + }) + + # Clean up tag body + body = frontmatter.parse(tag.body) + content = body[1] + + # Check for tags which can be hyperlinked + def sub(match: re.Match) -> str: + first, second = match.groups() + location = first + text, extra = match.group(), "" + + if second is not None: + # Possibly a tag group + try: + new_location = f"{first}/{second.strip()}" + utils.get_tag(new_location, skip_sync=True) + location = new_location + except Tag.DoesNotExist: + # Not a group, remove the second argument from the link + extra = text[text.find(second):] + text = text[:text.find(second)] + + link = reverse("content:tag", kwargs={"location": location}) + return f"[{text}]({link}){extra}" + content = COMMAND_REGEX.sub(sub, content) + + # Add support for some embed elements + if embed := body[0].get("embed"): + context["page_title"] = embed["title"] + if image := embed.get("image"): + content = f"![{embed['title']}]({image['url']})\n\n" + content + + # Insert the content + context["page"] = markdown.markdown(content, extensions=["pymdownx.superfences"]) + + @staticmethod + def _set_group_context(context: dict[str, any], tags: list[Tag]) -> None: + """Update the context with the information for a group of tags.""" + group = tags[0].group + context.update({ + "categories": {}, + "pages": utils.get_tag_category(tags, collapse_groups=False), + "page_title": group, + "icon": "fab fa-tags", + "is_tag_listing": True, + "app_name": "content:tag", + "path": f"{group}/", + "tag_url": f"{tags[0].URL_BASE}/{group}" + }) |