diff options
78 files changed, 1802 insertions, 548 deletions
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 860357868..876d32b15 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,10 +1,26 @@ repos: -- repo: local + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v2.5.0 hooks: - - id: flake8 + - id: check-merge-conflict + - id: check-toml + - id: check-yaml + args: [--unsafe] # Required due to custom constructors (e.g. !ENV) + - id: end-of-file-fixer + - id: mixed-line-ending + args: [--fix=lf] + - id: trailing-whitespace + args: [--markdown-linebreak-ext=md] + - repo: https://github.com/pre-commit/pygrep-hooks + rev: v1.5.1 + hooks: + - id: python-check-blanket-noqa + - repo: local + hooks: + - id: flake8 name: Flake8 description: This hook runs flake8 within our project's pipenv environment. - entry: pipenv run lint + entry: pipenv run flake8 language: python types: [python] - require_serial: true
\ No newline at end of file + require_serial: true diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 39f76c7b4..be591d17e 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,4 +1,4 @@ -# Contributing to one of our projects +# Contributing to one of Our Projects Our projects are open-source and are automatically deployed whenever commits are pushed to the `master` branch on each repository, so we've created a set of guidelines in order to keep everything clean and in working order. @@ -10,12 +10,12 @@ Note that contributions may be rejected on the basis of a contributor failing to 2. If you have direct access to the repository, **create a branch for your changes** and create a pull request for that branch. If not, create a branch on a fork of the repository and create a pull request from there. * It's common practice for a repository to reject direct pushes to `master`, so make branching a habit! * If PRing from your own fork, **ensure that "Allow edits from maintainers" is checked**. This gives permission for maintainers to commit changes directly to your fork, speeding up the review process. -3. **Adhere to the prevailing code style**, which we enforce using [flake8](http://flake8.pycqa.org/en/latest/index.html). - * Run `flake8` against your code **before** you push it. Your commit will be rejected by the build server if it fails to lint. - * [Git Hooks](https://git-scm.com/book/en/v2/Customizing-Git-Git-Hooks) are a powerful tool that can be a daunting to set up. Fortunately, [`pre-commit`](https://github.com/pre-commit/pre-commit) abstracts this process away from you and is provided as a dev dependency for this project. Run `pipenv run precommit` when setting up the project and you'll never have to worry about breaking the build for linting errors. +3. **Adhere to the prevailing code style**, which we enforce using [`flake8`](http://flake8.pycqa.org/en/latest/index.html) and [`pre-commit`](https://pre-commit.com/). + * Run `flake8` and `pre-commit` against your code [**before** you push it](https://soundcloud.com/lemonsaurusrex/lint-before-you-push). Your commit will be rejected by the build server if it fails to lint. + * [Git Hooks](https://git-scm.com/book/en/v2/Customizing-Git-Git-Hooks) are a powerful git feature for executing custom scripts when certain important git actions occur. The pre-commit hook is the first hook executed during the commit process and can be used to check the code being committed & abort the commit if issues, such as linting failures, are detected. While git hooks can seem daunting to configure, the `pre-commit` framework abstracts this process away from you and is provided as a dev dependency for this project. Run `pipenv run precommit` when setting up the project and you'll never have to worry about committing code that fails linting. 4. **Make great commits**. A well structured git log is key to a project's maintainability; it efficiently provides insight into when and *why* things were done for future maintainers of the project. * Commits should be as narrow in scope as possible. Commits that span hundreds of lines across multiple unrelated functions and/or files are very hard for maintainers to follow. After about a week they'll probably be hard for you to follow too. - * Try to avoid making minor commits for fixing typos or linting errors. Since you've already set up a pre-commit hook to run `flake8` before a commit, you shouldn't be committing linting issues anyway. + * Avoid making minor commits for fixing typos or linting errors. Since you've already set up a `pre-commit` hook to run the linting pipeline before a commit, you shouldn't be committing linting issues anyway. * A more in-depth guide to writing great commit messages can be found in Chris Beam's [*How to Write a Git Commit Message*](https://chris.beams.io/posts/git-commit/) 5. **Avoid frequent pushes to the main repository**. This goes for PRs opened against your fork as well. Our test build pipelines are triggered every time a push to the repository (or PR) is made. Try to batch your commits until you've finished working for that session, or you've reached a point where collaborators need your commits to continue their own work. This also provides you the opportunity to amend commits for minor changes rather than having to commit them on their own because you've already pushed. * This includes merging master into your branch. Try to leave merging from master for after your PR passes review; a maintainer will bring your PR up to date before merging. Exceptions to this include: resolving merge conflicts, needing something that was pushed to master for your branch, or something was pushed to master that could potentionally affect the functionality of what you're writing. @@ -24,13 +24,12 @@ Note that contributions may be rejected on the basis of a contributor failing to * One option is to fork the other contributor's repository and submit your changes to their branch with your own pull request. We suggest following these guidelines when interacting with their repository as well. * The author(s) of inactive PRs and claimed issues will be be pinged after a week of inactivity for an update. Continued inactivity may result in the issue being released back to the community and/or PR closure. 8. **Work as a team** and collaborate wherever possible. Keep things friendly and help each other out - these are shared projects and nobody likes to have their feet trodden on. -9. **Internal projects are internal**. As a contributor, you have access to information that the rest of the server does not. With this trust comes responsibility - do not release any information you have learned as a result of your contributor position. We are very strict about announcing things at specific times, and many staff members will not appreciate a disruption of the announcement schedule. -10. All static content, such as images or audio, **must be licensed for open public use**. +9. All static content, such as images or audio, **must be licensed for open public use**. * Static content must be hosted by a service designed to do so. Failing to do so is known as "leeching" and is frowned upon, as it generates extra bandwidth costs to the host without providing benefit. It would be best if appropriately licensed content is added to the repository itself so it can be served by PyDis' infrastructure. -Above all, the needs of our community should come before the wants of an individual. Work together, build solutions to problems and try to do so in a way that people can learn from easily. Abuse of our trust may result in the loss of your Contributor role, especially in relation to Rule 7. +Above all, the needs of our community should come before the wants of an individual. Work together, build solutions to problems and try to do so in a way that people can learn from easily. Abuse of our trust may result in the loss of your Contributor role. -## Changes to this arrangement +## Changes to this Arrangement All projects evolve over time, and this contribution guide is no different. This document is open to pull requests or changes by contributors. If you believe you have something valuable to add or change, please don't hesitate to do so in a PR. @@ -43,15 +42,19 @@ To provide a standalone development environment for this project, docker compose When pulling down changes from GitHub, remember to sync your environment using `pipenv sync --dev` to ensure you're using the most up-to-date versions the project's dependencies. ### Type Hinting -[PEP 484](https://www.python.org/dev/peps/pep-0484/) formally specifies type hints for Python functions, added to the Python Standard Library in version 3.5. Type hints are recognized by most modern code editing tools and provide useful insight into both the input and output types of a function, preventing the user from having to go through the codebase to determine these types. +[PEP 484](https://www.python.org/dev/peps/pep-0484/) formally specifies type hints for Python functions, added to the Python Standard Library in version 3.5. Type hints are recognized by most modern code editing tools and provide useful insight into both the input and output types of a function, preventing the user from having to go through the codebase to determine these types. For example: ```py -def foo(input_1: int, input_2: dict) -> bool: +import typing as t + + +def foo(input_1: int, input_2: t.Dict[str, str]) -> bool: + ... ``` -Tells us that `foo` accepts an `int` and a `dict` and returns a `bool`. +Tells us that `foo` accepts an `int` and a `dict`, with `str` keys and values, and returns a `bool`. All function declarations should be type hinted in code contributed to the PyDis organization. @@ -63,15 +66,19 @@ Many documentation packages provide support for automatic documentation generati For example: ```py -def foo(bar: int, baz: dict=None) -> bool: +import typing as t + + +def foo(bar: int, baz: t.Optional[t.Dict[str, str]] = None) -> bool: """ Does some things with some stuff. :param bar: Some input - :param baz: Optional, some other input + :param baz: Optional, some dictionary with string keys and values :return: Some boolean """ + ... ``` Since PyDis does not utilize automatic documentation generation, use of this syntax should not be used in code contributed to the organization. Should the purpose and type of the input variables not be easily discernable from the variable name and type annotation, a prose explanation can be used. Explicit references to variables, functions, classes, etc. should be wrapped with backticks (`` ` ``). @@ -79,25 +86,33 @@ Since PyDis does not utilize automatic documentation generation, use of this syn For example, the above docstring would become: ```py -def foo(bar: int, baz: dict=None) -> bool: +import typing as t + + +def foo(bar: int, baz: t.Optional[t.Dict[str, str]] = None) -> bool: """ Does some things with some stuff. This function takes an index, `bar` and checks for its presence in the database `baz`, passed as a dictionary. Returns `False` if `baz` is not passed. """ + ... ``` ### Logging Levels -The project currently defines [`logging`](https://docs.python.org/3/library/logging.html) levels as follows: -* **TRACE:** Use this for tracing every step of a complex process. That way we can see which step of the process failed. Err on the side of verbose. **Note:** This is a PyDis-implemented logging level. -* **DEBUG:** Someone is interacting with the application, and the application is behaving as expected. -* **INFO:** Something completely ordinary happened. Like a cog loading during startup. -* **WARNING:** Someone is interacting with the application in an unexpected way or the application is responding in an unexpected way, but without causing an error. -* **ERROR:** An error that affects the specific part that is being interacted with -* **CRITICAL:** An error that affects the whole application. +The project currently defines [`logging`](https://docs.python.org/3/library/logging.html) levels as follows, from lowest to highest severity: +* **TRACE:** These events should be used to provide a *verbose* trace of every step of a complex process. This is essentially the `logging` equivalent of sprinkling `print` statements throughout the code. + * **Note:** This is a PyDis-implemented logging level. +* **DEBUG:** These events should add context to what's happening in a development setup to make it easier to follow what's going while working on a project. This is in the same vein as **TRACE** logging but at a much lower level of verbosity. +* **INFO:** These events are normal and don't need direct attention but are worth keeping track of in production, like checking which cogs were loaded during a start-up. +* **WARNING:** These events are out of the ordinary and should be fixed, but have not caused a failure. + * **NOTE:** Events at this logging level and higher should be reserved for events that require the attention of the DevOps team. +* **ERROR:** These events have caused a failure in a specific part of the application and require urgent attention. +* **CRITICAL:** These events have caused the whole application to fail and require immediate intervention. + +Ensure that log messages are succinct. Should you want to pass additional useful information that would otherwise make the log message overly verbose the `logging` module accepts an `extra` kwarg, which can be used to pass a dictionary. This is used to populate the `__dict__` of the `LogRecord` created for the logging event with user-defined attributes that can be accessed by a log handler. Additional information and caveats may be found [in Python's `logging` documentation](https://docs.python.org/3/library/logging.html#logging.Logger.debug). ### Work in Progress (WIP) PRs -Github [has introduced a new PR feature](https://github.blog/2019-02-14-introducing-draft-pull-requests/) that allows the PR author to mark it as a WIP. This provides both a visual and functional indicator that the contents of the PR are in a draft state and not yet ready for formal review. +Github [provides a PR feature](https://github.blog/2019-02-14-introducing-draft-pull-requests/) that allows the PR author to mark it as a WIP. This provides both a visual and functional indicator that the contents of the PR are in a draft state and not yet ready for formal review. This feature should be utilized in place of the traditional method of prepending `[WIP]` to the PR title. @@ -31,17 +31,16 @@ flake8-import-order = "~=0.18" flake8-string-format = "~=0.2" flake8-tidy-imports = "~=4.0" flake8-todo = "~=0.7" +pep8-naming = "~=0.9" pre-commit = "~=2.1" -safety = "~=1.8" unittest-xml-reporting = "~=3.0" -dodgy = "~=0.1" [requires] python_version = "3.8" [scripts] start = "python -m bot" -lint = "python -m flake8" +lint = "pre-commit run --all-files" precommit = "pre-commit install" build = "docker build -t pythondiscord/bot:latest -f Dockerfile ." push = "docker push pythondiscord/bot:latest" diff --git a/Pipfile.lock b/Pipfile.lock index 9953aab40..ad9a3173a 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "fae6dcdb6a5ebf27e8ea5044f4ca2ab854774d17affb5fd64ac85f8d0ae71187" + "sha256": "2d3ba484e8467a115126b2ba39fa5f36f103ea455477813dd658797875c79cc9" }, "pipfile-spec": 6, "requires": { @@ -18,11 +18,11 @@ "default": { "aio-pika": { "hashes": [ - "sha256:4199122a450dffd8303b7857a9d82657bf1487fe329e489520833b40fbe92406", - "sha256:fe85c7456e5c060bce4eb9cffab5b2c4d3c563cb72177977b3556c54c8e3aeb6" + "sha256:9e4614636296e0040055bd6b304e97a38cc9796669ef391fc9b36649831d43ee", + "sha256:c9d242b3c7142d64b185feb6c5cce4154962610e89ec2e9b52bd69ef01f89b2f" ], "index": "pypi", - "version": "==6.5.2" + "version": "==6.6.0" }, "aiodns": { "hashes": [ @@ -159,11 +159,11 @@ }, "deepdiff": { "hashes": [ - "sha256:b3fa588d1eac7fa318ec1fb4f2004568e04cb120a1989feda8e5e7164bcbf07a", - "sha256:ed7342d3ed3c0c2058a3fb05b477c943c9959ef62223dca9baa3375718a25d87" + "sha256:59fc1e3e7a28dd0147b0f2b00e3e27181f0f0ef4286b251d5f214a5bcd9a9bc4", + "sha256:91360be1d9d93b1d9c13ae9c5048fa83d9cff17a88eb30afaa0d7ff2d0fee17d" ], "index": "pypi", - "version": "==4.2.0" + "version": "==4.3.2" }, "discord-py": { "hashes": [ @@ -189,10 +189,10 @@ }, "humanfriendly": { "hashes": [ - "sha256:cbe04ecf964ccb951a578f396091f258448ca4b4b4c6d4b6194f48ef458fe991", - "sha256:e8e2e4524409e55d5c5cbbb4c555a0c0a9599d5e8f74d0ce1ac504ba51ad1cd2" + "sha256:25c2108a45cfd1e8fbe9cdb30b825d34ef5d5675c8e11e4775c9aedbfb0bdee2", + "sha256:3a831920e40e55ad49adb64c9179ed50c604cabca72cd300e7bd5b51310e4ebb" ], - "version": "==7.2" + "version": "==8.1" }, "idna": { "hashes": [ @@ -331,10 +331,10 @@ }, "packaging": { "hashes": [ - "sha256:170748228214b70b672c581a3dd610ee51f733018650740e98c7df862a583f73", - "sha256:e665345f9eef0c621aa0bf2f8d78cf6d21904eef16a93f020240b704a57f1334" + "sha256:3c292b474fda1671ec57d46d739d072bfd495a4f51ad01a055121d81e952b7a3", + "sha256:82f77b9bee21c1bafbf35a84905d604d5d1223801d639cf3ed140bd651c08752" ], - "version": "==20.1" + "version": "==20.3" }, "pamqp": { "hashes": [ @@ -379,17 +379,17 @@ }, "pycparser": { "hashes": [ - "sha256:a988718abfad80b6b157acce7bf130a30876d27603738ac39f140993246b25b3", - "sha256:fd64020e8a5e0369de455adf9f22795a90fdb74e6bb999e9a13fd26b54f533ef" + "sha256:2d475327684562c3a96cc71adf7dc8c4f0565175cf86b6d7a404ff4c771f15f0", + "sha256:7582ad22678f0fcd81102833f60ef8d0e57288b6b5fb00323d101be910e35705" ], - "version": "==2.19" + "version": "==2.20" }, "pygments": { "hashes": [ - "sha256:2a3fe295e54a20164a9df49c75fa58526d3be48e14aceba6d6b1e8ac0bfd6f1b", - "sha256:98c8aa5a9f778fcd1026a17361ddaf7330d1b7c62ae97c3bb0ae73e0b9b6b0fe" + "sha256:647344a061c249a3b74e230c739f434d7ea4d8b1d5f3721bc0f3558049b38f44", + "sha256:ff7a40b4860b727ab48fad6360eb351cc1b33cbf9b15a0f689ca5353e9463324" ], - "version": "==2.5.2" + "version": "==2.6.1" }, "pyparsing": { "hashes": [ @@ -422,20 +422,20 @@ }, "pyyaml": { "hashes": [ - "sha256:059b2ee3194d718896c0ad077dd8c043e5e909d9180f387ce42012662a4946d6", - "sha256:1cf708e2ac57f3aabc87405f04b86354f66799c8e62c28c5fc5f88b5521b2dbf", - "sha256:24521fa2890642614558b492b473bee0ac1f8057a7263156b02e8b14c88ce6f5", - "sha256:4fee71aa5bc6ed9d5f116327c04273e25ae31a3020386916905767ec4fc5317e", - "sha256:70024e02197337533eef7b85b068212420f950319cc8c580261963aefc75f811", - "sha256:74782fbd4d4f87ff04159e986886931456a1894c61229be9eaf4de6f6e44b99e", - "sha256:940532b111b1952befd7db542c370887a8611660d2b9becff75d39355303d82d", - "sha256:cb1f2f5e426dc9f07a7681419fe39cee823bb74f723f36f70399123f439e9b20", - "sha256:dbbb2379c19ed6042e8f11f2a2c66d39cceb8aeace421bfc29d085d93eda3689", - "sha256:e3a057b7a64f1222b56e47bcff5e4b94c4f61faac04c7c4ecb1985e18caa3994", - "sha256:e9f45bd5b92c7974e59bcd2dcc8631a6b6cc380a904725fce7bc08872e691615" + "sha256:06a0d7ba600ce0b2d2fe2e78453a470b5a6e000a985dd4a4e54e436cc36b0e97", + "sha256:240097ff019d7c70a4922b6869d8a86407758333f02203e0fc6ff79c5dcede76", + "sha256:4f4b913ca1a7319b33cfb1369e91e50354d6f07a135f3b901aca02aa95940bd2", + "sha256:69f00dca373f240f842b2931fb2c7e14ddbacd1397d57157a9b005a6a9942648", + "sha256:73f099454b799e05e5ab51423c7bcf361c58d3206fa7b0d555426b1f4d9a3eaf", + "sha256:74809a57b329d6cc0fdccee6318f44b9b8649961fa73144a98735b0aaf029f1f", + "sha256:7739fc0fa8205b3ee8808aea45e968bc90082c10aef6ea95e855e10abf4a37b2", + "sha256:95f71d2af0ff4227885f7a6605c37fd53d3a106fcab511b8860ecca9fcf400ee", + "sha256:b8eac752c5e14d3eca0e6dd9199cd627518cb5ec06add0de9d32baeee6fe645d", + "sha256:cc8955cfbfc7a115fa81d85284ee61147059a753344bc51098f3ccd69b0d7e0c", + "sha256:d13155f591e6fcc1ec3b30685d50bf0711574e2c0dfffd7644babf8b5102ca1a" ], "index": "pypi", - "version": "==5.3" + "version": "==5.3.1" }, "requests": { "hashes": [ @@ -447,11 +447,11 @@ }, "sentry-sdk": { "hashes": [ - "sha256:480eee754e60bcae983787a9a13bc8f155a111aef199afaa4f289d6a76aa622a", - "sha256:a920387dc3ee252a66679d0afecd34479fb6fc52c2bc20763793ed69e5b0dcc0" + "sha256:23808d571d2461a4ce3784ec12bbee5bdb8c026c143fe79d36cef8a6d653e71f", + "sha256:bb90a4e19c7233a580715fc986cc44be2c48fc10b31e71580a2037e1c94b6950" ], "index": "pypi", - "version": "==0.14.2" + "version": "==0.14.3" }, "six": { "hashes": [ @@ -476,11 +476,11 @@ }, "sphinx": { "hashes": [ - "sha256:776ff8333181138fae52df65be733127539623bb46cc692e7fa0fcfc80d7aa88", - "sha256:ca762da97c3b5107cbf0ab9e11d3ec7ab8d3c31377266fd613b962ed971df709" + "sha256:b4c750d546ab6d7e05bdff6ac24db8ae3e8b8253a3569b754e445110a0a12b66", + "sha256:fc312670b56cb54920d6cc2ced455a22a547910de10b3142276495ced49231cb" ], "index": "pypi", - "version": "==2.4.3" + "version": "==2.4.4" }, "sphinxcontrib-applehelp": { "hashes": [ @@ -596,13 +596,6 @@ ], "version": "==19.3.0" }, - "certifi": { - "hashes": [ - "sha256:017c25db2a153ce562900032d5bc68e9f191e44e9a0f762f373977de9df1fbb3", - "sha256:25b64c7da4cd7479594d035c08c2d809eb4aab3a26e5a990ea98cc450c320f1f" - ], - "version": "==2019.11.28" - }, "cfgv": { "hashes": [ "sha256:1ccf53320421aeeb915275a196e23b3b8ae87dea8ac6698b1638001d4a486d53", @@ -610,79 +603,49 @@ ], "version": "==3.1.0" }, - "chardet": { - "hashes": [ - "sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae", - "sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691" - ], - "version": "==3.0.4" - }, - "click": { - "hashes": [ - "sha256:2335065e6395b9e67ca716de5f7526736bfa6ceead690adf616d925bdc622b13", - "sha256:5b94b49521f6456670fdb30cd82a4eca9412788a93fa6dd6df72c94d5a8ff2d7" - ], - "version": "==7.0" - }, "coverage": { "hashes": [ - "sha256:15cf13a6896048d6d947bf7d222f36e4809ab926894beb748fc9caa14605d9c3", - "sha256:1daa3eceed220f9fdb80d5ff950dd95112cd27f70d004c7918ca6dfc6c47054c", - "sha256:1e44a022500d944d42f94df76727ba3fc0a5c0b672c358b61067abb88caee7a0", - "sha256:25dbf1110d70bab68a74b4b9d74f30e99b177cde3388e07cc7272f2168bd1477", - "sha256:3230d1003eec018ad4a472d254991e34241e0bbd513e97a29727c7c2f637bd2a", - "sha256:3dbb72eaeea5763676a1a1efd9b427a048c97c39ed92e13336e726117d0b72bf", - "sha256:5012d3b8d5a500834783689a5d2292fe06ec75dc86ee1ccdad04b6f5bf231691", - "sha256:51bc7710b13a2ae0c726f69756cf7ffd4362f4ac36546e243136187cfcc8aa73", - "sha256:527b4f316e6bf7755082a783726da20671a0cc388b786a64417780b90565b987", - "sha256:722e4557c8039aad9592c6a4213db75da08c2cd9945320220634f637251c3894", - "sha256:76e2057e8ffba5472fd28a3a010431fd9e928885ff480cb278877c6e9943cc2e", - "sha256:77afca04240c40450c331fa796b3eab6f1e15c5ecf8bf2b8bee9706cd5452fef", - "sha256:7afad9835e7a651d3551eab18cbc0fdb888f0a6136169fbef0662d9cdc9987cf", - "sha256:9bea19ac2f08672636350f203db89382121c9c2ade85d945953ef3c8cf9d2a68", - "sha256:a8b8ac7876bc3598e43e2603f772d2353d9931709345ad6c1149009fd1bc81b8", - "sha256:b0840b45187699affd4c6588286d429cd79a99d509fe3de0f209594669bb0954", - "sha256:b26aaf69713e5674efbde4d728fb7124e429c9466aeaf5f4a7e9e699b12c9fe2", - "sha256:b63dd43f455ba878e5e9f80ba4f748c0a2156dde6e0e6e690310e24d6e8caf40", - "sha256:be18f4ae5a9e46edae3f329de2191747966a34a3d93046dbdf897319923923bc", - "sha256:c312e57847db2526bc92b9bfa78266bfbaabac3fdcd751df4d062cd4c23e46dc", - "sha256:c60097190fe9dc2b329a0eb03393e2e0829156a589bd732e70794c0dd804258e", - "sha256:c62a2143e1313944bf4a5ab34fd3b4be15367a02e9478b0ce800cb510e3bbb9d", - "sha256:cc1109f54a14d940b8512ee9f1c3975c181bbb200306c6d8b87d93376538782f", - "sha256:cd60f507c125ac0ad83f05803063bed27e50fa903b9c2cfee3f8a6867ca600fc", - "sha256:d513cc3db248e566e07a0da99c230aca3556d9b09ed02f420664e2da97eac301", - "sha256:d649dc0bcace6fcdb446ae02b98798a856593b19b637c1b9af8edadf2b150bea", - "sha256:d7008a6796095a79544f4da1ee49418901961c97ca9e9d44904205ff7d6aa8cb", - "sha256:da93027835164b8223e8e5af2cf902a4c80ed93cb0909417234f4a9df3bcd9af", - "sha256:e69215621707119c6baf99bda014a45b999d37602cb7043d943c76a59b05bf52", - "sha256:ea9525e0fef2de9208250d6c5aeeee0138921057cd67fcef90fbed49c4d62d37", - "sha256:fca1669d464f0c9831fd10be2eef6b86f5ebd76c724d1e0706ebdff86bb4adf0" + "sha256:03f630aba2b9b0d69871c2e8d23a69b7fe94a1e2f5f10df5049c0df99db639a0", + "sha256:046a1a742e66d065d16fb564a26c2a15867f17695e7f3d358d7b1ad8a61bca30", + "sha256:0a907199566269e1cfa304325cc3b45c72ae341fbb3253ddde19fa820ded7a8b", + "sha256:165a48268bfb5a77e2d9dbb80de7ea917332a79c7adb747bd005b3a07ff8caf0", + "sha256:1b60a95fc995649464e0cd48cecc8288bac5f4198f21d04b8229dc4097d76823", + "sha256:1f66cf263ec77af5b8fe14ef14c5e46e2eb4a795ac495ad7c03adc72ae43fafe", + "sha256:2e08c32cbede4a29e2a701822291ae2bc9b5220a971bba9d1e7615312efd3037", + "sha256:3844c3dab800ca8536f75ae89f3cf566848a3eb2af4d9f7b1103b4f4f7a5dad6", + "sha256:408ce64078398b2ee2ec08199ea3fcf382828d2f8a19c5a5ba2946fe5ddc6c31", + "sha256:443be7602c790960b9514567917af538cac7807a7c0c0727c4d2bbd4014920fd", + "sha256:4482f69e0701139d0f2c44f3c395d1d1d37abd81bfafbf9b6efbe2542679d892", + "sha256:4a8a259bf990044351baf69d3b23e575699dd60b18460c71e81dc565f5819ac1", + "sha256:513e6526e0082c59a984448f4104c9bf346c2da9961779ede1fc458e8e8a1f78", + "sha256:5f587dfd83cb669933186661a351ad6fc7166273bc3e3a1531ec5c783d997aac", + "sha256:62061e87071497951155cbccee487980524d7abea647a1b2a6eb6b9647df9006", + "sha256:641e329e7f2c01531c45c687efcec8aeca2a78a4ff26d49184dce3d53fc35014", + "sha256:65a7e00c00472cd0f59ae09d2fb8a8aaae7f4a0cf54b2b74f3138d9f9ceb9cb2", + "sha256:6ad6ca45e9e92c05295f638e78cd42bfaaf8ee07878c9ed73e93190b26c125f7", + "sha256:73aa6e86034dad9f00f4bbf5a666a889d17d79db73bc5af04abd6c20a014d9c8", + "sha256:7c9762f80a25d8d0e4ab3cb1af5d9dffbddb3ee5d21c43e3474c84bf5ff941f7", + "sha256:85596aa5d9aac1bf39fe39d9fa1051b0f00823982a1de5766e35d495b4a36ca9", + "sha256:86a0ea78fd851b313b2e712266f663e13b6bc78c2fb260b079e8b67d970474b1", + "sha256:8a620767b8209f3446197c0e29ba895d75a1e272a36af0786ec70fe7834e4307", + "sha256:922fb9ef2c67c3ab20e22948dcfd783397e4c043a5c5fa5ff5e9df5529074b0a", + "sha256:9fad78c13e71546a76c2f8789623eec8e499f8d2d799f4b4547162ce0a4df435", + "sha256:a37c6233b28e5bc340054cf6170e7090a4e85069513320275a4dc929144dccf0", + "sha256:c3fc325ce4cbf902d05a80daa47b645d07e796a80682c1c5800d6ac5045193e5", + "sha256:cda33311cb9fb9323958a69499a667bd728a39a7aa4718d7622597a44c4f1441", + "sha256:db1d4e38c9b15be1521722e946ee24f6db95b189d1447fa9ff18dd16ba89f732", + "sha256:eda55e6e9ea258f5e4add23bcf33dc53b2c319e70806e180aecbff8d90ea24de", + "sha256:f372cdbb240e09ee855735b9d85e7f50730dcfb6296b74b95a3e5dea0615c4c1" ], "index": "pypi", - "version": "==5.0.3" + "version": "==5.0.4" }, "distlib": { "hashes": [ - "sha256:2e166e231a26b36d6dfe35a48c4464346620f8645ed0ace01ee31822b288de21", - "sha256:9b183fb98f4870e02d315d5d17baef14be74c339d827346cae544f5597698555" + "sha256:2e166e231a26b36d6dfe35a48c4464346620f8645ed0ace01ee31822b288de21" ], "version": "==0.3.0" }, - "dodgy": { - "hashes": [ - "sha256:28323cbfc9352139fdd3d316fa17f325cc0e9ac74438cbba51d70f9b48f86c3a", - "sha256:51f54c0fd886fa3854387f354b19f429d38c04f984f38bc572558b703c0542a6" - ], - "index": "pypi", - "version": "==0.2.1" - }, - "dparse": { - "hashes": [ - "sha256:00a5fdfa900629e5159bf3600d44905b333f4059a3366f28e0dbd13eeab17b19", - "sha256:cef95156fa0adedaf042cd42f9990974bec76f25dfeca4dc01f381a243d5aa5b" - ], - "version": "==0.4.1" - }, "entrypoints": { "hashes": [ "sha256:589f874b313739ad35be6e0cd7efde2a4e9b6fea91edcc34e58ecbb8dbe56d19", @@ -707,11 +670,11 @@ }, "flake8-annotations": { "hashes": [ - "sha256:19a6637a5da1bb7ea7948483ca9e2b9e15b213e687e7bf5ff8c1bfc91c185006", - "sha256:bb033b72cdd3a2b0a530bbdf2081f12fbea7d70baeaaebb5899723a45f424b8e" + "sha256:a38b44d01abd480586a92a02a2b0a36231ec42dcc5e114de78fa5db016d8d3f9", + "sha256:d5b0e8704e4e7728b352fa1464e23539ff2341ba11cc153b536fa2cf921ee659" ], "index": "pypi", - "version": "==2.0.0" + "version": "==2.0.1" }, "flake8-bugbear": { "hashes": [ @@ -737,6 +700,13 @@ "index": "pypi", "version": "==0.18.1" }, + "flake8-polyfill": { + "hashes": [ + "sha256:12be6a34ee3ab795b19ca73505e7b55826d5f6ad7230d31b18e106400169b9e9", + "sha256:e44b087597f6da52ec6393a709e7108b2905317d0c0b744cdca6208e670d8eda" + ], + "version": "==1.0.2" + }, "flake8-string-format": { "hashes": [ "sha256:65f3da786a1461ef77fca3780b314edb2853c377f2e35069723348c8917deaa2", @@ -747,11 +717,11 @@ }, "flake8-tidy-imports": { "hashes": [ - "sha256:8aa34384b45137d4cf33f5818b8e7897dc903b1d1e10a503fa7dd193a9a710ba", - "sha256:b26461561bcc80e8012e46846630ecf0aaa59314f362a94cb7800dfdb32fa413" + "sha256:5b6e75cec6d751e66534c522fbdce7dac1c2738b1216b0f6b10453995932e188", + "sha256:cf26fbb3ab31a398f265d53b6f711d80006450c19221e41b2b7b0e0b14ac39c5" ], "index": "pypi", - "version": "==4.0.0" + "version": "==4.0.1" }, "flake8-todo": { "hashes": [ @@ -762,17 +732,10 @@ }, "identify": { "hashes": [ - "sha256:1222b648251bdcb8deb240b294f450fbf704c7984e08baa92507e4ea10b436d5", - "sha256:d824ebe21f38325c771c41b08a95a761db1982f1fc0eee37c6c97df3f1636b96" - ], - "version": "==1.4.11" - }, - "idna": { - "hashes": [ - "sha256:7588d1c14ae4c77d74036e8c22ff447b26d0fde8f007354fd48a7814db15b7cb", - "sha256:a068a21ceac8a4d63dbfd964670474107f541babbd2250d61922f029858365fa" + "sha256:a7577a1f55cee1d21953a5cf11a3c839ab87f5ef909a4cba6cf52ed72b4c6059", + "sha256:ab246293e6585a1c6361a505b68d5b501a0409310932b7de2c2ead667b564d89" ], - "version": "==2.9" + "version": "==1.4.13" }, "mccabe": { "hashes": [ @@ -787,20 +750,21 @@ ], "version": "==1.3.5" }, - "packaging": { + "pep8-naming": { "hashes": [ - "sha256:170748228214b70b672c581a3dd610ee51f733018650740e98c7df862a583f73", - "sha256:e665345f9eef0c621aa0bf2f8d78cf6d21904eef16a93f020240b704a57f1334" + "sha256:5d9f1056cb9427ce344e98d1a7f5665710e2f20f748438e308995852cfa24164", + "sha256:f3b4a5f9dd72b991bf7d8e2a341d2e1aa3a884a769b5aaac4f56825c1763bf3a" ], - "version": "==20.1" + "index": "pypi", + "version": "==0.10.0" }, "pre-commit": { "hashes": [ - "sha256:09ebe467f43ce24377f8c2f200fe3cd2570d328eb2ce0568c8e96ce19da45fa6", - "sha256:f8d555e31e2051892c7f7b3ad9f620bd2c09271d87e9eedb2ad831737d6211eb" + "sha256:487c675916e6f99d355ec5595ad77b325689d423ef4839db1ed2f02f639c9522", + "sha256:c0aa11bce04a7b46c5544723aedf4e81a4d5f64ad1205a30a9ea12d5e81969e1" ], "index": "pypi", - "version": "==2.1.1" + "version": "==2.2.0" }, "pycodestyle": { "hashes": [ @@ -823,45 +787,22 @@ ], "version": "==2.1.1" }, - "pyparsing": { - "hashes": [ - "sha256:4c830582a84fb022400b85429791bc551f1f4871c33f23e44f353119e92f969f", - "sha256:c342dccb5250c08d45fd6f8b4a559613ca603b57498511740e65cd11a2e7dcec" - ], - "version": "==2.4.6" - }, "pyyaml": { "hashes": [ - "sha256:059b2ee3194d718896c0ad077dd8c043e5e909d9180f387ce42012662a4946d6", - "sha256:1cf708e2ac57f3aabc87405f04b86354f66799c8e62c28c5fc5f88b5521b2dbf", - "sha256:24521fa2890642614558b492b473bee0ac1f8057a7263156b02e8b14c88ce6f5", - "sha256:4fee71aa5bc6ed9d5f116327c04273e25ae31a3020386916905767ec4fc5317e", - "sha256:70024e02197337533eef7b85b068212420f950319cc8c580261963aefc75f811", - "sha256:74782fbd4d4f87ff04159e986886931456a1894c61229be9eaf4de6f6e44b99e", - "sha256:940532b111b1952befd7db542c370887a8611660d2b9becff75d39355303d82d", - "sha256:cb1f2f5e426dc9f07a7681419fe39cee823bb74f723f36f70399123f439e9b20", - "sha256:dbbb2379c19ed6042e8f11f2a2c66d39cceb8aeace421bfc29d085d93eda3689", - "sha256:e3a057b7a64f1222b56e47bcff5e4b94c4f61faac04c7c4ecb1985e18caa3994", - "sha256:e9f45bd5b92c7974e59bcd2dcc8631a6b6cc380a904725fce7bc08872e691615" - ], - "index": "pypi", - "version": "==5.3" - }, - "requests": { - "hashes": [ - "sha256:43999036bfa82904b6af1d99e4882b560e5e2c68e5c4b0aa03b655f3d7d73fee", - "sha256:b3f43d496c6daba4493e7c431722aeb7dbc6288f52a6e04e7b6023b0247817e6" + "sha256:06a0d7ba600ce0b2d2fe2e78453a470b5a6e000a985dd4a4e54e436cc36b0e97", + "sha256:240097ff019d7c70a4922b6869d8a86407758333f02203e0fc6ff79c5dcede76", + "sha256:4f4b913ca1a7319b33cfb1369e91e50354d6f07a135f3b901aca02aa95940bd2", + "sha256:69f00dca373f240f842b2931fb2c7e14ddbacd1397d57157a9b005a6a9942648", + "sha256:73f099454b799e05e5ab51423c7bcf361c58d3206fa7b0d555426b1f4d9a3eaf", + "sha256:74809a57b329d6cc0fdccee6318f44b9b8649961fa73144a98735b0aaf029f1f", + "sha256:7739fc0fa8205b3ee8808aea45e968bc90082c10aef6ea95e855e10abf4a37b2", + "sha256:95f71d2af0ff4227885f7a6605c37fd53d3a106fcab511b8860ecca9fcf400ee", + "sha256:b8eac752c5e14d3eca0e6dd9199cd627518cb5ec06add0de9d32baeee6fe645d", + "sha256:cc8955cfbfc7a115fa81d85284ee61147059a753344bc51098f3ccd69b0d7e0c", + "sha256:d13155f591e6fcc1ec3b30685d50bf0711574e2c0dfffd7644babf8b5102ca1a" ], "index": "pypi", - "version": "==2.23.0" - }, - "safety": { - "hashes": [ - "sha256:0a3a8a178a9c96242b224f033ee8d1d130c0448b0e6622d12deaf37f6c3b4e59", - "sha256:5059f3ffab3648330548ea9c7403405bbfaf085b11235770825d14c58f24cb78" - ], - "index": "pypi", - "version": "==1.8.5" + "version": "==5.3.1" }, "six": { "hashes": [ @@ -892,19 +833,12 @@ "index": "pypi", "version": "==3.0.2" }, - "urllib3": { - "hashes": [ - "sha256:2f3db8b19923a873b3e5256dc9c2dedfa883e33d87c690d9c7913e1f40673cdc", - "sha256:87716c2d2a7121198ebcb7ce7cccf6ce5e9ba539041cfbaeecfb641dc0bf6acc" - ], - "version": "==1.25.8" - }, "virtualenv": { "hashes": [ - "sha256:30ea90b21dabd11da5f509710ad3be2ae47d40ccbc717dfdd2efe4367c10f598", - "sha256:4a36a96d785428278edd389d9c36d763c5755844beb7509279194647b1ef47f1" + "sha256:87831f1070534b636fea2241dd66f3afe37ac9041bcca6d0af3215cdcfbf7d82", + "sha256:f3128d882383c503003130389bf892856341c1da12c881ae24d6358c82561b55" ], - "version": "==20.0.7" + "version": "==20.0.13" } } } diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 35dea089a..16d1b7a2a 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -13,10 +13,12 @@ jobs: variables: PIP_CACHE_DIR: ".cache/pip" + PRE_COMMIT_HOME: $(Pipeline.Workspace)/pre-commit-cache steps: - task: UsePythonVersion@0 displayName: 'Set Python version' + name: PythonVersion inputs: versionSpec: '3.8.x' addToPath: true @@ -27,8 +29,26 @@ jobs: - script: pipenv install --dev --deploy --system displayName: 'Install project using pipenv' - - script: python -m flake8 - displayName: 'Run linter' + # Create an executable shell script which replaces the original pipenv binary. + # The shell script ignores the first argument and executes the rest of the args as a command. + # It makes the `pipenv run flake8` command in the pre-commit hook work by circumventing + # pipenv entirely, which is too dumb to know it should use the system interpreter rather than + # creating a new venv. + - script: | + printf '%s\n%s' '#!/bin/bash' '"${@:2}"' > $(PythonVersion.pythonLocation)/bin/pipenv \ + && chmod +x $(PythonVersion.pythonLocation)/bin/pipenv + displayName: 'Mock pipenv binary' + + - task: Cache@2 + displayName: 'Restore pre-commit environment' + inputs: + key: pre-commit | "$(PythonVersion.pythonLocation)" | .pre-commit-config.yaml + restoreKeys: | + pre-commit | "$(PythonVersion.pythonLocation)" + path: $(PRE_COMMIT_HOME) + + - script: pre-commit run --all-files + displayName: 'Run pre-commit hooks' - script: BOT_API_KEY=foo BOT_SENTRY_DSN=blah BOT_TOKEN=bar WOLFRAM_API_KEY=baz REDDIT_CLIENT_ID=spam REDDIT_SECRET=ham coverage run -m xmlrunner displayName: Run tests diff --git a/bot/__main__.py b/bot/__main__.py index 3df477a6d..8c3ae02e3 100644 --- a/bot/__main__.py +++ b/bot/__main__.py @@ -63,6 +63,7 @@ bot.load_extension("bot.cogs.tags") bot.load_extension("bot.cogs.token_remover") bot.load_extension("bot.cogs.utils") bot.load_extension("bot.cogs.watchchannels") +bot.load_extension("bot.cogs.webhook_remover") bot.load_extension("bot.cogs.wolfram") # Apply `message_edited_at` patch if discord.py did not yet release a bug fix. diff --git a/bot/api.py b/bot/api.py index e59916114..4b8520582 100644 --- a/bot/api.py +++ b/bot/api.py @@ -52,7 +52,7 @@ class APIClient: self._ready = asyncio.Event(loop=loop) self._creation_task = None - self._session_args = kwargs + self._default_session_kwargs = kwargs self.recreate() @@ -60,25 +60,41 @@ class APIClient: def _url_for(endpoint: str) -> str: return f"{URLs.site_schema}{URLs.site_api}/{quote_url(endpoint)}" - async def _create_session(self) -> None: - """Create the aiohttp session and set the ready event.""" - self.session = aiohttp.ClientSession(**self._session_args) + async def _create_session(self, **session_kwargs) -> None: + """ + Create the aiohttp session with `session_kwargs` and set the ready event. + + `session_kwargs` is merged with `_default_session_kwargs` and overwrites its values. + If an open session already exists, it will first be closed. + """ + await self.close() + self.session = aiohttp.ClientSession(**{**self._default_session_kwargs, **session_kwargs}) self._ready.set() async def close(self) -> None: """Close the aiohttp session and unset the ready event.""" - if not self._ready.is_set(): - return + if self.session: + await self.session.close() - await self.session.close() self._ready.clear() - def recreate(self) -> None: - """Schedule the aiohttp session to be created if it's been closed.""" - if self.session is None or self.session.closed: + def recreate(self, force: bool = False, **session_kwargs) -> None: + """ + Schedule the aiohttp session to be created with `session_kwargs` if it's been closed. + + If `force` is True, the session will be recreated even if an open one exists. If a task to + create the session is pending, it will be cancelled. + + `session_kwargs` is merged with the kwargs given when the `APIClient` was created and + overwrites those default kwargs. + """ + if force or self.session is None or self.session.closed: + if force and self._creation_task: + self._creation_task.cancel() + # Don't schedule a task if one is already in progress. - if self._creation_task is None or self._creation_task.done(): - self._creation_task = self.loop.create_task(self._create_session()) + if force or self._creation_task is None or self._creation_task.done(): + self._creation_task = self.loop.create_task(self._create_session(**session_kwargs)) async def maybe_raise_for_status(self, response: aiohttp.ClientResponse, should_raise: bool) -> None: """Raise ResponseCodeError for non-OK response if an exception should be raised.""" diff --git a/bot/bot.py b/bot/bot.py index 19b9035c4..950ac6751 100644 --- a/bot/bot.py +++ b/bot/bot.py @@ -1,6 +1,7 @@ import asyncio import logging import socket +import warnings from typing import Optional import aiohttp @@ -17,20 +18,20 @@ class Bot(commands.Bot): """A subclass of `discord.ext.commands.Bot` with an aiohttp session and an API client.""" def __init__(self, *args, **kwargs): - # Use asyncio for DNS resolution instead of threads so threads aren't spammed. - # Use AF_INET as its socket family to prevent HTTPS related problems both locally - # and in production. - self._connector = aiohttp.TCPConnector( - resolver=aiohttp.AsyncResolver(), - family=socket.AF_INET, - ) + if "connector" in kwargs: + warnings.warn( + "If login() is called (or the bot is started), the connector will be overwritten " + "with an internal one" + ) - super().__init__(*args, connector=self._connector, **kwargs) - - self._guild_available = asyncio.Event() + super().__init__(*args, **kwargs) self.http_session: Optional[aiohttp.ClientSession] = None - self.api_client = api.APIClient(loop=self.loop, connector=self._connector) + self.api_client = api.APIClient(loop=self.loop) + + self._connector = None + self._resolver = None + self._guild_available = asyncio.Event() def add_cog(self, cog: commands.Cog) -> None: """Adds a "cog" to the bot and logs the operation.""" @@ -38,22 +39,67 @@ class Bot(commands.Bot): log.info(f"Cog loaded: {cog.qualified_name}") def clear(self) -> None: - """Clears the internal state of the bot and resets the API client.""" + """ + Clears the internal state of the bot and recreates the connector and sessions. + + Will cause a DeprecationWarning if called outside a coroutine. + """ + # Because discord.py recreates the HTTPClient session, may as well follow suit and recreate + # our own stuff here too. + self._recreate() super().clear() - self.api_client.recreate() async def close(self) -> None: - """Close the aiohttp session after closing the Discord connection.""" + """Close the Discord connection and the aiohttp session, connector, and resolver.""" await super().close() - await self.http_session.close() await self.api_client.close() - async def start(self, *args, **kwargs) -> None: - """Open an aiohttp session before logging in and connecting to Discord.""" - self.http_session = aiohttp.ClientSession(connector=self._connector) + if self.http_session: + await self.http_session.close() + + if self._connector: + await self._connector.close() - await super().start(*args, **kwargs) + if self._resolver: + await self._resolver.close() + + async def login(self, *args, **kwargs) -> None: + """Re-create the connector and set up sessions before logging into Discord.""" + self._recreate() + await super().login(*args, **kwargs) + + def _recreate(self) -> None: + """Re-create the connector, aiohttp session, and the APIClient.""" + # Use asyncio for DNS resolution instead of threads so threads aren't spammed. + # Doesn't seem to have any state with regards to being closed, so no need to worry? + self._resolver = aiohttp.AsyncResolver() + + # Its __del__ does send a warning but it doesn't always show up for some reason. + if self._connector and not self._connector._closed: + log.warning( + "The previous connector was not closed; it will remain open and be overwritten" + ) + + # Use AF_INET as its socket family to prevent HTTPS related problems both locally + # and in production. + self._connector = aiohttp.TCPConnector( + resolver=self._resolver, + family=socket.AF_INET, + ) + + # Client.login() will call HTTPClient.static_login() which will create a session using + # this connector attribute. + self.http.connector = self._connector + + # Its __del__ does send a warning but it doesn't always show up for some reason. + if self.http_session and not self.http_session.closed: + log.warning( + "The previous session was not closed; it will remain open and be overwritten" + ) + + self.http_session = aiohttp.ClientSession(connector=self._connector) + self.api_client.recreate(force=True, connector=self._connector) async def on_guild_available(self, guild: discord.Guild) -> None: """ diff --git a/bot/cogs/alias.py b/bot/cogs/alias.py index 0b800575f..9001e18f0 100644 --- a/bot/cogs/alias.py +++ b/bot/cogs/alias.py @@ -26,9 +26,9 @@ class Alias (Cog): log.debug(f"{cmd_name} was invoked through an alias") cmd = self.bot.get_command(cmd_name) if not cmd: - return log.warning(f'Did not find command "{cmd_name}" to invoke.') + return log.info(f'Did not find command "{cmd_name}" to invoke.') elif not await cmd.can_run(ctx): - return log.warning( + return log.info( f'{str(ctx.author)} tried to run the command "{cmd_name}"' ) diff --git a/bot/cogs/antimalware.py b/bot/cogs/antimalware.py index 9e9e81364..79bf486a4 100644 --- a/bot/cogs/antimalware.py +++ b/bot/cogs/antimalware.py @@ -1,4 +1,5 @@ import logging +from os.path import splitext from discord import Embed, Message, NotFound from discord.ext.commands import Cog @@ -28,24 +29,30 @@ class AntiMalware(Cog): return embed = Embed() - for attachment in message.attachments: - filename = attachment.filename.lower() - if filename.endswith('.py'): - embed.description = ( - f"It looks like you tried to attach a Python file - please " - f"use a code-pasting service such as {URLs.site_schema}{URLs.site_paste}" - ) - break # Other detections irrelevant because we prioritize the .py message. - if not filename.endswith(tuple(AntiMalwareConfig.whitelist)): - whitelisted_types = ', '.join(AntiMalwareConfig.whitelist) - meta_channel = self.bot.get_channel(Channels.meta) - embed.description = ( - f"It looks like you tried to attach a file type that we " - f"do not allow. We currently allow the following file " - f"types: **{whitelisted_types}**. \n\n Feel free to ask " - f"in {meta_channel.mention} if you think this is a mistake." - ) + file_extensions = {splitext(attachment.filename.lower())[1] for attachment in message.attachments} + extensions_blocked = file_extensions - set(AntiMalwareConfig.whitelist) + blocked_extensions_str = ', '.join(extensions_blocked) + if ".py" in extensions_blocked: + # Short-circuit on *.py files to provide a pastebin link + embed.description = ( + "It looks like you tried to attach a Python file - " + f"please use a code-pasting service such as {URLs.site_schema}{URLs.site_paste}" + ) + elif extensions_blocked: + whitelisted_types = ', '.join(AntiMalwareConfig.whitelist) + meta_channel = self.bot.get_channel(Channels.meta) + embed.description = ( + f"It looks like you tried to attach file type(s) that we do not allow ({blocked_extensions_str}). " + f"We currently allow the following file types: **{whitelisted_types}**.\n\n" + f"Feel free to ask in {meta_channel.mention} if you think this is a mistake." + ) + if embed.description: + log.info( + f"User '{message.author}' ({message.author.id}) uploaded blacklisted file(s): {blocked_extensions_str}", + extra={"attachment_list": [attachment.filename for attachment in message.attachments]} + ) + await message.channel.send(f"Hey {message.author.mention}!", embed=embed) # Delete the offending message: diff --git a/bot/cogs/extensions.py b/bot/cogs/extensions.py index b312e1a1d..fb6cd9aa3 100644 --- a/bot/cogs/extensions.py +++ b/bot/cogs/extensions.py @@ -69,7 +69,7 @@ class Extensions(commands.Cog): @extensions_group.command(name="load", aliases=("l",)) async def load_command(self, ctx: Context, *extensions: Extension) -> None: - """ + r""" Load extensions given their fully qualified or unqualified names. If '\*' or '\*\*' is given as the name, all unloaded extensions will be loaded. @@ -86,7 +86,7 @@ class Extensions(commands.Cog): @extensions_group.command(name="unload", aliases=("ul",)) async def unload_command(self, ctx: Context, *extensions: Extension) -> None: - """ + r""" Unload currently loaded extensions given their fully qualified or unqualified names. If '\*' or '\*\*' is given as the name, all loaded extensions will be unloaded. @@ -109,7 +109,7 @@ class Extensions(commands.Cog): @extensions_group.command(name="reload", aliases=("r",)) async def reload_command(self, ctx: Context, *extensions: Extension) -> None: - """ + r""" Reload extensions given their fully qualified or unqualified names. If an extension fails to be reloaded, it will be rolled-back to the prior working state. diff --git a/bot/cogs/filtering.py b/bot/cogs/filtering.py index 74538542a..6651d38e4 100644 --- a/bot/cogs/filtering.py +++ b/bot/cogs/filtering.py @@ -6,6 +6,7 @@ import discord.errors from dateutil.relativedelta import relativedelta from discord import Colour, DMChannel, Member, Message, TextChannel from discord.ext.commands import Cog +from discord.utils import escape_markdown from bot.bot import Bot from bot.cogs.moderation import ModLog @@ -27,6 +28,7 @@ INVITE_RE = re.compile( flags=re.IGNORECASE ) +SPOILER_RE = re.compile(r"(\|\|.+?\|\|)", re.DOTALL) URL_RE = re.compile(r"(https?://[^\s]+)", flags=re.IGNORECASE) ZALGO_RE = re.compile(r"[\u0300-\u036F\u0489]") @@ -38,6 +40,14 @@ TOKEN_WATCHLIST_PATTERNS = [ ] +def expand_spoilers(text: str) -> str: + """Return a string containing all interpretations of a spoilered message.""" + split_text = SPOILER_RE.split(text) + return ''.join( + split_text[0::2] + split_text[1::2] + split_text + ) + + class Filtering(Cog): """Filtering out invites, blacklisting domains, and warning us of certain regular expressions.""" @@ -186,8 +196,8 @@ class Filtering(Cog): surroundings = match.string[max(match.start() - 10, 0): match.end() + 10] message_content = ( f"**Match:** '{match[0]}'\n" - f"**Location:** '...{surroundings}...'\n" - f"\n**Original Message:**\n{msg.content}" + f"**Location:** '...{escape_markdown(surroundings)}...'\n" + f"\n**Original Message:**\n{escape_markdown(msg.content)}" ) else: # Use content of discord Message message_content = msg.content @@ -244,6 +254,8 @@ class Filtering(Cog): Only matches words with boundaries before and after the expression. """ + if SPOILER_RE.search(text): + text = expand_spoilers(text) for regex_pattern in WORD_WATCHLIST_PATTERNS: match = regex_pattern.search(text) if match: diff --git a/bot/cogs/free.py b/bot/cogs/free.py index 02c02d067..33b55e79a 100644 --- a/bot/cogs/free.py +++ b/bot/cogs/free.py @@ -55,7 +55,7 @@ class Free(Cog): msg = messages[seek - 1] # Otherwise get last message else: - msg = await channel.history(limit=1).next() # noqa (False positive) + msg = await channel.history(limit=1).next() # noqa: B305 inactive = (datetime.utcnow() - msg.created_at).seconds if inactive > TIMEOUT: diff --git a/bot/cogs/information.py b/bot/cogs/information.py index 49beca15b..7921a4932 100644 --- a/bot/cogs/information.py +++ b/bot/cogs/information.py @@ -13,6 +13,7 @@ from discord.utils import escape_markdown from bot import constants from bot.bot import Bot from bot.decorators import InChannelCheckFailure, in_channel, with_role +from bot.pagination import LinePaginator from bot.utils.checks import cooldown_with_role_bypass, with_role_check from bot.utils.time import time_since @@ -32,20 +33,18 @@ class Information(Cog): # Sort the roles alphabetically and remove the @everyone role roles = sorted(ctx.guild.roles[1:], key=lambda role: role.name) - # Build a string - role_string = "" + # Build a list + role_list = [] for role in roles: - role_string += f"`{role.id}` - {role.mention}\n" + role_list.append(f"`{role.id}` - {role.mention}") # Build an embed embed = Embed( - title="Role information", - colour=Colour.blurple(), - description=role_string + title=f"Role information (Total {len(roles)} role{'s' * (len(role_list) > 1)})", + colour=Colour.blurple() ) - embed.set_footer(text=f"Total roles: {len(roles)}") - await ctx.send(embed=embed) + await LinePaginator.paginate(role_list, ctx, embed, empty=False) @with_role(*constants.MODERATION_ROLES) @command(name="role") diff --git a/bot/cogs/moderation/infractions.py b/bot/cogs/moderation/infractions.py index 9ea17b2b3..efa19f59e 100644 --- a/bot/cogs/moderation/infractions.py +++ b/bot/cogs/moderation/infractions.py @@ -67,7 +67,7 @@ class Infractions(InfractionScheduler, commands.Cog): @command() async def ban(self, ctx: Context, user: FetchedMember, *, reason: str = None) -> None: - """Permanently ban a user for the given reason.""" + """Permanently ban a user for the given reason and stop watching them with Big Brother.""" await self.apply_ban(ctx, user, reason) # endregion @@ -230,7 +230,11 @@ class Infractions(InfractionScheduler, commands.Cog): @respect_role_hierarchy() async def apply_ban(self, ctx: Context, user: UserSnowflake, reason: str, **kwargs) -> None: - """Apply a ban infraction with kwargs passed to `post_infraction`.""" + """ + Apply a ban infraction with kwargs passed to `post_infraction`. + + Will also remove the banned user from the Big Brother watch list if applicable. + """ if await utils.has_active_infraction(ctx, user, "ban"): return @@ -243,6 +247,20 @@ class Infractions(InfractionScheduler, commands.Cog): action = ctx.guild.ban(user, reason=reason, delete_message_days=0) await self.apply_infraction(ctx, infraction, user, action) + if infraction.get('expires_at') is not None: + log.trace(f"Ban isn't permanent; user {user} won't be unwatched by Big Brother.") + return + + bb_cog = self.bot.get_cog("Big Brother") + if not bb_cog: + log.error(f"Big Brother cog not loaded; perma-banned user {user} won't be unwatched.") + return + + log.trace(f"Big Brother cog loaded; attempting to unwatch perma-banned user {user}.") + + bb_reason = "User has been permanently banned from the server. Automatically removed." + await bb_cog.apply_unwatch(ctx, user, bb_reason, send_message=False) + # endregion # region: Base pardon functions diff --git a/bot/cogs/moderation/modlog.py b/bot/cogs/moderation/modlog.py index 59ae6b587..c63b4bab9 100644 --- a/bot/cogs/moderation/modlog.py +++ b/bot/cogs/moderation/modlog.py @@ -12,6 +12,7 @@ from deepdiff import DeepDiff from discord import Colour from discord.abc import GuildChannel from discord.ext.commands import Cog, Context +from discord.utils import escape_markdown from bot.bot import Bot from bot.constants import Channels, Colours, Emojis, Event, Guild as GuildConstant, Icons, URLs @@ -67,7 +68,7 @@ class ModLog(Cog, name="ModLog"): 'embeds': [embed.to_dict() for embed in message.embeds], 'attachments': attachment, } - for message, attachment in zip_longest(messages, attachments) + for message, attachment in zip_longest(messages, attachments, fillvalue=[]) ] } ) @@ -215,7 +216,10 @@ class ModLog(Cog, name="ModLog"): new = value["new_value"] old = value["old_value"] - changes.append(f"**{key.title()}:** `{old}` **→** `{new}`") + # Discord does not treat consecutive backticks ("``") as an empty inline code block, so the markdown + # formatting is broken when `new` and/or `old` are empty values. "None" is used for these cases so + # formatting is preserved. + changes.append(f"**{key.title()}:** `{old or 'None'}` **→** `{new or 'None'}`") done.append(key) @@ -386,7 +390,8 @@ class ModLog(Cog, name="ModLog"): if member.guild.id != GuildConstant.id: return - message = f"{member} (`{member.id}`)" + member_str = escape_markdown(str(member)) + message = f"{member_str} (`{member.id}`)" now = datetime.utcnow() difference = abs(relativedelta(now, member.created_at)) @@ -412,9 +417,10 @@ class ModLog(Cog, name="ModLog"): self._ignored[Event.member_remove].remove(member.id) return + member_str = escape_markdown(str(member)) await self.send_log_message( Icons.sign_out, Colours.soft_red, - "User left", f"{member} (`{member.id}`)", + "User left", f"{member_str} (`{member.id}`)", thumbnail=member.avatar_url_as(static_format="png"), channel_id=Channels.user_log ) @@ -429,9 +435,10 @@ class ModLog(Cog, name="ModLog"): self._ignored[Event.member_unban].remove(member.id) return + member_str = escape_markdown(str(member)) await self.send_log_message( Icons.user_unban, Colour.blurple(), - "User unbanned", f"{member} (`{member.id}`)", + "User unbanned", f"{member_str} (`{member.id}`)", thumbnail=member.avatar_url_as(static_format="png"), channel_id=Channels.mod_log ) @@ -523,7 +530,8 @@ class ModLog(Cog, name="ModLog"): for item in sorted(changes): message += f"{Emojis.bullet} {item}\n" - message = f"**{after}** (`{after.id}`)\n{message}" + member_str = escape_markdown(str(after)) + message = f"**{member_str}** (`{after.id}`)\n{message}" await self.send_log_message( Icons.user_update, Colour.blurple(), @@ -550,16 +558,17 @@ class ModLog(Cog, name="ModLog"): if author.bot: return + author_str = escape_markdown(str(author)) if channel.category: response = ( - f"**Author:** {author} (`{author.id}`)\n" + f"**Author:** {author_str} (`{author.id}`)\n" f"**Channel:** {channel.category}/#{channel.name} (`{channel.id}`)\n" f"**Message ID:** `{message.id}`\n" "\n" ) else: response = ( - f"**Author:** {author} (`{author.id}`)\n" + f"**Author:** {author_str} (`{author.id}`)\n" f"**Channel:** #{channel.name} (`{channel.id}`)\n" f"**Message ID:** `{message.id}`\n" "\n" @@ -646,6 +655,8 @@ class ModLog(Cog, name="ModLog"): return author = msg_before.author + author_str = escape_markdown(str(author)) + channel = msg_before.channel channel_name = f"{channel.category}/#{channel.name}" if channel.category else f"#{channel.name}" @@ -677,7 +688,7 @@ class ModLog(Cog, name="ModLog"): content_after.append(sub) response = ( - f"**Author:** {author} (`{author.id}`)\n" + f"**Author:** {author_str} (`{author.id}`)\n" f"**Channel:** {channel_name} (`{channel.id}`)\n" f"**Message ID:** `{msg_before.id}`\n" "\n" @@ -820,8 +831,9 @@ class ModLog(Cog, name="ModLog"): if not changes: return + member_str = escape_markdown(str(member)) message = "\n".join(f"{Emojis.bullet} {item}" for item in sorted(changes)) - message = f"**{member}** (`{member.id}`)\n{message}" + message = f"**{member_str}** (`{member.id}`)\n{message}" await self.send_log_message( icon_url=icon, diff --git a/bot/cogs/snekbox.py b/bot/cogs/snekbox.py index cff7c5786..454836921 100644 --- a/bot/cogs/snekbox.py +++ b/bot/cogs/snekbox.py @@ -232,7 +232,7 @@ class Snekbox(Cog): timeout=10 ) - code = new_message.content.split(' ', maxsplit=1)[1] + code = await self.get_code(new_message) await ctx.message.clear_reactions() with contextlib.suppress(HTTPException): await response.delete() @@ -243,6 +243,26 @@ class Snekbox(Cog): return code + async def get_code(self, message: Message) -> Optional[str]: + """ + Return the code from `message` to be evaluated. + + If the message is an invocation of the eval command, return the first argument or None if it + doesn't exist. Otherwise, return the full content of the message. + """ + log.trace(f"Getting context for message {message.id}.") + new_ctx = await self.bot.get_context(message) + + if new_ctx.command is self.eval_command: + log.trace(f"Message {message.id} invokes eval command.") + split = message.content.split(maxsplit=1) + code = split[1] if len(split) > 1 else None + else: + log.trace(f"Message {message.id} does not invoke eval command.") + code = message.content + + return code + @command(name="eval", aliases=("e",)) @guild_only() @in_channel(Channels.bot_commands, hidden_channels=(Channels.esoteric,), bypass_roles=EVAL_ROLES) diff --git a/bot/cogs/tags.py b/bot/cogs/tags.py index 5da9a4148..539105017 100644 --- a/bot/cogs/tags.py +++ b/bot/cogs/tags.py @@ -1,25 +1,26 @@ import logging import re import time -from typing import Dict, List, Optional +from pathlib import Path +from typing import Callable, Dict, Iterable, List, Optional from discord import Colour, Embed from discord.ext.commands import Cog, Context, group +from bot import constants from bot.bot import Bot -from bot.constants import Channels, Cooldowns, MODERATION_ROLES, Roles -from bot.converters import TagContentConverter, TagNameConverter -from bot.decorators import with_role +from bot.converters import TagNameConverter from bot.pagination import LinePaginator log = logging.getLogger(__name__) TEST_CHANNELS = ( - Channels.bot_commands, - Channels.helpers + constants.Channels.bot_commands, + constants.Channels.helpers ) REGEX_NON_ALPHABET = re.compile(r"[^a-z]", re.MULTILINE & re.IGNORECASE) +FOOTER_TEXT = f"To show a tag, type {constants.Bot.prefix}tags <tagname>." class Tags(Cog): @@ -28,21 +29,27 @@ class Tags(Cog): def __init__(self, bot: Bot): self.bot = bot self.tag_cooldowns = {} + self._cache = self.get_tags() - self._cache = {} - self._last_fetch: float = 0.0 - - async def _get_tags(self, is_forced: bool = False) -> None: + @staticmethod + def get_tags() -> dict: """Get all tags.""" - # refresh only when there's a more than 5m gap from last call. - time_now: float = time.time() - if is_forced or not self._last_fetch or time_now - self._last_fetch > 5 * 60: - tags = await self.bot.api_client.get('bot/tags') - self._cache = {tag['title'].lower(): tag for tag in tags} - self._last_fetch = time_now + # Save all tags in memory. + cache = {} + tag_files = Path("bot", "resources", "tags").iterdir() + for file in tag_files: + tag_title = file.stem + tag = { + "title": tag_title, + "embed": { + "description": file.read_text() + } + } + cache[tag_title] = tag + return cache @staticmethod - def _fuzzy_search(search: str, target: str) -> int: + def _fuzzy_search(search: str, target: str) -> float: """A simple scoring algorithm based on how many letters are found / total, with order in mind.""" current, index = 0, 0 _search = REGEX_NON_ALPHABET.sub('', search.lower()) @@ -78,19 +85,85 @@ class Tags(Cog): return [] - async def _get_tag(self, tag_name: str) -> list: + def _get_tag(self, tag_name: str) -> list: """Get a specific tag.""" - await self._get_tags() found = [self._cache.get(tag_name.lower(), None)] if not found[0]: return self._get_suggestions(tag_name) return found + def _get_tags_via_content(self, check: Callable[[Iterable], bool], keywords: str) -> list: + """ + Search for tags via contents. + + `predicate` will be the built-in any, all, or a custom callable. Must return a bool. + """ + keywords_processed: List[str] = [] + for keyword in keywords.split(','): + keyword_sanitized = keyword.strip().casefold() + if not keyword_sanitized: + # this happens when there are leading / trailing / consecutive comma. + continue + keywords_processed.append(keyword_sanitized) + + if not keywords_processed: + # after sanitizing, we can end up with an empty list, for example when keywords is ',' + # in that case, we simply want to search for such keywords directly instead. + keywords_processed = [keywords] + + matching_tags = [] + for tag in self._cache.values(): + if check(query in tag['embed']['description'].casefold() for query in keywords_processed): + matching_tags.append(tag) + + return matching_tags + + async def _send_matching_tags(self, ctx: Context, keywords: str, matching_tags: list) -> None: + """Send the result of matching tags to user.""" + if not matching_tags: + pass + elif len(matching_tags) == 1: + await ctx.send(embed=Embed().from_dict(matching_tags[0]['embed'])) + else: + is_plural = keywords.strip().count(' ') > 0 or keywords.strip().count(',') > 0 + embed = Embed( + title=f"Here are the tags containing the given keyword{'s' * is_plural}:", + description='\n'.join(tag['title'] for tag in matching_tags[:10]) + ) + await LinePaginator.paginate( + sorted(f"**»** {tag['title']}" for tag in matching_tags), + ctx, + embed, + footer_text=FOOTER_TEXT, + empty=False, + max_lines=15 + ) + @group(name='tags', aliases=('tag', 't'), invoke_without_command=True) async def tags_group(self, ctx: Context, *, tag_name: TagNameConverter = None) -> None: """Show all known tags, a single tag, or run a subcommand.""" await ctx.invoke(self.get_command, tag_name=tag_name) + @tags_group.group(name='search', invoke_without_command=True) + async def search_tag_content(self, ctx: Context, *, keywords: str) -> None: + """ + Search inside tags' contents for tags. Allow searching for multiple keywords separated by comma. + + Only search for tags that has ALL the keywords. + """ + matching_tags = self._get_tags_via_content(all, keywords) + await self._send_matching_tags(ctx, keywords, matching_tags) + + @search_tag_content.command(name='any') + async def search_tag_content_any_keyword(self, ctx: Context, *, keywords: Optional[str] = 'any') -> None: + """ + Search inside tags' contents for tags. Allow searching for multiple keywords separated by comma. + + Search for tags that has ANY of the keywords. + """ + matching_tags = self._get_tags_via_content(any, keywords or 'any') + await self._send_matching_tags(ctx, keywords, matching_tags) + @tags_group.command(name='get', aliases=('show', 'g')) async def get_command(self, ctx: Context, *, tag_name: TagNameConverter = None) -> None: """Get a specified tag, or a list of all tags if no tag is specified.""" @@ -105,7 +178,7 @@ class Tags(Cog): cooldown_conditions = ( tag_name and tag_name in self.tag_cooldowns - and (now - self.tag_cooldowns[tag_name]["time"]) < Cooldowns.tags + and (now - self.tag_cooldowns[tag_name]["time"]) < constants.Cooldowns.tags and self.tag_cooldowns[tag_name]["channel"] == ctx.channel.id ) @@ -114,17 +187,16 @@ class Tags(Cog): return False if _command_on_cooldown(tag_name): - time_left = Cooldowns.tags - (time.time() - self.tag_cooldowns[tag_name]["time"]) + time_elapsed = time.time() - self.tag_cooldowns[tag_name]["time"] + time_left = constants.Cooldowns.tags - time_elapsed log.info( f"{ctx.author} tried to get the '{tag_name}' tag, but the tag is on cooldown. " f"Cooldown ends in {time_left:.1f} seconds." ) return - await self._get_tags() - if tag_name is not None: - founds = await self._get_tag(tag_name) + founds = self._get_tag(tag_name) if len(founds) == 1: tag = founds[0] @@ -153,86 +225,11 @@ class Tags(Cog): sorted(f"**»** {tag['title']}" for tag in tags), ctx, embed, - footer_text="To show a tag, type !tags <tagname>.", + footer_text=FOOTER_TEXT, empty=False, max_lines=15 ) - @tags_group.command(name='set', aliases=('add', 's')) - @with_role(*MODERATION_ROLES) - async def set_command( - self, - ctx: Context, - tag_name: TagNameConverter, - *, - tag_content: TagContentConverter, - ) -> None: - """Create a new tag.""" - body = { - 'title': tag_name.lower().strip(), - 'embed': { - 'title': tag_name, - 'description': tag_content - } - } - - await self.bot.api_client.post('bot/tags', json=body) - self._cache[tag_name.lower()] = await self.bot.api_client.get(f'bot/tags/{tag_name}') - - log.debug(f"{ctx.author} successfully added the following tag to our database: \n" - f"tag_name: {tag_name}\n" - f"tag_content: '{tag_content}'\n") - - await ctx.send(embed=Embed( - title="Tag successfully added", - description=f"**{tag_name}** added to tag database.", - colour=Colour.blurple() - )) - - @tags_group.command(name='edit', aliases=('e', )) - @with_role(*MODERATION_ROLES) - async def edit_command( - self, - ctx: Context, - tag_name: TagNameConverter, - *, - tag_content: TagContentConverter, - ) -> None: - """Edit an existing tag.""" - body = { - 'embed': { - 'title': tag_name, - 'description': tag_content - } - } - - await self.bot.api_client.patch(f'bot/tags/{tag_name}', json=body) - self._cache[tag_name.lower()] = await self.bot.api_client.get(f'bot/tags/{tag_name}') - - log.debug(f"{ctx.author} successfully edited the following tag in our database: \n" - f"tag_name: {tag_name}\n" - f"tag_content: '{tag_content}'\n") - - await ctx.send(embed=Embed( - title="Tag successfully edited", - description=f"**{tag_name}** edited in the database.", - colour=Colour.blurple() - )) - - @tags_group.command(name='delete', aliases=('remove', 'rm', 'd')) - @with_role(Roles.admins, Roles.owners) - async def delete_command(self, ctx: Context, *, tag_name: TagNameConverter) -> None: - """Remove a tag from the database.""" - await self.bot.api_client.delete(f'bot/tags/{tag_name}') - self._cache.pop(tag_name.lower(), None) - - log.debug(f"{ctx.author} successfully deleted the tag called '{tag_name}'") - await ctx.send(embed=Embed( - title=tag_name, - description=f"Tag successfully removed: {tag_name}.", - colour=Colour.blurple() - )) - def setup(bot: Bot) -> None: """Load the Tags cog.""" diff --git a/bot/cogs/token_remover.py b/bot/cogs/token_remover.py index 82c01ae96..547ba8da0 100644 --- a/bot/cogs/token_remover.py +++ b/bot/cogs/token_remover.py @@ -96,12 +96,19 @@ class TokenRemover(Cog): if msg.author.bot: return False - maybe_match = TOKEN_RE.search(msg.content) - if maybe_match is None: + # Use findall rather than search to guard against method calls prematurely returning the + # token check (e.g. `message.channel.send` also matches our token pattern) + maybe_matches = TOKEN_RE.findall(msg.content) + if not maybe_matches: return False + return any(cls.is_maybe_token(substr) for substr in maybe_matches) + + @classmethod + def is_maybe_token(cls, test_str: str) -> bool: + """Check the provided string to see if it is a seemingly valid token.""" try: - user_id, creation_timestamp, hmac = maybe_match.group(0).split('.') + user_id, creation_timestamp, hmac = test_str.split('.') except ValueError: return False diff --git a/bot/cogs/utils.py b/bot/cogs/utils.py index 94b9d6b5a..f0b1172e3 100644 --- a/bot/cogs/utils.py +++ b/bot/cogs/utils.py @@ -1,14 +1,15 @@ +import difflib import logging import re import unicodedata from asyncio import TimeoutError, sleep from email.parser import HeaderParser from io import StringIO -from typing import Tuple +from typing import Tuple, Union from dateutil import relativedelta from discord import Colour, Embed, Message, Role -from discord.ext.commands import Cog, Context, command +from discord.ext.commands import BadArgument, Cog, Context, command from bot.bot import Bot from bot.constants import Channels, MODERATION_ROLES, Mention, STAFF_ROLES @@ -17,6 +18,30 @@ from bot.utils.time import humanize_delta log = logging.getLogger(__name__) +ZEN_OF_PYTHON = """\ +Beautiful is better than ugly. +Explicit is better than implicit. +Simple is better than complex. +Complex is better than complicated. +Flat is better than nested. +Sparse is better than dense. +Readability counts. +Special cases aren't special enough to break the rules. +Although practicality beats purity. +Errors should never pass silently. +Unless explicitly silenced. +In the face of ambiguity, refuse the temptation to guess. +There should be one-- and preferably only one --obvious way to do it. +Although that way may not be obvious at first unless you're Dutch. +Now is better than never. +Although never is often better than *right* now. +If the implementation is hard to explain, it's a bad idea. +If the implementation is easy to explain, it may be a good idea. +Namespaces are one honking great idea -- let's do more of those! +""" + +ICON_URL = "https://www.python.org/static/opengraph-icon-200x200.png" + class Utils(Cog): """A selection of utilities which don't have a clear category.""" @@ -36,6 +61,10 @@ class Utils(Cog): await ctx.invoke(self.bot.get_command("help"), "pep") return + # Handle PEP 0 directly because it's not in .rst or .txt so it can't be accessed like other PEPs. + if pep_number == 0: + return await self.send_pep_zero(ctx) + possible_extensions = ['.txt', '.rst'] found_pep = False for extension in possible_extensions: @@ -59,7 +88,7 @@ class Utils(Cog): description=f"[Link]({self.base_pep_url}{pep_number:04})", ) - pep_embed.set_thumbnail(url="https://www.python.org/static/opengraph-icon-200x200.png") + pep_embed.set_thumbnail(url=ICON_URL) # Add the interesting information fields_to_check = ("Status", "Python-Version", "Created", "Type") @@ -173,6 +202,101 @@ class Utils(Cog): f"as I detected unauthorised use by {msg.author} (ID: {msg.author.id})." ) + @command() + async def zen(self, ctx: Context, *, search_value: Union[int, str, None] = None) -> None: + """ + Show the Zen of Python. + + Without any arguments, the full Zen will be produced. + If an integer is provided, the line with that index will be produced. + If a string is provided, the line which matches best will be produced. + """ + embed = Embed( + colour=Colour.blurple(), + title="The Zen of Python", + description=ZEN_OF_PYTHON + ) + + if search_value is None: + embed.title += ", by Tim Peters" + await ctx.send(embed=embed) + return + + zen_lines = ZEN_OF_PYTHON.splitlines() + + # handle if it's an index int + if isinstance(search_value, int): + upper_bound = len(zen_lines) - 1 + lower_bound = -1 * upper_bound + if not (lower_bound <= search_value <= upper_bound): + raise BadArgument(f"Please provide an index between {lower_bound} and {upper_bound}.") + + embed.title += f" (line {search_value % len(zen_lines)}):" + embed.description = zen_lines[search_value] + await ctx.send(embed=embed) + return + + # handle if it's a search string + matcher = difflib.SequenceMatcher(None, search_value.lower()) + + best_match = "" + match_index = 0 + best_ratio = 0 + + for index, line in enumerate(zen_lines): + matcher.set_seq2(line.lower()) + + # the match ratio needs to be adjusted because, naturally, + # longer lines will have worse ratios than shorter lines when + # fuzzy searching for keywords. this seems to work okay. + adjusted_ratio = (len(line) - 5) ** 0.5 * matcher.ratio() + + if adjusted_ratio > best_ratio: + best_ratio = adjusted_ratio + best_match = line + match_index = index + + if not best_match: + raise BadArgument("I didn't get a match! Please try again with a different search term.") + + embed.title += f" (line {match_index}):" + embed.description = best_match + await ctx.send(embed=embed) + + @command(aliases=("poll",)) + @with_role(*MODERATION_ROLES) + async def vote(self, ctx: Context, title: str, *options: str) -> None: + """ + Build a quick voting poll with matching reactions with the provided options. + + A maximum of 20 options can be provided, as Discord supports a max of 20 + reactions on a single message. + """ + if len(options) < 2: + raise BadArgument("Please provide at least 2 options.") + if len(options) > 20: + raise BadArgument("I can only handle 20 options!") + + codepoint_start = 127462 # represents "regional_indicator_a" unicode value + options = {chr(i): f"{chr(i)} - {v}" for i, v in enumerate(options, start=codepoint_start)} + embed = Embed(title=title, description="\n".join(options.values())) + message = await ctx.send(embed=embed) + for reaction in options: + await message.add_reaction(reaction) + + async def send_pep_zero(self, ctx: Context) -> None: + """Send information about PEP 0.""" + pep_embed = Embed( + title=f"**PEP 0 - Index of Python Enhancement Proposals (PEPs)**", + description=f"[Link](https://www.python.org/dev/peps/)" + ) + pep_embed.set_thumbnail(url=ICON_URL) + pep_embed.add_field(name="Status", value="Active") + pep_embed.add_field(name="Created", value="13-Jul-2000") + pep_embed.add_field(name="Type", value="Informational") + + await ctx.send(embed=pep_embed) + def setup(bot: Bot) -> None: """Load the Utils cog.""" diff --git a/bot/cogs/verification.py b/bot/cogs/verification.py index 57b50c34f..b0a493e68 100644 --- a/bot/cogs/verification.py +++ b/bot/cogs/verification.py @@ -6,13 +6,9 @@ from discord import Colour, Forbidden, Message, NotFound, Object from discord.ext import tasks from discord.ext.commands import Cog, Context, command +from bot import constants from bot.bot import Bot from bot.cogs.moderation import ModLog -from bot.constants import ( - Bot as BotConfig, - Channels, Colours, Event, - Filter, Icons, MODERATION_ROLES, Roles -) from bot.decorators import InChannelCheckFailure, in_channel, without_role from bot.utils.checks import without_role_check @@ -29,18 +25,23 @@ your information removed here as well. Feel free to review them at any point! -Additionally, if you'd like to receive notifications for the announcements we post in <#{Channels.announcements}> \ -from time to time, you can send `!subscribe` to <#{Channels.bot_commands}> at any time to assign yourself the \ -**Announcements** role. We'll mention this role every time we make an announcement. +Additionally, if you'd like to receive notifications for the announcements \ +we post in <#{constants.Channels.announcements}> +from time to time, you can send `!subscribe` to <#{constants.Channels.bot_commands}> at any time \ +to assign yourself the **Announcements** role. We'll mention this role every time we make an announcement. If you'd like to unsubscribe from the announcement notifications, simply send `!unsubscribe` to \ -<#{Channels.bot_commands}>. +<#{constants.Channels.bot_commands}>. """ -PERIODIC_PING = ( - f"@everyone To verify that you have read our rules, please type `{BotConfig.prefix}accept`." - f" If you encounter any problems during the verification process, ping the <@&{Roles.admins}> role in this channel." -) +if constants.DEBUG_MODE: + PERIODIC_PING = "Periodic checkpoint message successfully sent." +else: + PERIODIC_PING = ( + f"@everyone To verify that you have read our rules, please type `{constants.Bot.prefix}accept`." + " If you encounter any problems during the verification process, " + f"ping the <@&{constants.Roles.admins}> role in this channel." + ) BOT_MESSAGE_DELETE_DELAY = 10 @@ -59,7 +60,7 @@ class Verification(Cog): @Cog.listener() async def on_message(self, message: Message) -> None: """Check new message event for messages to the checkpoint channel & process.""" - if message.channel.id != Channels.verification: + if message.channel.id != constants.Channels.verification: return # Only listen for #checkpoint messages if message.author.bot: @@ -85,20 +86,20 @@ class Verification(Cog): # Send pretty mod log embed to mod-alerts await self.mod_log.send_log_message( - icon_url=Icons.filtering, - colour=Colour(Colours.soft_red), + icon_url=constants.Icons.filtering, + colour=Colour(constants.Colours.soft_red), title=f"User/Role mentioned in {message.channel.name}", text=embed_text, thumbnail=message.author.avatar_url_as(static_format="png"), - channel_id=Channels.mod_alerts, - ping_everyone=Filter.ping_everyone, + channel_id=constants.Channels.mod_alerts, + ping_everyone=constants.Filter.ping_everyone, ) ctx: Context = await self.bot.get_context(message) if ctx.command is not None and ctx.command.name == "accept": return - if any(r.id == Roles.verified for r in ctx.author.roles): + if any(r.id == constants.Roles.verified for r in ctx.author.roles): log.info( f"{ctx.author} posted '{ctx.message.content}' " "in the verification channel, but is already verified." @@ -120,12 +121,12 @@ class Verification(Cog): await ctx.message.delete() @command(name='accept', aliases=('verify', 'verified', 'accepted'), hidden=True) - @without_role(Roles.verified) - @in_channel(Channels.verification) + @without_role(constants.Roles.verified) + @in_channel(constants.Channels.verification) async def accept_command(self, ctx: Context, *_) -> None: # We don't actually care about the args """Accept our rules and gain access to the rest of the server.""" log.debug(f"{ctx.author} called !accept. Assigning the 'Developer' role.") - await ctx.author.add_roles(Object(Roles.verified), reason="Accepted the rules") + await ctx.author.add_roles(Object(constants.Roles.verified), reason="Accepted the rules") try: await ctx.author.send(WELCOME_MESSAGE) except Forbidden: @@ -133,17 +134,17 @@ class Verification(Cog): finally: log.trace(f"Deleting accept message by {ctx.author}.") with suppress(NotFound): - self.mod_log.ignore(Event.message_delete, ctx.message.id) + self.mod_log.ignore(constants.Event.message_delete, ctx.message.id) await ctx.message.delete() @command(name='subscribe') - @in_channel(Channels.bot_commands) + @in_channel(constants.Channels.bot_commands) async def subscribe_command(self, ctx: Context, *_) -> None: # We don't actually care about the args """Subscribe to announcement notifications by assigning yourself the role.""" has_role = False for role in ctx.author.roles: - if role.id == Roles.announcements: + if role.id == constants.Roles.announcements: has_role = True break @@ -152,22 +153,22 @@ class Verification(Cog): return log.debug(f"{ctx.author} called !subscribe. Assigning the 'Announcements' role.") - await ctx.author.add_roles(Object(Roles.announcements), reason="Subscribed to announcements") + await ctx.author.add_roles(Object(constants.Roles.announcements), reason="Subscribed to announcements") log.trace(f"Deleting the message posted by {ctx.author}.") await ctx.send( - f"{ctx.author.mention} Subscribed to <#{Channels.announcements}> notifications.", + f"{ctx.author.mention} Subscribed to <#{constants.Channels.announcements}> notifications.", ) @command(name='unsubscribe') - @in_channel(Channels.bot_commands) + @in_channel(constants.Channels.bot_commands) async def unsubscribe_command(self, ctx: Context, *_) -> None: # We don't actually care about the args """Unsubscribe from announcement notifications by removing the role from yourself.""" has_role = False for role in ctx.author.roles: - if role.id == Roles.announcements: + if role.id == constants.Roles.announcements: has_role = True break @@ -176,12 +177,12 @@ class Verification(Cog): return log.debug(f"{ctx.author} called !unsubscribe. Removing the 'Announcements' role.") - await ctx.author.remove_roles(Object(Roles.announcements), reason="Unsubscribed from announcements") + await ctx.author.remove_roles(Object(constants.Roles.announcements), reason="Unsubscribed from announcements") log.trace(f"Deleting the message posted by {ctx.author}.") await ctx.send( - f"{ctx.author.mention} Unsubscribed from <#{Channels.announcements}> notifications." + f"{ctx.author.mention} Unsubscribed from <#{constants.Channels.announcements}> notifications." ) # This cannot be static (must have a __func__ attribute). @@ -193,7 +194,7 @@ class Verification(Cog): @staticmethod def bot_check(ctx: Context) -> bool: """Block any command within the verification channel that is not !accept.""" - if ctx.channel.id == Channels.verification and without_role_check(ctx, *MODERATION_ROLES): + if ctx.channel.id == constants.Channels.verification and without_role_check(ctx, *constants.MODERATION_ROLES): return ctx.command.name == "accept" else: return True @@ -201,7 +202,7 @@ class Verification(Cog): @tasks.loop(hours=12) async def periodic_ping(self) -> None: """Every week, mention @everyone to remind them to verify.""" - messages = self.bot.get_channel(Channels.verification).history(limit=10) + messages = self.bot.get_channel(constants.Channels.verification).history(limit=10) need_to_post = True # True if a new message needs to be sent. async for message in messages: @@ -215,7 +216,7 @@ class Verification(Cog): break if need_to_post: - await self.bot.get_channel(Channels.verification).send(PERIODIC_PING) + await self.bot.get_channel(constants.Channels.verification).send(PERIODIC_PING) @periodic_ping.before_loop async def before_ping(self) -> None: diff --git a/bot/cogs/watchchannels/bigbrother.py b/bot/cogs/watchchannels/bigbrother.py index c601e0d4d..903c87f85 100644 --- a/bot/cogs/watchchannels/bigbrother.py +++ b/bot/cogs/watchchannels/bigbrother.py @@ -52,6 +52,21 @@ class BigBrother(WatchChannel, Cog, name="Big Brother"): A `reason` for adding the user to Big Brother is required and will be displayed in the header when relaying messages of this user to the watchchannel. """ + await self.apply_watch(ctx, user, reason) + + @bigbrother_group.command(name='unwatch', aliases=('uw',)) + @with_role(*MODERATION_ROLES) + async def unwatch_command(self, ctx: Context, user: FetchedMember, *, reason: str) -> None: + """Stop relaying messages by the given `user`.""" + await self.apply_unwatch(ctx, user, reason) + + async def apply_watch(self, ctx: Context, user: FetchedMember, reason: str) -> None: + """ + Add `user` to watched users and apply a watch infraction with `reason`. + + A message indicating the result of the operation is sent to `ctx`. + The message will include `user`'s previous watch infraction history, if it exists. + """ if user.bot: await ctx.send(f":x: I'm sorry {ctx.author}, I'm afraid I can't do that. I only watch humans.") return @@ -90,10 +105,13 @@ class BigBrother(WatchChannel, Cog, name="Big Brother"): await ctx.send(msg) - @bigbrother_group.command(name='unwatch', aliases=('uw',)) - @with_role(*MODERATION_ROLES) - async def unwatch_command(self, ctx: Context, user: FetchedMember, *, reason: str) -> None: - """Stop relaying messages by the given `user`.""" + async def apply_unwatch(self, ctx: Context, user: FetchedMember, reason: str, send_message: bool = True) -> None: + """ + Remove `user` from watched users and mark their infraction as inactive with `reason`. + + If `send_message` is True, a message indicating the result of the operation is sent to + `ctx`. + """ active_watches = await self.bot.api_client.get( self.api_endpoint, params=ChainMap( @@ -102,6 +120,7 @@ class BigBrother(WatchChannel, Cog, name="Big Brother"): ) ) if active_watches: + log.trace("Active watches for user found. Attempting to remove.") [infraction] = active_watches await self.bot.api_client.patch( @@ -111,8 +130,20 @@ class BigBrother(WatchChannel, Cog, name="Big Brother"): await post_infraction(ctx, user, 'watch', f"Unwatched: {reason}", hidden=True, active=False) - await ctx.send(f":white_check_mark: Messages sent by {user} will no longer be relayed.") - self._remove_user(user.id) + + if not send_message: # Prevents a message being sent to the channel if part of a permanent ban + log.debug(f"Perma-banned user {user} was unwatched.") + return + log.trace("User is not banned. Sending message to channel") + message = f":white_check_mark: Messages sent by {user} will no longer be relayed." + else: - await ctx.send(":x: The specified user is currently not being watched.") + log.trace("No active watches found for user.") + if not send_message: # Prevents a message being sent to the channel if part of a permanent ban + log.debug(f"{user} was not on the watch list; no removal necessary.") + return + log.trace("User is not perma banned. Send the error message.") + message = ":x: The specified user is currently not being watched." + + await ctx.send(message) diff --git a/bot/cogs/watchchannels/watchchannel.py b/bot/cogs/watchchannels/watchchannel.py index 3667a80e8..479820444 100644 --- a/bot/cogs/watchchannels/watchchannel.py +++ b/bot/cogs/watchchannels/watchchannel.py @@ -9,7 +9,7 @@ from typing import Optional import dateutil.parser import discord -from discord import Color, Embed, HTTPException, Message, errors +from discord import Color, DMChannel, Embed, HTTPException, Message, errors from discord.ext.commands import Cog, Context from bot.api import ResponseCodeError @@ -273,7 +273,14 @@ class WatchChannel(metaclass=CogABCMeta): reason = self.watched_users[user_id]['reason'] - embed = Embed(description=f"{msg.author.mention} in [#{msg.channel.name}]({msg.jump_url})") + if isinstance(msg.channel, DMChannel): + # If a watched user DMs the bot there won't be a channel name or jump URL + # This could technically include a GroupChannel but bot's can't be in those + message_jump = "via DM" + else: + message_jump = f"in [#{msg.channel.name}]({msg.jump_url})" + + embed = Embed(description=f"{msg.author.mention} {message_jump}") embed.set_footer(text=f"Added {time_delta} by {actor} | Reason: {reason}") await self.webhook_send(embed=embed, username=msg.author.display_name, avatar_url=msg.author.avatar_url) diff --git a/bot/cogs/webhook_remover.py b/bot/cogs/webhook_remover.py new file mode 100644 index 000000000..49692113d --- /dev/null +++ b/bot/cogs/webhook_remover.py @@ -0,0 +1,72 @@ +import logging +import re + +from discord import Colour, Message +from discord.ext.commands import Cog + +from bot.bot import Bot +from bot.cogs.moderation.modlog import ModLog +from bot.constants import Channels, Colours, Event, Icons + +WEBHOOK_URL_RE = re.compile(r"((?:https?://)?discordapp\.com/api/webhooks/\d+/)\S+/?", re.I) + +ALERT_MESSAGE_TEMPLATE = ( + "{user}, looks like you posted a Discord webhook URL. Therefore, your " + "message has been removed. Your webhook may have been **compromised** so " + "please re-create the webhook **immediately**. If you believe this was " + "mistake, please let us know." +) + +log = logging.getLogger(__name__) + + +class WebhookRemover(Cog): + """Scan messages to detect Discord webhooks links.""" + + def __init__(self, bot: Bot): + self.bot = bot + + @property + def mod_log(self) -> ModLog: + """Get current instance of `ModLog`.""" + return self.bot.get_cog("ModLog") + + async def delete_and_respond(self, msg: Message, redacted_url: str) -> None: + """Delete `msg` and send a warning that it contained the Discord webhook `redacted_url`.""" + # Don't log this, due internal delete, not by user. Will make different entry. + self.mod_log.ignore(Event.message_delete, msg.id) + await msg.delete() + await msg.channel.send(ALERT_MESSAGE_TEMPLATE.format(user=msg.author.mention)) + + message = ( + f"{msg.author} (`{msg.author.id}`) posted a Discord webhook URL " + f"to #{msg.channel}. Webhook URL was `{redacted_url}`" + ) + log.debug(message) + + # Send entry to moderation alerts. + await self.mod_log.send_log_message( + icon_url=Icons.token_removed, + colour=Colour(Colours.soft_red), + title="Discord webhook URL removed!", + text=message, + thumbnail=msg.author.avatar_url_as(static_format="png"), + channel_id=Channels.mod_alerts + ) + + @Cog.listener() + async def on_message(self, msg: Message) -> None: + """Check if a Discord webhook URL is in `message`.""" + matches = WEBHOOK_URL_RE.search(msg.content) + if matches: + await self.delete_and_respond(msg, matches[1] + "xxx") + + @Cog.listener() + async def on_message_edit(self, before: Message, after: Message) -> None: + """Check if a Discord webhook URL is in the edited message `after`.""" + await self.on_message(after) + + +def setup(bot: Bot) -> None: + """Load `WebhookRemover` cog.""" + bot.add_cog(WebhookRemover(bot)) diff --git a/bot/resources/tags/args-kwargs.md b/bot/resources/tags/args-kwargs.md new file mode 100644 index 000000000..b440a2346 --- /dev/null +++ b/bot/resources/tags/args-kwargs.md @@ -0,0 +1,17 @@ +`*args` and `**kwargs` + +These special parameters allow functions to take arbitrary amounts of positional and keyword arguments. The names `args` and `kwargs` are purely convention, and could be named any other valid variable name. The special functionality comes from the single and double asterisks (`*`). If both are used in a function signature, `*args` **must** appear before `**kwargs`. + +**Single asterisk** +`*args` will ingest an arbitrary amount of **positional arguments**, and store it in a tuple. If there are parameters after `*args` in the parameter list with no default value, they will become **required** keyword arguments by default. + +**Double asterisk** +`**kwargs` will ingest an arbitrary amount of **keyword arguments**, and store it in a dictionary. There can be **no** additional parameters **after** `**kwargs` in the parameter list. + +**Use cases** +• **Decorators** (see `!tags decorators`) +• **Inheritance** (overriding methods) +• **Future proofing** (in the case of the first two bullet points, if the parameters change, your code won't break) +• **Flexibility** (writing functions that behave like `dict()` or `print()`) + +*See* `!tags positional-keyword` *for information about positional and keyword arguments* diff --git a/bot/resources/tags/ask.md b/bot/resources/tags/ask.md new file mode 100644 index 000000000..e2c2a88f6 --- /dev/null +++ b/bot/resources/tags/ask.md @@ -0,0 +1,9 @@ +Asking good questions will yield a much higher chance of a quick response: + +• Don't ask to ask your question, just go ahead and tell us your problem. +• Don't ask if anyone is knowledgeable in some area, filtering serves no purpose. +• Try to solve the problem on your own first, we're not going to write code for you. +• Show us the code you've tried and any errors or unexpected results it's giving. +• Be patient while we're helping you. + +You can find a much more detailed explanation [on our website](https://pythondiscord.com/pages/asking-good-questions/). diff --git a/bot/resources/tags/class.md b/bot/resources/tags/class.md new file mode 100644 index 000000000..4f73fc974 --- /dev/null +++ b/bot/resources/tags/class.md @@ -0,0 +1,25 @@ +**Classes** + +Classes are used to create objects that have specific behavior. + +Every object in python has a class, including `list`s, `dict`ionaries and even numbers. Using a class to group code and data like this is the foundation of Object Oriented Programming. Classes allow you to expose a simple, consistent interface while hiding the more complicated details. This simplifies the rest of your program and makes it easier to separately maintain and debug each component. + +Here is an example class: + +```python +class Foo: + def __init__(self, somedata): + self.my_attrib = somedata + + def show(self): + print(self.my_attrib) +``` + +To use a class, you need to instantiate it. The following creates a new object named `bar`, with `Foo` as its class. + +```python +bar = Foo('data') +bar.show() +``` + +We can access any of `Foo`'s methods via `bar.my_method()`, and access any of `bar`s data via `bar.my_attribute`. diff --git a/bot/resources/tags/classmethod.md b/bot/resources/tags/classmethod.md new file mode 100644 index 000000000..a4e803093 --- /dev/null +++ b/bot/resources/tags/classmethod.md @@ -0,0 +1,20 @@ +Although most methods are tied to an _object instance_, it can sometimes be useful to create a method that does something with _the class itself_. To achieve this in Python, you can use the `@classmethod` decorator. This is often used to provide alternative constructors for a class. + +For example, you may be writing a class that takes some magic token (like an API key) as a constructor argument, but you sometimes read this token from a configuration file. You could make use of a `@classmethod` to create an alternate constructor for when you want to read from the configuration file. +```py +class Bot: + def __init__(self, token: str): + self._token = token + + @classmethod + def from_config(cls, config: dict) -> Bot: + token = config['token'] + return cls(token) + +# now we can create the bot instance like this +alternative_bot = Bot.from_config(default_config) + +# but this still works, too +regular_bot = Bot("tokenstring") +``` +This is just one of the many use cases of `@classmethod`. A more in-depth explanation can be found [here](https://stackoverflow.com/questions/12179271/meaning-of-classmethod-and-staticmethod-for-beginner#12179752). diff --git a/bot/resources/tags/codeblock.md b/bot/resources/tags/codeblock.md new file mode 100644 index 000000000..a28ae397b --- /dev/null +++ b/bot/resources/tags/codeblock.md @@ -0,0 +1,17 @@ +Discord has support for Markdown, which allows you to post code with full syntax highlighting. Please use these whenever you paste code, as this helps improve the legibility and makes it easier for us to help you. + +To do this, use the following method: + +\```python +print('Hello world!') +\``` + +Note: +• **These are backticks, not quotes.** Backticks can usually be found on the tilde key. +• You can also use py as the language instead of python +• The language must be on the first line next to the backticks with **no** space between them + +This will result in the following: +```py +print('Hello world!') +``` diff --git a/bot/resources/tags/decorators.md b/bot/resources/tags/decorators.md new file mode 100644 index 000000000..39c943f0a --- /dev/null +++ b/bot/resources/tags/decorators.md @@ -0,0 +1,31 @@ +**Decorators** + +A decorator is a function that modifies another function. + +Consider the following example of a timer decorator: +```py +>>> import time +>>> def timer(f): +... def inner(*args, **kwargs): +... start = time.time() +... result = f(*args, **kwargs) +... print('Time elapsed:', time.time() - start) +... return result +... return inner +... +>>> @timer +... def slow(delay=1): +... time.sleep(delay) +... return 'Finished!' +... +>>> print(slow()) +Time elapsed: 1.0011568069458008 +Finished! +>>> print(slow(3)) +Time elapsed: 3.000307321548462 +Finished! +``` + +More information: +• [Corey Schafer's video on decorators](https://youtu.be/FsAPt_9Bf3U) +• [Real python article](https://realpython.com/primer-on-python-decorators/) diff --git a/bot/resources/tags/dictcomps.md b/bot/resources/tags/dictcomps.md new file mode 100644 index 000000000..11867d77b --- /dev/null +++ b/bot/resources/tags/dictcomps.md @@ -0,0 +1,20 @@ +**Dictionary Comprehensions** + +Like lists, there is a convenient way of creating dictionaries: +```py +>>> ftoc = {f: round((5/9)*(f-32)) for f in range(-40,101,20)} +>>> print(ftoc) +{-40: -40, -20: -29, 0: -18, 20: -7, 40: 4, 60: 16, 80: 27, 100: 38} +``` +In the example above, I created a dictionary of temperatures in Fahrenheit, that are mapped to (*roughly*) their Celsius counterpart within a small range. These comprehensions are useful for succinctly creating dictionaries from some other sequence. + +They are also very useful for inverting the key value pairs of a dictionary that already exists, such that the value in the old dictionary is now the key, and the corresponding key is now its value: +```py +>>> ctof = {v:k for k, v in ftoc.items()} +>>> print(ctof) +{-40: -40, -29: -20, -18: 0, -7: 20, 4: 40, 16: 60, 27: 80, 38: 100} +``` + +Also like list comprehensions, you can add a conditional to it in order to filter out items you don't want. + +For more information and examples, check [PEP 274](https://www.python.org/dev/peps/pep-0274/) diff --git a/bot/resources/tags/enumerate.md b/bot/resources/tags/enumerate.md new file mode 100644 index 000000000..dd984af52 --- /dev/null +++ b/bot/resources/tags/enumerate.md @@ -0,0 +1,13 @@ +Ever find yourself in need of the current iteration number of your `for` loop? You should use **enumerate**! Using `enumerate`, you can turn code that looks like this: +```py +index = 0 +for item in my_list: + print(f"{index}: {item}") + index += 1 +``` +into beautiful, _pythonic_ code: +```py +for index, item in enumerate(my_list): + print(f"{index}: {item}") +``` +For more information, check out [the official docs](https://docs.python.org/3/library/functions.html#enumerate), or [PEP 279](https://www.python.org/dev/peps/pep-0279/). diff --git a/bot/resources/tags/except.md b/bot/resources/tags/except.md new file mode 100644 index 000000000..8f0abf156 --- /dev/null +++ b/bot/resources/tags/except.md @@ -0,0 +1,17 @@ +A key part of the Python philosophy is to ask for forgiveness, not permission. This means that it's okay to write code that may produce an error, as long as you specify how that error should be handled. Code written this way is readable and resilient. +```py +try: + number = int(user_input) +except ValueError: + print("failed to convert user_input to a number. setting number to 0.") + number = 0 +``` +You should always specify the exception type if it is possible to do so, and your `try` block should be as short as possible. Attempting to handle broad categories of unexpected exceptions can silently hide serious problems. +```py +try: + number = int(user_input) + item = some_list[number] +except: + print("An exception was raised, but we have no idea if it was a ValueError or an IndexError.") +``` +For more information about exception handling, see [the official Python docs](https://docs.python.org/3/tutorial/errors.html), or watch [Corey Schafer's video on exception handling](https://www.youtube.com/watch?v=NIWwJbo-9_8). diff --git a/bot/resources/tags/exit().md b/bot/resources/tags/exit().md new file mode 100644 index 000000000..27da9f866 --- /dev/null +++ b/bot/resources/tags/exit().md @@ -0,0 +1,8 @@ +**Exiting Programmatically** + +If you want to exit your code programmatically, you might think to use the functions `exit()` or `quit()`, however this is bad practice. These functions are constants added by the [`site`](https://docs.python.org/3/library/site.html#module-site) module as a convenient method for exiting the interactive interpreter shell, and should not be used in programs. + +You should use either [`SystemExit`](https://docs.python.org/3/library/exceptions.html#SystemExit) or [`sys.exit()`](https://docs.python.org/3/library/sys.html#sys.exit) instead. +There's not much practical difference between these two other than having to `import sys` for the latter. Both take an optional argument to provide an exit status. + +[Official documentation](https://docs.python.org/3/library/constants.html#exit) with the warning not to use `exit()` or `quit()` in source code. diff --git a/bot/resources/tags/f-strings.md b/bot/resources/tags/f-strings.md new file mode 100644 index 000000000..69bc82487 --- /dev/null +++ b/bot/resources/tags/f-strings.md @@ -0,0 +1,17 @@ +In Python, there are several ways to do string interpolation, including using `%s`\'s and by using the `+` operator to concatenate strings together. However, because some of these methods offer poor readability and require typecasting to prevent errors, you should for the most part be using a feature called format strings. + +**In Python 3.6 or later, we can use f-strings like this:** +```py +snake = "Pythons" +print(f"{snake} are some of the largest snakes in the world") +``` +**In earlier versions of Python or in projects where backwards compatibility is very important, use str.format() like this:** +```py +snake = "Pythons" + +# With str.format() you can either use indexes +print("{0} are some of the largest snakes in the world".format(snake)) + +# Or keyword arguments +print("{family} are some of the largest snakes in the world".format(family=snake)) +``` diff --git a/bot/resources/tags/foo.md b/bot/resources/tags/foo.md new file mode 100644 index 000000000..98529bfc0 --- /dev/null +++ b/bot/resources/tags/foo.md @@ -0,0 +1,10 @@ +**Metasyntactic variables** + +A specific word or set of words identified as a placeholder used in programming. They are used to name entities such as variables, functions, etc, whose exact identity is unimportant and serve only to demonstrate a concept, which is useful for teaching programming. + +Common examples include `foobar`, `foo`, `bar`, `baz`, and `qux`. +Python has its own metasyntactic variables, namely `spam`, `eggs`, and `bacon`. This is a reference to a [Monty Python](https://en.wikipedia.org/wiki/Monty_Python) sketch (the eponym of the language). + +More information: +• [History of foobar](https://en.wikipedia.org/wiki/Foobar) +• [Monty Python sketch](https://en.wikipedia.org/wiki/Spam_%28Monty_Python%29) diff --git a/bot/resources/tags/functions-are-objects.md b/bot/resources/tags/functions-are-objects.md new file mode 100644 index 000000000..01af7a721 --- /dev/null +++ b/bot/resources/tags/functions-are-objects.md @@ -0,0 +1,39 @@ +**Calling vs. Referencing functions** + +When assigning a new name to a function, storing it in a container, or passing it as an argument, a common mistake made is to call the function. Instead of getting the actual function, you'll get its return value. + +In Python you can treat function names just like any other variable. Assume there was a function called `now` that returns the current time. If you did `x = now()`, the current time would be assigned to `x`, but if you did `x = now`, the function `now` itself would be assigned to `x`. `x` and `now` would both equally reference the function. + +**Examples** +```py +# assigning new name + +def foo(): + return 'bar' + +def spam(): + return 'eggs' + +baz = foo +baz() # returns 'bar' + +ham = spam +ham() # returns 'eggs' +``` +```py +# storing in container + +import math +functions = [math.sqrt, math.factorial, math.log] +functions[0](25) # returns 5.0 +# the above equivalent to math.sqrt(25) +``` +```py +# passing as argument + +class C: + builtin_open = staticmethod(open) + +# open function is passed +# to the staticmethod class +``` diff --git a/bot/resources/tags/global.md b/bot/resources/tags/global.md new file mode 100644 index 000000000..64c316b62 --- /dev/null +++ b/bot/resources/tags/global.md @@ -0,0 +1,16 @@ +When adding functions or classes to a program, it can be tempting to reference inaccessible variables by declaring them as global. Doing this can result in code that is harder to read, debug and test. Instead of using globals, pass variables or objects as parameters and receive return values. + +Instead of writing +```py +def update_score(): + global score, roll + score = score + roll +update_score() +``` +do this instead +```py +def update_score(score, roll): + return score + roll +score = update_score(score, roll) +``` +For in-depth explanations on why global variables are bad news in a variety of situations, see [this Stack Overflow answer](https://stackoverflow.com/questions/19158339/why-are-global-variables-evil/19158418#19158418). diff --git a/bot/resources/tags/if-name-main.md b/bot/resources/tags/if-name-main.md new file mode 100644 index 000000000..9d88bb897 --- /dev/null +++ b/bot/resources/tags/if-name-main.md @@ -0,0 +1,26 @@ +`if __name__ == '__main__'` + +This is a statement that is only true if the module (your source code) it appears in is being run directly, as opposed to being imported into another module. When you run your module, the `__name__` special variable is automatically set to the string `'__main__'`. Conversely, when you import that same module into a different one, and run that, `__name__` is instead set to the filename of your module minus the `.py` extension. + +**Example** +```py +# foo.py + +print('spam') + +if __name__ == '__main__': + print('eggs') +``` +If you run the above module `foo.py` directly, both `'spam'`and `'eggs'` will be printed. Now consider this next example: +```py +# bar.py + +import foo +``` +If you run this module named `bar.py`, it will execute the code in `foo.py`. First it will print `'spam'`, and then the `if` statement will fail, because `__name__` will now be the string `'foo'`. + +**Why would I do this?** + +• Your module is a library, but also has a special case where it can be run directly +• Your module is a library and you want to safeguard it against people running it directly (like what `pip` does) +• Your module is the main program, but has unit tests and the testing framework works by importing your module, and you want to avoid having your main code run during the test diff --git a/bot/resources/tags/indent.md b/bot/resources/tags/indent.md new file mode 100644 index 000000000..dec8407b0 --- /dev/null +++ b/bot/resources/tags/indent.md @@ -0,0 +1,24 @@ +**Indentation** + +Indentation is leading whitespace (spaces and tabs) at the beginning of a line of code. In the case of Python, they are used to determine the grouping of statements. + +Spaces should be preferred over tabs. To be clear, this is in reference to the character itself, not the keys on a keyboard. Your editor/IDE should be configured to insert spaces when the TAB key is pressed. The amount of spaces should be a multiple of 4, except optionally in the case of continuation lines. + +**Example** +```py +def foo(): + bar = 'baz' # indented one level + if bar == 'baz': + print('ham') # indented two levels + return bar # indented one level +``` +The first line is not indented. The next two lines are indented to be inside of the function definition. They will only run when the function is called. The fourth line is indented to be inside the `if` statement, and will only run if the `if` statement evaluates to `True`. The fifth and last line is like the 2nd and 3rd and will always run when the function is called. It effectively closes the `if` statement above as no more lines can be inside the `if` statement below that line. + +**Indentation is used after:** +**1.** [Compound statements](https://docs.python.org/3/reference/compound_stmts.html) (eg. `if`, `while`, `for`, `try`, `with`, `def`, `class`, and their counterparts) +**2.** [Continuation lines](https://www.python.org/dev/peps/pep-0008/#indentation) + +**More Info** +**1.** [Indentation style guide](https://www.python.org/dev/peps/pep-0008/#indentation) +**2.** [Tabs or Spaces?](https://www.python.org/dev/peps/pep-0008/#tabs-or-spaces) +**3.** [Official docs on indentation](https://docs.python.org/3/reference/lexical_analysis.html#indentation) diff --git a/bot/resources/tags/inline.md b/bot/resources/tags/inline.md new file mode 100644 index 000000000..a6a7c35d6 --- /dev/null +++ b/bot/resources/tags/inline.md @@ -0,0 +1,16 @@ +**Inline codeblocks** + +In addition to multi-line codeblocks, discord has support for inline codeblocks as well. These are small codeblocks that are usually a single line, that can fit between non-codeblocks on the same line. + +The following is an example of how it's done: + +The \`\_\_init\_\_\` method customizes the newly created instance. + +And results in the following: + +The `__init__` method customizes the newly created instance. + +**Note:** +• These are **backticks** not quotes +• Avoid using them for multiple lines +• Useful for negating formatting you don't want diff --git a/bot/resources/tags/iterate-dict.md b/bot/resources/tags/iterate-dict.md new file mode 100644 index 000000000..78c067b20 --- /dev/null +++ b/bot/resources/tags/iterate-dict.md @@ -0,0 +1,10 @@ +There are two common ways to iterate over a dictionary in Python. To iterate over the keys: +```py +for key in my_dict: + print(key) +``` +To iterate over both the keys and values: +```py +for key, val in my_dict.items(): + print(key, val) +``` diff --git a/bot/resources/tags/listcomps.md b/bot/resources/tags/listcomps.md new file mode 100644 index 000000000..0003b9bb8 --- /dev/null +++ b/bot/resources/tags/listcomps.md @@ -0,0 +1,14 @@ +Do you ever find yourself writing something like: +```py +even_numbers = [] +for n in range(20): + if n % 2 == 0: + even_numbers.append(n) +``` +Using list comprehensions can simplify this significantly, and greatly improve code readability. If we rewrite the example above to use list comprehensions, it would look like this: +```py +even_numbers = [n for n in range(20) if n % 2 == 0] +``` +This also works for generators, dicts and sets by using `()` or `{}` instead of `[]`. + +For more info, see [this pythonforbeginners.com post](http://www.pythonforbeginners.com/basics/list-comprehensions-in-python) or [PEP 202](https://www.python.org/dev/peps/pep-0202/). diff --git a/bot/resources/tags/mutable-default-args.md b/bot/resources/tags/mutable-default-args.md new file mode 100644 index 000000000..a8f0c38b3 --- /dev/null +++ b/bot/resources/tags/mutable-default-args.md @@ -0,0 +1,48 @@ +**Mutable Default Arguments** + +Default arguments in python are evaluated *once* when the function is +**defined**, *not* each time the function is **called**. This means that if +you have a mutable default argument and mutate it, you will have +mutated that object for all future calls to the function as well. + +For example, the following `append_one` function appends `1` to a list +and returns it. `foo` is set to an empty list by default. +```python +>>> def append_one(foo=[]): +... foo.append(1) +... return foo +... +``` +See what happens when we call it a few times: +```python +>>> append_one() +[1] +>>> append_one() +[1, 1] +>>> append_one() +[1, 1, 1] +``` +Each call appends an additional `1` to our list `foo`. It does not +receive a new empty list on each call, it is the same list everytime. + +To avoid this problem, you have to create a new object every time the +function is **called**: +```python +>>> def append_one(foo=None): +... if foo is None: +... foo = [] +... foo.append(1) +... return foo +... +>>> append_one() +[1] +>>> append_one() +[1] +``` + +**Note**: + +• This behavior can be used intentionally to maintain state between +calls of a function (eg. when writing a caching function). +• This behavior is not unique to mutable objects, all default +arguments are evaulated only once when the function is defined. diff --git a/bot/resources/tags/names.md b/bot/resources/tags/names.md new file mode 100644 index 000000000..3e76269f7 --- /dev/null +++ b/bot/resources/tags/names.md @@ -0,0 +1,37 @@ +**Naming and Binding** + +A name is a piece of text that is bound to an object. They are a **reference** to an object. Examples are function names, class names, module names, variables, etc. + +**Note:** Names **cannot** reference other names, and assignment **never** creates a copy. +```py +x = 1 # x is bound to 1 +y = x # y is bound to VALUE of x +x = 2 # x is bound to 2 +print(x, y) # 2 1 +``` +When doing `y = x`, the name `y` is being bound to the *value* of `x` which is `1`. Neither `x` nor `y` are the 'real' name. The object `1` simply has *multiple* names. They are the exact same object. +``` +>>> x = 1 +x ━━ 1 + +>>> y = x +x ━━ 1 +y ━━━┛ + +>>> x = 2 +x ━━ 2 +y ━━ 1 +``` +**Names are created in multiple ways** +You might think that the only way to bind a name to an object is by using assignment, but that isn't the case. All of the following work exactly the same as assignment: +• `import` statements +• `class` and `def` +• `for` loop headers +• `as` keyword when used with `except`, `import`, and `with` +• formal parameters in function headers + +There is also `del` which has the purpose of *unbinding* a name. + +**More info** +• Please watch [Ned Batchelder's talk](https://youtu.be/_AEJHKGk9ns) on names in python for a detailed explanation with examples +• [Official documentation](https://docs.python.org/3/reference/executionmodel.html#naming-and-binding) diff --git a/bot/resources/tags/off-topic.md b/bot/resources/tags/off-topic.md new file mode 100644 index 000000000..c7f98a813 --- /dev/null +++ b/bot/resources/tags/off-topic.md @@ -0,0 +1,8 @@ +**Off-topic channels** + +There are three off-topic channels: +• <#291284109232308226> +• <#463035241142026251> +• <#463035268514185226> + +Their names change randomly every 24 hours, but you can always find them under the `OFF-TOPIC/GENERAL` category in the channel list. diff --git a/bot/resources/tags/open.md b/bot/resources/tags/open.md new file mode 100644 index 000000000..13b4555b9 --- /dev/null +++ b/bot/resources/tags/open.md @@ -0,0 +1,26 @@ +**Opening files** + +The built-in function `open()` is one of several ways to open files on your computer. It accepts many different parameters, so this tag will only go over two of them (`file` and `mode`). For more extensive documentation on all these parameters, consult the [official documentation](https://docs.python.org/3/library/functions.html#open). The object returned from this function is a [file object or stream](https://docs.python.org/3/glossary.html#term-file-object), for which the full documentation can be found [here](https://docs.python.org/3/library/io.html#io.TextIOBase). + +See also: +• `!tags with` for information on context managers +• `!tags pathlib` for an alternative way of opening files +• `!tags seek` for information on changing your position in a file + +**The `file` parameter** + +This should be a [path-like object](https://docs.python.org/3/glossary.html#term-path-like-object) denoting the name or path (absolute or relative) to the file you want to open. + +An absolute path is the full path from your root directory to the file you want to open. Generally this is the option you should choose so it doesn't matter what directory you're in when you execute your module. + +See `!tags relative-path` for more information on relative paths. + +**The `mode` parameter** + +This is an optional string that specifies the mode in which the file should be opened. There's not enough room to discuss them all, but listed below are some of the more confusing modes. + +`'r+'` Opens for reading and writing (file must already exist) +`'w+'` Opens for reading and writing and truncates (can create files) +`'x'` Creates file and opens for writing (file must **not** already exist) +`'x+'` Creates file and opens for reading and writing (file must **not** already exist) +`'a+'` Opens file for reading and writing at **end of file** (can create files) diff --git a/bot/resources/tags/or-gotcha.md b/bot/resources/tags/or-gotcha.md new file mode 100644 index 000000000..00c2db1f8 --- /dev/null +++ b/bot/resources/tags/or-gotcha.md @@ -0,0 +1,17 @@ +When checking if something is equal to one thing or another, you might think that this is possible: +```py +if favorite_fruit == 'grapefruit' or 'lemon': + print("That's a weird favorite fruit to have.") +``` +After all, that's how you would normally phrase it in plain English. In Python, however, you have to have _complete instructions on both sides of the logical operator_. + +So, if you want to check if something is equal to one thing or another, there are two common ways: +```py +# Like this... +if favorite_fruit == 'grapefruit' or favorite_fruit == 'lemon': + print("That's a weird favorite fruit to have.") + +# ...or like this. +if favorite_fruit in ('grapefruit', 'lemon'): + print("That's a weird favorite fruit to have.") +``` diff --git a/bot/resources/tags/param-arg.md b/bot/resources/tags/param-arg.md new file mode 100644 index 000000000..88069d8bd --- /dev/null +++ b/bot/resources/tags/param-arg.md @@ -0,0 +1,12 @@ +**Parameters vs. Arguments** + +A parameter is a variable defined in a function signature (the line with `def` in it), while arguments are objects passed to a function call. + +```py +def square(n): # n is the parameter + return n*n + +print(square(5)) # 5 is the argument +``` + +Note that `5` is the argument passed to `square`, but `square(5)` in its entirety is the argument passed to `print` diff --git a/bot/resources/tags/paste.md b/bot/resources/tags/paste.md new file mode 100644 index 000000000..2ed51def7 --- /dev/null +++ b/bot/resources/tags/paste.md @@ -0,0 +1,6 @@ +**Pasting large amounts of code** + +If your code is too long to fit in a codeblock in discord, you can paste your code here: +https://paste.pydis.com/ + +After pasting your code, **save** it by clicking the floppy disk icon in the top right, or by typing `ctrl + S`. After doing that, the URL should **change**. Copy the URL and post it here so others can see it. diff --git a/bot/resources/tags/pathlib.md b/bot/resources/tags/pathlib.md new file mode 100644 index 000000000..dfeb7ecac --- /dev/null +++ b/bot/resources/tags/pathlib.md @@ -0,0 +1,21 @@ +**Pathlib** + +Python 3 comes with a new module named `Pathlib`. Since Python 3.6, `pathlib.Path` objects work nearly everywhere that `os.path` can be used, meaning you can integrate your new code directly into legacy code without having to rewrite anything. Pathlib makes working with paths way simpler than `os.path` does. + +**Feature spotlight**: + +• Normalizes file paths for all platforms automatically +• Has glob-like utilites (eg. `Path.glob`, `Path.rglob`) for searching files +• Can read and write files, and close them automatically +• Convenient syntax, utilising the `/` operator (e.g. `Path('~') / 'Documents'`) +• Can easily pick out components of a path (eg. name, parent, stem, suffix, anchor) +• Supports method chaining +• Move and delete files +• And much more + +**More Info**: + +• [**Why you should use pathlib** - Trey Hunner](https://treyhunner.com/2018/12/why-you-should-be-using-pathlib/) +• [**Answering concerns about pathlib** - Trey Hunner](https://treyhunner.com/2019/01/no-really-pathlib-is-great/) +• [**Official Documentation**](https://docs.python.org/3/library/pathlib.html) +• [**PEP 519** - Adding a file system path protocol](https://www.python.org/dev/peps/pep-0519/) diff --git a/bot/resources/tags/pep8.md b/bot/resources/tags/pep8.md new file mode 100644 index 000000000..cab4c4db8 --- /dev/null +++ b/bot/resources/tags/pep8.md @@ -0,0 +1,3 @@ +**PEP 8** is the official style guide for Python. It includes comprehensive guidelines for code formatting, variable naming, and making your code easy to read. Professional Python developers are usually required to follow the guidelines, and will often use code-linters like `flake8` to verify that the code they\'re writing complies with the style guide. + +You can find the PEP 8 document [here](https://www.python.org/dev/peps/pep-0008). diff --git a/bot/resources/tags/positional-keyword.md b/bot/resources/tags/positional-keyword.md new file mode 100644 index 000000000..dd6ddfc4b --- /dev/null +++ b/bot/resources/tags/positional-keyword.md @@ -0,0 +1,38 @@ +**Positional vs. Keyword arguments** + +Functions can take two different kinds of arguments. A positional argument is just the object itself. A keyword argument is a name assigned to an object. + +**Example** +```py +>>> print('Hello', 'world!', sep=', ') +Hello, world! +``` +The first two strings `'Hello'` and `world!'` are positional arguments. +The `sep=', '` is a keyword argument. + +**Note** +A keyword argument can be passed positionally in some cases. +```py +def sum(a, b=1): + return a + b + +sum(1, b=5) +sum(1, 5) # same as above +``` +[Somtimes this is forced](https://www.python.org/dev/peps/pep-0570/#history-of-positional-only-parameter-semantics-in-python), in the case of the `pow()` function. + +The reverse is also true: +```py +>>> def foo(a, b): +... print(a, b) +... +>>> foo(a=1, b=2) +1 2 +>>> foo(b=1, a=2) +2 1 +``` + +**More info** +• [Keyword only arguments](https://www.python.org/dev/peps/pep-3102/) +• [Positional only arguments](https://www.python.org/dev/peps/pep-0570/) +• `!tags param-arg` (Parameters vs. Arguments) diff --git a/bot/resources/tags/precedence.md b/bot/resources/tags/precedence.md new file mode 100644 index 000000000..ed399143c --- /dev/null +++ b/bot/resources/tags/precedence.md @@ -0,0 +1,13 @@ +**Operator Precedence** + +Operator precedence is essentially like an order of operations for python's operators. + +**Example 1** (arithmetic) +`2 * 3 + 1` is `7` because multiplication is first +`2 * (3 + 1)` is `8` because the parenthesis change the precedence allowing the sum to be first + +**Example 2** (logic) +`not True or True` is `True` because the `not` is first +`not (True or True)` is `False` because the `or` is first + +The full table of precedence from lowest to highest is [here](https://docs.python.org/3/reference/expressions.html#operator-precedence) diff --git a/bot/resources/tags/quotes.md b/bot/resources/tags/quotes.md new file mode 100644 index 000000000..8421748a1 --- /dev/null +++ b/bot/resources/tags/quotes.md @@ -0,0 +1,20 @@ +**String Quotes** + +Single and Double quoted strings are the **same** in Python. The choice of which one to use is up to you, just make sure that you **stick to that choice**. + +With that said, there are exceptions to this that are more important than consistency. If a single or double quote is needed *inside* the string, using the opposite quotation is better than using escape characters. + +Example: +```py +'My name is "Guido"' # good +"My name is \"Guido\"" # bad + +"Don't go in there" # good +'Don\'t go in there' # bad +``` +**Note:** +If you need both single and double quotes inside your string, use the version that would result in the least amount of escapes. In the case of a tie, use the quotation you use the most. + +**References:** +• [pep-8 on quotes](https://www.python.org/dev/peps/pep-0008/#string-quotes) +• [convention for triple quoted strings](https://www.python.org/dev/peps/pep-0257/) diff --git a/bot/resources/tags/relative-path.md b/bot/resources/tags/relative-path.md new file mode 100644 index 000000000..6e97b78af --- /dev/null +++ b/bot/resources/tags/relative-path.md @@ -0,0 +1,7 @@ +**Relative Path** + +A relative path is a partial path that is relative to your current working directory. A common misconception is that your current working directory is the location of the module you're executing, **but this is not the case**. Your current working directory is actually the **directory you were in when you ran the python interpreter**. The reason for this misconception is because a common way to run your code is to navigate to the directory your module is stored, and run `python <module>.py`. Thus, in this case your current working directory will be the same as the location of the module. However, if we instead did `python path/to/<module>.py`, our current working directory would no longer be the same as the location of the module we're executing. + +**Why is this important?** + +When opening files in python, relative paths won't always work since it's dependent on what directory you were in when you ran your code. A common issue people face is running their code in an IDE thinking they can open files that are in the same directory as their module, but the current working directory will be different than what they expect and so they won't find the file. The way to avoid this problem is by using absolute paths, which is the full path from your root directory to the file you want to open. diff --git a/bot/resources/tags/repl.md b/bot/resources/tags/repl.md new file mode 100644 index 000000000..875b4ec47 --- /dev/null +++ b/bot/resources/tags/repl.md @@ -0,0 +1,13 @@ +**Read-Eval-Print Loop** + +A REPL is an interactive language shell environment. It first **reads** one or more expressions entered by the user, **evaluates** it, yields the result, and **prints** it out to the user. It will then **loop** back to the **read** step. + +To use python's REPL, execute the interpreter with no arguments. This will drop you into the interactive interpreter shell, print out some relevant information, and then prompt you with the primary prompt `>>>`. At this point it is waiting for your input. + +Firstly you can start typing in some valid python expressions, pressing <return> to either bring you to the **eval** step, or prompting you with the secondary prompt `...` (or no prompt at all depending on your environment), meaning your expression isn't yet terminated and it's waiting for more input. This is useful for code that requires multiple lines like loops, functions, and classes. If you reach the secondary prompt in a clause that can have an arbitrary amount of expressions, you can terminate it by pressing <return> on a blank line. In other words, for the last expression you write in the clause, <return> must be pressed twice in a row. + +Alternatively, you can make use of the builtin `help()` function. `help(thing)` to get help on some `thing` object, or `help()` to start an interactive help session. This mode is extremely powerful, read the instructions when first entering the session to learn how to use it. + +Lastly you can run your code with the `-i` flag to execute your code normally, but be dropped into the REPL once execution is finished, giving you access to all your global variables/functions in the REPL. + +To **exit** either a help session, or normal REPL prompt, you must send an EOF signal to the prompt. In *nix systems, this is done with `ctrl + D`, and in windows systems it is `ctrl + Z`. You can also exit the normal REPL prompt with the dedicated functions `exit()` or `quit()`. diff --git a/bot/resources/tags/return.md b/bot/resources/tags/return.md new file mode 100644 index 000000000..e37f0eebc --- /dev/null +++ b/bot/resources/tags/return.md @@ -0,0 +1,35 @@ +**Return Statement** + +When calling a function, you'll often want it to give you a value back. In order to do that, you must `return` it. The reason for this is because functions have their own scope. Any values defined within the function body are inaccessible outside of that function. + +*For more information about scope, see `!tags scope`* + +Consider the following function: +```py +def square(n): + return n*n +``` +If we wanted to store 5 squared in a variable called `x`, we could do that like so: +`x = square(5)`. `x` would now equal `25`. + +**Common Mistakes** +```py +>>> def square(n): +... n*n # calculates then throws away, returns None +... +>>> x = square(5) +>>> print(x) +None +>>> def square(n): +... print(n*n) # calculates and prints, then throws away and returns None +... +>>> x = square(5) +25 +>>> print(x) +None +``` +**Things to note** +• `print()` and `return` do **not** accomplish the same thing. `print()` will only print the value, it will not be accessible outside of the function afterwards. +• A function will return `None` if it ends without reaching an explicit `return` statement. +• When you want to print a value calculated in a function, instead of printing inside the function, it is often better to return the value and print the *function call* instead. +• [Official documentation for `return`](https://docs.python.org/3/reference/simple_stmts.html#the-return-statement) diff --git a/bot/resources/tags/round.md b/bot/resources/tags/round.md new file mode 100644 index 000000000..0392bb41b --- /dev/null +++ b/bot/resources/tags/round.md @@ -0,0 +1,24 @@ +**Round half to even** + +Python 3 uses bankers' rounding (also known by other names), where if the fractional part of a number is `.5`, it's rounded to the nearest **even** result instead of away from zero. + +Example: +```py +>>> round(2.5) +2 +>>> round(1.5) +2 +``` +In the first example, there is a tie between 2 and 3, and since 3 is odd and 2 is even, the result is 2. +In the second example, the tie is between 1 and 2, and so 2 is also the result. + +**Why this is done:** +The round half up technique creates a slight bias towards the larger number. With a large amount of calculations, this can be significant. The round half to even technique eliminates this bias. + +It should be noted that round half to even distorts the distribution by increasing the probability of evens relative to odds, however this is considered less important than the bias explained above. + +**References:** +• [Wikipedia article about rounding](https://en.wikipedia.org/wiki/Rounding#Round_half_to_even) +• [Documentation on `round` function](https://docs.python.org/3/library/functions.html#round) +• [`round` in what's new in python 3](https://docs.python.org/3/whatsnew/3.0.html#builtins) (4th bullet down) +• [How to force rounding technique](https://stackoverflow.com/a/10826537/4607272) diff --git a/bot/resources/tags/scope.md b/bot/resources/tags/scope.md new file mode 100644 index 000000000..5c1e64e1c --- /dev/null +++ b/bot/resources/tags/scope.md @@ -0,0 +1,24 @@ +**Scoping Rules** + +A *scope* defines the visibility of a name within a block, where a block is a piece of python code executed as a unit. For simplicity, this would be a module, a function body, and a class definition. A name refers to text bound to an object. + +*For more information about names, see `!tags names`* + +A module is the source code file itself, and encompasses all blocks defined within it. Therefore if a variable is defined at the module level (top-level code block), it is a global variable and can be accessed anywhere in the module as long as the block in which it's referenced is executed after it was defined. + +Alternatively if a variable is defined within a function block for example, it is a local variable. It is not accessible at the module level, as that would be *outside* its scope. This is the purpose of the `return` statement, as it hands an object back to the scope of its caller. Conversely if a function was defined *inside* the previously mentioned block, it *would* have access to that variable, because it is within the first function's scope. +```py +>>> def outer(): +... foo = 'bar' # local variable to outer +... def inner(): +... print(foo) # has access to foo from scope of outer +... return inner # brings inner to scope of caller +... +>>> inner = outer() # get inner function +>>> inner() # prints variable foo without issue +bar +``` +**Official Documentation** +**1.** [Program structure, name binding and resolution](https://docs.python.org/3/reference/executionmodel.html#execution-model) +**2.** [`global` statement](https://docs.python.org/3/reference/simple_stmts.html#the-global-statement) +**3.** [`nonlocal` statement](https://docs.python.org/3/reference/simple_stmts.html#the-nonlocal-statement) diff --git a/bot/resources/tags/seek.md b/bot/resources/tags/seek.md new file mode 100644 index 000000000..bc013fe03 --- /dev/null +++ b/bot/resources/tags/seek.md @@ -0,0 +1,22 @@ +**Seek** + +In the context of a [file object](https://docs.python.org/3/glossary.html#term-file-object), the `seek` function changes the stream position to a given byte offset, with an optional argument of where to offset from. While you can find the official documentation [here](https://docs.python.org/3/library/io.html#io.IOBase.seek), it can be unclear how to actually use this feature, so keep reading to see examples on how to use it. + +File named `example`: +``` +foobar +spam eggs +``` +Open file for reading in byte mode: +```py +f = open('example', 'rb') +``` +Note that stream positions start from 0 in much the same way that the index for a list does. If we do `f.seek(3, 0)`, our stream position will move 3 bytes forward relative to the **beginning** of the stream. Now if we then did `f.read(1)` to read a single byte from where we are in the stream, it would return the string `'b'` from the 'b' in 'foobar'. Notice that the 'b' is the 4th character. Also note that after we did `f.read(1)`, we moved the stream position again 1 byte forward relative to the **current** position in the stream. So the stream position is now currently at position 4. + +Now lets do `f.seek(4, 1)`. This will move our stream position 4 bytes forward relative to our **current** position in the stream. Now if we did `f.read(1)`, it would return the string `'p'` from the 'p' in 'spam' on the next line. Note this time that the character at position 6 is the newline character `'\n'`. + +Finally, lets do `f.seek(-4, 2)`, moving our stream position *backwards* 4 bytes relative to the **end** of the stream. Now if we did `f.read()` to read everything after our position in the file, it would return the string `'eggs'` and also move our stream position to the end of the file. + +**Note** +• For the second argument in `seek()`, use `os.SEEK_SET`, `os.SEEK_CUR`, and `os.SEEK_END` in place of 0, 1, and 2 respectively. +• `os.SEEK_CUR` is only usable when the file is in byte mode. diff --git a/bot/resources/tags/self.md b/bot/resources/tags/self.md new file mode 100644 index 000000000..d20154fd5 --- /dev/null +++ b/bot/resources/tags/self.md @@ -0,0 +1,25 @@ +**Class instance** + +When calling a method from a class instance (ie. `instance.method()`), the instance itself will automatically be passed as the first argument implicitly. By convention, we call this `self`, but it could technically be called any valid variable name. + +```py +class Foo: + def bar(self): + print('bar') + + def spam(self, eggs): + print(eggs) + +foo = Foo() +``` + +If we call `foo.bar()`, it is equivalent to doing `Foo.bar(foo)`. Our instance `foo` is passed for us to the `bar` function, so while we initially gave zero arguments, it is actually called with one. + +Similarly if we call `foo.spam('ham')`, it is equivalent to +doing `Foo.spam(foo, 'ham')`. + +**Why is this useful?** + +Methods do not inherently have access to attributes defined in the class. In order for any one method to be able to access other methods or variables defined in the class, it must have access to the instance. + +Consider if outside the class, we tried to do this: `spam(foo, 'ham')`. This would give an error, because we don't have access to the `spam` method directly, we have to call it by doing `foo.spam('ham')`. This is also the case inside of the class. If we wanted to call the `bar` method inside the `spam` method, we'd have to do `self.bar()`, just doing `bar()` would give an error. diff --git a/bot/resources/tags/star-imports.md b/bot/resources/tags/star-imports.md new file mode 100644 index 000000000..2be6aab6e --- /dev/null +++ b/bot/resources/tags/star-imports.md @@ -0,0 +1,48 @@ +**Star / Wildcard imports** + +Wildcard imports are import statements in the form `from <module_name> import *`. What imports like these do is that they import everything **[1]** from the module into the current module's namespace **[2]**. This allows you to use names defined in the imported module without prefixing the module's name. + +Example: +```python +>>> from math import * +>>> sin(pi / 2) +1.0 +``` +**This is discouraged, for various reasons:** + +Example: +```python +>>> from custom_sin import sin +>>> from math import * +>>> sin(pi / 2) # uses sin from math rather than your custom sin +``` + +• Potential namespace collision. Names defined from a previous import might get shadowed by a wildcard import. + +• Causes ambiguity. From the example, it is unclear which `sin` function is actually being used. From the Zen of Python **[3]**: `Explicit is better than implicit.` + +• Makes import order significant, which they shouldn't. Certain IDE's `sort import` functionality may end up breaking code due to namespace collision. + +**How should you import?** + +• Import the module under the module's namespace (Only import the name of the module, and names defined in the module can be used by prefixing the module's name) + +```python +>>> import math +>>> math.sin(math.pi / 2) +``` + +• Explicitly import certain names from the module + +```python +>>> from math import sin, pi +>>> sin(pi / 2) +``` + +Conclusion: Namespaces are one honking great idea -- let's do more of those! *[3]* + +**[1]** If the module defines the variable `__all__`, the names defined in `__all__` will get imported by the wildcard import, otherwise all the names in the module get imported (except for names with a leading underscore) + +**[2]** [Namespaces and scopes](https://www.programiz.com/python-programming/namespace) + +**[3]** [Zen of Python](https://www.python.org/dev/peps/pep-0020/) diff --git a/bot/resources/tags/traceback.md b/bot/resources/tags/traceback.md new file mode 100644 index 000000000..46ef40aa1 --- /dev/null +++ b/bot/resources/tags/traceback.md @@ -0,0 +1,18 @@ +Please provide a full traceback to your exception in order for us to identify your issue. + +A full traceback could look like: +```py +Traceback (most recent call last): + File "tiny", line 3, in + do_something() + File "tiny", line 2, in do_something + a = 6 / 0 +ZeroDivisionError: integer division or modulo by zero +``` +The best way to read your traceback is bottom to top. + +• Identify the exception raised (e.g. ZeroDivisonError) +• Make note of the line number, and navigate there in your program. +• Try to understand why the error occurred. + +To read more about exceptions and errors, please refer to the [PyDis Wiki](https://pythondiscord.com/pages/asking-good-questions/#examining-tracebacks) or the [official Python tutorial.](https://docs.python.org/3.7/tutorial/errors.html) diff --git a/bot/resources/tags/windows-path.md b/bot/resources/tags/windows-path.md new file mode 100644 index 000000000..da8edf685 --- /dev/null +++ b/bot/resources/tags/windows-path.md @@ -0,0 +1,30 @@ +**PATH on Windows** + +If you have installed Python but you forgot to check the *Add Python to PATH* option during the installation you may still be able to access your installation with ease. + +If you did not uncheck the option to install the Python launcher then you will find a `py` command on your system. If you want to be able to open your Python installation by running `python` then your best option is to re-install Python. + +Otherwise, you can access your install using the `py` command in Command Prompt. Where you may type something with the `python` command like: +``` +C:\Users\Username> python3 my_application_file.py +``` + +You can achieve the same result using the `py` command like this: +``` +C:\Users\Username> py -3 my_application_file.py +``` + +You can pass any options to the Python interpreter after you specify a version, for example, to install a Python module using `pip` you can run: +``` +C:\Users\Username> py -3 -m pip install numpy +``` + +You can also access different versions of Python using the version flag, like so: +``` +C:\Users\Username> py -3.7 +... Python 3.7 starts ... +C:\Users\Username> py -3.6 +... Python 3.6 stars ... +C:\Users\Username> py -2 +... Python 2 (any version installed) starts ... +``` diff --git a/bot/resources/tags/with.md b/bot/resources/tags/with.md new file mode 100644 index 000000000..62d5612f2 --- /dev/null +++ b/bot/resources/tags/with.md @@ -0,0 +1,8 @@ +The `with` keyword triggers a context manager. Context managers automatically set up and take down data connections, or any other kind of object that implements the magic methods `__enter__` and `__exit__`. +```py +with open("test.txt", "r") as file: + do_things(file) +``` +The above code automatically closes `file` when the `with` block exits, so you never have to manually do a `file.close()`. Most connection types, including file readers and database connections, support this. + +For more information, read [the official docs](https://docs.python.org/3/reference/compound_stmts.html#with), watch [Corey Schafer\'s context manager video](https://www.youtube.com/watch?v=-aKFBoZpiqA), or see [PEP 343](https://www.python.org/dev/peps/pep-0343/). diff --git a/bot/resources/tags/xy-problem.md b/bot/resources/tags/xy-problem.md new file mode 100644 index 000000000..b77bd27e8 --- /dev/null +++ b/bot/resources/tags/xy-problem.md @@ -0,0 +1,7 @@ +**xy-problem** + +Asking about your attempted solution rather than your actual problem. + +Often programmers will get distracted with a potential solution they've come up with, and will try asking for help getting it to work. However, it's possible this solution either wouldn't work as they expect, or there's a much better solution instead. + +For more information and examples: http://xyproblem.info/ diff --git a/bot/resources/tags/ytdl.md b/bot/resources/tags/ytdl.md new file mode 100644 index 000000000..e34ecff44 --- /dev/null +++ b/bot/resources/tags/ytdl.md @@ -0,0 +1,12 @@ +Per [PyDis' Rule 5](https://pythondiscord.com/pages/rules), we are unable to assist with questions related to youtube-dl, commonly used by Discord bots to stream audio, as its use violates YouTube's Terms of Service. + +For reference, this usage is covered by the following clauses in [YouTube's TOS](https://www.youtube.com/static?template=terms), as of 2019-07-22: +``` +The following restrictions apply to your use of the Service. You are not allowed to: + +1. access, reproduce, download, distribute, transmit, broadcast, display, sell, license, alter, modify or otherwise use any part of the Service or any Content except: (a) as specifically permitted by the Service; (b) with prior written permission from YouTube and, if applicable, the respective rights holders; or (c) as permitted by applicable law; + +3. access the Service using any automated means (such as robots, botnets or scrapers) except: (a) in the case of public search engines, in accordance with YouTube’s robots.txt file; (b) with YouTube’s prior written permission; or (c) as permitted by applicable law; + +9. use the Service to view or listen to Content other than for personal, non-commercial use (for example, you may not publicly screen videos or stream music from the Service) +``` diff --git a/bot/resources/tags/zip.md b/bot/resources/tags/zip.md new file mode 100644 index 000000000..6b05f0282 --- /dev/null +++ b/bot/resources/tags/zip.md @@ -0,0 +1,12 @@ +The zip function allows you to iterate through multiple iterables simultaneously. It joins the iterables together, almost like a zipper, so that each new element is a tuple with one element from each iterable. + +```py +letters = 'abc' +numbers = [1, 2, 3] +# zip(letters, numbers) --> [('a', 1), ('b', 2), ('c', 3)] +for letter, number in zip(letters, numbers): + print(letter, number) +``` +The `zip()` iterator is exhausted after the length of the shortest iterable is exceeded. If you would like to retain the other values, consider using [itertools.zip_longest](https://docs.python.org/3/library/itertools.html#itertools.zip_longest). + +For more information on zip, please refer to the [official documentation](https://docs.python.org/3/library/functions.html#zip). diff --git a/bot/utils/__init__.py b/bot/utils/__init__.py index 3e4b15ce4..9b32e515d 100644 --- a/bot/utils/__init__.py +++ b/bot/utils/__init__.py @@ -1,5 +1,4 @@ from abc import ABCMeta -from typing import Any, Hashable from discord.ext.commands import CogMeta @@ -8,59 +7,3 @@ class CogABCMeta(CogMeta, ABCMeta): """Metaclass for ABCs meant to be implemented as Cogs.""" pass - - -class CaseInsensitiveDict(dict): - """ - We found this class on StackOverflow. Thanks to m000 for writing it! - - https://stackoverflow.com/a/32888599/4022104 - """ - - @classmethod - def _k(cls, key: Hashable) -> Hashable: - """Return lowered key if a string-like is passed, otherwise pass key straight through.""" - return key.lower() if isinstance(key, str) else key - - def __init__(self, *args, **kwargs): - super(CaseInsensitiveDict, self).__init__(*args, **kwargs) - self._convert_keys() - - def __getitem__(self, key: Hashable) -> Any: - """Case insensitive __setitem__.""" - return super(CaseInsensitiveDict, self).__getitem__(self.__class__._k(key)) - - def __setitem__(self, key: Hashable, value: Any): - """Case insensitive __setitem__.""" - super(CaseInsensitiveDict, self).__setitem__(self.__class__._k(key), value) - - def __delitem__(self, key: Hashable) -> Any: - """Case insensitive __delitem__.""" - return super(CaseInsensitiveDict, self).__delitem__(self.__class__._k(key)) - - def __contains__(self, key: Hashable) -> bool: - """Case insensitive __contains__.""" - return super(CaseInsensitiveDict, self).__contains__(self.__class__._k(key)) - - def pop(self, key: Hashable, *args, **kwargs) -> Any: - """Case insensitive pop.""" - return super(CaseInsensitiveDict, self).pop(self.__class__._k(key), *args, **kwargs) - - def get(self, key: Hashable, *args, **kwargs) -> Any: - """Case insensitive get.""" - return super(CaseInsensitiveDict, self).get(self.__class__._k(key), *args, **kwargs) - - def setdefault(self, key: Hashable, *args, **kwargs) -> Any: - """Case insensitive setdefault.""" - return super(CaseInsensitiveDict, self).setdefault(self.__class__._k(key), *args, **kwargs) - - def update(self, E: Any = None, **F) -> None: - """Case insensitive update.""" - super(CaseInsensitiveDict, self).update(self.__class__(E)) - super(CaseInsensitiveDict, self).update(self.__class__(**F)) - - def _convert_keys(self) -> None: - """Helper method to lowercase all existing string-like keys.""" - for k in list(self.keys()): - v = super(CaseInsensitiveDict, self).pop(k) - self.__setitem__(k, v) diff --git a/config-default.yml b/config-default.yml index 9beb610cc..5788d1e12 100644 --- a/config-default.yml +++ b/config-default.yml @@ -353,6 +353,7 @@ filter: - *ADMINS_ROLE - *MODS_ROLE - *OWNERS_ROLE + - *HELPERS_ROLE - *PY_COMMUNITY_ROLE diff --git a/tests/README.md b/tests/README.md index be78821bf..4f62edd68 100644 --- a/tests/README.md +++ b/tests/README.md @@ -83,7 +83,7 @@ TagContentConverter should return correct values for valid input. As we are trying to test our "units" of code independently, we want to make sure that we do not rely objects and data generated by "external" code. If we we did, then we wouldn't know if the failure we're observing was caused by the code we are actually trying to test or something external to it. -However, the features that we are trying to test often depend on those objects generated by external pieces of code. It would be difficult to test a bot command without having access to a `Context` instance. Fortunately, there's a solution for that: we use fake objects that act like the true object. We call these fake objects "mocks". +However, the features that we are trying to test often depend on those objects generated by external pieces of code. It would be difficult to test a bot command without having access to a `Context` instance. Fortunately, there's a solution for that: we use fake objects that act like the true object. We call these fake objects "mocks". To create these mock object, we mainly use the [`unittest.mock`](https://docs.python.org/3/library/unittest.mock.html) module. In addition, we have also defined a couple of specialized mock objects that mock specific `discord.py` types (see the section on the below.). @@ -114,13 +114,13 @@ class BotCogTests(unittest.TestCase): ### Mocking coroutines -By default, the `unittest.mock.Mock` and `unittest.mock.MagicMock` classes cannot mock coroutines, since the `__call__` method they provide is synchronous. In anticipation of the `AsyncMock` that will be [introduced in Python 3.8](https://docs.python.org/3.9/whatsnew/3.8.html#unittest), we have added an `AsyncMock` helper to [`helpers.py`](/tests/helpers.py). Do note that this drop-in replacement only implements an asynchronous `__call__` method, not the additional assertions that will come with the new `AsyncMock` type in Python 3.8. +By default, the `unittest.mock.Mock` and `unittest.mock.MagicMock` classes cannot mock coroutines, since the `__call__` method they provide is synchronous. In anticipation of the `AsyncMock` that will be [introduced in Python 3.8](https://docs.python.org/3.9/whatsnew/3.8.html#unittest), we have added an `AsyncMock` helper to [`helpers.py`](/tests/helpers.py). Do note that this drop-in replacement only implements an asynchronous `__call__` method, not the additional assertions that will come with the new `AsyncMock` type in Python 3.8. ### Special mocks for some `discord.py` types To quote Ned Batchelder, Mock objects are "automatic chameleons". This means that they will happily allow the access to any attribute or method and provide a mocked value in return. One downside to this is that if the code you are testing gets the name of the attribute wrong, your mock object will not complain and the test may still pass. -In order to avoid that, we have defined a number of Mock types in [`helpers.py`](/tests/helpers.py) that follow the specifications of the actual Discord types they are mocking. This means that trying to access an attribute or method on a mocked object that does not exist on the equivalent `discord.py` object will result in an `AttributeError`. In addition, these mocks have some sensible defaults and **pass `isinstance` checks for the types they are mocking**. +In order to avoid that, we have defined a number of Mock types in [`helpers.py`](/tests/helpers.py) that follow the specifications of the actual Discord types they are mocking. This means that trying to access an attribute or method on a mocked object that does not exist on the equivalent `discord.py` object will result in an `AttributeError`. In addition, these mocks have some sensible defaults and **pass `isinstance` checks for the types they are mocking**. These special mocks are added when they are needed, so if you think it would be sensible to add another one, feel free to propose one in your PR. @@ -144,7 +144,7 @@ Finally, there are some considerations to make when writing tests, both for writ ### Test coverage is a starting point -Having test coverage is a good starting point for unit testing: If a part of your code was not covered by a test, we know that we have not tested it properly. The reverse is unfortunately not true: Even if the code we are testing has 100% branch coverage, it does not mean it's fully tested or guaranteed to work. +Having test coverage is a good starting point for unit testing: If a part of your code was not covered by a test, we know that we have not tested it properly. The reverse is unfortunately not true: Even if the code we are testing has 100% branch coverage, it does not mean it's fully tested or guaranteed to work. One problem is that 100% branch coverage may be misleading if we haven't tested our code against all the realistic input it may get in production. For instance, take a look at the following `member_information` function and the test we've written for it: @@ -169,7 +169,7 @@ class FunctionsTests(unittest.TestCase): If you were to run this test, not only would the function pass the test, `coverage.py` will also tell us that the test provides 100% branch coverage for the function. Can you spot the bug the test suite did not catch? -The problem here is that we have only tested our function with a member object that had `None` for the `member.joined` attribute. This means that `member.joined.stfptime("%d-%m-%Y")` was never executed during our test, leading to us missing the spelling mistake in `stfptime` (it should be `strftime`). +The problem here is that we have only tested our function with a member object that had `None` for the `member.joined` attribute. This means that `member.joined.stfptime("%d-%m-%Y")` was never executed during our test, leading to us missing the spelling mistake in `stfptime` (it should be `strftime`). Adding another test would not increase the test coverage we have, but it does ensure that we'll notice that this function can fail with realistic data: diff --git a/tests/base.py b/tests/base.py index 42174e911..d99b9ac31 100644 --- a/tests/base.py +++ b/tests/base.py @@ -31,7 +31,7 @@ class LoggingTestsMixin: """ @contextmanager - def assertNotLogs(self, logger=None, level=None, msg=None): + def assertNotLogs(self, logger=None, level=None, msg=None): # noqa: N802 """ Asserts that no logs of `level` and higher were emitted by `logger`. @@ -81,7 +81,7 @@ class LoggingTestsMixin: class CommandTestCase(unittest.IsolatedAsyncioTestCase): """TestCase with additional assertions that are useful for testing Discord commands.""" - async def assertHasPermissionsCheck( + async def assertHasPermissionsCheck( # noqa: N802 self, cmd: commands.Command, permissions: Dict[str, bool], diff --git a/tests/bot/cogs/sync/test_base.py b/tests/bot/cogs/sync/test_base.py index fe0594efe..6ee9dfda6 100644 --- a/tests/bot/cogs/sync/test_base.py +++ b/tests/bot/cogs/sync/test_base.py @@ -84,7 +84,7 @@ class SyncerSendPromptTests(unittest.IsolatedAsyncioTestCase): method.assert_called_once_with(constants.Channels.dev_core) - async def test_send_prompt_returns_None_if_channel_fetch_fails(self): + async def test_send_prompt_returns_none_if_channel_fetch_fails(self): """None should be returned if there's an HTTPException when fetching the channel.""" self.bot.get_channel.return_value = None self.bot.fetch_channel.side_effect = discord.HTTPException(mock.MagicMock(), "test error!") diff --git a/tests/bot/cogs/test_cogs.py b/tests/bot/cogs/test_cogs.py new file mode 100644 index 000000000..39f6492cb --- /dev/null +++ b/tests/bot/cogs/test_cogs.py @@ -0,0 +1,80 @@ +"""Test suite for general tests which apply to all cogs.""" + +import importlib +import pkgutil +import typing as t +import unittest +from collections import defaultdict +from types import ModuleType +from unittest import mock + +from discord.ext import commands + +from bot import cogs + + +class CommandNameTests(unittest.TestCase): + """Tests for shadowing command names and aliases.""" + + @staticmethod + def walk_commands(cog: commands.Cog) -> t.Iterator[commands.Command]: + """An iterator that recursively walks through `cog`'s commands and subcommands.""" + # Can't use Bot.walk_commands() or Cog.get_commands() cause those are instance methods. + for command in cog.__cog_commands__: + if command.parent is None: + yield command + if isinstance(command, commands.GroupMixin): + # Annoyingly it returns duplicates for each alias so use a set to fix that + yield from set(command.walk_commands()) + + @staticmethod + def walk_modules() -> t.Iterator[ModuleType]: + """Yield imported modules from the bot.cogs subpackage.""" + def on_error(name: str) -> t.NoReturn: + raise ImportError(name=name) + + # The mock prevents asyncio.get_event_loop() from being called. + with mock.patch("discord.ext.tasks.loop"): + for module in pkgutil.walk_packages(cogs.__path__, "bot.cogs.", onerror=on_error): + if not module.ispkg: + yield importlib.import_module(module.name) + + @staticmethod + def walk_cogs(module: ModuleType) -> t.Iterator[commands.Cog]: + """Yield all cogs defined in an extension.""" + for obj in module.__dict__.values(): + # Check if it's a class type cause otherwise issubclass() may raise a TypeError. + is_cog = isinstance(obj, type) and issubclass(obj, commands.Cog) + if is_cog and obj.__module__ == module.__name__: + yield obj + + @staticmethod + def get_qualified_names(command: commands.Command) -> t.List[str]: + """Return a list of all qualified names, including aliases, for the `command`.""" + names = [f"{command.full_parent_name} {alias}".strip() for alias in command.aliases] + names.append(command.qualified_name) + + return names + + def get_all_commands(self) -> t.Iterator[commands.Command]: + """Yield all commands for all cogs in all extensions.""" + for module in self.walk_modules(): + for cog in self.walk_cogs(module): + for cmd in self.walk_commands(cog): + yield cmd + + def test_names_dont_shadow(self): + """Names and aliases of commands should be unique.""" + all_names = defaultdict(list) + for cmd in self.get_all_commands(): + func_name = f"{cmd.module}.{cmd.callback.__qualname__}" + + for name in self.get_qualified_names(cmd): + with self.subTest(cmd=func_name, name=name): + if name in all_names: + conflicts = ", ".join(all_names.get(name, "")) + self.fail( + f"Name '{name}' of the command {func_name} conflicts with {conflicts}." + ) + + all_names[name].append(func_name) diff --git a/tests/bot/cogs/test_information.py b/tests/bot/cogs/test_information.py index 5693d2946..3c26374f5 100644 --- a/tests/bot/cogs/test_information.py +++ b/tests/bot/cogs/test_information.py @@ -45,10 +45,9 @@ class InformationCogTests(unittest.TestCase): _, kwargs = self.ctx.send.call_args embed = kwargs.pop('embed') - self.assertEqual(embed.title, "Role information") + self.assertEqual(embed.title, "Role information (Total 1 role)") self.assertEqual(embed.colour, discord.Colour.blurple()) - self.assertEqual(embed.description, f"`{self.moderator_role.id}` - {self.moderator_role.mention}\n") - self.assertEqual(embed.footer.text, "Total roles: 1") + self.assertEqual(embed.description, f"\n`{self.moderator_role.id}` - {self.moderator_role.mention}\n") def test_role_info_command(self): """Tests the `role info` command.""" diff --git a/tests/bot/cogs/test_snekbox.py b/tests/bot/cogs/test_snekbox.py index 9cd7f0154..1dec0ccaf 100644 --- a/tests/bot/cogs/test_snekbox.py +++ b/tests/bot/cogs/test_snekbox.py @@ -1,11 +1,13 @@ import asyncio import logging import unittest -from unittest.mock import AsyncMock, MagicMock, Mock, call, patch +from unittest.mock import AsyncMock, MagicMock, Mock, call, create_autospec, patch +from discord.ext import commands + +from bot import constants from bot.cogs import snekbox from bot.cogs.snekbox import Snekbox -from bot.constants import URLs from tests.helpers import MockBot, MockContext, MockMessage, MockReaction, MockUser @@ -23,7 +25,7 @@ class SnekboxTests(unittest.IsolatedAsyncioTestCase): self.assertEqual(await self.cog.post_eval("import random"), "return") self.bot.http_session.post.assert_called_with( - URLs.snekbox_eval_api, + constants.URLs.snekbox_eval_api, json={"input": "import random"}, raise_for_status=True ) @@ -43,10 +45,10 @@ class SnekboxTests(unittest.IsolatedAsyncioTestCase): self.assertEqual( await self.cog.upload_output("My awesome output"), - URLs.paste_service.format(key=key) + constants.URLs.paste_service.format(key=key) ) self.bot.http_session.post.assert_called_with( - URLs.paste_service.format(key="documents"), + constants.URLs.paste_service.format(key="documents"), data="My awesome output", raise_for_status=True ) @@ -89,15 +91,15 @@ class SnekboxTests(unittest.IsolatedAsyncioTestCase): self.assertEqual(actual, expected) @patch('bot.cogs.snekbox.Signals', side_effect=ValueError) - def test_get_results_message_invalid_signal(self, mock_Signals: Mock): + def test_get_results_message_invalid_signal(self, mock_signals: Mock): self.assertEqual( self.cog.get_results_message({'stdout': '', 'returncode': 127}), ('Your eval job has completed with return code 127', '') ) @patch('bot.cogs.snekbox.Signals') - def test_get_results_message_valid_signal(self, mock_Signals: Mock): - mock_Signals.return_value.name = 'SIGTEST' + def test_get_results_message_valid_signal(self, mock_signals: Mock): + mock_signals.return_value.name = 'SIGTEST' self.assertEqual( self.cog.get_results_message({'stdout': '', 'returncode': 127}), ('Your eval job has completed with return code 127 (SIGTEST)', '') @@ -279,11 +281,14 @@ class SnekboxTests(unittest.IsolatedAsyncioTestCase): """Test that the continue_eval function does continue if required conditions are met.""" ctx = MockContext(message=MockMessage(add_reaction=AsyncMock(), clear_reactions=AsyncMock())) response = MockMessage(delete=AsyncMock()) - new_msg = MockMessage(content='!e NewCode') + new_msg = MockMessage() self.bot.wait_for.side_effect = ((None, new_msg), None) + expected = "NewCode" + self.cog.get_code = create_autospec(self.cog.get_code, spec_set=True, return_value=expected) actual = await self.cog.continue_eval(ctx, response) - self.assertEqual(actual, 'NewCode') + self.cog.get_code.assert_awaited_once_with(new_msg) + self.assertEqual(actual, expected) self.bot.wait_for.assert_has_awaits( ( call('message_edit', check=partial_mock(snekbox.predicate_eval_message_edit, ctx), timeout=10), @@ -302,6 +307,32 @@ class SnekboxTests(unittest.IsolatedAsyncioTestCase): self.assertEqual(actual, None) ctx.message.clear_reactions.assert_called_once() + async def test_get_code(self): + """Should return 1st arg (or None) if eval cmd in message, otherwise return full content.""" + prefix = constants.Bot.prefix + subtests = ( + (self.cog.eval_command, f"{prefix}{self.cog.eval_command.name} print(1)", "print(1)"), + (self.cog.eval_command, f"{prefix}{self.cog.eval_command.name}", None), + (MagicMock(spec=commands.Command), f"{prefix}tags get foo"), + (None, "print(123)") + ) + + for command, content, *expected_code in subtests: + if not expected_code: + expected_code = content + else: + [expected_code] = expected_code + + with self.subTest(content=content, expected_code=expected_code): + self.bot.get_context.reset_mock() + self.bot.get_context.return_value = MockContext(command=command) + message = MockMessage(content=content) + + actual_code = await self.cog.get_code(message) + + self.bot.get_context.assert_awaited_once_with(message) + self.assertEqual(actual_code, expected_code) + def test_predicate_eval_message_edit(self): """Test the predicate_eval_message_edit function.""" msg0 = MockMessage(id=1, content='abc') diff --git a/tests/bot/test_utils.py b/tests/bot/test_utils.py deleted file mode 100644 index d7bcc3ba6..000000000 --- a/tests/bot/test_utils.py +++ /dev/null @@ -1,37 +0,0 @@ -import unittest - -from bot import utils - - -class CaseInsensitiveDictTests(unittest.TestCase): - """Tests for the `CaseInsensitiveDict` container.""" - - def test_case_insensitive_key_access(self): - """Tests case insensitive key access and storage.""" - instance = utils.CaseInsensitiveDict() - - key = 'LEMON' - value = 'trees' - - instance[key] = value - self.assertIn(key, instance) - self.assertEqual(instance.get(key), value) - self.assertEqual(instance.get(key.casefold()), value) - self.assertEqual(instance.pop(key.casefold()), value) - self.assertNotIn(key, instance) - self.assertNotIn(key.casefold(), instance) - - instance.setdefault(key, value) - del instance[key] - self.assertNotIn(key, instance) - - def test_initialization_from_kwargs(self): - """Tests creating the dictionary from keyword arguments.""" - instance = utils.CaseInsensitiveDict({'FOO': 'bar'}) - self.assertEqual(instance['foo'], 'bar') - - def test_update_from_other_mapping(self): - """Tests updating the dictionary from another mapping.""" - instance = utils.CaseInsensitiveDict() - instance.update({'FOO': 'bar'}) - self.assertEqual(instance['foo'], 'bar') |