From c4213744c18be23e3e4484f126ae0b2d0eba4437 Mon Sep 17 00:00:00 2001 From: Sebastiaan Zeeff <33516116+SebastiaanZ@users.noreply.github.com> Date: Wed, 2 Oct 2019 16:59:03 +0200 Subject: Migrate pytest to unittest After a discussion in the core developers channel, we have decided to migrate from `pytest` to `unittest` as the testing framework. This commit sets up the repository to use `unittest` and migrates the first couple of tests files to the new framework. What I have done to migrate to `unitest`: - Removed all `pytest` test files, since they are incompatible. - Removed `pytest`-related dependencies from the Pipfile. - Added `coverage.py` to the Pipfile dev-packages and relocked. - Added convenience scripts to Pipfile for running the test suite. - Adjust to `azure-pipelines.yml` to use `coverage.py` and `unittest`. - Migrated four test files from `pytest` to `unittest` format. In addition, I've added five helper Mock subclasses in `helpers.py` and created a `TestCase` subclass in `base.py` to add an assertion that asserts that no log records were logged within the context of the context manager. Obviously, these new utility functions and classes are fully tested in their respective `test_` files. Finally, I've started with an introductory guide for writing tests for our bot in `README.md`. --- tests/base.py | 70 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 70 insertions(+) create mode 100644 tests/base.py (limited to 'tests/base.py') diff --git a/tests/base.py b/tests/base.py new file mode 100644 index 000000000..625dcc0a8 --- /dev/null +++ b/tests/base.py @@ -0,0 +1,70 @@ +import logging +import unittest +from contextlib import contextmanager + + +class _CaptureLogHandler(logging.Handler): + """ + A logging handler capturing all (raw and formatted) logging output. + """ + + def __init__(self): + super().__init__() + self.records = [] + + def flush(self): + pass + + def emit(self, record): + self.records.append(record) + + +class LoggingTestCase(unittest.TestCase): + """TestCase subclass that adds more logging assertion tools.""" + + @contextmanager + def assertNotLogs(self, logger=None, level=None, msg=None): + """ + Asserts that no logs of `level` and higher were emitted by `logger`. + + You can specify a specific `logger`, the minimum `logging` level we want to watch and a + custom `msg` to be added to the `AssertionError` if thrown. If the assertion fails, the + recorded log records will be outputted with the `AssertionError` message. The context + manager does not yield a live `look` into the logging records, since we use this context + manager when we're testing under the assumption that no log records will be emitted. + """ + if not isinstance(logger, logging.Logger): + logger = logging.getLogger(logger) + + if level: + level = logging._nameToLevel.get(level, level) + else: + level = logging.INFO + + handler = _CaptureLogHandler() + old_handlers = logger.handlers[:] + old_level = logger.level + old_propagate = logger.propagate + + logger.handlers = [handler] + logger.setLevel(level) + logger.propagate = False + + try: + yield + except Exception as exc: + raise exc + finally: + logger.handlers = old_handlers + logger.propagate = old_propagate + logger.setLevel(old_level) + + if handler.records: + level_name = logging.getLevelName(level) + n_logs = len(handler.records) + base_message = f"{n_logs} logs of {level_name} or higher were triggered on {logger.name}:\n" + records = [str(record) for record in handler.records] + record_message = "\n".join(records) + standard_message = self._truncateMessage(base_message, record_message) + msg = self._formatMessage(msg, standard_message) + self.fail(msg) -- cgit v1.2.3 From 70fb1315199d83f53d24b0772c940e66422d4cd4 Mon Sep 17 00:00:00 2001 From: Sebastiaan Zeeff <33516116+SebastiaanZ@users.noreply.github.com> Date: Fri, 11 Oct 2019 18:11:43 +0200 Subject: Add tests for tests.base I forgot to test some aspects of the `tests.base` module, including some branches of the `self.assertNotLogs` method. I've corrected that by including a couple of tests. I also removed the test result publishing from the Azure pipeline, since I've not configured an XML test runner yet. The coverage report is still published, of course and test output will be available in standard out, so information is readily available. --- azure-pipelines.yml | 7 ------- tests/base.py | 3 --- tests/test_base.py | 32 +++++++++++++++++++++++++++++++- 3 files changed, 31 insertions(+), 11 deletions(-) (limited to 'tests/base.py') diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 3d0932398..15470f9be 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -43,13 +43,6 @@ jobs: codeCoverageTool: Cobertura summaryFileLocation: coverage.xml - - task: PublishTestResults@2 - displayName: 'Publish Test Results' - condition: succeededOrFailed() - inputs: - testResultsFiles: junit.xml - testRunTitle: 'Bot Test results' - - job: build displayName: 'Build & Push Container' dependsOn: 'test' diff --git a/tests/base.py b/tests/base.py index 625dcc0a8..029a249ed 100644 --- a/tests/base.py +++ b/tests/base.py @@ -12,9 +12,6 @@ class _CaptureLogHandler(logging.Handler): super().__init__() self.records = [] - def flush(self): - pass - def emit(self, record): self.records.append(record) diff --git a/tests/test_base.py b/tests/test_base.py index b7c1e0037..a16e2af8f 100644 --- a/tests/test_base.py +++ b/tests/test_base.py @@ -3,7 +3,7 @@ import unittest import unittest.mock -from tests.base import LoggingTestCase +from tests.base import LoggingTestCase, _CaptureLogHandler class LoggingTestCaseTests(unittest.TestCase): @@ -59,3 +59,33 @@ class LoggingTestCaseTests(unittest.TestCase): self.assertEqual(self.log.handlers, old_handlers) self.assertEqual(self.log.level, old_level) self.assertEqual(self.log.propagate, old_propagate) + + def test_logging_test_case_works_with_logger_instance(self): + """Test if the LoggingTestCase captures logging for provided logger.""" + log = logging.getLogger("new_logger") + with self.assertRaises(AssertionError): + with LoggingTestCase.assertNotLogs(self, logger=log): + log.info("Hello, this should raise an AssertionError") + + def test_logging_test_case_respects_alternative_logger(self): + """Test if LoggingTestCase only checks the provided logger.""" + log_one = logging.getLogger("log one") + log_two = logging.getLogger("log two") + with LoggingTestCase.assertNotLogs(self, logger=log_one): + log_two.info("Hello, this should not raise an AssertionError") + + def test_logging_test_case_respects_logging_level(self): + """Test if LoggingTestCase does not raise for a logging level lower than provided.""" + with LoggingTestCase.assertNotLogs(self, level=logging.CRITICAL): + self.log.info("Hello, this should raise an AssertionError") + + def test_capture_log_handler_default_initialization(self): + """Test if the _CaptureLogHandler is initialized properly.""" + handler = _CaptureLogHandler() + self.assertFalse(handler.records) + + def test_capture_log_handler_saves_record_on_emit(self): + """Test if the _CaptureLogHandler saves the log record when it's emitted.""" + handler = _CaptureLogHandler() + handler.emit("Log message") + self.assertIn("Log message", handler.records) -- cgit v1.2.3