From c4213744c18be23e3e4484f126ae0b2d0eba4437 Mon Sep 17 00:00:00 2001 From: Sebastiaan Zeeff <33516116+SebastiaanZ@users.noreply.github.com> Date: Wed, 2 Oct 2019 16:59:03 +0200 Subject: Migrate pytest to unittest After a discussion in the core developers channel, we have decided to migrate from `pytest` to `unittest` as the testing framework. This commit sets up the repository to use `unittest` and migrates the first couple of tests files to the new framework. What I have done to migrate to `unitest`: - Removed all `pytest` test files, since they are incompatible. - Removed `pytest`-related dependencies from the Pipfile. - Added `coverage.py` to the Pipfile dev-packages and relocked. - Added convenience scripts to Pipfile for running the test suite. - Adjust to `azure-pipelines.yml` to use `coverage.py` and `unittest`. - Migrated four test files from `pytest` to `unittest` format. In addition, I've added five helper Mock subclasses in `helpers.py` and created a `TestCase` subclass in `base.py` to add an assertion that asserts that no log records were logged within the context of the context manager. Obviously, these new utility functions and classes are fully tested in their respective `test_` files. Finally, I've started with an introductory guide for writing tests for our bot in `README.md`. --- tests/test_base.py | 61 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 61 insertions(+) create mode 100644 tests/test_base.py (limited to 'tests/test_base.py') diff --git a/tests/test_base.py b/tests/test_base.py new file mode 100644 index 000000000..b7c1e0037 --- /dev/null +++ b/tests/test_base.py @@ -0,0 +1,61 @@ +import logging +import unittest +import unittest.mock + + +from tests.base import LoggingTestCase + + +class LoggingTestCaseTests(unittest.TestCase): + """Tests for the LoggingTestCase.""" + + @classmethod + def setUpClass(cls): + cls.log = logging.getLogger(__name__) + + def test_assert_not_logs_does_not_raise_with_no_logs(self): + """Test if LoggingTestCase.assertNotLogs does not raise when no logs were emitted.""" + try: + with LoggingTestCase.assertNotLogs(self, level=logging.DEBUG): + pass + except AssertionError: + self.fail("`self.assertNotLogs` raised an AssertionError when it should not!") + + @unittest.mock.patch("tests.base.LoggingTestCase.assertNotLogs") + def test_the_test_function_assert_not_logs_does_not_raise_with_no_logs(self, assertNotLogs): + """Test if test_assert_not_logs_does_not_raise_with_no_logs captures exception correctly.""" + assertNotLogs.return_value = iter([None]) + assertNotLogs.side_effect = AssertionError + + message = "`self.assertNotLogs` raised an AssertionError when it should not!" + with self.assertRaises(AssertionError, msg=message): + self.test_assert_not_logs_does_not_raise_with_no_logs() + + def test_assert_not_logs_raises_correct_assertion_error_when_logs_are_emitted(self): + """Test if LoggingTestCase.assertNotLogs raises AssertionError when logs were emitted.""" + msg_regex = ( + r"1 logs of DEBUG or higher were triggered on root:\n" + r'' + ) + with self.assertRaisesRegex(AssertionError, msg_regex): + with LoggingTestCase.assertNotLogs(self, level=logging.DEBUG): + self.log.debug("Log!") + + def test_assert_not_logs_reraises_unexpected_exception_in_managed_context(self): + """Test if LoggingTestCase.assertNotLogs reraises an unexpected exception.""" + with self.assertRaises(ValueError, msg="test exception"): + with LoggingTestCase.assertNotLogs(self, level=logging.DEBUG): + raise ValueError("test exception") + + def test_assert_not_logs_restores_old_logging_settings(self): + """Test if LoggingTestCase.assertNotLogs reraises an unexpected exception.""" + old_handlers = self.log.handlers[:] + old_level = self.log.level + old_propagate = self.log.propagate + + with LoggingTestCase.assertNotLogs(self, level=logging.DEBUG): + pass + + self.assertEqual(self.log.handlers, old_handlers) + self.assertEqual(self.log.level, old_level) + self.assertEqual(self.log.propagate, old_propagate) -- cgit v1.2.3 From 70fb1315199d83f53d24b0772c940e66422d4cd4 Mon Sep 17 00:00:00 2001 From: Sebastiaan Zeeff <33516116+SebastiaanZ@users.noreply.github.com> Date: Fri, 11 Oct 2019 18:11:43 +0200 Subject: Add tests for tests.base I forgot to test some aspects of the `tests.base` module, including some branches of the `self.assertNotLogs` method. I've corrected that by including a couple of tests. I also removed the test result publishing from the Azure pipeline, since I've not configured an XML test runner yet. The coverage report is still published, of course and test output will be available in standard out, so information is readily available. --- azure-pipelines.yml | 7 ------- tests/base.py | 3 --- tests/test_base.py | 32 +++++++++++++++++++++++++++++++- 3 files changed, 31 insertions(+), 11 deletions(-) (limited to 'tests/test_base.py') diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 3d0932398..15470f9be 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -43,13 +43,6 @@ jobs: codeCoverageTool: Cobertura summaryFileLocation: coverage.xml - - task: PublishTestResults@2 - displayName: 'Publish Test Results' - condition: succeededOrFailed() - inputs: - testResultsFiles: junit.xml - testRunTitle: 'Bot Test results' - - job: build displayName: 'Build & Push Container' dependsOn: 'test' diff --git a/tests/base.py b/tests/base.py index 625dcc0a8..029a249ed 100644 --- a/tests/base.py +++ b/tests/base.py @@ -12,9 +12,6 @@ class _CaptureLogHandler(logging.Handler): super().__init__() self.records = [] - def flush(self): - pass - def emit(self, record): self.records.append(record) diff --git a/tests/test_base.py b/tests/test_base.py index b7c1e0037..a16e2af8f 100644 --- a/tests/test_base.py +++ b/tests/test_base.py @@ -3,7 +3,7 @@ import unittest import unittest.mock -from tests.base import LoggingTestCase +from tests.base import LoggingTestCase, _CaptureLogHandler class LoggingTestCaseTests(unittest.TestCase): @@ -59,3 +59,33 @@ class LoggingTestCaseTests(unittest.TestCase): self.assertEqual(self.log.handlers, old_handlers) self.assertEqual(self.log.level, old_level) self.assertEqual(self.log.propagate, old_propagate) + + def test_logging_test_case_works_with_logger_instance(self): + """Test if the LoggingTestCase captures logging for provided logger.""" + log = logging.getLogger("new_logger") + with self.assertRaises(AssertionError): + with LoggingTestCase.assertNotLogs(self, logger=log): + log.info("Hello, this should raise an AssertionError") + + def test_logging_test_case_respects_alternative_logger(self): + """Test if LoggingTestCase only checks the provided logger.""" + log_one = logging.getLogger("log one") + log_two = logging.getLogger("log two") + with LoggingTestCase.assertNotLogs(self, logger=log_one): + log_two.info("Hello, this should not raise an AssertionError") + + def test_logging_test_case_respects_logging_level(self): + """Test if LoggingTestCase does not raise for a logging level lower than provided.""" + with LoggingTestCase.assertNotLogs(self, level=logging.CRITICAL): + self.log.info("Hello, this should raise an AssertionError") + + def test_capture_log_handler_default_initialization(self): + """Test if the _CaptureLogHandler is initialized properly.""" + handler = _CaptureLogHandler() + self.assertFalse(handler.records) + + def test_capture_log_handler_saves_record_on_emit(self): + """Test if the _CaptureLogHandler saves the log record when it's emitted.""" + handler = _CaptureLogHandler() + handler.emit("Log message") + self.assertIn("Log message", handler.records) -- cgit v1.2.3