diff --git a/README.md b/README.md index 03c73d6..9e1e076 100644 --- a/README.md +++ b/README.md @@ -64,6 +64,7 @@ the file: * `# N: ` - we expect a mypy note message * `# W: ` - we expect a mypy warning message * `# E: ` - we expect a mypy error message +* `# F: ` - we expect a mypy fatal error message * `# R: ` - we expect a mypy note message `Revealed type is ''`. This is useful to easily check `reveal_type` output: ```python @@ -73,13 +74,15 @@ the file: reveal_type(456) # R: Literal[456]? ``` -## mypy Error Codes +## mypy Error Code Matching The algorithm matching messages parses mypy error code both in the -output generated by mypy and in the Python comments. If both the mypy -output and the Python comment contain an error code, then the codes -must match. So the following test case expects that mypy writes out an -``assignment`` error code: +output generated by mypy and in the Python comments. + +If both the mypy output and the Python comment contain an error code +and a full message, then the messages and the error codes must +match. The following test case expects that mypy writes out an +``assignment`` error code and a specific error message: ``` python @pytest.mark.mypy_testing @@ -89,7 +92,27 @@ def mypy_test_invalid_assignment() -> None: ``` If the Python comment does not contain an error code, then the error -code written out by mypy (if any) is simply ignored. +code written out by mypy (if any) is ignored. The following test case +expects a specific error message from mypy, but ignores the error code +produced by mypy: + +``` python +@pytest.mark.mypy_testing +def mypy_test_invalid_assignment() -> None: + foo = "abc" + foo = 123 # E: Incompatible types in assignment (expression has type "int", variable has type "str") +``` + +If the Python comment specifies only an error code, then the message +written out by mypy is ignored, i.e., the following test case checks +that mypy reports an `assignment` error: + +``` python +@pytest.mark.mypy_testing +def mypy_test_invalid_assignment() -> None: + foo = "abc" + foo = 123 # E: [assignment] +``` ## Skipping and Expected Failures @@ -114,9 +137,15 @@ decorators are extracted from the ast. # Changelog +## v0.1.1 + +* Compare just mypy error codes if given and no error message is given + in the test case Python comment ([#36][i36], [#43][p43]) + ## v0.1.0 -* Implement support for flexible matching of mypy error codes (towards [#36][i36], [#41][p41]) +* Implement support for flexible matching of mypy error codes (towards + [#36][i36], [#41][p41]) * Add support for pytest 7.2.x ([#42][p42]) * Add support for mypy 1.0.x ([#42][p42]) * Add support for Python 3.11 ([#42][p42]) @@ -192,3 +221,4 @@ decorators are extracted from the ast. [p40]: https://github.com/davidfritzsche/pytest-mypy-testing/pull/40 [p41]: https://github.com/davidfritzsche/pytest-mypy-testing/pull/41 [p42]: https://github.com/davidfritzsche/pytest-mypy-testing/pull/42 +[p43]: https://github.com/davidfritzsche/pytest-mypy-testing/pull/43 diff --git a/pytest.ini b/pytest.ini index 0a01c47..2bfb95e 100644 --- a/pytest.ini +++ b/pytest.ini @@ -2,9 +2,12 @@ # SPDX-License-Identifier: CC0-1.0 [pytest] testpaths = - tests mypy_tests + tests mypy_tests pytest_mypy_testing addopts = + --doctest-continue-on-failure + --doctest-modules --failed-first + --pyargs --showlocals -p no:mypy-testing doctest_optionflags = NORMALIZE_WHITESPACE IGNORE_EXCEPTION_DETAIL ELLIPSIS diff --git a/src/pytest_mypy_testing/message.py b/src/pytest_mypy_testing/message.py index 62f1587..ff2ecd5 100644 --- a/src/pytest_mypy_testing/message.py +++ b/src/pytest_mypy_testing/message.py @@ -22,6 +22,7 @@ class Severity(enum.Enum): NOTE = 1 WARNING = 2 ERROR = 3 + FATAL = 4 @classmethod def from_string(cls, string: str) -> "Severity": @@ -39,6 +40,7 @@ def __repr__(self) -> str: "N": Severity.NOTE, "W": Severity.WARNING, "E": Severity.ERROR, + "F": Severity.FATAL, } _COMMENT_MESSAGES = frozenset( @@ -55,11 +57,11 @@ def __repr__(self) -> str: class Message: """Mypy message""" - filename: str - lineno: int - colno: Optional[int] - severity: Severity - message: str + filename: str = "" + lineno: int = 0 + colno: Optional[int] = None + severity: Severity = Severity.ERROR + message: str = "" revealed_type: Optional[str] = None error_code: Optional[str] = None @@ -72,19 +74,22 @@ class Message: COMMENT_RE = re.compile( r"^(?:# *type: *ignore *)?(?:# *)?" r"(?P[RENW]):" - r"((?P\d+):)? *" - r"(?P[^#]*?)" - r"(?: +\[(?P[^\]]*)\])?" + r"((?P\d+):)?" + r" *" + r"(?P[^#]*)" r"(?:#.*?)?$" ) + MESSAGE_AND_ERROR_CODE = re.compile( + r"(?P[^\[][^#]*?)" r" +" r"\[(?P[^\]]*)\]" + ) + OUTPUT_RE = re.compile( r"^(?P([a-zA-Z]:)?[^:]+):" r"(?P[0-9]+):" r"((?P[0-9]+):)?" r" *(?P(error|note|warning)):" - r"(?P.*?)" - r"(?: +\[(?P[^\]]*)\])?" + r"(?P.*?)" r"$" ) @@ -128,7 +133,7 @@ def astuple(self, *, normalized: bool = False) -> "Message.TupleType": >>> m = Message("foo.py", 1, 1, Severity.NOTE, 'Revealed type is "float"') >>> m.astuple() - ('foo.py', 1, 1, Severity.NOTE, 'Revealed type is "float"', 'float') + ('foo.py', 1, 1, Severity.NOTE, 'Revealed type is "float"', 'float', None) """ return ( self.filename, @@ -144,7 +149,11 @@ def is_comment(self) -> bool: return (self.severity, self.message) in _COMMENT_MESSAGES def _as_short_tuple( - self, *, normalized: bool = False, default_error_code: Optional[str] = None + self, + *, + normalized: bool = False, + default_message: str = "", + default_error_code: Optional[str] = None, ) -> "Message.TupleType": if normalized: message = self.normalized_message @@ -155,32 +164,73 @@ def _as_short_tuple( self.lineno, None, self.severity, - message, + message or default_message, self.revealed_type, self.error_code or default_error_code, ) + def __hash__(self) -> int: + t = (self.filename, self.lineno, self.severity, self.revealed_type) + return hash(t) + def __eq__(self, other): + """Compare if *self* and *other* are equal. + + Returns `True` if *other* is a :obj:`Message:` object + considered to be equal to *self*. + + >>> Message() == Message() + True + >>> Message(error_code="baz") == Message(message="some text", error_code="baz") + True + >>> Message(message="some text") == Message(message="some text", error_code="baz") + True + + >>> Message() == Message(message="some text", error_code="baz") + False + >>> Message(error_code="baz") == Message(error_code="bax") + False + """ if isinstance(other, Message): default_error_code = self.error_code or other.error_code - if self.colno is None or other.colno is None: - a = self._as_short_tuple( - normalized=True, default_error_code=default_error_code - ) - b = other._as_short_tuple( - normalized=True, default_error_code=default_error_code + if self.error_code and other.error_code: + default_message = self.normalized_message or other.normalized_message + else: + default_message = "" + + def to_tuple(m: Message): + return m._as_short_tuple( + normalized=True, + default_message=default_message, + default_error_code=default_error_code, ) - return a == b + + if self.colno is None or other.colno is None: + return to_tuple(self) == to_tuple(other) else: return self.astuple(normalized=True) == other.astuple(normalized=True) else: return NotImplemented - def __hash__(self) -> int: - return hash(self._as_short_tuple(normalized=True)) - def __str__(self) -> str: - return f"{self._prefix} {self.severity.name.lower()}: {self.message}" + return self.to_string(prefix=f"{self._prefix} ") + + def to_string(self, prefix: Optional[str] = None) -> str: + prefix = prefix or f"{self._prefix} " + error_code = f" [{self.error_code}]" if self.error_code else "" + return f"{prefix}{self.severity.name.lower()}: {self.message}{error_code}" + + @classmethod + def __split_message_and_error_code(cls, msg: str) -> Tuple[str, Optional[str]]: + msg = msg.strip() + if msg.startswith("[") and msg.endswith("]"): + return "", msg[1:-1] + else: + m = cls.MESSAGE_AND_ERROR_CODE.fullmatch(msg) + if m: + return m.group("message"), m.group("error_code") + else: + return msg, None @classmethod def from_comment( @@ -189,13 +239,17 @@ def from_comment( """Create message object from Python *comment*. >>> Message.from_comment("foo.py", 1, "R: foo") - Message(filename='foo.py', lineno=1, colno=None, severity=Severity.NOTE, message="Revealed type is 'foo'", revealed_type='foo') + Message(filename='foo.py', lineno=1, colno=None, severity=Severity.NOTE, message="Revealed type is 'foo'", revealed_type='foo', error_code=None) + >>> Message.from_comment("foo.py", 1, "E: [assignment]") + Message(filename='foo.py', lineno=1, colno=None, severity=Severity.ERROR, message='', revealed_type=None, error_code='assignment') """ m = cls.COMMENT_RE.match(comment.strip()) if not m: raise ValueError("Not a valid mypy message comment") colno = int(m.group("colno")) if m.group("colno") else None - message = m.group("message").strip() + message, error_code = cls.__split_message_and_error_code( + m.group("message_and_error_code") + ) if m.group("severity") == "R": revealed_type = message message = "Revealed type is {!r}".format(message) @@ -208,7 +262,7 @@ def from_comment( severity=Severity.from_string(m.group("severity")), message=message, revealed_type=revealed_type, - error_code=m.group("error_code") or None, + error_code=error_code, ) @classmethod @@ -216,29 +270,37 @@ def from_output(cls, line: str) -> "Message": """Create message object from mypy output line. >>> m = Message.from_output("z.py:1: note: bar") - >>> (m.lineno, m.colno, m.severity, m.message, m.revealed_type) - (1, None, Severity.NOTE, 'bar', None) + >>> (m.lineno, m.colno, m.severity, m.message, m.revealed_type, m.error_code) + (1, None, Severity.NOTE, 'bar', None, None) >>> m = Message.from_output("z.py:1:13: note: bar") - >>> (m.lineno, m.colno, m.severity, m.message, m.revealed_type) - (1, 13, Severity.NOTE, 'bar', None) + >>> (m.lineno, m.colno, m.severity, m.message, m.revealed_type, m.error_code) + (1, 13, Severity.NOTE, 'bar', None, None) >>> m = Message.from_output("z.py:1: note: Revealed type is 'bar'") - >>> (m.lineno, m.colno, m.severity, m.message, m.revealed_type) - (1, None, Severity.NOTE, "Revealed type is 'bar'", 'bar') + >>> (m.lineno, m.colno, m.severity, m.message, m.revealed_type, m.error_code) + (1, None, Severity.NOTE, "Revealed type is 'bar'", 'bar', None) >>> m = Message.from_output('z.py:1: note: Revealed type is "bar"') - >>> (m.lineno, m.colno, m.severity, m.message, m.revealed_type) - (1, None, Severity.NOTE, 'Revealed type is "bar"', 'bar') + >>> (m.lineno, m.colno, m.severity, m.message, m.revealed_type, m.error_code) + (1, None, Severity.NOTE, 'Revealed type is "bar"', 'bar', None) + + >>> m = Message.from_output("z.py:1:13: error: bar [baz]") + >>> (m.lineno, m.colno, m.severity, m.message, m.revealed_type, m.error_code) + (1, 13, Severity.ERROR, 'bar', None, 'baz') """ m = cls.OUTPUT_RE.match(line) if not m: raise ValueError("Not a valid mypy message") + message, error_code = cls.__split_message_and_error_code( + m.group("message_and_error_code") + ) return cls( os.path.abspath(m.group("fname")), lineno=int(m.group("lineno")), colno=int(m.group("colno")) if m.group("colno") else None, severity=Severity[m.group("severity").upper()], - message=m.group("message").strip(), + message=message, + error_code=error_code, ) diff --git a/src/pytest_mypy_testing/output_processing.py b/src/pytest_mypy_testing/output_processing.py index 08e272d..ddf9235 100644 --- a/src/pytest_mypy_testing/output_processing.py +++ b/src/pytest_mypy_testing/output_processing.py @@ -46,9 +46,7 @@ def __post_init__(self) -> None: def _fmt(msg: Message, actual_expected: str = "", *, indent: str = " ") -> str: if actual_expected: actual_expected += ": " - return ( - f"{indent}{actual_expected}{msg.severity.name.lower()}: {msg.message}" - ) + return msg.to_string(prefix=f"{indent}{actual_expected}") if not any([self.actual, self.expected]): raise ValueError("At least one of actual and expected must be given") diff --git a/src/pytest_mypy_testing/plugin.py b/src/pytest_mypy_testing/plugin.py index 10bf5c9..fed5ae9 100644 --- a/src/pytest_mypy_testing/plugin.py +++ b/src/pytest_mypy_testing/plugin.py @@ -172,6 +172,7 @@ def _run_mypy(self, filename: Union[pathlib.Path, os.PathLike, str]) -> MypyResu "--no-silence-site-packages", "--no-warn-unused-configs", "--show-column-numbers", + "--show-error-codes", "--show-traceback", str(filename), ] diff --git a/tests/test_basics.mypy-testing b/tests/test_basics.mypy-testing index e004529..2272972 100644 --- a/tests/test_basics.mypy-testing +++ b/tests/test_basics.mypy-testing @@ -11,6 +11,39 @@ def mypy_test_invalid_assginment(): foo = 123 # E: Incompatible types in assignment (expression has type "int", variable has type "str") +@pytest.mark.mypy_testing +def mypy_test_invalid_assginment_with_error_code(): + foo = "abc" + foo = 123 # E: Incompatible types in assignment (expression has type "int", variable has type "str") [assignment] + + +@pytest.mark.xfail +@pytest.mark.mypy_testing +def mypy_test_invalid_assginment_with_error_code__message_does_not_match(): + foo = "abc" + foo = 123 # E: Invalid assignment [assignment] + + +@pytest.mark.mypy_testing +def mypy_test_invalid_assginment_only_error_code(): + foo = "abc" + foo = 123 # E: [assignment] + + +@pytest.mark.xfail +@pytest.mark.mypy_testing +def mypy_test_invalid_assginment_only_error_code__error_code_does_not_match(): + foo = "abc" + foo = 123 # E: [baz] + + +@pytest.mark.xfail +@pytest.mark.mypy_testing +def mypy_test_invalid_assginment_no_message_and_no_error_code(): + foo = "abc" + foo = 123 # E: + + @pytest.mark.mypy_testing def mypy_test_use_reveal_type(): reveal_type(123) # N: Revealed type is 'Literal[123]?'