test: cover the validator, the directory and the site

tests/test_coverage.py

@@ -0,0 +1,50 @@
+"""The validator against the spec it implements.
+
+These are the tests that notice when the spec moves and the validator doesn't.
+"""
+
+from pathlib import Path
+import re
+
+from mews.lint import CHECKS, NOT_CHECKED
+
+SPEC = Path(__file__).resolve().parent.parent / "doc" / "SPEC.md"
+KEYWORD = re.compile(r"\b(MUST NOT|MUST|SHOULD NOT|SHOULD)\b")
+HEADING = re.compile(r"^#{2,3} (\d+(?:\.\d+)?)[. ]")
+
+
+def sections_with_rules() -> set[str]:
+    """Every section of SPEC.md 3 to 7 that states a MUST or a SHOULD."""
+    found: set[str] = set()
+    current = ""
+    for line in SPEC.read_text().splitlines():
+        heading = HEADING.match(line)
+        if heading:
+            current = heading.group(1)
+            continue
+        if current[:1] in "34567" and KEYWORD.search(line):
+            found.add(current)
+    return found
+
+
+def test_every_rule_in_the_spec_is_checked_or_listed_as_unchecked():
+    covered = {check.section for check in CHECKS} | set(NOT_CHECKED)
+    missing = sections_with_rules() - covered
+    assert not missing, (
+        f"sections {sorted(missing)} of SPEC.md state rules that mews.lint "
+        "neither checks nor lists in NOT_CHECKED"
+    )
+
+
+def test_the_unchecked_list_says_why():
+    for section, reason in NOT_CHECKED.items():
+        assert len(reason) > 20, section
+
+
+def test_check_codes_are_unique():
+    codes = [check.code for check in CHECKS]
+    assert len(codes) == len(set(codes))
+
+
+def test_every_check_is_a_must_or_a_should():
+    assert {check.level for check in CHECKS} == {"must", "should"}