scripts.audit_test_purity

Reference › scripts › audit_test_purity

Audits the test suite for forbidden patterns: mocks, stubs, fakes, monkeypatching, skips and banned-library imports. It enforces the real-infrastructure rule for tests. The banned library names come from banned_libraries through _banned_library_names. tests/scripts/test_audit_test_purity.py imports it, and it runs in the pre-commit battery and as a standalone check.

Invocation: main takes zero or more paths (test trees or individual files; default tests/), --fix to produce a fix report, and --banned-import-only PATH... to sweep extra trees for the banned-import category alone. The latter exists so production code can be checked for banned libraries without false positives on legitimate tooling that merely has names such as stub.

scan_file parses one file and returns violations grouped by category with line numbers. Name and attribute-chain matching (_name_matches, _attr_chain_matches, _normalize_pattern) catches both calls and attribute uses, and _check_banned_library_imports flags imports. _collect_py_files expands the given paths. main prints the violations and exits non-zero when any are found.

At a glance

Class diagram

No classes; see Functions.

Dependency diagram

API

Functions

Source

Permalink: https://gitlab.com/MSamy/orchestrata-ultimate-architecture/-/blob/a4cbc2bdf/scripts/audit_test_purity.py

#!/usr/bin/env python3
"""Audit test suite for forbidden patterns: mocks, stubs, fakes, monkeypatch, skips.

Scans tests/ directory and reports every file that uses forbidden patterns.
Returns non-zero exit code if any violations are found.

Usage:
    python scripts/audit_test_purity.py [--fix]
    python scripts/audit_test_purity.py tests/ --banned-import-only src/ scripts/
"""

import argparse
import ast
import sys
from functools import lru_cache
from pathlib import Path

from orchestrata_ultimate.core.banned_libraries import (
    BANNED_LIBRARIES,
    banned_names,
    is_banned,
)

# Repo root, resolved from this script's own location so the registry loads
# correctly regardless of the caller's cwd (AGENTS.md says "run from the repo
# root", but health_check.py and the pre-commit hook both subprocess this
# script, and a cwd assumption that holds for one caller and not the other is
# exactly the kind of environment-dependent gap RUL-014 exists to close).
_REPO_ROOT = Path(__file__).resolve().parents[1]


@lru_cache(maxsize=1)
def _banned_library_names() -> frozenset[str]:
    """The shared ``.claude/banned-libraries.txt`` registry, read through the
    ONE canonical parser (``core.banned_libraries``) rather than a second,
    independent copy of the format. A hardcoded second parser is the exact
    defect that module's own docstring documents twice already
    (``serialize_codebase_ast.py``'s ``_FORBIDDEN_IMPORTS = {"pyswip"}``;
    ``refactor/codemod/review.py``'s hardcoded ban tuple) — this checker is
    not adding a third. Cached because every ``scan_file`` call would
    otherwise re-read and re-parse the registry file once per test module."""
    return frozenset(banned_names(_REPO_ROOT / BANNED_LIBRARIES))


FORBIDDEN_PATTERNS = {
    "mock": [
        "MagicMock",
        "AsyncMock",
        "Mock",
        "patch(",
        "mock.patch",
        "unittest.mock",
        "pytest_mock",
        "mocker",
    ],
    "stub": [
        # Class names / instantiations only (not variable prefixes)
        "Fake(",
        "Stub(",
        "Dummy(",
        " Fake",
        " Stub",
        " Dummy",  # class definitions / type annotations
        "=Fake",
        "=Stub",
        "=Dummy",
        "_fake_",
        "fake_",
        "_stub_",
        "stub_",
    ],
    "monkeypatch": [
        "monkeypatch",
    ],
    "skip": [
        "pytest.skip",
        "pytest.mark.skip",
        "skipif",
        "pytest.mark.skipif",
        # Module-level conditional skip: silently greens the whole module when
        # the import is absent (found live 2026-07-29 guarding a DECLARED dep).
        "importorskip",
        "pytest.importorskip",
    ],
}


def _normalize_pattern(pattern: str) -> str:
    """Strip the source-context decorations a pattern carries for
    documentation purposes: a trailing ``(`` call marker, or a leading
    space/``=`` marking annotation/assignment context (e.g. ``"Fake("``,
    ``" Stub"``, ``"=Dummy"``). None of those characters can ever appear in
    an ``ast.Name.id`` or in a dotted attribute chain, so comparing the raw
    pattern text against either would never match — this recovers the bare
    identifier the pattern was written to catch."""
    return pattern.rstrip("(").lstrip(" =")


def _name_matches(name: str, pattern: str, *, is_call: bool = False) -> bool:
    """Match a bare identifier (``ast.Name.id``) against a forbidden pattern.

    Patterns ending in ``_`` (``"fake_"``, ``"_stub_"``) describe a
    name *prefix*, matched with ``startswith`` directly against the
    pattern (no extra ``_`` is appended — the pattern already carries
    one). All other patterns describe a symbol name and are matched
    against its normalized (paren/space/equals-stripped) form, either
    exactly or as a ``symbol_suffix`` prefix.

    A pattern written with a trailing ``(`` (``"patch("``, ``"Fake("``)
    requires the node to be the function of a real call — normalizing the
    paren away must not widen a call pattern to plain reads (a semver
    ``.patch`` field access was falsely flagged as mock usage; audit
    2026-07-10).
    """
    if pattern.endswith("(") and not is_call:
        return False
    if pattern.endswith("_"):
        return name.startswith(pattern)
    bare = _normalize_pattern(pattern)
    return name == bare or name.startswith(bare + "_")


def _attr_chain_matches(
    attr_chain: str, attr_name: str, pattern: str, *, is_call: bool = False
) -> bool:
    """Match an attribute access against a forbidden pattern.

    A *dotted* pattern (contains ``.``, e.g. ``"pytest.mark.skip"``,
    ``"mock.patch"``) is matched as a substring of the full dotted chain —
    that's the whole point of writing it dotted. A *bare* pattern
    (normalized down to a single identifier, e.g. ``"patch("`` ->
    ``"patch"``) is matched against just the immediate attribute name
    (``node.attr``) via ``_name_matches``, NOT as a substring of the whole
    chain: a naive chain-substring check would also match unrelated
    attributes that merely contain the pattern as a substring (e.g.
    ``"patch"`` inside ``obj.dispatch``). Call-marked patterns (trailing
    ``(``) additionally require the attribute to be genuinely called —
    the check must run on the ORIGINAL pattern, before normalization
    strips the paren (a semver ``.patch`` field read was falsely flagged
    as mock usage; audit 2026-07-10).
    """
    if pattern.endswith("(") and not is_call:
        return False
    bare = _normalize_pattern(pattern)
    if "." in bare:
        return bare in attr_chain
    return _name_matches(attr_name, bare, is_call=is_call)


def _check_banned_library_imports(
    tree: ast.AST, lines: list[str], banned: frozenset[str]
) -> list[tuple[int, str]]:
    """Flag any ``import``/``from ... import`` whose target is a banned
    library — the shared ``.claude/banned-libraries.txt`` registry, not a
    second copy of it. This is a LIBRARY-IMPORT check, distinct from every
    category in ``FORBIDDEN_PATTERNS`` above: those match hand-rolled
    ``class FakeX``/``StubX``/``DummyX`` usage sites by NAME, so a real
    mock/fake LIBRARY import (``import fakeredis.aioredis as fake_redis``)
    only happened to be caught by the ``stub`` category's ``"fake_"`` prefix
    pattern matching the alias — and was then actively un-caught by the
    since-removed ``EXCLUDED_PATTERNS`` carve-out that whitelisted any line
    containing the substring ``"fakeredis"``. A checker that recognizes
    ``fakeredis`` only by an alias someone happened to name ``fake_...`` (or
    that whitelists it outright) does not generalize to ``docker``,
    ``pyswip``, or the next fake library added to the registry with no
    "fake" in its name at all — reading the registry directly does.
    """
    found: list[tuple[int, str]] = []
    for node in ast.walk(tree):
        if isinstance(node, ast.Import):
            for alias in node.names:
                if is_banned(alias.name, banned):
                    lineno = node.lineno
                    line_text = lines[lineno - 1] if lineno <= len(lines) else ""
                    found.append((lineno, line_text.strip()))
                    break
        elif isinstance(node, ast.ImportFrom):
            if node.module and is_banned(node.module, banned):
                lineno = node.lineno
                line_text = lines[lineno - 1] if lineno <= len(lines) else ""
                found.append((lineno, line_text.strip()))
    return found


def scan_file(
    py_file: Path, banned: frozenset[str] | None = None
) -> dict[str, list[tuple[int, str]]]:
    """Scan a single test file for forbidden patterns.

    ``banned`` is the set of banned-library names to check imports against
    (default: the real ``.claude/banned-libraries.txt`` registry, lazily
    loaded once and cached). Overridable so a test can exercise the
    import-level check against a synthetic registry without touching the
    real one.
    """
    violations: dict[str, list[tuple[int, str]]] = {
        "mock": [],
        "stub": [],
        "monkeypatch": [],
        "skip": [],
        "banned_import": [],
    }

    try:
        content = py_file.read_text(encoding="utf-8")
        tree = ast.parse(content)
    except Exception:
        return violations

    lines = content.splitlines()

    # Nodes that are the function of a real call — call-marked patterns
    # ("patch(", "Fake(") only match these, never plain reads.
    called_nodes = {id(n.func) for n in ast.walk(tree) if isinstance(n, ast.Call)}

    violations["banned_import"] = _check_banned_library_imports(
        tree, lines, banned if banned is not None else _banned_library_names()
    )

    for node in ast.walk(tree):
        # Check for monkeypatch fixture parameter
        if isinstance(node, ast.FunctionDef | ast.AsyncFunctionDef):
            for arg in node.args.args + node.args.kwonlyargs:
                if arg.arg == "monkeypatch":
                    lineno = node.lineno
                    line_text = lines[lineno - 1] if lineno <= len(lines) else ""
                    violations["monkeypatch"].append((lineno, line_text.strip()))

        # Check imports and names
        if isinstance(node, ast.Name):
            name = node.id
            for category, patterns in FORBIDDEN_PATTERNS.items():
                if category == "monkeypatch":
                    continue  # handled above
                for pattern in patterns:
                    if _name_matches(name, pattern, is_call=id(node) in called_nodes):
                        lineno = getattr(node, "lineno", 0)
                        line_text = (
                            lines[lineno - 1] if lineno and lineno <= len(lines) else ""
                        )
                        violations[category].append((lineno, line_text.strip()))

        # Check attribute access (e.g., pytest.mark.skip)
        if isinstance(node, ast.Attribute):
            attr_chain = _get_attr_chain(node)
            for category, patterns in FORBIDDEN_PATTERNS.items():
                if category == "monkeypatch":
                    continue
                for pattern in patterns:
                    if _attr_chain_matches(
                        attr_chain,
                        node.attr,
                        pattern,
                        is_call=id(node) in called_nodes,
                    ):
                        lineno = getattr(node, "lineno", 0)
                        line_text = (
                            lines[lineno - 1] if lineno and lineno <= len(lines) else ""
                        )
                        violations[category].append((lineno, line_text.strip()))

    # Deduplicate
    for category in violations:
        seen = set()
        deduped = []
        for lineno, line_text in violations[category]:
            key = (lineno, line_text)
            if key not in seen:
                seen.add(key)
                deduped.append((lineno, line_text))
        violations[category] = deduped

    return violations


def _get_attr_chain(node: ast.Attribute) -> str:
    """Get the full attribute chain as a string (e.g., 'pytest.mark.skip')."""
    parts = [node.attr]
    current = node.value
    while isinstance(current, ast.Attribute):
        parts.append(current.attr)
        current = current.value
    if isinstance(current, ast.Name):
        parts.append(current.id)
    return ".".join(reversed(parts))


def _collect_py_files(raw_paths: list[str]) -> list[Path]:
    """Resolve tree/file arguments to a deduplicated, order-preserving list
    of ``.py`` files — directories walked via ``rglob``, files taken as-is.
    Shared by the two scan groups ``main()`` builds (full-category and
    banned-import-only) so both dedup and walk identically."""
    collected: list[Path] = []
    seen: set[Path] = set()
    for raw in raw_paths:
        target = Path(raw)
        found = sorted(target.rglob("*.py")) if target.is_dir() else [target]
        for path in found:
            if path not in seen:
                seen.add(path)
                collected.append(path)
    return collected


def main() -> int:
    parser = argparse.ArgumentParser(description="Audit tests for forbidden patterns")
    parser.add_argument("--fix", action="store_true", help="Generate fix report")
    parser.add_argument(
        "paths",
        nargs="*",
        default=["tests/"],
        help="Test trees OR individual test files to audit for the full "
        "mock/stub/monkeypatch/skip/banned_import category set (default: "
        "tests/). Files are accepted so a change of a handful of modules can "
        "be audited for the pre-commit battery without rescanning the whole "
        "tree; production always audits the full tests/.",
    )
    parser.add_argument(
        "--banned-import-only",
        nargs="+",
        metavar="PATH",
        default=[],
        dest="banned_import_only_paths",
        help="Additional trees OR files scanned for the 'banned_import' "
        "category ONLY — never mock/stub/monkeypatch/skip. Those four exist "
        "to enforce test-purity (real infra, no doubles) and do not "
        "generalize to production code: src/orchestrata_tools/"
        "test_generator/{combined_generator,stub_generator}.py legitimately "
        "name a 'stub_gen' attribute because they GENERATE test-stub files "
        "(a first-class AST-toolkit capability), not because they fake a "
        "dependency, and the full category set flags that as a violation it "
        "is not. A banned-library import (fakeredis, pyswip, docker, "
        "unittest.mock, grpcio, xdist, ...) IS an architecture violation "
        "wherever it appears, so this flag lets a tree be swept for that one "
        "category without false positives on legitimate 'stub'-named "
        "production tooling.",
    )
    args = parser.parse_args()

    # Keep the caller's spelling for the error message — Path("tests/") prints
    # as "tests", and echoing back something the caller did not type makes the
    # message harder to match against what they ran.
    given = args.paths or ["tests/"]
    given_banned_only = args.banned_import_only_paths
    all_raw = given + given_banned_only
    all_targets = [Path(p) for p in all_raw]
    missing = [raw for raw, t in zip(all_raw, all_targets) if not t.exists()]
    if missing:
        for raw in missing:
            print(f"ERROR: {raw} not found")
        return 1

    # file -> allowed categories. None means every category (mock, stub,
    # monkeypatch, skip, banned_import). A file reached ONLY through
    # --banned-import-only is narrowed to {"banned_import"}; a file ALSO
    # reached through the full-scope `paths` keeps the wider scope regardless
    # of which group's argument came first — narrowing an already-full-scope
    # file would silently drop coverage a caller already had.
    file_scope: dict[Path, frozenset[str] | None] = {}
    for path in _collect_py_files(given):
        file_scope[path] = None
    for path in _collect_py_files(given_banned_only):
        file_scope.setdefault(path, frozenset({"banned_import"}))

    py_files: list[Path] = list(file_scope)

    total_violations = 0
    file_count = 0

    for py_file in py_files:
        if "__pycache__" in str(py_file):
            continue
        if "_generated_broken" in str(py_file) or "generated_recovery" in str(py_file):
            continue

        violations = scan_file(py_file)
        allowed = file_scope[py_file]
        if allowed is not None:
            violations = {
                category: (items if category in allowed else [])
                for category, items in violations.items()
            }
        has_violations = any(v for v in violations.values())

        if has_violations:
            file_count += 1
            print(f"\n{'=' * 60}")
            # Not relative_to(<root>) any more: targets can be a mix of trees
            # and individual files, so there is no single root to relativize
            # against. The path as given is what the caller can act on.
            print(f"VIOLATIONS: {py_file}")
            print("=" * 60)
            for category, items in violations.items():
                if items:
                    print(f"\n  [{category.upper()}] ({len(items)})")
                    for lineno, line_text in items[:10]:
                        print(f"    Line {lineno}: {line_text[:80]}")
                    if len(items) > 10:
                        print(f"    ... and {len(items) - 10} more")
                    total_violations += len(items)

    print(f"\n{'=' * 60}")
    print(f"SUMMARY: {file_count} files with {total_violations} total violations")
    print("=" * 60)

    return 1 if total_violations > 0 else 0


if __name__ == "__main__":
    sys.exit(main())