scripts.doc_lint

Reference › scripts › doc_lint

The doc-truth lint: it catches hand-written documentation claims that drift from the facts. tests/scripts/test_doc_lint.py imports it; a gate or a human runs it.

Invocation: main takes any number of files or directories (default docs/ and README.md), --warn-only (report findings but exit 0) and --incus-json FILE (read the Incus ground truth from a file instead of running incus list). If the ground truth is unavailable, meaning no live Incus and no JSON file, it fails with exit 2 even under --warn-only.

lint scans the Markdown files (iter_markdown_files) with three checks. check_stale_infra flags stale infrastructure references. check_test_counts compares test counts quoted in the documentation with the actual count read from a run-tests JSON file (load_actual_test_count). check_container_ips compares IP addresses quoted in the documentation with the container addresses known to Incus (load_incus_ips). A line can be marked exempt (_is_exempt). Each failure is a Finding, and the summary line counts failures, exempt lines and files.

At a glance

Class diagram

Dependency diagram

No internal imports and no importers.

API

Constants

Classes

Finding (dataclass)

Decorators: dataclass · Line 96

LintReport (dataclass)

Decorators: dataclass · Line 107

Functions

Source

Permalink: https://gitlab.com/MSamy/orchestrata-ultimate-architecture/-/blob/a4cbc2bdf/scripts/doc_lint.py

#!/usr/bin/env python3
"""Doc-truth lint — catches hand-written documentation claims drifting from
ground reality (the project's corrections-register defect-#6 class:
hand-written claims drifting from reality; see ``plans/tooling-wiring.md``).

MVP scope — exactly three checks, each comparing a doc claim to a live
ground-truth source, never to another document:

1. STALE-INFRA PHRASES — ``docs/**/*.md`` + ``README.md`` scanned for
   retired-infrastructure references: IPs matching ``10.222.9.\\d+``, the
   phrase ``swiplserver`` or ``MQI`` presented as the CURRENT bridge, and
   ``gatekeeper-neo4j`` presented as live. A line (or its containing
   markdown table row — table rows are one physical line) that also
   contains "stale", "was", "superseded", "destroyed", "delta",
   "historical", or "originally" is EXEMPT — reconciliation tables
   legitimately quote old values. An inline ``<!-- doc-lint: allow -->``
   comment on the same or the immediately preceding line is also an
   exemption.

2. TEST-COUNT CLAIMS — ``\\b([0-9]{1,3}(?:,[0-9]{3})+|[0-9]{4,})\\s+tests\\b``
   claims are compared against the real total (``passed`` + ``failed``)
   read from ``reports/run-tests-latest.json``. A claim differing from
   reality by more than 10% is a FAIL naming file:line, claimed, actual.

3. CONTAINER/IP CLAIMS — every ``10.53.1.\\d+`` mention is checked against
   live ``incus list --format json`` output (or ``--incus-json PATH`` for
   tests/CI). If neither ground truth is reachable the run exits 2 with
   "ground truth unavailable" — it never silently passes.

Output is one line per finding: ``FAIL <file>:<line> <check>: <detail>``,
followed by a summary line ``doc-lint: N FAIL, M exempt, K files``.

Usage:
    python scripts/doc_lint.py [--warn-only] [--incus-json PATH] [paths...]

Exit codes: 0 clean, 1 findings (unless ``--warn-only``), 2 ground-truth
unavailable (unconditional — ``--warn-only`` does not suppress this).
"""

from __future__ import annotations

import argparse
import json
import re
import subprocess
import sys
from dataclasses import dataclass
from pathlib import Path

REPO_ROOT = Path(__file__).resolve().parent.parent
RUN_TESTS_LATEST_JSON = REPO_ROOT / "reports" / "run-tests-latest.json"
DEFAULT_PATHS = [Path("docs"), Path("README.md")]

# --- check 1: stale-infra phrases -------------------------------------------------

_STALE_PATTERNS: tuple[tuple[str, re.Pattern[str]], ...] = (
    ("10.222.9.x IP", re.compile(r"\b10\.222\.9\.\d+\b")),
    ("swiplserver", re.compile(r"swiplserver", re.IGNORECASE)),
    ("MQI", re.compile(r"\bMQI\b")),
    ("gatekeeper-neo4j", re.compile(r"gatekeeper-neo4j", re.IGNORECASE)),
)

_EXEMPTION_KEYWORDS = (
    "stale",
    "was",
    "superseded",
    "destroyed",
    "delta",
    "historical",
    "originally",
)
_EXEMPTION_RE = re.compile(
    r"\b(" + "|".join(_EXEMPTION_KEYWORDS) + r")\b", re.IGNORECASE
)
_INLINE_ALLOW_RE = re.compile(r"<!--\s*doc-lint:\s*allow\s*-->")

# --- check 2: test-count claims ---------------------------------------------------

_TEST_COUNT_RE = re.compile(r"\b([0-9]{1,3}(?:,[0-9]{3})+|[0-9]{4,})\s+tests\b")
_TEST_COUNT_TOLERANCE = 0.10

# --- check 3: container/IP claims -------------------------------------------------

# (?!/\d) skips CIDR network references like 10.53.1.0/24 — those describe
# the subnet, not a claimed container host (false positive found 2026-08-02).
_CONTAINER_IP_RE = re.compile(r"\b10\.53\.1\.\d+\b(?!/\d)")

# Strikethrough (~~...~~) marks a RETIRED/DELETED container row by convention in
# INFRASTRUCTURE.md. A struck IP is explicitly no-longer-live, so it must NOT be
# checked against incus ground truth — that is exactly the record it documents
# (false positive found 2026-08-03: 7 retired Neo4j/Memgraph rows).
_STRIKETHROUGH_RE = re.compile(r"~~.*?~~")


@dataclass
class Finding:
    path: Path
    line_no: int
    check: str
    detail: str

    def format(self) -> str:
        return f"FAIL {self.path}:{self.line_no} {self.check}: {self.detail}"


@dataclass
class LintReport:
    findings: list[Finding]
    exempt_count: int
    file_count: int
    ground_truth_available: bool


def iter_markdown_files(paths: list[Path]) -> list[Path]:
    """Expand CLI path args (files or directories) into a sorted, de-duplicated
    list of ``.md`` files. Directories are scanned recursively; missing paths
    are silently skipped (a default ``docs/`` in a doc-less checkout is not a
    lint error)."""
    files: set[Path] = set()
    for p in paths:
        if p.is_dir():
            files.update(p.rglob("*.md"))
        elif p.is_file():
            files.add(p)
    return sorted(files)


def _is_exempt(lines: list[str], idx: int) -> bool:
    """``idx`` is the 0-based index of the line under test. Exempt if the
    line itself carries an exemption keyword or the inline allow comment, or
    if the immediately preceding line carries the inline allow comment."""
    line = lines[idx]
    if _EXEMPTION_RE.search(line) or _INLINE_ALLOW_RE.search(line):
        return True
    if idx > 0 and _INLINE_ALLOW_RE.search(lines[idx - 1]):
        return True
    return False


def check_stale_infra(path: Path, lines: list[str]) -> tuple[list[Finding], int]:
    """Returns (findings, exempt_count) — exempt_count is the number of
    matches that WOULD have been findings but were exempted, for the summary
    line."""
    findings: list[Finding] = []
    exempt_count = 0
    for i, line in enumerate(lines, start=1):
        matches = [
            (label, m.group(0))
            for label, rx in _STALE_PATTERNS
            if (m := rx.search(line)) is not None
        ]
        if not matches:
            continue
        if _is_exempt(lines, i - 1):
            exempt_count += len(matches)
            continue
        findings.extend(
            Finding(path, i, "STALE-INFRA", f"retired {label} reference {text!r}")
            for label, text in matches
        )
    return findings, exempt_count


def load_actual_test_count(run_tests_json: Path) -> int:
    """Real test total = passed + failed, read from run-tests-latest.json.
    (The file's only count keys are ``aborted``/``passed``/``failed`` — there
    is no ``total`` key, so the real count a doc claim means is the sum of
    the two outcome counts.)"""
    data = json.loads(run_tests_json.read_text(encoding="utf-8"))
    return int(data.get("passed", 0)) + int(data.get("failed", 0))


def check_test_counts(path: Path, lines: list[str], actual: int) -> list[Finding]:
    findings: list[Finding] = []
    for i, line in enumerate(lines, start=1):
        for m in _TEST_COUNT_RE.finditer(line):
            claimed = int(m.group(1).replace(",", ""))
            if actual == 0:
                differs = claimed != 0
            else:
                differs = abs(claimed - actual) / actual > _TEST_COUNT_TOLERANCE
            if differs:
                findings.append(
                    Finding(
                        path,
                        i,
                        "TEST-COUNT",
                        f"claimed {claimed:,} tests, actual {actual:,}",
                    )
                )
    return findings


def load_incus_ips(incus_json_path: Path | None) -> set[str] | None:
    """Return the set of live IPv4 addresses known to incus ground truth, or
    None if ground truth could not be obtained from either source (caller
    must then exit 2 — never silently pass)."""
    if incus_json_path is not None:
        try:
            data = json.loads(incus_json_path.read_text(encoding="utf-8"))
        except OSError:
            return None
        except json.JSONDecodeError:
            return None
    else:
        try:
            result = subprocess.run(
                ["incus", "list", "--format", "json"],
                capture_output=True,
                text=True,
                timeout=30,
                check=False,
            )
        except OSError:
            return None
        except subprocess.TimeoutExpired:
            return None
        if result.returncode != 0:
            return None
        try:
            data = json.loads(result.stdout)
        except json.JSONDecodeError:
            return None

    ips: set[str] = set()
    for container in data:
        state = container.get("state") or {}
        network = state.get("network") or {}
        for iface in network.values():
            for addr in iface.get("addresses") or []:
                if addr.get("family") == "inet" and addr.get("address"):
                    ips.add(addr["address"])
    return ips


def check_container_ips(
    path: Path, lines: list[str], known_ips: set[str]
) -> list[Finding]:
    findings: list[Finding] = []
    for i, line in enumerate(lines, start=1):
        # Drop struck-through spans first: a ~~retired~~ container's IP is not a
        # live-ground-truth claim (line numbers are unaffected — we scan the
        # residue of the same line).
        scan_line = _STRIKETHROUGH_RE.sub("", line)
        for m in _CONTAINER_IP_RE.finditer(scan_line):
            ip = m.group(0)
            if ip not in known_ips:
                findings.append(
                    Finding(
                        path,
                        i,
                        "CONTAINER-IP",
                        f"{ip} not found in incus ground truth",
                    )
                )
    return findings


def lint(paths: list[Path], incus_json_path: Path | None) -> LintReport:
    md_files = iter_markdown_files(paths)

    known_ips = load_incus_ips(incus_json_path)
    if known_ips is None:
        return LintReport(
            findings=[],
            exempt_count=0,
            file_count=len(md_files),
            ground_truth_available=False,
        )

    actual_tests = load_actual_test_count(RUN_TESTS_LATEST_JSON)

    findings: list[Finding] = []
    exempt_count = 0
    for f in md_files:
        lines = f.read_text(encoding="utf-8", errors="replace").splitlines()
        stale_findings, file_exempt = check_stale_infra(f, lines)
        findings.extend(stale_findings)
        exempt_count += file_exempt
        findings.extend(check_test_counts(f, lines, actual_tests))
        findings.extend(check_container_ips(f, lines, known_ips))

    return LintReport(
        findings=findings,
        exempt_count=exempt_count,
        file_count=len(md_files),
        ground_truth_available=True,
    )


def main(argv: list[str] | None = None) -> int:
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument(
        "paths",
        nargs="*",
        type=Path,
        default=[],
        help="Files/directories to scan (default: docs/ README.md)",
    )
    parser.add_argument(
        "--warn-only",
        action="store_true",
        help="Report findings but exit 0 (does not affect the ground-truth-unavailable exit 2)",
    )
    parser.add_argument(
        "--incus-json",
        type=Path,
        default=None,
        help="Read incus ground truth from this JSON file instead of running `incus list`",
    )
    args = parser.parse_args(argv)

    paths = args.paths if args.paths else DEFAULT_PATHS
    report = lint(paths, args.incus_json)

    if not report.ground_truth_available:
        print(
            "FAIL doc-lint: ground truth unavailable "
            "(no live `incus` and no --incus-json provided)",
            file=sys.stderr,
        )
        return 2

    for finding in report.findings:
        print(finding.format())
    print(
        f"doc-lint: {len(report.findings)} FAIL, "
        f"{report.exempt_count} exempt, {report.file_count} files"
    )

    if report.findings and not args.warn_only:
        return 1
    return 0


if __name__ == "__main__":
    sys.exit(main())