scripts.check_staged_secrets

Reference › scripts › check_staged_secrets

A pre-commit gate that refuses a commit which stages a plaintext secret. It looks only at what is staged, and its sibling check_secret_leaks covers the tracked tree. It uses the waiver mechanism in gate_waivers, and tests/scripts/test_check_staged_secrets.py and tests/scripts/test_check_staged_secrets_gaps.py import it.

Invocation: main takes only --repo-root (default the current directory). It prints a report and returns the fail code when violations exist; otherwise it returns success.

run_check lists the staged files (staged_files), reads the staged content of each (staged_content) and the set of added line numbers (staged_added_lines), so only lines added by the commit can fail it. find_secret_assignments finds assignments of a literal to a credential-like name, skipping unquoted values, placeholders and f-string interpolations (is_quoted_literal, is_placeholder, _is_fstring_interpolation). Each hit is a Violation, and _print_report lists them without echoing the value.

At a glance

Class diagram

Dependency diagram

API

Classes

Violation (dataclass)

Decorators: dataclass · Line 123

Functions

Source

Permalink: https://gitlab.com/MSamy/orchestrata-ultimate-architecture/-/blob/a4cbc2bdf/scripts/check_staged_secrets.py

#!/usr/bin/env python3
"""Pre-commit gate: refuse a commit that stages a plaintext secret.

Structural half of the secrets design (see scripts/secrets_cli.py's module
docstring for the full threat model): SOPS + age keep the tracked
``secrets/env.sops.env`` safe to commit, but nothing stops a developer (or
an agent) from pasting a live key straight into a ``.env`` that gets
accidentally ``git add``-ed, or hardcoding one in source. This gate makes
that structurally hard to *commit*, not just discouraged — it scans every
staged file (except the tracked ciphertext itself, which is SUPPOSED to
carry ``ENC[...]`` values) for an assignment whose NAME looks like a secret
(``KEY``/``TOKEN``/``SECRET``/``PASSWORD``/``PASS``/``API``/``CREDENTIAL``/
``AUTH``, case-insensitive substring match) and whose VALUE is neither empty
nor a recognised template placeholder (``changeme``, ``""``, ...) — exactly
the shape ``.env.example`` uses throughout this repo, which is why it must
never trip this gate.

Scans the STAGED blob content (``git show :<path>``), not the working-tree
file, so a partially-staged file is judged by what would actually land in
the commit.

Wired into ``scripts/hooks/pre-commit`` — see that file. Exit codes mirror
the rest of this repo's ``check_*`` gates: 0 clean, 1 violation(s) found,
2 could not determine what's staged (not a git repo, git unavailable —
never silently treated as "clean" the way a swallowed exception would).
"""

from __future__ import annotations

import argparse
import re
import subprocess
import sys
from dataclasses import dataclass
from pathlib import Path

sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src"))
from orchestrata_ultimate.core.gate_waivers import waived_lines

EXIT_OK = 0
EXIT_FAIL = 1
EXIT_PRECONDITION = 2

# The tracked ciphertext artifact is exempt by construction — SOPS-encrypted
# values are MEANT to sit in plain sight in git history; that is the entire
# point of secrets/env.sops.env (see scripts/secrets_cli.py).
EXEMPT_RELPATHS = frozenset({"secrets/env.sops.env"})

# NAME match: a case-insensitive substring, exactly the alternation the
# design calls for — deliberately unanchored (so e.g. ``API_KEY`` and
# ``deepseek_api_key`` both match), which is over-inclusive by design for a
# security gate: a false positive costs a `--no-verify`, a false negative
# costs a leaked credential.
# Credential-shaped NAMES. The first pass matched the bare substrings
# KEY|TOKEN|SECRET|PASSWORD|PASS|API|CREDENTIAL|AUTH, which fires on PASSED,
# BYPASS, RAPID, AUTHENTICATING -- and, measured on one commit in this repo,
# on eighteen Redis key names (EVENTS_KEY, per_thread_key, stream_key,
# key_prefix) and two message-type enum members. A Redis key is not a
# cryptographic key, and in a codebase whose parallel-safety gate is entirely
# about Redis key namespacing that over-match blocks ordinary work.
#
# So: words that mean "credential" on their own stay bare, and the ambiguous
# ones (KEY, AUTH) must carry a qualifier that supplies the credential sense.
_CRED_QUALIFIER = r"(?:API|SECRET|PRIVATE|PUBLIC|ACCESS|REFRESH|BEARER|SIGNING|ENCRYPT\w*|DECRYPT\w*|LICENSE|SSH|GPG|AUTH|APP|CLIENT|MASTER|ROOT|ADMIN|SERVICE|WEBHOOK)"
_SECRET_NAME_RE = re.compile(
    r"""(?:
          SECRET | PASSWORD | PASSWD | PASSPHRASE | CREDENTIALS? | APIKEY
        | \bTOKEN\b | \w*_TOKEN\b
        | """
    + _CRED_QUALIFIER
    + r"""\w*_?KEY\b
        )""",
    re.IGNORECASE | re.VERBOSE,
)

# A simple assignment at (near) the start of a line: dotenv (`NAME=value`),
# shell (`export NAME=value`), or a bare Python/JS module-level constant
# (`NAME = "value"`). `=(?!=)` excludes `==` comparisons. The optional
# trailing `,?` absorbs a Python kwarg's statement-terminating comma
# (`password="",` inside a multi-line constructor call) — without it the
# captured value for an EMPTY kwarg was `"",` (quote, quote, comma), which
# is neither a properly-quoted empty string nor a member of
# PLACEHOLDER_VALUES, so it fell through as a false positive. Measured
# 2026-08-12: tests/semantic_graph/test_memgraph_adapter.py flagged 10
# sites, every one a `password=""` / `password_key="MEMGRAPH_PASSWORD"`
# kwarg — a config-field name and an intentionally-unset test credential,
# neither a leaked secret.
_ASSIGNMENT_RE = re.compile(
    r"^\s*(?:export\s+)?([A-Za-z_][A-Za-z0-9_]*)\s*=(?!=)\s*(.*?)\s*,?\s*$"
)

# Case-insensitive, whole-value placeholder tokens a template file is
# expected to carry instead of a live secret. ".env.example" in this repo
# uses "" (blank) and "changeme" for every secret-shaped key throughout —
# both MUST be here or that file itself would fail this gate. The rest are
# common placeholder conventions, admitted defensively.
PLACEHOLDER_VALUES = frozenset(
    {
        "",
        "changeme",
        "change_me",
        "change-me",
        "your-key-here",
        "your_key_here",
        "<placeholder>",
        "placeholder",
        "xxx",
        "xxxx",
        "todo",
        "fixme",
        "replace_me",
        "replaceme",
        "example",
        "dummy",
        "n/a",
        "none",
        "null",
    }
)


@dataclass
class Violation:
    file: str
    line: int
    name: str


# In SOURCE files a hardcoded credential is necessarily a QUOTED STRING
# LITERAL. Everything else on the right-hand side is an expression — a number,
# a call, a collection, a name — and cannot itself be a secret.
#
# WHY THIS EXISTS (measured 2026-08-12): the name match above is deliberately
# over-inclusive, and its header accepts the cost as "a false positive costs a
# --no-verify". Staging this repo's own secrets subsystem produced 19 false
# positives and ZERO true ones, including the gate flagging its own regex
# (_SECRET_NAME_RE = re.compile(...)), a counter (key_count = 6) and a list
# (differing_keys = [...]). A gate that routinely demands --no-verify is a gate
# that gets bypassed, and a bypassed gate catches nothing — so the accepted cost
# was not the real cost.
#
# This does NOT loosen the security-relevant half. The NAME match is unchanged,
# and a real hardcoded credential (API_KEY = "sk-...") is quoted and still fires.
# Dotenv and shell are deliberately EXCLUDED from this rule: NAME=value with an
# unquoted scalar is exactly how a secret is written in those formats.
_CODE_SUFFIXES = (".py", ".pyi", ".js", ".jsx", ".mjs", ".ts", ".tsx")
_STRING_PREFIXES = ("f", "r", "b", "u")


def is_quoted_literal(raw: str) -> bool:
    """True when the RAW right-hand side begins a quoted string literal."""
    value = raw.strip()
    while value[:1].lower() in _STRING_PREFIXES and value[1:2] in ("'", '"'):
        value = value[1:]
    return value[:1] in ("'", '"')


def is_placeholder(value: str) -> bool:
    return value.strip().lower() in PLACEHOLDER_VALUES


def _strip_value(raw: str) -> str:
    """Drop a trailing ``# comment`` (dotenv/shell style — only when the
    value itself isn't quoted, since a quoted value may legitimately
    contain ``#``) and matching surrounding quotes."""
    value = raw.strip()
    if not (value.startswith('"') or value.startswith("'")):
        hash_idx = value.find("#")
        if hash_idx != -1 and (hash_idx == 0 or value[hash_idx - 1].isspace()):
            value = value[:hash_idx].strip()
    if len(value) >= 2 and value[0] == value[-1] and value[0] in "\"'":
        value = value[1:-1]
    return value.strip()


# A value that interpolates a variable is a reference, not a literal secret:
# "$root/scripts/x.py", "${HOME}/.config", "%(token)s". Anchored so it matches
# only when the expansion spans the whole value, leaving mixed literals
# (KEY="sk-live-abc${suffix}") flagged.
_EXPANSION_RE = re.compile(
    r"""^\s*(?:
          \$\{?[A-Za-z_][A-Za-z0-9_]*\}?   # $VAR or ${VAR}
        | %\([A-Za-z_][A-Za-z0-9_]*\)s      # %(name)s
        | \{[A-Za-z_][A-Za-z0-9_]*\}        # {name}
        )[\w./:@~+-]*\s*$""",
    re.VERBOSE,
)


def _is_fstring_interpolation(raw: str) -> bool:
    """True when the value is an f-string carrying an interpolation.

    The Python sibling of _EXPANSION_RE: an f-string with a ``{...}`` field
    is a template resolved at run time, so the literal bytes in the source
    are not the credential. A plain (non-f) quoted string is untouched --
    that is a literal and still gets flagged.
    """
    v = raw.strip()
    if not v[:1].lower() == "f":
        return False
    return v[1:2] in ("'", '"') and "{" in v and "}" in v


def find_secret_assignments(text: str, *, filename: str) -> list[Violation]:
    """Every line assigning a non-empty, non-placeholder value to a
    secret-shaped NAME. Pure — exercised directly against fabricated text,
    no git/subprocess involved."""
    hits: list[Violation] = []
    waived = waived_lines(text)
    for lineno, line in enumerate(text.splitlines(), start=1):
        stripped = line.strip()
        if not stripped or stripped.startswith("#"):
            continue
        match = _ASSIGNMENT_RE.match(line)
        if not match:
            continue
        name, raw_value = match.group(1), match.group(2)
        if not _SECRET_NAME_RE.search(name):
            continue
        if lineno in waived:
            # A reasoned "# gate-ok: <reason>" exception, recorded AT the line by
            # a person. This gate was the ONLY one of its family that did not
            # honour the shared convention (check_secret_leaks.py and
            # gate_parallel_safety.py both do), so its only escape was
            # --no-verify, which bypasses the whole gate rather than one line.
            continue
        if filename.endswith(_CODE_SUFFIXES) and not is_quoted_literal(raw_value):
            # An expression, not a literal credential — see _CODE_SUFFIXES.
            continue
        value = _strip_value(raw_value)
        if not value or is_placeholder(value):
            continue
        if re.search(r"[A-Za-z0-9]", value) is None:
            # AD-85 (Mohamed, 2026-10-17): a value with no ASCII letter and no
            # digit cannot be a credential. Measured: the governance landing's
            # `_NAF_TOKEN = "\\+"` (Prolog negation-as-failure) blocked its
            # commit only because the name ends in _TOKEN. One letter or digit
            # anywhere keeps the value a finding.
            continue
        if _is_fstring_interpolation(raw_value):
            # f"sess:{test_id}:keys" is CONSTRUCTED at run time, so the bytes
            # here are a template, not a credential. Measured: this gate
            # flagged two test-id-derived Redis keys on one commit for no
            # reason other than their names.
            continue
        if _EXPANSION_RE.search(value):
            # A reference, not a literal. `SECRETS_GATE="$root/scripts/x.py"`
            # was flagged by this gate purely because the NAME contains
            # "SECRET" -- it blocked its own commit. A value that interpolates
            # a shell/format variable cannot BE the credential: the bytes are
            # resolved elsewhere at run time, so there is nothing here to leak.
            # Deliberately narrow -- an opaque literal alongside an expansion
            # (KEY="sk-live-...${suffix}") still carries real bytes, so the
            # expansion must account for essentially the whole value.
            continue
        hits.append(Violation(file=filename, line=lineno, name=name))
    return hits


def _git(repo_root: Path, *args: str) -> subprocess.CompletedProcess[str]:
    return subprocess.run(
        ["git", *args], cwd=repo_root, capture_output=True, text=True, timeout=30
    )


def staged_files(repo_root: Path) -> list[str]:
    """Staged files about to be committed — added/copied/modified. Never
    deleted: a file being removed from the tree cannot introduce a new
    plaintext value."""
    result = _git(
        repo_root, "diff", "--cached", "--name-only", "--diff-filter=ACM", "-z"
    )
    if result.returncode != 0:
        raise RuntimeError(result.stderr.strip() or "git diff --cached failed")
    return [p for p in result.stdout.split("\0") if p]


def staged_content(repo_root: Path, relpath: str) -> str | None:
    """The STAGED blob's content — never the working-tree file, which can
    differ from what a partial `git add -p` actually staged. Returns None
    for a binary blob (embedded NUL) or anything that isn't valid UTF-8."""
    result = _git(repo_root, "show", f":{relpath}")
    if result.returncode != 0:
        return None
    if "\x00" in result.stdout:
        return None
    return result.stdout


_HUNK_RE = re.compile(r"^@@ -\d+(?:,\d+)? \+(\d+)(?:,(\d+))? @@")


def staged_added_lines(repo_root: Path, relpath: str) -> set[int] | None:
    """Line numbers this commit ADDS to ``relpath``, numbered in the staged blob.

    ``None`` means "treat every line as added" — a new file, or no HEAD to diff
    against (the initial commit). Both are cases where the whole blob genuinely
    is new content.

    Why this exists: the gate used to scan the entire staged blob, so touching a
    file for ANY reason re-flagged every pre-existing line in it. That blocked a
    commit whose diff added only requirement ids, with 12 findings across four
    spec files — all of them illustrative code already committed in HEAD
    (``token = await exchange_code(code)``, ``api_token="«redacted»"``,
    ``secret = pyotp.random_base32()``). The only way through was ``--no-verify``,
    which trains exactly the reflex a secrets gate exists to prevent, on the exact
    commits where a real leak would be easiest to wave past.

    Scoping to added lines is also the correct division of labour: a secret
    already committed is ``check_secret_leaks.py``'s subject — it scans TRACKED
    source — while this gate's subject is what THIS commit introduces.
    """
    if _git(repo_root, "rev-parse", "--verify", "HEAD").returncode != 0:
        return None
    result = _git(repo_root, "diff", "--cached", "-U0", "HEAD", "--", relpath)
    if result.returncode != 0:
        return None
    added: set[int] = set()
    for line in result.stdout.splitlines():
        match = _HUNK_RE.match(line)
        if match is None:
            continue
        start = int(match.group(1))
        count = 1 if match.group(2) is None else int(match.group(2))
        added.update(range(start, start + count))
    return added


def run_check(repo_root: Path) -> tuple[int, list[Violation]]:
    """Injection seam: tests pass a throwaway tmp_path repo root here —
    never chdir the test process, never touch the live repo/index."""
    try:
        files = staged_files(repo_root)
    except (RuntimeError, FileNotFoundError, subprocess.TimeoutExpired) as exc:
        print(
            f"[staged-secrets] could not list staged files in {repo_root}: {exc}",
            file=sys.stderr,
        )
        return EXIT_PRECONDITION, []

    violations: list[Violation] = []
    for relpath in files:
        if relpath in EXEMPT_RELPATHS:
            continue
        content = staged_content(repo_root, relpath)
        if content is None:
            continue
        found = find_secret_assignments(content, filename=relpath)
        added = staged_added_lines(repo_root, relpath)
        if added is not None:
            found = [v for v in found if v.line in added]
        violations.extend(found)

    return (EXIT_FAIL if violations else EXIT_OK), violations


def _print_report(violations: list[Violation]) -> None:
    print("secrets: refusing to commit — plaintext secret(s) staged:", file=sys.stderr)
    for v in violations:
        print(f"  {v.file}:{v.line}: {v.name}=<redacted>", file=sys.stderr)
    print("", file=sys.stderr)
    print(
        "Move the value into the encrypted secret store instead "
        "(python scripts/secrets_cli.py encrypt), or use a real placeholder "
        "(blank, 'changeme', ...) if this is a template file.",
        file=sys.stderr,
    )
    print("To bypass once: git commit --no-verify", file=sys.stderr)


def main(argv: list[str] | None = None) -> int:
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument(
        "--repo-root",
        type=Path,
        default=None,
        help="Repo root whose staged changes to scan (default: cwd).",
    )
    args = parser.parse_args(argv)
    repo_root = args.repo_root.resolve() if args.repo_root else Path.cwd()

    exit_code, violations = run_check(repo_root)
    if exit_code == EXIT_FAIL:
        _print_report(violations)
    return exit_code


if __name__ == "__main__":
    raise SystemExit(main())