scripts.build_ft_cpython

Reference › scripts › build_ft_cpython

Builds a patched free-threaded CPython 3.14 interpreter, as part of the SPEC-023 GIL-off doctrine. A human runs it when a free-threaded interpreter is needed; tests/scripts/test_build_ft_cpython.py imports it.

Invocation: main takes --prefix (install location), --ref (the CPython commit to build, pinned by default), --jobs (default the CPU count), --optimizations (profile-guided and link-time optimization: a slower build and a faster interpreter) and --src (checkout directory, by default under TMPDIR). The work runs in six numbered steps, each printed. It fetches the pinned commit with a shallow git fetch, verifies that the fix for upstream issue gh-153809 is present in the checkout (_verify_fix_present), configures with --disable-gil --enable-shared and an rpath so the installed binary finds its own libpython, then builds and installs. An existing install at the prefix is removed first. _run runs each command and can write its output to a log file such as configure.log.

At a glance

Class diagram

No classes; see Functions.

Dependency diagram

No internal imports and no importers.

API

Functions

Source

Permalink: https://gitlab.com/MSamy/orchestrata-ultimate-architecture/-/blob/a4cbc2bdf/scripts/build_ft_cpython.py

#!/usr/bin/env python3
"""Build a patched free-threaded CPython 3.14 interpreter (SPEC-023 GIL-off doctrine).

Sibling of ``scripts/build_ft_grpcio.py`` / ``scripts/build_janus_ft.py``: pins an
exact upstream source revision, verifies the fix it carries, and builds a
free-threaded (``--disable-gil``) interpreter WITH debug symbols.

Why this exists
---------------
Two distinct free-threading defects motivate a source-built interpreter:

1. **cpython gh-153809** (``TaskObj_dealloc`` GC-untrack ordering; fix
   gh-154563, 3.14 backport gh-154666). Present in the released 3.14.6t: an
   asyncio Task can sit at refcount 0 while still GC-tracked, so a concurrent
   ``gc.collect()`` hangs or segfaults. The fix landed on the ``3.14`` branch
   2026-07-25, AFTER 3.14.6 (2026-06-10), and 3.14.7 was not yet released, so
   no downloadable interpreter carried it. The pinned commit below does.
   VERIFIED: the gh-153809 reproducer hangs on stock 3.14.6t and exits clean on
   this build.

2. The broader free-threading **heap-corruption race class** behind the
   ``tests/knowledge_graph/`` interpreter-exit hang (GC ``update_refs`` spinning
   in mimalloc ``mi_heap_visit_blocks`` over a corrupted heap). Fixing gh-153809
   alone does NOT resolve it (measured 2026-08-03: the KG suite still hangs on
   this build). See ``tests/knowledge_graph/conftest.py`` and the project
   memory ``python314t-heap-corruption-investigation``. This build keeps debug
   symbols precisely so gdb can name the corruptor -- the prior investigation
   was blocked by uv/python-build-standalone shipping no DWARF.

Usage
-----
    python scripts/build_ft_cpython.py            # build to the default prefix
    python scripts/build_ft_cpython.py --jobs 8
    uv venv --python <prefix>/bin/python3.14t .venv && uv sync --dev

The build is NOT wired into the pinned project interpreter: it fixes gh-153809
but not the KG hang, so adopting it is a deliberate choice, not a default.
"""

from __future__ import annotations

import argparse
import os
import re
import subprocess
import sys
from pathlib import Path

# Pinned CPython 3.14-branch revision that carries the gh-153809 fix
# (gh-154666). Reported as "3.14.6+" — post-3.14.6, pre-3.14.7.
CPYTHON_REF = "1f574b1ce58067f9fdaad0b5ff9a3bfb87151e4c"
CPYTHON_REPO = "https://github.com/python/cpython.git"
DEFAULT_PREFIX = Path.home() / ".local" / "share" / "cpython-3.14-ft-153809"


def _run(cmd: list[str], *, cwd: Path | None = None, log: Path | None = None) -> None:
    """Run ``cmd``, streaming to ``log`` if given; raise on non-zero exit."""
    where = f" (cwd={cwd})" if cwd else ""
    print(f"  $ {' '.join(cmd)}{where}", flush=True)
    if log is not None:
        with log.open("w") as fh:
            proc = subprocess.run(cmd, cwd=cwd, stdout=fh, stderr=subprocess.STDOUT)
    else:
        proc = subprocess.run(cmd, cwd=cwd)
    if proc.returncode != 0:
        tail = ""
        if log is not None and log.exists():
            tail = "\n".join(log.read_text().splitlines()[-30:])
        raise SystemExit(f"FAILED ({proc.returncode}): {' '.join(cmd)}\n{tail}")


def _verify_fix_present(src: Path) -> None:
    """Prove gh-153809's fix is in this checkout: GC_UnTrack before unregister_task."""
    body_src = (src / "Modules" / "_asynciomodule.c").read_text()
    match = re.search(
        r"TaskObj_dealloc\(PyObject \*self\)\s*\{(.*?)\n\}", body_src, re.S
    )
    if match is None:
        raise SystemExit("could not locate TaskObj_dealloc — CPython layout changed")
    body = match.group(1)
    untrack = body.find("PyObject_GC_UnTrack")
    unregister = body.find("unregister_task")
    if not (0 <= untrack < unregister):
        raise SystemExit(
            "gh-153809 fix NOT present: PyObject_GC_UnTrack must precede "
            f"unregister_task in TaskObj_dealloc (untrack@{untrack}, "
            f"unregister@{unregister})"
        )
    print(f"  gh-153809 fix present (UnTrack@{untrack} < unregister_task@{unregister})")


def main() -> int:
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument("--prefix", type=Path, default=DEFAULT_PREFIX)
    parser.add_argument("--ref", default=CPYTHON_REF, help="CPython commit to build")
    parser.add_argument("--jobs", type=int, default=os.cpu_count() or 4)
    parser.add_argument(
        "--optimizations",
        action="store_true",
        help="configure --enable-optimizations (PGO; slower build, faster runtime)",
    )
    parser.add_argument(
        "--src",
        type=Path,
        default=Path(os.environ.get("TMPDIR", "/tmp")) / "cpython-ft-src",
    )
    args = parser.parse_args()

    src: Path = args.src
    prefix: Path = args.prefix
    print(f"[1/6] fetching CPython {args.ref[:12]} into {src}")
    if not (src / ".git").exists():
        src.mkdir(parents=True, exist_ok=True)
        _run(["git", "init", "-q"], cwd=src)
        _run(["git", "remote", "add", "origin", CPYTHON_REPO], cwd=src)
    _run(["git", "fetch", "--depth", "1", "origin", args.ref], cwd=src)
    _run(["git", "checkout", "-q", "FETCH_HEAD"], cwd=src)

    print("[2/6] verifying gh-153809 fix is present")
    _verify_fix_present(src)

    print(f"[3/6] configure --disable-gil --enable-shared (prefix={prefix})")
    if prefix.exists():
        _run(["rm", "-rf", str(prefix)])
    configure = [
        "./configure",
        "--disable-gil",
        "--enable-shared",
        f"--prefix={prefix}",
    ]
    if args.optimizations:
        configure += ["--enable-optimizations", "--with-lto"]
    # rpath so the installed binary finds its own libpython at runtime.
    configure.append(f"LDFLAGS=-Wl,-rpath={prefix}/lib")
    _run(configure, cwd=src, log=src / "configure.log")

    print(f"[4/6] make -j{args.jobs} (debug symbols kept for gdb diagnosis)")
    _run(["make", f"-j{args.jobs}"], cwd=src, log=src / "build.log")

    print("[5/6] make install")
    _run(["make", "install"], cwd=src, log=src / "install.log")

    print("[6/6] sanity check")
    py = prefix / "bin" / "python3.14t"
    _run([str(py), "-VV"])
    _run(
        [
            str(py),
            "-c",
            "import sys; assert not sys._is_gil_enabled(), 'GIL is ENABLED'; "
            "print('gil_enabled =', sys._is_gil_enabled())",
        ]
    )
    print(f"\nBUILD_DONE — interpreter at {py}")
    print("Create a venv with:")
    print(f"  uv venv --python {py} .venv && uv sync --dev")
    return 0


if __name__ == "__main__":
    sys.exit(main())