orchestrata_ultimate.refactor.codemod._parse_cache

Reference › orchestrata_ultimate › refactor › codemod › _parse_cache

A disk cache of parsed LosslessModule objects, used by workspace when load is given a cache_dir, so unchanged files are unpickled instead of re-parsed. cache_key(data) hashes the file bytes with xxhash3 and prefixes a format version and the ctok cache version, so a tokenizer or parser upgrade never serves stale entries. ParseCache.put pickles a module to a temporary file and renames it into place; a write failure is logged and never breaks a load. get returns the cached module, or None on a miss; an unreadable entry, or one that is not a LosslessModule, is logged, evicted and treated as a miss. It depends on the ctok package for the module type.

At a glance

Class diagram

Dependency diagram

API

Classes

ParseCache

Line 49

Directory-backed cache of pickled LosslessModules.

Functions

Source

Permalink: https://gitlab.com/MSamy/orchestrata-ultimate-architecture/-/blob/a4cbc2bdf/src/orchestrata_ultimate/refactor/codemod/_parse_cache.py

"""Content-addressed disk cache for parsed LosslessModules.

Repeat loads (daemon restart, CLI reruns, CI on unchanged files) can skip
re-parsing entirely: unpickling a materialized LosslessModule (TokenStream
+ ast overlay + NodeIndex) is cheaper than re-tokenizing and re-parsing.

Cache entries are keyed by xxhash3_64 of the file bytes plus the ctok
build id and a format version, so invalidation is exact — a changed file
or a tokenizer/parser upgrade produces a different key and falls back to
parsing.

The cache is opt-in: ``CodemodWorkspace.load(..., cache_dir=...)``. The
REQ-110-003 load benchmarks run without it (they measure cold parse).
"""

from __future__ import annotations

import logging
import os
import pickle
import tempfile
from pathlib import Path

import codemod_fast

from orchestrata_ultimate.refactor.ctok import LosslessModule

logger = logging.getLogger(__name__)

# Bump when the cache entry format changes.
_FORMAT_VERSION = "1"

# Bump when the ctok tokenizer/parser layer changes in a way that could
# produce a different LosslessModule for the same source bytes (mirrors
# the old libcst-version pin, which existed for the same reason).
CTOK_CACHE_VERSION = "1"


def cache_key(data: bytes) -> str:
    """Content-addressed key for a source file's bytes.

    Incorporates the ctok build version and cache format version so a
    tokenizer/parser upgrade never serves stale modules.
    """
    digest = codemod_fast.xxhash3_64(data)
    return f"{_FORMAT_VERSION}-{CTOK_CACHE_VERSION}-{digest}"


class ParseCache:
    """Directory-backed cache of pickled LosslessModules."""

    def __init__(self, cache_dir: str | Path) -> None:
        self._dir = Path(cache_dir)
        self._dir.mkdir(parents=True, exist_ok=True)

    @property
    def directory(self) -> Path:
        return self._dir

    def _path_for(self, key: str) -> Path:
        # Key characters are [0-9a-f.-] — safe as a filename.
        return self._dir / f"{key}.pickle"

    def get(self, key: str) -> LosslessModule | None:
        """Return the cached LosslessModule for key, or None on miss/error."""
        path = self._path_for(key)
        try:
            with path.open("rb") as f:
                module = pickle.load(f)
        except FileNotFoundError:
            return None
        except Exception as e:
            # Corrupt or incompatible entry: log, evict, treat as miss.
            logger.warning("Parse cache entry %s unreadable (%s); evicting", key, e)
            try:
                path.unlink()
            except OSError as unlink_err:
                logger.warning("Could not evict %s: %s", path, unlink_err)
            return None
        if not isinstance(module, LosslessModule):
            logger.warning(
                "Parse cache entry %s is not a LosslessModule; evicting", key
            )
            path.unlink()
            return None
        return module

    def put(self, key: str, module: LosslessModule) -> None:
        """Store a LosslessModule under key (atomic temp-file + rename)."""
        path = self._path_for(key)
        try:
            fd, tmp = tempfile.mkstemp(dir=self._dir, suffix=".tmp")
            try:
                with os.fdopen(fd, "wb") as f:
                    pickle.dump(module, f, protocol=5)
                os.replace(tmp, path)
            except Exception:
                try:
                    os.unlink(tmp)
                except OSError as e:
                    logger.warning("Could not remove temp cache file %s: %s", tmp, e)
                raise
        except Exception as e:
            # A cache that cannot write must never break a load.
            logger.warning("Parse cache write failed for %s: %s", key, e)