144 lines
4.9 KiB
Python
144 lines
4.9 KiB
Python
"""Walk .state/comments/ and run extract_comment over every dir.
|
|
|
|
Parallelized with ThreadPoolExecutor. PyMuPDF releases the GIL during
|
|
PDF decoding so threads (not processes) are the right primitive — keeps
|
|
shared state simple and avoids fork overhead per attachment.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import logging
|
|
from collections.abc import Callable
|
|
from concurrent.futures import ThreadPoolExecutor, as_completed
|
|
from pathlib import Path
|
|
|
|
from rex.comments.combine import (
|
|
derive_siblings_from_combined,
|
|
extract_comment,
|
|
is_current,
|
|
)
|
|
|
|
log = logging.getLogger(__name__)
|
|
|
|
|
|
def walk_and_extract(
|
|
root: Path,
|
|
*,
|
|
inline_body_lookup: Callable[[str], str],
|
|
docket: str | None = None,
|
|
limit: int | None = None,
|
|
workers: int = 8,
|
|
force: bool = False,
|
|
on_extracted: Callable[[str, Path], None] | None = None,
|
|
skip_dockets: set[str] | frozenset[str] = frozenset(),
|
|
reattach: bool = False,
|
|
) -> dict[str, int]:
|
|
"""Process every comment dir under *root*.
|
|
|
|
*root* is the dir containing per-docket subdirs (typically
|
|
``.state/comments/``). *inline_body_lookup* is called with each
|
|
``comment_id`` and returns the inline body text from bib.sqlite (or
|
|
"" if missing).
|
|
|
|
A dir is *current* when its combined.md is newer than every source
|
|
file (``combine.is_current``); current dirs are skipped without being
|
|
read. *on_extracted* fires for newly written dirs; with *reattach*
|
|
it also fires for skipped dirs (after deriving any missing sibling
|
|
MDs), which is the repair path for bib notes. *skip_dockets* names
|
|
docket dirs to ignore entirely (sealed dockets), unless *force*.
|
|
|
|
Returns counts: ``{"written": N, "skipped": N, "failed": N}``, plus
|
|
``"skipped_sealed": N`` when any docket was skipped via *skip_dockets*.
|
|
"""
|
|
candidate_dirs, skipped_dirs, sealed = _collect_dirs(
|
|
root, docket=docket, limit=limit, force=force, skip_dockets=skip_dockets
|
|
)
|
|
stats = {"written": 0, "skipped": len(skipped_dirs), "failed": 0}
|
|
if sealed:
|
|
stats["skipped_sealed"] = sealed
|
|
|
|
if candidate_dirs:
|
|
with ThreadPoolExecutor(max_workers=workers) as pool:
|
|
futures = {
|
|
pool.submit(_one, d, inline_body_lookup, force): d
|
|
for d in candidate_dirs
|
|
}
|
|
for fut in as_completed(futures):
|
|
status = fut.result()
|
|
stats[status] += 1
|
|
if on_extracted and status == "written":
|
|
cdir = futures[fut]
|
|
on_extracted(cdir.name, cdir)
|
|
|
|
if on_extracted and reattach:
|
|
for cdir in skipped_dirs:
|
|
try:
|
|
derive_siblings_from_combined(cdir)
|
|
except Exception as e: # noqa: BLE001
|
|
log.warning("sibling backfill failed for %s: %s", cdir, e)
|
|
on_extracted(cdir.name, cdir)
|
|
|
|
return stats
|
|
|
|
|
|
def _collect_dirs(
|
|
root: Path,
|
|
*,
|
|
docket: str | None,
|
|
limit: int | None,
|
|
force: bool,
|
|
skip_dockets: set[str] | frozenset[str] = frozenset(),
|
|
) -> tuple[list[Path], list[Path], int]:
|
|
"""Return ``(dirs_to_process, dirs_skipped, sealed_docket_count)``.
|
|
|
|
``dirs_skipped`` are dirs that are already current (``is_current``)
|
|
and would not be re-processed. We return them as a list (not a count)
|
|
so callers can still drive per-dir post-processing — e.g. attaching
|
|
the existing combined.md to bib on a re-run (``reattach``).
|
|
|
|
``sealed_docket_count`` counts docket dirs skipped entirely because
|
|
their name is in *skip_dockets* (and *force* is False).
|
|
"""
|
|
todo: list[Path] = []
|
|
skipped: list[Path] = []
|
|
sealed = 0
|
|
for docket_dir in sorted(root.iterdir()):
|
|
if not docket_dir.is_dir():
|
|
continue
|
|
if docket and docket_dir.name != docket:
|
|
continue
|
|
if docket_dir.name in skip_dockets and not force:
|
|
sealed += 1
|
|
continue
|
|
for cdir in sorted(docket_dir.iterdir()):
|
|
if not cdir.is_dir():
|
|
continue
|
|
if not force and is_current(cdir):
|
|
skipped.append(cdir)
|
|
continue
|
|
todo.append(cdir)
|
|
if limit and len(todo) >= limit:
|
|
return todo, skipped, sealed
|
|
return todo, skipped, sealed
|
|
|
|
|
|
def _one(
|
|
comment_dir: Path,
|
|
inline_body_lookup: Callable[[str], str],
|
|
force: bool,
|
|
) -> str:
|
|
"""Process one comment dir. Returns "written" / "skipped" / "failed"."""
|
|
if not force and is_current(comment_dir):
|
|
return "skipped"
|
|
try:
|
|
body = inline_body_lookup(comment_dir.name)
|
|
except Exception as e: # noqa: BLE001
|
|
log.warning("inline body lookup failed for %s: %s", comment_dir.name, e)
|
|
body = ""
|
|
try:
|
|
extract_comment(comment_dir, inline_body=body, force=True)
|
|
except Exception as e: # noqa: BLE001
|
|
log.warning("extract_comment failed for %s: %s", comment_dir, e)
|
|
return "failed"
|
|
return "written"
|