From 87c2e8427cc3887fd744bb7bb4ae0967962b0f75 Mon Sep 17 00:00:00 2001 From: kert Date: Wed, 11 Nov 2026 15:05:54 -0500 Subject: [PATCH] fix(comments): extract attachment text nightly, sealed or not Nothing ran 'stack comments extract' on a schedule, and extract skipped every sealed docket outright. 28 dockets (~29k comments with attachments, including every OPPS/IPPS docket) were sealed before anyone extracted them, so the tagger and the chat index only ever saw their 'see attached' stubs. Comment Pull now extracts between pull and index, and a seal skips only a fully extracted docket. --- .gitea/workflows/comment-pull.yml | 4 ++++ dev/scripts/backends/gitea.py | 9 ++++++++- src/cli/comments.py | 14 +++++++++++++- tests/cli/test_comments_dockets.py | 28 ++++++++++++++++++++++++++-- 4 files changed, 51 insertions(+), 4 deletions(-) diff --git a/.gitea/workflows/comment-pull.yml b/.gitea/workflows/comment-pull.yml index 70da2f7..e247b8b 100644 --- a/.gitea/workflows/comment-pull.yml +++ b/.gitea/workflows/comment-pull.yml @@ -29,6 +29,10 @@ jobs: run: | docker exec api uv run --no-sync stack bib pull-open-dockets + - name: Extract attachment text (PDF/DOCX to combined.md) + run: | + docker exec api uv run --no-sync stack comments extract --workers 4 + - name: Index new comments for the chat search run: | docker exec llm uv run --no-sync stack llm index --collection comments diff --git a/dev/scripts/backends/gitea.py b/dev/scripts/backends/gitea.py index 2f2ae86..8c6f9a5 100644 --- a/dev/scripts/backends/gitea.py +++ b/dev/scripts/backends/gitea.py @@ -825,7 +825,10 @@ def _gen_comment_pull(runner: str, uv_version: str, **_kw: object) -> tuple[str, docket quiet long enough after its comment period seals itself, so the job costs nothing between rulemaking seasons. `fetch-pfs-comments` for the current year registers a new PFS proposed rule (July) the night it - appears. Indexing runs in the llm container (its embeddings host and + appears. Attachment text is extracted in the api container (it mounts + .state/comments read-write); nothing ran that step on a schedule, so 28 + dockets were tagged and indexed from "see attached" stubs until + 2026-11-11. Indexing runs in the llm container (its embeddings host and pgvector settings live there). 02:30 runs before the 03:30 notebooks, 03:40 golden and 04:15 Zotero jobs. """ @@ -855,6 +858,10 @@ jobs: run: | docker exec api uv run --no-sync stack bib pull-open-dockets + - name: Extract attachment text (PDF/DOCX to combined.md) + run: | + docker exec api uv run --no-sync stack comments extract --workers 4 + - name: Index new comments for the chat search run: | docker exec llm uv run --no-sync stack llm index --collection comments diff --git a/src/cli/comments.py b/src/cli/comments.py index 7c8532f..cfb46ea 100644 --- a/src/cli/comments.py +++ b/src/cli/comments.py @@ -122,6 +122,14 @@ def _bib_lookup_factory(use_bib: bool): return body +def _extraction_complete(docket_dir: Path) -> bool: + """Every comment dir under *docket_dir* has its combined.md (a + missing dir counts as complete: nothing to extract).""" + if not docket_dir.is_dir(): + return True + return all((d / "combined.md").exists() for d in docket_dir.iterdir() if d.is_dir()) + + @app.command() def extract( root: Path = typer.Option(_DEFAULT_ROOT, "--root", help="Comments root dir."), @@ -162,8 +170,12 @@ def extract( lookup, attach_cb, store = _build_bib_helpers(use_bib, attach=use_bib and attach) try: + # A seal skips finished work only: a docket sealed before anyone + # extracted it (28 were, by Nov 2026) is still extracted. skip = ( - set(store.sealed_dockets()) if (store is not None and not force) else set() + {d for d in store.sealed_dockets() if _extraction_complete(root / d)} + if (store is not None and not force) + else set() ) stats = walk_and_extract( root, diff --git a/tests/cli/test_comments_dockets.py b/tests/cli/test_comments_dockets.py index 6878a90..0af275f 100644 --- a/tests/cli/test_comments_dockets.py +++ b/tests/cli/test_comments_dockets.py @@ -112,7 +112,11 @@ def test_dockets_discover_populates_from_tags(mc_connect, mc_client, mc_disc, st @patch("bib.connect") -def test_extract_skips_sealed_docket(mc_connect, tmp_path: Path, store): +def test_extract_still_extracts_a_sealed_docket_with_unextracted_dirs( + mc_connect, tmp_path: Path, store +): + # 28 dockets were sealed before anyone ran extract, and the seal then + # kept them unextracted forever: a seal only skips finished work. s = store s.docket_upsert( Docket( @@ -125,8 +129,28 @@ def test_extract_skips_sealed_docket(mc_connect, tmp_path: Path, store): app, ["extract", "--root", str(tmp_path), "--workers", "1", "--no-attach"] ) assert result.exit_code == 0, result.output + assert (cdir / "combined.md").exists() + + +@patch("bib.connect") +def test_extract_skips_a_fully_extracted_sealed_docket( + mc_connect, tmp_path: Path, store +): + s = store + s.docket_upsert( + Docket( + id="CMS-2024-0001", sealed_at="2025-01-01T00:00:00Z", seal_reason="manual" + ) + ) + mc_connect.return_value = s + cdir = _seed_dir(tmp_path, "CMS-2024-0001") + (cdir / "combined.md").write_text("done") + result = runner.invoke( + app, ["extract", "--root", str(tmp_path), "--workers", "1", "--no-attach"] + ) + assert result.exit_code == 0, result.output assert "skipped_sealed: 1" in result.output - assert not (cdir / "combined.md").exists() + assert (cdir / "combined.md").read_text() == "done" @patch("bib.connect")