diff --git a/.gitea/workflows/comment-pull.yml b/.gitea/workflows/comment-pull.yml index 70da2f7..e247b8b 100644 --- a/.gitea/workflows/comment-pull.yml +++ b/.gitea/workflows/comment-pull.yml @@ -29,6 +29,10 @@ jobs: run: | docker exec api uv run --no-sync stack bib pull-open-dockets + - name: Extract attachment text (PDF/DOCX to combined.md) + run: | + docker exec api uv run --no-sync stack comments extract --workers 4 + - name: Index new comments for the chat search run: | docker exec llm uv run --no-sync stack llm index --collection comments diff --git a/dev/scripts/backends/gitea.py b/dev/scripts/backends/gitea.py index 2f2ae86..8c6f9a5 100644 --- a/dev/scripts/backends/gitea.py +++ b/dev/scripts/backends/gitea.py @@ -825,7 +825,10 @@ def _gen_comment_pull(runner: str, uv_version: str, **_kw: object) -> tuple[str, docket quiet long enough after its comment period seals itself, so the job costs nothing between rulemaking seasons. `fetch-pfs-comments` for the current year registers a new PFS proposed rule (July) the night it - appears. Indexing runs in the llm container (its embeddings host and + appears. Attachment text is extracted in the api container (it mounts + .state/comments read-write); nothing ran that step on a schedule, so 28 + dockets were tagged and indexed from "see attached" stubs until + 2026-11-11. Indexing runs in the llm container (its embeddings host and pgvector settings live there). 02:30 runs before the 03:30 notebooks, 03:40 golden and 04:15 Zotero jobs. """ @@ -855,6 +858,10 @@ jobs: run: | docker exec api uv run --no-sync stack bib pull-open-dockets + - name: Extract attachment text (PDF/DOCX to combined.md) + run: | + docker exec api uv run --no-sync stack comments extract --workers 4 + - name: Index new comments for the chat search run: | docker exec llm uv run --no-sync stack llm index --collection comments diff --git a/src/cli/comments.py b/src/cli/comments.py index 7c8532f..cfb46ea 100644 --- a/src/cli/comments.py +++ b/src/cli/comments.py @@ -122,6 +122,14 @@ def _bib_lookup_factory(use_bib: bool): return body +def _extraction_complete(docket_dir: Path) -> bool: + """Every comment dir under *docket_dir* has its combined.md (a + missing dir counts as complete: nothing to extract).""" + if not docket_dir.is_dir(): + return True + return all((d / "combined.md").exists() for d in docket_dir.iterdir() if d.is_dir()) + + @app.command() def extract( root: Path = typer.Option(_DEFAULT_ROOT, "--root", help="Comments root dir."), @@ -162,8 +170,12 @@ def extract( lookup, attach_cb, store = _build_bib_helpers(use_bib, attach=use_bib and attach) try: + # A seal skips finished work only: a docket sealed before anyone + # extracted it (28 were, by Nov 2026) is still extracted. skip = ( - set(store.sealed_dockets()) if (store is not None and not force) else set() + {d for d in store.sealed_dockets() if _extraction_complete(root / d)} + if (store is not None and not force) + else set() ) stats = walk_and_extract( root, diff --git a/tests/cli/test_comments_dockets.py b/tests/cli/test_comments_dockets.py index 6878a90..0af275f 100644 --- a/tests/cli/test_comments_dockets.py +++ b/tests/cli/test_comments_dockets.py @@ -112,7 +112,11 @@ def test_dockets_discover_populates_from_tags(mc_connect, mc_client, mc_disc, st @patch("bib.connect") -def test_extract_skips_sealed_docket(mc_connect, tmp_path: Path, store): +def test_extract_still_extracts_a_sealed_docket_with_unextracted_dirs( + mc_connect, tmp_path: Path, store +): + # 28 dockets were sealed before anyone ran extract, and the seal then + # kept them unextracted forever: a seal only skips finished work. s = store s.docket_upsert( Docket( @@ -125,8 +129,28 @@ def test_extract_skips_sealed_docket(mc_connect, tmp_path: Path, store): app, ["extract", "--root", str(tmp_path), "--workers", "1", "--no-attach"] ) assert result.exit_code == 0, result.output + assert (cdir / "combined.md").exists() + + +@patch("bib.connect") +def test_extract_skips_a_fully_extracted_sealed_docket( + mc_connect, tmp_path: Path, store +): + s = store + s.docket_upsert( + Docket( + id="CMS-2024-0001", sealed_at="2025-01-01T00:00:00Z", seal_reason="manual" + ) + ) + mc_connect.return_value = s + cdir = _seed_dir(tmp_path, "CMS-2024-0001") + (cdir / "combined.md").write_text("done") + result = runner.invoke( + app, ["extract", "--root", str(tmp_path), "--workers", "1", "--no-attach"] + ) + assert result.exit_code == 0, result.output assert "skipped_sealed: 1" in result.output - assert not (cdir / "combined.md").exists() + assert (cdir / "combined.md").read_text() == "done" @patch("bib.connect")