fix(comments): extract attachment text nightly, sealed or not
Some checks failed
CI / lint (push) Successful in 1m26s
CI / notebooks-smoke (push) Successful in 2m36s
CI / test (push) Has been cancelled
Deploy / changes (push) Has started running
Deploy / notebooks (push) Has been cancelled
Deploy / zotero (push) Has been cancelled
Deploy / docs (push) Has been cancelled
Deploy / api (push) Has been cancelled
Deploy / bcda (push) Has been cancelled
Deploy / llm (push) Has been cancelled
Deploy / mc (push) Has been cancelled
Deploy / report (push) Has been cancelled
Infra CI / mc (push) Has been cancelled
Infra CI / notebooks (push) Has been cancelled
Infra CI / zotero (push) Has been cancelled
Infra CI / docs (push) Has been cancelled
Infra CI / api (push) Has been cancelled
Infra CI / bcda (push) Has been cancelled
Infra CI / llm (push) Has been cancelled
Some checks failed
CI / lint (push) Successful in 1m26s
CI / notebooks-smoke (push) Successful in 2m36s
CI / test (push) Has been cancelled
Deploy / changes (push) Has started running
Deploy / notebooks (push) Has been cancelled
Deploy / zotero (push) Has been cancelled
Deploy / docs (push) Has been cancelled
Deploy / api (push) Has been cancelled
Deploy / bcda (push) Has been cancelled
Deploy / llm (push) Has been cancelled
Deploy / mc (push) Has been cancelled
Deploy / report (push) Has been cancelled
Infra CI / mc (push) Has been cancelled
Infra CI / notebooks (push) Has been cancelled
Infra CI / zotero (push) Has been cancelled
Infra CI / docs (push) Has been cancelled
Infra CI / api (push) Has been cancelled
Infra CI / bcda (push) Has been cancelled
Infra CI / llm (push) Has been cancelled
Nothing ran 'stack comments extract' on a schedule, and extract skipped every sealed docket outright. 28 dockets (~29k comments with attachments, including every OPPS/IPPS docket) were sealed before anyone extracted them, so the tagger and the chat index only ever saw their 'see attached' stubs. Comment Pull now extracts between pull and index, and a seal skips only a fully extracted docket.
This commit is contained in:
@@ -29,6 +29,10 @@ jobs:
|
||||
run: |
|
||||
docker exec api uv run --no-sync stack bib pull-open-dockets
|
||||
|
||||
- name: Extract attachment text (PDF/DOCX to combined.md)
|
||||
run: |
|
||||
docker exec api uv run --no-sync stack comments extract --workers 4
|
||||
|
||||
- name: Index new comments for the chat search
|
||||
run: |
|
||||
docker exec llm uv run --no-sync stack llm index --collection comments
|
||||
|
||||
@@ -825,7 +825,10 @@ def _gen_comment_pull(runner: str, uv_version: str, **_kw: object) -> tuple[str,
|
||||
docket quiet long enough after its comment period seals itself, so the
|
||||
job costs nothing between rulemaking seasons. `fetch-pfs-comments` for
|
||||
the current year registers a new PFS proposed rule (July) the night it
|
||||
appears. Indexing runs in the llm container (its embeddings host and
|
||||
appears. Attachment text is extracted in the api container (it mounts
|
||||
.state/comments read-write); nothing ran that step on a schedule, so 28
|
||||
dockets were tagged and indexed from "see attached" stubs until
|
||||
2026-11-11. Indexing runs in the llm container (its embeddings host and
|
||||
pgvector settings live there). 02:30 runs before the 03:30 notebooks,
|
||||
03:40 golden and 04:15 Zotero jobs.
|
||||
"""
|
||||
@@ -855,6 +858,10 @@ jobs:
|
||||
run: |
|
||||
docker exec api uv run --no-sync stack bib pull-open-dockets
|
||||
|
||||
- name: Extract attachment text (PDF/DOCX to combined.md)
|
||||
run: |
|
||||
docker exec api uv run --no-sync stack comments extract --workers 4
|
||||
|
||||
- name: Index new comments for the chat search
|
||||
run: |
|
||||
docker exec llm uv run --no-sync stack llm index --collection comments
|
||||
|
||||
@@ -122,6 +122,14 @@ def _bib_lookup_factory(use_bib: bool):
|
||||
return body
|
||||
|
||||
|
||||
def _extraction_complete(docket_dir: Path) -> bool:
|
||||
"""Every comment dir under *docket_dir* has its combined.md (a
|
||||
missing dir counts as complete: nothing to extract)."""
|
||||
if not docket_dir.is_dir():
|
||||
return True
|
||||
return all((d / "combined.md").exists() for d in docket_dir.iterdir() if d.is_dir())
|
||||
|
||||
|
||||
@app.command()
|
||||
def extract(
|
||||
root: Path = typer.Option(_DEFAULT_ROOT, "--root", help="Comments root dir."),
|
||||
@@ -162,8 +170,12 @@ def extract(
|
||||
|
||||
lookup, attach_cb, store = _build_bib_helpers(use_bib, attach=use_bib and attach)
|
||||
try:
|
||||
# A seal skips finished work only: a docket sealed before anyone
|
||||
# extracted it (28 were, by Nov 2026) is still extracted.
|
||||
skip = (
|
||||
set(store.sealed_dockets()) if (store is not None and not force) else set()
|
||||
{d for d in store.sealed_dockets() if _extraction_complete(root / d)}
|
||||
if (store is not None and not force)
|
||||
else set()
|
||||
)
|
||||
stats = walk_and_extract(
|
||||
root,
|
||||
|
||||
@@ -112,7 +112,11 @@ def test_dockets_discover_populates_from_tags(mc_connect, mc_client, mc_disc, st
|
||||
|
||||
|
||||
@patch("bib.connect")
|
||||
def test_extract_skips_sealed_docket(mc_connect, tmp_path: Path, store):
|
||||
def test_extract_still_extracts_a_sealed_docket_with_unextracted_dirs(
|
||||
mc_connect, tmp_path: Path, store
|
||||
):
|
||||
# 28 dockets were sealed before anyone ran extract, and the seal then
|
||||
# kept them unextracted forever: a seal only skips finished work.
|
||||
s = store
|
||||
s.docket_upsert(
|
||||
Docket(
|
||||
@@ -125,8 +129,28 @@ def test_extract_skips_sealed_docket(mc_connect, tmp_path: Path, store):
|
||||
app, ["extract", "--root", str(tmp_path), "--workers", "1", "--no-attach"]
|
||||
)
|
||||
assert result.exit_code == 0, result.output
|
||||
assert (cdir / "combined.md").exists()
|
||||
|
||||
|
||||
@patch("bib.connect")
|
||||
def test_extract_skips_a_fully_extracted_sealed_docket(
|
||||
mc_connect, tmp_path: Path, store
|
||||
):
|
||||
s = store
|
||||
s.docket_upsert(
|
||||
Docket(
|
||||
id="CMS-2024-0001", sealed_at="2025-01-01T00:00:00Z", seal_reason="manual"
|
||||
)
|
||||
)
|
||||
mc_connect.return_value = s
|
||||
cdir = _seed_dir(tmp_path, "CMS-2024-0001")
|
||||
(cdir / "combined.md").write_text("done")
|
||||
result = runner.invoke(
|
||||
app, ["extract", "--root", str(tmp_path), "--workers", "1", "--no-attach"]
|
||||
)
|
||||
assert result.exit_code == 0, result.output
|
||||
assert "skipped_sealed: 1" in result.output
|
||||
assert not (cdir / "combined.md").exists()
|
||||
assert (cdir / "combined.md").read_text() == "done"
|
||||
|
||||
|
||||
@patch("bib.connect")
|
||||
|
||||
Reference in New Issue
Block a user