fix(docs): self-consistent, hermetic library export (closes #623)

The exporter read tags in a second query after the minutes-long item
hydration; a tag edit landing in between (P37 controller tag ops during
#618's suite run) produced a library.json whose item tags were missing
from the tag list, failing test_tags_in_items_match_tag_list on every
run until regeneration. The tag list is now derived from the serialized
items themselves (self-consistent by construction) and the output file
is swapped in atomically.

Tests no longer assert on live data/bib.sqlite: TestExportLibrary
exports a fixture store via new STACK_BIB_DB/STACK_LIBRARY_JSON
overrides — the live-store contract is the docs build's own concern.

Co-designed with a peer session that landed the derived-tags exporter.

docs/scripts/export_library.py

@@ -4,6 +4,14 @@ Reads ``data/bib.sqlite`` and writes ``docs/static/library.json``
 with items, collections, and tags. Gracefully writes empty JSON if
 the database is missing.
 
+The tag list is derived from the serialized items rather than queried
+separately, so the export is self-consistent even if a writer mutates
+tags during the minutes-long item scan (#623), and the output file is
+swapped in atomically so concurrent readers never see a partial write.
+
+``STACK_BIB_DB`` / ``STACK_LIBRARY_JSON`` override the default paths
+(used by the hermetic tests in tests/docs/test_docs_build.py).
+
 Usage::
 
     uv run python docs/scripts/export_library.py
@@ -12,20 +20,28 @@ Usage::
 from __future__ import annotations
 
 import json
+import os
 import sys
+from collections import Counter
 from pathlib import Path
 
-BIB_DB = Path("data/bib.sqlite")
-OUT_PATH = Path("docs/static/library.json")
+BIB_DB = Path(os.environ.get("STACK_BIB_DB", "data/bib.sqlite"))
+OUT_PATH = Path(os.environ.get("STACK_LIBRARY_JSON", "docs/static/library.json"))
+
+EMPTY = {"items": [], "collections": [], "tags": []}
+
+
+def _write(data: dict) -> None:
+    OUT_PATH.parent.mkdir(parents=True, exist_ok=True)
+    tmp = OUT_PATH.with_name(OUT_PATH.name + ".tmp")
+    tmp.write_text(json.dumps(data, indent=2, default=str))
+    os.replace(tmp, OUT_PATH)
 
 
 def main() -> None:
     if not BIB_DB.exists():
         print(f"bib.sqlite not found at {BIB_DB}, writing empty library.json")
-        OUT_PATH.parent.mkdir(parents=True, exist_ok=True)
-        OUT_PATH.write_text(
-            json.dumps({"items": [], "collections": [], "tags": []}, indent=2)
-        )
+        _write(EMPTY)
         return
 
     # Import here so griffe stage doesn't need bib deps
@@ -34,16 +50,13 @@ def main() -> None:
         from bib.store import Store
     except ImportError as exc:
         print(f"bib deps not available ({exc}), writing empty library.json")
-        OUT_PATH.parent.mkdir(parents=True, exist_ok=True)
-        OUT_PATH.write_text(
-            json.dumps({"items": [], "collections": [], "tags": []}, indent=2)
-        )
+        _write(EMPTY)
         return
 
     store = Store(str(BIB_DB))
+    con = store._con()
     items = store.list_items()
     raw_collections = store.list_collections()
-    tags = store.list_tags()
 
     # item_count from list_collections() is direct-only, which is what
     # the library page filter uses (i.collections.includes(key)).
@@ -51,9 +64,9 @@ def main() -> None:
     collections = [c for c in raw_collections if c["item_count"] > 0]
 
     serialized_items = []
+    tag_counts: Counter[str] = Counter()
     for item in items:
-        abstract = item.abstract or ""
-        creator_rows = store._con().execute(
+        creator_rows = con.execute(
             "SELECT c.last_name, c.first_name FROM item_creators ic "
             "JOIN creators c ON ic.creator_id = c.id "
             "WHERE ic.item_id = (SELECT id FROM items WHERE key = ?) "
@@ -66,6 +79,7 @@ def main() -> None:
             else r["last_name"]
             for r in creator_rows
         ]
+        tag_counts.update(item.tags)
         serialized_items.append(
             {
                 "key": item.key,
@@ -73,24 +87,17 @@ def main() -> None:
                 "title": item.title,
                 "url": item.url,
                 "date_published": item.date_published,
-                "abstract": abstract,
+                "abstract": item.abstract or "",
                 "tags": item.tags,
                 "collections": item.collections,
                 "institution": item.institution,
                 "creators": creators,
             }
         )
+    store.close()
 
-    data = {
-        "items": serialized_items,
-        "collections": collections,
-        "tags": tags,
-    }
-
-    OUT_PATH.parent.mkdir(parents=True, exist_ok=True)
-    OUT_PATH.write_text(json.dumps(data, indent=2, default=str))
-    if serialized_items:
-        assert len(serialized_items) > 0, "library.json has no items"
+    tags = [{"name": n, "count": c} for n, c in sorted(tag_counts.items())]
+    _write({"items": serialized_items, "collections": collections, "tags": tags})
     print(f"Wrote {len(serialized_items)} items to {OUT_PATH}")
 
 

tests/docs/test_docs_build.py

@@ -1,8 +1,14 @@
-"""Integration tests for the docs build pipeline."""
+"""Integration tests for the docs build pipeline.
+
+TestExportLibrary is hermetic (#623): it exports a small fixture store
+via STACK_BIB_DB/STACK_LIBRARY_JSON overrides instead of asserting on
+the live, constantly-mutating data/bib.sqlite.
+"""
 
 from __future__ import annotations
 
 import json
+import os
 import re
 import subprocess
 import sys
@@ -11,9 +17,7 @@ from pathlib import Path
 import pytest
 
 _ROOT = Path(__file__).resolve().parents[2]
-BIB_DB = _ROOT / "data/bib.sqlite"
 EXPORT_SCRIPT = _ROOT / "docs/scripts/export_library.py"
-LIBRARY_JSON = _ROOT / "docs/static/library.json"
 DOCS_DIR = _ROOT / "docs/docs"
 INTRO_MD = DOCS_DIR / "intro.md"
 
@@ -21,46 +25,54 @@ INTRO_MD = DOCS_DIR / "intro.md"
 class TestExportLibrary:
     """Tests for docs/scripts/export_library.py."""
 
-    def test_export_runs_without_error(self):
+    @pytest.fixture
+    def exported(self, tmp_path):
+        """Export a 2-item fixture store; return the parsed payload."""
+        from bib.item import Item
+        from bib.store import Store
+
+        db = tmp_path / "bib.sqlite"
+        store = Store(str(db), storage_dir=tmp_path / "storage")
+        store.create(
+            Item(
+                item_type="rule",
+                title="CY2027 PFS proposed rule",
+                url="https://example.gov/nprm",
+                abstract="A" * 400,
+                tags=["module:pfs", "year:2027"],
+            )
+        )
+        store.create(
+            Item(
+                item_type="source",
+                title="Comment letter",
+                url="https://example.gov/comment",
+                tags=["module:pfs", "docket:CMS-2026-2377"],
+            )
+        )
+        store.close()
+
+        out = tmp_path / "library.json"
         result = subprocess.run(
             [sys.executable, str(EXPORT_SCRIPT)],
             capture_output=True,
             text=True,
+            cwd=_ROOT,
+            env={
+                **os.environ,
+                "STACK_BIB_DB": str(db),
+                "STACK_LIBRARY_JSON": str(out),
+            },
         )
         assert result.returncode == 0, result.stderr
+        return json.loads(out.read_text())
 
-    def test_library_json_has_items(self):
-        """library.json should have >0 items when bib.sqlite has data."""
-        if not BIB_DB.exists():
-            pytest.skip("bib.sqlite not available")
-        # Verify bib.sqlite is a real database with items
-        import sqlite3
+    def test_export_writes_fixture_items(self, exported):
+        assert len(exported["items"]) == 2
+        titles = {i["title"] for i in exported["items"]}
+        assert titles == {"CY2027 PFS proposed rule", "Comment letter"}
 
-        try:
-            con = sqlite3.connect(str(BIB_DB))
-            count = con.execute("SELECT count(*) FROM items").fetchone()[0]
-            con.close()
-        except (sqlite3.OperationalError, sqlite3.DatabaseError):
-            pytest.skip("bib.sqlite is not a valid database")
-        if count == 0:
-            pytest.skip("bib.sqlite has no items")
-        # Run the export
-        subprocess.run([sys.executable, str(EXPORT_SCRIPT)], check=True)
-        data = json.loads(LIBRARY_JSON.read_text())
-        assert len(data["items"]) > 0, "library.json has no items"
-        assert len(data["tags"]) > 0, "library.json has no tags"
-
-    def test_item_has_required_fields(self):
-        """Each item must have all required fields."""
-        if not LIBRARY_JSON.exists():
-            import pytest
-
-            pytest.skip("library.json not generated")
-        data = json.loads(LIBRARY_JSON.read_text())
-        if not data["items"]:
-            import pytest
-
-            pytest.skip("no items in library.json")
+    def test_item_has_required_fields(self, exported):
         required = {
             "key",
             "item_type",
@@ -71,38 +83,43 @@ class TestExportLibrary:
             "collections",
             "creators",
         }
-        for item in data["items"][:10]:  # spot check first 10
+        for item in exported["items"]:
             missing = required - set(item.keys())
             assert not missing, f"Item {item.get('key', '?')} missing fields: {missing}"
 
-    def test_abstracts_not_truncated(self):
-        """Abstracts should not be truncated to 300 chars."""
-        if not LIBRARY_JSON.exists():
-            import pytest
+    def test_abstracts_not_truncated(self, exported):
+        by_title = {i["title"]: i for i in exported["items"]}
+        assert by_title["CY2027 PFS proposed rule"]["abstract"] == "A" * 400
 
-            pytest.skip("library.json not generated")
-        data = json.loads(LIBRARY_JSON.read_text())
-        # Find items with long abstracts
-        truncated = [
-            i["key"]
-            for i in data["items"]
-            if i["abstract"].endswith("...") and len(i["abstract"]) == 300
+    def test_tags_match_item_tags(self, exported):
+        """The tag list is exactly the tags on serialized items, with
+        counts — self-consistent by construction, so a tag edit landing
+        mid-export can no longer orphan item tags (#623)."""
+        assert exported["tags"] == [
+            {"name": "docket:CMS-2026-2377", "count": 1},
+            {"name": "module:pfs", "count": 2},
+            {"name": "year:2027", "count": 1},
         ]
-        assert not truncated, f"Found truncated abstracts: {truncated[:5]}"
 
-    def test_tags_in_items_match_tag_list(self):
-        """Every tag referenced by items should appear in the tags list."""
-        if not LIBRARY_JSON.exists():
-            import pytest
-
-            pytest.skip("library.json not generated")
-        data = json.loads(LIBRARY_JSON.read_text())
-        tag_names = {t["name"] for t in data["tags"]}
-        item_tags = set()
-        for item in data["items"]:
-            item_tags.update(item["tags"])
-        orphans = item_tags - tag_names
-        assert not orphans, f"Tags on items but not in tags list: {orphans}"
+    def test_missing_db_writes_empty(self, tmp_path):
+        out = tmp_path / "library.json"
+        result = subprocess.run(
+            [sys.executable, str(EXPORT_SCRIPT)],
+            capture_output=True,
+            text=True,
+            cwd=_ROOT,
+            env={
+                **os.environ,
+                "STACK_BIB_DB": str(tmp_path / "nope.sqlite"),
+                "STACK_LIBRARY_JSON": str(out),
+            },
+        )
+        assert result.returncode == 0, result.stderr
+        assert json.loads(out.read_text()) == {
+            "items": [],
+            "collections": [],
+            "tags": [],
+        }
 
 
 class TestIntroLinks: