fix(llm): search returns each citation label once — the same FR paragraph indexed under two bib items (2019-16041, 2024-14828 each have two rule items) showed twice; keep the closest hit

src/llm/search.py

@@ -102,6 +102,23 @@ def _theme_ok(md: dict[str, Any], theme: str) -> bool:
     return theme in str(md.get("themes", "")).split(",")
 
 
+def dedupe_by_label(hits: list[dict]) -> list[dict]:
+    """One result per citation label, keeping the closest: the same FR
+    paragraph can be indexed under two bib items that hold the same rule
+    document, and the same chunk can be reached through two collections;
+    a reader wants each cite once."""
+    best: dict[str, dict] = {}
+    for h in hits:
+        key = h.get("label") or h.get("id") or ""
+        if not key:
+            best[id(h)] = h  # type: ignore[index]
+            continue
+        cur = best.get(key)
+        if cur is None or float(h.get("distance", 0)) < float(cur.get("distance", 0)):
+            best[key] = h
+    return list(best.values())
+
+
 def _hit_to_result(doc: Any, distance: float) -> dict:
     md = {k: str(v) for k, v in (doc.metadata or {}).items()}
     result = as_source(md, doc.page_content, _similarity(float(distance)))
@@ -157,6 +174,7 @@ def search(
             if not _theme_ok(doc.metadata or {}, theme):
                 continue
             hits.append(_hit_to_result(doc, distance))
+    hits = dedupe_by_label(hits)
     hits.sort(key=lambda h: h["distance"])
     return hits[offset : offset + limit]
 

tests/llm/test_search.py

@@ -406,3 +406,22 @@ class TestThemeFilter:
             "$and": [{"doctype": "manual"}, theme_or]
         }
         assert _store_filter({"theme": "telehealth"}) == theme_or
+
+
+class TestDedupeByLabel:
+    def test_keeps_the_closest_per_label(self):
+        from llm.search import dedupe_by_label
+
+        hits = [
+            {"label": "89 FR 62002 ¶5", "distance": 0.30, "item_key": "A"},
+            {"label": "89 FR 62002 ¶5", "distance": 0.25, "item_key": "B"},
+            {"label": "IOM 100-02 ch.10 §30.1.1", "distance": 0.4},
+            {"label": "", "id": "", "distance": 0.5},
+        ]
+        out = dedupe_by_label(hits)
+        assert [h.get("label") for h in out] == [
+            "89 FR 62002 ¶5",
+            "IOM 100-02 ch.10 §30.1.1",
+            "",
+        ]
+        assert out[0]["item_key"] == "B"