From 84a1d54c31dbb2626ce9f63821c735df326e47ce Mon Sep 17 00:00:00 2001
From: kert <kert@rack.localdomain>
Date: Thu, 24 Sep 2026 20:29:59 -0400
Subject: [PATCH] =?UTF-8?q?fix(llm):=20search=20returns=20each=20citation?=
 =?UTF-8?q?=20label=20once=20=E2=80=94=20the=20same=20FR=20paragraph=20ind?=
 =?UTF-8?q?exed=20under=20two=20bib=20items=20(2019-16041,=202024-14828=20?=
 =?UTF-8?q?each=20have=20two=20rule=20items)=20showed=20twice;=20keep=20th?=
 =?UTF-8?q?e=20closest=20hit?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 src/llm/search.py        | 18 ++++++++++++++++++
 tests/llm/test_search.py | 19 +++++++++++++++++++
 2 files changed, 37 insertions(+)

diff --git a/src/llm/search.py b/src/llm/search.py
index 7ffb76a..e300dd8 100644
--- a/src/llm/search.py
+++ b/src/llm/search.py
@@ -102,6 +102,23 @@ def _theme_ok(md: dict[str, Any], theme: str) -> bool:
     return theme in str(md.get("themes", "")).split(",")
 
 
+def dedupe_by_label(hits: list[dict]) -> list[dict]:
+    """One result per citation label, keeping the closest: the same FR
+    paragraph can be indexed under two bib items that hold the same rule
+    document, and the same chunk can be reached through two collections;
+    a reader wants each cite once."""
+    best: dict[str, dict] = {}
+    for h in hits:
+        key = h.get("label") or h.get("id") or ""
+        if not key:
+            best[id(h)] = h  # type: ignore[index]
+            continue
+        cur = best.get(key)
+        if cur is None or float(h.get("distance", 0)) < float(cur.get("distance", 0)):
+            best[key] = h
+    return list(best.values())
+
+
 def _hit_to_result(doc: Any, distance: float) -> dict:
     md = {k: str(v) for k, v in (doc.metadata or {}).items()}
     result = as_source(md, doc.page_content, _similarity(float(distance)))
@@ -157,6 +174,7 @@ def search(
             if not _theme_ok(doc.metadata or {}, theme):
                 continue
             hits.append(_hit_to_result(doc, distance))
+    hits = dedupe_by_label(hits)
     hits.sort(key=lambda h: h["distance"])
     return hits[offset : offset + limit]
 
diff --git a/tests/llm/test_search.py b/tests/llm/test_search.py
index d0d3b2d..1f18bc6 100644
--- a/tests/llm/test_search.py
+++ b/tests/llm/test_search.py
@@ -406,3 +406,22 @@ class TestThemeFilter:
             "$and": [{"doctype": "manual"}, theme_or]
         }
         assert _store_filter({"theme": "telehealth"}) == theme_or
+
+
+class TestDedupeByLabel:
+    def test_keeps_the_closest_per_label(self):
+        from llm.search import dedupe_by_label
+
+        hits = [
+            {"label": "89 FR 62002 ¶5", "distance": 0.30, "item_key": "A"},
+            {"label": "89 FR 62002 ¶5", "distance": 0.25, "item_key": "B"},
+            {"label": "IOM 100-02 ch.10 §30.1.1", "distance": 0.4},
+            {"label": "", "id": "", "distance": 0.5},
+        ]
+        out = dedupe_by_label(hits)
+        assert [h.get("label") for h in out] == [
+            "89 FR 62002 ¶5",
+            "IOM 100-02 ch.10 §30.1.1",
+            "",
+        ]
+        assert out[0]["item_key"] == "B"