diff --git a/src/bib/store.py b/src/bib/store.py index 4981e98..189f8d7 100644 --- a/src/bib/store.py +++ b/src/bib/store.py @@ -324,14 +324,23 @@ class Store: # every time, so a column-wise empty check can't apply — merge # the decoded dict per key instead: a stored key survives unless # the incoming payload sets it to a non-empty value. - cur_extra = json.loads(cur_row.get("extra_json") or "{}") + # Read the stored JSON itself, not current.to_row(): the model + # drops keys it does not declare (the tagger's llm_tags state, a + # rule's federal_register_citation), and rebuilding from it would + # erase them on any updating upsert. + stored_raw = con.execute( + "SELECT extra_json FROM items WHERE key = ?", (ekey,) + ).fetchone()[0] + cur_extra = json.loads(stored_raw or "{}") new_extra = json.loads(new_row.get("extra_json") or "{}") merged_extra = {**cur_extra, **{k: v for k, v in new_extra.items() if v}} merged_extra_json = json.dumps(merged_extra) new_row["extra_json"] = merged_extra_json - same_cols = all( - new_row.get(c, "") == cur_row.get(c, "") for c in self._COMPARE_COLS + same_cols = merged_extra == cur_extra and all( + new_row.get(c, "") == cur_row.get(c, "") + for c in self._COMPARE_COLS + if c != "extra_json" ) if ( same_cols diff --git a/tests/bib/test_store_upsert_noop.py b/tests/bib/test_store_upsert_noop.py index 9acfa3f..df2ca7a 100644 --- a/tests/bib/test_store_upsert_noop.py +++ b/tests/bib/test_store_upsert_noop.py @@ -143,3 +143,29 @@ def test_non_empty_incoming_extra_json_key_updates_that_key_only(): # ...but the blank incoming effective_date did not blank the stored one. assert stored.effective_date == "2026-01-01" assert stored.cms_id == "CMS-1848-P" + + +def test_unmodeled_extra_json_key_survives_an_updating_upsert(): + """Keys the item model does not declare (the tagger's llm_tags state, + a rule's federal_register_citation) live only in the stored JSON. An + upsert that changes the row must carry them over, not rebuild the + stored side from the model and drop them.""" + import json + + s = _store() + key, _ = s.upsert_status(_item(abstract="body")) + s.merge_extra(key, {"llm_tags": {"version": 2, "tags": {"telehealth": {}}}}) + _, status = s.upsert_status(_item(abstract="body, enriched")) + assert status == "updated" + row = ( + s._con().execute("SELECT extra_json FROM items WHERE key=?", (key,)).fetchone() + ) + assert json.loads(row[0])["llm_tags"]["version"] == 2 + + +def test_unmodeled_extra_json_key_alone_does_not_make_upsert_dirty(): + s = _store() + key, _ = s.upsert_status(_item()) + s.merge_extra(key, {"llm_tags": {"version": 2}}) + _, status = s.upsert_status(_item()) + assert status == "unchanged"