5930a295bd
Backfills codes/families/elements onto the 1.15M chunks already in
pgvector without re-embedding: anchor_metadata is a pure function of
chunk text, so it's recomputed and merged into cmetadata with a jsonb
`||`, one transaction per batch, keyset-paginated on id. Idempotent —
a row whose three keys already match the freshly computed values is
skipped; a row missing a key (the pre-backfill norm) is patched even
when the computed value is "", since the acceptance check is key
presence (`cmetadata ? 'families'`), not value equality.
`stack llm restamp --collection all|comments|rules|corpus [--batch]
[--dry-run]` mirrors `index`'s config/engine setup, runs migrate() and
ensure_hnsw() first so the GIN indexes over families/elements exist,
refreshes the derived family registry from the DuckDB replica when
present, and builds the code->family index once per run via
pfs.anchors.code_family_index.
Live backfill (rig, pgvector at LLM_PG_HOST):
rules: scanned=67,340 updated=67,340 (168.4s, then 12,389 on the
A5 re-run after excluding single-code families)
corpus: scanned=235,140 updated=235,140 (626.1s, ~376 rows/s)
comments (~918k rows) launched in the background per Ruling A3.
54 lines
1.5 KiB
Python
54 lines
1.5 KiB
Python
"""llm.restamp — metadata-only anchor backfill on already-indexed chunks."""
|
|
|
|
import json
|
|
import os
|
|
|
|
import pytest
|
|
|
|
from llm.restamp import SELECT_SQL, UPDATE_SQL, plan_patches, restamp
|
|
|
|
|
|
def test_plan_skips_unchanged_and_patches_changed():
|
|
rows = [
|
|
(
|
|
"a",
|
|
"Use 99439 in conjunction with 99490; consent",
|
|
{
|
|
"codes": "99439 99490",
|
|
"families": "CCM",
|
|
"elements": "activity=consent relation=addon-of",
|
|
},
|
|
),
|
|
(
|
|
"b",
|
|
"Use 99439 in conjunction with 99490; consent",
|
|
{"codes": "99439 99490"},
|
|
),
|
|
("c", "nothing", {}),
|
|
]
|
|
patches = plan_patches(rows)
|
|
ids = [p[0] for p in patches]
|
|
assert ids == ["b", "c"]
|
|
assert json.loads(patches[0][1])["families"] == "CCM"
|
|
assert json.loads(patches[1][1]) == {"codes": "", "families": "", "elements": ""}
|
|
|
|
|
|
def test_sql_shapes():
|
|
assert (
|
|
"ORDER BY e.id" in SELECT_SQL
|
|
and ":after" in SELECT_SQL
|
|
and ":batch" in SELECT_SQL
|
|
)
|
|
assert "cmetadata || CAST(:patch AS jsonb)" in UPDATE_SQL
|
|
|
|
|
|
@pytest.mark.skipif(not os.environ.get("LLM_DB_PASSWORD"), reason="needs pgvector")
|
|
def test_restamp_dry_run_scans_live_rules():
|
|
from llm import config as llm_config
|
|
from llm.index import _engine
|
|
|
|
cfg = llm_config.load()
|
|
engine = _engine(cfg)
|
|
stats = restamp(engine, collection="rules", batch=50, dry_run=True)
|
|
assert stats["scanned"] > 0
|