b1e6d6ef3a
`stack llm restamp` called `ensure_hnsw`, which ALTERs the embedding column's type and rebuilds the HNSW index — ACCESS EXCLUSIVE for the life of a rebuild over 1M+ rows, locking out every reader for a metadata-only backfill that never touches embeddings. Split `_HNSW_DDL` into the column/HNSW part (unchanged `ensure_hnsw`, still called by `stack llm index` after a fresh run) and a new `ensure_metadata_indexes(engine)` holding only the three GIN indexes on codes/families/elements — no ACCESS EXCLUSIVE lock, safe on a live table. Restamp now calls only the latter, and skips it entirely under --dry-run.
76 lines
3.2 KiB
Python
76 lines
3.2 KiB
Python
"""llm.migrate — idempotent DDL for the llm database."""
|
|
|
|
from unittest.mock import MagicMock
|
|
|
|
from llm import migrate
|
|
|
|
|
|
class TestDdl:
|
|
def test_index_state_ddl_is_idempotent_sql(self):
|
|
assert "CREATE TABLE IF NOT EXISTS index_state" in migrate.INDEX_STATE_DDL
|
|
assert "PRIMARY KEY (item_key, collection)" in migrate.INDEX_STATE_DDL
|
|
|
|
def test_migrate_executes_ddl(self):
|
|
engine = MagicMock()
|
|
conn = engine.begin.return_value.__enter__.return_value
|
|
migrate.migrate(engine)
|
|
executed = " ".join(str(call.args[0]) for call in conn.execute.call_args_list)
|
|
assert "index_state" in executed
|
|
|
|
def test_ensure_hnsw_types_column_then_indexes(self):
|
|
engine = MagicMock()
|
|
conn = engine.begin.return_value.__enter__.return_value
|
|
migrate.ensure_hnsw(engine, 768)
|
|
executed = " ".join(str(call.args[0]) for call in conn.execute.call_args_list)
|
|
assert "vector(768)" in executed
|
|
assert "USING hnsw" in executed
|
|
assert (
|
|
"ix_embedding_collection" in executed
|
|
and "langchain_pg_embedding (collection_id)" in executed
|
|
)
|
|
# F6: the metadata GIN indexes are ensure_metadata_indexes' job now,
|
|
# not ensure_hnsw's — this must never take the ACCESS EXCLUSIVE lock.
|
|
assert "ix_embedding_codes" not in executed
|
|
assert "ix_embedding_families" not in executed
|
|
assert "ix_embedding_elements" not in executed
|
|
|
|
def test_ensure_metadata_indexes_issues_exactly_the_three_gin_statements(self):
|
|
engine = MagicMock()
|
|
conn = engine.begin.return_value.__enter__.return_value
|
|
migrate.ensure_metadata_indexes(engine)
|
|
calls = [str(call.args[0]) for call in conn.execute.call_args_list]
|
|
assert len(calls) == 3
|
|
executed = " ".join(calls)
|
|
# code-cited lookups scan cmetadata->>'codes' inside one collection
|
|
assert (
|
|
"ix_embedding_codes" in executed
|
|
and "USING gin (string_to_array" in executed
|
|
)
|
|
assert (
|
|
"ix_embedding_families" in executed and "cmetadata->>'families'" in executed
|
|
)
|
|
assert (
|
|
"ix_embedding_elements" in executed and "cmetadata->>'elements'" in executed
|
|
)
|
|
# No vector-column ALTER, no HNSW build — this must be safe on a
|
|
# live table.
|
|
assert "ALTER TABLE" not in executed
|
|
assert "hnsw" not in executed.lower()
|
|
|
|
def test_fingerprint_column_and_docket_state(self):
|
|
assert "fingerprint" in migrate.INDEX_STATE_DDL
|
|
assert "ADD COLUMN IF NOT EXISTS fingerprint" in migrate.INDEX_STATE_ALTER
|
|
assert (
|
|
"CREATE TABLE IF NOT EXISTS index_docket_state"
|
|
in migrate.INDEX_DOCKET_STATE_DDL
|
|
)
|
|
assert "PRIMARY KEY (collection, docket)" in migrate.INDEX_DOCKET_STATE_DDL
|
|
|
|
def test_migrate_runs_alter_and_docket_state(self):
|
|
engine = MagicMock()
|
|
conn = engine.begin.return_value.__enter__.return_value
|
|
migrate.migrate(engine)
|
|
executed = " ".join(str(call.args[0]) for call in conn.execute.call_args_list)
|
|
assert "ADD COLUMN IF NOT EXISTS fingerprint" in executed
|
|
assert "index_docket_state" in executed
|