dbf71a6594
Fix Zotero table models for Zotero 9:
- Remove stale Annotations/Highlights/Transaction* models
- Add ItemAnnotations, RetractedItems, DeletedCollections,
DeletedSearches, DbDebug1
- Fix ItemAttachments, Libraries, Users column mismatches
New test files covering all major modules:
- cli/{bib,prisma,rec,zot,mail,run} deep exercising tests
- mail/{droplet,postmark,resend,cloudflare} lifecycle tests
- bib/{iom,oig,pincite,sync,regulations_gov,email_ingest,format,store}
- prisma/{vpn,fetch,export,llm,screen,eligibility,extract,project,ingest,flow}
- aco/lake/{unity,quality,deploy} + api/aco coverage gaps
- zot/{ops,db,extract,duck} + rec/{report,engine,base,pricers}
- pfs/{pipe,rules,eq,files}
Add pytest-xdist for parallel test execution.
Tracks #353
233 lines
8.0 KiB
Python
233 lines
8.0 KiB
Python
"""Deeper tests for zot.ops — fix_dates + fix_keys with real data."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import sqlite3
|
|
|
|
from tests.zot.test_db import ZOTERO_SCHEMA, _seed_schema_maps
|
|
from zot.db import TYPE_MAP, Db
|
|
from zot.ops import dump_schema, fix_dates, fix_fields, fix_keys
|
|
|
|
|
|
class TestFixDatesWithData:
|
|
def test_normalizes_iso(self, tmp_path):
|
|
path = str(tmp_path / "z.sqlite")
|
|
con = sqlite3.connect(path)
|
|
con.executescript(ZOTERO_SCHEMA)
|
|
_seed_schema_maps(con)
|
|
con.commit()
|
|
con.close()
|
|
with Db(path) as db:
|
|
iid = db.create_item(TYPE_MAP["document"])
|
|
db.con.execute(
|
|
"UPDATE items SET dateAdded=? WHERE itemID=?",
|
|
("2023-07-15 10:30:00", iid),
|
|
)
|
|
db.commit()
|
|
result = fix_dates(path)
|
|
assert isinstance(result, dict)
|
|
|
|
|
|
class TestFixKeysWithData:
|
|
def test_replaces_short_keys(self, tmp_path):
|
|
path = str(tmp_path / "z.sqlite")
|
|
con = sqlite3.connect(path)
|
|
con.executescript(ZOTERO_SCHEMA)
|
|
_seed_schema_maps(con)
|
|
# Insert an item with a too-short key
|
|
con.execute("INSERT INTO items (itemTypeID, key) VALUES (14, 'AB')")
|
|
con.commit()
|
|
con.close()
|
|
result = fix_keys(path)
|
|
assert isinstance(result, dict)
|
|
|
|
|
|
class TestFixKeysDeeper:
|
|
def test_with_storage_rename(self, tmp_path):
|
|
path = str(tmp_path / "z.sqlite")
|
|
storage = tmp_path / "storage"
|
|
storage.mkdir()
|
|
con = sqlite3.connect(path)
|
|
con.executescript(ZOTERO_SCHEMA)
|
|
_seed_schema_maps(con)
|
|
con.execute(
|
|
"INSERT INTO items (itemTypeID, libraryID, key, dateAdded, dateModified, clientDateModified) "
|
|
"VALUES (14, 1, 'AB', '', '', '')"
|
|
)
|
|
con.commit()
|
|
con.close()
|
|
(storage / "AB").mkdir()
|
|
result = fix_keys(path, storage_dir=str(storage))
|
|
assert result["items"] >= 1
|
|
assert result["storage_renames"] >= 1
|
|
assert result["remaining"] == 0
|
|
|
|
def test_with_backup(self, tmp_path):
|
|
path = str(tmp_path / "z.sqlite")
|
|
con = sqlite3.connect(path)
|
|
con.executescript(ZOTERO_SCHEMA)
|
|
_seed_schema_maps(con)
|
|
con.execute(
|
|
"INSERT INTO items (itemTypeID, libraryID, key, dateAdded, dateModified, clientDateModified) "
|
|
"VALUES (14, 1, 'XY', '', '', '')"
|
|
)
|
|
con.commit()
|
|
con.close()
|
|
result = fix_keys(path, backup=True)
|
|
assert result["items"] >= 1
|
|
import pathlib
|
|
|
|
bak = pathlib.Path(path + ".pre-keyfix.bak")
|
|
assert bak.exists()
|
|
|
|
def test_missing_file(self, tmp_path):
|
|
import pytest
|
|
|
|
with pytest.raises(FileNotFoundError):
|
|
fix_keys(str(tmp_path / "nope.sqlite"))
|
|
|
|
def test_bad_collection_keys(self, tmp_path):
|
|
path = str(tmp_path / "z.sqlite")
|
|
con = sqlite3.connect(path)
|
|
con.executescript(ZOTERO_SCHEMA)
|
|
_seed_schema_maps(con)
|
|
con.execute(
|
|
"INSERT INTO collections (collectionName, libraryID, key) "
|
|
"VALUES ('Test', 1, '!@')"
|
|
)
|
|
con.commit()
|
|
con.close()
|
|
result = fix_keys(path)
|
|
assert result["collections"] >= 1
|
|
|
|
|
|
class TestFixFields:
|
|
"""fix_fields needs real Zotero combined views that don't exist in test schemas."""
|
|
|
|
def test_on_copy_of_real_db(self, tmp_path):
|
|
import shutil
|
|
from pathlib import Path
|
|
|
|
real = Path("data/zotero/data/zotero.sqlite")
|
|
if not real.exists():
|
|
import pytest
|
|
|
|
pytest.skip("real zotero.sqlite not available")
|
|
copy = tmp_path / "zotero_copy.sqlite"
|
|
shutil.copy2(real, copy)
|
|
result = fix_fields(str(copy))
|
|
assert isinstance(result, dict)
|
|
|
|
|
|
class TestFixKeysValidKeysSkipped:
|
|
"""Lines 171, 182: valid keys are skipped (continue); lines 200, 203: remaining count."""
|
|
|
|
def test_valid_items_skipped(self, tmp_path):
|
|
"""Items with valid 8-char keys are skipped; remaining is 0."""
|
|
path = str(tmp_path / "z.sqlite")
|
|
con = sqlite3.connect(path)
|
|
con.executescript(ZOTERO_SCHEMA)
|
|
_seed_schema_maps(con)
|
|
# Insert items with VALID keys — they should not be rewritten
|
|
con.execute(
|
|
"INSERT INTO items (itemTypeID, libraryID, key, dateAdded, dateModified, clientDateModified) "
|
|
"VALUES (14, 1, 'ABCD2345', '', '', '')"
|
|
)
|
|
con.execute(
|
|
"INSERT INTO items (itemTypeID, libraryID, key, dateAdded, dateModified, clientDateModified) "
|
|
"VALUES (14, 1, 'EFGH6789', '', '', '')"
|
|
)
|
|
# Also add a valid collection key
|
|
con.execute(
|
|
"INSERT INTO collections (collectionName, libraryID, key) "
|
|
"VALUES ('ValidCol', 1, 'MNPQ2345')"
|
|
)
|
|
con.commit()
|
|
con.close()
|
|
result = fix_keys(path, backup=False)
|
|
assert result["items"] == 0
|
|
assert result["collections"] == 0
|
|
assert result["remaining"] == 0
|
|
|
|
def test_remaining_count_with_bad_item_and_collection(self, tmp_path):
|
|
"""Lines 200, 203: remaining increments for invalid keys left behind."""
|
|
path = str(tmp_path / "z.sqlite")
|
|
con = sqlite3.connect(path)
|
|
con.executescript(ZOTERO_SCHEMA)
|
|
_seed_schema_maps(con)
|
|
con.execute(
|
|
"INSERT INTO items (itemTypeID, libraryID, key, dateAdded, dateModified, clientDateModified) "
|
|
"VALUES (14, 1, 'ab', '', '', '')"
|
|
)
|
|
con.commit()
|
|
con.close()
|
|
# This should fix the key; remaining should be 0
|
|
result = fix_keys(path, backup=False)
|
|
assert result["items"] >= 1
|
|
assert result["remaining"] == 0
|
|
|
|
|
|
class TestFixFieldsWithCombinedViews:
|
|
"""Lines 255, 261-263, 267-268, 272, 274, 278, 280, 284: fix_fields logic."""
|
|
|
|
def test_fix_fields_remap_and_delete(self, tmp_path):
|
|
"""Test with a real schema that has combined views."""
|
|
import shutil
|
|
from pathlib import Path
|
|
|
|
real = Path("data/zotero/data/zotero.sqlite")
|
|
if not real.exists():
|
|
import pytest
|
|
|
|
pytest.skip("real zotero.sqlite not available")
|
|
copy = tmp_path / "zotero_copy.sqlite"
|
|
shutil.copy2(real, copy)
|
|
|
|
con = sqlite3.connect(str(copy))
|
|
con.row_factory = sqlite3.Row
|
|
# Find a valid type and its base field mapping
|
|
bfm = con.execute(
|
|
"SELECT itemTypeID, baseFieldID, fieldID FROM baseFieldMappingsCombined LIMIT 1"
|
|
).fetchone()
|
|
if bfm:
|
|
type_id = bfm["itemTypeID"]
|
|
base_fid = bfm["baseFieldID"]
|
|
bfm["fieldID"]
|
|
# Create an item with the base fieldID (invalid for this type)
|
|
con.execute(
|
|
"INSERT INTO items (itemTypeID, libraryID, key, dateAdded, dateModified, clientDateModified) "
|
|
"VALUES (?, 1, 'FIXF2345', '', '', '')",
|
|
(type_id,),
|
|
)
|
|
iid = con.execute("SELECT last_insert_rowid()").fetchone()[0]
|
|
vid = con.execute(
|
|
"INSERT INTO itemDataValues (value) VALUES ('test_val')"
|
|
).lastrowid
|
|
con.execute(
|
|
"INSERT INTO itemData (itemID, fieldID, valueID) VALUES (?, ?, ?)",
|
|
(iid, base_fid, vid),
|
|
)
|
|
con.commit()
|
|
con.close()
|
|
|
|
result = fix_fields(str(copy))
|
|
assert isinstance(result, dict)
|
|
assert "remapped" in result
|
|
assert "deleted_no_mapping" in result
|
|
assert "deleted_conflict" in result
|
|
|
|
|
|
class TestDumpSchemaOutput:
|
|
def test_all_maps_present(self, tmp_path):
|
|
path = str(tmp_path / "z.sqlite")
|
|
con = sqlite3.connect(path)
|
|
con.executescript(ZOTERO_SCHEMA)
|
|
_seed_schema_maps(con)
|
|
con.commit()
|
|
con.close()
|
|
result = dump_schema(path)
|
|
assert "TYPE_MAP" in result
|
|
assert "FIELD_MAP" in result
|
|
assert "CREATOR_TYPES" in result
|
|
assert len(result["TYPE_MAP"]) > 30
|