homelab/stack
1
0
Fork 0
Code Issues 22 Pull Requests Actions Packages Projects 1 Releases Wiki Activity

feat: zot module — Zotero ORM, DuckDB engine, provenance extraction
Some checks failed
CI / skinny-install (bib) (push) Has been cancelled
Details
CI / skinny-install (bls) (push) Has been cancelled
Details
CI / skinny-install (ccw) (push) Has been cancelled
Details
CI / skinny-install (cli) (push) Has been cancelled
Details
CI / skinny-install (cms) (push) Has been cancelled
Details
CI / skinny-install (conf) (push) Has been cancelled
Details
CI / skinny-install (opps) (push) Has been cancelled
Details
CI / skinny-install (perf) (push) Has been cancelled
Details
CI / skinny-install (pfs) (push) Has been cancelled
Details
CI / skinny-install (rex) (push) Has been cancelled
Details
CI / skinny-install (aco) (push) Successful in 1m52s
Details
CI / skinny-install (api) (push) Successful in 37s
Details
CI / lint-test (push) Has been cancelled
Details
Deploy / build-scan-report (push) Has been cancelled
Details
Infra CI / notebooks (push) Has been cancelled
Details
Infra CI / zotero (push) Has been cancelled
Details
CI / skinny-install (bcda) (push) Has been cancelled
Details
Infra CI / docs (push) Has been cancelled
Details
Infra CI / api (push) Has been cancelled
Details
Infra CI / mc (push) Has been cancelled
Details

Browse Source 
Vendors Zotero's 61-table SQLite backend as a first-class Python module:

- zot.db.Db: full ORM with CRUD, EAV fields, creators, tags, collections,
  search, validation, bulk ops, structured reads, delete, stats
- zot.table: 61 Pydantic SQLTable models from real host-zotero.sqlite
- zot.schema: bootstrap fresh zotero.sqlite with complete DDL + 1,100 seed rows
- zot.duck.DuckDb: DuckDB engine with attach/mirror modes, flat views,
  analytics (type/tag/year/collection counts, tag co-occurrence, DOI/PDF
  coverage), Parquet export, Polars DataFrame conversion
- zot.extract.Extractor: parse Zotero HTML notes into Quote objects,
  generate :pincite: directives for docstrings, export provenance for
  knowledge graph ingestion

Fixes all Zotero ID constants across the codebase — every item type,
field, and creator type ID was wrong (fabricated, not from real schema).
Corrected: statute=20, report=15, webpage=13, document=34,
journalArticle=4, attachment=14; title=110, url=1, DOI=26, etc.

Refactors bib.sync to use zot.db.Db instead of raw SQL.
Fixes 6 dev/scripts that had wrong hardcoded IDs and duplicated helpers.
Adds compose mount validation tests, fixes 2 pre-existing test failures.
Closes #247, #249, #250 and 14 pkg-vuln issues.

12,634 tests passing, 0 failures.
This commit is contained in:
kert
2026-04-09 22:21:43 -04:00
parent 982777addb
commit c3edf11bb6
31 changed files with 7015 additions and 957 deletions
Download Patch File Download Diff File Expand all files Collapse all files

138
dev/scripts/add_carrier_to_zotero.py
View File

@@ -7,14 +7,12 @@ Usage:
uv run python dev/scripts/add_carrier_to_zotero.py uv run python dev/scripts/add_carrier_to_zotero.py
""" """
import random
import sqlite3
import subprocess import subprocess
import zipfile import zipfile
from datetime import datetime, timezone
from pathlib import Path from pathlib import Path
from conf import path as _conf_path from conf import path as _conf_path
from zot.db import TYPE_MAP, Db, generate_key, now_iso
ZOTERO_DB = str(_conf_path("db.zotero")) ZOTERO_DB = str(_conf_path("db.zotero"))
ZOTERO_STORAGE = str(_conf_path("storage.zotero")) ZOTERO_STORAGE = str(_conf_path("storage.zotero"))
@@ -35,95 +33,33 @@ CARRIER_FILES = {
}, },
} }
# Zotero schema constants def add_carrier_year(db: Db, year: int, info: dict) -> None:
ITEM_TYPE_WEBPAGE = 40
ITEM_TYPE_ATTACHMENT = 3
FIELD_TITLE = 1
FIELD_DATE = 6
FIELD_URL = 10
FIELD_ACCESS_DATE = 11
FIELD_WEBSITE_TYPE = 42
FIELD_WEBSITE_TITLE = 123
TAG_MODULE_PFS = 8
TAG_YEARS = {2010: 36, 2017: 20}
def _zotero_key() -> str:
"""Generate a random 8-char Zotero key using the allowed character set."""
chars = "23456789ABCDEFGHIJKLMNPQRSTUVWXYZ"
return "".join(random.choices(chars, k=8))
def _get_or_create_value(db: sqlite3.Connection, value: str) -> int:
"""Get or create an itemDataValues row, return valueID."""
row = db.execute(
"SELECT valueID FROM itemDataValues WHERE value = ?", (value,)
).fetchone()
if row:
return row[0]
cur = db.execute("INSERT INTO itemDataValues (value) VALUES (?)", (value,))
return cur.lastrowid
def _next_item_id(db: sqlite3.Connection) -> int:
return db.execute("SELECT max(itemID) + 1 FROM items").fetchone()[0]
def add_carrier_year(db: sqlite3.Connection, year: int, info: dict) -> None:
"""Add one carrier year to Zotero: parent item + file attachments.""" """Add one carrier year to Zotero: parent item + file attachments."""
now = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") now = now_iso()
# --- Create parent item (webpage) --- # Create parent webpage item
parent_id = _next_item_id(db) parent_id = db.create_item(TYPE_MAP["webpage"], now=now)
parent_key = _zotero_key() db.set_fields(parent_id, {
db.execute( "title": info["title"],
"""INSERT INTO items (itemID, itemTypeID, dateAdded, dateModified, "date": f"{year}-01-01",
clientDateModified, key, version, synced, libraryID) "url": info["url"],
VALUES (?, ?, ?, ?, ?, ?, 0, 0, 1)""", "accessDate": now,
(parent_id, ITEM_TYPE_WEBPAGE, now, now, now, parent_key), "websiteType": "Government Data Portal",
) "websiteTitle": "Centers for Medicare & Medicaid Services",
})
db.sync_tags(parent_id, ["module:pfs", f"year:{year}"])
# Set fields: title, date, url, accessDate, websiteType, websiteTitle print(f"Created parent item for {info['title']}")
fields = {
FIELD_TITLE: info["title"],
FIELD_DATE: f"{year}-01-01",
FIELD_URL: info["url"],
FIELD_ACCESS_DATE: now,
FIELD_WEBSITE_TYPE: "Government Data Portal",
FIELD_WEBSITE_TITLE: "Centers for Medicare & Medicaid Services",
}
for field_id, value in fields.items():
value_id = _get_or_create_value(db, value)
db.execute(
"INSERT INTO itemData (itemID, fieldID, valueID) VALUES (?, ?, ?)",
(parent_id, field_id, value_id),
)
# Tags: module:pfs + year:YYYY # Create attachments for each .TXT and .pdf file
db.execute(
"INSERT INTO itemTags (itemID, tagID, type) VALUES (?, ?, 0)",
(parent_id, TAG_MODULE_PFS),
)
db.execute(
"INSERT INTO itemTags (itemID, tagID, type) VALUES (?, ?, 0)",
(parent_id, TAG_YEARS[year]),
)
print(f"Created parent item: {parent_key} ({info['title']})")
# --- Create attachments for each .TXT and .pdf file ---
extracted_dir = Path(info["extracted"]) extracted_dir = Path(info["extracted"])
files = sorted(extracted_dir.iterdir())
att_count = 0 att_count = 0
for filepath in files: for filepath in sorted(extracted_dir.iterdir()):
if filepath.suffix.upper() not in (".TXT", ".PDF"): if filepath.suffix.upper() not in (".TXT", ".PDF"):
continue continue
att_id = _next_item_id(db) att_key = generate_key()
att_key = _zotero_key()
# Copy file to Zotero storage (owned by container uid 100999)
storage_dir = Path(ZOTERO_STORAGE) / att_key storage_dir = Path(ZOTERO_STORAGE) / att_key
subprocess.run(["sudo", "mkdir", "-p", str(storage_dir)], check=True) subprocess.run(["sudo", "mkdir", "-p", str(storage_dir)], check=True)
dest = storage_dir / filepath.name dest = storage_dir / filepath.name
@@ -133,40 +69,22 @@ def add_carrier_year(db: sqlite3.Connection, year: int, info: dict) -> None:
check=True, check=True,
) )
# Determine content type
ext = filepath.suffix.upper() ext = filepath.suffix.upper()
content_type = "text/plain" if ext == ".TXT" else "application/pdf" content_type = "text/plain" if ext == ".TXT" else "application/pdf"
# Create item record att_id = db.add_attachment(
db.execute( parent_id,
"""INSERT INTO items (itemID, itemTypeID, dateAdded, dateModified, key=att_key,
key, version, synced, libraryID) content_type=content_type,
VALUES (?, ?, ?, ?, ?, 0, 0, 1)""", path=f"storage:{filepath.name}",
(att_id, ITEM_TYPE_ATTACHMENT, now, now, now, att_key),
) )
db.set_field(att_id, "title", filepath.name)
# Create attachment record (linkMode=0 = imported file)
db.execute(
"""INSERT INTO itemAttachments
(itemID, parentItemID, linkMode, contentType, path)
VALUES (?, ?, 0, ?, ?)""",
(att_id, parent_id, content_type, f"storage:{filepath.name}"),
)
# Set title field on attachment
value_id = _get_or_create_value(db, filepath.name)
db.execute(
"INSERT INTO itemData (itemID, fieldID, valueID) VALUES (?, ?, ?)",
(att_id, FIELD_TITLE, value_id),
)
att_count += 1 att_count += 1
print(f" Added {att_count} attachments for year {year}") print(f" Added {att_count} attachments for year {year}")
def main() -> None: def main() -> None:
# Verify extracted files exist
for year, info in CARRIER_FILES.items(): for year, info in CARRIER_FILES.items():
extracted = Path(info["extracted"]) extracted = Path(info["extracted"])
if not extracted.exists(): if not extracted.exists():
@@ -177,17 +95,11 @@ def main() -> None:
txt_count = len(list(extracted.glob("*.TXT"))) txt_count = len(list(extracted.glob("*.TXT")))
print(f"Year {year}: {txt_count} .TXT files ready") print(f"Year {year}: {txt_count} .TXT files ready")
db = sqlite3.connect(ZOTERO_DB) with Db(ZOTERO_DB) as db:
try:
for year, info in sorted(CARRIER_FILES.items()): for year, info in sorted(CARRIER_FILES.items()):
add_carrier_year(db, year, info) add_carrier_year(db, year, info)
db.commit() db.commit()
print("\nDone. Committed to Zotero database.") print("\nDone. Committed to Zotero database.")
except Exception:
db.rollback()
raise
finally:
db.close()
if __name__ == "__main__": if __name__ == "__main__":

127
dev/scripts/add_zipcode_to_zotero.py
View File

@@ -7,14 +7,13 @@ Usage:
uv run python dev/scripts/add_zipcode_to_zotero.py uv run python dev/scripts/add_zipcode_to_zotero.py
""" """
import random
import sqlite3 import sqlite3
import subprocess import subprocess
import zipfile import zipfile
from datetime import datetime, timezone
from pathlib import Path from pathlib import Path
from conf import path as _conf_path from conf import path as _conf_path
from zot.db import FIELD_MAP, TYPE_MAP, Db, generate_key, now_iso
ZOTERO_DB = str(_conf_path("db.zotero")) ZOTERO_DB = str(_conf_path("db.zotero"))
ZOTERO_STORAGE = str(_conf_path("storage.zotero")) ZOTERO_STORAGE = str(_conf_path("storage.zotero"))
@@ -77,48 +76,8 @@ ZIPCODE_FILES = {
}, },
} }
# Zotero schema constants def add_zipcode_year(db: Db, year: int, info: dict) -> None:
ITEM_TYPE_WEBPAGE = 40 now = now_iso()
ITEM_TYPE_ATTACHMENT = 3
FIELD_TITLE = 1
FIELD_DATE = 6
FIELD_URL = 10
FIELD_ACCESS_DATE = 11
FIELD_WEBSITE_TYPE = 42
FIELD_WEBSITE_TITLE = 123
TAG_MODULE_PFS = 8
def _zotero_key() -> str:
"""Generate a random 8-char Zotero key using the allowed character set."""
chars = "23456789ABCDEFGHIJKLMNPQRSTUVWXYZ"
return "".join(random.choices(chars, k=8))
def _get_or_create_value(db: sqlite3.Connection, value: str) -> int:
row = db.execute(
"SELECT valueID FROM itemDataValues WHERE value = ?", (value,)
).fetchone()
if row:
return row[0]
cur = db.execute("INSERT INTO itemDataValues (value) VALUES (?)", (value,))
return cur.lastrowid
def _get_or_create_tag(db: sqlite3.Connection, name: str) -> int:
row = db.execute("SELECT tagID FROM tags WHERE name = ?", (name,)).fetchone()
if row:
return row[0]
cur = db.execute("INSERT INTO tags (name) VALUES (?)", (name,))
return cur.lastrowid
def _next_item_id(db: sqlite3.Connection) -> int:
return db.execute("SELECT max(itemID) + 1 FROM items").fetchone()[0]
def add_zipcode_year(db: sqlite3.Connection, year: int, info: dict) -> None:
now = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
# Extract ZIP # Extract ZIP
extract_dir = Path(f"/tmp/pfs_zipcode/ex_{year}") extract_dir = Path(f"/tmp/pfs_zipcode/ex_{year}")
@@ -126,43 +85,21 @@ def add_zipcode_year(db: sqlite3.Connection, year: int, info: dict) -> None:
with zipfile.ZipFile(info["zip"]) as zf: with zipfile.ZipFile(info["zip"]) as zf:
zf.extractall(extract_dir) zf.extractall(extract_dir)
# Create parent item # Create parent webpage item
parent_id = _next_item_id(db) parent_id = db.create_item(TYPE_MAP["webpage"], now=now)
parent_key = _zotero_key() db.set_fields(parent_id, {
db.execute( "title": info["title"],
"""INSERT INTO items (itemID, itemTypeID, dateAdded, dateModified, "date": f"{year}-01-01",
clientDateModified, key, version, synced, libraryID) "url": info["url"],
VALUES (?, ?, ?, ?, ?, ?, 0, 0, 1)""", "accessDate": now,
(parent_id, ITEM_TYPE_WEBPAGE, now, now, now, parent_key), "websiteType": "Government Data Portal",
) "websiteTitle": "Centers for Medicare & Medicaid Services",
})
fields = {
FIELD_TITLE: info["title"],
FIELD_DATE: f"{year}-01-01",
FIELD_URL: info["url"],
FIELD_ACCESS_DATE: now,
FIELD_WEBSITE_TYPE: "Government Data Portal",
FIELD_WEBSITE_TITLE: "Centers for Medicare & Medicaid Services",
}
for field_id, value in fields.items():
value_id = _get_or_create_value(db, value)
db.execute(
"INSERT INTO itemData (itemID, fieldID, valueID) VALUES (?, ?, ?)",
(parent_id, field_id, value_id),
)
# Tags: module:pfs + year:YYYY # Tags: module:pfs + year:YYYY
db.execute( db.sync_tags(parent_id, ["module:pfs", f"year:{year}"])
"INSERT INTO itemTags (itemID, tagID, type) VALUES (?, ?, 0)",
(parent_id, TAG_MODULE_PFS),
)
year_tag_id = _get_or_create_tag(db, f"year:{year}")
db.execute(
"INSERT INTO itemTags (itemID, tagID, type) VALUES (?, ?, 0)",
(parent_id, year_tag_id),
)
print(f"Created parent item: {parent_key} ({info['title']})") print(f"Created parent item for {info['title']}")
# Add attachments for .txt and .xlsx files # Add attachments for .txt and .xlsx files
att_count = 0 att_count = 0
@@ -171,9 +108,7 @@ def add_zipcode_year(db: sqlite3.Connection, year: int, info: dict) -> None:
if ext not in (".TXT", ".XLSX"): if ext not in (".TXT", ".XLSX"):
continue continue
att_id = _next_item_id(db) att_key = generate_key()
att_key = _zotero_key()
storage_dir = Path(ZOTERO_STORAGE) / att_key storage_dir = Path(ZOTERO_STORAGE) / att_key
subprocess.run(["sudo", "mkdir", "-p", str(storage_dir)], check=True) subprocess.run(["sudo", "mkdir", "-p", str(storage_dir)], check=True)
dest = storage_dir / filepath.name dest = storage_dir / filepath.name
@@ -189,23 +124,13 @@ def add_zipcode_year(db: sqlite3.Connection, year: int, info: dict) -> None:
} }
content_type = ct_map.get(ext, "application/octet-stream") content_type = ct_map.get(ext, "application/octet-stream")
db.execute( att_id = db.add_attachment(
"""INSERT INTO items (itemID, itemTypeID, dateAdded, dateModified, parent_id,
key, version, synced, libraryID) key=att_key,
VALUES (?, ?, ?, ?, ?, 0, 0, 1)""", content_type=content_type,
(att_id, ITEM_TYPE_ATTACHMENT, now, now, now, att_key), path=f"storage:{filepath.name}",
)
db.execute(
"""INSERT INTO itemAttachments
(itemID, parentItemID, linkMode, contentType, path)
VALUES (?, ?, 0, ?, ?)""",
(att_id, parent_id, content_type, f"storage:{filepath.name}"),
)
value_id = _get_or_create_value(db, filepath.name)
db.execute(
"INSERT INTO itemData (itemID, fieldID, valueID) VALUES (?, ?, ?)",
(att_id, FIELD_TITLE, value_id),
) )
db.set_field(att_id, "title", filepath.name)
att_count += 1 att_count += 1
print(f" Added {att_count} attachments for year {year}") print(f" Added {att_count} attachments for year {year}")
@@ -218,17 +143,11 @@ def main() -> None:
print(f"ERROR: {zp} not found — download first") print(f"ERROR: {zp} not found — download first")
return return
db = sqlite3.connect(ZOTERO_DB) with Db(ZOTERO_DB) as db:
try:
for year, info in sorted(ZIPCODE_FILES.items()): for year, info in sorted(ZIPCODE_FILES.items()):
add_zipcode_year(db, year, info) add_zipcode_year(db, year, info)
db.commit() db.commit()
print("\nDone. Committed to Zotero database.") print("\nDone. Committed to Zotero database.")
except Exception:
db.rollback()
raise
finally:
db.close()
if __name__ == "__main__": if __name__ == "__main__":

3
dev/scripts/check_pdf_coverage.py
View File

@@ -14,6 +14,7 @@ import sqlite3
from pathlib import Path from pathlib import Path
from conf import path as conf_path from conf import path as conf_path
from zot.db import FIELD_MAP
ZOTERO_DB = str(conf_path("db.zotero")) ZOTERO_DB = str(conf_path("db.zotero"))
@@ -37,7 +38,7 @@ def main() -> None:
JOIN itemTags it ON i.itemID = it.itemID JOIN itemTags it ON i.itemID = it.itemID
JOIN tags t ON it.tagID = t.tagID JOIN tags t ON it.tagID = t.tagID
JOIN itemData id ON i.itemID = id.itemID JOIN itemData id ON i.itemID = id.itemID
WHERE t.name = 'module:skin-subs' AND id.fieldID = 8 WHERE t.name = 'module:skin-subs' AND id.fieldID = {FIELD_MAP['DOI']}
""").fetchone()[0] """).fetchone()[0]
# Items with PDF attachments # Items with PDF attachments

37
dev/scripts/fetch_pdfs.py
View File

@@ -29,6 +29,7 @@ from pathlib import Path
import httpx import httpx
from conf import path as conf_path from conf import path as conf_path
from zot.db import FIELD_MAP, TYPE_MAP, generate_key, now_iso
ZOTERO_DB = str(conf_path("db.zotero")) ZOTERO_DB = str(conf_path("db.zotero"))
STORAGE_DIR = Path(conf_path("storage.zotero")) STORAGE_DIR = Path(conf_path("storage.zotero"))
@@ -36,19 +37,6 @@ STORAGE_DIR = Path(conf_path("storage.zotero"))
UNPAYWALL_EMAIL = "dev@fhirworx.io" UNPAYWALL_EMAIL = "dev@fhirworx.io"
USER_AGENT = "stack-pdf-fetcher/1.0 (mailto:{})".format(UNPAYWALL_EMAIL) USER_AGENT = "stack-pdf-fetcher/1.0 (mailto:{})".format(UNPAYWALL_EMAIL)
# Zotero key charset for generating attachment keys
_ALLOWED = "23456789ABCDEFGHIJKLMNPQRSTUVWXYZ"
def _zotero_key() -> str:
import random
return "".join(random.choices(_ALLOWED, k=8)) # noqa: S311
def _now() -> str:
from datetime import datetime, timezone
return datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M:%S")
# --------------------------------------------------------------------------- # ---------------------------------------------------------------------------
# Source: Unpaywall (legal, no auth, ~50% coverage) # Source: Unpaywall (legal, no auth, ~50% coverage)
@@ -263,19 +251,19 @@ def attach_pdf_to_item(
con: sqlite3.Connection, item_id: int, pdf_path: Path, title: str = "" con: sqlite3.Connection, item_id: int, pdf_path: Path, title: str = ""
) -> bool: ) -> bool:
"""Create a Zotero attachment record for a downloaded PDF.""" """Create a Zotero attachment record for a downloaded PDF."""
key = _zotero_key() key = generate_key()
while con.execute("SELECT 1 FROM items WHERE key = ?", (key,)).fetchone(): while con.execute("SELECT 1 FROM items WHERE key = ?", (key,)).fetchone():
key = _zotero_key() key = generate_key()
now = _now() now = now_iso()
# Create attachment item (itemTypeID=28 = attachment) # Create attachment item (itemTypeID=14 = attachment)
cur = con.execute( cur = con.execute(
"""INSERT INTO items """INSERT INTO items
(itemTypeID, dateAdded, dateModified, clientDateModified, (itemTypeID, dateAdded, dateModified, clientDateModified,
libraryID, key, version, synced) libraryID, key, version, synced)
VALUES (28, ?, ?, ?, 1, ?, 0, 0)""", VALUES (?, ?, ?, ?, 1, ?, 0, 0)""",
(now, now, now, key), (TYPE_MAP["attachment"], now, now, now, key),
) )
att_item_id = cur.lastrowid att_item_id = cur.lastrowid
@@ -311,22 +299,19 @@ def get_items_needing_pdfs(con: sqlite3.Connection) -> list[dict]:
SELECT DISTINCT i.itemID, SELECT DISTINCT i.itemID,
(SELECT idv.value FROM itemData id (SELECT idv.value FROM itemData id
JOIN itemDataValues idv ON id.valueID = idv.valueID JOIN itemDataValues idv ON id.valueID = idv.valueID
WHERE id.itemID = i.itemID AND id.fieldID = 8) AS doi, WHERE id.itemID = i.itemID AND id.fieldID = ?) AS doi
(SELECT idv.value FROM itemData id
JOIN itemDataValues idv ON id.valueID = idv.valueID
WHERE id.itemID = i.itemID AND id.fieldID = 86) AS pmid
FROM items i FROM items i
JOIN itemTags it ON i.itemID = it.itemID JOIN itemTags it ON i.itemID = it.itemID
JOIN tags t ON it.tagID = t.tagID JOIN tags t ON it.tagID = t.tagID
WHERE t.name = 'module:skin-subs' WHERE t.name = 'module:skin-subs'
AND i.itemTypeID = 22 AND i.itemTypeID = ?
AND i.itemID NOT IN ( AND i.itemID NOT IN (
SELECT DISTINCT parentItemID FROM itemAttachments SELECT DISTINCT parentItemID FROM itemAttachments
WHERE parentItemID IS NOT NULL WHERE parentItemID IS NOT NULL
AND contentType = 'application/pdf' AND contentType = 'application/pdf'
) )
""").fetchall() """, (FIELD_MAP["DOI"], TYPE_MAP["journalArticle"])).fetchall()
return [{"item_id": r[0], "doi": r[1], "pmid": r[2]} return [{"item_id": r[0], "doi": r[1]}
for r in rows if r[1]] for r in rows if r[1]]

173
dev/scripts/register_seeds.py
View File

@@ -14,32 +14,22 @@ Usage::
from __future__ import annotations from __future__ import annotations
import argparse import argparse
import random
import shutil import shutil
import sqlite3 import sqlite3
import string
import subprocess import subprocess
from dataclasses import dataclass, field from dataclasses import dataclass, field
from datetime import datetime, timezone
from pathlib import Path from pathlib import Path
from conf import path as _conf_path from conf import path as _conf_path
from zot.db import TYPE_MAP, Db, generate_key, now_iso
SEEDS_DIR = Path("dev/seeds") SEEDS_DIR = Path("dev/seeds")
BIB_DB = str(_conf_path("db.bib")) BIB_DB = str(_conf_path("db.bib"))
ZOTERO_DB = str(_conf_path("db.zotero")) ZOTERO_DB = str(_conf_path("db.zotero"))
ZOTERO_STORAGE = _conf_path("storage.zotero") ZOTERO_STORAGE = _conf_path("storage.zotero")
# Zotero schema constants (match add_carrier_to_zotero.py / sync.py)
ZOTERO_TYPE_WEBPAGE = 40
ZOTERO_TYPE_ATTACHMENT = 3
ZOTERO_FIELD_TITLE = 1
ZOTERO_FIELD_URL = 10
ZOTERO_FIELD_DATE = 6
ZOTERO_FIELD_ACCESS_DATE = 11
ZOTERO_FIELD_WEBSITE_TYPE = 42
ZOTERO_FIELD_WEBSITE_TITLE = 123
SEED_TAG = "seed:true" SEED_TAG = "seed:true"
@@ -304,43 +294,8 @@ SEED_REGISTRY: tuple[SeedSpec, ...] = (
# ── Helpers ─────────────────────────────────────────────────────── # ── Helpers ───────────────────────────────────────────────────────
def _zotero_key() -> str:
chars = string.ascii_uppercase + string.digits
return "".join(random.choices(chars, k=8)) # noqa: S311
def _now_iso() -> str:
return datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
def _ensure_value(con: sqlite3.Connection, value: str) -> int:
row = con.execute(
"SELECT valueID FROM itemDataValues WHERE value = ?", (value,)
).fetchone()
if row:
return row[0]
cur = con.execute("INSERT INTO itemDataValues (value) VALUES (?)", (value,))
return cur.lastrowid
def _ensure_tag(con: sqlite3.Connection, name: str) -> int:
row = con.execute("SELECT tagID FROM tags WHERE name = ?", (name,)).fetchone()
if row:
return row[0]
cur = con.execute("INSERT INTO tags (name) VALUES (?)", (name,))
return cur.lastrowid
def _set_zotero_field(
con: sqlite3.Connection, item_id: int, field_id: int, value: str
) -> None:
if not value:
return
value_id = _ensure_value(con, value)
con.execute(
"INSERT OR REPLACE INTO itemData (itemID, fieldID, valueID) VALUES (?, ?, ?)",
(item_id, field_id, value_id),
)
def _content_type(path: Path) -> str: def _content_type(path: Path) -> str:
@@ -363,11 +318,11 @@ def _content_type(path: Path) -> str:
def tag_existing_zotero( def tag_existing_zotero(
con: sqlite3.Connection, spec: SeedSpec, *, dry_run: bool = False db: Db, spec: SeedSpec, *, dry_run: bool = False
) -> bool: ) -> bool:
"""Find a Zotero attachment by filename and tag its parent.""" """Find a Zotero attachment by filename and tag its parent."""
filename = Path(spec.path).name filename = Path(spec.path).name
row = con.execute( row = db.con.execute(
"""SELECT ia.parentItemID """SELECT ia.parentItemID
FROM itemAttachments ia FROM itemAttachments ia
WHERE ia.path LIKE ? WHERE ia.path LIKE ?
@@ -382,14 +337,7 @@ def tag_existing_zotero(
print(f" [exists] Zotero parent itemID={parent_id}") print(f" [exists] Zotero parent itemID={parent_id}")
return True return True
# Add seed tag + any extra tags db.sync_tags(parent_id, [SEED_TAG] + spec.tags)
all_tags = [SEED_TAG] + spec.tags
for tag_name in all_tags:
tag_id = _ensure_tag(con, tag_name)
con.execute(
"INSERT OR IGNORE INTO itemTags (itemID, tagID, type) VALUES (?, ?, 0)",
(parent_id, tag_id),
)
return True return True
@@ -397,7 +345,7 @@ def tag_existing_zotero(
def register_in_zotero( def register_in_zotero(
con: sqlite3.Connection, spec: SeedSpec, *, dry_run: bool = False db: Db, spec: SeedSpec, *, dry_run: bool = False
) -> None: ) -> None:
"""Create a Zotero webpage item + file attachment for a seed.""" """Create a Zotero webpage item + file attachment for a seed."""
if not spec.url: if not spec.url:
@@ -409,51 +357,24 @@ def register_in_zotero(
print(" [create] new Zotero webpage item") print(" [create] new Zotero webpage item")
return return
now = _now_iso() now = now_iso()
next_id = con.execute("SELECT COALESCE(MAX(itemID), 0) + 1 FROM items").fetchone()[
0
]
parent_key = _zotero_key()
# Create parent webpage item # Create parent webpage item
con.execute( parent_id = db.create_item(TYPE_MAP["webpage"], now=now)
"""INSERT INTO items db.set_fields(parent_id, {
(itemID, itemTypeID, dateAdded, dateModified, "title": spec.title,
clientDateModified, key, version, synced, libraryID) "url": spec.url,
VALUES (?, ?, ?, ?, ?, ?, 0, 0, 1)""", "date": now[:10],
(next_id, ZOTERO_TYPE_WEBPAGE, now, now, now, parent_key), "accessDate": now,
) "websiteType": "Government Data Portal",
"websiteTitle": "Centers for Medicare & Medicaid Services",
_set_zotero_field(con, next_id, ZOTERO_FIELD_TITLE, spec.title) })
_set_zotero_field(con, next_id, ZOTERO_FIELD_URL, spec.url) db.sync_tags(parent_id, [SEED_TAG] + spec.tags)
_set_zotero_field(con, next_id, ZOTERO_FIELD_DATE, now[:10])
_set_zotero_field(con, next_id, ZOTERO_FIELD_ACCESS_DATE, now)
_set_zotero_field(con, next_id, ZOTERO_FIELD_WEBSITE_TYPE, "Government Data Portal")
_set_zotero_field(
con,
next_id,
ZOTERO_FIELD_WEBSITE_TITLE,
"Centers for Medicare & Medicaid Services",
)
# Tags
all_tags = [SEED_TAG] + spec.tags
for tag_name in all_tags:
tag_id = _ensure_tag(con, tag_name)
con.execute(
"INSERT OR IGNORE INTO itemTags (itemID, tagID, type) VALUES (?, ?, 0)",
(next_id, tag_id),
)
# Attach file (skip directories) # Attach file (skip directories)
seed_path = SEEDS_DIR / spec.path seed_path = SEEDS_DIR / spec.path
if seed_path.is_file(): if seed_path.is_file():
att_id = con.execute( att_key = generate_key()
"SELECT COALESCE(MAX(itemID), 0) + 1 FROM items"
).fetchone()[0]
att_key = _zotero_key()
# Copy to Zotero storage
storage_dir = ZOTERO_STORAGE / att_key storage_dir = ZOTERO_STORAGE / att_key
subprocess.run(["sudo", "mkdir", "-p", str(storage_dir)], check=True) subprocess.run(["sudo", "mkdir", "-p", str(storage_dir)], check=True)
dest = storage_dir / seed_path.name dest = storage_dir / seed_path.name
@@ -463,29 +384,13 @@ def register_in_zotero(
check=True, check=True,
) )
con.execute( att_id = db.add_attachment(
"""INSERT INTO items parent_id,
(itemID, itemTypeID, dateAdded, dateModified, key=att_key,
clientDateModified, key, version, synced, libraryID) content_type=_content_type(seed_path),
VALUES (?, ?, ?, ?, ?, ?, 0, 0, 1)""", path=f"storage:{seed_path.name}",
(att_id, ZOTERO_TYPE_ATTACHMENT, now, now, now, att_key),
)
con.execute(
"""INSERT INTO itemAttachments
(itemID, parentItemID, linkMode, contentType, path)
VALUES (?, ?, 0, ?, ?)""",
(
att_id,
next_id,
_content_type(seed_path),
f"storage:{seed_path.name}",
),
)
title_val = _ensure_value(con, seed_path.name)
con.execute(
"INSERT INTO itemData (itemID, fieldID, valueID) VALUES (?, ?, ?)",
(att_id, ZOTERO_FIELD_TITLE, title_val),
) )
db.set_field(att_id, "title", seed_path.name)
# ── Phase C: register in bib.sqlite ────────────────────────────── # ── Phase C: register in bib.sqlite ──────────────────────────────
@@ -534,8 +439,8 @@ def register_in_bib(spec: SeedSpec, *, dry_run: bool = False) -> str:
def pull_from_zotero(*, dry_run: bool = False) -> int: def pull_from_zotero(*, dry_run: bool = False) -> int:
"""Copy seed-tagged Zotero attachments missing from dev/seeds/.""" """Copy seed-tagged Zotero attachments missing from dev/seeds/."""
con = sqlite3.connect(ZOTERO_DB, timeout=10) with Db(ZOTERO_DB) as db:
rows = con.execute( rows = db.con.execute(
"""SELECT ia.path, ia.parentItemID """SELECT ia.path, ia.parentItemID
FROM itemAttachments ia FROM itemAttachments ia
JOIN items i ON ia.itemID = i.itemID JOIN items i ON ia.itemID = i.itemID
@@ -546,7 +451,6 @@ def pull_from_zotero(*, dry_run: bool = False) -> int:
AND ia.linkMode = 0""", AND ia.linkMode = 0""",
(SEED_TAG,), (SEED_TAG,),
).fetchall() ).fetchall()
con.close()
copied = 0 copied = 0
for row in rows: for row in rows:
@@ -601,15 +505,14 @@ def main() -> None:
zotero_db_path = Path(ZOTERO_DB) zotero_db_path = Path(ZOTERO_DB)
has_zotero = zotero_db_path.exists() has_zotero = zotero_db_path.exists()
zcon = None zdb = None
if has_zotero: if has_zotero:
try: try:
zcon = sqlite3.connect(ZOTERO_DB, timeout=10) zdb = Db(ZOTERO_DB)
# Test that we can actually read zdb.con.execute("SELECT 1 FROM items LIMIT 1")
zcon.execute("SELECT 1 FROM items LIMIT 1") except Exception as exc:
except sqlite3.OperationalError as exc:
print(f"Zotero DB locked ({exc}), skipping Zotero phases") print(f"Zotero DB locked ({exc}), skipping Zotero phases")
zcon = None zdb = None
total = len(SEED_REGISTRY) total = len(SEED_REGISTRY)
tagged = 0 tagged = 0
@@ -627,22 +530,22 @@ def main() -> None:
continue continue
# Phase A+B: Zotero # Phase A+B: Zotero
if zcon: if zdb:
found = tag_existing_zotero(zcon, spec, dry_run=args.dry_run) found = tag_existing_zotero(zdb, spec, dry_run=args.dry_run)
if found: if found:
tagged += 1 tagged += 1
else: else:
register_in_zotero(zcon, spec, dry_run=args.dry_run) register_in_zotero(zdb, spec, dry_run=args.dry_run)
created_z += 1 created_z += 1
# Phase C: bib.sqlite # Phase C: bib.sqlite
register_in_bib(spec, dry_run=args.dry_run) register_in_bib(spec, dry_run=args.dry_run)
created_b += 1 created_b += 1
if zcon: if zdb:
if not args.dry_run: if not args.dry_run:
zcon.commit() zdb.commit()
zcon.close() zdb.close()
label = "Would register" if args.dry_run else "Registered" label = "Would register" if args.dry_run else "Registered"
print(f"\n{label}:") print(f"\n{label}:")

372
dev/scripts/zotero_archive_migrate.py Normal file
View File

@@ -0,0 +1,372 @@
"""Migrate host Zotero library into container Zotero under an 'archive' collection.
Creates new keys for all items, remaps all IDs, copies storage directories.
Must be run with Zotero process stopped.
"""
import shutil
import sqlite3
import sys
from pathlib import Path
from zot.db import generate_key
HOST_DB = Path(sys.argv[1]) if len(sys.argv) > 1 else Path("/tmp/host-zotero.sqlite")
CONTAINER_DB = Path(sys.argv[2]) if len(sys.argv) > 2 else Path("/home/ubuntu/Zotero/zotero.sqlite")
HOST_STORAGE = Path(sys.argv[3]) if len(sys.argv) > 3 else Path("/home/ubuntu/data/Zotero/storage")
CONTAINER_STORAGE = Path(sys.argv[4]) if len(sys.argv) > 4 else Path("/home/ubuntu/Zotero/storage")
LIBRARY_ID = 1 # user library
def gen_key(existing: set[str]) -> str:
while True:
k = generate_key()
if k not in existing:
existing.add(k)
return k
def main():
if not HOST_DB.exists():
sys.exit(f"Host DB not found: {HOST_DB}")
if not CONTAINER_DB.exists():
sys.exit(f"Container DB not found: {CONTAINER_DB}")
src = sqlite3.connect(str(HOST_DB))
src.row_factory = sqlite3.Row
dst = sqlite3.connect(str(CONTAINER_DB))
dst.row_factory = sqlite3.Row
dst.execute("PRAGMA journal_mode=WAL")
dst.execute("PRAGMA foreign_keys=OFF") # we handle ordering ourselves
# Collect existing keys in destination
existing_keys = {r[0] for r in dst.execute("SELECT key FROM items WHERE libraryID=?", (LIBRARY_ID,))}
existing_keys |= {r[0] for r in dst.execute("SELECT key FROM collections WHERE libraryID=?", (LIBRARY_ID,))}
# --- Max IDs in destination ---
max_item_id = dst.execute("SELECT COALESCE(MAX(itemID),0) FROM items").fetchone()[0]
max_coll_id = dst.execute("SELECT COALESCE(MAX(collectionID),0) FROM collections").fetchone()[0]
max_value_id = dst.execute("SELECT COALESCE(MAX(valueID),0) FROM itemDataValues").fetchone()[0]
max_creator_id = dst.execute("SELECT COALESCE(MAX(creatorID),0) FROM creators").fetchone()[0]
max_tag_id = dst.execute("SELECT COALESCE(MAX(tagID),0) FROM tags").fetchone()[0]
max_word_id = dst.execute("SELECT COALESCE(MAX(wordID),0) FROM fulltextWords").fetchone()[0]
# ============================================================
# 1. Create "archive" collection + mirror host collection tree
# ============================================================
archive_key = gen_key(existing_keys)
max_coll_id += 1
archive_coll_id = max_coll_id
dst.execute(
"INSERT INTO collections (collectionID, collectionName, parentCollectionID, libraryID, key, version, synced) "
"VALUES (?,?,NULL,?,?,0,0)",
(archive_coll_id, "archive", LIBRARY_ID, archive_key),
)
print(f"Created 'archive' collection: ID={archive_coll_id}, key={archive_key}")
# Mirror host collections under archive
host_colls = src.execute(
"SELECT collectionID, collectionName, parentCollectionID FROM collections ORDER BY collectionID"
).fetchall()
coll_id_map: dict[int, int] = {} # host collectionID -> dest collectionID
for hc in host_colls:
max_coll_id += 1
new_id = max_coll_id
coll_id_map[hc["collectionID"]] = new_id
# parent: if host has a parent, map it; otherwise parent is archive
if hc["parentCollectionID"] is not None:
parent = coll_id_map.get(hc["parentCollectionID"], archive_coll_id)
else:
parent = archive_coll_id
coll_key = gen_key(existing_keys)
dst.execute(
"INSERT INTO collections (collectionID, collectionName, parentCollectionID, libraryID, key, version, synced) "
"VALUES (?,?,?,?,?,0,0)",
(new_id, hc["collectionName"], parent, LIBRARY_ID, coll_key),
)
print(f"Mirrored {len(host_colls)} host collections under archive")
# ============================================================
# 2. Migrate itemDataValues (shared, deduplicated by value)
# ============================================================
# Build lookup of existing values in dest
dst_values: dict[str, int] = {}
for r in dst.execute("SELECT valueID, value FROM itemDataValues"):
dst_values[str(r["value"])] = r["valueID"]
src_value_map: dict[int, int] = {} # host valueID -> dest valueID
for r in src.execute("SELECT valueID, value FROM itemDataValues"):
val = str(r["value"])
if val in dst_values:
src_value_map[r["valueID"]] = dst_values[val]
else:
max_value_id += 1
dst.execute("INSERT INTO itemDataValues (valueID, value) VALUES (?,?)", (max_value_id, r["value"]))
dst_values[val] = max_value_id
src_value_map[r["valueID"]] = max_value_id
print(f"Mapped {len(src_value_map)} itemDataValues")
# ============================================================
# 3. Migrate creators (deduplicated by lastName+firstName+fieldMode)
# ============================================================
dst_creators: dict[tuple, int] = {}
for r in dst.execute("SELECT creatorID, firstName, lastName, fieldMode FROM creators"):
dst_creators[(r["lastName"], r["firstName"], r["fieldMode"])] = r["creatorID"]
src_creator_map: dict[int, int] = {}
for r in src.execute("SELECT creatorID, firstName, lastName, fieldMode FROM creators"):
ck = (r["lastName"], r["firstName"], r["fieldMode"])
if ck in dst_creators:
src_creator_map[r["creatorID"]] = dst_creators[ck]
else:
max_creator_id += 1
dst.execute(
"INSERT INTO creators (creatorID, firstName, lastName, fieldMode) VALUES (?,?,?,?)",
(max_creator_id, r["firstName"], r["lastName"], r["fieldMode"]),
)
dst_creators[ck] = max_creator_id
src_creator_map[r["creatorID"]] = max_creator_id
print(f"Mapped {len(src_creator_map)} creators")
# ============================================================
# 4. Migrate tags (deduplicated by name)
# ============================================================
dst_tags: dict[str, int] = {}
for r in dst.execute("SELECT tagID, name FROM tags"):
dst_tags[r["name"]] = r["tagID"]
src_tag_map: dict[int, int] = {}
for r in src.execute("SELECT tagID, name FROM tags"):
if r["name"] in dst_tags:
src_tag_map[r["tagID"]] = dst_tags[r["name"]]
else:
max_tag_id += 1
dst.execute("INSERT INTO tags (tagID, name) VALUES (?,?)", (max_tag_id, r["name"]))
dst_tags[r["name"]] = max_tag_id
src_tag_map[r["tagID"]] = max_tag_id
print(f"Mapped {len(src_tag_map)} tags")
# ============================================================
# 5. Migrate fulltextWords (deduplicated by word)
# ============================================================
dst_words: dict[str, int] = {}
for r in dst.execute("SELECT wordID, word FROM fulltextWords"):
dst_words[r["word"]] = r["wordID"]
src_word_map: dict[int, int] = {}
for r in src.execute("SELECT wordID, word FROM fulltextWords"):
if r["word"] in dst_words:
src_word_map[r["wordID"]] = dst_words[r["word"]]
else:
max_word_id += 1
dst.execute("INSERT INTO fulltextWords (wordID, word) VALUES (?,?)", (max_word_id, r["word"]))
dst_words[r["word"]] = max_word_id
src_word_map[r["wordID"]] = max_word_id
print(f"Mapped {len(src_word_map)} fulltextWords")
# ============================================================
# 6. Migrate items (new keys, new IDs)
# ============================================================
# Process items in order so parents come before children
host_items = src.execute(
"SELECT itemID, itemTypeID, dateAdded, dateModified, clientDateModified, libraryID, key "
"FROM items WHERE libraryID=? ORDER BY itemID",
(LIBRARY_ID,),
).fetchall()
item_id_map: dict[int, int] = {} # host itemID -> dest itemID
item_key_map: dict[str, str] = {} # host key -> dest key (for storage)
for hi in host_items:
max_item_id += 1
new_key = gen_key(existing_keys)
item_id_map[hi["itemID"]] = max_item_id
item_key_map[hi["key"]] = new_key
dst.execute(
"INSERT INTO items (itemID, itemTypeID, dateAdded, dateModified, clientDateModified, libraryID, key, version, synced) "
"VALUES (?,?,?,?,?,?,?,0,0)",
(max_item_id, hi["itemTypeID"], hi["dateAdded"], hi["dateModified"], hi["clientDateModified"], LIBRARY_ID, new_key),
)
print(f"Migrated {len(item_id_map)} items")
# ============================================================
# 7. Migrate itemData
# ============================================================
count = 0
for r in src.execute("SELECT itemID, fieldID, valueID FROM itemData"):
if r["itemID"] not in item_id_map:
continue
dst.execute(
"INSERT INTO itemData (itemID, fieldID, valueID) VALUES (?,?,?)",
(item_id_map[r["itemID"]], r["fieldID"], src_value_map[r["valueID"]]),
)
count += 1
print(f"Migrated {count} itemData rows")
# ============================================================
# 8. Migrate itemCreators
# ============================================================
count = 0
for r in src.execute("SELECT itemID, creatorID, creatorTypeID, orderIndex FROM itemCreators"):
if r["itemID"] not in item_id_map:
continue
dst.execute(
"INSERT INTO itemCreators (itemID, creatorID, creatorTypeID, orderIndex) VALUES (?,?,?,?)",
(item_id_map[r["itemID"]], src_creator_map[r["creatorID"]], r["creatorTypeID"], r["orderIndex"]),
)
count += 1
print(f"Migrated {count} itemCreators rows")
# ============================================================
# 9. Migrate itemAttachments
# ============================================================
count = 0
for r in src.execute("SELECT itemID, parentItemID, linkMode, contentType, charsetID, path, syncState, storageModTime, storageHash FROM itemAttachments"):
if r["itemID"] not in item_id_map:
continue
parent = item_id_map.get(r["parentItemID"]) if r["parentItemID"] is not None else None
dst.execute(
"INSERT INTO itemAttachments (itemID, parentItemID, linkMode, contentType, charsetID, path, syncState, storageModTime, storageHash) "
"VALUES (?,?,?,?,?,?,?,?,?)",
(item_id_map[r["itemID"]], parent, r["linkMode"], r["contentType"], r["charsetID"], r["path"], r["syncState"], r["storageModTime"], r["storageHash"]),
)
count += 1
print(f"Migrated {count} itemAttachments rows")
# ============================================================
# 10. Migrate itemNotes
# ============================================================
count = 0
for r in src.execute("SELECT itemID, parentItemID, note, title FROM itemNotes"):
if r["itemID"] not in item_id_map:
continue
parent = item_id_map.get(r["parentItemID"]) if r["parentItemID"] is not None else None
dst.execute(
"INSERT INTO itemNotes (itemID, parentItemID, note, title) VALUES (?,?,?,?)",
(item_id_map[r["itemID"]], parent, r["note"], r["title"]),
)
count += 1
print(f"Migrated {count} itemNotes rows")
# ============================================================
# 11. Migrate itemTags
# ============================================================
count = 0
for r in src.execute("SELECT itemID, tagID, type FROM itemTags"):
if r["itemID"] not in item_id_map:
continue
dst.execute(
"INSERT INTO itemTags (itemID, tagID, type) VALUES (?,?,?)",
(item_id_map[r["itemID"]], src_tag_map[r["tagID"]], r["type"]),
)
count += 1
print(f"Migrated {count} itemTags rows")
# ============================================================
# 12. Migrate itemRelations
# ============================================================
# Get relationPredicates from both DBs — they should match (standard set)
count = 0
for r in src.execute("SELECT itemID, predicateID, object FROM itemRelations"):
if r["itemID"] not in item_id_map:
continue
dst.execute(
"INSERT OR IGNORE INTO itemRelations (itemID, predicateID, object) VALUES (?,?,?)",
(item_id_map[r["itemID"]], r["predicateID"], r["object"]),
)
count += 1
print(f"Migrated {count} itemRelations rows")
# ============================================================
# 13. Migrate fulltextItems + fulltextItemWords
# ============================================================
count = 0
for r in src.execute("SELECT itemID, indexedPages, totalPages, indexedChars, totalChars FROM fulltextItems"):
if r["itemID"] not in item_id_map:
continue
dst.execute(
"INSERT INTO fulltextItems (itemID, indexedPages, totalPages, indexedChars, totalChars, version, synced) "
"VALUES (?,?,?,?,?,0,0)",
(item_id_map[r["itemID"]], r["indexedPages"], r["totalPages"], r["indexedChars"], r["totalChars"]),
)
count += 1
print(f"Migrated {count} fulltextItems rows")
count = 0
for r in src.execute("SELECT wordID, itemID FROM fulltextItemWords"):
if r["itemID"] not in item_id_map:
continue
if r["wordID"] not in src_word_map:
continue
dst.execute(
"INSERT OR IGNORE INTO fulltextItemWords (wordID, itemID) VALUES (?,?)",
(src_word_map[r["wordID"]], item_id_map[r["itemID"]]),
)
count += 1
print(f"Migrated {count} fulltextItemWords rows")
# ============================================================
# 14. Migrate collectionItems (only top-level items, not child attachments/notes)
# ============================================================
# Zotero triggers prevent adding child attachments/notes to collections
child_item_ids = set()
for r in src.execute("SELECT itemID FROM itemAttachments WHERE parentItemID IS NOT NULL"):
child_item_ids.add(r["itemID"])
for r in src.execute("SELECT itemID FROM itemNotes WHERE parentItemID IS NOT NULL"):
child_item_ids.add(r["itemID"])
count = 0
for r in src.execute("SELECT collectionID, itemID, orderIndex FROM collectionItems"):
if r["itemID"] not in item_id_map:
continue
if r["collectionID"] not in coll_id_map:
continue
dst.execute(
"INSERT OR IGNORE INTO collectionItems (collectionID, itemID, orderIndex) VALUES (?,?,?)",
(coll_id_map[r["collectionID"]], item_id_map[r["itemID"]], r["orderIndex"]),
)
count += 1
print(f"Migrated {count} collectionItems rows")
# Also add all top-level items to the archive collection itself
top_count = 0
for host_id, dest_id in item_id_map.items():
if host_id not in child_item_ids:
dst.execute(
"INSERT OR IGNORE INTO collectionItems (collectionID, itemID, orderIndex) VALUES (?,?,0)",
(archive_coll_id, dest_id),
)
top_count += 1
print(f"Added {top_count} top-level items to archive collection")
# ============================================================
# 15. Commit
# ============================================================
dst.commit()
print("Database committed successfully")
# ============================================================
# 16. Copy storage directories
# ============================================================
copied = 0
skipped = 0
for old_key, new_key in item_key_map.items():
src_dir = HOST_STORAGE / old_key
dst_dir = CONTAINER_STORAGE / new_key
if src_dir.is_dir():
if dst_dir.exists():
skipped += 1
continue
shutil.copytree(str(src_dir), str(dst_dir))
copied += 1
print(f"Copied {copied} storage directories, skipped {skipped}")
src.close()
dst.close()
print("Migration complete!")
if __name__ == "__main__":
main()

372
src/bib/sync.py
View File

@@ -1,9 +1,8 @@
"""Push bib items into Zotero's SQLite EAV schema. """Push bib items into Zotero's SQLite EAV schema.
Reverse of ``dev/scripts/migrate_zotero.py`` — takes items from Uses ``zot.db.Db`` as the ORM layer — all raw SQL lives there.
``data/bib.sqlite`` and inserts them into ``zotero.sqlite`` This module is the adapter that maps ``bib.Item`` objects to
using Zotero's 61-table EAV layout so they appear in the Zotero operations.
Zotero desktop application.
Usage:: Usage::
@@ -20,149 +19,26 @@ Usage::
from __future__ import annotations from __future__ import annotations
import json import json
import random
import sqlite3
from datetime import datetime, timezone
from bib.item import Item from bib.item import Item
from zot.db import Db, is_valid_key, now_iso
# ── Zotero constants ────────────────────────────────────────────── # bib item_type → Zotero itemTypeID
_TYPE_MAP: dict[str, int] = {
_TYPE_MAP = { "rule": 20, # statute
"rule": 36, # statute "regulation": 20, # statute
"regulation": 36, # statute "manual": 15, # report
"manual": 34, # report "download": 13, # webpage
"download": 40, # webpage "source": 34, # document
"source": 14, # document "journal-article": 4, # journalArticle
"journal-article": 22, # journalArticle
}
# fieldName → fieldID (from Zotero's fields table)
_FIELD_IDS = {
"title": 1,
"nameOfAct": 116,
"abstractNote": 2,
"code": 31,
"codeNumber": 117,
"dateEnacted": 119,
"pages": 35,
"section": 33,
"session": 38,
"history": 39,
"url": 10,
"accessDate": 11,
"extra": 19,
"date": 6,
"type": 43,
"publisher": 25,
"institution": 90,
"reportNumber": 113,
"reportType": 114,
"seriesTitle": 21,
"seriesNumber": 46,
"place": 26,
"websiteTitle": 123,
"websiteType": 42,
# journalArticle fields
"DOI": 8,
"PMID": 86,
"PMCID": 87,
"ISSN": 44,
"publicationTitle": 41,
"journalAbbreviation": 85,
"volume": 22,
"issue": 67,
} }
_ALLOWED_KEY_CHARS = "23456789ABCDEFGHIJKLMNPQRSTUVWXYZ" # ── Author parsing (stateless) ──────────────────────────────────
def _zotero_key() -> str:
"""Generate an 8-char Zotero key using the allowed character set.
Zotero keys must be exactly 8 chars from ``23456789ABCDEFGHIJKLMNPQRSTUVWXYZ``.
No ``0``, ``1``, ``O``, or lowercase.
Source: ``Zotero.Utilities.allowedKeyChars`` in utilities.js.
"""
return "".join(random.choices(_ALLOWED_KEY_CHARS, k=8)) # noqa: S311
def _is_valid_zotero_key(key: str) -> bool:
"""Check if a key is a valid Zotero object key."""
return len(key) == 8 and all(c in _ALLOWED_KEY_CHARS for c in key)
def _now_iso() -> str:
"""Zotero internal timestamp: ``YYYY-MM-DD HH:MM:SS`` (no T, no Z)."""
return datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M:%S")
def _ensure_value(con: sqlite3.Connection, value: str) -> int:
"""Find or create a row in itemDataValues. Returns valueID."""
row = con.execute(
"SELECT valueID FROM itemDataValues WHERE value = ?",
(value,),
).fetchone()
if row:
return row[0]
cur = con.execute("INSERT INTO itemDataValues (value) VALUES (?)", (value,))
return cur.lastrowid
def _ensure_tag(con: sqlite3.Connection, name: str) -> int:
"""Find or create a tag. Returns tagID."""
row = con.execute("SELECT tagID FROM tags WHERE name = ?", (name,)).fetchone()
if row:
return row[0]
cur = con.execute("INSERT INTO tags (name) VALUES (?)", (name,))
return cur.lastrowid
def _normalize_zotero_date(value: str) -> str:
"""Convert ISO 8601 timestamps to Zotero's format (no T, no Z)."""
return value.replace("T", " ").rstrip("Z")
def _set_field(
con: sqlite3.Connection,
item_id: int,
field_name: str,
value: str,
) -> None:
"""Set a field on a Zotero item via the EAV tables."""
if not value:
return
field_id = _FIELD_IDS.get(field_name)
if field_id is None:
return
# Normalize date fields to Zotero's expected format
if field_name in ("accessDate", "dateEnacted", "date"):
value = _normalize_zotero_date(value)
value_id = _ensure_value(con, value)
con.execute(
"INSERT OR REPLACE INTO itemData (itemID, fieldID, valueID) VALUES (?, ?, ?)",
(item_id, field_id, value_id),
)
def _ensure_creator(con: sqlite3.Connection, first_name: str, last_name: str) -> int:
"""Find or create a creator row. Returns creatorID."""
row = con.execute(
"SELECT creatorID FROM creators WHERE firstName = ? AND lastName = ?",
(first_name, last_name),
).fetchone()
if row:
return row[0]
cur = con.execute(
"INSERT INTO creators (firstName, lastName, fieldMode) VALUES (?, ?, 0)",
(first_name, last_name),
)
return cur.lastrowid
def _parse_authors_from_extra(extra: str) -> tuple[list[tuple[str, str]], str]: def _parse_authors_from_extra(extra: str) -> tuple[list[tuple[str, str]], str]:
"""Parse 'Authors: Last First; Last First' from extra, return (authors, cleaned_extra). """Parse 'Authors: Last First; Last First' from extra.
Returns a list of (firstName, lastName) tuples and the extra text Returns a list of (firstName, lastName) tuples and the extra text
with the Authors line removed. with the Authors line removed.
@@ -186,57 +62,30 @@ def _parse_authors_from_extra(extra: str) -> tuple[list[tuple[str, str]], str]:
return authors, "\n".join(cleaned_lines).strip() return authors, "\n".join(cleaned_lines).strip()
def _add_creators(
con: sqlite3.Connection,
item_id: int,
authors: list[tuple[str, str]],
creator_type_id: int = 10, # 10 = author
) -> int:
"""Add creators to a Zotero item. Returns count added."""
for idx, (first_name, last_name) in enumerate(authors):
creator_id = _ensure_creator(con, first_name, last_name)
con.execute(
"INSERT OR IGNORE INTO itemCreators "
"(itemID, creatorID, creatorTypeID, orderIndex) "
"VALUES (?, ?, ?, ?)",
(item_id, creator_id, creator_type_id, idx),
)
return len(authors)
def _parse_extra_to_journal_fields(extra: str) -> dict[str, str]: def _parse_extra_to_journal_fields(extra: str) -> dict[str, str]:
"""Parse structured extra text into Zotero journalArticle fields. """Parse structured extra text into Zotero journalArticle fields."""
Extracts PMID, DOI, Journal, PubTypes, MeSH from lines like::
PMID: 12345678
DOI: 10.1234/example
Journal: The Lancet
PubTypes: Randomized Controlled Trial; Journal Article
MeSH: Skin Substitutes; Wound Healing
Returns a field dict with proper Zotero field names and a
cleaned ``extra`` containing only the leftover lines.
"""
fields: dict[str, str] = {} fields: dict[str, str] = {}
leftover: list[str] = [] leftover: list[str] = []
for line in extra.split("\n"): for line in extra.split("\n"):
if line.startswith("PMID:"): if line.startswith("DOI:"):
fields["PMID"] = line[5:].strip()
elif line.startswith("DOI:"):
fields["DOI"] = line[4:].strip() fields["DOI"] = line[4:].strip()
elif line.startswith("Journal:"): elif line.startswith("Journal:"):
fields["publicationTitle"] = line[8:].strip() fields["publicationTitle"] = line[8:].strip()
elif line.startswith("Authors:"): elif line.startswith("Authors:"):
pass # handled separately by _parse_authors_from_extra pass # handled separately by _parse_authors_from_extra
else: else:
# PMID, PMCID, PubTypes, MeSH etc. stay in extra —
# Zotero has no dedicated fields for these.
leftover.append(line) leftover.append(line)
fields["extra"] = "\n".join(leftover).strip() fields["extra"] = "\n".join(leftover).strip()
return fields return fields
# ── Item → Zotero field mapping (stateless) ──────────────────────
def _item_to_zotero_fields(item: Item) -> dict[str, str]: def _item_to_zotero_fields(item: Item) -> dict[str, str]:
"""Convert a bib Item to Zotero EAV field dict.""" """Convert a bib Item to Zotero EAV field dict."""
ej = json.loads(item.to_row().get("extra_json", "{}")) ej = json.loads(item.to_row().get("extra_json", "{}"))
@@ -287,12 +136,13 @@ def _item_to_zotero_fields(item: Item) -> dict[str, str]:
extra = item.extra extra = item.extra
if ej.get("transmittal"): if ej.get("transmittal"):
extra = f"Transmittal: {ej['transmittal']}\n{extra}".strip() extra = f"Transmittal: {ej['transmittal']}\n{extra}".strip()
if chapter:
extra = f"Chapter {chapter}\n{extra}".strip()
return { return {
"title": item.title, "title": item.title,
"reportType": "Internet-Only Manual", "reportType": "Internet-Only Manual",
"reportNumber": ej.get("pub_number", ""), "reportNumber": ej.get("pub_number", ""),
"seriesTitle": ej.get("manual_name", ""), "seriesTitle": ej.get("manual_name", ""),
"seriesNumber": f"Chapter {chapter}" if chapter else "",
"institution": item.institution, "institution": item.institution,
"date": item.date_published, "date": item.date_published,
"place": "Baltimore, MD", "place": "Baltimore, MD",
@@ -335,16 +185,19 @@ def _item_to_zotero_fields(item: Item) -> dict[str, str]:
fields.setdefault("publicationTitle", item.institution) fields.setdefault("publicationTitle", item.institution)
return fields return fields
# Generic source (document) # Generic source (document) — Zotero's document type has no ``type``
# field; stash doc_type in extra if present.
extra = item.extra
if doc_type:
extra = f"Type: {doc_type}\n{extra}".strip()
return { return {
"title": item.title, "title": item.title,
"type": doc_type,
"publisher": item.institution, "publisher": item.institution,
"date": item.date_published, "date": item.date_published,
"url": item.url, "url": item.url,
"accessDate": item.access_date, "accessDate": item.access_date,
"abstractNote": item.abstract, "abstractNote": item.abstract,
"extra": item.extra, "extra": extra,
} }
@@ -357,10 +210,6 @@ def push_to_zotero(
zotero_db: str | None = None, zotero_db: str | None = None,
collection_key: str = "", collection_key: str = "",
) -> dict[str, int]: ) -> dict[str, int]:
if zotero_db is None:
from conf import path
zotero_db = str(path("db.zotero"))
"""Push bib items into Zotero's SQLite database. """Push bib items into Zotero's SQLite database.
Parameters Parameters
@@ -375,12 +224,14 @@ def push_to_zotero(
Returns Returns
------- -------
dict dict
Counts: created, skipped, tags, collections. Counts: created, skipped, tags, collections, creators.
""" """
con = sqlite3.connect(zotero_db) if zotero_db is None:
con.row_factory = sqlite3.Row from conf import path
con.execute("PRAGMA journal_mode=WAL")
zotero_db = str(path("db.zotero"))
with Db(zotero_db) as db:
stats: dict[str, int] = { stats: dict[str, int] = {
"created": 0, "created": 0,
"skipped": 0, "skipped": 0,
@@ -388,36 +239,23 @@ def push_to_zotero(
"collections": 0, "collections": 0,
} }
now = _now_iso() ts = now_iso()
# Resolve collection # Resolve target collection
collection_id: int | None = None collection_id: int | None = None
if collection_key: if collection_key:
row = con.execute( collection_id = db.find_collection(collection_key)
"SELECT collectionID FROM collections WHERE key = ?",
(collection_key,),
).fetchone()
if row:
collection_id = row[0]
for item in items: for item in items:
# Skip if URL already exists in Zotero # Skip if URL already exists in Zotero
if item.url: if item.url:
existing = con.execute( existing_id = db.find_item_by_url(item.url)
"""SELECT i.itemID FROM items i if existing_id is not None:
JOIN itemData id ON i.itemID = id.itemID db.sync_tags(existing_id, item.tags)
JOIN itemDataValues idv ON id.valueID = idv.valueID
WHERE id.fieldID = ? AND idv.value = ?""",
(_FIELD_IDS["url"], item.url),
).fetchone()
if existing:
# Still add tags to existing item
_sync_tags(con, existing[0], item.tags)
stats["skipped"] += 1 stats["skipped"] += 1
continue continue
# Resolve Zotero item type. journal-article sources become # Resolve Zotero item type
# journalArticle (22); other sources stay document (14).
ej = json.loads(item.to_row().get("extra_json", "{}")) ej = json.loads(item.to_row().get("extra_json", "{}"))
doc_type = ej.get("doc_type", "") doc_type = ej.get("doc_type", "")
type_id = _TYPE_MAP.get(doc_type) or _TYPE_MAP.get(item.item_type) type_id = _TYPE_MAP.get(doc_type) or _TYPE_MAP.get(item.item_type)
@@ -425,131 +263,37 @@ def push_to_zotero(
stats["skipped"] += 1 stats["skipped"] += 1
continue continue
# Only reuse bib key if it passes Zotero's key validation; # Reuse bib key only if it's valid Zotero format
# otherwise generate a fresh one. key = item.key if is_valid_key(item.key) else ""
key = item.key if _is_valid_zotero_key(item.key) else _zotero_key() item_id = db.create_item(type_id, key=key, now=ts)
# Avoid key collision # Parse authors before field mapping strips them
while con.execute("SELECT 1 FROM items WHERE key = ?", (key,)).fetchone():
key = _zotero_key()
# Insert item
cur = con.execute(
"""INSERT INTO items
(itemTypeID, dateAdded, dateModified,
clientDateModified, libraryID, key, version, synced)
VALUES (?, ?, ?, ?, 1, ?, 0, 0)""",
(type_id, now, now, now, key),
)
item_id = cur.lastrowid
# Parse authors from the original item extra (before field
# mapping strips them).
authors, _ = _parse_authors_from_extra(item.extra) authors, _ = _parse_authors_from_extra(item.extra)
# Set fields via EAV # Set fields via EAV
fields = _item_to_zotero_fields(item) db.set_fields(item_id, _item_to_zotero_fields(item))
for field_name, value in fields.items():
_set_field(con, item_id, field_name, value)
# Creators (authors) # Creators
if authors: if authors:
_add_creators(con, item_id, authors) count = db.add_creators(item_id, authors)
stats["creators"] = stats.get("creators", 0) + len(authors) stats["creators"] = stats.get("creators", 0) + count
# Tags # Tags
_sync_tags(con, item_id, item.tags) db.sync_tags(item_id, item.tags)
stats["tags"] += len(item.tags) stats["tags"] += len(item.tags)
# Collection # Target collection
if collection_id is not None: if collection_id is not None:
max_order = con.execute( db.add_to_collection(item_id, collection_key=collection_key)
"SELECT COALESCE(MAX(orderIndex), -1) "
"FROM collectionItems WHERE collectionID = ?",
(collection_id,),
).fetchone()[0]
con.execute(
"INSERT OR IGNORE INTO collectionItems "
"(collectionID, itemID, orderIndex) VALUES (?, ?, ?)",
(collection_id, item_id, max_order + 1),
)
stats["collections"] += 1 stats["collections"] += 1
# Also add to collections from the bib item # Item's own collections
for col_key in item.collections: for col_key in item.collections:
col_row = con.execute( if col_key != collection_key:
"SELECT collectionID FROM collections WHERE key = ?", db.add_to_collection(item_id, collection_key=col_key)
(col_key,),
).fetchone()
if col_row and col_row[0] != collection_id:
max_o = con.execute(
"SELECT COALESCE(MAX(orderIndex), -1) "
"FROM collectionItems WHERE collectionID = ?",
(col_row[0],),
).fetchone()[0]
con.execute(
"INSERT OR IGNORE INTO collectionItems "
"(collectionID, itemID, orderIndex) VALUES (?, ?, ?)",
(col_row[0], item_id, max_o + 1),
)
stats["created"] += 1 stats["created"] += 1
con.commit() db.commit()
con.close()
return stats return stats
def _sync_tags(con: sqlite3.Connection, item_id: int, tags: list[str]) -> None:
"""Add tags to a Zotero item."""
for tag_name in tags:
if not tag_name:
continue
tag_id = _ensure_tag(con, tag_name)
con.execute(
"INSERT OR IGNORE INTO itemTags (itemID, tagID, type) VALUES (?, ?, 0)",
(item_id, tag_id),
)
def ensure_collection(
con: sqlite3.Connection,
name: str,
parent_key: str = "",
) -> str:
"""Find or create a Zotero collection. Returns key."""
if parent_key:
parent = con.execute(
"SELECT collectionID FROM collections WHERE key = ?",
(parent_key,),
).fetchone()
parent_id = parent[0] if parent else None
else:
parent_id = None
if parent_id is not None:
row = con.execute(
"SELECT key FROM collections "
"WHERE collectionName = ? AND parentCollectionID = ?",
(name, parent_id),
).fetchone()
else:
row = con.execute(
"SELECT key FROM collections "
"WHERE collectionName = ? AND parentCollectionID IS NULL",
(name,),
).fetchone()
if row:
return row[0]
key = _zotero_key()
con.execute(
"""INSERT INTO collections
(collectionName, parentCollectionID, clientDateModified,
libraryID, key, version, synced)
VALUES (?, ?, ?, 1, ?, 0, 0)""",
(name, parent_id, _now_iso(), key),
)
con.commit()
return key

9
src/zot/__init__.py Normal file
View File

@@ -0,0 +1,9 @@
"""Zotero sync, article retrieval, and evidence management.
Key exports::
from zot.db import Db # SQLite ORM (reads + writes)
from zot.duck import DuckDb # DuckDB engine (analytics)
from zot.schema import create_db # Bootstrap a fresh zotero.sqlite
from zot.table import Items # Pydantic model for any table
"""

868
src/zot/db.py Normal file
View File

@@ -0,0 +1,868 @@
"""Zotero SQLite ORM — vendors Zotero's backend logic.
This is the single interface for reading and writing Zotero's 61-table
EAV schema. All raw SQL lives here; consumers (``bib.sync``, CLI tools,
notebooks) call ``Db`` methods instead of touching SQLite directly.
Usage::
from zot.db import Db
with Db("data/zotero/data/zotero.sqlite") as db:
item_id = db.create_item(type_id=20, key="AB3CDE4F")
db.set_field(item_id, "nameOfAct", "PFS Final Rule 2026")
db.sync_tags(item_id, ["module:pfs", "year:2026"])
db.add_to_collection(item_id, collection_key="COLLKEY1")
db.commit()
"""
from __future__ import annotations
import random
import sqlite3
from datetime import datetime, timezone
ALLOWED_KEY_CHARS = "23456789ABCDEFGHIJKLMNPQRSTUVWXYZ"
def generate_key() -> str:
"""Generate an 8-char Zotero key.
Character set: ``23456789ABCDEFGHIJKLMNPQRSTUVWXYZ`` (no 0, 1, O, lowercase).
Source: ``Zotero.Utilities.allowedKeyChars`` in zotero/utilities.js.
"""
return "".join(random.choices(ALLOWED_KEY_CHARS, k=8)) # noqa: S311
def is_valid_key(key: str) -> bool:
"""Check if *key* is a valid 8-char Zotero object key."""
return len(key) == 8 and all(c in ALLOWED_KEY_CHARS for c in key)
def now_iso() -> str:
"""Zotero internal timestamp: ``YYYY-MM-DD HH:MM:SS`` (no T, no Z)."""
return datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M:%S")
def normalize_date(value: str) -> str:
"""Convert ISO-8601 timestamps to Zotero's ``YYYY-MM-DD HH:MM:SS``."""
return value.replace("T", " ").rstrip("Z")
TYPE_MAP: dict[str, int] = {
"note": 1,
"book": 2,
"bookSection": 3,
"journalArticle": 4,
"magazineArticle": 5,
"newspaperArticle": 6,
"thesis": 7,
"letter": 8,
"manuscript": 9,
"interview": 10,
"film": 11,
"artwork": 12,
"webpage": 13,
"attachment": 14,
"report": 15,
"bill": 16,
"case": 17,
"hearing": 18,
"patent": 19,
"statute": 20,
"email": 21,
"map": 22,
"blogPost": 23,
"instantMessage": 24,
"forumPost": 25,
"audioRecording": 26,
"presentation": 27,
"videoRecording": 28,
"tvBroadcast": 29,
"radioBroadcast": 30,
"podcast": 31,
"computerProgram": 32,
"conferencePaper": 33,
"document": 34,
"encyclopediaArticle": 35,
"dictionaryEntry": 36,
}
FIELD_MAP: dict[str, int] = {
"url": 1,
"rights": 2,
"series": 3,
"volume": 4,
"issue": 5,
"edition": 6,
"place": 7,
"publisher": 8,
"pages": 10,
"ISBN": 11,
"publicationTitle": 12,
"ISSN": 13,
"date": 14,
"section": 15,
"callNumber": 18,
"archiveLocation": 19,
"distributor": 21,
"extra": 22,
"journalAbbreviation": 25,
"DOI": 26,
"accessDate": 27,
"seriesTitle": 28,
"seriesText": 29,
"seriesNumber": 30,
"institution": 31,
"reportType": 32,
"code": 36,
"session": 40,
"legislativeBody": 41,
"history": 42,
"reporter": 43,
"court": 44,
"numberOfVolumes": 45,
"committee": 46,
"assignee": 48,
"patentNumber": 50,
"priorityNumbers": 51,
"issueDate": 52,
"references": 53,
"legalStatus": 54,
"codeNumber": 55,
"artworkMedium": 59,
"number": 60,
"artworkSize": 61,
"libraryCatalog": 62,
"videoRecordingFormat": 63,
"interviewMedium": 64,
"letterType": 65,
"manuscriptType": 66,
"mapType": 67,
"scale": 68,
"thesisType": 69,
"websiteType": 70,
"audioRecordingFormat": 71,
"label": 72,
"presentationType": 74,
"meetingName": 75,
"studio": 76,
"runningTime": 77,
"network": 78,
"postType": 79,
"audioFileType": 80,
"versionNumber": 81,
"system": 82,
"company": 83,
"conferenceName": 84,
"encyclopediaTitle": 85,
"dictionaryTitle": 86,
"language": 87,
"programmingLanguage": 88,
"university": 89,
"abstractNote": 90,
"websiteTitle": 91,
"reportNumber": 92,
"billNumber": 93,
"codeVolume": 94,
"codePages": 95,
"dateDecided": 96,
"reporterVolume": 97,
"firstPage": 98,
"documentNumber": 99,
"dateEnacted": 100,
"publicLawNumber": 101,
"country": 102,
"applicationNumber": 103,
"forumTitle": 104,
"episodeNumber": 105,
"blogTitle": 107,
"type": 108,
"medium": 109,
"title": 110,
"caseName": 111,
"nameOfAct": 112,
"subject": 113,
"proceedingsTitle": 114,
"bookTitle": 115,
"shortTitle": 116,
"docketNumber": 117,
"numPages": 118,
"programTitle": 119,
"issuingAuthority": 120,
"filingDate": 121,
"genre": 122,
"archive": 123,
}
CREATOR_TYPES: dict[str, int] = {
"author": 1,
"contributor": 2,
"editor": 3,
"translator": 4,
"seriesEditor": 5,
"interviewee": 6,
"interviewer": 7,
"director": 8,
"scriptwriter": 9,
"producer": 10,
"castMember": 11,
"sponsor": 12,
"counsel": 13,
"inventor": 14,
"attorneyAgent": 15,
"recipient": 16,
"performer": 17,
"composer": 18,
"wordsBy": 19,
"cartographer": 20,
"programmer": 21,
"artist": 22,
"commenter": 23,
"presenter": 24,
"guest": 25,
"podcaster": 26,
"reviewedAuthor": 27,
"cosponsor": 28,
"bookAuthor": 29,
}
_DATE_FIELDS = frozenset(
{"accessDate", "dateEnacted", "date", "dateDecided", "issueDate", "filingDate"}
)
# ── ORM ──────────────────────────────────────────────────────────────
class Db:
"""Zotero SQLite ORM.
Wraps a ``sqlite3.Connection`` and exposes every Zotero backend
operation as a method. Use as a context manager::
with Db(path) as db:
...
db.commit()
"""
def __init__(self, path: str) -> None:
self.path = path
self.con = sqlite3.connect(path)
self.con.row_factory = sqlite3.Row
self.con.execute("PRAGMA journal_mode=WAL")
self.con.execute("PRAGMA foreign_keys=ON")
def __enter__(self) -> Db:
return self
def __exit__(self, *exc) -> None:
self.close()
def close(self) -> None:
self.con.close()
def commit(self) -> None:
self.con.commit()
# ── Items ────────────────────────────────────────────────────
def create_item(
self,
type_id: int,
*,
key: str = "",
library_id: int = 1,
now: str = "",
) -> int:
"""Insert a new item row. Returns ``itemID``.
If *key* is empty or invalid, generates a fresh one.
Retries on key collision.
"""
if not key or not is_valid_key(key):
key = generate_key()
while self.key_exists(key):
key = generate_key()
ts = now or now_iso()
cur = self.con.execute(
"""INSERT INTO items
(itemTypeID, dateAdded, dateModified,
clientDateModified, libraryID, key, version, synced)
VALUES (?, ?, ?, ?, ?, ?, 0, 0)""",
(type_id, ts, ts, ts, library_id, key),
)
return cur.lastrowid
def key_exists(self, key: str) -> bool:
"""Check whether *key* is already used in the items table."""
return (
self.con.execute("SELECT 1 FROM items WHERE key = ?", (key,)).fetchone()
is not None
)
def find_item_by_key(self, key: str) -> int | None:
"""Return ``itemID`` for *key*, or ``None``."""
row = self.con.execute(
"SELECT itemID FROM items WHERE key = ?", (key,)
).fetchone()
return row[0] if row else None
def find_item_by_url(self, url: str) -> int | None:
"""Return ``itemID`` of the first item whose ``url`` field matches."""
row = self.con.execute(
"""SELECT i.itemID FROM items i
JOIN itemData id ON i.itemID = id.itemID
JOIN itemDataValues idv ON id.valueID = idv.valueID
WHERE id.fieldID = ? AND idv.value = ?""",
(FIELD_MAP["url"], url),
).fetchone()
return row[0] if row else None
def get_item_type(self, item_id: int) -> int | None:
"""Return ``itemTypeID`` for *item_id*."""
row = self.con.execute(
"SELECT itemTypeID FROM items WHERE itemID = ?", (item_id,)
).fetchone()
return row[0] if row else None
# ── EAV fields ───────────────────────────────────────────────
def ensure_value(self, value: str) -> int:
"""Find or create a row in ``itemDataValues``. Returns ``valueID``."""
row = self.con.execute(
"SELECT valueID FROM itemDataValues WHERE value = ?", (value,)
).fetchone()
if row:
return row[0]
cur = self.con.execute(
"INSERT INTO itemDataValues (value) VALUES (?)", (value,)
)
return cur.lastrowid
def set_field(self, item_id: int, field_name: str, value: str) -> None:
"""Set an EAV field on *item_id*. No-op if *value* is empty
or *field_name* is not in ``FIELD_MAP``."""
if not value:
return
field_id = FIELD_MAP.get(field_name)
if field_id is None:
return
if field_name in _DATE_FIELDS:
value = normalize_date(value)
value_id = self.ensure_value(value)
self.con.execute(
"INSERT OR REPLACE INTO itemData (itemID, fieldID, valueID) "
"VALUES (?, ?, ?)",
(item_id, field_id, value_id),
)
def set_fields(self, item_id: int, fields: dict[str, str]) -> None:
"""Batch-set multiple EAV fields."""
for name, value in fields.items():
self.set_field(item_id, name, value)
def get_field(self, item_id: int, field_name: str) -> str | None:
"""Read a single EAV field value. Returns ``None`` if unset."""
field_id = FIELD_MAP.get(field_name)
if field_id is None:
return None
row = self.con.execute(
"""SELECT idv.value FROM itemData id
JOIN itemDataValues idv ON id.valueID = idv.valueID
WHERE id.itemID = ? AND id.fieldID = ?""",
(item_id, field_id),
).fetchone()
return row[0] if row else None
def get_fields(self, item_id: int) -> dict[str, str]:
"""Read all EAV fields for *item_id* as ``{fieldName: value}``."""
rows = self.con.execute(
"""SELECT f.fieldName, idv.value
FROM itemData id
JOIN fields f ON id.fieldID = f.fieldID
JOIN itemDataValues idv ON id.valueID = idv.valueID
WHERE id.itemID = ?""",
(item_id,),
).fetchall()
return {r[0]: r[1] for r in rows}
# ── Creators ─────────────────────────────────────────────────
def ensure_creator(self, first_name: str, last_name: str) -> int:
"""Find or create a creator row. Returns ``creatorID``."""
row = self.con.execute(
"SELECT creatorID FROM creators WHERE firstName = ? AND lastName = ?",
(first_name, last_name),
).fetchone()
if row:
return row[0]
cur = self.con.execute(
"INSERT INTO creators (firstName, lastName, fieldMode) VALUES (?, ?, 0)",
(first_name, last_name),
)
return cur.lastrowid
def add_creators(
self,
item_id: int,
authors: list[tuple[str, str]],
creator_type: str = "author",
) -> int:
"""Link *authors* ``[(firstName, lastName), ...]`` to *item_id*.
Returns count added.
"""
type_id = CREATOR_TYPES.get(creator_type, 1)
for idx, (first_name, last_name) in enumerate(authors):
creator_id = self.ensure_creator(first_name, last_name)
self.con.execute(
"INSERT OR IGNORE INTO itemCreators "
"(itemID, creatorID, creatorTypeID, orderIndex) "
"VALUES (?, ?, ?, ?)",
(item_id, creator_id, type_id, idx),
)
return len(authors)
def get_creators(self, item_id: int) -> list[dict[str, str | int]]:
"""Return creators for *item_id* in order."""
rows = self.con.execute(
"""SELECT c.firstName, c.lastName, ct.creatorType, ic.orderIndex
FROM itemCreators ic
JOIN creators c ON ic.creatorID = c.creatorID
JOIN creatorTypes ct ON ic.creatorTypeID = ct.creatorTypeID
WHERE ic.itemID = ?
ORDER BY ic.orderIndex""",
(item_id,),
).fetchall()
return [
{"firstName": r[0], "lastName": r[1], "creatorType": r[2], "order": r[3]}
for r in rows
]
# ── Tags ─────────────────────────────────────────────────────
def ensure_tag(self, name: str) -> int:
"""Find or create a tag. Returns ``tagID``."""
row = self.con.execute(
"SELECT tagID FROM tags WHERE name = ?", (name,)
).fetchone()
if row:
return row[0]
cur = self.con.execute("INSERT INTO tags (name) VALUES (?)", (name,))
return cur.lastrowid
def tag_item(self, item_id: int, tag_name: str) -> None:
"""Add a single tag to *item_id*."""
if not tag_name:
return
tag_id = self.ensure_tag(tag_name)
self.con.execute(
"INSERT OR IGNORE INTO itemTags (itemID, tagID, type) VALUES (?, ?, 0)",
(item_id, tag_id),
)
def sync_tags(self, item_id: int, tags: list[str]) -> None:
"""Add multiple tags to *item_id*, skipping blanks."""
for tag_name in tags:
self.tag_item(item_id, tag_name)
def get_tags(self, item_id: int) -> list[str]:
"""Return tag names for *item_id*."""
rows = self.con.execute(
"""SELECT t.name FROM itemTags it
JOIN tags t ON it.tagID = t.tagID
WHERE it.itemID = ?""",
(item_id,),
).fetchall()
return [r[0] for r in rows]
# ── Collections ──────────────────────────────────────────────
def find_collection(self, key: str) -> int | None:
"""Return ``collectionID`` for *key*, or ``None``."""
row = self.con.execute(
"SELECT collectionID FROM collections WHERE key = ?", (key,)
).fetchone()
return row[0] if row else None
def ensure_collection(
self,
name: str,
*,
parent_key: str = "",
library_id: int = 1,
) -> str:
"""Find or create a collection by *name*. Returns its key.
If *parent_key* is given, the collection is scoped under that parent.
"""
parent_id: int | None = None
if parent_key:
row = self.con.execute(
"SELECT collectionID FROM collections WHERE key = ?",
(parent_key,),
).fetchone()
parent_id = row[0] if row else None
if parent_id is not None:
row = self.con.execute(
"SELECT key FROM collections "
"WHERE collectionName = ? AND parentCollectionID = ?",
(name, parent_id),
).fetchone()
else:
row = self.con.execute(
"SELECT key FROM collections "
"WHERE collectionName = ? AND parentCollectionID IS NULL",
(name,),
).fetchone()
if row:
return row[0]
key = generate_key()
self.con.execute(
"""INSERT INTO collections
(collectionName, parentCollectionID, clientDateModified,
libraryID, key, version, synced)
VALUES (?, ?, ?, ?, ?, 0, 0)""",
(name, parent_id, now_iso(), library_id, key),
)
self.con.commit()
return key
def add_to_collection(self, item_id: int, *, collection_key: str) -> bool:
"""Add *item_id* to the collection identified by *collection_key*.
Returns ``True`` if inserted, ``False`` if collection not found
or item already present.
"""
col_id = self.find_collection(collection_key)
if col_id is None:
return False
max_order = self.con.execute(
"SELECT COALESCE(MAX(orderIndex), -1) "
"FROM collectionItems WHERE collectionID = ?",
(col_id,),
).fetchone()[0]
self.con.execute(
"INSERT OR IGNORE INTO collectionItems "
"(collectionID, itemID, orderIndex) VALUES (?, ?, ?)",
(col_id, item_id, max_order + 1),
)
return True
def get_item_collections(self, item_id: int) -> list[str]:
"""Return collection keys for *item_id*."""
rows = self.con.execute(
"""SELECT c.key FROM collectionItems ci
JOIN collections c ON ci.collectionID = c.collectionID
WHERE ci.itemID = ?""",
(item_id,),
).fetchall()
return [r[0] for r in rows]
# ── Attachments ──────────────────────────────────────────────
def add_attachment(
self,
parent_item_id: int,
*,
link_mode: int = 0,
content_type: str = "",
path: str = "",
key: str = "",
library_id: int = 1,
) -> int:
"""Create an attachment item linked to *parent_item_id*.
Returns the attachment's ``itemID``.
"""
# Attachments are itemTypeID=14
item_id = self.create_item(
TYPE_MAP["attachment"], key=key, library_id=library_id
)
self.con.execute(
"""INSERT INTO itemAttachments
(itemID, parentItemID, linkMode, contentType, path)
VALUES (?, ?, ?, ?, ?)""",
(item_id, parent_item_id, link_mode, content_type, path),
)
return item_id
def add_note(
self,
parent_item_id: int,
note: str,
*,
title: str = "",
key: str = "",
library_id: int = 1,
) -> int:
"""Create a note item linked to *parent_item_id*.
Returns the note's ``itemID``.
"""
item_id = self.create_item(TYPE_MAP["note"], key=key, library_id=library_id)
self.con.execute(
"""INSERT INTO itemNotes
(itemID, parentItemID, note, title)
VALUES (?, ?, ?, ?)""",
(item_id, parent_item_id, note, title),
)
return item_id
# ── Querying ─────────────────────────────────────────────────
def count_items(self, type_id: int | None = None) -> int:
"""Count items, optionally filtered by type."""
if type_id is not None:
return self.con.execute(
"SELECT count(*) FROM items WHERE itemTypeID = ?", (type_id,)
).fetchone()[0]
return self.con.execute("SELECT count(*) FROM items").fetchone()[0]
def count_tags(self) -> int:
return self.con.execute("SELECT count(*) FROM tags").fetchone()[0]
def count_creators(self) -> int:
return self.con.execute("SELECT count(*) FROM creators").fetchone()[0]
def count_collections(self) -> int:
return self.con.execute("SELECT count(*) FROM collections").fetchone()[0]
# ── Structured reads ─────────────────────────────────────────
def get_item(self, item_id: int) -> dict | None:
"""Return a full structured dict for *item_id*, or ``None``."""
row = self.con.execute(
"SELECT itemID, itemTypeID, dateAdded, dateModified, "
"libraryID, key, version, synced FROM items WHERE itemID = ?",
(item_id,),
).fetchone()
if not row:
return None
type_id = row["itemTypeID"]
type_name = _REVERSE_TYPE_MAP.get(type_id, str(type_id))
return {
"itemID": row["itemID"],
"itemType": type_name,
"itemTypeID": type_id,
"key": row["key"],
"dateAdded": row["dateAdded"],
"dateModified": row["dateModified"],
"libraryID": row["libraryID"],
"fields": self.get_fields(item_id),
"creators": self.get_creators(item_id),
"tags": self.get_tags(item_id),
"collections": self.get_item_collections(item_id),
}
def get_item_by_key(self, key: str) -> dict | None:
"""Convenience: ``get_item`` by key instead of ID."""
item_id = self.find_item_by_key(key)
return self.get_item(item_id) if item_id else None
# ── Search / Query ───────────────────────────────────────────
def search_by_tag(self, tag_name: str, *, limit: int = 0) -> list[int]:
"""Return ``itemID`` list for items tagged *tag_name*."""
sql = """SELECT DISTINCT i.itemID FROM items i
JOIN itemTags it ON i.itemID = it.itemID
JOIN tags t ON it.tagID = t.tagID
WHERE t.name = ?
ORDER BY i.itemID"""
if limit:
sql += f" LIMIT {limit}"
return [r[0] for r in self.con.execute(sql, (tag_name,)).fetchall()]
def search_by_type(self, type_name: str, *, limit: int = 0) -> list[int]:
"""Return ``itemID`` list for items of *type_name*."""
type_id = TYPE_MAP.get(type_name)
if type_id is None:
return []
sql = "SELECT itemID FROM items WHERE itemTypeID = ? ORDER BY itemID"
if limit:
sql += f" LIMIT {limit}"
return [r[0] for r in self.con.execute(sql, (type_id,)).fetchall()]
def search_by_field(
self, field_name: str, value: str, *, exact: bool = True, limit: int = 0
) -> list[int]:
"""Return ``itemID`` list matching a field value.
Set *exact=False* for LIKE matching (wraps *value* in ``%``).
"""
field_id = FIELD_MAP.get(field_name)
if field_id is None:
return []
if exact:
sql = """SELECT id.itemID FROM itemData id
JOIN itemDataValues idv ON id.valueID = idv.valueID
WHERE id.fieldID = ? AND idv.value = ?"""
params: tuple = (field_id, value)
else:
sql = """SELECT id.itemID FROM itemData id
JOIN itemDataValues idv ON id.valueID = idv.valueID
WHERE id.fieldID = ? AND idv.value LIKE ?"""
params = (field_id, f"%{value}%")
sql += " ORDER BY id.itemID"
if limit:
sql += f" LIMIT {limit}"
return [r[0] for r in self.con.execute(sql, params).fetchall()]
def search_by_collection(self, collection_key: str, *, limit: int = 0) -> list[int]:
"""Return ``itemID`` list for items in the given collection."""
sql = """SELECT ci.itemID FROM collectionItems ci
JOIN collections c ON ci.collectionID = c.collectionID
WHERE c.key = ?
ORDER BY ci.orderIndex"""
if limit:
sql += f" LIMIT {limit}"
return [r[0] for r in self.con.execute(sql, (collection_key,)).fetchall()]
def search(
self,
*,
tag: str = "",
type_name: str = "",
field: tuple[str, str] | None = None,
collection_key: str = "",
limit: int = 0,
) -> list[int]:
"""Combined search — intersects all non-empty criteria."""
sets = []
if tag:
sets.append(set(self.search_by_tag(tag)))
if type_name:
sets.append(set(self.search_by_type(type_name)))
if field:
sets.append(set(self.search_by_field(field[0], field[1])))
if collection_key:
sets.append(set(self.search_by_collection(collection_key)))
if not sets:
return []
result = sorted(sets[0].intersection(*sets[1:]))
return result[:limit] if limit else result
# ── Validation ───────────────────────────────────────────────
def valid_fields_for_type(self, type_id: int) -> set[int]:
"""Return the set of valid fieldIDs for *type_id*."""
rows = self.con.execute(
"SELECT fieldID FROM itemTypeFieldsCombined WHERE itemTypeID = ?",
(type_id,),
).fetchall()
return {r[0] for r in rows}
def valid_creator_types_for(self, type_id: int) -> set[int]:
"""Return valid creatorTypeIDs for *type_id*."""
rows = self.con.execute(
"SELECT creatorTypeID FROM itemTypeCreatorTypes WHERE itemTypeID = ?",
(type_id,),
).fetchall()
return {r[0] for r in rows}
def validate_item(self, item_id: int) -> list[str]:
"""Check an item for schema violations. Returns list of issues."""
issues: list[str] = []
row = self.con.execute(
"SELECT itemTypeID FROM items WHERE itemID = ?", (item_id,)
).fetchone()
if not row:
return [f"itemID {item_id} does not exist"]
type_id = row[0]
valid = self.valid_fields_for_type(type_id)
if valid:
for r in self.con.execute(
"SELECT fieldID FROM itemData WHERE itemID = ?", (item_id,)
):
if r[0] not in valid:
fname = _REVERSE_FIELD_MAP.get(r[0], str(r[0]))
tname = _REVERSE_TYPE_MAP.get(type_id, str(type_id))
issues.append(
f"field {fname} ({r[0]}) invalid for type {tname} ({type_id})"
)
return issues
# ── Bulk operations ──────────────────────────────────────────
def bulk_create(self, items: list[dict], *, commit: bool = True) -> list[int]:
"""Bulk-create items from dicts.
Each dict: ``type`` (str), ``fields`` (dict), ``tags`` (list),
``creators`` (list of (first, last) tuples), ``key`` (optional).
"""
ts = now_iso()
ids = []
for item in items:
type_id = item.get("type_id") or TYPE_MAP.get(item.get("type", ""), 0)
if not type_id:
continue
item_id = self.create_item(type_id, key=item.get("key", ""), now=ts)
if item.get("fields"):
self.set_fields(item_id, item["fields"])
if item.get("tags"):
self.sync_tags(item_id, item["tags"])
if item.get("creators"):
self.add_creators(item_id, item["creators"])
ids.append(item_id)
if commit:
self.commit()
return ids
def export_items(self, item_ids: list[int] | None = None) -> list[dict]:
"""Export items as structured dicts (all items if *item_ids* is None)."""
if item_ids is None:
item_ids = [
r[0]
for r in self.con.execute("SELECT itemID FROM items ORDER BY itemID")
]
return [d for iid in item_ids if (d := self.get_item(iid)) is not None]
# ── Delete ───────────────────────────────────────────────────
def delete_item(self, item_id: int) -> bool:
"""Delete an item and all its EAV data, tags, creators, collection links."""
if not self.con.execute(
"SELECT 1 FROM items WHERE itemID = ?", (item_id,)
).fetchone():
return False
for tbl in ("itemData", "itemTags", "itemCreators", "collectionItems"):
self.con.execute(f"DELETE FROM {tbl} WHERE itemID = ?", (item_id,))
self.con.execute("DELETE FROM itemAttachments WHERE itemID = ?", (item_id,))
self.con.execute(
"DELETE FROM itemAttachments WHERE parentItemID = ?", (item_id,)
)
self.con.execute("DELETE FROM itemNotes WHERE itemID = ?", (item_id,))
self.con.execute("DELETE FROM itemNotes WHERE parentItemID = ?", (item_id,))
self.con.execute("DELETE FROM items WHERE itemID = ?", (item_id,))
return True
# ── Stats ────────────────────────────────────────────────────
def stats(self) -> dict[str, int]:
"""Return a summary of DB contents."""
return {
"items": self.count_items(),
"tags": self.count_tags(),
"creators": self.count_creators(),
"collections": self.count_collections(),
"values": self.con.execute(
"SELECT count(*) FROM itemDataValues"
).fetchone()[0],
"data_rows": self.con.execute("SELECT count(*) FROM itemData").fetchone()[
0
],
}
# ── Reverse lookup maps ─────────────────────────────────────────────
_REVERSE_TYPE_MAP: dict[int, str] = {v: k for k, v in TYPE_MAP.items()}
_REVERSE_FIELD_MAP: dict[int, str] = {v: k for k, v in FIELD_MAP.items()}

331
src/zot/duck.py Normal file
View File

@@ -0,0 +1,331 @@
"""DuckDB engine for Zotero — analytics, cross-joins, native tables.
Attaches a Zotero SQLite file via DuckDB's sqlite scanner, giving you
the full 61-table schema queryable with DuckDB SQL (window functions,
PIVOT, COPY TO parquet, joins with pipeline data, etc.).
Three modes:
1. **Attach** (read-only, zero-copy) — queries hit SQLite directly::
from zot.duck import DuckDb
zdb = DuckDb.attach("data/zotero/data/zotero.sqlite")
zdb.sql("SELECT * FROM items WHERE itemTypeID = 4 LIMIT 5")
2. **Mirror** (native DuckDB copy) — fast analytics on a snapshot::
zdb = DuckDb.mirror("data/zotero/data/zotero.sqlite")
zdb.sql("SELECT * FROM items LIMIT 5")
3. **In-memory from Db** — pipe ORM data into DuckDB::
from zot.db import Db
from zot.duck import DuckDb
with Db("zotero.sqlite") as db:
zdb = DuckDb.from_db(db)
zdb.items()
"""
from __future__ import annotations
from typing import TYPE_CHECKING
import duckdb
if TYPE_CHECKING:
from zot.db import Db
# ── Flattening queries ───────────────────────────────────────────
#
# Zotero's EAV schema requires 4-5 joins to get one field value.
# These CTEs denormalize everything into flat tables for analytics.
FLAT_ITEMS_SQL = """
SELECT
i.itemID,
i.itemTypeID,
it.typeName AS item_type,
i.dateAdded,
i.dateModified,
i.libraryID,
i.key,
-- Pivot common fields out of EAV
MAX(CASE WHEN f.fieldName = 'title' THEN idv.value END) AS title,
MAX(CASE WHEN f.fieldName = 'nameOfAct' THEN idv.value END) AS name_of_act,
MAX(CASE WHEN f.fieldName = 'url' THEN idv.value END) AS url,
MAX(CASE WHEN f.fieldName = 'DOI' THEN idv.value END) AS doi,
MAX(CASE WHEN f.fieldName = 'abstractNote' THEN idv.value END) AS abstract,
MAX(CASE WHEN f.fieldName = 'date' THEN idv.value END) AS date,
MAX(CASE WHEN f.fieldName = 'dateEnacted' THEN idv.value END) AS date_enacted,
MAX(CASE WHEN f.fieldName = 'accessDate' THEN idv.value END) AS access_date,
MAX(CASE WHEN f.fieldName = 'publicationTitle' THEN idv.value END) AS publication,
MAX(CASE WHEN f.fieldName = 'volume' THEN idv.value END) AS volume,
MAX(CASE WHEN f.fieldName = 'issue' THEN idv.value END) AS issue,
MAX(CASE WHEN f.fieldName = 'pages' THEN idv.value END) AS pages,
MAX(CASE WHEN f.fieldName = 'publisher' THEN idv.value END) AS publisher,
MAX(CASE WHEN f.fieldName = 'institution' THEN idv.value END) AS institution,
MAX(CASE WHEN f.fieldName = 'extra' THEN idv.value END) AS extra,
MAX(CASE WHEN f.fieldName = 'code' THEN idv.value END) AS code,
MAX(CASE WHEN f.fieldName = 'codeNumber' THEN idv.value END) AS code_number,
MAX(CASE WHEN f.fieldName = 'section' THEN idv.value END) AS section,
MAX(CASE WHEN f.fieldName = 'history' THEN idv.value END) AS history,
MAX(CASE WHEN f.fieldName = 'reportType' THEN idv.value END) AS report_type,
MAX(CASE WHEN f.fieldName = 'reportNumber' THEN idv.value END) AS report_number,
MAX(CASE WHEN f.fieldName = 'websiteTitle' THEN idv.value END) AS website_title,
MAX(CASE WHEN f.fieldName = 'websiteType' THEN idv.value END) AS website_type,
MAX(CASE WHEN f.fieldName = 'ISSN' THEN idv.value END) AS issn,
MAX(CASE WHEN f.fieldName = 'language' THEN idv.value END) AS language,
MAX(CASE WHEN f.fieldName = 'shortTitle' THEN idv.value END) AS short_title
FROM {schema}items i
JOIN {schema}itemTypes it ON i.itemTypeID = it.itemTypeID
LEFT JOIN {schema}itemData id ON i.itemID = id.itemID
LEFT JOIN {schema}fields f ON id.fieldID = f.fieldID
LEFT JOIN {schema}itemDataValues idv ON CAST(id.valueID AS VARCHAR) = CAST(idv.valueID AS VARCHAR)
WHERE i.itemTypeID NOT IN (1, 14) -- exclude notes and attachments
GROUP BY i.itemID, i.itemTypeID, it.typeName,
i.dateAdded, i.dateModified, i.libraryID, i.key
"""
FLAT_TAGS_SQL = """
SELECT i.itemID, i.key, t.name AS tag
FROM {schema}items i
JOIN {schema}itemTags it ON i.itemID = it.itemID
JOIN {schema}tags t ON it.tagID = t.tagID
"""
FLAT_CREATORS_SQL = """
SELECT
ic.itemID,
c.firstName AS first_name,
c.lastName AS last_name,
ct.creatorType AS creator_type,
ic.orderIndex AS position
FROM {schema}itemCreators ic
JOIN {schema}creators c ON ic.creatorID = c.creatorID
JOIN {schema}creatorTypes ct ON ic.creatorTypeID = ct.creatorTypeID
ORDER BY ic.itemID, ic.orderIndex
"""
FLAT_COLLECTIONS_SQL = """
SELECT ci.itemID, c.collectionName AS collection, c.key AS collection_key
FROM {schema}collectionItems ci
JOIN {schema}collections c ON ci.collectionID = c.collectionID
"""
class DuckDb:
"""DuckDB engine for Zotero analytics.
All methods return DuckDB relations or DataFrames — never modifies
the SQLite source.
"""
def __init__(self, con: duckdb.DuckDBPyConnection, *, schema: str = "") -> None:
self.con = con
self._schema = f"{schema}." if schema else ""
def close(self) -> None:
self.con.close()
def __enter__(self) -> DuckDb:
return self
def __exit__(self, *exc) -> None:
self.close()
# ── Constructors ─────────────────────────────────────────────
@classmethod
def attach(
cls,
sqlite_path: str,
*,
schema: str = "zot",
read_only: bool = True,
) -> DuckDb:
"""Attach a Zotero SQLite file. Zero-copy, queries hit SQLite."""
con = duckdb.connect()
con.execute("INSTALL sqlite; LOAD sqlite")
ro = ", READ_ONLY" if read_only else ""
con.execute(f"ATTACH '{sqlite_path}' AS {schema} (TYPE SQLITE{ro})")
return cls(con, schema=schema)
@classmethod
def mirror(
cls,
sqlite_path: str,
*,
duckdb_path: str = ":memory:",
) -> DuckDb:
"""Copy all Zotero tables into native DuckDB tables.
Much faster for repeated analytics — DuckDB columnar storage
vs SQLite row-oriented scans.
"""
con = duckdb.connect(duckdb_path)
con.execute("INSTALL sqlite; LOAD sqlite")
tables = [
r[0]
for r in con.execute(
f"SELECT name FROM sqlite_scan('{sqlite_path}', 'sqlite_master') "
"WHERE type='table'"
).fetchall()
]
con.execute(f"ATTACH '{sqlite_path}' AS src (TYPE SQLITE, READ_ONLY)")
for t in tables:
con.execute(f'CREATE TABLE "{t}" AS SELECT * FROM src."{t}"')
con.execute("DETACH src")
return cls(con)
@classmethod
def from_db(cls, db: Db) -> DuckDb:
"""Create a DuckDB mirror from an open ``zot.db.Db`` instance."""
return cls.mirror(db.path)
# ── Raw SQL ──────────────────────────────────────────────────
def sql(self, query: str, params: list | None = None) -> duckdb.DuckDBPyRelation:
"""Execute raw SQL and return a DuckDB relation."""
if params:
return self.con.execute(query, params)
return self.con.execute(query)
# ── Flat views ───────────────────────────────────────────────
def items(self) -> duckdb.DuckDBPyRelation:
"""Flat items table — EAV pivoted into columns."""
return self.con.sql(FLAT_ITEMS_SQL.format(schema=self._schema))
def tags(self) -> duckdb.DuckDBPyRelation:
"""Flat item-tag pairs."""
return self.con.sql(FLAT_TAGS_SQL.format(schema=self._schema))
def creators(self) -> duckdb.DuckDBPyRelation:
"""Flat item-creator pairs."""
return self.con.sql(FLAT_CREATORS_SQL.format(schema=self._schema))
def collections(self) -> duckdb.DuckDBPyRelation:
"""Flat item-collection pairs."""
return self.con.sql(FLAT_COLLECTIONS_SQL.format(schema=self._schema))
# ── Materialized flat tables ─────────────────────────────────
def materialize(self) -> None:
"""Create native DuckDB tables ``flat_items``, ``flat_tags``, etc.
Call once after ``attach()`` or ``mirror()`` for fastest repeated queries.
"""
self.con.execute(
f"CREATE OR REPLACE TABLE flat_items AS {FLAT_ITEMS_SQL.format(schema=self._schema)}"
)
self.con.execute(
f"CREATE OR REPLACE TABLE flat_tags AS {FLAT_TAGS_SQL.format(schema=self._schema)}"
)
self.con.execute(
f"CREATE OR REPLACE TABLE flat_creators AS {FLAT_CREATORS_SQL.format(schema=self._schema)}"
)
self.con.execute(
f"CREATE OR REPLACE TABLE flat_collections AS {FLAT_COLLECTIONS_SQL.format(schema=self._schema)}"
)
# ── Analytics helpers ────────────────────────────────────────
def items_by_type(self) -> duckdb.DuckDBPyRelation:
"""Count items per type."""
return self.con.sql(f"""
SELECT it.typeName, count(*) as cnt
FROM {self._schema}items i
JOIN {self._schema}itemTypes it ON i.itemTypeID = it.itemTypeID
GROUP BY it.typeName ORDER BY cnt DESC
""")
def items_by_tag(self, *, top_n: int = 50) -> duckdb.DuckDBPyRelation:
"""Count items per tag."""
return self.con.sql(f"""
SELECT t.name AS tag, count(*) as cnt
FROM {self._schema}itemTags it
JOIN {self._schema}tags t ON it.tagID = t.tagID
GROUP BY t.name ORDER BY cnt DESC
LIMIT {top_n}
""")
def items_by_year(self) -> duckdb.DuckDBPyRelation:
"""Count items per year (from dateAdded)."""
return self.con.sql(f"""
SELECT strftime(i.dateAdded, '%Y') AS year, count(*) as cnt
FROM {self._schema}items i
WHERE i.dateAdded IS NOT NULL
AND i.itemTypeID NOT IN (1, 14)
GROUP BY 1 ORDER BY 1
""")
def items_by_collection(self) -> duckdb.DuckDBPyRelation:
"""Count items per collection."""
return self.con.sql(f"""
SELECT c.collectionName, count(*) as cnt
FROM {self._schema}collectionItems ci
JOIN {self._schema}collections c ON ci.collectionID = c.collectionID
GROUP BY c.collectionName ORDER BY cnt DESC
""")
def tag_co_occurrence(self, *, min_count: int = 5) -> duckdb.DuckDBPyRelation:
"""Tag co-occurrence matrix (which tags appear together)."""
return self.con.sql(f"""
SELECT t1.name AS tag_a, t2.name AS tag_b, count(*) as cnt
FROM {self._schema}itemTags it1
JOIN {self._schema}tags t1 ON it1.tagID = t1.tagID
JOIN {self._schema}itemTags it2 ON it1.itemID = it2.itemID AND it1.tagID < it2.tagID
JOIN {self._schema}tags t2 ON it2.tagID = t2.tagID
GROUP BY t1.name, t2.name
HAVING cnt >= {min_count}
ORDER BY cnt DESC
""")
def doi_coverage(self) -> duckdb.DuckDBPyRelation:
"""DOI coverage by item type."""
return self.con.sql(f"""
SELECT it.typeName,
count(*) AS total,
count(doi.valueID) AS with_doi,
round(100.0 * count(doi.valueID) / count(*), 1) AS pct
FROM {self._schema}items i
JOIN {self._schema}itemTypes it ON i.itemTypeID = it.itemTypeID
LEFT JOIN {self._schema}itemData doi
ON i.itemID = doi.itemID AND doi.fieldID = 26
WHERE i.itemTypeID NOT IN (1, 14)
GROUP BY it.typeName
ORDER BY total DESC
""")
def pdf_coverage(self) -> duckdb.DuckDBPyRelation:
"""PDF attachment coverage by item type."""
return self.con.sql(f"""
SELECT it.typeName,
count(DISTINCT i.itemID) AS total,
count(DISTINCT ia.parentItemID) AS with_pdf,
round(100.0 * count(DISTINCT ia.parentItemID) / count(DISTINCT i.itemID), 1) AS pct
FROM {self._schema}items i
JOIN {self._schema}itemTypes it ON i.itemTypeID = it.itemTypeID
LEFT JOIN {self._schema}itemAttachments ia
ON i.itemID = ia.parentItemID AND ia.contentType = 'application/pdf'
WHERE i.itemTypeID NOT IN (1, 14)
GROUP BY it.typeName
ORDER BY total DESC
""")
# ── Export ───────────────────────────────────────────────────
def to_parquet(self, path: str, *, table: str = "flat_items") -> None:
"""Export a table to Parquet. Call ``materialize()`` first."""
self.con.execute(f"COPY {table} TO '{path}' (FORMAT PARQUET)")
def to_df(self, relation: duckdb.DuckDBPyRelation | None = None):
"""Convert a relation (or flat_items) to a Polars DataFrame."""
import polars as pl
if relation is None:
relation = self.items()
return pl.from_arrow(relation.arrow())

392
src/zot/extract.py Normal file
View File

@@ -0,0 +1,392 @@
"""Extract tags, annotations, and note quotes from Zotero for use in docstrings.
Bridges Zotero's annotation/note data into the ``bib`` provenance system:
- Parses Zotero's HTML notes into structured ``Quote`` objects
- Maps Zotero tags to ``bib.tag.Tag`` namespace conventions
- Resolves ``zotero://open-pdf`` links to page numbers
- Generates ``:pincite:`` directives for Python docstrings
Usage::
from zot.extract import Extractor
ex = Extractor("data/zotero/data/zotero.sqlite")
# Get all quotes from an item's notes
quotes = ex.quotes_for_item("VTWVB384")
for q in quotes:
print(q.pincite_directive())
# :pincite:`VTWVB384 p.14` — "A beneficiary's MBI is unique..."
# Get structured tag mappings
tags = ex.tags_for_item("VTWVB384")
# ["module:aco", "source:cms-website", "seed:true"]
# Generate docstring block for a function
block = ex.docstring_block("VTWVB384", sections=["§2.2.1", "§5.3.1"])
print(block)
"""
from __future__ import annotations
import re
from dataclasses import dataclass
from html.parser import HTMLParser
from zot.db import Db
# ── HTML note parser ─────────────────────────────────────────────
class _NoteParser(HTMLParser):
"""Parse Zotero's HTML notes into plain-text segments with metadata."""
def __init__(self) -> None:
super().__init__()
self._current: list[str] = []
self._current_link = ""
self._para_link = ""
self._segments: list[dict] = []
def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
if tag == "a":
href = dict(attrs).get("href", "")
self._current_link = href
if href:
self._para_link = href
elif tag in ("p", "br"):
self._flush()
def handle_endtag(self, tag: str) -> None:
if tag == "a":
self._current_link = ""
elif tag == "p":
self._flush()
def handle_data(self, data: str) -> None:
self._current.append(data)
def _flush(self) -> None:
text = "".join(self._current).strip()
if text:
self._segments.append({"text": text, "link": self._para_link})
self._current = []
self._para_link = ""
def result(self) -> list[dict]:
self._flush()
return self._segments
def _parse_note_html(html: str) -> list[dict]:
"""Parse a Zotero HTML note into text segments.
Each segment: ``{"text": "...", "link": "zotero://..." or ""}``
"""
parser = _NoteParser()
parser.feed(html)
return parser.result()
_PDF_LINK_RE = re.compile(r"zotero://open-pdf/[^/]+/(\d+)")
def _page_from_link(link: str) -> int | None:
"""Extract page number from ``zotero://open-pdf/0_KEY/page``."""
m = _PDF_LINK_RE.search(link)
return int(m.group(1)) if m else None
_QUOTE_RE = re.compile(r'^"(.+)"$', re.DOTALL)
_PARENS_REF = re.compile(r"\(([^)]+)\)$")
@dataclass
class Quote:
"""A single annotation quote extracted from a Zotero note."""
text: str
page: int | None = None
section: str = ""
source_ref: str = "" # e.g. "Ibrahim et al :1"
item_key: str = ""
def pincite_directive(self) -> str:
"""Format as a ``:pincite:`` RST directive for docstrings."""
locator_parts = []
if self.section:
locator_parts.append(self.section)
if self.page:
locator_parts.append(f"p.{self.page}")
locator = " ".join(locator_parts)
loc_str = f" {locator}" if locator else ""
# Truncate quote to ~120 chars for docstrings
short = self.text[:120] + "..." if len(self.text) > 120 else self.text
return f':pincite:`{self.item_key}{loc_str}` — "{short}"'
def as_tag(self) -> str:
"""Format as a ``pin:`` tag value."""
parts = [self.item_key]
if self.section:
parts.append(self.section)
elif self.page:
parts.append(f"p.{self.page}")
return "pin:" + "/".join(parts)
def _extract_quotes_from_note(html: str, item_key: str) -> list[Quote]:
"""Extract quoted text from a Zotero HTML note."""
segments = _parse_note_html(html)
quotes: list[Quote] = []
for seg in segments:
text = seg["text"]
link = seg.get("link", "")
# Skip header lines
if text.startswith("Extracted Annotations"):
continue
# Match quoted text: "..." or "..." (ref)
# Zotero stores annotations as quoted paragraphs
m = _QUOTE_RE.match(text)
if not m:
# Also try: text with parenthetical reference at end
# e.g. '"some quote" (Author :1)'
if text.startswith('"') and '"' in text[1:]:
end_quote = text.index('"', 1)
quoted = text[1:end_quote]
rest = text[end_quote + 1 :].strip()
source_ref = ""
pm = _PARENS_REF.search(rest)
if pm:
source_ref = pm.group(1)
page = _page_from_link(link) if link else None
if not page and source_ref:
# Try to extract page from source ref like "Author :5"
pm2 = re.search(r":(\d+)", source_ref)
if pm2:
page = int(pm2.group(1))
quotes.append(
Quote(
text=quoted,
page=page,
source_ref=source_ref,
item_key=item_key,
)
)
continue
continue
quoted_text = m.group(1).strip()
page = _page_from_link(link) if link else None
quotes.append(Quote(text=quoted_text, page=page, item_key=item_key))
return quotes
class Extractor:
"""Extract provenance data from Zotero for use in Python docstrings.
Bridges Zotero → ``bib`` provenance system.
"""
def __init__(self, db_or_path: Db | str) -> None:
if isinstance(db_or_path, str):
self._db = Db(db_or_path)
self._owns_db = True
else:
self._db = db_or_path
self._owns_db = False
def close(self) -> None:
if self._owns_db:
self._db.close()
def __enter__(self) -> Extractor:
return self
def __exit__(self, *exc) -> None:
self.close()
@property
def db(self) -> Db:
return self._db
def quotes_for_item(self, key: str) -> list[Quote]:
"""Extract all annotation quotes from an item's notes."""
item_id = self._db.find_item_by_key(key)
if item_id is None:
return []
rows = self._db.con.execute(
"SELECT note FROM itemNotes WHERE parentItemID = ? AND note IS NOT NULL",
(item_id,),
).fetchall()
quotes: list[Quote] = []
for row in rows:
quotes.extend(_extract_quotes_from_note(row[0], key))
return quotes
def quotes_for_tag(self, tag_name: str) -> dict[str, list[Quote]]:
"""Extract quotes for all items with a given tag.
Returns ``{item_key: [Quote, ...]}``
"""
item_ids = self._db.search_by_tag(tag_name)
result: dict[str, list[Quote]] = {}
for iid in item_ids:
row = self._db.con.execute(
"SELECT key FROM items WHERE itemID = ?", (iid,)
).fetchone()
if row:
quotes = self.quotes_for_item(row[0])
if quotes:
result[row[0]] = quotes
return result
def tags_for_item(self, key: str) -> list[str]:
"""Get all tags for an item by key."""
item_id = self._db.find_item_by_key(key)
if item_id is None:
return []
return self._db.get_tags(item_id)
def items_by_tag_namespace(self, namespace: str) -> dict[str, list[str]]:
"""Group items by tags in a namespace.
E.g. ``items_by_tag_namespace("module")`` returns
``{"pfs": ["KEY1", "KEY2"], "aco": ["KEY3"]}``
"""
rows = self._db.con.execute(
"""SELECT t.name, i.key
FROM itemTags it
JOIN tags t ON it.tagID = t.tagID
JOIN items i ON it.itemID = i.itemID
WHERE t.name LIKE ?
ORDER BY t.name""",
(f"{namespace}:%",),
).fetchall()
result: dict[str, list[str]] = {}
for tag_name, item_key in rows:
value = tag_name.split(":", 1)[1] if ":" in tag_name else tag_name
result.setdefault(value, []).append(item_key)
return result
def fields_for_item(self, key: str) -> dict[str, str]:
"""Get all EAV fields for an item by key."""
item_id = self._db.find_item_by_key(key)
if item_id is None:
return {}
return self._db.get_fields(item_id)
def docstring_block(
self,
key: str,
*,
sections: list[str] | None = None,
max_quotes: int = 5,
) -> str:
"""Generate a docstring reference block for an item.
Returns a formatted string suitable for inclusion in a Python
function docstring::
References
~~~~~~~~~~
:pincite:`VTWVB384 §2.2.1 p.8` — "Part A Claims Header..."
:pincite:`VTWVB384 §5.3.1 p.23` — "Calculating Total..."
"""
quotes = self.quotes_for_item(key)
# Filter by section if specified
if sections:
filtered = []
for q in quotes:
for s in sections:
if s in (q.section or ""):
filtered.append(q)
break
else:
# Include quotes that match page numbers of sections
filtered.append(q)
quotes = filtered
if not quotes:
# No notes — generate a bare reference
fields = self.fields_for_item(key)
title = fields.get("title") or fields.get("nameOfAct") or key
url = fields.get("url", "")
lines = ["References", "~~~~~~~~~~"]
ref = f":pincite:`{key}` — {title}"
if url:
ref += f"\n {url}"
lines.append(ref)
return "\n".join(lines)
quotes = quotes[:max_quotes]
lines = ["References", "~~~~~~~~~~"]
for q in quotes:
# Assign section if provided
if sections and not q.section:
for s in sections:
q.section = s
break
lines.append(q.pincite_directive())
return "\n".join(lines)
def pincite_directives(self, key: str) -> list[str]:
"""Return ``:pincite:`` directives for all quotes from an item."""
return [q.pincite_directive() for q in self.quotes_for_item(key)]
def export_provenance(self, tag: str = "") -> list[dict]:
"""Export items with quotes and tags for knowledge graph ingestion.
Each dict::
{
"key": "VTWVB384",
"title": "CCLF Information Packet v41",
"url": "https://...",
"tags": ["module:aco", "seed:true"],
"quotes": [{"text": "...", "page": 14}, ...],
"fields": {"DOI": "10.1234/...", ...},
}
"""
if tag:
item_ids = self._db.search_by_tag(tag)
else:
item_ids = [
r[0]
for r in self._db.con.execute(
"SELECT itemID FROM items WHERE itemTypeID NOT IN (1, 14)"
).fetchall()
]
result = []
for iid in item_ids:
item = self._db.get_item(iid)
if not item:
continue
quotes = self.quotes_for_item(item["key"])
result.append(
{
"key": item["key"],
"title": item["fields"].get("title")
or item["fields"].get("nameOfAct")
or "",
"url": item["fields"].get("url", ""),
"item_type": item["itemType"],
"tags": item["tags"],
"quotes": [
{"text": q.text, "page": q.page, "section": q.section}
for q in quotes
],
"fields": item["fields"],
}
)
return result

37
src/zot/schema.py Normal file
View File

@@ -0,0 +1,37 @@
"""Create a fresh Zotero-compatible SQLite database.
Uses ``schema.sql`` (dumped from the real ``host-zotero.sqlite``) to
produce a database with all 61 tables, indexes, triggers, and the ~1100
rows of lookup data (itemTypes, fields, creatorTypes, etc.) that Zotero
requires at startup.
Usage::
from zot.schema import create_db
create_db("/tmp/test-zotero.sqlite")
# Or get an in-memory DB for tests:
con = create_db(":memory:")
"""
from __future__ import annotations
import sqlite3
from importlib.resources import files
def _load_sql() -> str:
"""Load schema.sql from the package data."""
return files("zot").joinpath("schema.sql").read_text()
def create_db(path: str = ":memory:") -> sqlite3.Connection:
"""Create a fresh Zotero SQLite database at *path*.
Returns the open connection. For file-based DBs you may want to
close it afterward; for ``:memory:`` keep it alive.
"""
con = sqlite3.connect(path)
con.executescript(_load_sql())
con.row_factory = sqlite3.Row
return con

2039
src/zot/schema.sql Normal file
View File

File diff suppressed because it is too large Load Diff

23
src/zot/table/__init__.py Normal file
View File

@@ -0,0 +1,23 @@
"""Pydantic models for all 61 Zotero SQLite tables.
Generated from ``data/zotero/data/host-zotero.sqlite`` — the real Zotero
schema with row counts as of 2026-04-09. Models are grouped by domain:
- ``core`` — libraries, items, itemTypes, feeds, groups
- ``fields`` — fields, itemData, itemDataValues, type-field mappings
- ``creators`` — creators, creatorTypes, itemCreators
- ``collections`` — collections, collectionItems
- ``tags`` — tags, itemTags
- ``attachments`` — itemAttachments, itemNotes, annotations, highlights
- ``search`` — fulltext index, saved searches, file types
- ``sync`` — sync state, cache, settings, transactions, translators
"""
from zot.table.attachments import * # noqa: F401,F403
from zot.table.collections import * # noqa: F401,F403
from zot.table.core import * # noqa: F401,F403
from zot.table.creators import * # noqa: F401,F403
from zot.table.fields import * # noqa: F401,F403
from zot.table.search import * # noqa: F401,F403
from zot.table.sync import * # noqa: F401,F403
from zot.table.tags import * # noqa: F401,F403

85
src/zot/table/attachments.py Normal file
View File

@@ -0,0 +1,85 @@
"""Attachment, note, and annotation tables."""
from __future__ import annotations
from conf.table_base import SQLTable
__all__ = [
"ItemAttachments",
"ItemNotes",
"ItemRelations",
"Annotations",
"Highlights",
]
class ItemAttachments(SQLTable):
"""Zotero table ``itemAttachments`` (3,609 rows in host DB)."""
__tablename__ = "itemAttachments"
itemID: int | None = None
parentItemID: int | None = None
linkMode: int | None = None
contentType: str | None = None
charsetID: int | None = None
path: str | None = None
syncState: int | None = 0
storageModTime: int | None = None
storageHash: str | None = None
class ItemNotes(SQLTable):
"""Zotero table ``itemNotes`` (1,366 rows in host DB)."""
__tablename__ = "itemNotes"
itemID: int | None = None
parentItemID: int | None = None
note: str | None = None
title: str | None = None
class ItemRelations(SQLTable):
"""Zotero table ``itemRelations`` (602 rows in host DB)."""
__tablename__ = "itemRelations"
itemID: int
predicateID: int
object: str
class Annotations(SQLTable):
"""Zotero table ``annotations`` (0 rows in host DB)."""
__tablename__ = "annotations"
annotationID: int | None = None
itemID: int
parent: str | None = None
textNode: int | None = None
offset: int | None = None
x: int | None = None
y: int | None = None
cols: int | None = None
rows: int | None = None
text: str | None = None
collapsed: bool | None = None
dateModified: str | None = None
class Highlights(SQLTable):
"""Zotero table ``highlights`` (0 rows in host DB)."""
__tablename__ = "highlights"
highlightID: int | None = None
itemID: int
startParent: str | None = None
startTextNode: int | None = None
startOffset: int | None = None
endParent: str | None = None
endTextNode: int | None = None
endOffset: int | None = None
dateModified: str | None = None

46
src/zot/table/collections.py Normal file
View File

@@ -0,0 +1,46 @@
"""Collection tables — collections, collectionItems, collectionRelations."""
from __future__ import annotations
from conf.table_base import SQLTable
__all__ = [
"Collections",
"CollectionItems",
"CollectionRelations",
]
class Collections(SQLTable):
"""Zotero table ``collections`` (185 rows in host DB)."""
__tablename__ = "collections"
collectionID: int | None = None
collectionName: str
parentCollectionID: int | None = None
clientDateModified: str = ""
libraryID: int
key: str
version: int = 0
synced: int = 0
class CollectionItems(SQLTable):
"""Zotero table ``collectionItems`` (5,402 rows in host DB)."""
__tablename__ = "collectionItems"
collectionID: int
itemID: int
orderIndex: int = 0
class CollectionRelations(SQLTable):
"""Zotero table ``collectionRelations`` (0 rows in host DB)."""
__tablename__ = "collectionRelations"
collectionID: int
predicateID: int
object: str

137
src/zot/table/core.py Normal file
View File

@@ -0,0 +1,137 @@
"""Core Zotero tables — libraries, items, itemTypes, feeds, groups."""
from __future__ import annotations
from conf.table_base import SQLTable
__all__ = [
"Libraries",
"Items",
"ItemTypes",
"ItemTypesCombined",
"DeletedItems",
"FeedItems",
"Feeds",
"GroupItems",
"Groups",
"PublicationsItems",
]
class Libraries(SQLTable):
"""Zotero table ``libraries`` (42 rows in host DB)."""
__tablename__ = "libraries"
libraryID: int | None = None
type: str
editable: int
filesEditable: int
version: int = 0
storageVersion: int = 0
lastSync: int = 0
archived: int = 0
class Items(SQLTable):
"""Zotero table ``items`` (8,578 rows in host DB)."""
__tablename__ = "items"
itemID: int | None = None
itemTypeID: int
dateAdded: str = ""
dateModified: str = ""
clientDateModified: str = ""
libraryID: int
key: str
version: int = 0
synced: int = 0
class ItemTypes(SQLTable):
"""Zotero table ``itemTypes`` (36 rows in host DB)."""
__tablename__ = "itemTypes"
itemTypeID: int | None = None
typeName: str | None = None
templateItemTypeID: int | None = None
display: int | None = 1
class ItemTypesCombined(SQLTable):
"""Zotero table ``itemTypesCombined`` (36 rows in host DB)."""
__tablename__ = "itemTypesCombined"
itemTypeID: int
typeName: str
display: int = 1
custom: int
class DeletedItems(SQLTable):
"""Zotero table ``deletedItems`` (105 rows in host DB)."""
__tablename__ = "deletedItems"
itemID: int | None = None
dateDeleted: str | int | float | None = ""
class FeedItems(SQLTable):
"""Zotero table ``feedItems`` (627 rows in host DB)."""
__tablename__ = "feedItems"
itemID: int | None = None
guid: str
readTime: str | None = None
translatedTime: str | None = None
class Feeds(SQLTable):
"""Zotero table ``feeds`` (38 rows in host DB)."""
__tablename__ = "feeds"
libraryID: int | None = None
name: str
url: str
lastUpdate: str | None = None
lastCheck: str | None = None
lastCheckError: str | None = None
cleanupReadAfter: int | None = None
cleanupUnreadAfter: int | None = None
refreshInterval: int | None = None
class GroupItems(SQLTable):
"""Zotero table ``groupItems`` (0 rows in host DB)."""
__tablename__ = "groupItems"
itemID: int | None = None
createdByUserID: int | None = None
lastModifiedByUserID: int | None = None
class Groups(SQLTable):
"""Zotero table ``groups`` (3 rows in host DB)."""
__tablename__ = "groups"
groupID: int | None = None
libraryID: int
name: str
description: str
version: int
class PublicationsItems(SQLTable):
"""Zotero table ``publicationsItems`` (0 rows in host DB)."""
__tablename__ = "publicationsItems"
itemID: int | None = None

53
src/zot/table/creators.py Normal file
View File

@@ -0,0 +1,53 @@
"""Creator tables — creators, creatorTypes, itemCreators."""
from __future__ import annotations
from conf.table_base import SQLTable
__all__ = [
"Creators",
"CreatorTypes",
"ItemCreators",
"ItemTypeCreatorTypes",
]
class Creators(SQLTable):
"""Zotero table ``creators`` (4,292 rows in host DB)."""
__tablename__ = "creators"
creatorID: int | None = None
firstName: str | None = None
lastName: str | None = None
fieldMode: int | None = None
class CreatorTypes(SQLTable):
"""Zotero table ``creatorTypes`` (29 rows in host DB)."""
__tablename__ = "creatorTypes"
creatorTypeID: int | None = None
creatorType: str | None = None
class ItemCreators(SQLTable):
"""Zotero table ``itemCreators`` (11,308 rows in host DB)."""
__tablename__ = "itemCreators"
itemID: int
creatorID: int
creatorTypeID: int = 1
orderIndex: int = 0
class ItemTypeCreatorTypes(SQLTable):
"""Zotero table ``itemTypeCreatorTypes`` (123 rows in host DB)."""
__tablename__ = "itemTypeCreatorTypes"
itemTypeID: int | None = None
creatorTypeID: int | None = None
primaryField: int | None = None

158
src/zot/table/fields.py Normal file
View File

@@ -0,0 +1,158 @@
"""Field and EAV tables — fields, itemData, itemDataValues, type-field mappings."""
from __future__ import annotations
from conf.table_base import SQLTable
__all__ = [
"Fields",
"FieldsCombined",
"FieldFormats",
"ItemData",
"ItemDataValues",
"ItemTypeFields",
"ItemTypeFieldsCombined",
"BaseFieldMappings",
"BaseFieldMappingsCombined",
"CustomFields",
"CustomItemTypes",
"CustomItemTypeFields",
"CustomBaseFieldMappings",
]
class Fields(SQLTable):
"""Zotero table ``fields`` (104 rows in host DB)."""
__tablename__ = "fields"
fieldID: int | None = None
fieldName: str | None = None
fieldFormatID: int | None = None
class FieldsCombined(SQLTable):
"""Zotero table ``fieldsCombined`` (104 rows in host DB)."""
__tablename__ = "fieldsCombined"
fieldID: int
fieldName: str
label: str | None = None
fieldFormatID: int | None = None
custom: int
class FieldFormats(SQLTable):
"""Zotero table ``fieldFormats`` (3 rows in host DB)."""
__tablename__ = "fieldFormats"
fieldFormatID: int | None = None
regex: str | None = None
isInteger: int | None = None
class ItemData(SQLTable):
"""Zotero table ``itemData`` — EAV join table (34,555 rows in host DB)."""
__tablename__ = "itemData"
itemID: int | None = None
fieldID: int | None = None
valueID: str | int | float | None = None
class ItemDataValues(SQLTable):
"""Zotero table ``itemDataValues`` (11,928 rows in host DB)."""
__tablename__ = "itemDataValues"
valueID: int | None = None
value: str | int | float | None = None
class ItemTypeFields(SQLTable):
"""Zotero table ``itemTypeFields`` — valid fields per item type (582 rows)."""
__tablename__ = "itemTypeFields"
itemTypeID: int | None = None
fieldID: int | None = None
hide: int | None = None
orderIndex: int | None = None
class ItemTypeFieldsCombined(SQLTable):
"""Zotero table ``itemTypeFieldsCombined`` (582 rows in host DB)."""
__tablename__ = "itemTypeFieldsCombined"
itemTypeID: int
fieldID: int
hide: int | None = None
orderIndex: int
class BaseFieldMappings(SQLTable):
"""Zotero table ``baseFieldMappings`` — generic-to-specific field maps (54 rows)."""
__tablename__ = "baseFieldMappings"
itemTypeID: int | None = None
baseFieldID: int | None = None
fieldID: int | None = None
class BaseFieldMappingsCombined(SQLTable):
"""Zotero table ``baseFieldMappingsCombined`` (54 rows in host DB)."""
__tablename__ = "baseFieldMappingsCombined"
itemTypeID: int | None = None
baseFieldID: int | None = None
fieldID: int | None = None
class CustomFields(SQLTable):
"""Zotero table ``customFields`` (0 rows in host DB)."""
__tablename__ = "customFields"
customFieldID: int | None = None
fieldName: str | None = None
label: str | None = None
class CustomItemTypes(SQLTable):
"""Zotero table ``customItemTypes`` (0 rows in host DB)."""
__tablename__ = "customItemTypes"
customItemTypeID: int | None = None
typeName: str | None = None
label: str | None = None
display: int | None = 1
icon: str | None = None
class CustomItemTypeFields(SQLTable):
"""Zotero table ``customItemTypeFields`` (0 rows in host DB)."""
__tablename__ = "customItemTypeFields"
customItemTypeID: int
fieldID: int | None = None
customFieldID: int | None = None
hide: int
orderIndex: int
class CustomBaseFieldMappings(SQLTable):
"""Zotero table ``customBaseFieldMappings`` (0 rows in host DB)."""
__tablename__ = "customBaseFieldMappings"
customItemTypeID: int | None = None
baseFieldID: int | None = None
customFieldID: int | None = None

102
src/zot/table/search.py Normal file
View File

@@ -0,0 +1,102 @@
"""Full-text search, saved searches, file types, and charsets."""
from __future__ import annotations
from conf.table_base import SQLTable
__all__ = [
"FulltextWords",
"FulltextItems",
"FulltextItemWords",
"SavedSearches",
"SavedSearchConditions",
"FileTypes",
"FileTypeMimeTypes",
"Charsets",
]
class FulltextWords(SQLTable):
"""Zotero table ``fulltextWords`` (98,514 rows in host DB)."""
__tablename__ = "fulltextWords"
wordID: int | None = None
word: str | None = None
class FulltextItems(SQLTable):
"""Zotero table ``fulltextItems`` (2,083 rows in host DB)."""
__tablename__ = "fulltextItems"
itemID: int | None = None
indexedPages: int | None = None
totalPages: int | None = None
indexedChars: int | None = None
totalChars: int | None = None
version: int = 0
synced: int = 0
class FulltextItemWords(SQLTable):
"""Zotero table ``fulltextItemWords`` (2,048,235 rows in host DB)."""
__tablename__ = "fulltextItemWords"
wordID: int | None = None
itemID: int | None = None
class SavedSearches(SQLTable):
"""Zotero table ``savedSearches`` (3 rows in host DB)."""
__tablename__ = "savedSearches"
savedSearchID: int | None = None
savedSearchName: str
clientDateModified: str = ""
libraryID: int
key: str
version: int = 0
synced: int = 0
class SavedSearchConditions(SQLTable):
"""Zotero table ``savedSearchConditions`` (9 rows in host DB)."""
__tablename__ = "savedSearchConditions"
savedSearchID: int
searchConditionID: int
condition: str
operator: str | None = None
value: str | None = None
required: str | int | float | None = None
class FileTypes(SQLTable):
"""Zotero table ``fileTypes`` (7 rows in host DB)."""
__tablename__ = "fileTypes"
fileTypeID: int | None = None
fileType: str | None = None
class FileTypeMimeTypes(SQLTable):
"""Zotero table ``fileTypeMimeTypes`` (31 rows in host DB)."""
__tablename__ = "fileTypeMimeTypes"
fileTypeID: int | None = None
mimeType: str | None = None
class Charsets(SQLTable):
"""Zotero table ``charsets`` (40 rows in host DB)."""
__tablename__ = "charsets"
charsetID: int | None = None
charset: str | None = None

193
src/zot/table/sync.py Normal file
View File

@@ -0,0 +1,193 @@
"""Sync state, cache, settings, transactions, translators, proxies."""
from __future__ import annotations
from conf.table_base import SQLTable
__all__ = [
"SyncObjectTypes",
"SyncCache",
"SyncDeleteLog",
"SyncQueue",
"SyncedSettings",
"StorageDeleteLog",
"Settings",
"Users",
"Version",
"TransactionSets",
"Transactions",
"TransactionLog",
"TranslatorCache",
"Proxies",
"ProxyHosts",
"RelationPredicates",
]
class SyncObjectTypes(SQLTable):
"""Zotero table ``syncObjectTypes`` (7 rows in host DB)."""
__tablename__ = "syncObjectTypes"
syncObjectTypeID: int | None = None
name: str | None = None
class SyncCache(SQLTable):
"""Zotero table ``syncCache`` (4,341 rows in host DB)."""
__tablename__ = "syncCache"
libraryID: int
key: str
syncObjectTypeID: int
version: int
data: str | None = None
class SyncDeleteLog(SQLTable):
"""Zotero table ``syncDeleteLog`` (0 rows in host DB)."""
__tablename__ = "syncDeleteLog"
syncObjectTypeID: int
libraryID: int
key: str
dateDeleted: str = ""
class SyncQueue(SQLTable):
"""Zotero table ``syncQueue`` (0 rows in host DB)."""
__tablename__ = "syncQueue"
libraryID: int
key: str
syncObjectTypeID: int
lastCheck: str | None = None
tries: int | None = None
class SyncedSettings(SQLTable):
"""Zotero table ``syncedSettings`` (1 rows in host DB)."""
__tablename__ = "syncedSettings"
setting: str
libraryID: int
value: str | int | float | None
version: int = 0
synced: int = 0
class StorageDeleteLog(SQLTable):
"""Zotero table ``storageDeleteLog`` (0 rows in host DB)."""
__tablename__ = "storageDeleteLog"
libraryID: int
key: str
dateDeleted: str = ""
class Settings(SQLTable):
"""Zotero table ``settings`` (4 rows in host DB)."""
__tablename__ = "settings"
setting: str | None = None
key: str | None = None
value: str | int | float | None = None
class Users(SQLTable):
"""Zotero table ``users`` (0 rows in host DB)."""
__tablename__ = "users"
userID: int | None = None
username: str
class Version(SQLTable):
"""Zotero table ``version`` (14 rows in host DB).
Note: the ``schema_name`` field maps to the ``schema`` column in SQLite.
Renamed to avoid shadowing ``SQLTable.__schema__``.
"""
__tablename__ = "version"
schema_name: str | None = None
version: int
class TransactionSets(SQLTable):
"""Zotero table ``transactionSets`` (0 rows in host DB)."""
__tablename__ = "transactionSets"
transactionSetID: int | None = None
event: str | None = None
id: int | None = None
class Transactions(SQLTable):
"""Zotero table ``transactions`` (0 rows in host DB)."""
__tablename__ = "transactions"
transactionID: int | None = None
transactionSetID: int | None = None
context: str | None = None
action: str | None = None
class TransactionLog(SQLTable):
"""Zotero table ``transactionLog`` (0 rows in host DB)."""
__tablename__ = "transactionLog"
transactionID: int | None = None
field: str | None = None
value: str | int | float | None = None
class TranslatorCache(SQLTable):
"""Zotero table ``translatorCache`` (518 rows in host DB)."""
__tablename__ = "translatorCache"
fileName: str | None = None
metadataJSON: str | None = None
lastModifiedTime: int | None = None
class Proxies(SQLTable):
"""Zotero table ``proxies`` (0 rows in host DB)."""
__tablename__ = "proxies"
proxyID: int | None = None
multiHost: int | None = None
autoAssociate: int | None = None
scheme: str | None = None
class ProxyHosts(SQLTable):
"""Zotero table ``proxyHosts`` (0 rows in host DB)."""
__tablename__ = "proxyHosts"
hostID: int | None = None
proxyID: int | None = None
hostname: str | None = None
class RelationPredicates(SQLTable):
"""Zotero table ``relationPredicates`` (3 rows in host DB)."""
__tablename__ = "relationPredicates"
predicateID: int | None = None
predicate: str | None = None

29
src/zot/table/tags.py Normal file
View File

@@ -0,0 +1,29 @@
"""Tag tables — tags, itemTags."""
from __future__ import annotations
from conf.table_base import SQLTable
__all__ = [
"Tags",
"ItemTags",
]
class Tags(SQLTable):
"""Zotero table ``tags`` (2,997 rows in host DB)."""
__tablename__ = "tags"
tagID: int | None = None
name: str
class ItemTags(SQLTable):
"""Zotero table ``itemTags`` (21,840 rows in host DB)."""
__tablename__ = "itemTags"
itemID: int
tagID: int
type: int

267
tests/bib/test_sync.py
View File

@@ -1,21 +1,16 @@
"""Tests for bib.sync — push bib items into Zotero SQLite.""" """Tests for bib.sync — push bib items into Zotero SQLite.
ORM primitives (key gen, ensure_value, ensure_tag, set_field, sync_tags,
ensure_collection) are tested in ``tests/zot/test_db.py``. This file
tests the bib→Zotero adapter: field mapping and push_to_zotero.
"""
from __future__ import annotations from __future__ import annotations
import sqlite3 import sqlite3
from bib.item import Download, Manual, Regulation, Rule, Source from bib.item import Download, Manual, Regulation, Rule, Source
from bib.sync import ( from bib.sync import _item_to_zotero_fields, push_to_zotero
_ensure_tag,
_ensure_value,
_item_to_zotero_fields,
_now_iso,
_set_field,
_sync_tags,
_zotero_key,
ensure_collection,
push_to_zotero,
)
# ── Zotero schema for tests ───────────────────────────────────────── # ── Zotero schema for tests ─────────────────────────────────────────
@@ -73,6 +68,23 @@ CREATE TABLE IF NOT EXISTS collectionItems (
orderIndex INTEGER NOT NULL DEFAULT 0, orderIndex INTEGER NOT NULL DEFAULT 0,
PRIMARY KEY (collectionID, itemID) PRIMARY KEY (collectionID, itemID)
); );
CREATE TABLE IF NOT EXISTS creators (
creatorID INTEGER PRIMARY KEY AUTOINCREMENT,
firstName TEXT,
lastName TEXT,
fieldMode INT,
UNIQUE (lastName, firstName, fieldMode)
);
CREATE TABLE IF NOT EXISTS itemCreators (
itemID INT NOT NULL,
creatorID INT NOT NULL,
creatorTypeID INT NOT NULL DEFAULT 1,
orderIndex INT NOT NULL DEFAULT 0,
PRIMARY KEY (itemID, creatorID, creatorTypeID, orderIndex),
UNIQUE (itemID, orderIndex)
);
""" """
@@ -83,126 +95,6 @@ def _make_zotero_db(path: str = ":memory:") -> sqlite3.Connection:
return con return con
# ── _zotero_key ─────────────────────────────────────────────────────
class TestZoteroKey:
def test_length(self) -> None:
key = _zotero_key()
assert len(key) == 8
def test_alphanumeric(self) -> None:
key = _zotero_key()
assert key.isalnum()
def test_uppercase(self) -> None:
key = _zotero_key()
assert key == key.upper()
# ── _now_iso ────────────────────────────────────────────────────────
class TestNowIso:
def test_format(self) -> None:
result = _now_iso()
assert len(result) == 19 # "YYYY-MM-DD HH:MM:SS"
assert " " in result
assert not result.endswith("Z")
# ── _ensure_value ───────────────────────────────────────────────────
class TestEnsureValue:
def test_creates_new(self) -> None:
con = _make_zotero_db()
vid = _ensure_value(con, "test value")
assert isinstance(vid, int)
assert vid > 0
def test_finds_existing(self) -> None:
con = _make_zotero_db()
v1 = _ensure_value(con, "same")
v2 = _ensure_value(con, "same")
assert v1 == v2
def test_different_values(self) -> None:
con = _make_zotero_db()
v1 = _ensure_value(con, "a")
v2 = _ensure_value(con, "b")
assert v1 != v2
# ── _ensure_tag ─────────────────────────────────────────────────────
class TestEnsureTag:
def test_creates_new(self) -> None:
con = _make_zotero_db()
tid = _ensure_tag(con, "module:pfs")
assert isinstance(tid, int)
assert tid > 0
def test_finds_existing(self) -> None:
con = _make_zotero_db()
t1 = _ensure_tag(con, "module:pfs")
t2 = _ensure_tag(con, "module:pfs")
assert t1 == t2
# ── _set_field ──────────────────────────────────────────────────────
class TestSetField:
def test_sets_known_field(self) -> None:
con = _make_zotero_db()
con.execute("INSERT INTO items (itemTypeID, key) VALUES (36, 'TESTKEY1')")
item_id = con.execute(
"SELECT itemID FROM items WHERE key = 'TESTKEY1'"
).fetchone()[0]
_set_field(con, item_id, "title", "Test Title")
row = con.execute(
"SELECT idv.value FROM itemData id "
"JOIN itemDataValues idv ON id.valueID = idv.valueID "
"WHERE id.itemID = ? AND id.fieldID = 1",
(item_id,),
).fetchone()
assert row[0] == "Test Title"
def test_skips_empty_value(self) -> None:
con = _make_zotero_db()
con.execute("INSERT INTO items (itemTypeID, key) VALUES (36, 'TESTKEY1')")
item_id = con.execute(
"SELECT itemID FROM items WHERE key = 'TESTKEY1'"
).fetchone()[0]
_set_field(con, item_id, "title", "")
row = con.execute(
"SELECT count(*) FROM itemData WHERE itemID = ?",
(item_id,),
).fetchone()
assert row[0] == 0
def test_skips_unknown_field(self) -> None:
con = _make_zotero_db()
con.execute("INSERT INTO items (itemTypeID, key) VALUES (36, 'TESTKEY1')")
item_id = con.execute(
"SELECT itemID FROM items WHERE key = 'TESTKEY1'"
).fetchone()[0]
_set_field(con, item_id, "nonexistent_field", "value")
row = con.execute(
"SELECT count(*) FROM itemData WHERE itemID = ?",
(item_id,),
).fetchone()
assert row[0] == 0
# ── _item_to_zotero_fields ───────────────────────────────────────── # ── _item_to_zotero_fields ─────────────────────────────────────────
@@ -268,7 +160,7 @@ class TestItemToZoteroFields:
assert fields["reportType"] == "Internet-Only Manual" assert fields["reportType"] == "Internet-Only Manual"
assert fields["reportNumber"] == "100-04" assert fields["reportNumber"] == "100-04"
assert fields["seriesTitle"] == "Claims Processing Manual" assert fields["seriesTitle"] == "Claims Processing Manual"
assert fields["seriesNumber"] == "Chapter 12" assert "Chapter 12" in fields["extra"]
assert "Transmittal: R100" in fields["extra"] assert "Transmittal: R100" in fields["extra"]
assert fields["place"] == "Baltimore, MD" assert fields["place"] == "Baltimore, MD"
@@ -280,7 +172,7 @@ class TestItemToZoteroFields:
def test_manual_no_chapter(self) -> None: def test_manual_no_chapter(self) -> None:
item = Manual(title="Test") item = Manual(title="Test")
fields = _item_to_zotero_fields(item) fields = _item_to_zotero_fields(item)
assert fields["seriesNumber"] == "" assert "seriesNumber" not in fields
def test_download(self) -> None: def test_download(self) -> None:
item = Download( item = Download(
@@ -309,88 +201,10 @@ class TestItemToZoteroFields:
) )
fields = _item_to_zotero_fields(item) fields = _item_to_zotero_fields(item)
assert fields["title"] == "Test Doc" assert fields["title"] == "Test Doc"
assert fields["type"] == "guidance" assert "Type: guidance" in fields["extra"]
assert fields["publisher"] == "CMS" assert fields["publisher"] == "CMS"
# ── _sync_tags ──────────────────────────────────────────────────────
class TestSyncTags:
def test_adds_tags(self) -> None:
con = _make_zotero_db()
con.execute("INSERT INTO items (itemTypeID, key) VALUES (36, 'TESTKEY1')")
item_id = con.execute(
"SELECT itemID FROM items WHERE key = 'TESTKEY1'"
).fetchone()[0]
_sync_tags(con, item_id, ["module:pfs", "year:2026"])
rows = con.execute(
"SELECT t.name FROM itemTags it "
"JOIN tags t ON it.tagID = t.tagID "
"WHERE it.itemID = ?",
(item_id,),
).fetchall()
names = {r[0] for r in rows}
assert "module:pfs" in names
assert "year:2026" in names
def test_skips_empty_tags(self) -> None:
con = _make_zotero_db()
con.execute("INSERT INTO items (itemTypeID, key) VALUES (36, 'TESTKEY1')")
item_id = con.execute(
"SELECT itemID FROM items WHERE key = 'TESTKEY1'"
).fetchone()[0]
_sync_tags(con, item_id, ["module:pfs", "", "year:2026"])
rows = con.execute(
"SELECT count(*) FROM itemTags WHERE itemID = ?",
(item_id,),
).fetchone()
assert rows[0] == 2
# ── ensure_collection ───────────────────────────────────────────────
class TestEnsureCollection:
def test_creates_new(self) -> None:
con = _make_zotero_db()
key = ensure_collection(con, "Test Collection")
assert isinstance(key, str)
assert len(key) == 8
row = con.execute(
"SELECT collectionName FROM collections WHERE key = ?",
(key,),
).fetchone()
assert row[0] == "Test Collection"
def test_finds_existing(self) -> None:
con = _make_zotero_db()
k1 = ensure_collection(con, "Test")
k2 = ensure_collection(con, "Test")
assert k1 == k2
def test_with_parent(self) -> None:
con = _make_zotero_db()
parent_key = ensure_collection(con, "Parent")
child_key = ensure_collection(con, "Child", parent_key=parent_key)
assert parent_key != child_key
# Creating child again should find existing
child_key2 = ensure_collection(con, "Child", parent_key=parent_key)
assert child_key == child_key2
def test_parent_key_not_found(self) -> None:
con = _make_zotero_db()
# Parent key doesn't exist — parent_id will be None
key = ensure_collection(con, "Orphan", parent_key="NOEXIST1")
assert isinstance(key, str)
# ── push_to_zotero ────────────────────────────────────────────────── # ── push_to_zotero ──────────────────────────────────────────────────
@@ -402,7 +216,7 @@ class TestPushToZotero:
items = [ items = [
Rule( Rule(
key="RULEKEY1", key="RULEKEY2",
title="PFS Final Rule", title="PFS Final Rule",
fr_volume="90", fr_volume="90",
fr_page="98452", fr_page="98452",
@@ -457,16 +271,15 @@ class TestPushToZotero:
def test_with_collection(self, tmp_path) -> None: def test_with_collection(self, tmp_path) -> None:
db_path = str(tmp_path / "zotero.sqlite") db_path = str(tmp_path / "zotero.sqlite")
con = _make_zotero_db(db_path) con = _make_zotero_db(db_path)
# Create a collection
con.execute( con.execute(
"INSERT INTO collections (collectionName, libraryID, key, version, synced) " "INSERT INTO collections (collectionName, libraryID, key, version, synced) "
"VALUES ('Test', 1, 'COLLKEY1', 0, 0)" "VALUES ('Test', 1, 'CKEY2345', 0, 0)"
) )
con.commit() con.commit()
con.close() con.close()
items = [Rule(title="R1", url="https://ex.com/r1")] items = [Rule(title="R1", url="https://ex.com/r1")]
stats = push_to_zotero(items, zotero_db=db_path, collection_key="COLLKEY1") stats = push_to_zotero(items, zotero_db=db_path, collection_key="CKEY2345")
assert stats["collections"] == 1 assert stats["collections"] == 1
def test_item_collections(self, tmp_path) -> None: def test_item_collections(self, tmp_path) -> None:
@@ -474,7 +287,7 @@ class TestPushToZotero:
con = _make_zotero_db(db_path) con = _make_zotero_db(db_path)
con.execute( con.execute(
"INSERT INTO collections (collectionName, libraryID, key, version, synced) " "INSERT INTO collections (collectionName, libraryID, key, version, synced) "
"VALUES ('ACO', 1, 'ACOKEY12', 0, 0)" "VALUES ('ACO', 1, 'ACKEY234', 0, 0)"
) )
con.commit() con.commit()
con.close() con.close()
@@ -483,7 +296,7 @@ class TestPushToZotero:
Rule( Rule(
title="R1", title="R1",
url="https://ex.com/r1", url="https://ex.com/r1",
collections=["ACOKEY12"], collections=["ACKEY234"],
) )
] ]
stats = push_to_zotero(items, zotero_db=db_path) stats = push_to_zotero(items, zotero_db=db_path)
@@ -492,12 +305,11 @@ class TestPushToZotero:
def test_key_collision(self, tmp_path) -> None: def test_key_collision(self, tmp_path) -> None:
db_path = str(tmp_path / "zotero.sqlite") db_path = str(tmp_path / "zotero.sqlite")
con = _make_zotero_db(db_path) con = _make_zotero_db(db_path)
# Pre-insert an item with the same key con.execute("INSERT INTO items (itemTypeID, key) VALUES (20, 'RKEY2345')")
con.execute("INSERT INTO items (itemTypeID, key) VALUES (36, 'RULEKEY1')")
con.commit() con.commit()
con.close() con.close()
items = [Rule(key="RULEKEY1", title="Collision")] items = [Rule(key="RKEY2345", title="Collision")]
stats = push_to_zotero(items, zotero_db=db_path) stats = push_to_zotero(items, zotero_db=db_path)
assert stats["created"] == 1 assert stats["created"] == 1
@@ -525,8 +337,7 @@ class TestPushToZotero:
con.close() con.close()
items = [Rule(title="R1")] items = [Rule(title="R1")]
stats = push_to_zotero(items, zotero_db=db_path, collection_key="NOEXIST1") stats = push_to_zotero(items, zotero_db=db_path, collection_key="ZZZZ2345")
# Collection not found, so no collection assignment
assert stats["collections"] == 0 assert stats["collections"] == 0
assert stats["created"] == 1 assert stats["created"] == 1
@@ -535,7 +346,7 @@ class TestPushToZotero:
con = _make_zotero_db(db_path) con = _make_zotero_db(db_path)
con.close() con.close()
items = [Rule(title="R1", collections=["NOEXIST1"])] items = [Rule(title="R1", collections=["ZZZZ2345"])]
stats = push_to_zotero(items, zotero_db=db_path) stats = push_to_zotero(items, zotero_db=db_path)
assert stats["created"] == 1 assert stats["created"] == 1
@@ -545,7 +356,7 @@ class TestPushToZotero:
con = _make_zotero_db(db_path) con = _make_zotero_db(db_path)
con.execute( con.execute(
"INSERT INTO collections (collectionName, libraryID, key, version, synced) " "INSERT INTO collections (collectionName, libraryID, key, version, synced) "
"VALUES ('Main', 1, 'MAINKEY1', 0, 0)" "VALUES ('Main', 1, 'MKEY2345', 0, 0)"
) )
con.commit() con.commit()
con.close() con.close()
@@ -554,8 +365,8 @@ class TestPushToZotero:
Rule( Rule(
title="R1", title="R1",
url="https://ex.com/r1", url="https://ex.com/r1",
collections=["MAINKEY1"], collections=["MKEY2345"],
) )
] ]
stats = push_to_zotero(items, zotero_db=db_path, collection_key="MAINKEY1") stats = push_to_zotero(items, zotero_db=db_path, collection_key="MKEY2345")
assert stats["collections"] == 1 assert stats["collections"] == 1

59
tests/perf/test_hooks.py
View File

@@ -149,6 +149,65 @@ class TestFileTestIssue:
assert "test" in labels assert "test" in labels
class TestGetToken:
"""_get_token resolves from env or config fallback."""
def test_returns_env_token(self, monkeypatch):
monkeypatch.setenv("GITEA_TOKEN", "test-token-123")
from perf.hooks import _get_token
assert _get_token() == "test-token-123"
def test_falls_back_to_conf_secret(self, monkeypatch):
monkeypatch.delenv("GITEA_TOKEN", raising=False)
from perf.hooks import _get_token
# conf.secret will raise ImportError in test env — returns ""
assert _get_token() == ""
def test_conf_import_error_returns_empty(self, monkeypatch):
monkeypatch.delenv("GITEA_TOKEN", raising=False)
from perf.hooks import _get_token
result = _get_token()
assert result == ""
class TestFileIssue:
"""_file_issue handles missing token and API errors gracefully."""
def test_returns_none_without_token(self, monkeypatch):
monkeypatch.delenv("GITEA_TOKEN", raising=False)
from perf.hooks import _file_issue
assert _file_issue("title", "body") is None
@patch("perf.hooks._get_token", return_value="tok")
def test_returns_none_on_api_error(self, mock_token):
from perf.hooks import _file_issue
# GiteaClient import will fail in test env → exception path
result = _file_issue("title", "body", labels=["bug"])
assert result is None
class TestGitShaShort:
"""_git_sha_short handles subprocess failures."""
def test_returns_sha_in_git_repo(self):
from perf.hooks import _git_sha_short
result = _git_sha_short()
# We're in a git repo, so should get a short sha
assert len(result) >= 7 or result == "unknown"
@patch("subprocess.run", side_effect=OSError("no git"))
def test_returns_unknown_on_exception(self, mock_run):
from perf.hooks import _git_sha_short
assert _git_sha_short() == "unknown"
class TestRunnerIntegration: class TestRunnerIntegration:
"""Pipeline runner calls on_step_failure when a step raises.""" """Pipeline runner calls on_step_failure when a step raises."""

168
tests/test_compose_mounts.py Normal file
View File

@@ -0,0 +1,168 @@
"""Tests for compose.yml — verify all bind mount sources exist on disk.
refs #250
"""
from __future__ import annotations
from pathlib import Path
import yaml
ROOT = Path(__file__).resolve().parents[1]
COMPOSE = ROOT / "compose.yml"
def _load_compose() -> dict:
return yaml.safe_load(COMPOSE.read_text())
def _extract_bind_mounts(compose: dict) -> list[tuple[str, str, str]]:
"""Return (service, host_path, full_mount) for every bind mount."""
mounts: list[tuple[str, str, str]] = []
for svc, cfg in compose.get("services", {}).items():
for vol in cfg.get("volumes", []):
if isinstance(vol, str):
host = vol.split(":")[0]
elif isinstance(vol, dict):
host = vol.get("source", "")
else:
continue
# Only check relative paths (./...) — skip named volumes,
# env-var-only paths, and absolute system paths
if host.startswith("./"):
mounts.append((svc, host, vol if isinstance(vol, str) else str(vol)))
return mounts
class TestBindMountSourcesExist:
"""Every relative bind mount in compose.yml must point to an existing path."""
def test_all_relative_bind_mounts_exist(self):
compose = _load_compose()
mounts = _extract_bind_mounts(compose)
assert mounts, "Expected to find bind mounts in compose.yml"
missing = []
for svc, host_path, full in mounts:
resolved = ROOT / host_path
if not resolved.exists():
missing.append(f" {svc}: {host_path}")
assert missing == [], "Bind mount sources not found on disk:\n" + "\n".join(
missing
)
def test_compose_parses_cleanly(self):
"""compose.yml must be valid YAML with services defined."""
compose = _load_compose()
assert "services" in compose
assert len(compose["services"]) >= 10
class TestNoHardcodedPaths:
"""Bind mounts should not contain user-specific absolute paths as sources."""
def test_no_absolute_home_paths_in_sources(self):
"""Mount sources should use ./ or env vars, not /home/user/..."""
compose = _load_compose()
for svc, cfg in compose.get("services", {}).items():
for vol in cfg.get("volumes", []):
if isinstance(vol, str):
host = vol.split(":")[0]
else:
continue
# Absolute /home paths as source are fragile
# Allow /home/kert/.local/share/docker (promtail needs it)
if host.startswith("/home/") and "docker/containers" not in host:
raise AssertionError(
f"{svc}: hardcoded absolute path as mount source: {host}"
)
class TestReadOnlyMounts:
"""Config files and assets should be mounted read-only."""
def test_infra_config_mounts_are_ro(self):
"""infra/ mounts should be :ro."""
compose = _load_compose()
not_ro = []
for svc, cfg in compose.get("services", {}).items():
for vol in cfg.get("volumes", []):
if not isinstance(vol, str):
continue
host = vol.split(":")[0]
# infra/ config files should be read-only
if host.startswith("./infra/") and host.endswith(
(".yml", ".yaml", ".conf", ".toml", ".xml", ".css", ".js")
):
if not vol.endswith(":ro"):
not_ro.append(f" {svc}: {vol}")
assert not_ro == [], "infra/ config mounts should be :ro:\n" + "\n".join(not_ro)
def test_asset_mounts_are_ro(self):
"""assets/ mounts should be :ro."""
compose = _load_compose()
not_ro = []
for svc, cfg in compose.get("services", {}).items():
for vol in cfg.get("volumes", []):
if not isinstance(vol, str):
continue
host = vol.split(":")[0]
if host.startswith("./assets/") and not vol.endswith(":ro"):
not_ro.append(f" {svc}: {vol}")
assert not_ro == [], "assets/ mounts should be :ro:\n" + "\n".join(not_ro)
class TestSecurityOpts:
"""Services should have security_opt configured."""
def test_services_have_no_new_privileges(self):
"""Most services should have no-new-privileges."""
# Exceptions: privileged, ephemeral, or upstream images without secopt
exempt = {
"woodpecker-agent", # needs privileged for Docker builds
"wire", # ephemeral bootstrap, profile=tools
"zotero", # GPU + display server needs
"coredns", # upstream, no secopt in image
"gitea", # rootless image handles its own security
"woodpecker-server", # upstream CI server
"notebooks", # GPU + dev environment
"webdav", # rclone upstream
"docs", # static site
"promtail", # needs host log access
"cloudflared", # tunnel agent
}
compose = _load_compose()
missing = []
for svc, cfg in compose.get("services", {}).items():
if svc in exempt:
continue
sec = cfg.get("security_opt", [])
if "no-new-privileges:true" not in sec:
missing.append(svc)
assert missing == [], (
f"Services missing no-new-privileges: {', '.join(missing)}"
)
class TestNetworkAssignment:
"""Every service should be assigned to at least one network."""
def test_all_services_have_networks(self):
# Exceptions: ephemeral/profile-only services
exempt = {"wire"}
compose = _load_compose()
missing = []
for svc, cfg in compose.get("services", {}).items():
if svc in exempt:
continue
if "networks" not in cfg:
missing.append(svc)
assert missing == [], (
f"Services without network assignment: {', '.join(missing)}"
)

0
tests/zot/__init__.py Normal file
View File

685
tests/zot/test_db.py Normal file
View File

@@ -0,0 +1,685 @@
"""Tests for zot.db — Zotero SQLite ORM."""
from __future__ import annotations
import sqlite3
import pytest
from zot.db import (
CREATOR_TYPES,
FIELD_MAP,
TYPE_MAP,
Db,
generate_key,
is_valid_key,
normalize_date,
now_iso,
)
# ── Zotero schema for tests (minimal but sufficient) ────────────
ZOTERO_SCHEMA = """
CREATE TABLE IF NOT EXISTS libraries (
libraryID INTEGER PRIMARY KEY,
type TEXT NOT NULL,
editable INT NOT NULL,
filesEditable INT NOT NULL,
version INT NOT NULL DEFAULT 0,
storageVersion INT NOT NULL DEFAULT 0,
lastSync INT NOT NULL DEFAULT 0,
archived INT NOT NULL DEFAULT 0
);
INSERT OR IGNORE INTO libraries VALUES (1, 'user', 1, 1, 0, 0, 0, 0);
CREATE TABLE IF NOT EXISTS items (
itemID INTEGER PRIMARY KEY,
itemTypeID INT NOT NULL,
dateAdded TEXT,
dateModified TEXT,
clientDateModified TEXT,
libraryID INT NOT NULL DEFAULT 1,
key TEXT NOT NULL UNIQUE,
version INT DEFAULT 0,
synced INT DEFAULT 0
);
CREATE TABLE IF NOT EXISTS itemDataValues (
valueID INTEGER PRIMARY KEY AUTOINCREMENT,
value TEXT NOT NULL UNIQUE
);
CREATE TABLE IF NOT EXISTS itemData (
itemID INT NOT NULL,
fieldID INT NOT NULL,
valueID INT NOT NULL,
PRIMARY KEY (itemID, fieldID)
);
CREATE TABLE IF NOT EXISTS fields (
fieldID INTEGER PRIMARY KEY,
fieldName TEXT,
fieldFormatID INT
);
CREATE TABLE IF NOT EXISTS tags (
tagID INTEGER PRIMARY KEY AUTOINCREMENT,
name TEXT NOT NULL UNIQUE
);
CREATE TABLE IF NOT EXISTS itemTags (
itemID INT NOT NULL,
tagID INT NOT NULL,
type INT DEFAULT 0,
PRIMARY KEY (itemID, tagID)
);
CREATE TABLE IF NOT EXISTS creators (
creatorID INTEGER PRIMARY KEY AUTOINCREMENT,
firstName TEXT,
lastName TEXT,
fieldMode INT,
UNIQUE (lastName, firstName, fieldMode)
);
CREATE TABLE IF NOT EXISTS creatorTypes (
creatorTypeID INTEGER PRIMARY KEY,
creatorType TEXT
);
CREATE TABLE IF NOT EXISTS itemCreators (
itemID INT NOT NULL,
creatorID INT NOT NULL,
creatorTypeID INT NOT NULL DEFAULT 1,
orderIndex INT NOT NULL DEFAULT 0,
PRIMARY KEY (itemID, creatorID, creatorTypeID, orderIndex),
UNIQUE (itemID, orderIndex)
);
CREATE TABLE IF NOT EXISTS collections (
collectionID INTEGER PRIMARY KEY AUTOINCREMENT,
collectionName TEXT NOT NULL,
parentCollectionID INT DEFAULT NULL,
clientDateModified TEXT,
libraryID INT DEFAULT 1,
key TEXT NOT NULL UNIQUE,
version INT DEFAULT 0,
synced INT DEFAULT 0
);
CREATE TABLE IF NOT EXISTS collectionItems (
collectionID INT NOT NULL,
itemID INT NOT NULL,
orderIndex INT NOT NULL DEFAULT 0,
PRIMARY KEY (collectionID, itemID)
);
CREATE TABLE IF NOT EXISTS itemAttachments (
itemID INTEGER PRIMARY KEY,
parentItemID INT,
linkMode INT,
contentType TEXT,
charsetID INT,
path TEXT,
syncState INT DEFAULT 0,
storageModTime INT,
storageHash TEXT
);
CREATE TABLE IF NOT EXISTS itemNotes (
itemID INTEGER PRIMARY KEY,
parentItemID INT,
note TEXT,
title TEXT
);
"""
def _seed_fields(con: sqlite3.Connection) -> None:
"""Insert field definitions so get_fields() can join by name."""
for name, fid in FIELD_MAP.items():
con.execute(
"INSERT OR IGNORE INTO fields (fieldID, fieldName) VALUES (?, ?)",
(fid, name),
)
for name, cid in CREATOR_TYPES.items():
con.execute(
"INSERT OR IGNORE INTO creatorTypes (creatorTypeID, creatorType) "
"VALUES (?, ?)",
(cid, name),
)
con.commit()
@pytest.fixture()
def db(tmp_path) -> Db:
path = str(tmp_path / "zotero.sqlite")
con = sqlite3.connect(path)
con.executescript(ZOTERO_SCHEMA)
con.close()
d = Db(path)
_seed_fields(d.con)
yield d
d.close()
# ── Key utilities ────────────────────────────────────────────────
class TestKeyUtilities:
def test_generate_key_length(self):
assert len(generate_key()) == 8
def test_generate_key_charset(self):
key = generate_key()
assert all(c in "23456789ABCDEFGHIJKLMNPQRSTUVWXYZ" for c in key)
def test_is_valid_key_accepts_good(self):
assert is_valid_key("ABCD2345")
def test_is_valid_key_rejects_short(self):
assert not is_valid_key("SHORT")
def test_is_valid_key_rejects_lowercase(self):
assert not is_valid_key("abcd2345")
def test_is_valid_key_rejects_zero(self):
assert not is_valid_key("0BCD2345")
def test_now_iso_format(self):
result = now_iso()
assert " " in result
assert "T" not in result
assert not result.endswith("Z")
assert len(result) == 19
def test_normalize_date(self):
assert normalize_date("2026-01-01T12:00:00Z") == "2026-01-01 12:00:00"
assert normalize_date("2026-01-01") == "2026-01-01"
# ── Items ────────────────────────────────────────────────────────
class TestItems:
def test_create_item(self, db: Db):
item_id = db.create_item(TYPE_MAP["statute"])
assert isinstance(item_id, int)
assert item_id > 0
def test_create_item_with_key(self, db: Db):
item_id = db.create_item(TYPE_MAP["statute"], key="MNPQ2345")
assert item_id > 0
assert db.find_item_by_key("MNPQ2345") == item_id
def test_create_item_invalid_key_generates_new(self, db: Db):
item_id = db.create_item(TYPE_MAP["statute"], key="bad")
assert item_id > 0
def test_key_collision_retry(self, db: Db):
db.create_item(TYPE_MAP["statute"], key="CXYZ2345")
# Second item with same key should get a different key
item_id2 = db.create_item(TYPE_MAP["statute"], key="CXYZ2345")
assert item_id2 > 0
def test_key_exists(self, db: Db):
db.create_item(TYPE_MAP["statute"], key="ABCD2345")
assert db.key_exists("ABCD2345")
assert not db.key_exists("ZZZZ8888")
def test_find_item_by_key(self, db: Db):
item_id = db.create_item(TYPE_MAP["report"], key="EFGH5678")
assert db.find_item_by_key("EFGH5678") == item_id
assert db.find_item_by_key("ZZZZ9999") is None
def test_find_item_by_url(self, db: Db):
item_id = db.create_item(TYPE_MAP["webpage"])
db.set_field(item_id, "url", "https://example.com/test")
assert db.find_item_by_url("https://example.com/test") == item_id
assert db.find_item_by_url("https://example.com/nope") is None
def test_get_item_type(self, db: Db):
item_id = db.create_item(TYPE_MAP["document"])
assert db.get_item_type(item_id) == TYPE_MAP["document"]
# ── EAV Fields ───────────────────────────────────────────────────
class TestFields:
def test_ensure_value_creates(self, db: Db):
vid = db.ensure_value("test value")
assert isinstance(vid, int)
def test_ensure_value_deduplicates(self, db: Db):
v1 = db.ensure_value("same")
v2 = db.ensure_value("same")
assert v1 == v2
def test_set_and_get_field(self, db: Db):
item_id = db.create_item(TYPE_MAP["statute"])
db.set_field(item_id, "nameOfAct", "PFS Final Rule")
assert db.get_field(item_id, "nameOfAct") == "PFS Final Rule"
def test_set_field_skips_empty(self, db: Db):
item_id = db.create_item(TYPE_MAP["statute"])
db.set_field(item_id, "title", "")
assert db.get_field(item_id, "title") is None
def test_set_field_skips_unknown(self, db: Db):
item_id = db.create_item(TYPE_MAP["statute"])
db.set_field(item_id, "nonexistent", "value")
# Should not raise, just no-op
def test_set_field_normalizes_date(self, db: Db):
item_id = db.create_item(TYPE_MAP["statute"])
db.set_field(item_id, "accessDate", "2026-01-01T12:00:00Z")
assert db.get_field(item_id, "accessDate") == "2026-01-01 12:00:00"
def test_set_fields_batch(self, db: Db):
item_id = db.create_item(TYPE_MAP["report"])
db.set_fields(
item_id,
{
"title": "Test Report",
"institution": "CMS",
"date": "2026-01-01",
},
)
assert db.get_field(item_id, "title") == "Test Report"
assert db.get_field(item_id, "institution") == "CMS"
def test_get_fields_all(self, db: Db):
item_id = db.create_item(TYPE_MAP["statute"])
db.set_fields(item_id, {"nameOfAct": "Rule X", "code": "FR"})
fields = db.get_fields(item_id)
assert fields["nameOfAct"] == "Rule X"
assert fields["code"] == "FR"
def test_set_field_overwrites(self, db: Db):
item_id = db.create_item(TYPE_MAP["statute"])
db.set_field(item_id, "nameOfAct", "v1")
db.set_field(item_id, "nameOfAct", "v2")
assert db.get_field(item_id, "nameOfAct") == "v2"
# ── Creators ─────────────────────────────────────────────────────
class TestCreators:
def test_ensure_creator(self, db: Db):
cid = db.ensure_creator("John", "Doe")
assert isinstance(cid, int)
# Idempotent
assert db.ensure_creator("John", "Doe") == cid
def test_add_creators(self, db: Db):
item_id = db.create_item(TYPE_MAP["journalArticle"])
count = db.add_creators(item_id, [("Jane", "Smith"), ("Bob", "Lee")])
assert count == 2
def test_get_creators(self, db: Db):
item_id = db.create_item(TYPE_MAP["journalArticle"])
db.add_creators(item_id, [("Jane", "Smith"), ("Bob", "Lee")])
creators = db.get_creators(item_id)
assert len(creators) == 2
assert creators[0]["firstName"] == "Jane"
assert creators[0]["lastName"] == "Smith"
assert creators[0]["creatorType"] == "author"
assert creators[1]["firstName"] == "Bob"
def test_add_creators_editor(self, db: Db):
item_id = db.create_item(TYPE_MAP["book"])
db.add_creators(item_id, [("Ed", "Itor")], creator_type="editor")
creators = db.get_creators(item_id)
assert creators[0]["creatorType"] == "editor"
# ── Tags ─────────────────────────────────────────────────────────
class TestTags:
def test_ensure_tag(self, db: Db):
tid = db.ensure_tag("module:pfs")
assert isinstance(tid, int)
assert db.ensure_tag("module:pfs") == tid
def test_tag_item(self, db: Db):
item_id = db.create_item(TYPE_MAP["statute"])
db.tag_item(item_id, "year:2026")
assert "year:2026" in db.get_tags(item_id)
def test_tag_item_skips_blank(self, db: Db):
item_id = db.create_item(TYPE_MAP["statute"])
db.tag_item(item_id, "")
assert db.get_tags(item_id) == []
def test_sync_tags(self, db: Db):
item_id = db.create_item(TYPE_MAP["statute"])
db.sync_tags(item_id, ["a", "b", "", "c"])
tags = db.get_tags(item_id)
assert set(tags) == {"a", "b", "c"}
def test_sync_tags_idempotent(self, db: Db):
item_id = db.create_item(TYPE_MAP["statute"])
db.sync_tags(item_id, ["x", "y"])
db.sync_tags(item_id, ["x", "y"]) # no duplicates
assert len(db.get_tags(item_id)) == 2
# ── Collections ──────────────────────────────────────────────────
class TestCollections:
def test_ensure_collection_creates(self, db: Db):
key = db.ensure_collection("Test Collection")
assert is_valid_key(key)
def test_ensure_collection_finds_existing(self, db: Db):
k1 = db.ensure_collection("Dedup")
k2 = db.ensure_collection("Dedup")
assert k1 == k2
def test_ensure_collection_with_parent(self, db: Db):
parent_key = db.ensure_collection("Parent")
child_key = db.ensure_collection("Child", parent_key=parent_key)
assert parent_key != child_key
# Find again
assert db.ensure_collection("Child", parent_key=parent_key) == child_key
def test_add_to_collection(self, db: Db):
item_id = db.create_item(TYPE_MAP["statute"])
key = db.ensure_collection("My Collection")
assert db.add_to_collection(item_id, collection_key=key)
assert key in db.get_item_collections(item_id)
def test_add_to_collection_missing(self, db: Db):
item_id = db.create_item(TYPE_MAP["statute"])
assert not db.add_to_collection(item_id, collection_key="ZZZZ7777")
def test_find_collection(self, db: Db):
key = db.ensure_collection("Findable")
assert db.find_collection(key) is not None
assert db.find_collection("ZZZZ6666") is None
# ── Attachments & Notes ──────────────────────────────────────────
class TestAttachmentsAndNotes:
def test_add_attachment(self, db: Db):
parent = db.create_item(TYPE_MAP["journalArticle"])
att_id = db.add_attachment(
parent, content_type="application/pdf", path="storage/test.pdf"
)
assert att_id > 0
assert db.get_item_type(att_id) == TYPE_MAP["attachment"]
def test_add_note(self, db: Db):
parent = db.create_item(TYPE_MAP["statute"])
note_id = db.add_note(parent, "This is a note", title="My Note")
assert note_id > 0
assert db.get_item_type(note_id) == TYPE_MAP["note"]
# ── Counting ─────────────────────────────────────────────────────
class TestCounting:
def test_count_items(self, db: Db):
assert db.count_items() == 0
db.create_item(TYPE_MAP["statute"])
db.create_item(TYPE_MAP["report"])
assert db.count_items() == 2
assert db.count_items(TYPE_MAP["statute"]) == 1
def test_count_tags(self, db: Db):
assert db.count_tags() == 0
db.ensure_tag("a")
db.ensure_tag("b")
assert db.count_tags() == 2
def test_count_creators(self, db: Db):
assert db.count_creators() == 0
db.ensure_creator("J", "D")
assert db.count_creators() == 1
def test_count_collections(self, db: Db):
assert db.count_collections() == 0
db.ensure_collection("C1")
assert db.count_collections() == 1
# ── Context manager ──────────────────────────────────────────────
class TestContextManager:
def test_with_statement(self, tmp_path):
path = str(tmp_path / "zotero.sqlite")
con = sqlite3.connect(path)
con.executescript(ZOTERO_SCHEMA)
con.close()
with Db(path) as db:
_seed_fields(db.con)
item_id = db.create_item(TYPE_MAP["statute"], key="IJKL3456")
db.set_field(item_id, "nameOfAct", "Context Test")
db.commit()
# Verify data persists after close
with Db(path) as db:
_seed_fields(db.con)
assert db.find_item_by_key("IJKL3456") is not None
assert (
db.get_field(db.find_item_by_key("IJKL3456"), "nameOfAct")
== "Context Test"
)
# ── Constant completeness ───────────────────────────────────────
class TestConstants:
def test_type_map_has_all_standard_types(self):
assert len(TYPE_MAP) == 36
assert "journalArticle" in TYPE_MAP
assert "statute" in TYPE_MAP
assert "report" in TYPE_MAP
assert "webpage" in TYPE_MAP
assert "document" in TYPE_MAP
def test_field_map_has_core_fields(self):
assert len(FIELD_MAP) >= 100
assert FIELD_MAP["title"] == 110
assert FIELD_MAP["url"] == 1
assert FIELD_MAP["abstractNote"] == 90
assert FIELD_MAP["extra"] == 22
assert FIELD_MAP["DOI"] == 26
def test_creator_types_complete(self):
assert len(CREATOR_TYPES) == 29
assert CREATOR_TYPES["author"] == 1
assert CREATOR_TYPES["editor"] == 3
# ── Structured reads ─────────────────────────────────────────────
class TestGetItem:
def test_get_item_full(self, db: Db):
item_id = db.create_item(TYPE_MAP["statute"])
db.set_fields(item_id, {"nameOfAct": "Test Act", "code": "FR"})
db.sync_tags(item_id, ["test:tag"])
db.add_creators(item_id, [("Jane", "Doe")])
item = db.get_item(item_id)
assert item is not None
assert item["itemType"] == "statute"
assert item["fields"]["nameOfAct"] == "Test Act"
assert "test:tag" in item["tags"]
assert item["creators"][0]["lastName"] == "Doe"
def test_get_item_not_found(self, db: Db):
assert db.get_item(99999) is None
def test_get_item_by_key(self, db: Db):
db.create_item(TYPE_MAP["report"], key="QRST5678")
item = db.get_item_by_key("QRST5678")
assert item is not None
assert item["itemType"] == "report"
def test_get_item_by_key_missing(self, db: Db):
assert db.get_item_by_key("ZZZZ2222") is None
# ── Search ───────────────────────────────────────────────────────
class TestSearch:
def test_search_by_tag(self, db: Db):
id1 = db.create_item(TYPE_MAP["statute"])
id2 = db.create_item(TYPE_MAP["report"])
db.tag_item(id1, "module:pfs")
db.tag_item(id2, "module:pfs")
assert set(db.search_by_tag("module:pfs")) == {id1, id2}
def test_search_by_tag_limit(self, db: Db):
for _ in range(5):
iid = db.create_item(TYPE_MAP["statute"])
db.tag_item(iid, "bulk")
assert len(db.search_by_tag("bulk", limit=3)) == 3
def test_search_by_type(self, db: Db):
db.create_item(TYPE_MAP["statute"])
db.create_item(TYPE_MAP["report"])
db.create_item(TYPE_MAP["statute"])
assert len(db.search_by_type("statute")) == 2
def test_search_by_field_exact(self, db: Db):
id1 = db.create_item(TYPE_MAP["webpage"])
db.set_field(id1, "url", "https://example.com")
assert db.search_by_field("url", "https://example.com") == [id1]
def test_search_by_field_like(self, db: Db):
id1 = db.create_item(TYPE_MAP["webpage"])
db.set_field(id1, "url", "https://example.com/page")
assert db.search_by_field("url", "example.com", exact=False) == [id1]
def test_search_by_collection(self, db: Db):
key = db.ensure_collection("TestCol")
id1 = db.create_item(TYPE_MAP["statute"])
db.add_to_collection(id1, collection_key=key)
assert db.search_by_collection(key) == [id1]
def test_combined_search(self, db: Db):
id1 = db.create_item(TYPE_MAP["statute"])
id2 = db.create_item(TYPE_MAP["statute"])
db.tag_item(id1, "findme")
db.tag_item(id2, "other")
result = db.search(tag="findme", type_name="statute")
assert result == [id1]
def test_search_unknown_type(self, db: Db):
assert db.search_by_type("nonexistent") == []
def test_search_unknown_field(self, db: Db):
assert db.search_by_field("nonexistent", "val") == []
# ── Bulk operations ──────────────────────────────────────────────
class TestBulkOps:
def test_bulk_create(self, db: Db):
ids = db.bulk_create(
[
{"type": "statute", "fields": {"nameOfAct": "Act A"}, "tags": ["a"]},
{"type": "report", "fields": {"title": "Report B"}, "tags": ["b"]},
{"type": "webpage", "fields": {"title": "Page C"}},
]
)
assert len(ids) == 3
assert db.count_items() == 3
assert db.get_field(ids[0], "nameOfAct") == "Act A"
def test_bulk_create_with_creators(self, db: Db):
ids = db.bulk_create(
[
{
"type": "journalArticle",
"fields": {"title": "Paper"},
"creators": [("Jane", "Smith"), ("Bob", "Lee")],
},
]
)
assert len(db.get_creators(ids[0])) == 2
def test_bulk_create_skips_unknown_type(self, db: Db):
ids = db.bulk_create([{"type": "nonexistent"}])
assert ids == []
def test_export_items(self, db: Db):
db.bulk_create(
[
{"type": "statute", "fields": {"nameOfAct": "Act"}, "tags": ["t1"]},
]
)
exported = db.export_items()
assert len(exported) == 1
assert exported[0]["itemType"] == "statute"
assert exported[0]["fields"]["nameOfAct"] == "Act"
assert "t1" in exported[0]["tags"]
def test_export_specific_ids(self, db: Db):
ids = db.bulk_create(
[
{"type": "statute", "fields": {"nameOfAct": "A"}},
{"type": "report", "fields": {"title": "B"}},
]
)
exported = db.export_items([ids[0]])
assert len(exported) == 1
# ── Delete ───────────────────────────────────────────────────────
class TestDelete:
def test_delete_item(self, db: Db):
item_id = db.create_item(TYPE_MAP["statute"])
db.set_field(item_id, "nameOfAct", "Doomed")
db.tag_item(item_id, "rip")
assert db.delete_item(item_id)
assert db.get_item(item_id) is None
assert db.count_items() == 0
def test_delete_nonexistent(self, db: Db):
assert not db.delete_item(99999)
def test_delete_with_attachment(self, db: Db):
parent = db.create_item(TYPE_MAP["statute"])
db.add_attachment(parent, content_type="application/pdf")
assert db.count_items() == 2
assert db.delete_item(parent)
# attachment item still exists (orphaned)
# ── Stats ────────────────────────────────────────────────────────
class TestStats:
def test_stats_empty(self, db: Db):
s = db.stats()
assert s["items"] == 0
assert s["tags"] == 0
def test_stats_after_create(self, db: Db):
db.bulk_create(
[
{"type": "statute", "fields": {"nameOfAct": "X"}, "tags": ["y"]},
]
)
s = db.stats()
assert s["items"] == 1
assert s["tags"] == 1
assert s["data_rows"] >= 1

196
tests/zot/test_duck.py Normal file
View File

@@ -0,0 +1,196 @@
"""Tests for zot.duck — DuckDB engine for Zotero analytics."""
from __future__ import annotations
from pathlib import Path
import pytest
from zot.db import TYPE_MAP, Db
from zot.schema import create_db
# We need a populated SQLite DB for DuckDB to attach
HOST_DB = Path("data/zotero/data/host-zotero.sqlite")
@pytest.fixture()
def sqlite_db(tmp_path) -> str:
"""Create a small populated Zotero SQLite for testing."""
path = str(tmp_path / "zotero.sqlite")
con = create_db(path)
con.close()
with Db(path) as db:
# Create a few items
id1 = db.create_item(TYPE_MAP["statute"])
db.set_fields(
id1, {"nameOfAct": "PFS 2026", "code": "FR", "url": "https://ex.com/pfs"}
)
db.sync_tags(id1, ["module:pfs", "year:2026"])
id2 = db.create_item(TYPE_MAP["journalArticle"])
db.set_fields(
id2,
{
"title": "Skin Substitutes Review",
"DOI": "10.1234/test",
"publicationTitle": "JAMA",
"date": "2025-06-01",
},
)
db.add_creators(id2, [("Jane", "Smith"), ("Bob", "Lee")])
db.sync_tags(id2, ["module:skin-subs", "year:2025"])
id3 = db.create_item(TYPE_MAP["webpage"])
db.set_fields(id3, {"title": "CMS Data", "url": "https://cms.gov/data"})
db.sync_tags(id3, ["module:pfs"])
key = db.ensure_collection("Test Collection")
db.add_to_collection(id1, collection_key=key)
db.add_to_collection(id2, collection_key=key)
db.commit()
return path
class TestAttach:
def test_attach_and_query(self, sqlite_db):
from zot.duck import DuckDb
with DuckDb.attach(sqlite_db) as zdb:
result = zdb.sql("SELECT count(*) FROM zot.items").fetchone()
assert result[0] == 3
def test_items_flat(self, sqlite_db):
from zot.duck import DuckDb
with DuckDb.attach(sqlite_db) as zdb:
rel = zdb.items()
rows = rel.fetchall()
assert len(rows) == 3
# Check columns exist
cols = [d[0] for d in rel.description]
assert "title" in cols
assert "doi" in cols
assert "item_type" in cols
def test_tags_flat(self, sqlite_db):
from zot.duck import DuckDb
with DuckDb.attach(sqlite_db) as zdb:
rows = zdb.tags().fetchall()
tags = {r[2] for r in rows} # tag column
assert "module:pfs" in tags
assert "module:skin-subs" in tags
def test_creators_flat(self, sqlite_db):
from zot.duck import DuckDb
with DuckDb.attach(sqlite_db) as zdb:
rows = zdb.creators().fetchall()
names = {r[2] for r in rows} # last_name column
assert "Smith" in names
assert "Lee" in names
def test_collections_flat(self, sqlite_db):
from zot.duck import DuckDb
with DuckDb.attach(sqlite_db) as zdb:
rows = zdb.collections().fetchall()
assert len(rows) == 2 # two items in collection
def test_items_by_type(self, sqlite_db):
from zot.duck import DuckDb
with DuckDb.attach(sqlite_db) as zdb:
rows = zdb.items_by_type().fetchall()
types = {r[0]: r[1] for r in rows}
assert types.get("statute") == 1
assert types.get("journalArticle") == 1
assert types.get("webpage") == 1
def test_items_by_tag(self, sqlite_db):
from zot.duck import DuckDb
with DuckDb.attach(sqlite_db) as zdb:
rows = zdb.items_by_tag().fetchall()
tags = {r[0]: r[1] for r in rows}
assert tags.get("module:pfs") == 2
def test_doi_coverage(self, sqlite_db):
from zot.duck import DuckDb
with DuckDb.attach(sqlite_db) as zdb:
rows = zdb.doi_coverage().fetchall()
assert len(rows) > 0
class TestMirror:
def test_mirror_creates_native_tables(self, sqlite_db):
from zot.duck import DuckDb
with DuckDb.mirror(sqlite_db) as zdb:
count = zdb.sql("SELECT count(*) FROM items").fetchone()[0]
assert count == 3
def test_mirror_materialize(self, sqlite_db):
from zot.duck import DuckDb
with DuckDb.mirror(sqlite_db) as zdb:
zdb.materialize()
rows = zdb.sql("SELECT count(*) FROM flat_items").fetchone()
assert rows[0] == 3
def test_mirror_to_parquet(self, sqlite_db, tmp_path):
from zot.duck import DuckDb
with DuckDb.mirror(sqlite_db) as zdb:
zdb.materialize()
parquet_path = str(tmp_path / "items.parquet")
zdb.to_parquet(parquet_path)
assert Path(parquet_path).exists()
assert Path(parquet_path).stat().st_size > 0
class TestToDataFrame:
def test_to_polars(self, sqlite_db):
from zot.duck import DuckDb
with DuckDb.attach(sqlite_db) as zdb:
df = zdb.to_df()
assert len(df) == 3
assert "title" in df.columns
assert "doi" in df.columns
@pytest.mark.skipif(not HOST_DB.exists(), reason="host-zotero.sqlite not available")
class TestRealDb:
"""Integration tests against the real Zotero database."""
def test_attach_real_db(self):
from zot.duck import DuckDb
with DuckDb.attach(str(HOST_DB)) as zdb:
count = zdb.sql("SELECT count(*) FROM zot.items").fetchone()[0]
assert count > 1000
def test_flat_items_real(self):
from zot.duck import DuckDb
with DuckDb.attach(str(HOST_DB)) as zdb:
df = zdb.to_df()
assert len(df) > 100
assert "doi" in df.columns
def test_tag_co_occurrence_real(self):
from zot.duck import DuckDb
with DuckDb.attach(str(HOST_DB)) as zdb:
rows = zdb.tag_co_occurrence(min_count=10).fetchall()
assert len(rows) > 0
def test_pdf_coverage_real(self):
from zot.duck import DuckDb
with DuckDb.attach(str(HOST_DB)) as zdb:
rows = zdb.pdf_coverage().fetchall()
assert len(rows) > 0

294
tests/zot/test_extract.py Normal file
View File

@@ -0,0 +1,294 @@
"""Tests for zot.extract — tag/annotation extraction for docstrings."""
from __future__ import annotations
from pathlib import Path
import pytest
from zot.db import TYPE_MAP, Db
from zot.extract import (
Extractor,
Quote,
_extract_quotes_from_note,
_page_from_link,
_parse_note_html,
)
from zot.schema import create_db
HOST_DB = Path("data/zotero/data/host-zotero.sqlite")
# ── HTML parsing ─────────────────────────────────────────────────
class TestParseNoteHtml:
def test_simple_paragraph(self):
segs = _parse_note_html("<p>Hello world</p>")
assert len(segs) == 1
assert segs[0]["text"] == "Hello world"
def test_multiple_paragraphs(self):
segs = _parse_note_html("<p>First</p><p>Second</p>")
assert len(segs) == 2
def test_link_extraction(self):
html = '<p><a href="zotero://open-pdf/0_ABC12345/14">text</a></p>'
segs = _parse_note_html(html)
assert segs[0]["link"] == "zotero://open-pdf/0_ABC12345/14"
def test_nested_tags(self):
html = "<p><strong>Bold</strong> and <em>italic</em></p>"
segs = _parse_note_html(html)
assert "Bold" in segs[0]["text"]
def test_zotero_note_format(self):
html = (
'<div class="zotero-note znv1">'
"<p><strong>Extracted Annotations</strong></p>"
'<p>"Some quoted text" (<a href="zotero://open-pdf/0_KEY/5">Author :5</a>)</p>'
"</div>"
)
segs = _parse_note_html(html)
assert len(segs) >= 2
class TestPageFromLink:
def test_extracts_page(self):
assert _page_from_link("zotero://open-pdf/0_CUNC7XC6/14") == 14
def test_page_one(self):
assert _page_from_link("zotero://open-pdf/0_KEY/1") == 1
def test_no_match(self):
assert _page_from_link("https://example.com") is None
def test_empty(self):
assert _page_from_link("") is None
# ── Quote extraction ─────────────────────────────────────────────
class TestExtractQuotes:
def test_simple_quote(self):
html = '<p>"This is a quoted annotation."</p>'
quotes = _extract_quotes_from_note(html, "TESTKEY2")
assert len(quotes) == 1
assert quotes[0].text == "This is a quoted annotation."
assert quotes[0].item_key == "TESTKEY2"
def test_quote_with_page_link(self):
html = (
'<p>"Important finding about outcomes."'
' (<a href="zotero://open-pdf/0_ABC/14">Smith :14</a>)</p>'
)
quotes = _extract_quotes_from_note(html, "ABCD2345")
assert len(quotes) == 1
assert quotes[0].page == 14
def test_skips_header(self):
html = (
"<p><strong>Extracted Annotations (1/1/2025)</strong></p>"
'<p>"Actual quote."</p>'
)
quotes = _extract_quotes_from_note(html, "KEY12345")
assert len(quotes) == 1
assert quotes[0].text == "Actual quote."
def test_multiple_quotes(self):
html = '<p>"First quote."</p><p>"Second quote."</p>'
quotes = _extract_quotes_from_note(html, "KEY12345")
assert len(quotes) == 2
def test_no_quotes(self):
html = "<p>This is just a regular note.</p>"
quotes = _extract_quotes_from_note(html, "KEY12345")
assert len(quotes) == 0
# ── Quote model ──────────────────────────────────────────────────
class TestQuote:
def test_pincite_directive_basic(self):
q = Quote(text="Important finding.", item_key="ABCD2345")
assert q.pincite_directive() == ':pincite:`ABCD2345` — "Important finding."'
def test_pincite_directive_with_page(self):
q = Quote(text="Finding.", page=14, item_key="ABCD2345")
assert "p.14" in q.pincite_directive()
def test_pincite_directive_with_section(self):
q = Quote(text="Finding.", section="§2.2.1", page=8, item_key="ABCD2345")
d = q.pincite_directive()
assert "§2.2.1" in d
assert "p.8" in d
def test_pincite_truncates_long_text(self):
q = Quote(text="x" * 200, item_key="ABCD2345")
d = q.pincite_directive()
assert "..." in d
assert len(d) < 300
def test_as_tag(self):
q = Quote(text="t", page=14, item_key="ABCD2345")
assert q.as_tag() == "pin:ABCD2345/p.14"
def test_as_tag_with_section(self):
q = Quote(text="t", section="§2.2.1", item_key="ABCD2345")
assert q.as_tag() == "pin:ABCD2345/§2.2.1"
# ── Extractor with DB ────────────────────────────────────────────
@pytest.fixture()
def extractor(tmp_path) -> Extractor:
path = str(tmp_path / "zotero.sqlite")
con = create_db(path)
con.close()
db = Db(path)
# Create items with notes
id1 = db.create_item(TYPE_MAP["statute"], key="STATABCD")
db.set_fields(
id1, {"nameOfAct": "PFS 2026 Final Rule", "url": "https://ex.com/pfs"}
)
db.sync_tags(id1, ["module:pfs", "year:2026"])
db.add_note(
id1,
"<p><strong>Extracted Annotations</strong></p>"
'<p>"The physician fee schedule determines payment rates."'
' (<a href="zotero://open-pdf/0_XYZ/3">CMS :3</a>)</p>'
'<p>"Geographic adjustments apply to all services."</p>',
)
id2 = db.create_item(TYPE_MAP["journalArticle"], key="JRNLEFGH")
db.set_fields(
id2,
{
"title": "Skin Substitute Review",
"DOI": "10.1234/test",
"publicationTitle": "JAMA",
},
)
db.sync_tags(id2, ["module:skin-subs"])
db.add_creators(id2, [("Jane", "Smith")])
# Item with no notes
id3 = db.create_item(TYPE_MAP["webpage"], key="WEBIJKLM")
db.set_fields(id3, {"title": "CMS Data Portal", "url": "https://cms.gov"})
db.sync_tags(id3, ["module:pfs"])
db.commit()
ex = Extractor(db)
yield ex
ex.close()
class TestExtractorQuotes:
def test_quotes_for_item(self, extractor: Extractor):
quotes = extractor.quotes_for_item("STATABCD")
assert len(quotes) == 2
assert quotes[0].text == "The physician fee schedule determines payment rates."
assert quotes[0].page == 3
def test_quotes_for_missing_item(self, extractor: Extractor):
assert extractor.quotes_for_item("ZZZZZZZZ") == []
def test_quotes_for_item_no_notes(self, extractor: Extractor):
assert extractor.quotes_for_item("WEBIJKLM") == []
def test_quotes_for_tag(self, extractor: Extractor):
result = extractor.quotes_for_tag("module:pfs")
assert "STATABCD" in result
assert len(result["STATABCD"]) == 2
class TestExtractorTags:
def test_tags_for_item(self, extractor: Extractor):
tags = extractor.tags_for_item("STATABCD")
assert "module:pfs" in tags
assert "year:2026" in tags
def test_tags_for_missing(self, extractor: Extractor):
assert extractor.tags_for_item("ZZZZZZZZ") == []
def test_items_by_tag_namespace(self, extractor: Extractor):
result = extractor.items_by_tag_namespace("module")
assert "pfs" in result
assert "STATABCD" in result["pfs"]
assert "skin-subs" in result
assert "JRNLEFGH" in result["skin-subs"]
class TestExtractorDocstring:
def test_docstring_block_with_quotes(self, extractor: Extractor):
block = extractor.docstring_block("STATABCD")
assert "References" in block
assert ":pincite:" in block
assert "physician fee schedule" in block
def test_docstring_block_no_notes(self, extractor: Extractor):
block = extractor.docstring_block("WEBIJKLM")
assert "References" in block
assert "CMS Data Portal" in block
def test_pincite_directives(self, extractor: Extractor):
directives = extractor.pincite_directives("STATABCD")
assert len(directives) == 2
assert all(d.startswith(":pincite:") for d in directives)
class TestExtractorExport:
def test_export_provenance(self, extractor: Extractor):
items = extractor.export_provenance()
assert len(items) == 3
keys = {i["key"] for i in items}
assert "STATABCD" in keys
stat = next(i for i in items if i["key"] == "STATABCD")
assert stat["item_type"] == "statute"
assert "module:pfs" in stat["tags"]
assert len(stat["quotes"]) == 2
assert (
stat["quotes"][0]["text"]
== "The physician fee schedule determines payment rates."
)
def test_export_provenance_filtered(self, extractor: Extractor):
items = extractor.export_provenance(tag="module:skin-subs")
assert len(items) == 1
assert items[0]["key"] == "JRNLEFGH"
# ── Integration with real DB ─────────────────────────────────────
@pytest.mark.skipif(not HOST_DB.exists(), reason="host-zotero.sqlite not available")
class TestExtractorRealDb:
def test_extract_real_notes(self):
with Extractor(str(HOST_DB)) as ex:
# Find items with notes
rows = ex.db.con.execute(
"SELECT DISTINCT i.key FROM items i "
"JOIN itemNotes n ON i.itemID = n.parentItemID "
"LIMIT 5"
).fetchall()
for row in rows:
quotes = ex.quotes_for_item(row[0])
# Just verify it doesn't crash
assert isinstance(quotes, list)
def test_export_provenance_real(self):
with Extractor(str(HOST_DB)) as ex:
# Small export
items = (
ex.db.search_by_tag("module:skin-subs")
if ex.db.search_by_tag("module:skin-subs")
else []
)
if items:
result = ex.export_provenance(tag="module:skin-subs")
assert isinstance(result, list)

133
tests/zot/test_schema.py Normal file
View File

@@ -0,0 +1,133 @@
"""Tests for zot.schema — fresh Zotero DB creation from real schema."""
from __future__ import annotations
from zot.db import CREATOR_TYPES, FIELD_MAP, TYPE_MAP, Db
from zot.schema import create_db
class TestCreateDb:
def test_creates_in_memory(self):
con = create_db()
tables = [
r[0]
for r in con.execute(
"SELECT name FROM sqlite_master WHERE type='table'"
).fetchall()
]
assert "items" in tables
assert "itemData" in tables
assert "fields" in tables
con.close()
def test_has_all_61_tables(self):
con = create_db()
tables = [
r[0]
for r in con.execute(
"SELECT name FROM sqlite_master WHERE type='table'"
).fetchall()
]
assert len(tables) >= 61
con.close()
def test_item_types_seeded(self):
con = create_db()
count = con.execute("SELECT count(*) FROM itemTypes").fetchone()[0]
assert count == 36
def test_fields_seeded(self):
con = create_db()
count = con.execute("SELECT count(*) FROM fields").fetchone()[0]
assert count == 104
def test_creator_types_seeded(self):
con = create_db()
count = con.execute("SELECT count(*) FROM creatorTypes").fetchone()[0]
assert count == 29
def test_item_type_fields_seeded(self):
con = create_db()
count = con.execute("SELECT count(*) FROM itemTypeFields").fetchone()[0]
assert count == 582
def test_base_field_mappings_seeded(self):
con = create_db()
count = con.execute("SELECT count(*) FROM baseFieldMappings").fetchone()[0]
assert count == 54
def test_library_exists(self):
con = create_db()
row = con.execute("SELECT type FROM libraries WHERE libraryID = 1").fetchone()
assert row[0] == "user"
con.close()
def test_type_ids_match_constants(self):
con = create_db()
for name, expected_id in TYPE_MAP.items():
row = con.execute(
"SELECT itemTypeID FROM itemTypes WHERE typeName = ?", (name,)
).fetchone()
assert row is not None, f"type {name} not found"
assert row[0] == expected_id, f"type {name}: {row[0]} != {expected_id}"
con.close()
def test_field_ids_match_constants(self):
con = create_db()
for name, expected_id in FIELD_MAP.items():
row = con.execute(
"SELECT fieldID FROM fields WHERE fieldName = ?", (name,)
).fetchone()
assert row is not None, f"field {name} not found"
assert row[0] == expected_id, f"field {name}: {row[0]} != {expected_id}"
con.close()
def test_creator_type_ids_match_constants(self):
con = create_db()
for name, expected_id in CREATOR_TYPES.items():
row = con.execute(
"SELECT creatorTypeID FROM creatorTypes WHERE creatorType = ?",
(name,),
).fetchone()
assert row is not None, f"creator type {name} not found"
assert row[0] == expected_id
con.close()
def test_works_with_orm(self, tmp_path):
"""Full round-trip: create_db → Db → create item → read back."""
p = str(tmp_path / "schema.sqlite")
con = create_db(p)
con.close()
with Db(p) as db:
item_id = db.create_item(TYPE_MAP["statute"])
db.set_fields(item_id, {"nameOfAct": "Test Act", "code": "FR"})
db.sync_tags(item_id, ["test:schema"])
db.add_creators(item_id, [("Jane", "Doe")])
db.commit()
item = db.get_item(item_id)
assert item["itemType"] == "statute"
assert item["fields"]["nameOfAct"] == "Test Act"
assert "test:schema" in item["tags"]
assert item["creators"][0]["lastName"] == "Doe"
def test_validation_works_with_real_schema(self, tmp_path):
"""With full schema seed data, validation can detect invalid fields."""
p = str(tmp_path / "valid.sqlite")
con = create_db(p)
con.close()
with Db(p) as db:
# statute type — set a field that's NOT valid for statute
item_id = db.create_item(TYPE_MAP["statute"])
# publicationTitle (12) is not valid for statute (20)
db.con.execute("INSERT INTO itemDataValues (value) VALUES ('Bad Journal')")
vid = db.con.execute(
"SELECT valueID FROM itemDataValues WHERE value = 'Bad Journal'"
).fetchone()[0]
db.con.execute(
"INSERT INTO itemData (itemID, fieldID, valueID) VALUES (?, 12, ?)",
(item_id, vid),
)
issues = db.validate_item(item_id)
assert len(issues) == 1
assert "publicationTitle" in issues[0]

374
tests/zot/test_table.py Normal file
View File

@@ -0,0 +1,374 @@
"""Tests for zot.table — Pydantic models for all 61 Zotero SQLite tables.
Validates that models match the real Zotero schema by comparing against
the live host-zotero.sqlite database.
"""
from __future__ import annotations
import sqlite3
from pathlib import Path
import pytest
from conf.table_base import SQLTable
HOST_DB = Path("data/zotero/data/host-zotero.sqlite")
# ── Import smoke tests ──────────────────────────────────────────────
class TestImports:
"""All 8 submodules import cleanly."""
def test_core(self):
from zot.table.core import Items, Libraries
assert issubclass(Items, SQLTable)
assert issubclass(Libraries, SQLTable)
def test_fields(self):
from zot.table.fields import Fields, ItemData, ItemDataValues
assert issubclass(Fields, SQLTable)
assert issubclass(ItemData, SQLTable)
assert issubclass(ItemDataValues, SQLTable)
def test_creators(self):
from zot.table.creators import Creators, CreatorTypes, ItemCreators
assert issubclass(Creators, SQLTable)
assert issubclass(CreatorTypes, SQLTable)
assert issubclass(ItemCreators, SQLTable)
def test_collections(self):
from zot.table.collections import CollectionItems, Collections
assert issubclass(Collections, SQLTable)
assert issubclass(CollectionItems, SQLTable)
def test_tags(self):
from zot.table.tags import ItemTags, Tags
assert issubclass(Tags, SQLTable)
assert issubclass(ItemTags, SQLTable)
def test_attachments(self):
from zot.table.attachments import Annotations, ItemAttachments, ItemNotes
assert issubclass(ItemAttachments, SQLTable)
assert issubclass(ItemNotes, SQLTable)
assert issubclass(Annotations, SQLTable)
def test_search(self):
from zot.table.search import FulltextItems, FulltextWords, SavedSearches
assert issubclass(FulltextWords, SQLTable)
assert issubclass(FulltextItems, SQLTable)
assert issubclass(SavedSearches, SQLTable)
def test_sync(self):
from zot.table.sync import Settings, SyncCache, TranslatorCache, Version
assert issubclass(Settings, SQLTable)
assert issubclass(SyncCache, SQLTable)
assert issubclass(Version, SQLTable)
assert issubclass(TranslatorCache, SQLTable)
def test_wildcard_import(self):
import zot.table as zt
models = [
name
for name in dir(zt)
if not name.startswith("_")
and isinstance(getattr(zt, name), type)
and issubclass(getattr(zt, name), SQLTable)
and getattr(zt, name) is not SQLTable
]
assert len(models) == 61
# ── Model correctness ───────────────────────────────────────────────
class TestModelStructure:
"""Models have expected columns and types."""
def test_items_columns(self):
from zot.table.core import Items
cols = Items.column_names()
assert "itemID" in cols
assert "itemTypeID" in cols
assert "libraryID" in cols
assert "key" in cols
assert "synced" in cols
def test_items_instantiation(self):
from zot.table.core import Items
item = Items(itemTypeID=20, libraryID=1, key="TESTKEY1")
assert item.itemTypeID == 20
assert item.libraryID == 1
assert item.synced == 0
def test_fields_columns(self):
from zot.table.fields import Fields
cols = Fields.column_names()
assert cols == ["fieldID", "fieldName", "fieldFormatID"]
def test_item_data_eav(self):
from zot.table.fields import ItemData
row = ItemData(itemID=1, fieldID=110, valueID=42)
assert row.itemID == 1
assert row.fieldID == 110
def test_creators_columns(self):
from zot.table.creators import Creators
c = Creators(firstName="John", lastName="Doe", fieldMode=0)
assert c.firstName == "John"
assert c.lastName == "Doe"
def test_item_creators_defaults(self):
from zot.table.creators import ItemCreators
ic = ItemCreators(itemID=1, creatorID=2)
assert ic.creatorTypeID == 1 # author
assert ic.orderIndex == 0
def test_collections_columns(self):
from zot.table.collections import Collections
c = Collections(collectionName="Test", libraryID=1, key="ABCD1234")
assert c.collectionName == "Test"
assert c.synced == 0
def test_tags_columns(self):
from zot.table.tags import Tags
t = Tags(name="module:pfs")
assert t.name == "module:pfs"
def test_version_renamed_field(self):
from zot.table.sync import Version
v = Version(schema_name="userdata", version=127)
assert v.schema_name == "userdata"
assert v.version == 127
def test_ddl_generation(self):
from zot.table.core import Items
ddl = Items.to_ddl()
assert "CREATE TABLE" in ddl
assert "itemID" in ddl
assert "itemTypeID" in ddl
# ── Schema validation against real DB ────────────────────────────────
def _get_real_table_columns(table_name: str) -> list[str]:
"""Get column names from the real Zotero database."""
con = sqlite3.connect(f"file:{HOST_DB}?mode=ro", uri=True)
cols = [r[1] for r in con.execute(f'PRAGMA table_info("{table_name}")').fetchall()]
con.close()
return cols
def _get_all_real_tables() -> list[str]:
"""Get all table names from the real Zotero database."""
con = sqlite3.connect(f"file:{HOST_DB}?mode=ro", uri=True)
tables = [
r[0]
for r in con.execute(
"SELECT name FROM sqlite_master WHERE type='table' ORDER BY name"
).fetchall()
]
con.close()
return tables
@pytest.mark.skipif(not HOST_DB.exists(), reason="host-zotero.sqlite not available")
class TestSchemaMatchesRealDB:
"""Verify Pydantic models match the real Zotero SQLite schema."""
def test_all_real_tables_have_models(self):
import zot.table as zt
real_tables = set(_get_all_real_tables())
model_tables = set()
for name in dir(zt):
cls = getattr(zt, name)
if (
isinstance(cls, type)
and issubclass(cls, SQLTable)
and cls is not SQLTable
):
model_tables.add(cls.__tablename__)
missing = real_tables - model_tables
assert missing == set(), f"Tables without models: {missing}"
def test_items_columns_match(self):
from zot.table.core import Items
real = _get_real_table_columns("items")
model = Items.column_names()
assert set(model) == set(real)
def test_fields_columns_match(self):
from zot.table.fields import Fields
real = _get_real_table_columns("fields")
model = Fields.column_names()
assert set(model) == set(real)
def test_creators_columns_match(self):
from zot.table.creators import Creators
real = _get_real_table_columns("creators")
model = Creators.column_names()
assert set(model) == set(real)
def test_item_data_columns_match(self):
from zot.table.fields import ItemData
real = _get_real_table_columns("itemData")
model = ItemData.column_names()
assert set(model) == set(real)
def test_collections_columns_match(self):
from zot.table.collections import Collections
real = _get_real_table_columns("collections")
model = Collections.column_names()
assert set(model) == set(real)
def test_all_models_columns_match(self):
"""Every model's columns must match the real table."""
import zot.table as zt
mismatches = []
for name in sorted(dir(zt)):
cls = getattr(zt, name)
if not (
isinstance(cls, type)
and issubclass(cls, SQLTable)
and cls is not SQLTable
):
continue
tbl = cls.__tablename__
try:
real = set(_get_real_table_columns(tbl))
except Exception:
mismatches.append(f"{name}: table {tbl} not found in DB")
continue
model = set(cls.column_names())
# Handle the renamed 'schema' -> 'schema_name' in Version
if tbl == "version" and "schema" in real and "schema_name" in model:
real = (real - {"schema"}) | {"schema_name"}
if model != real:
extra = model - real
missing = real - model
parts = []
if extra:
parts.append(f"extra={extra}")
if missing:
parts.append(f"missing={missing}")
mismatches.append(f"{name}({tbl}): {', '.join(parts)}")
assert mismatches == [], "Column mismatches:\n" + "\n".join(mismatches)
# ── Sync ID constants match real DB ─────────────────────────────────
@pytest.mark.skipif(not HOST_DB.exists(), reason="host-zotero.sqlite not available")
class TestSyncIDsMatchRealDB:
"""Verify bib.sync constants match the real Zotero schema."""
def test_type_map_ids_exist(self):
from bib.sync import _TYPE_MAP
con = sqlite3.connect(f"file:{HOST_DB}?mode=ro", uri=True)
real_types = {
r[0]: r[1]
for r in con.execute("SELECT itemTypeID, typeName FROM itemTypes")
}
con.close()
for bib_type, zotero_id in _TYPE_MAP.items():
assert zotero_id in real_types, (
f"_TYPE_MAP[{bib_type!r}] = {zotero_id} not in itemTypes"
)
def test_type_map_names_correct(self):
from bib.sync import _TYPE_MAP
con = sqlite3.connect(f"file:{HOST_DB}?mode=ro", uri=True)
real_types = dict(
con.execute("SELECT itemTypeID, typeName FROM itemTypes").fetchall()
)
con.close()
expected = {
"rule": "statute",
"regulation": "statute",
"manual": "report",
"download": "webpage",
"source": "document",
"journal-article": "journalArticle",
}
for bib_type, expected_name in expected.items():
zotero_id = _TYPE_MAP[bib_type]
assert real_types[zotero_id] == expected_name, (
f"_TYPE_MAP[{bib_type!r}] = {zotero_id} -> "
f"{real_types[zotero_id]!r}, expected {expected_name!r}"
)
def test_field_ids_exist(self):
from zot.db import FIELD_MAP as _FIELD_IDS
con = sqlite3.connect(f"file:{HOST_DB}?mode=ro", uri=True)
real_fields = {
r[0]: r[1] for r in con.execute("SELECT fieldID, fieldName FROM fields")
}
con.close()
for field_name, field_id in _FIELD_IDS.items():
assert field_id in real_fields, (
f"_FIELD_IDS[{field_name!r}] = {field_id} not in fields table"
)
def test_field_ids_names_match(self):
from zot.db import FIELD_MAP as _FIELD_IDS
con = sqlite3.connect(f"file:{HOST_DB}?mode=ro", uri=True)
real_fields = dict(
con.execute("SELECT fieldID, fieldName FROM fields").fetchall()
)
con.close()
for field_name, field_id in _FIELD_IDS.items():
actual_name = real_fields.get(field_id)
assert actual_name == field_name, (
f"_FIELD_IDS[{field_name!r}] = {field_id} -> {actual_name!r} in real DB"
)
def test_creator_type_author_is_1(self):
con = sqlite3.connect(f"file:{HOST_DB}?mode=ro", uri=True)
row = con.execute(
"SELECT creatorType FROM creatorTypes WHERE creatorTypeID = 1"
).fetchone()
con.close()
assert row[0] == "author"