feat(zotero): attach landed AMA files whose folder is not an exact OpenLibrary title
Some checks failed
CI / lint (push) Successful in 34s
CI / notebooks-smoke (push) Successful in 1m29s
CI / lint (pull_request) Successful in 38s
CI / test (push) Successful in 17m37s
CI / notebooks-smoke (pull_request) Successful in 1m28s
CI / test (pull_request) Successful in 16m58s
Infra CI / zotero (pull_request) Successful in 14s
Infra CI / docs (pull_request) Successful in 15s
Infra CI / notebooks (pull_request) Failing after 1m12s
Infra CI / api (pull_request) Successful in 12s
Infra CI / llm (pull_request) Successful in 13s
Infra CI / mc (pull_request) Successful in 13s
Some checks failed
CI / lint (push) Successful in 34s
CI / notebooks-smoke (push) Successful in 1m29s
CI / lint (pull_request) Successful in 38s
CI / test (push) Successful in 17m37s
CI / notebooks-smoke (pull_request) Successful in 1m28s
CI / test (pull_request) Successful in 16m58s
Infra CI / zotero (pull_request) Successful in 14s
Infra CI / docs (pull_request) Successful in 15s
Infra CI / notebooks (pull_request) Failing after 1m12s
Infra CI / api (pull_request) Successful in 12s
Infra CI / llm (pull_request) Successful in 13s
Infra CI / mc (pull_request) Successful in 13s
Year-aware containment match (keeps ICD-10-CM apart from ICD-10-PCS and a 4th edition from a 6th); a local item is minted when no title covers a landed book, so every file on disk is filed.
This commit is contained in:
@@ -51,10 +51,51 @@ def _norm(s: str) -> str:
|
|||||||
|
|
||||||
def _ol_url(key: str) -> str:
|
def _ol_url(key: str) -> str:
|
||||||
olid = key.rsplit("/", 1)[-1]
|
olid = key.rsplit("/", 1)[-1]
|
||||||
|
if key.startswith("/local/"):
|
||||||
|
# Landed file with no OpenLibrary work: a stable local identity.
|
||||||
|
return f"local://ama-coding/{olid}"
|
||||||
kind = "works" if olid.endswith("W") else "books"
|
kind = "works" if olid.endswith("W") else "books"
|
||||||
return f"https://openlibrary.org/{kind}/{olid}"
|
return f"https://openlibrary.org/{kind}/{olid}"
|
||||||
|
|
||||||
|
|
||||||
|
_YEAR_RE = re.compile(r"\b(19|20)\d\d\b")
|
||||||
|
|
||||||
|
|
||||||
|
def _year_of(s: str) -> str | None:
|
||||||
|
m = _YEAR_RE.search(s)
|
||||||
|
return m.group(0) if m else None
|
||||||
|
|
||||||
|
|
||||||
|
def _folder_matches(folder: str, entries: list[dict]) -> dict | None:
|
||||||
|
"""Best title entry for a landed folder: exact normalised title, else a
|
||||||
|
title that contains it (or is contained by it) once spaces are dropped,
|
||||||
|
with agreeing years. Containment keeps ICD-10-CM apart from ICD-10-PCS
|
||||||
|
and a 4th edition apart from a 6th; a similarity ratio does not."""
|
||||||
|
target = _norm(folder)
|
||||||
|
by_norm = {_norm(e["title"]): e for e in entries}
|
||||||
|
if target in by_norm:
|
||||||
|
return by_norm[target]
|
||||||
|
year = _year_of(folder)
|
||||||
|
tight = target.replace(" ", "")
|
||||||
|
best, best_len = None, 0
|
||||||
|
for e in entries:
|
||||||
|
ey = _year_of(e["title"]) or (str(e["year"]) if e.get("year") else None)
|
||||||
|
if year and ey and year != ey:
|
||||||
|
continue
|
||||||
|
cand = _norm(e["title"]).replace(" ", "")
|
||||||
|
if cand in tight or tight in cand:
|
||||||
|
if len(cand) > best_len:
|
||||||
|
best, best_len = e, len(cand)
|
||||||
|
return best
|
||||||
|
|
||||||
|
|
||||||
|
def _folder_entry(folder: str) -> dict:
|
||||||
|
"""A synthetic entry for a landed book no OpenLibrary title covers."""
|
||||||
|
year = _year_of(folder)
|
||||||
|
return {"key": f"/local/{_norm(folder).replace(' ', '-')}", "title": folder,
|
||||||
|
"year": int(year) if year else None, "isbns": []}
|
||||||
|
|
||||||
|
|
||||||
def _index_books(books_dir: Path) -> dict[str, list[Path]]:
|
def _index_books(books_dir: Path) -> dict[str, list[Path]]:
|
||||||
"""Map normalised folder title -> ebook files LazyLibrarian has landed."""
|
"""Map normalised folder title -> ebook files LazyLibrarian has landed."""
|
||||||
index: dict[str, list[Path]] = {}
|
index: dict[str, list[Path]] = {}
|
||||||
@@ -113,8 +154,9 @@ def _ensure_item(db: Db, entry: dict, collection_key: str, *, dry_run: bool) ->
|
|||||||
"accessDate": now,
|
"accessDate": now,
|
||||||
"language": "en",
|
"language": "en",
|
||||||
"libraryCatalog": "Open Library",
|
"libraryCatalog": "Open Library",
|
||||||
"extra": f"OpenLibrary: {entry['key'].rsplit('/', 1)[-1]}",
|
|
||||||
}
|
}
|
||||||
|
if not entry["key"].startswith("/local/"):
|
||||||
|
fields["extra"] = f"OpenLibrary: {entry['key'].rsplit('/', 1)[-1]}"
|
||||||
if entry.get("year"):
|
if entry.get("year"):
|
||||||
fields["date"] = str(entry["year"])
|
fields["date"] = str(entry["year"])
|
||||||
if entry.get("isbns"):
|
if entry.get("isbns"):
|
||||||
@@ -160,6 +202,23 @@ def main() -> None:
|
|||||||
continue
|
continue
|
||||||
_attach(db, item_id, f, dry_run=args.dry_run)
|
_attach(db, item_id, f, dry_run=args.dry_run)
|
||||||
attached += 1
|
attached += 1
|
||||||
|
# Folders the exact-title pass did not consume: fuzzy-match a title
|
||||||
|
# (same year), else mint a local item so every landed file is filed.
|
||||||
|
exact = {_norm(e["title"]) for e in entries}
|
||||||
|
for folder_norm, files in books.items():
|
||||||
|
if folder_norm in exact:
|
||||||
|
continue
|
||||||
|
folder = files[0].parent.name
|
||||||
|
entry = _folder_matches(folder, entries) or _folder_entry(folder)
|
||||||
|
item_id = _ensure_item(db, entry, collection_key, dry_run=args.dry_run)
|
||||||
|
if item_id is None:
|
||||||
|
continue
|
||||||
|
have = _existing_attachment_names(db, item_id)
|
||||||
|
for f in files:
|
||||||
|
if f.name in have:
|
||||||
|
continue
|
||||||
|
_attach(db, item_id, f, dry_run=args.dry_run)
|
||||||
|
attached += 1
|
||||||
if not args.dry_run:
|
if not args.dry_run:
|
||||||
db.commit()
|
db.commit()
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user