feat: scipdf plugin + PDF coverage tracker for skin-subs retrieval (refs #257)
Some checks failed
CI / skinny-install (aco) (push) Successful in 50s
CI / skinny-install (ccw) (push) Successful in 34s
CI / skinny-install (api) (push) Successful in 35s
CI / lint-test (push) Failing after 1m30s
CI / skinny-install (bib) (push) Successful in 29s
CI / skinny-install (bcda) (push) Successful in 34s
CI / skinny-install (bls) (push) Successful in 22s
CI / skinny-install (cli) (push) Successful in 32s
CI / skinny-install (cms) (push) Successful in 26s
CI / skinny-install (conf) (push) Successful in 25s
CI / skinny-install (perf) (push) Successful in 30s
CI / skinny-install (pfs) (push) Successful in 36s
CI / skinny-install (rex) (push) Successful in 27s
Deploy / build-scan-report (push) Successful in 3m31s
Some checks failed
CI / skinny-install (aco) (push) Successful in 50s
CI / skinny-install (ccw) (push) Successful in 34s
CI / skinny-install (api) (push) Successful in 35s
CI / lint-test (push) Failing after 1m30s
CI / skinny-install (bib) (push) Successful in 29s
CI / skinny-install (bcda) (push) Successful in 34s
CI / skinny-install (bls) (push) Successful in 22s
CI / skinny-install (cli) (push) Successful in 32s
CI / skinny-install (cms) (push) Successful in 26s
CI / skinny-install (conf) (push) Successful in 25s
CI / skinny-install (perf) (push) Successful in 30s
CI / skinny-install (pfs) (push) Successful in 36s
CI / skinny-install (rex) (push) Successful in 27s
Deploy / build-scan-report (push) Successful in 3m31s
Installed zotero-scipdf v8.0.4 (syt2/zotero-scipdf) in Zotero 8 container: - 7 SciHub mirrors registered as Zotero PDF resolvers - Works with Zotero's native "Find Available PDF" feature - Also installed alongside ZotMoov (existing) PDF coverage script: dev/scripts/check_pdf_coverage.py - Reports DOI count, PDF attachment count, coverage by item type - Current baseline: 9,658 DOIs (94%), 0 PDFs retrieved To retrieve: open Zotero GUI, select Skin Substitutes collection, Ctrl+A, right-click → Find Available PDF. Plugin handles Unpaywall first, then SciHub mirrors sequentially.
This commit is contained in:
92
dev/scripts/check_pdf_coverage.py
Normal file
92
dev/scripts/check_pdf_coverage.py
Normal file
@@ -0,0 +1,92 @@
|
|||||||
|
"""Check PDF retrieval coverage for skin substitute evidence base.
|
||||||
|
|
||||||
|
Reports how many Zotero items have PDF attachments vs DOI-bearing items
|
||||||
|
that could potentially be retrieved. Run after using Zotero's
|
||||||
|
"Find Available PDF" with the scipdf plugin.
|
||||||
|
|
||||||
|
Usage:
|
||||||
|
uv run python dev/scripts/check_pdf_coverage.py
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import sqlite3
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
from conf import path as conf_path
|
||||||
|
|
||||||
|
ZOTERO_DB = str(conf_path("db.zotero"))
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> None:
|
||||||
|
print("Checking PDF coverage for skin-subs items in Zotero ...\n")
|
||||||
|
|
||||||
|
con = sqlite3.connect(ZOTERO_DB, timeout=10)
|
||||||
|
con.row_factory = sqlite3.Row
|
||||||
|
|
||||||
|
# Total skin-subs items
|
||||||
|
total = con.execute("""
|
||||||
|
SELECT count(DISTINCT it.itemID) FROM itemTags it
|
||||||
|
JOIN tags t ON it.tagID = t.tagID
|
||||||
|
WHERE t.name = 'module:skin-subs'
|
||||||
|
""").fetchone()[0]
|
||||||
|
|
||||||
|
# Items with DOI (retrievable)
|
||||||
|
with_doi = con.execute("""
|
||||||
|
SELECT count(DISTINCT i.itemID) FROM items i
|
||||||
|
JOIN itemTags it ON i.itemID = it.itemID
|
||||||
|
JOIN tags t ON it.tagID = t.tagID
|
||||||
|
JOIN itemData id ON i.itemID = id.itemID
|
||||||
|
WHERE t.name = 'module:skin-subs' AND id.fieldID = 8
|
||||||
|
""").fetchone()[0]
|
||||||
|
|
||||||
|
# Items with PDF attachments
|
||||||
|
with_pdf = con.execute("""
|
||||||
|
SELECT count(DISTINCT parent.itemID) FROM items parent
|
||||||
|
JOIN itemTags it ON parent.itemID = it.itemID
|
||||||
|
JOIN tags t ON it.tagID = t.tagID
|
||||||
|
JOIN itemAttachments ia ON ia.parentItemID = parent.itemID
|
||||||
|
JOIN items att ON ia.itemID = att.itemID
|
||||||
|
WHERE t.name = 'module:skin-subs'
|
||||||
|
AND ia.contentType = 'application/pdf'
|
||||||
|
""").fetchone()[0]
|
||||||
|
|
||||||
|
print(f"Total skin-subs items: {total:>6}")
|
||||||
|
print(f"Items with DOI: {with_doi:>6} ({with_doi * 100 // total}%)")
|
||||||
|
print(f"Items with PDF attached: {with_pdf:>6} ({with_pdf * 100 // max(total, 1)}%)")
|
||||||
|
print(f"DOIs without PDF: {with_doi - with_pdf:>6}")
|
||||||
|
print(f"Coverage (of DOI items): {with_pdf * 100 // max(with_doi, 1)}%")
|
||||||
|
|
||||||
|
# Breakdown by item type
|
||||||
|
print("\nBy item type:")
|
||||||
|
for r in con.execute("""
|
||||||
|
SELECT it2.typeName,
|
||||||
|
count(DISTINCT i.itemID) as total,
|
||||||
|
count(DISTINCT CASE WHEN ia.itemID IS NOT NULL THEN i.itemID END) as with_pdf
|
||||||
|
FROM items i
|
||||||
|
JOIN itemTags it ON i.itemID = it.itemID
|
||||||
|
JOIN tags t ON it.tagID = t.tagID
|
||||||
|
JOIN itemTypes it2 ON i.itemTypeID = it2.itemTypeID
|
||||||
|
LEFT JOIN itemAttachments ia ON ia.parentItemID = i.itemID
|
||||||
|
AND ia.contentType = 'application/pdf'
|
||||||
|
WHERE t.name = 'module:skin-subs'
|
||||||
|
GROUP BY it2.typeName
|
||||||
|
ORDER BY total DESC
|
||||||
|
""").fetchall():
|
||||||
|
pct = r["with_pdf"] * 100 // max(r["total"], 1)
|
||||||
|
print(f" {r['typeName']:20s} total={r['total']:>5} pdf={r['with_pdf']:>5} ({pct}%)")
|
||||||
|
|
||||||
|
# Storage size
|
||||||
|
storage = Path(conf_path("storage.zotero"))
|
||||||
|
if storage.exists():
|
||||||
|
total_bytes = sum(f.stat().st_size for f in storage.rglob("*") if f.is_file())
|
||||||
|
print(f"\nZotero storage size: {total_bytes / 1024 / 1024:.1f} MB")
|
||||||
|
|
||||||
|
con.close()
|
||||||
|
print("\nTo retrieve PDFs: open Zotero, select 'Skin Substitutes' collection,")
|
||||||
|
print("select all items (Ctrl+A), right-click → Find Available PDF.")
|
||||||
|
print("scipdf plugin (v8.0.4) will try Unpaywall then 7 SciHub mirrors.")
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
Reference in New Issue
Block a user