From ff4099d1d2fadd6b776d0388161880cd08250537 Mon Sep 17 00:00:00 2001 From: kert Date: Thu, 12 Nov 2026 22:57:36 -0500 Subject: [PATCH] feat(hoard): register every on-disk ACO dataset version in bib, dedup by URL --- dev/scripts/hoard_aco_versions.py | 51 +++++++++++++++++-------------- 1 file changed, 28 insertions(+), 23 deletions(-) diff --git a/dev/scripts/hoard_aco_versions.py b/dev/scripts/hoard_aco_versions.py index 923bd54..74f9d4d 100644 --- a/dev/scripts/hoard_aco_versions.py +++ b/dev/scripts/hoard_aco_versions.py @@ -111,47 +111,52 @@ PROGRAM = { "number_of_accountable_care": "mssp", "county_level_aggregate": "mssp", "next_generation": "ngaco", "pioneer": "pioneer", "reach": "reach", "comprehensive_end_stage": "cec", "kidney_care": "kcc", + "advance_investment": "mssp", "prepaid_shared": "mssp", "ambulatory_specialty": "asm", } def register(dry: bool) -> int: - """One bib source item per newly fetched dataset version (file attached). + """One bib source item per on-disk dataset version (file attached). - Same tags as import_lead_corpus (``source:lead-corpus`` + - ``lead-dataset:``) plus ``doc:dataset`` and ``program:``. - Items match on URL, so re-running is safe. + Tags as import_lead_corpus (``source:lead-corpus``, + ``lead-dataset:``) plus ``doc:dataset``, + ``source:cms-website`` and ``program:``. Items match on URL (the + upsert key), so re-running is safe; dry-run reports what exists. """ from collections import Counter - manifest = json.loads(MANIFEST.read_text()) - rows = [m for m in manifest if m["status"] == "fetched"] - stats: Counter = Counter() - if not dry: - from bib.item import Source - from bib.store import Store + from bib.item import Source + from bib.store import Store - store = Store() - col = store.ensure_collections( - {"LEAD corpus": {"Dataset documentation": {}}} - )["Dataset documentation"] - for m in rows: + store = Store() + col = store.ensure_collections( + {"LEAD corpus": {"Dataset documentation": {}}} + )["Dataset documentation"] if not dry else None + have = {r[0] for r in store._con().execute("SELECT url FROM items WHERE url <> ''")} + before: Counter = Counter() + total: Counter = Counter() + for m in json.loads(MANIFEST.read_text()): + total[m["source_id"]] += 1 + before[m["source_id"]] += m["url"] in have + if m["url"] in have and dry: + continue prog = next((v for k, v in PROGRAM.items() if k in m["source_id"]), "") - tags = ["source:lead-corpus", f"lead-dataset:{m['source_id']}", "doc:dataset"] + tags = ["source:lead-corpus", "source:cms-website", + f"lead-dataset:{m['source_id']}", "doc:dataset"] tags += [f"program:{prog}"] if prog else [] - title = f"{m['series']} : {m['release'].replace('_', '-')}" if dry: - print("would register", title, tags) - stats["dry"] += 1 + print("would register", m["series"], m["release"]) continue item = Source( - title=title, url=m["url"], institution="CMS / data.cms.gov", + title=f"{m['series']} : {m['release'].replace('_', '-')}", url=m["url"], + institution="CMS / data.cms.gov", date_published=m["release"].replace("_", "-")[:10], extra=f"sha256: {m['sha256']}", doc_type="dataset", ) - key, status = store.upsert_status(item, tags=tags, collection=col) + key, _ = store.upsert_status(item, tags=tags, collection=col) store.attach_file(key, ROOT / m["source_id"] / m["release"] / m["name"]) - stats[status] += 1 - print("register:", dict(stats)) + for sid in sorted(total): + print(f"{sid}: bib before {before[sid]}/{total[sid]}") return 0