feat(hoard): register every on-disk ACO dataset version in bib, dedup by URL
Some checks failed
CI / lint (push) Successful in 44s
CI / notebooks-smoke (push) Successful in 3m44s
Deploy / changes (push) Successful in 1m3s
Infra CI / notebooks (push) Successful in 1m46s
Infra CI / zotero (push) Successful in 33s
Infra CI / docs (push) Successful in 30s
CI / test (push) Failing after 8m56s
Infra CI / api (push) Successful in 2m26s
Infra CI / bcda (push) Successful in 5m7s
Infra CI / mc (push) Successful in 28s
Infra CI / llm (push) Successful in 4m29s
Deploy / notebooks (push) Has been skipped
Deploy / zotero (push) Has been skipped
Deploy / docs (push) Has been skipped
Deploy / api (push) Successful in 1m35s
Renovate / renovate (push) Successful in 3m6s
Deploy / bcda (push) Successful in 2m33s
Deploy / mc (push) Has been skipped
Deploy / llm (push) Successful in 2m8s
Zotero Sync / zotero-sync (push) Successful in 4m23s
Deploy / report (push) Has been cancelled

This commit is contained in:
kert
2026-11-12 22:57:36 -05:00
parent 96b74c545c
commit ff4099d1d2

View File

@@ -111,47 +111,52 @@ PROGRAM = {
"number_of_accountable_care": "mssp", "county_level_aggregate": "mssp",
"next_generation": "ngaco", "pioneer": "pioneer", "reach": "reach",
"comprehensive_end_stage": "cec", "kidney_care": "kcc",
"advance_investment": "mssp", "prepaid_shared": "mssp", "ambulatory_specialty": "asm",
}
def register(dry: bool) -> int:
"""One bib source item per newly fetched dataset version (file attached).
"""One bib source item per on-disk dataset version (file attached).
Same tags as import_lead_corpus (``source:lead-corpus`` +
``lead-dataset:<source_id>``) plus ``doc:dataset`` and ``program:<x>``.
Items match on URL, so re-running is safe.
Tags as import_lead_corpus (``source:lead-corpus``,
``lead-dataset:<source_id>``) plus ``doc:dataset``,
``source:cms-website`` and ``program:<x>``. Items match on URL (the
upsert key), so re-running is safe; dry-run reports what exists.
"""
from collections import Counter
manifest = json.loads(MANIFEST.read_text())
rows = [m for m in manifest if m["status"] == "fetched"]
stats: Counter = Counter()
if not dry:
from bib.item import Source
from bib.store import Store
from bib.item import Source
from bib.store import Store
store = Store()
col = store.ensure_collections(
{"LEAD corpus": {"Dataset documentation": {}}}
)["Dataset documentation"]
for m in rows:
store = Store()
col = store.ensure_collections(
{"LEAD corpus": {"Dataset documentation": {}}}
)["Dataset documentation"] if not dry else None
have = {r[0] for r in store._con().execute("SELECT url FROM items WHERE url <> ''")}
before: Counter = Counter()
total: Counter = Counter()
for m in json.loads(MANIFEST.read_text()):
total[m["source_id"]] += 1
before[m["source_id"]] += m["url"] in have
if m["url"] in have and dry:
continue
prog = next((v for k, v in PROGRAM.items() if k in m["source_id"]), "")
tags = ["source:lead-corpus", f"lead-dataset:{m['source_id']}", "doc:dataset"]
tags = ["source:lead-corpus", "source:cms-website",
f"lead-dataset:{m['source_id']}", "doc:dataset"]
tags += [f"program:{prog}"] if prog else []
title = f"{m['series']} : {m['release'].replace('_', '-')}"
if dry:
print("would register", title, tags)
stats["dry"] += 1
print("would register", m["series"], m["release"])
continue
item = Source(
title=title, url=m["url"], institution="CMS / data.cms.gov",
title=f"{m['series']} : {m['release'].replace('_', '-')}", url=m["url"],
institution="CMS / data.cms.gov",
date_published=m["release"].replace("_", "-")[:10],
extra=f"sha256: {m['sha256']}", doc_type="dataset",
)
key, status = store.upsert_status(item, tags=tags, collection=col)
key, _ = store.upsert_status(item, tags=tags, collection=col)
store.attach_file(key, ROOT / m["source_id"] / m["release"] / m["name"])
stats[status] += 1
print("register:", dict(stats))
for sid in sorted(total):
print(f"{sid}: bib before {before[sid]}/{total[sid]}")
return 0