feat(hoard): register every on-disk ACO dataset version in bib, dedup by URL
Some checks failed
CI / lint (push) Successful in 44s
CI / notebooks-smoke (push) Successful in 3m44s
Deploy / changes (push) Successful in 1m3s
Infra CI / notebooks (push) Successful in 1m46s
Infra CI / zotero (push) Successful in 33s
Infra CI / docs (push) Successful in 30s
CI / test (push) Failing after 8m56s
Infra CI / api (push) Successful in 2m26s
Infra CI / bcda (push) Successful in 5m7s
Infra CI / mc (push) Successful in 28s
Infra CI / llm (push) Successful in 4m29s
Deploy / notebooks (push) Has been skipped
Deploy / zotero (push) Has been skipped
Deploy / docs (push) Has been skipped
Deploy / api (push) Successful in 1m35s
Renovate / renovate (push) Successful in 3m6s
Deploy / bcda (push) Successful in 2m33s
Deploy / mc (push) Has been skipped
Deploy / llm (push) Successful in 2m8s
Zotero Sync / zotero-sync (push) Successful in 4m23s
Deploy / report (push) Has been cancelled
Some checks failed
CI / lint (push) Successful in 44s
CI / notebooks-smoke (push) Successful in 3m44s
Deploy / changes (push) Successful in 1m3s
Infra CI / notebooks (push) Successful in 1m46s
Infra CI / zotero (push) Successful in 33s
Infra CI / docs (push) Successful in 30s
CI / test (push) Failing after 8m56s
Infra CI / api (push) Successful in 2m26s
Infra CI / bcda (push) Successful in 5m7s
Infra CI / mc (push) Successful in 28s
Infra CI / llm (push) Successful in 4m29s
Deploy / notebooks (push) Has been skipped
Deploy / zotero (push) Has been skipped
Deploy / docs (push) Has been skipped
Deploy / api (push) Successful in 1m35s
Renovate / renovate (push) Successful in 3m6s
Deploy / bcda (push) Successful in 2m33s
Deploy / mc (push) Has been skipped
Deploy / llm (push) Successful in 2m8s
Zotero Sync / zotero-sync (push) Successful in 4m23s
Deploy / report (push) Has been cancelled
This commit is contained in:
@@ -111,47 +111,52 @@ PROGRAM = {
|
||||
"number_of_accountable_care": "mssp", "county_level_aggregate": "mssp",
|
||||
"next_generation": "ngaco", "pioneer": "pioneer", "reach": "reach",
|
||||
"comprehensive_end_stage": "cec", "kidney_care": "kcc",
|
||||
"advance_investment": "mssp", "prepaid_shared": "mssp", "ambulatory_specialty": "asm",
|
||||
}
|
||||
|
||||
|
||||
def register(dry: bool) -> int:
|
||||
"""One bib source item per newly fetched dataset version (file attached).
|
||||
"""One bib source item per on-disk dataset version (file attached).
|
||||
|
||||
Same tags as import_lead_corpus (``source:lead-corpus`` +
|
||||
``lead-dataset:<source_id>``) plus ``doc:dataset`` and ``program:<x>``.
|
||||
Items match on URL, so re-running is safe.
|
||||
Tags as import_lead_corpus (``source:lead-corpus``,
|
||||
``lead-dataset:<source_id>``) plus ``doc:dataset``,
|
||||
``source:cms-website`` and ``program:<x>``. Items match on URL (the
|
||||
upsert key), so re-running is safe; dry-run reports what exists.
|
||||
"""
|
||||
from collections import Counter
|
||||
|
||||
manifest = json.loads(MANIFEST.read_text())
|
||||
rows = [m for m in manifest if m["status"] == "fetched"]
|
||||
stats: Counter = Counter()
|
||||
if not dry:
|
||||
from bib.item import Source
|
||||
from bib.store import Store
|
||||
from bib.item import Source
|
||||
from bib.store import Store
|
||||
|
||||
store = Store()
|
||||
col = store.ensure_collections(
|
||||
{"LEAD corpus": {"Dataset documentation": {}}}
|
||||
)["Dataset documentation"]
|
||||
for m in rows:
|
||||
store = Store()
|
||||
col = store.ensure_collections(
|
||||
{"LEAD corpus": {"Dataset documentation": {}}}
|
||||
)["Dataset documentation"] if not dry else None
|
||||
have = {r[0] for r in store._con().execute("SELECT url FROM items WHERE url <> ''")}
|
||||
before: Counter = Counter()
|
||||
total: Counter = Counter()
|
||||
for m in json.loads(MANIFEST.read_text()):
|
||||
total[m["source_id"]] += 1
|
||||
before[m["source_id"]] += m["url"] in have
|
||||
if m["url"] in have and dry:
|
||||
continue
|
||||
prog = next((v for k, v in PROGRAM.items() if k in m["source_id"]), "")
|
||||
tags = ["source:lead-corpus", f"lead-dataset:{m['source_id']}", "doc:dataset"]
|
||||
tags = ["source:lead-corpus", "source:cms-website",
|
||||
f"lead-dataset:{m['source_id']}", "doc:dataset"]
|
||||
tags += [f"program:{prog}"] if prog else []
|
||||
title = f"{m['series']} : {m['release'].replace('_', '-')}"
|
||||
if dry:
|
||||
print("would register", title, tags)
|
||||
stats["dry"] += 1
|
||||
print("would register", m["series"], m["release"])
|
||||
continue
|
||||
item = Source(
|
||||
title=title, url=m["url"], institution="CMS / data.cms.gov",
|
||||
title=f"{m['series']} : {m['release'].replace('_', '-')}", url=m["url"],
|
||||
institution="CMS / data.cms.gov",
|
||||
date_published=m["release"].replace("_", "-")[:10],
|
||||
extra=f"sha256: {m['sha256']}", doc_type="dataset",
|
||||
)
|
||||
key, status = store.upsert_status(item, tags=tags, collection=col)
|
||||
key, _ = store.upsert_status(item, tags=tags, collection=col)
|
||||
store.attach_file(key, ROOT / m["source_id"] / m["release"] / m["name"])
|
||||
stats[status] += 1
|
||||
print("register:", dict(stats))
|
||||
for sid in sorted(total):
|
||||
print(f"{sid}: bib before {before[sid]}/{total[sid]}")
|
||||
return 0
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user