homelab/stack
1
0
Fork 0
Code Issues 20 Pull Requests Actions Packages Projects 1 Releases Wiki Activity
Files
5628b505485653b2bc962ae2d3b40961adb87bc5
stack/dev/scripts/build_skin_subs_evidence_base.py
kert 90b37fe4c2 chore: prune dead code and refresh stale docs 
- Delete 4 empty, unreferenced orphan scripts in dev/scripts/
  (find_databricks_workspace, scrape_bls_data, generate_bls_express,
  test_databricks_connection).
- Remove the unconditionally-skipped TestFixFields.test_runs in
  tests/zot/test_ops.py — fix_fields is already exercised end-to-end by
  test_ops_deep.py::test_fix_fields_remap_and_delete (which builds the
  itemTypeFieldsCombined view the stub said it needed).
- Remove the empty .github/workflows/ orphan left from the Gitea move
  (CI lives entirely in .gitea/workflows/).
- Drop two dead locals in dev/scripts (F841): an unused authors_section
  loop and a `tags` assignment immediately overwritten by fresh_tags.
- docs/docs/intro.md listed only 9 of 17 modules — sync the packages
  table and src/ tree with the actual module set.
2026-07-08 10:12:24 -04:00

432 lines
13 KiB
Python
Raw Blame History

"""Build the Skin Substitutes evidence base in bib.sqlite.
Creates collection hierarchy, assigns items to sub-collections,
enriches PubMed articles with conflict-of-interest and funding
analysis, and verifies tag schema coverage.
Addresses issues #244 (Zotero evidence base) and #237 (partial —
COI/funding enrichment of PubMed results).
Usage:
uv run python dev/scripts/build_skin_subs_evidence_base.py
uv run python dev/scripts/build_skin_subs_evidence_base.py --dry-run
"""
from __future__ import annotations
import argparse
import re
from datetime import datetime
from bib.store import Store
# ---------------------------------------------------------------------------
# Collection hierarchy for skin-subs research
# ---------------------------------------------------------------------------
SKIN_SUBS_COLLECTIONS = {
"Skin Substitutes": {
"Clinical Evidence": {
"RCTs": {},
"Systematic Reviews": {},
"Meta-Analyses": {},
"Observational Studies": {},
},
"CMS Policy": {
"Final Rules (OPPS/PFS)": {},
"Benefit Policy Manuals": {},
"ASP Pricing Files": {},
},
"OIG Reports": {},
"GAO & MedPAC": {},
"Enforcement": {
"DOJ Press Releases": {},
"Court Filings": {},
},
"MAC LCDs": {},
"Market Data": {},
"Industry & Societies": {},
"Cost-Effectiveness": {},
"Fraud & Abuse Literature": {},
},
}
# ---------------------------------------------------------------------------
# Known manufacturer names for COI detection
# ---------------------------------------------------------------------------
MANUFACTURERS = [
"organogenesis",
"mimedx",
"smith nephew",
"smith & nephew",
"integra",
"solsys",
"amnioexcel",
"derma sciences",
"healthpoint",
"shire",
"acelity",
"kci",
"3m",
"molnlycke",
"medline",
"hollister",
"coloplast",
"stryker",
"zimmer biomet",
"wright medical",
"solventum",
"apria",
"anika",
"musculoskeletal transplant",
"surmodics",
"tissue regenix",
"nuo therapeutics",
"sanara medtech",
"kerecis",
"human bioprocessing",
"alphatec",
"biosig technologies",
]
# Brand names that indicate manufacturer-linked studies
BRAND_NAMES = [
"apligraf",
"dermagraft",
"epifix",
"grafix",
"amnioexcel",
"dermacell",
"oasis",
"primatrix",
"integra",
"graftjacket",
"dermapure",
"affinity",
"biovance",
"cytal",
"endoform",
"kerecis omega3",
"novafix",
"puraply",
"restorigin",
"surgicraft",
"theraskin",
"amnioburn",
"clarix",
"epicord",
"genesis",
"grafix core",
"grafix prime",
"innovamatrix",
"nushield",
"stravix",
"woundex",
]
# Patterns suggesting industry funding
FUNDING_PATTERNS = [
r"funded by .{0,50}(organogenesis|mimedx|smith|integra|solsys|amnio)",
r"grant from .{0,50}(organogenesis|mimedx|smith|integra|solsys)",
r"financial support.{0,50}(organogenesis|mimedx|smith|integra)",
r"supported by .{0,50}(organogenesis|mimedx|smith|integra|solsys)",
r"sponsored by .{0,50}(organogenesis|mimedx|smith|integra|solsys)",
r"employee of .{0,50}(organogenesis|mimedx|smith|integra|solsys)",
r"consultant.{0,30}(organogenesis|mimedx|smith|integra|solsys)",
r"speaker.{0,30}(organogenesis|mimedx|smith|integra|solsys)",
r"advisory board.{0,30}(organogenesis|mimedx|smith|integra|solsys)",
r"honorari.{0,30}(organogenesis|mimedx|smith|integra|solsys)",
r"conflict.{0,50}interest",
r"disclosur.{0,80}(stock|equity|consult|employ|honorar|speaker|grant)",
]
# Patterns suggesting independent/skeptical perspective
SKEPTICAL_PATTERNS = [
r"no (significant )?difference",
r"lack.{0,20}evidence",
r"insufficient evidence",
r"low.{0,15}quality evidence",
r"limited evidence",
r"no.{0,15}superiority",
r"similar (outcome|result|efficacy|effectiveness)",
r"(waste|fraud|abuse|overutiliz|unnecessary|inappropriate)",
r"(overspend|excessive.{0,15}cost|cost concern)",
r"(marketing|promotional|commercial bias)",
]
# ---------------------------------------------------------------------------
# Collection assignment rules
# ---------------------------------------------------------------------------
def assign_collection(tags: list[str], title: str, abstract: str) -> str:
"""Return the most specific sub-collection name for an item."""
tag_set = set(tags)
# Grey literature — by source tag
if "source:oig" in tag_set:
return "OIG Reports"
if "source:gao" in tag_set or "source:medpac" in tag_set:
return "GAO & MedPAC"
if "source:doj" in tag_set:
return "DOJ Press Releases"
if "source:court" in tag_set:
return "Court Filings"
if "source:mac-lcd" in tag_set:
return "MAC LCDs"
if "source:industry" in tag_set:
return "Industry & Societies"
if "source:cms" in tag_set:
for t in tag_set:
if t.startswith("type:rule"):
return "Final Rules (OPPS/PFS)"
if t.startswith("type:manual"):
return "Benefit Policy Manuals"
return "CMS Policy"
# PubMed — by type tag
if "type:meta-analysis" in tag_set:
return "Meta-Analyses"
if "type:review" in tag_set:
return "Systematic Reviews"
if "type:rct" in tag_set:
return "RCTs"
if "type:economic" in tag_set:
return "Cost-Effectiveness"
if "type:fraud" in tag_set:
return "Fraud & Abuse Literature"
if "type:clinical" in tag_set:
return "Observational Studies"
return "Skin Substitutes" # fallback to root
def detect_coi(extra: str, abstract: str, title: str) -> list[str]:
"""Detect conflict-of-interest indicators, return enrichment tags."""
tags: list[str] = []
text = f"{extra}\n{abstract}\n{title}".lower()
# Industry funding patterns
for pattern in FUNDING_PATTERNS:
if re.search(pattern, text, re.IGNORECASE):
tags.append("coi:industry-linked")
break
# Single-product studies (often manufacturer-funded)
brand_count = sum(1 for b in BRAND_NAMES if b in text)
if brand_count == 1:
tags.append("coi:single-product")
# Skeptical / critical perspective
for pattern in SKEPTICAL_PATTERNS:
if re.search(pattern, text, re.IGNORECASE):
tags.append("stance:skeptical")
break
# Pro-product sentiment (positive claims in title)
title_lower = title.lower()
pro_patterns = [
r"(effective|superior|promising|excellent|favorable|beneficial)",
r"(accelerat|improv|enhanc|advanc).{0,20}(heal|wound|outcome)",
r"(novel|innovative|breakthrough).{0,20}(treatment|therapy|approach)",
]
for pattern in pro_patterns:
if re.search(pattern, title_lower):
tags.append("stance:favorable")
break
return tags
# ---------------------------------------------------------------------------
# Main
# ---------------------------------------------------------------------------
def main() -> None:
parser = argparse.ArgumentParser(description="Build skin-subs evidence base")
parser.add_argument("--dry-run", action="store_true")
args = parser.parse_args()
print("=" * 70)
print("Skin Substitutes Evidence Base Builder")
print(f"Date: {datetime.now().strftime('%Y-%m-%d %H:%M')}")
print("=" * 70)
store = Store()
con = store._con()
# --- Step 1: Create collection hierarchy ---
print("\n--- Step 1: Creating collection hierarchy ---")
col_map = store.ensure_collections(SKIN_SUBS_COLLECTIONS)
for name, key in sorted(col_map.items()):
print(f" {key} {name}")
print(f" Total collections created/verified: {len(col_map)}")
# --- Step 2: Load all skin-subs items ---
print("\n--- Step 2: Loading skin-subs items ---")
rows = con.execute(
"""SELECT DISTINCT i.id, i.key, i.title, i.abstract, i.extra
FROM items i
JOIN item_tags it ON i.id = it.item_id
JOIN tags t ON it.tag_id = t.id
WHERE t.name = 'module:skin-subs'"""
).fetchall()
print(f" Total items: {len(rows)}")
# Pre-load tags for each item
item_tags: dict[int, list[str]] = {}
tag_rows = con.execute(
"""SELECT it.item_id, t.name
FROM item_tags it
JOIN tags t ON it.tag_id = t.id
WHERE it.item_id IN (
SELECT DISTINCT i.id FROM items i
JOIN item_tags it2 ON i.id = it2.item_id
JOIN tags t2 ON it2.tag_id = t2.id
WHERE t2.name = 'module:skin-subs'
)"""
).fetchall()
for tr in tag_rows:
item_tags.setdefault(tr["item_id"], []).append(tr["name"])
# --- Step 3: Assign to collections ---
print("\n--- Step 3: Assigning items to collections ---")
collection_counts: dict[str, int] = {}
assignments: list[tuple[int, str]] = [] # (item_id, collection_key)
for row in rows:
tags = item_tags.get(row["id"], [])
col_name = assign_collection(tags, row["title"] or "", row["abstract"] or "")
if col_name in col_map:
assignments.append((row["id"], col_map[col_name]))
collection_counts[col_name] = collection_counts.get(col_name, 0) + 1
print(" Assignment distribution:")
for name, count in sorted(collection_counts.items(), key=lambda x: -x[1]):
print(f" {name:30s}: {count:>5}")
# --- Step 4: COI / funding enrichment ---
print("\n--- Step 4: COI / funding / stance enrichment ---")
coi_tags_to_add: dict[int, list[str]] = {}
coi_counts: dict[str, int] = {}
for row in rows:
tags = item_tags.get(row["id"], [])
# Only enrich PubMed items
if "source:pubmed" not in tags:
continue
new_tags = detect_coi(
row["extra"] or "", row["abstract"] or "", row["title"] or ""
)
if new_tags:
coi_tags_to_add[row["id"]] = new_tags
for t in new_tags:
coi_counts[t] = coi_counts.get(t, 0) + 1
print(" Enrichment tag counts:")
for tag, count in sorted(coi_counts.items(), key=lambda x: -x[1]):
print(f" {tag:30s}: {count:>5}")
print(f" Items enriched: {len(coi_tags_to_add)} / {len(rows)}")
if args.dry_run:
print("\n[DRY RUN] Skipping writes")
store.close()
return
# --- Step 5: Write collection assignments ---
print("\n--- Step 5: Writing collection assignments ---")
for item_id, col_key in assignments:
col_row = con.execute(
"SELECT id FROM collections WHERE key = ?", (col_key,)
).fetchone()
if col_row:
con.execute(
"INSERT OR IGNORE INTO collection_items "
"(collection_id, item_id) VALUES (?, ?)",
(col_row["id"], item_id),
)
con.commit()
print(f" Assigned {len(assignments)} items to collections")
# --- Step 6: Write COI enrichment tags ---
print("\n--- Step 6: Writing COI enrichment tags ---")
total_tags_added = 0
for item_id, new_tags in coi_tags_to_add.items():
for tag in new_tags:
tag_id = store._ensure_tag(tag)
con.execute(
"INSERT OR IGNORE INTO item_tags (item_id, tag_id) VALUES (?, ?)",
(item_id, tag_id),
)
total_tags_added += 1
con.commit()
print(f" Added {total_tags_added} enrichment tags")
# --- Step 7: Verify ---
print("\n--- Step 7: Verification ---")
# Tag schema coverage
expected_tags = [
"module:skin-subs",
"source:pubmed",
"source:oig",
"source:cms",
"source:court",
"source:doj",
"source:gao",
"source:medpac",
"source:mac-lcd",
"source:industry",
"type:clinical",
"type:economic",
"type:fraud",
"type:rct",
"type:review",
"type:meta-analysis",
"type:report",
"type:rule",
"type:filing",
"type:lcd",
"type:position",
"type:press-release",
]
for tag in expected_tags:
count = con.execute(
"SELECT count(*) FROM item_tags it "
"JOIN tags t ON it.tag_id = t.id WHERE t.name = ?",
(tag,),
).fetchone()[0]
status = "OK" if count > 0 else "MISSING"
print(f" {status:7s} {tag:30s}: {count:>5}")
# Collection item counts
print("\n Collection item counts:")
for name, key in sorted(col_map.items()):
count = con.execute(
"SELECT count(*) FROM collection_items ci "
"JOIN collections c ON ci.collection_id = c.id "
"WHERE c.key = ?",
(key,),
).fetchone()[0]
if count > 0:
print(f" {name:30s}: {count:>5}")
# Total
total = con.execute(
"""SELECT count(DISTINCT i.id) FROM items i
JOIN item_tags it ON i.id = it.item_id
JOIN tags t ON it.tag_id = t.id
WHERE t.name = 'module:skin-subs'"""
).fetchone()[0]
print(f"\n Total module:skin-subs items: {total}")
store.close()
print("\nDone.")
if __name__ == "__main__":
main()
Reference in New Issue View Git Blame Copy Permalink