stack/dev/scripts/extract_study_characteristics.py

"""Extract study characteristics table from PubMed articles in bib.sqlite.

Builds skin_subs.study_characteristics in DuckDB with structured fields
parsed from article metadata: PMID, first author, year, publication type,
journal, products mentioned, sample size (if detectable), COI/stance flags.

Addresses #235 checklist item: "Extract study characteristics table".

Usage:
    uv run python dev/scripts/extract_study_characteristics.py
"""

from __future__ import annotations

import re
from pathlib import Path

import duckdb

from bib.store import Store

ROOT = Path(__file__).resolve().parents[2]
DUCKDB_PATH = ROOT / "data" / "aco.duckdb"

# ---------------------------------------------------------------------------
# Product detection — map brand names to HCPCS + manufacturer
# ---------------------------------------------------------------------------

PRODUCTS = {
    "apligraf": ("Q4101", "Organogenesis"),
    "oasis wound matrix": ("Q4102", "Smith & Nephew"),
    "oasis burn matrix": ("Q4103", "Smith & Nephew"),
    "integra": ("Q4104/Q4105", "Integra LifeSciences"),
    "dermagraft": ("Q4106", "Organogenesis"),
    "graftjacket": ("Q4107", "Wright Medical"),
    "dermacell": ("Q4122", "LifeNet Health"),
    "omnigraft": ("Q4125", "Integra LifeSciences"),
    "amnioexcel": ("Q4126", "Derma Sciences"),
    "talymed": ("Q4127", "Marine Polymer Technologies"),
    "grafix core": ("Q4132", "Osiris/Smith & Nephew"),
    "grafix prime": ("Q4133", "Osiris/Smith & Nephew"),
    "grafix": ("Q4132/Q4133", "Osiris/Smith & Nephew"),
    "hmatrix": ("Q4134", "Bacterin"),
    "mediskin": ("Q4135", "MediWound"),
    "epifix": ("Q4186", "MiMedx"),
    "epicord": ("Q4187", "MiMedx"),
    "amnioband": ("Q4151", "MTF Biologics"),
    "biovance": ("Q4154", "Celularity"),
    "neox": ("Q4148", "Amnio Technology"),
    "clarix": ("Q4148", "Amnio Technology"),
    "dermapure": ("Q4152", "Tissue Regenix"),
    "affinity": ("Q4159", "Organogenesis"),
    "nushield": ("Q4160", "Organogenesis"),
    "novafix": ("Q4194", "Organogenesis"),
    "surgicraft": ("Q4162", "Solsys Medical"),
    "puraply": ("Q4195", "Organogenesis"),
    "cytal": ("Q4189", "Acell"),
    "endoform": ("Q4163", "Aroa Biosurgery"),
    "kerecis": ("Q4158", "Kerecis"),
    "theraskin": ("Q4121", "Solsys Medical"),
    "stravix": ("Q4193", "Osiris"),
    "primatrix": ("Q4110", "TEI Biosciences"),
    "alloderm": ("Q4116", "LifeCell/Allergan"),
    "matristem": ("Q4118", "Acell"),
    "restorigin": ("Q4191", "Acell"),
    "woundex": ("Q4196", "Skye Biologics"),
}

# Patterns for sample size extraction
SAMPLE_SIZE_PATTERNS = [
    r"(?:n\s*=\s*)(\d{2,5})",
    r"(\d{2,5})\s*(?:patients|subjects|participants|wounds|ulcers)",
    r"(?:enrolled|included|randomized|recruited)\s+(\d{2,5})",
    r"(?:sample size|sample of)\s+(\d{2,5})",
    r"(\d{2,5})\s*(?:were (?:enrolled|included|randomized))",
]


def extract_products(text: str) -> list[str]:
    """Find product brand names in text, return sorted unique list."""
    text_lower = text.lower()
    found = set()
    for brand in PRODUCTS:
        if brand in text_lower:
            found.add(brand)
    # Deduplicate: if "grafix core" found, don't also add "grafix"
    if "grafix core" in found or "grafix prime" in found:
        found.discard("grafix")
    return sorted(found)


def extract_sample_size(text: str) -> int | None:
    """Try to extract sample size from abstract text."""
    for pattern in SAMPLE_SIZE_PATTERNS:
        m = re.search(pattern, text, re.IGNORECASE)
        if m:
            n = int(m.group(1))
            if 10 <= n <= 50000:  # sanity bounds
                return n
    return None


def extract_first_author(extra: str) -> str:
    """Extract first author surname from extra metadata."""
    for line in extra.split("\n"):
        if line.startswith("Authors:"):
            authors = line[8:].strip()
            if authors:
                first = authors.split(";")[0].strip()
                # "LastName ForeName" → "LastName"
                parts = first.split()
                if parts:
                    return parts[0]
    return ""


def classify_pub_type(tags: list[str], pub_types_str: str) -> str:
    """Classify into RCT, review, meta-analysis, or other."""
    tag_set = set(tags)
    if "type:meta-analysis" in tag_set:
        return "meta-analysis"
    if "type:rct" in tag_set:
        return "rct"
    if "type:review" in tag_set:
        return "review"
    pt_lower = pub_types_str.lower()
    if "randomized" in pt_lower or "clinical trial" in pt_lower:
        return "rct"
    if "meta-analysis" in pt_lower:
        return "meta-analysis"
    if "review" in pt_lower:
        return "review"
    if "case report" in pt_lower:
        return "case-report"
    if "observational" in pt_lower or "cohort" in pt_lower:
        return "observational"
    return "other"


def extract_journal(extra: str) -> str:
    """Extract journal name from extra metadata."""
    for line in extra.split("\n"):
        if line.startswith("Journal:"):
            return line[8:].strip()
    return ""


def extract_pmid(extra: str) -> str:
    """Extract PMID from extra metadata."""
    for line in extra.split("\n"):
        if line.startswith("PMID:"):
            return line[5:].strip()
    return ""


def extract_pub_types(extra: str) -> str:
    """Extract PubTypes string from extra."""
    for line in extra.split("\n"):
        if line.startswith("PubTypes:"):
            return line[9:].strip()
    return ""


def main() -> None:
    print("Extracting study characteristics from bib.sqlite ...")

    store = Store()
    con = store._con()

    # Load all PubMed skin-subs items with their tags
    rows = con.execute(
        """SELECT DISTINCT i.id, i.key, i.title, i.abstract, i.extra,
                  i.date_published, i.institution, i.url
           FROM items i
           JOIN item_tags it ON i.id = it.item_id
           JOIN tags t ON it.tag_id = t.id
           WHERE t.name = 'module:skin-subs'
             AND i.url LIKE '%pubmed%'"""
    ).fetchall()
    print(f"  PubMed articles: {len(rows)}")

    # Pre-load tags
    item_tags: dict[int, list[str]] = {}
    tag_rows = con.execute(
        """SELECT it.item_id, t.name FROM item_tags it
           JOIN tags t ON it.tag_id = t.id
           WHERE it.item_id IN (
               SELECT DISTINCT i.id FROM items i
               JOIN item_tags it2 ON i.id = it2.item_id
               JOIN tags t2 ON it2.tag_id = t2.id
               WHERE t2.name = 'module:skin-subs'
                 AND i.url LIKE '%pubmed%'
           )"""
    ).fetchall()
    for tr in tag_rows:
        item_tags.setdefault(tr["item_id"], []).append(tr["name"])

    # Build characteristics
    chars: list[dict] = []
    for row in rows:
        tags = item_tags.get(row["id"], [])
        extra = row["extra"] or ""
        abstract = row["abstract"] or ""
        title = row["title"] or ""
        text = f"{title} {abstract} {extra}"

        pmid = extract_pmid(extra)
        first_author = extract_first_author(extra)
        year = row["date_published"] or ""
        pub_types_str = extract_pub_types(extra)
        pub_type = classify_pub_type(tags, pub_types_str)
        journal = extract_journal(extra)
        products = extract_products(text)
        sample_size = extract_sample_size(abstract)

        # COI/stance flags
        tag_set = set(tags)
        is_industry_linked = "coi:industry-linked" in tag_set
        is_single_product = "coi:single-product" in tag_set
        stance = ""
        if "stance:skeptical" in tag_set:
            stance = "skeptical"
        elif "stance:favorable" in tag_set:
            stance = "favorable"

        # Domain tags
        domains = []
        if "type:clinical" in tag_set:
            domains.append("clinical")
        if "type:economic" in tag_set:
            domains.append("economic")
        if "type:fraud" in tag_set:
            domains.append("fraud")

        chars.append(
            {
                "pmid": pmid,
                "bib_key": row["key"],
                "first_author": first_author,
                "year": year,
                "pub_type": pub_type,
                "journal": journal,
                "title": title[:200],
                "products": "; ".join(products) if products else "",
                "product_count": len(products),
                "sample_size": sample_size,
                "is_industry_linked": is_industry_linked,
                "is_single_product": is_single_product,
                "stance": stance,
                "domains": "; ".join(domains),
                "url": row["url"] or "",
            }
        )

    store.close()

    # Summary stats
    print(f"\n  Study characteristics extracted: {len(chars)}")
    type_counts: dict[str, int] = {}
    for c in chars:
        type_counts[c["pub_type"]] = type_counts.get(c["pub_type"], 0) + 1
    print("  By pub type:")
    for t, ct in sorted(type_counts.items(), key=lambda x: -x[1]):
        print(f"    {t:20s}: {ct:>5}")

    with_products = sum(1 for c in chars if c["product_count"] > 0)
    with_sample = sum(1 for c in chars if c["sample_size"] is not None)
    print(f"  With product mentions: {with_products}")
    print(f"  With sample size: {with_sample}")

    # Top products
    product_counts: dict[str, int] = {}
    for c in chars:
        for p in c["products"].split("; ") if c["products"] else []:
            product_counts[p] = product_counts.get(p, 0) + 1
    print("\n  Top 15 products mentioned:")
    for p, ct in sorted(product_counts.items(), key=lambda x: -x[1])[:15]:
        hcpcs, mfr = PRODUCTS.get(p, ("?", "?"))
        print(f"    {p:25s} ({hcpcs:12s} {mfr:25s}): {ct:>4}")

    # Load into DuckDB
    print(f"\nLoading into DuckDB at {DUCKDB_PATH} ...")
    ddb = duckdb.connect(str(DUCKDB_PATH))
    ddb.execute("CREATE SCHEMA IF NOT EXISTS skin_subs")
    ddb.execute("DROP TABLE IF EXISTS skin_subs.study_characteristics")

    # Register Python list as table
    import pyarrow as pa

    schema = pa.schema(
        [
            ("pmid", pa.string()),
            ("bib_key", pa.string()),
            ("first_author", pa.string()),
            ("year", pa.string()),
            ("pub_type", pa.string()),
            ("journal", pa.string()),
            ("title", pa.string()),
            ("products", pa.string()),
            ("product_count", pa.int32()),
            ("sample_size", pa.int32()),
            ("is_industry_linked", pa.bool_()),
            ("is_single_product", pa.bool_()),
            ("stance", pa.string()),
            ("domains", pa.string()),
            ("url", pa.string()),
        ]
    )

    arrays = [
        pa.array([c["pmid"] for c in chars]),
        pa.array([c["bib_key"] for c in chars]),
        pa.array([c["first_author"] for c in chars]),
        pa.array([c["year"] for c in chars]),
        pa.array([c["pub_type"] for c in chars]),
        pa.array([c["journal"] for c in chars]),
        pa.array([c["title"] for c in chars]),
        pa.array([c["products"] for c in chars]),
        pa.array([c["product_count"] for c in chars]),
        pa.array([c["sample_size"] for c in chars]),
        pa.array([c["is_industry_linked"] for c in chars]),
        pa.array([c["is_single_product"] for c in chars]),
        pa.array([c["stance"] for c in chars]),
        pa.array([c["domains"] for c in chars]),
        pa.array([c["url"] for c in chars]),
    ]
    arrow_tbl = pa.table(dict(zip([f.name for f in schema], arrays)), schema=schema)

    ddb.register("arrow_tbl", arrow_tbl)
    ddb.execute(
        "CREATE TABLE skin_subs.study_characteristics AS SELECT * FROM arrow_tbl"
    )
    count = ddb.execute(
        "SELECT count(*) FROM skin_subs.study_characteristics"
    ).fetchone()[0]
    print(f"  Loaded {count} rows into skin_subs.study_characteristics")

    # Quick validation queries
    print("\n  Validation:")
    for q, label in [
        (
            "SELECT pub_type, count(*) c FROM skin_subs.study_characteristics GROUP BY 1 ORDER BY c DESC LIMIT 5",
            "pub_type",
        ),
        (
            "SELECT count(*) FROM skin_subs.study_characteristics WHERE product_count > 0",
            "with_products",
        ),
        (
            "SELECT count(*) FROM skin_subs.study_characteristics WHERE sample_size IS NOT NULL",
            "with_sample_size",
        ),
        (
            "SELECT count(*) FROM skin_subs.study_characteristics WHERE is_industry_linked",
            "industry_linked",
        ),
    ]:
        result = ddb.execute(q).fetchall()
        print(f"    {label}: {result}")

    ddb.close()
    print("\nDone.")


if __name__ == "__main__":
    main()