diff --git a/src/lead/explore/app.py b/src/lead/explore/app.py index a4052ed..0a89df9 100644 --- a/src/lead/explore/app.py +++ b/src/lead/explore/app.py @@ -36,9 +36,10 @@ def main() -> None: st.page_link(item, label=item.title or "", icon=item.icon) st.link_button("OSF registration", "https://osf.io/8wq3m/") st.caption(views.db_path()) - if st.button("Reconnect"): - views.connection.clear() - st.rerun() + st.caption( + "The mirror is opened for this page and closed when the page " + "finishes, so a load can take the write lock between visits." + ) page.run() diff --git a/src/lead/explore/views.py b/src/lead/explore/views.py index ff17d8b..65c8b38 100644 --- a/src/lead/explore/views.py +++ b/src/lead/explore/views.py @@ -40,19 +40,25 @@ def db_path() -> str: return os.environ.get("LEAD_PUBLIC_DB", DEFAULT_DB) -@st.cache_resource(show_spinner=False) -def connection(path: str): - return open_mirror(path) - - def _open(): + """Open the mirror for this page run. + + A read-only DuckDB connection still holds the file lock. Caching it + for the life of the process blocks ``stack lead load``. The caller + closes the connection before returning. + """ path = db_path() try: - return connection(path), path, None + return open_mirror(path), path, None except Exception as exc: return None, path, exc +def _close(con) -> None: + if con is not None: + con.close() + + def _show_error(path: str, exc: BaseException) -> None: text = str(exc) if "lock" in text.lower(): @@ -86,22 +92,32 @@ def _files_frame(rows) -> pd.DataFrame: def writeup() -> None: brand.title("Public data mirror", "lead · evidence chassis") con, path, err = _open() - st.caption(path) - if err is not None or con is None: - _show_error(path, err or FileNotFoundError(path)) - return - for section in build(con): - st.subheader(section.heading) - for paragraph in section.body: - st.markdown(paragraph) - if section.rows: - frame = pd.DataFrame(section.rows, columns=list(section.headers)) - st.dataframe(frame, hide_index=True, width="stretch") + try: + st.caption(path) + if err is not None or con is None: + _show_error(path, err or FileNotFoundError(path)) + return + for section in build(con): + st.subheader(section.heading) + for paragraph in section.body: + st.markdown(paragraph) + if section.rows: + frame = pd.DataFrame(section.rows, columns=list(section.headers)) + st.dataframe(frame, hide_index=True, width="stretch") + finally: + _close(con) def coverage() -> None: brand.title("Coverage", "lead · load log") con, path, err = _open() + try: + _coverage(con, path, err) + finally: + _close(con) + + +def _coverage(con, path: str, err) -> None: st.caption(path) if err is not None or con is None: _show_error(path, err or FileNotFoundError(path)) @@ -132,6 +148,13 @@ def coverage() -> None: def browse() -> None: brand.title("Tables", "lead · one table") con, path, err = _open() + try: + _browse(con, path, err) + finally: + _close(con) + + +def _browse(con, path: str, err) -> None: st.caption(path) if err is not None or con is None: _show_error(path, err or FileNotFoundError(path)) @@ -210,6 +233,13 @@ def browse() -> None: def place() -> None: brand.title("Place", "lead · one geography") con, path, err = _open() + try: + _place(con, path, err) + finally: + _close(con) + + +def _place(con, path: str, err) -> None: st.caption(path) if err is not None or con is None: _show_error(path, err or FileNotFoundError(path)) @@ -265,6 +295,13 @@ def place() -> None: def series() -> None: brand.title("Series", "lead · one measure over time") con, path, err = _open() + try: + _series(con, path, err) + finally: + _close(con) + + +def _series(con, path: str, err) -> None: st.caption(path) if err is not None or con is None: _show_error(path, err or FileNotFoundError(path)) @@ -334,6 +371,13 @@ def series() -> None: def reader() -> None: brand.title("SQL", "lead · one read") con, path, err = _open() + try: + _reader(con, path, err) + finally: + _close(con) + + +def _reader(con, path: str, err) -> None: st.caption(path) if err is not None or con is None: _show_error(path, err or FileNotFoundError(path)) diff --git a/src/lead/explore/writeup.py b/src/lead/explore/writeup.py index 550f907e..2870199 100644 --- a/src/lead/explore/writeup.py +++ b/src/lead/explore/writeup.py @@ -6,6 +6,7 @@ how a row was built. Every count is computed from the open connection. from __future__ import annotations +import re from dataclasses import dataclass from lead.explore.catalog import FileRow, TableCard, latest_files, status_counts, tables @@ -208,9 +209,10 @@ def _mirror(files: tuple[FileRow, ...], cards: tuple[TableCard, ...]) -> Section "This is the local DuckDB mirror of public files for the LEAD " "evidence chassis. `stack lead load` reads files that are already " "on disk, from the enrichment manifest and from the strand-5 " - "inventory, and writes them here. The app opens that file read-only. " - "It does not download, it does not publish, and it does not write " - "Unity Catalog.", + "inventory, and writes them here. The app opens that file read-only " + "for the page and closes it afterward, so a load can take the " + "write lock between visits. It does not download, it does not " + "publish, and it does not write Unity Catalog.", state, "A row count in this write-up is the number the loader recorded " "for the latest successful read of each file. Opening this page " @@ -241,13 +243,14 @@ def _recorded(counts: dict[str, int], loaded_rows: int, sources: int) -> Section ( f"{_n(loaded_n, 'file')} loaded, {loaded_rows:,} rows recorded, " f"across {_n(sources, 'source id')}.", - "Skipped is a file the loader refused on purpose: the manifest " - "kind was not data, the member was a program or a readme, or the " - "same bytes were already stored from a loose file. Absent is a " - "file the inventory marked missing before any read, including a " - "404 and FRED. A hash mismatch is recorded and that file is not " - "inserted. A file the reader could not open is not in this log: " - "the load command prints it as unreadable, and a later run retries it.", + "Skipped is not a missing dataset. The loader refused the member " + "because the observation file is already stored, or because the " + "member is a dictionary, a guide, a sample, or not a table. " + "Absent means the inventory had no file on disk before any read: " + "a 404, or a host that did not answer. A hash mismatch is recorded " + "and that file is not inserted. A file the reader could not open " + "is not in this log: the load command prints it as unreadable, " + "and a later run retries it.", ), ("Status", "Files"), rows, @@ -347,6 +350,96 @@ def _geography() -> Section: ) +_SKIP_WHY = { + "kind:dictionary": "a data dictionary; the data file is loaded separately", + "kind:reference": "a methods paper or guide", + "kind:sample": "a sample of a file that is loaded in full", + "not_tabular": "documentation or an archive with no table", + "duplicate": "the same extract is already loaded from another file", + "http_22": "the fetch failed and another copy is already loaded", + "covered_by_manifest": "the same file is already loaded from the manifest", +} + + +def _year_phrase(years: list[int]) -> str: + if not years: + return "no year in the path" + ordered = sorted(set(years)) + runs: list[tuple[int, int]] = [] + start = previous = ordered[0] + for year in ordered[1:]: + if year == previous + 1: + previous = year + continue + runs.append((start, previous)) + start = previous = year + runs.append((start, previous)) + return ", ".join( + str(left) if left == right else f"{left}–{right}" for left, right in runs + ) + + +def _skip_line(skipped: dict[str, int]) -> str: + if not skipped: + return "No file's latest status is skipped." + parts = [] + for reason, count in sorted(skipped.items()): + why = _SKIP_WHY.get(reason) + parts.append(f"{reason} {count} ({why})" if why else f"{reason} {count}") + return "Skipped is not missing data. " + ", ".join(parts) + "." + + +def _qcew_line(absent: list[FileRow]) -> str: + qcew = [row for row in absent if row.source_id == "qcew"] + if not qcew: + return "" + years = [ + int(year) for row in qcew for year in re.findall(r"(?:19|20)\d{2}", row.path) + ] + reasons = ", ".join(sorted({row.reason or "unspecified" for row in qcew})) + return ( + f"{_n(len(qcew), 'absent QCEW file')} " + f"({_year_phrase(years)}, {reasons}). Nothing was stored: the URL " + "404'd or the annual file was not published yet. Loaded QCEW starts " + "at 1990. These rows are not years the loader dropped." + ) + + +def _fred_line(files: tuple[FileRow, ...], cards: tuple[TableCard, ...]) -> str: + fred_rows = [row for row in files if row.source_id == "fred"] + absent = [row for row in fred_rows if row.status == "absent"] + names = {card.qualified for card in cards} + has_econ = "econ.fred" in names + has_public = "lead_public.fred" in names + if absent and not has_econ and not has_public: + why = absent[-1].reason or "unreachable" + return ( + "FRED is absent. The log records the graph URL as " + f"{why}, and this file has no `econ.fred` table and no " + "`lead_public.fred` table. The market series live in the aco " + "database, not here. `stack lead load` reads a CSV that is " + "already on disk; it does not delete this URL row." + ) + if absent and (has_econ or has_public): + which = "econ.fred" if has_econ else "lead_public.fred" + why = absent[-1].reason or "unreachable" + return ( + "The FRED graph URL is still marked absent " + f"({why}). `{which}` holds the CSV loaded after that fetch. " + "The old URL row stays in the log, so the absent count still " + "includes it." + ) + if has_econ or has_public: + which = "econ.fred" if has_econ else "lead_public.fred" + return f"`{which}` is present in this file." + if fred_rows: + return ( + "The log has a FRED row whose latest status is " + f"{fred_rows[-1].status}, reason {fred_rows[-1].reason or 'blank'}." + ) + return "The load log has no FRED row, and `econ.fred` is not in this file." + + def _gaps(files: tuple[FileRow, ...], cards: tuple[TableCard, ...]) -> Section: absent = [row for row in files if row.status == "absent"] skipped: dict[str, int] = {} @@ -354,40 +447,25 @@ def _gaps(files: tuple[FileRow, ...], cards: tuple[TableCard, ...]) -> Section: if row.status == "skipped": reason = row.reason or "(blank)" skipped[reason] = skipped.get(reason, 0) + 1 - fred_rows = [row for row in files if row.source_id == "fred"] - has_fred_table = any(card.qualified == "econ.fred" for card in cards) - if any(row.status == "absent" for row in fred_rows) and not has_fred_table: - fred = ( - "FRED is absent. The log records it that way, and there is no " - "`econ.fred` table. Those series were not fetched." - ) - elif has_fred_table: - fred = "`econ.fred` is present in this file." - elif fred_rows: - fred = ( - "The log has a FRED row whose latest status is " - f"{fred_rows[-1].status}, reason {fred_rows[-1].reason or 'blank'}." - ) - else: - fred = "The load log has no FRED row, and `econ.fred` is not in this file." rows = tuple( (row.source_id, _base(row.path) or row.release, row.reason or "") for row in absent ) - skip_line = ( - "Skipped files by reason: " - + ", ".join(f"{reason} {count}" for reason, count in sorted(skipped.items())) - + "." - if skipped - else "No file's latest status is skipped." - ) + body = [ + _fred_line(files, cards), + ( + f"{len(absent):,} files have latest status absent. " + "Absent means the inventory had no file on disk. It is not a " + "table the loader chose to drop." + ), + _skip_line(skipped), + ] + qcew = _qcew_line(absent) + if qcew: + body.insert(1, qcew) return Section( "Gaps", - ( - fred, - f"{len(absent):,} files have latest status absent.", - skip_line, - ), + tuple(body), ("Source", "File", "Reason"), rows, ) diff --git a/tests/lead/test_explore.py b/tests/lead/test_explore.py index 86b6e52..5bcdae2 100644 --- a/tests/lead/test_explore.py +++ b/tests/lead/test_explore.py @@ -307,6 +307,8 @@ def test_the_writeup_counts_the_log_and_states_the_contract(tmp_path): assert largest.rows[0][3] == "yes" gaps = next(section for section in sections if section.heading == "Gaps") assert ("fred", "fred.csv", "unreachable") in gaps.rows + assert "not missing data" in "\n".join(gaps.body) + assert "kind:note 1" in "\n".join(gaps.body) spine = next( section for section in sections if section.heading == "How a row is keyed" ) @@ -314,6 +316,83 @@ def test_the_writeup_counts_the_log_and_states_the_contract(tmp_path): assert "000" in fips[1] +def test_gaps_name_a_dictionary_and_the_qcew_404_years(tmp_path): + con = duckdb.connect(str(tmp_path / "gaps.duckdb")) + ensure_log(con) + record( + con, + source_id="cms_dict", + release="2026", + sha256="e" * 64, + path="/data/dictionary__dict.pdf", + table="", + status="skipped", + rows=0, + reason="kind:dictionary", + actor="stack", + ) + record( + con, + source_id="qcew", + release="", + sha256="", + path="https://data.bls.gov/cew/data/files/1975/csv/1975_annual_by_industry.zip", + table="", + status="absent", + rows=0, + reason="http_404", + actor="stack", + ) + record( + con, + source_id="qcew", + release="", + sha256="", + path="https://data.bls.gov/cew/data/files/2026/csv/2026_annual_by_industry.zip", + table="", + status="absent", + rows=0, + reason="http_404", + actor="stack", + ) + record( + con, + source_id="fred", + release="UNRATE.csv", + sha256="f" * 64, + path="/data/lead/strand5/fred/UNRATE.csv", + table="lead_public.fred", + status="loaded", + rows=900, + reason="", + actor="stack", + ) + record( + con, + source_id="fred", + release="", + sha256="", + path="https://fred.stlouisfed.org/graph/fredgraph.csv?id=UNRATE", + table="", + status="absent", + rows=0, + reason="unreachable", + actor="stack", + ) + con.execute("CREATE SCHEMA IF NOT EXISTS lead_public") + con.execute( + "CREATE TABLE lead_public.fred (observation_date VARCHAR, UNRATE VARCHAR)" + ) + gaps = next(section for section in writeup.build(con) if section.heading == "Gaps") + text = "\n".join(gaps.body) + assert "data dictionary" in text + assert "1975, 2026" in text + assert "Loaded QCEW starts at 1990" in text + assert "lead_public.fred" in text + assert "still marked absent" in text + con.close() + + def test_writeup_page_renders_the_recorded_count(tmp_path, monkeypatch): con = _mirror(tmp_path) path = str(tmp_path / "public.duckdb") diff --git a/tests/lead/test_explore_pages.py b/tests/lead/test_explore_pages.py index c9219b0..2a23c97 100644 --- a/tests/lead/test_explore_pages.py +++ b/tests/lead/test_explore_pages.py @@ -5,6 +5,9 @@ from __future__ import annotations import sys from pathlib import Path +import pytest + +from lead.explore import catalog from tests.lead.test_explore import _mirror from zot.db import TYPE_MAP, Db from zot.schema import create_db @@ -91,6 +94,25 @@ def test_coverage_filters_the_log_and_a_missing_file_is_an_error(tmp_path, monke assert "No mirror" in _blob(missing) +def test_a_page_closes_the_mirror(tmp_path, monkeypatch): + import duckdb + + held: list = [] + monkeypatch.setenv("LEAD_PUBLIC_DB", _mirror_path(tmp_path)) + + def wrapped(path): + con = catalog.open_mirror(path) + held.append(con) + return con + + monkeypatch.setattr("lead.explore.views.open_mirror", wrapped) + at = _run("from lead.explore.views import coverage\ncoverage()\n") + assert not at.exception, at.exception + assert held + with pytest.raises(duckdb.Error): + held[0].execute("SELECT 1") + + def test_a_locked_mirror_and_any_other_open_error_are_shown(tmp_path, monkeypatch): monkeypatch.setenv("LEAD_PUBLIC_DB", str(tmp_path / "locked.duckdb")) @@ -272,7 +294,7 @@ def test_the_shell_links_osf_and_reconnects(tmp_path, monkeypatch): at = _run("from lead.explore.app import main\nmain()\n") urls = [str(getattr(item, "url", "")) for item in at.link_button] assert any("osf.io/8wq3m" in url for url in urls) or "osf.io/8wq3m" in _blob(at) - _widget(at, "button", "Reconnect").click().run() + assert "closes it when the page finishes" in _blob(at) or "write lock" in _blob(at) assert not at.exception, at.exception