fix(lead): close the public mirror after each explorer page

A cached read-only DuckDB connection held the lock for the life of
the process and blocked stack lead load. Each page opens the mirror
and closes it. The write-up now says skipped files were refused on
purpose and absent files were never in the inventory.
This commit is contained in:
kert
2026-11-17 16:00:19 -05:00
parent df29e8d958
commit a8ed6dde9a
5 changed files with 283 additions and 59 deletions

View File

@@ -36,9 +36,10 @@ def main() -> None:
st.page_link(item, label=item.title or "", icon=item.icon)
st.link_button("OSF registration", "https://osf.io/8wq3m/")
st.caption(views.db_path())
if st.button("Reconnect"):
views.connection.clear()
st.rerun()
st.caption(
"The mirror is opened for this page and closed when the page "
"finishes, so a load can take the write lock between visits."
)
page.run()

View File

@@ -40,19 +40,25 @@ def db_path() -> str:
return os.environ.get("LEAD_PUBLIC_DB", DEFAULT_DB)
@st.cache_resource(show_spinner=False)
def connection(path: str):
return open_mirror(path)
def _open():
"""Open the mirror for this page run.
A read-only DuckDB connection still holds the file lock. Caching it
for the life of the process blocks ``stack lead load``. The caller
closes the connection before returning.
"""
path = db_path()
try:
return connection(path), path, None
return open_mirror(path), path, None
except Exception as exc:
return None, path, exc
def _close(con) -> None:
if con is not None:
con.close()
def _show_error(path: str, exc: BaseException) -> None:
text = str(exc)
if "lock" in text.lower():
@@ -86,22 +92,32 @@ def _files_frame(rows) -> pd.DataFrame:
def writeup() -> None:
brand.title("Public data mirror", "lead · evidence chassis")
con, path, err = _open()
st.caption(path)
if err is not None or con is None:
_show_error(path, err or FileNotFoundError(path))
return
for section in build(con):
st.subheader(section.heading)
for paragraph in section.body:
st.markdown(paragraph)
if section.rows:
frame = pd.DataFrame(section.rows, columns=list(section.headers))
st.dataframe(frame, hide_index=True, width="stretch")
try:
st.caption(path)
if err is not None or con is None:
_show_error(path, err or FileNotFoundError(path))
return
for section in build(con):
st.subheader(section.heading)
for paragraph in section.body:
st.markdown(paragraph)
if section.rows:
frame = pd.DataFrame(section.rows, columns=list(section.headers))
st.dataframe(frame, hide_index=True, width="stretch")
finally:
_close(con)
def coverage() -> None:
brand.title("Coverage", "lead · load log")
con, path, err = _open()
try:
_coverage(con, path, err)
finally:
_close(con)
def _coverage(con, path: str, err) -> None:
st.caption(path)
if err is not None or con is None:
_show_error(path, err or FileNotFoundError(path))
@@ -132,6 +148,13 @@ def coverage() -> None:
def browse() -> None:
brand.title("Tables", "lead · one table")
con, path, err = _open()
try:
_browse(con, path, err)
finally:
_close(con)
def _browse(con, path: str, err) -> None:
st.caption(path)
if err is not None or con is None:
_show_error(path, err or FileNotFoundError(path))
@@ -210,6 +233,13 @@ def browse() -> None:
def place() -> None:
brand.title("Place", "lead · one geography")
con, path, err = _open()
try:
_place(con, path, err)
finally:
_close(con)
def _place(con, path: str, err) -> None:
st.caption(path)
if err is not None or con is None:
_show_error(path, err or FileNotFoundError(path))
@@ -265,6 +295,13 @@ def place() -> None:
def series() -> None:
brand.title("Series", "lead · one measure over time")
con, path, err = _open()
try:
_series(con, path, err)
finally:
_close(con)
def _series(con, path: str, err) -> None:
st.caption(path)
if err is not None or con is None:
_show_error(path, err or FileNotFoundError(path))
@@ -334,6 +371,13 @@ def series() -> None:
def reader() -> None:
brand.title("SQL", "lead · one read")
con, path, err = _open()
try:
_reader(con, path, err)
finally:
_close(con)
def _reader(con, path: str, err) -> None:
st.caption(path)
if err is not None or con is None:
_show_error(path, err or FileNotFoundError(path))

View File

@@ -6,6 +6,7 @@ how a row was built. Every count is computed from the open connection.
from __future__ import annotations
import re
from dataclasses import dataclass
from lead.explore.catalog import FileRow, TableCard, latest_files, status_counts, tables
@@ -208,9 +209,10 @@ def _mirror(files: tuple[FileRow, ...], cards: tuple[TableCard, ...]) -> Section
"This is the local DuckDB mirror of public files for the LEAD "
"evidence chassis. `stack lead load` reads files that are already "
"on disk, from the enrichment manifest and from the strand-5 "
"inventory, and writes them here. The app opens that file read-only. "
"It does not download, it does not publish, and it does not write "
"Unity Catalog.",
"inventory, and writes them here. The app opens that file read-only "
"for the page and closes it afterward, so a load can take the "
"write lock between visits. It does not download, it does not "
"publish, and it does not write Unity Catalog.",
state,
"A row count in this write-up is the number the loader recorded "
"for the latest successful read of each file. Opening this page "
@@ -241,13 +243,14 @@ def _recorded(counts: dict[str, int], loaded_rows: int, sources: int) -> Section
(
f"{_n(loaded_n, 'file')} loaded, {loaded_rows:,} rows recorded, "
f"across {_n(sources, 'source id')}.",
"Skipped is a file the loader refused on purpose: the manifest "
"kind was not data, the member was a program or a readme, or the "
"same bytes were already stored from a loose file. Absent is a "
"file the inventory marked missing before any read, including a "
"404 and FRED. A hash mismatch is recorded and that file is not "
"inserted. A file the reader could not open is not in this log: "
"the load command prints it as unreadable, and a later run retries it.",
"Skipped is not a missing dataset. The loader refused the member "
"because the observation file is already stored, or because the "
"member is a dictionary, a guide, a sample, or not a table. "
"Absent means the inventory had no file on disk before any read: "
"a 404, or a host that did not answer. A hash mismatch is recorded "
"and that file is not inserted. A file the reader could not open "
"is not in this log: the load command prints it as unreadable, "
"and a later run retries it.",
),
("Status", "Files"),
rows,
@@ -347,6 +350,96 @@ def _geography() -> Section:
)
_SKIP_WHY = {
"kind:dictionary": "a data dictionary; the data file is loaded separately",
"kind:reference": "a methods paper or guide",
"kind:sample": "a sample of a file that is loaded in full",
"not_tabular": "documentation or an archive with no table",
"duplicate": "the same extract is already loaded from another file",
"http_22": "the fetch failed and another copy is already loaded",
"covered_by_manifest": "the same file is already loaded from the manifest",
}
def _year_phrase(years: list[int]) -> str:
if not years:
return "no year in the path"
ordered = sorted(set(years))
runs: list[tuple[int, int]] = []
start = previous = ordered[0]
for year in ordered[1:]:
if year == previous + 1:
previous = year
continue
runs.append((start, previous))
start = previous = year
runs.append((start, previous))
return ", ".join(
str(left) if left == right else f"{left}–{right}" for left, right in runs
)
def _skip_line(skipped: dict[str, int]) -> str:
if not skipped:
return "No file's latest status is skipped."
parts = []
for reason, count in sorted(skipped.items()):
why = _SKIP_WHY.get(reason)
parts.append(f"{reason} {count} ({why})" if why else f"{reason} {count}")
return "Skipped is not missing data. " + ", ".join(parts) + "."
def _qcew_line(absent: list[FileRow]) -> str:
qcew = [row for row in absent if row.source_id == "qcew"]
if not qcew:
return ""
years = [
int(year) for row in qcew for year in re.findall(r"(?:19|20)\d{2}", row.path)
]
reasons = ", ".join(sorted({row.reason or "unspecified" for row in qcew}))
return (
f"{_n(len(qcew), 'absent QCEW file')} "
f"({_year_phrase(years)}, {reasons}). Nothing was stored: the URL "
"404'd or the annual file was not published yet. Loaded QCEW starts "
"at 1990. These rows are not years the loader dropped."
)
def _fred_line(files: tuple[FileRow, ...], cards: tuple[TableCard, ...]) -> str:
fred_rows = [row for row in files if row.source_id == "fred"]
absent = [row for row in fred_rows if row.status == "absent"]
names = {card.qualified for card in cards}
has_econ = "econ.fred" in names
has_public = "lead_public.fred" in names
if absent and not has_econ and not has_public:
why = absent[-1].reason or "unreachable"
return (
"FRED is absent. The log records the graph URL as "
f"{why}, and this file has no `econ.fred` table and no "
"`lead_public.fred` table. The market series live in the aco "
"database, not here. `stack lead load` reads a CSV that is "
"already on disk; it does not delete this URL row."
)
if absent and (has_econ or has_public):
which = "econ.fred" if has_econ else "lead_public.fred"
why = absent[-1].reason or "unreachable"
return (
"The FRED graph URL is still marked absent "
f"({why}). `{which}` holds the CSV loaded after that fetch. "
"The old URL row stays in the log, so the absent count still "
"includes it."
)
if has_econ or has_public:
which = "econ.fred" if has_econ else "lead_public.fred"
return f"`{which}` is present in this file."
if fred_rows:
return (
"The log has a FRED row whose latest status is "
f"{fred_rows[-1].status}, reason {fred_rows[-1].reason or 'blank'}."
)
return "The load log has no FRED row, and `econ.fred` is not in this file."
def _gaps(files: tuple[FileRow, ...], cards: tuple[TableCard, ...]) -> Section:
absent = [row for row in files if row.status == "absent"]
skipped: dict[str, int] = {}
@@ -354,40 +447,25 @@ def _gaps(files: tuple[FileRow, ...], cards: tuple[TableCard, ...]) -> Section:
if row.status == "skipped":
reason = row.reason or "(blank)"
skipped[reason] = skipped.get(reason, 0) + 1
fred_rows = [row for row in files if row.source_id == "fred"]
has_fred_table = any(card.qualified == "econ.fred" for card in cards)
if any(row.status == "absent" for row in fred_rows) and not has_fred_table:
fred = (
"FRED is absent. The log records it that way, and there is no "
"`econ.fred` table. Those series were not fetched."
)
elif has_fred_table:
fred = "`econ.fred` is present in this file."
elif fred_rows:
fred = (
"The log has a FRED row whose latest status is "
f"{fred_rows[-1].status}, reason {fred_rows[-1].reason or 'blank'}."
)
else:
fred = "The load log has no FRED row, and `econ.fred` is not in this file."
rows = tuple(
(row.source_id, _base(row.path) or row.release, row.reason or "")
for row in absent
)
skip_line = (
"Skipped files by reason: "
+ ", ".join(f"{reason} {count}" for reason, count in sorted(skipped.items()))
+ "."
if skipped
else "No file's latest status is skipped."
)
body = [
_fred_line(files, cards),
(
f"{len(absent):,} files have latest status absent. "
"Absent means the inventory had no file on disk. It is not a "
"table the loader chose to drop."
),
_skip_line(skipped),
]
qcew = _qcew_line(absent)
if qcew:
body.insert(1, qcew)
return Section(
"Gaps",
(
fred,
f"{len(absent):,} files have latest status absent.",
skip_line,
),
tuple(body),
("Source", "File", "Reason"),
rows,
)

View File

@@ -307,6 +307,8 @@ def test_the_writeup_counts_the_log_and_states_the_contract(tmp_path):
assert largest.rows[0][3] == "yes"
gaps = next(section for section in sections if section.heading == "Gaps")
assert ("fred", "fred.csv", "unreachable") in gaps.rows
assert "not missing data" in "\n".join(gaps.body)
assert "kind:note 1" in "\n".join(gaps.body)
spine = next(
section for section in sections if section.heading == "How a row is keyed"
)
@@ -314,6 +316,83 @@ def test_the_writeup_counts_the_log_and_states_the_contract(tmp_path):
assert "000" in fips[1]
def test_gaps_name_a_dictionary_and_the_qcew_404_years(tmp_path):
con = duckdb.connect(str(tmp_path / "gaps.duckdb"))
ensure_log(con)
record(
con,
source_id="cms_dict",
release="2026",
sha256="e" * 64,
path="/data/dictionary__dict.pdf",
table="",
status="skipped",
rows=0,
reason="kind:dictionary",
actor="stack",
)
record(
con,
source_id="qcew",
release="",
sha256="",
path="https://data.bls.gov/cew/data/files/1975/csv/1975_annual_by_industry.zip",
table="",
status="absent",
rows=0,
reason="http_404",
actor="stack",
)
record(
con,
source_id="qcew",
release="",
sha256="",
path="https://data.bls.gov/cew/data/files/2026/csv/2026_annual_by_industry.zip",
table="",
status="absent",
rows=0,
reason="http_404",
actor="stack",
)
record(
con,
source_id="fred",
release="UNRATE.csv",
sha256="f" * 64,
path="/data/lead/strand5/fred/UNRATE.csv",
table="lead_public.fred",
status="loaded",
rows=900,
reason="",
actor="stack",
)
record(
con,
source_id="fred",
release="",
sha256="",
path="https://fred.stlouisfed.org/graph/fredgraph.csv?id=UNRATE",
table="",
status="absent",
rows=0,
reason="unreachable",
actor="stack",
)
con.execute("CREATE SCHEMA IF NOT EXISTS lead_public")
con.execute(
"CREATE TABLE lead_public.fred (observation_date VARCHAR, UNRATE VARCHAR)"
)
gaps = next(section for section in writeup.build(con) if section.heading == "Gaps")
text = "\n".join(gaps.body)
assert "data dictionary" in text
assert "1975, 2026" in text
assert "Loaded QCEW starts at 1990" in text
assert "lead_public.fred" in text
assert "still marked absent" in text
con.close()
def test_writeup_page_renders_the_recorded_count(tmp_path, monkeypatch):
con = _mirror(tmp_path)
path = str(tmp_path / "public.duckdb")

View File

@@ -5,6 +5,9 @@ from __future__ import annotations
import sys
from pathlib import Path
import pytest
from lead.explore import catalog
from tests.lead.test_explore import _mirror
from zot.db import TYPE_MAP, Db
from zot.schema import create_db
@@ -91,6 +94,25 @@ def test_coverage_filters_the_log_and_a_missing_file_is_an_error(tmp_path, monke
assert "No mirror" in _blob(missing)
def test_a_page_closes_the_mirror(tmp_path, monkeypatch):
import duckdb
held: list = []
monkeypatch.setenv("LEAD_PUBLIC_DB", _mirror_path(tmp_path))
def wrapped(path):
con = catalog.open_mirror(path)
held.append(con)
return con
monkeypatch.setattr("lead.explore.views.open_mirror", wrapped)
at = _run("from lead.explore.views import coverage\ncoverage()\n")
assert not at.exception, at.exception
assert held
with pytest.raises(duckdb.Error):
held[0].execute("SELECT 1")
def test_a_locked_mirror_and_any_other_open_error_are_shown(tmp_path, monkeypatch):
monkeypatch.setenv("LEAD_PUBLIC_DB", str(tmp_path / "locked.duckdb"))
@@ -272,7 +294,7 @@ def test_the_shell_links_osf_and_reconnects(tmp_path, monkeypatch):
at = _run("from lead.explore.app import main\nmain()\n")
urls = [str(getattr(item, "url", "")) for item in at.link_button]
assert any("osf.io/8wq3m" in url for url in urls) or "osf.io/8wq3m" in _blob(at)
_widget(at, "button", "Reconnect").click().run()
assert "closes it when the page finishes" in _blob(at) or "write lock" in _blob(at)
assert not at.exception, at.exception