fix(lead): close the public mirror after each explorer page
A cached read-only DuckDB connection held the lock for the life of the process and blocked stack lead load. Each page opens the mirror and closes it. The write-up now says skipped files were refused on purpose and absent files were never in the inventory.
This commit is contained in:
@@ -36,9 +36,10 @@ def main() -> None:
|
||||
st.page_link(item, label=item.title or "", icon=item.icon)
|
||||
st.link_button("OSF registration", "https://osf.io/8wq3m/")
|
||||
st.caption(views.db_path())
|
||||
if st.button("Reconnect"):
|
||||
views.connection.clear()
|
||||
st.rerun()
|
||||
st.caption(
|
||||
"The mirror is opened for this page and closed when the page "
|
||||
"finishes, so a load can take the write lock between visits."
|
||||
)
|
||||
page.run()
|
||||
|
||||
|
||||
|
||||
@@ -40,19 +40,25 @@ def db_path() -> str:
|
||||
return os.environ.get("LEAD_PUBLIC_DB", DEFAULT_DB)
|
||||
|
||||
|
||||
@st.cache_resource(show_spinner=False)
|
||||
def connection(path: str):
|
||||
return open_mirror(path)
|
||||
|
||||
|
||||
def _open():
|
||||
"""Open the mirror for this page run.
|
||||
|
||||
A read-only DuckDB connection still holds the file lock. Caching it
|
||||
for the life of the process blocks ``stack lead load``. The caller
|
||||
closes the connection before returning.
|
||||
"""
|
||||
path = db_path()
|
||||
try:
|
||||
return connection(path), path, None
|
||||
return open_mirror(path), path, None
|
||||
except Exception as exc:
|
||||
return None, path, exc
|
||||
|
||||
|
||||
def _close(con) -> None:
|
||||
if con is not None:
|
||||
con.close()
|
||||
|
||||
|
||||
def _show_error(path: str, exc: BaseException) -> None:
|
||||
text = str(exc)
|
||||
if "lock" in text.lower():
|
||||
@@ -86,22 +92,32 @@ def _files_frame(rows) -> pd.DataFrame:
|
||||
def writeup() -> None:
|
||||
brand.title("Public data mirror", "lead · evidence chassis")
|
||||
con, path, err = _open()
|
||||
st.caption(path)
|
||||
if err is not None or con is None:
|
||||
_show_error(path, err or FileNotFoundError(path))
|
||||
return
|
||||
for section in build(con):
|
||||
st.subheader(section.heading)
|
||||
for paragraph in section.body:
|
||||
st.markdown(paragraph)
|
||||
if section.rows:
|
||||
frame = pd.DataFrame(section.rows, columns=list(section.headers))
|
||||
st.dataframe(frame, hide_index=True, width="stretch")
|
||||
try:
|
||||
st.caption(path)
|
||||
if err is not None or con is None:
|
||||
_show_error(path, err or FileNotFoundError(path))
|
||||
return
|
||||
for section in build(con):
|
||||
st.subheader(section.heading)
|
||||
for paragraph in section.body:
|
||||
st.markdown(paragraph)
|
||||
if section.rows:
|
||||
frame = pd.DataFrame(section.rows, columns=list(section.headers))
|
||||
st.dataframe(frame, hide_index=True, width="stretch")
|
||||
finally:
|
||||
_close(con)
|
||||
|
||||
|
||||
def coverage() -> None:
|
||||
brand.title("Coverage", "lead · load log")
|
||||
con, path, err = _open()
|
||||
try:
|
||||
_coverage(con, path, err)
|
||||
finally:
|
||||
_close(con)
|
||||
|
||||
|
||||
def _coverage(con, path: str, err) -> None:
|
||||
st.caption(path)
|
||||
if err is not None or con is None:
|
||||
_show_error(path, err or FileNotFoundError(path))
|
||||
@@ -132,6 +148,13 @@ def coverage() -> None:
|
||||
def browse() -> None:
|
||||
brand.title("Tables", "lead · one table")
|
||||
con, path, err = _open()
|
||||
try:
|
||||
_browse(con, path, err)
|
||||
finally:
|
||||
_close(con)
|
||||
|
||||
|
||||
def _browse(con, path: str, err) -> None:
|
||||
st.caption(path)
|
||||
if err is not None or con is None:
|
||||
_show_error(path, err or FileNotFoundError(path))
|
||||
@@ -210,6 +233,13 @@ def browse() -> None:
|
||||
def place() -> None:
|
||||
brand.title("Place", "lead · one geography")
|
||||
con, path, err = _open()
|
||||
try:
|
||||
_place(con, path, err)
|
||||
finally:
|
||||
_close(con)
|
||||
|
||||
|
||||
def _place(con, path: str, err) -> None:
|
||||
st.caption(path)
|
||||
if err is not None or con is None:
|
||||
_show_error(path, err or FileNotFoundError(path))
|
||||
@@ -265,6 +295,13 @@ def place() -> None:
|
||||
def series() -> None:
|
||||
brand.title("Series", "lead · one measure over time")
|
||||
con, path, err = _open()
|
||||
try:
|
||||
_series(con, path, err)
|
||||
finally:
|
||||
_close(con)
|
||||
|
||||
|
||||
def _series(con, path: str, err) -> None:
|
||||
st.caption(path)
|
||||
if err is not None or con is None:
|
||||
_show_error(path, err or FileNotFoundError(path))
|
||||
@@ -334,6 +371,13 @@ def series() -> None:
|
||||
def reader() -> None:
|
||||
brand.title("SQL", "lead · one read")
|
||||
con, path, err = _open()
|
||||
try:
|
||||
_reader(con, path, err)
|
||||
finally:
|
||||
_close(con)
|
||||
|
||||
|
||||
def _reader(con, path: str, err) -> None:
|
||||
st.caption(path)
|
||||
if err is not None or con is None:
|
||||
_show_error(path, err or FileNotFoundError(path))
|
||||
|
||||
@@ -6,6 +6,7 @@ how a row was built. Every count is computed from the open connection.
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from dataclasses import dataclass
|
||||
|
||||
from lead.explore.catalog import FileRow, TableCard, latest_files, status_counts, tables
|
||||
@@ -208,9 +209,10 @@ def _mirror(files: tuple[FileRow, ...], cards: tuple[TableCard, ...]) -> Section
|
||||
"This is the local DuckDB mirror of public files for the LEAD "
|
||||
"evidence chassis. `stack lead load` reads files that are already "
|
||||
"on disk, from the enrichment manifest and from the strand-5 "
|
||||
"inventory, and writes them here. The app opens that file read-only. "
|
||||
"It does not download, it does not publish, and it does not write "
|
||||
"Unity Catalog.",
|
||||
"inventory, and writes them here. The app opens that file read-only "
|
||||
"for the page and closes it afterward, so a load can take the "
|
||||
"write lock between visits. It does not download, it does not "
|
||||
"publish, and it does not write Unity Catalog.",
|
||||
state,
|
||||
"A row count in this write-up is the number the loader recorded "
|
||||
"for the latest successful read of each file. Opening this page "
|
||||
@@ -241,13 +243,14 @@ def _recorded(counts: dict[str, int], loaded_rows: int, sources: int) -> Section
|
||||
(
|
||||
f"{_n(loaded_n, 'file')} loaded, {loaded_rows:,} rows recorded, "
|
||||
f"across {_n(sources, 'source id')}.",
|
||||
"Skipped is a file the loader refused on purpose: the manifest "
|
||||
"kind was not data, the member was a program or a readme, or the "
|
||||
"same bytes were already stored from a loose file. Absent is a "
|
||||
"file the inventory marked missing before any read, including a "
|
||||
"404 and FRED. A hash mismatch is recorded and that file is not "
|
||||
"inserted. A file the reader could not open is not in this log: "
|
||||
"the load command prints it as unreadable, and a later run retries it.",
|
||||
"Skipped is not a missing dataset. The loader refused the member "
|
||||
"because the observation file is already stored, or because the "
|
||||
"member is a dictionary, a guide, a sample, or not a table. "
|
||||
"Absent means the inventory had no file on disk before any read: "
|
||||
"a 404, or a host that did not answer. A hash mismatch is recorded "
|
||||
"and that file is not inserted. A file the reader could not open "
|
||||
"is not in this log: the load command prints it as unreadable, "
|
||||
"and a later run retries it.",
|
||||
),
|
||||
("Status", "Files"),
|
||||
rows,
|
||||
@@ -347,6 +350,96 @@ def _geography() -> Section:
|
||||
)
|
||||
|
||||
|
||||
_SKIP_WHY = {
|
||||
"kind:dictionary": "a data dictionary; the data file is loaded separately",
|
||||
"kind:reference": "a methods paper or guide",
|
||||
"kind:sample": "a sample of a file that is loaded in full",
|
||||
"not_tabular": "documentation or an archive with no table",
|
||||
"duplicate": "the same extract is already loaded from another file",
|
||||
"http_22": "the fetch failed and another copy is already loaded",
|
||||
"covered_by_manifest": "the same file is already loaded from the manifest",
|
||||
}
|
||||
|
||||
|
||||
def _year_phrase(years: list[int]) -> str:
|
||||
if not years:
|
||||
return "no year in the path"
|
||||
ordered = sorted(set(years))
|
||||
runs: list[tuple[int, int]] = []
|
||||
start = previous = ordered[0]
|
||||
for year in ordered[1:]:
|
||||
if year == previous + 1:
|
||||
previous = year
|
||||
continue
|
||||
runs.append((start, previous))
|
||||
start = previous = year
|
||||
runs.append((start, previous))
|
||||
return ", ".join(
|
||||
str(left) if left == right else f"{left}–{right}" for left, right in runs
|
||||
)
|
||||
|
||||
|
||||
def _skip_line(skipped: dict[str, int]) -> str:
|
||||
if not skipped:
|
||||
return "No file's latest status is skipped."
|
||||
parts = []
|
||||
for reason, count in sorted(skipped.items()):
|
||||
why = _SKIP_WHY.get(reason)
|
||||
parts.append(f"{reason} {count} ({why})" if why else f"{reason} {count}")
|
||||
return "Skipped is not missing data. " + ", ".join(parts) + "."
|
||||
|
||||
|
||||
def _qcew_line(absent: list[FileRow]) -> str:
|
||||
qcew = [row for row in absent if row.source_id == "qcew"]
|
||||
if not qcew:
|
||||
return ""
|
||||
years = [
|
||||
int(year) for row in qcew for year in re.findall(r"(?:19|20)\d{2}", row.path)
|
||||
]
|
||||
reasons = ", ".join(sorted({row.reason or "unspecified" for row in qcew}))
|
||||
return (
|
||||
f"{_n(len(qcew), 'absent QCEW file')} "
|
||||
f"({_year_phrase(years)}, {reasons}). Nothing was stored: the URL "
|
||||
"404'd or the annual file was not published yet. Loaded QCEW starts "
|
||||
"at 1990. These rows are not years the loader dropped."
|
||||
)
|
||||
|
||||
|
||||
def _fred_line(files: tuple[FileRow, ...], cards: tuple[TableCard, ...]) -> str:
|
||||
fred_rows = [row for row in files if row.source_id == "fred"]
|
||||
absent = [row for row in fred_rows if row.status == "absent"]
|
||||
names = {card.qualified for card in cards}
|
||||
has_econ = "econ.fred" in names
|
||||
has_public = "lead_public.fred" in names
|
||||
if absent and not has_econ and not has_public:
|
||||
why = absent[-1].reason or "unreachable"
|
||||
return (
|
||||
"FRED is absent. The log records the graph URL as "
|
||||
f"{why}, and this file has no `econ.fred` table and no "
|
||||
"`lead_public.fred` table. The market series live in the aco "
|
||||
"database, not here. `stack lead load` reads a CSV that is "
|
||||
"already on disk; it does not delete this URL row."
|
||||
)
|
||||
if absent and (has_econ or has_public):
|
||||
which = "econ.fred" if has_econ else "lead_public.fred"
|
||||
why = absent[-1].reason or "unreachable"
|
||||
return (
|
||||
"The FRED graph URL is still marked absent "
|
||||
f"({why}). `{which}` holds the CSV loaded after that fetch. "
|
||||
"The old URL row stays in the log, so the absent count still "
|
||||
"includes it."
|
||||
)
|
||||
if has_econ or has_public:
|
||||
which = "econ.fred" if has_econ else "lead_public.fred"
|
||||
return f"`{which}` is present in this file."
|
||||
if fred_rows:
|
||||
return (
|
||||
"The log has a FRED row whose latest status is "
|
||||
f"{fred_rows[-1].status}, reason {fred_rows[-1].reason or 'blank'}."
|
||||
)
|
||||
return "The load log has no FRED row, and `econ.fred` is not in this file."
|
||||
|
||||
|
||||
def _gaps(files: tuple[FileRow, ...], cards: tuple[TableCard, ...]) -> Section:
|
||||
absent = [row for row in files if row.status == "absent"]
|
||||
skipped: dict[str, int] = {}
|
||||
@@ -354,40 +447,25 @@ def _gaps(files: tuple[FileRow, ...], cards: tuple[TableCard, ...]) -> Section:
|
||||
if row.status == "skipped":
|
||||
reason = row.reason or "(blank)"
|
||||
skipped[reason] = skipped.get(reason, 0) + 1
|
||||
fred_rows = [row for row in files if row.source_id == "fred"]
|
||||
has_fred_table = any(card.qualified == "econ.fred" for card in cards)
|
||||
if any(row.status == "absent" for row in fred_rows) and not has_fred_table:
|
||||
fred = (
|
||||
"FRED is absent. The log records it that way, and there is no "
|
||||
"`econ.fred` table. Those series were not fetched."
|
||||
)
|
||||
elif has_fred_table:
|
||||
fred = "`econ.fred` is present in this file."
|
||||
elif fred_rows:
|
||||
fred = (
|
||||
"The log has a FRED row whose latest status is "
|
||||
f"{fred_rows[-1].status}, reason {fred_rows[-1].reason or 'blank'}."
|
||||
)
|
||||
else:
|
||||
fred = "The load log has no FRED row, and `econ.fred` is not in this file."
|
||||
rows = tuple(
|
||||
(row.source_id, _base(row.path) or row.release, row.reason or "")
|
||||
for row in absent
|
||||
)
|
||||
skip_line = (
|
||||
"Skipped files by reason: "
|
||||
+ ", ".join(f"{reason} {count}" for reason, count in sorted(skipped.items()))
|
||||
+ "."
|
||||
if skipped
|
||||
else "No file's latest status is skipped."
|
||||
)
|
||||
body = [
|
||||
_fred_line(files, cards),
|
||||
(
|
||||
f"{len(absent):,} files have latest status absent. "
|
||||
"Absent means the inventory had no file on disk. It is not a "
|
||||
"table the loader chose to drop."
|
||||
),
|
||||
_skip_line(skipped),
|
||||
]
|
||||
qcew = _qcew_line(absent)
|
||||
if qcew:
|
||||
body.insert(1, qcew)
|
||||
return Section(
|
||||
"Gaps",
|
||||
(
|
||||
fred,
|
||||
f"{len(absent):,} files have latest status absent.",
|
||||
skip_line,
|
||||
),
|
||||
tuple(body),
|
||||
("Source", "File", "Reason"),
|
||||
rows,
|
||||
)
|
||||
|
||||
@@ -307,6 +307,8 @@ def test_the_writeup_counts_the_log_and_states_the_contract(tmp_path):
|
||||
assert largest.rows[0][3] == "yes"
|
||||
gaps = next(section for section in sections if section.heading == "Gaps")
|
||||
assert ("fred", "fred.csv", "unreachable") in gaps.rows
|
||||
assert "not missing data" in "\n".join(gaps.body)
|
||||
assert "kind:note 1" in "\n".join(gaps.body)
|
||||
spine = next(
|
||||
section for section in sections if section.heading == "How a row is keyed"
|
||||
)
|
||||
@@ -314,6 +316,83 @@ def test_the_writeup_counts_the_log_and_states_the_contract(tmp_path):
|
||||
assert "000" in fips[1]
|
||||
|
||||
|
||||
def test_gaps_name_a_dictionary_and_the_qcew_404_years(tmp_path):
|
||||
con = duckdb.connect(str(tmp_path / "gaps.duckdb"))
|
||||
ensure_log(con)
|
||||
record(
|
||||
con,
|
||||
source_id="cms_dict",
|
||||
release="2026",
|
||||
sha256="e" * 64,
|
||||
path="/data/dictionary__dict.pdf",
|
||||
table="",
|
||||
status="skipped",
|
||||
rows=0,
|
||||
reason="kind:dictionary",
|
||||
actor="stack",
|
||||
)
|
||||
record(
|
||||
con,
|
||||
source_id="qcew",
|
||||
release="",
|
||||
sha256="",
|
||||
path="https://data.bls.gov/cew/data/files/1975/csv/1975_annual_by_industry.zip",
|
||||
table="",
|
||||
status="absent",
|
||||
rows=0,
|
||||
reason="http_404",
|
||||
actor="stack",
|
||||
)
|
||||
record(
|
||||
con,
|
||||
source_id="qcew",
|
||||
release="",
|
||||
sha256="",
|
||||
path="https://data.bls.gov/cew/data/files/2026/csv/2026_annual_by_industry.zip",
|
||||
table="",
|
||||
status="absent",
|
||||
rows=0,
|
||||
reason="http_404",
|
||||
actor="stack",
|
||||
)
|
||||
record(
|
||||
con,
|
||||
source_id="fred",
|
||||
release="UNRATE.csv",
|
||||
sha256="f" * 64,
|
||||
path="/data/lead/strand5/fred/UNRATE.csv",
|
||||
table="lead_public.fred",
|
||||
status="loaded",
|
||||
rows=900,
|
||||
reason="",
|
||||
actor="stack",
|
||||
)
|
||||
record(
|
||||
con,
|
||||
source_id="fred",
|
||||
release="",
|
||||
sha256="",
|
||||
path="https://fred.stlouisfed.org/graph/fredgraph.csv?id=UNRATE",
|
||||
table="",
|
||||
status="absent",
|
||||
rows=0,
|
||||
reason="unreachable",
|
||||
actor="stack",
|
||||
)
|
||||
con.execute("CREATE SCHEMA IF NOT EXISTS lead_public")
|
||||
con.execute(
|
||||
"CREATE TABLE lead_public.fred (observation_date VARCHAR, UNRATE VARCHAR)"
|
||||
)
|
||||
gaps = next(section for section in writeup.build(con) if section.heading == "Gaps")
|
||||
text = "\n".join(gaps.body)
|
||||
assert "data dictionary" in text
|
||||
assert "1975, 2026" in text
|
||||
assert "Loaded QCEW starts at 1990" in text
|
||||
assert "lead_public.fred" in text
|
||||
assert "still marked absent" in text
|
||||
con.close()
|
||||
|
||||
|
||||
def test_writeup_page_renders_the_recorded_count(tmp_path, monkeypatch):
|
||||
con = _mirror(tmp_path)
|
||||
path = str(tmp_path / "public.duckdb")
|
||||
|
||||
@@ -5,6 +5,9 @@ from __future__ import annotations
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from lead.explore import catalog
|
||||
from tests.lead.test_explore import _mirror
|
||||
from zot.db import TYPE_MAP, Db
|
||||
from zot.schema import create_db
|
||||
@@ -91,6 +94,25 @@ def test_coverage_filters_the_log_and_a_missing_file_is_an_error(tmp_path, monke
|
||||
assert "No mirror" in _blob(missing)
|
||||
|
||||
|
||||
def test_a_page_closes_the_mirror(tmp_path, monkeypatch):
|
||||
import duckdb
|
||||
|
||||
held: list = []
|
||||
monkeypatch.setenv("LEAD_PUBLIC_DB", _mirror_path(tmp_path))
|
||||
|
||||
def wrapped(path):
|
||||
con = catalog.open_mirror(path)
|
||||
held.append(con)
|
||||
return con
|
||||
|
||||
monkeypatch.setattr("lead.explore.views.open_mirror", wrapped)
|
||||
at = _run("from lead.explore.views import coverage\ncoverage()\n")
|
||||
assert not at.exception, at.exception
|
||||
assert held
|
||||
with pytest.raises(duckdb.Error):
|
||||
held[0].execute("SELECT 1")
|
||||
|
||||
|
||||
def test_a_locked_mirror_and_any_other_open_error_are_shown(tmp_path, monkeypatch):
|
||||
monkeypatch.setenv("LEAD_PUBLIC_DB", str(tmp_path / "locked.duckdb"))
|
||||
|
||||
@@ -272,7 +294,7 @@ def test_the_shell_links_osf_and_reconnects(tmp_path, monkeypatch):
|
||||
at = _run("from lead.explore.app import main\nmain()\n")
|
||||
urls = [str(getattr(item, "url", "")) for item in at.link_button]
|
||||
assert any("osf.io/8wq3m" in url for url in urls) or "osf.io/8wq3m" in _blob(at)
|
||||
_widget(at, "button", "Reconnect").click().run()
|
||||
assert "closes it when the page finishes" in _blob(at) or "write lock" in _blob(at)
|
||||
assert not at.exception, at.exception
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user