fix(pfs): stance and reaction --write re-read the primary inside the batch and replace only their own reaction rows (I5)
This commit is contained in:
@@ -50,6 +50,7 @@ from pfs.codetables import (
|
||||
read_reaction,
|
||||
read_reviews,
|
||||
read_stances,
|
||||
replace_reaction_rows,
|
||||
write_cms_summaries,
|
||||
write_commenters,
|
||||
write_elements,
|
||||
@@ -854,15 +855,14 @@ def reaction(
|
||||
# Ruling A13: the classifier and every family's series (docket
|
||||
# counts, the stance sample — ~1-2s/sample — and FR pairs) all run
|
||||
# against pgvector/bib, not DuckDB, so they run BEFORE any DuckDB
|
||||
# batch is opened. The batch below only ever calls write_reaction
|
||||
# batch is opened. The batch below only re-reads the primary, writes
|
||||
# and publishes — it never holds the single-writer lock (#508-#514)
|
||||
# while the slow work happens.
|
||||
classify = _classifier() if stance_sample > 0 else None
|
||||
engine = _engine()
|
||||
by_family: dict[str, list[Any]] = {}
|
||||
rows: list[Any] = []
|
||||
series_of: dict[str, list[Any]] = {}
|
||||
for key, codes, name in targets:
|
||||
fam_rows = reaction_series(
|
||||
series_of[key] = reaction_series(
|
||||
engine,
|
||||
store,
|
||||
key,
|
||||
@@ -871,19 +871,29 @@ def reaction(
|
||||
stance_sample_n=stance_sample,
|
||||
name=name,
|
||||
)
|
||||
fam_rows = _overlay_stances(
|
||||
by_family = {
|
||||
key: _overlay_stances(
|
||||
fam_rows, stored[key], old_rows[key], n_comm.get(key, Counter())
|
||||
)
|
||||
by_family[key] = fam_rows
|
||||
rows.extend(fam_rows)
|
||||
for key, fam_rows in series_of.items()
|
||||
}
|
||||
|
||||
if write:
|
||||
with _batch() as con:
|
||||
ensure_tables(con)
|
||||
for key, fam_rows in by_family.items():
|
||||
write_reaction(con, key, fam_rows)
|
||||
for key, fam_rows in series_of.items():
|
||||
# I5: overlay the stances and method record the primary holds
|
||||
# now (a stance --write may have published during the series
|
||||
# work); replace only this run's own rows
|
||||
by_family[key] = _overlay_stances(
|
||||
fam_rows,
|
||||
_stored(read_stances, con, key),
|
||||
_stored(read_reaction, con, key),
|
||||
Counter(c.docket for c in _stored(_commenters_of, con, key)),
|
||||
)
|
||||
replace_reaction_rows(con, key, by_family[key])
|
||||
_publish()
|
||||
_print_reaction(rows)
|
||||
_print_reaction([r for fam_rows in by_family.values() for r in fam_rows])
|
||||
|
||||
|
||||
def _print_exposures(rows: list[Any]) -> None:
|
||||
@@ -2059,7 +2069,6 @@ def stance(
|
||||
commenter_rows = [
|
||||
c for c in read_commenters(con, family=fam.key) if c.docket == docket
|
||||
]
|
||||
stored = _stored(read_reaction, con, fam.key)
|
||||
finally:
|
||||
con.close()
|
||||
if not commenter_rows:
|
||||
@@ -2146,15 +2155,21 @@ def stance(
|
||||
golden, method, name, fam.key, generation, tuple(heldout)
|
||||
),
|
||||
)
|
||||
keep = [
|
||||
r
|
||||
for r in (stored or series_rows)
|
||||
if r.period != docket or r.period_kind != "docket"
|
||||
]
|
||||
with _batch() as bcon:
|
||||
ensure_tables(bcon)
|
||||
n = write_stances(bcon, fam.key, docket, rows)
|
||||
write_reaction(bcon, fam.key, [*keep, new])
|
||||
# I5: the primary's rows as they are now (never a replica read from
|
||||
# before the model work): only this docket's row is replaced; a
|
||||
# family with no rows yet is seeded with the series
|
||||
if _stored(read_reaction, bcon, fam.key):
|
||||
replace_reaction_rows(bcon, fam.key, [new])
|
||||
else:
|
||||
seed = [
|
||||
r
|
||||
for r in series_rows
|
||||
if r.period != docket or r.period_kind != "docket"
|
||||
]
|
||||
write_reaction(bcon, fam.key, [*seed, new])
|
||||
typer.echo(f"wrote {n} stances")
|
||||
_publish()
|
||||
|
||||
|
||||
@@ -916,6 +916,20 @@ def write_reaction(con: Any, family: str, rows: Sequence[ReactionRow]) -> int:
|
||||
return _insert(con, "pfs.code_reaction", rows)
|
||||
|
||||
|
||||
def replace_reaction_rows(con: Any, family: str, rows: Sequence[ReactionRow]) -> int:
|
||||
"""Replace only the ``(family, period, period_kind)`` rows given; the
|
||||
family's other rows stay as the primary holds them (I5: a run never
|
||||
writes its own stale copy of another run's rows)."""
|
||||
ensure_tables(con)
|
||||
for r in rows:
|
||||
con.execute(
|
||||
"DELETE FROM pfs.code_reaction "
|
||||
"WHERE family = ? AND period = ? AND period_kind = ?",
|
||||
[family, r.period, r.period_kind],
|
||||
)
|
||||
return _insert(con, "pfs.code_reaction", rows)
|
||||
|
||||
|
||||
def read_reaction(con: Any, family: str) -> list[ReactionRow]:
|
||||
rows = con.execute(
|
||||
"SELECT * FROM pfs.code_reaction WHERE family = ? "
|
||||
|
||||
@@ -2126,6 +2126,44 @@ class TestStance:
|
||||
assert rows["D0"].events_json == "{}"
|
||||
assert con.published == [True]
|
||||
|
||||
def test_write_rereads_the_primary_inside_the_batch(self, con, monkeypatch):
|
||||
"""I5: rows another run publishes during the model work (or rows a
|
||||
stale replica lacks) survive; only the run's own docket row is
|
||||
replaced."""
|
||||
from pfs.codetables import write_reaction
|
||||
|
||||
write_reaction(
|
||||
con._real,
|
||||
"HOME-VISITS",
|
||||
[
|
||||
ReactionRow("HOME-VISITS", "D1", "docket", 2023, 2, 9, 0, 0, 0, 0, "[]"),
|
||||
ReactionRow("HOME-VISITS", "D0", "docket", 2020, 1, 4, 0, 0, 0, 0, "[]"),
|
||||
],
|
||||
) # fmt: skip
|
||||
self._patch(con, monkeypatch)
|
||||
slow = pfs_cli.stances_for
|
||||
ev = json.dumps({"recall": 0.5, "direction": 0.5})
|
||||
|
||||
def concurrent(*a, **kw):
|
||||
# another docket's stance --write publishes meanwhile
|
||||
con._real.execute("DELETE FROM pfs.code_reaction WHERE period = 'D0'")
|
||||
write_reaction_rows = [
|
||||
ReactionRow("HOME-VISITS", "D0", "docket", 2020, 1, 4, 1, 0, 0, 0, "[]", "{}", 1, 1, stance_eval=ev),
|
||||
ReactionRow("HOME-VISITS", "D2", "docket", 2021, 1, 4, 0, 0, 0, 0, "[]"),
|
||||
] # fmt: skip
|
||||
from pfs.codetables import _insert
|
||||
|
||||
_insert(con._real, "pfs.code_reaction", write_reaction_rows)
|
||||
return slow(*a, **kw)
|
||||
|
||||
monkeypatch.setattr(pfs_cli, "stances_for", concurrent)
|
||||
res = runner.invoke(app, self._args("--write"))
|
||||
assert res.exit_code == 0, res.output
|
||||
rows = {r.period: r for r in read_reaction(con._real, "HOME-VISITS")}
|
||||
assert set(rows) == {"D0", "D1", "D2"}
|
||||
assert rows["D0"].stance_eval == ev and rows["D0"].n_classified == 1
|
||||
assert (rows["D1"].stance_support, rows["D1"].stance_oppose) == (1, 1)
|
||||
|
||||
def test_no_llm_replays_the_ledger(self, con, monkeypatch):
|
||||
seen = self._patch(con, monkeypatch)
|
||||
monkeypatch.setattr(pfs_cli, "_asker", lambda: 1 / 0)
|
||||
@@ -2277,6 +2315,49 @@ class TestStance:
|
||||
)
|
||||
assert (rows["D2"].stance_method, rows["D2"].stance_eval) == (None, None)
|
||||
|
||||
def test_reaction_write_overlays_the_primary_read_inside_the_batch(
|
||||
self, con, monkeypatch
|
||||
):
|
||||
"""I5: a stance --write that publishes while reaction gathers its
|
||||
series keeps its counts and method record; the replica read before
|
||||
the slow work is not what the batch writes over."""
|
||||
from pfs.codetables import (
|
||||
CommenterRow,
|
||||
StanceRow,
|
||||
write_commenters,
|
||||
write_reaction,
|
||||
write_stances,
|
||||
)
|
||||
|
||||
ev = json.dumps({"recall": 0.5, "direction": 0.5, "in_sample": False})
|
||||
|
||||
def series(e, s, f, c, **kw):
|
||||
# the concurrent stance --write lands during the slow work
|
||||
write_commenters(
|
||||
con._real,
|
||||
[CommenterRow(k, "D1", k, "", "individual", "", "", 0, "regs_gov_fields", "", "regs", "CCM") for k in ("K1", "K2")],
|
||||
) # fmt: skip
|
||||
write_stances(
|
||||
con._real, "CCM", "D1",
|
||||
[StanceRow("CCM", "K1", "D1", "CCM:2023:other", "oppose", "", "", "x")],
|
||||
) # fmt: skip
|
||||
write_reaction(
|
||||
con._real, "CCM",
|
||||
[ReactionRow("CCM", "D1", "docket", 2023, 5, 20, 0, 0, 1, 0, "[]", '{"CCM:2023:other": {}}', 1, 2, stance_method="m", stance_eval=ev),
|
||||
ReactionRow("CCM", "D9", "docket", 2019, 1, 3, 0, 0, 0, 0, "[]")],
|
||||
) # fmt: skip
|
||||
return [ReactionRow(f, "D1", "docket", 2023, 6, 21, 0, 0, 0, 0, "[]")]
|
||||
|
||||
monkeypatch.setattr(pfs_cli, "_engine", lambda: "ENGINE")
|
||||
monkeypatch.setattr(pfs_cli, "reaction_series", series)
|
||||
res = runner.invoke(app, ["pfs", "reaction", "--family", "ccm", "--write"])
|
||||
assert res.exit_code == 0, res.output
|
||||
rows = {r.period: r for r in read_reaction(con._real, "CCM")}
|
||||
assert set(rows) == {"D1", "D9"}
|
||||
d1 = rows["D1"]
|
||||
assert (d1.n_items, d1.n_total, d1.stance_oppose) == (6, 21, 1)
|
||||
assert (d1.stance_method, d1.stance_eval, d1.n_classified) == ("m", ev, 1)
|
||||
|
||||
def test_stance_sample_refused_when_stances_stored(self, con, monkeypatch):
|
||||
from pfs.codetables import StanceRow, write_stances
|
||||
|
||||
|
||||
Reference in New Issue
Block a user