fix(pfs): stance and reaction --write re-read the primary inside the batch and replace only their own reaction rows (I5)

This commit is contained in:
kert
2026-11-13 02:46:02 -05:00
parent 3b2175cb80
commit bdccc343fc
3 changed files with 127 additions and 17 deletions

View File

@@ -50,6 +50,7 @@ from pfs.codetables import (
read_reaction,
read_reviews,
read_stances,
replace_reaction_rows,
write_cms_summaries,
write_commenters,
write_elements,
@@ -854,15 +855,14 @@ def reaction(
# Ruling A13: the classifier and every family's series (docket
# counts, the stance sample — ~1-2s/sample — and FR pairs) all run
# against pgvector/bib, not DuckDB, so they run BEFORE any DuckDB
# batch is opened. The batch below only ever calls write_reaction
# batch is opened. The batch below only re-reads the primary, writes
# and publishes — it never holds the single-writer lock (#508-#514)
# while the slow work happens.
classify = _classifier() if stance_sample > 0 else None
engine = _engine()
by_family: dict[str, list[Any]] = {}
rows: list[Any] = []
series_of: dict[str, list[Any]] = {}
for key, codes, name in targets:
fam_rows = reaction_series(
series_of[key] = reaction_series(
engine,
store,
key,
@@ -871,19 +871,29 @@ def reaction(
stance_sample_n=stance_sample,
name=name,
)
fam_rows = _overlay_stances(
by_family = {
key: _overlay_stances(
fam_rows, stored[key], old_rows[key], n_comm.get(key, Counter())
)
by_family[key] = fam_rows
rows.extend(fam_rows)
for key, fam_rows in series_of.items()
}
if write:
with _batch() as con:
ensure_tables(con)
for key, fam_rows in by_family.items():
write_reaction(con, key, fam_rows)
for key, fam_rows in series_of.items():
# I5: overlay the stances and method record the primary holds
# now (a stance --write may have published during the series
# work); replace only this run's own rows
by_family[key] = _overlay_stances(
fam_rows,
_stored(read_stances, con, key),
_stored(read_reaction, con, key),
Counter(c.docket for c in _stored(_commenters_of, con, key)),
)
replace_reaction_rows(con, key, by_family[key])
_publish()
_print_reaction(rows)
_print_reaction([r for fam_rows in by_family.values() for r in fam_rows])
def _print_exposures(rows: list[Any]) -> None:
@@ -2059,7 +2069,6 @@ def stance(
commenter_rows = [
c for c in read_commenters(con, family=fam.key) if c.docket == docket
]
stored = _stored(read_reaction, con, fam.key)
finally:
con.close()
if not commenter_rows:
@@ -2146,15 +2155,21 @@ def stance(
golden, method, name, fam.key, generation, tuple(heldout)
),
)
keep = [
r
for r in (stored or series_rows)
if r.period != docket or r.period_kind != "docket"
]
with _batch() as bcon:
ensure_tables(bcon)
n = write_stances(bcon, fam.key, docket, rows)
write_reaction(bcon, fam.key, [*keep, new])
# I5: the primary's rows as they are now (never a replica read from
# before the model work): only this docket's row is replaced; a
# family with no rows yet is seeded with the series
if _stored(read_reaction, bcon, fam.key):
replace_reaction_rows(bcon, fam.key, [new])
else:
seed = [
r
for r in series_rows
if r.period != docket or r.period_kind != "docket"
]
write_reaction(bcon, fam.key, [*seed, new])
typer.echo(f"wrote {n} stances")
_publish()

View File

@@ -916,6 +916,20 @@ def write_reaction(con: Any, family: str, rows: Sequence[ReactionRow]) -> int:
return _insert(con, "pfs.code_reaction", rows)
def replace_reaction_rows(con: Any, family: str, rows: Sequence[ReactionRow]) -> int:
"""Replace only the ``(family, period, period_kind)`` rows given; the
family's other rows stay as the primary holds them (I5: a run never
writes its own stale copy of another run's rows)."""
ensure_tables(con)
for r in rows:
con.execute(
"DELETE FROM pfs.code_reaction "
"WHERE family = ? AND period = ? AND period_kind = ?",
[family, r.period, r.period_kind],
)
return _insert(con, "pfs.code_reaction", rows)
def read_reaction(con: Any, family: str) -> list[ReactionRow]:
rows = con.execute(
"SELECT * FROM pfs.code_reaction WHERE family = ? "

View File

@@ -2126,6 +2126,44 @@ class TestStance:
assert rows["D0"].events_json == "{}"
assert con.published == [True]
def test_write_rereads_the_primary_inside_the_batch(self, con, monkeypatch):
"""I5: rows another run publishes during the model work (or rows a
stale replica lacks) survive; only the run's own docket row is
replaced."""
from pfs.codetables import write_reaction
write_reaction(
con._real,
"HOME-VISITS",
[
ReactionRow("HOME-VISITS", "D1", "docket", 2023, 2, 9, 0, 0, 0, 0, "[]"),
ReactionRow("HOME-VISITS", "D0", "docket", 2020, 1, 4, 0, 0, 0, 0, "[]"),
],
) # fmt: skip
self._patch(con, monkeypatch)
slow = pfs_cli.stances_for
ev = json.dumps({"recall": 0.5, "direction": 0.5})
def concurrent(*a, **kw):
# another docket's stance --write publishes meanwhile
con._real.execute("DELETE FROM pfs.code_reaction WHERE period = 'D0'")
write_reaction_rows = [
ReactionRow("HOME-VISITS", "D0", "docket", 2020, 1, 4, 1, 0, 0, 0, "[]", "{}", 1, 1, stance_eval=ev),
ReactionRow("HOME-VISITS", "D2", "docket", 2021, 1, 4, 0, 0, 0, 0, "[]"),
] # fmt: skip
from pfs.codetables import _insert
_insert(con._real, "pfs.code_reaction", write_reaction_rows)
return slow(*a, **kw)
monkeypatch.setattr(pfs_cli, "stances_for", concurrent)
res = runner.invoke(app, self._args("--write"))
assert res.exit_code == 0, res.output
rows = {r.period: r for r in read_reaction(con._real, "HOME-VISITS")}
assert set(rows) == {"D0", "D1", "D2"}
assert rows["D0"].stance_eval == ev and rows["D0"].n_classified == 1
assert (rows["D1"].stance_support, rows["D1"].stance_oppose) == (1, 1)
def test_no_llm_replays_the_ledger(self, con, monkeypatch):
seen = self._patch(con, monkeypatch)
monkeypatch.setattr(pfs_cli, "_asker", lambda: 1 / 0)
@@ -2277,6 +2315,49 @@ class TestStance:
)
assert (rows["D2"].stance_method, rows["D2"].stance_eval) == (None, None)
def test_reaction_write_overlays_the_primary_read_inside_the_batch(
self, con, monkeypatch
):
"""I5: a stance --write that publishes while reaction gathers its
series keeps its counts and method record; the replica read before
the slow work is not what the batch writes over."""
from pfs.codetables import (
CommenterRow,
StanceRow,
write_commenters,
write_reaction,
write_stances,
)
ev = json.dumps({"recall": 0.5, "direction": 0.5, "in_sample": False})
def series(e, s, f, c, **kw):
# the concurrent stance --write lands during the slow work
write_commenters(
con._real,
[CommenterRow(k, "D1", k, "", "individual", "", "", 0, "regs_gov_fields", "", "regs", "CCM") for k in ("K1", "K2")],
) # fmt: skip
write_stances(
con._real, "CCM", "D1",
[StanceRow("CCM", "K1", "D1", "CCM:2023:other", "oppose", "", "", "x")],
) # fmt: skip
write_reaction(
con._real, "CCM",
[ReactionRow("CCM", "D1", "docket", 2023, 5, 20, 0, 0, 1, 0, "[]", '{"CCM:2023:other": {}}', 1, 2, stance_method="m", stance_eval=ev),
ReactionRow("CCM", "D9", "docket", 2019, 1, 3, 0, 0, 0, 0, "[]")],
) # fmt: skip
return [ReactionRow(f, "D1", "docket", 2023, 6, 21, 0, 0, 0, 0, "[]")]
monkeypatch.setattr(pfs_cli, "_engine", lambda: "ENGINE")
monkeypatch.setattr(pfs_cli, "reaction_series", series)
res = runner.invoke(app, ["pfs", "reaction", "--family", "ccm", "--write"])
assert res.exit_code == 0, res.output
rows = {r.period: r for r in read_reaction(con._real, "CCM")}
assert set(rows) == {"D1", "D9"}
d1 = rows["D1"]
assert (d1.n_items, d1.n_total, d1.stance_oppose) == (6, 21, 1)
assert (d1.stance_method, d1.stance_eval, d1.n_classified) == ("m", ev, 1)
def test_stance_sample_refused_when_stances_stored(self, con, monkeypatch):
from pfs.codetables import StanceRow, write_stances