From bdccc343fc00c0549cd795ae2344fccd14d17ce5 Mon Sep 17 00:00:00 2001 From: kert Date: Fri, 13 Nov 2026 02:46:02 -0500 Subject: [PATCH] fix(pfs): stance and reaction --write re-read the primary inside the batch and replace only their own reaction rows (I5) --- src/cli/pfs.py | 49 +++++++++++++++-------- src/pfs/codetables.py | 14 +++++++ tests/cli/test_pfs_cli.py | 81 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 127 insertions(+), 17 deletions(-) diff --git a/src/cli/pfs.py b/src/cli/pfs.py index 77033bf..1ab1481 100644 --- a/src/cli/pfs.py +++ b/src/cli/pfs.py @@ -50,6 +50,7 @@ from pfs.codetables import ( read_reaction, read_reviews, read_stances, + replace_reaction_rows, write_cms_summaries, write_commenters, write_elements, @@ -854,15 +855,14 @@ def reaction( # Ruling A13: the classifier and every family's series (docket # counts, the stance sample — ~1-2s/sample — and FR pairs) all run # against pgvector/bib, not DuckDB, so they run BEFORE any DuckDB - # batch is opened. The batch below only ever calls write_reaction + # batch is opened. The batch below only re-reads the primary, writes # and publishes — it never holds the single-writer lock (#508-#514) # while the slow work happens. classify = _classifier() if stance_sample > 0 else None engine = _engine() - by_family: dict[str, list[Any]] = {} - rows: list[Any] = [] + series_of: dict[str, list[Any]] = {} for key, codes, name in targets: - fam_rows = reaction_series( + series_of[key] = reaction_series( engine, store, key, @@ -871,19 +871,29 @@ def reaction( stance_sample_n=stance_sample, name=name, ) - fam_rows = _overlay_stances( + by_family = { + key: _overlay_stances( fam_rows, stored[key], old_rows[key], n_comm.get(key, Counter()) ) - by_family[key] = fam_rows - rows.extend(fam_rows) + for key, fam_rows in series_of.items() + } if write: with _batch() as con: ensure_tables(con) - for key, fam_rows in by_family.items(): - write_reaction(con, key, fam_rows) + for key, fam_rows in series_of.items(): + # I5: overlay the stances and method record the primary holds + # now (a stance --write may have published during the series + # work); replace only this run's own rows + by_family[key] = _overlay_stances( + fam_rows, + _stored(read_stances, con, key), + _stored(read_reaction, con, key), + Counter(c.docket for c in _stored(_commenters_of, con, key)), + ) + replace_reaction_rows(con, key, by_family[key]) _publish() - _print_reaction(rows) + _print_reaction([r for fam_rows in by_family.values() for r in fam_rows]) def _print_exposures(rows: list[Any]) -> None: @@ -2059,7 +2069,6 @@ def stance( commenter_rows = [ c for c in read_commenters(con, family=fam.key) if c.docket == docket ] - stored = _stored(read_reaction, con, fam.key) finally: con.close() if not commenter_rows: @@ -2146,15 +2155,21 @@ def stance( golden, method, name, fam.key, generation, tuple(heldout) ), ) - keep = [ - r - for r in (stored or series_rows) - if r.period != docket or r.period_kind != "docket" - ] with _batch() as bcon: ensure_tables(bcon) n = write_stances(bcon, fam.key, docket, rows) - write_reaction(bcon, fam.key, [*keep, new]) + # I5: the primary's rows as they are now (never a replica read from + # before the model work): only this docket's row is replaced; a + # family with no rows yet is seeded with the series + if _stored(read_reaction, bcon, fam.key): + replace_reaction_rows(bcon, fam.key, [new]) + else: + seed = [ + r + for r in series_rows + if r.period != docket or r.period_kind != "docket" + ] + write_reaction(bcon, fam.key, [*seed, new]) typer.echo(f"wrote {n} stances") _publish() diff --git a/src/pfs/codetables.py b/src/pfs/codetables.py index 9f56ac8..b5400b6 100644 --- a/src/pfs/codetables.py +++ b/src/pfs/codetables.py @@ -916,6 +916,20 @@ def write_reaction(con: Any, family: str, rows: Sequence[ReactionRow]) -> int: return _insert(con, "pfs.code_reaction", rows) +def replace_reaction_rows(con: Any, family: str, rows: Sequence[ReactionRow]) -> int: + """Replace only the ``(family, period, period_kind)`` rows given; the + family's other rows stay as the primary holds them (I5: a run never + writes its own stale copy of another run's rows).""" + ensure_tables(con) + for r in rows: + con.execute( + "DELETE FROM pfs.code_reaction " + "WHERE family = ? AND period = ? AND period_kind = ?", + [family, r.period, r.period_kind], + ) + return _insert(con, "pfs.code_reaction", rows) + + def read_reaction(con: Any, family: str) -> list[ReactionRow]: rows = con.execute( "SELECT * FROM pfs.code_reaction WHERE family = ? " diff --git a/tests/cli/test_pfs_cli.py b/tests/cli/test_pfs_cli.py index 274970e..2abc35a 100644 --- a/tests/cli/test_pfs_cli.py +++ b/tests/cli/test_pfs_cli.py @@ -2126,6 +2126,44 @@ class TestStance: assert rows["D0"].events_json == "{}" assert con.published == [True] + def test_write_rereads_the_primary_inside_the_batch(self, con, monkeypatch): + """I5: rows another run publishes during the model work (or rows a + stale replica lacks) survive; only the run's own docket row is + replaced.""" + from pfs.codetables import write_reaction + + write_reaction( + con._real, + "HOME-VISITS", + [ + ReactionRow("HOME-VISITS", "D1", "docket", 2023, 2, 9, 0, 0, 0, 0, "[]"), + ReactionRow("HOME-VISITS", "D0", "docket", 2020, 1, 4, 0, 0, 0, 0, "[]"), + ], + ) # fmt: skip + self._patch(con, monkeypatch) + slow = pfs_cli.stances_for + ev = json.dumps({"recall": 0.5, "direction": 0.5}) + + def concurrent(*a, **kw): + # another docket's stance --write publishes meanwhile + con._real.execute("DELETE FROM pfs.code_reaction WHERE period = 'D0'") + write_reaction_rows = [ + ReactionRow("HOME-VISITS", "D0", "docket", 2020, 1, 4, 1, 0, 0, 0, "[]", "{}", 1, 1, stance_eval=ev), + ReactionRow("HOME-VISITS", "D2", "docket", 2021, 1, 4, 0, 0, 0, 0, "[]"), + ] # fmt: skip + from pfs.codetables import _insert + + _insert(con._real, "pfs.code_reaction", write_reaction_rows) + return slow(*a, **kw) + + monkeypatch.setattr(pfs_cli, "stances_for", concurrent) + res = runner.invoke(app, self._args("--write")) + assert res.exit_code == 0, res.output + rows = {r.period: r for r in read_reaction(con._real, "HOME-VISITS")} + assert set(rows) == {"D0", "D1", "D2"} + assert rows["D0"].stance_eval == ev and rows["D0"].n_classified == 1 + assert (rows["D1"].stance_support, rows["D1"].stance_oppose) == (1, 1) + def test_no_llm_replays_the_ledger(self, con, monkeypatch): seen = self._patch(con, monkeypatch) monkeypatch.setattr(pfs_cli, "_asker", lambda: 1 / 0) @@ -2277,6 +2315,49 @@ class TestStance: ) assert (rows["D2"].stance_method, rows["D2"].stance_eval) == (None, None) + def test_reaction_write_overlays_the_primary_read_inside_the_batch( + self, con, monkeypatch + ): + """I5: a stance --write that publishes while reaction gathers its + series keeps its counts and method record; the replica read before + the slow work is not what the batch writes over.""" + from pfs.codetables import ( + CommenterRow, + StanceRow, + write_commenters, + write_reaction, + write_stances, + ) + + ev = json.dumps({"recall": 0.5, "direction": 0.5, "in_sample": False}) + + def series(e, s, f, c, **kw): + # the concurrent stance --write lands during the slow work + write_commenters( + con._real, + [CommenterRow(k, "D1", k, "", "individual", "", "", 0, "regs_gov_fields", "", "regs", "CCM") for k in ("K1", "K2")], + ) # fmt: skip + write_stances( + con._real, "CCM", "D1", + [StanceRow("CCM", "K1", "D1", "CCM:2023:other", "oppose", "", "", "x")], + ) # fmt: skip + write_reaction( + con._real, "CCM", + [ReactionRow("CCM", "D1", "docket", 2023, 5, 20, 0, 0, 1, 0, "[]", '{"CCM:2023:other": {}}', 1, 2, stance_method="m", stance_eval=ev), + ReactionRow("CCM", "D9", "docket", 2019, 1, 3, 0, 0, 0, 0, "[]")], + ) # fmt: skip + return [ReactionRow(f, "D1", "docket", 2023, 6, 21, 0, 0, 0, 0, "[]")] + + monkeypatch.setattr(pfs_cli, "_engine", lambda: "ENGINE") + monkeypatch.setattr(pfs_cli, "reaction_series", series) + res = runner.invoke(app, ["pfs", "reaction", "--family", "ccm", "--write"]) + assert res.exit_code == 0, res.output + rows = {r.period: r for r in read_reaction(con._real, "CCM")} + assert set(rows) == {"D1", "D9"} + d1 = rows["D1"] + assert (d1.n_items, d1.n_total, d1.stance_oppose) == (6, 21, 1) + assert (d1.stance_method, d1.stance_eval, d1.n_classified) == ("m", ev, 1) + def test_stance_sample_refused_when_stances_stored(self, con, monkeypatch): from pfs.codetables import StanceRow, write_stances