Files
stack/dev/scripts/lex_probe.py
kert 0082ec80ce
Some checks failed
CI / lint (push) Successful in 58s
CI / notebooks-smoke (push) Successful in 2m3s
Deploy / changes (push) Successful in 18s
Infra CI / notebooks (push) Successful in 59s
CI / test (push) Failing after 4m32s
Infra CI / zotero (push) Successful in 29s
Infra CI / docs (push) Successful in 38s
Infra CI / api (push) Successful in 1m32s
Infra CI / bcda (push) Successful in 2m29s
Infra CI / llm (push) Successful in 1m26s
Deploy / notebooks (push) Has been skipped
Deploy / zotero (push) Has been skipped
Deploy / docs (push) Has been skipped
Infra CI / mc (push) Successful in 33s
Deploy / api (push) Successful in 1m42s
Deploy / report (push) Has been cancelled
Deploy / mc (push) Has been cancelled
Deploy / bcda (push) Has been cancelled
Deploy / llm (push) Has been cancelled
fix(lex): facts index carries also-cite locators and the probe matches book also-cites exactly (S1-P10); the current tab scrolls into view on phones; book title only on licensed books; a citeless [F#] stays as text (S1)
2026-11-13 03:42:52 -05:00

388 lines
14 KiB
Python
Raw Permalink Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""Drive a real Chromium through the lex app (lex S1 D13).
Two modes:
* Fixture mode (``--fixture DIR --assets DIR --page FILE``), before the roll:
``page.route`` answers every request to ``--base`` the way the three lex
routers would (see :func:`route_target`): ``/api/policy/…`` from the files
``dev/scripts/lex_dump_api.py`` wrote to DIR, ``/assets/dossier/<name>.js``
from ``--assets``, and anything else with the page.
* Live mode (``--map HOST=TARGET``): Chromium resolves HOST to TARGET
(``--host-resolver-rules``), e.g. a throwaway Traefik on the gateway network.
Checks:
* ``smoke``: every view at 1440×900 and 390×844 must reach
``main[data-ready=<view>]`` within 15 s, with no page error, no console error
(failed Google Fonts loads aside), no sideways page scroll, and its tab in
view inside the tab strip (which scrolls sideways on phones). Screenshots go
to ``--shots``.
* ``facts``: every fact of the family's facts index must be rendered on its
view with its citation: the link's href is the fact's url, or the locator
span's text is its locator; each also-url is linked too. When the fact's
element carries no cite, it is clicked and the evidence drawer is checked.
Prints ``facts N rendered N links ok N locator-only M``.
Exits 1 on any miss. Run in the pinned Playwright image::
docker run --rm -v /home/kert/stack:/w:ro -v "$S/lex":/lex \\
mcr.microsoft.com/playwright/python:v1.61.0-noble bash -c \\
'pip install -q playwright==1.61.0 && python /w/dev/scripts/lex_probe.py \\
--base http://lex.test --fixture /lex/api \\
--assets /w/infra/gitea/custom/public/assets/dossier \\
--page /lex/index.html --family HOME-VISITS --check smoke --shots /lex/shots'
"""
from __future__ import annotations
import argparse
import json
import re
import sys
from collections import OrderedDict
from pathlib import Path
from typing import Any
from urllib.parse import parse_qs, quote, urlsplit
#: state.js VIEWS (a test keeps the two equal)
VIEWS = (
"contents",
"dossier",
"timeline",
"valuation",
"payment",
"politics",
"sources",
"grid",
"evaluate",
"predict",
"draft",
"review",
)
VIEWPORTS = ((1440, 900), (390, 844))
READY_MS = 15_000
FONT_HOSTS = ("fonts.googleapis.com", "fonts.gstatic.com")
_API = re.compile(r"^/api/policy/")
_ASSET = re.compile(r"^/assets/dossier/([a-z][a-z0-9-]*[.]js)$")
_SECTION = re.compile(r"^/api/policy/dossier/([a-z]+)$")
_FACT_ID = re.compile(r"^F\d{1,6}$")
#: the cites inside an element: hrefs of a.cite, texts of .cite.book
_CITES_JS = """el => ({
links: [...el.querySelectorAll("a.cite")].map((a) => a.getAttribute("href")),
books: [...el.querySelectorAll(".cite.book")].map((s) => s.textContent.trim()),
})"""
#: "" when the current tab lies inside the tab strip's visible box
_TAB_JS = """() => {
const on = document.querySelector("#tabs a.on");
if (!on) return "no current tab";
const w = on.closest(".tabs-wrap").getBoundingClientRect();
const r = on.getBoundingClientRect();
if (r.left >= w.left - 1 && r.right <= w.right + 1) return "";
return `the current tab is outside the tab strip (${Math.round(r.left)}-${Math.round(r.right)} of ${Math.round(w.left)}-${Math.round(w.right)})`;
}"""
_READY_JS = """v => {
const m = document.querySelector("main");
return !!m && (m.dataset.ready === v || m.dataset.ready === "error");
}"""
def route_target(path: str) -> tuple[str, str]:
"""Which lex router takes a request path: ``lex-api`` (PathPrefix
``/api/policy/``), ``lex-assets`` (flat module names only) or the page."""
if _API.match(path):
return ("api", path)
m = _ASSET.match(path)
if m:
return ("asset", m.group(1))
return ("page", "")
def view_url(base: str, view: str, family: str) -> str:
return f"{base.rstrip('/')}/{view}?family={quote(family, safe='')}"
def host_rules(maps: list[str]) -> str | None:
"""``HOST=TARGET`` pairs as a Chromium ``--host-resolver-rules`` value."""
rules = []
for m in maps:
host, _, target = m.partition("=")
if not host or not target or " " in m:
raise SystemExit(f"--map {m!r}: expected HOST=TARGET")
rules.append(f"MAP {host} {target}")
return ", ".join(rules) or None
def ignorable(url: str) -> bool:
"""A console error from a Google Fonts load (no network in the probe)."""
return bool(url) and urlsplit(url).hostname in FONT_HOSTS
def _norm(family: str) -> str:
return (family or "").strip().upper()
def _missing(message: str, **extra: str) -> tuple[int, dict[str, Any]]:
return (404, {"detail": {"message": message, **extra}})
def fixture_file(
directory: Path, path: str, query: str, family: str
) -> tuple[int, Path | dict[str, Any]]:
"""The dump file that answers an API request, or a 404 JSON body."""
directory = Path(directory)
if path == "/api/policy/contents":
f = directory / "contents.json"
return (200, f) if f.is_file() else _missing("contents not in the dump")
qs = parse_qs(query)
asked = _norm(qs.get("family", [""])[0])
section = _SECTION.match(path)
if not (section or path == "/api/policy/fact"):
return _missing("not in the dump")
if not asked:
return _missing("family is required")
if asked != _norm(family):
return _missing(
f"No dossier is built for {asked} yet.", family=asked, status="unknown"
)
if section:
f = directory / "dossier" / f"{section.group(1)}.json"
return (200, f) if f.is_file() else _missing("unknown section")
fid = qs.get("id", [""])[0]
f = directory / "fact" / f"{fid}.json"
if _FACT_ID.match(fid) and f.is_file():
return (200, f)
return _missing("unknown fact", family=asked, id=fid)
def fact_misses(entry: dict[str, Any], links: list[str], books: list[str]) -> list[str]:
"""What is missing of one fact's citation among the rendered cites: each
cite (the fact's own and every also-cite) is either a link whose href is
its url, or, with no url, a locator span whose text is its locator (else
its label), as the page renders it (R10)."""
fid = entry["id"]
out = []
for what, c in [("", entry), *(("also ", a) for a in entry["also"])]:
if c["url"]:
if c["url"] not in links:
out.append(f"{fid}: {what}link {c['url']} not rendered")
else:
want = c["locator"] or c["label"]
if want not in books:
out.append(f"{fid}: {what}locator {want!r} not rendered")
return out
def facts_line(n: int, rendered: int, links_ok: int, locator_only: int) -> str:
return (
f"facts {n} rendered {rendered} links ok {links_ok} locator-only {locator_only}"
)
def _origin(url: str) -> str:
u = urlsplit(url)
return f"{u.scheme}://{u.netloc}"
def _install_fixture(context: Any, args: argparse.Namespace) -> None:
origin = _origin(args.base)
page_body = Path(args.page).read_bytes()
fixture, assets = Path(args.fixture), Path(args.assets)
def handle(route: Any) -> None:
u = urlsplit(route.request.url)
if f"{u.scheme}://{u.netloc}" != origin:
route.continue_()
return
kind, arg = route_target(u.path)
if kind == "api":
status, body = fixture_file(fixture, u.path, u.query, args.family)
data = body.read_bytes() if isinstance(body, Path) else json.dumps(body)
route.fulfill(status=status, content_type="application/json", body=data)
elif kind == "asset":
p = assets / arg
if p.is_file():
route.fulfill(
status=200,
content_type="text/javascript",
body=p.read_bytes(),
headers={"Cache-Control": "no-cache"},
)
else:
route.fulfill(status=404, content_type="text/plain", body="not found")
else:
route.fulfill(
status=200, content_type="text/html; charset=utf-8", body=page_body
)
context.route("**/*", handle)
def _context(browser: Any, args: argparse.Namespace, width: int, height: int) -> Any:
ctx = browser.new_context(
viewport={"width": width, "height": height}, ignore_https_errors=True
)
if args.fixture:
_install_fixture(ctx, args)
return ctx
def _open(page: Any, url: str, view: str, timeout_error: type) -> str | None:
page.goto(url, wait_until="domcontentloaded")
try:
page.wait_for_function(_READY_JS, arg=view, timeout=READY_MS)
except timeout_error:
return None
return page.evaluate("document.querySelector('main').dataset.ready")
def smoke(browser: Any, args: argparse.Namespace, timeout_error: type) -> int:
shots = Path(args.shots) if args.shots else None
if shots:
shots.mkdir(parents=True, exist_ok=True)
failed = 0
for width, height in VIEWPORTS:
ctx = _context(browser, args, width, height)
page = ctx.new_page()
errors: list[str] = []
page.on("pageerror", lambda e: errors.append(f"page error: {e}"))
def on_console(msg: Any, errors: list[str] = errors) -> None:
where = (msg.location or {}).get("url", "")
if (
msg.type == "error"
and not ignorable(where)
and not any(h in msg.text for h in FONT_HOSTS)
):
errors.append(f"console: {msg.text} ({where})")
page.on("console", on_console)
for view in VIEWS:
errors.clear()
ready = _open(
page, view_url(args.base, view, args.family), view, timeout_error
)
page.evaluate("async () => { await document.fonts.ready; return true; }")
sw, iw = page.evaluate(
"[document.documentElement.scrollWidth, window.innerWidth]"
)
problems = []
if ready != view:
problems.append(f"main[data-ready] is {ready!r}")
problems += errors
tab = page.evaluate(_TAB_JS)
if tab:
problems.append(tab)
if sw > iw:
problems.append(f"the page scrolls sideways ({sw} > {iw})")
if shots:
page.screenshot(path=str(shots / f"{view}-{width}.png"), full_page=True)
failed += bool(problems)
tag = "FAIL" if problems else "ok "
detail = (": " + "; ".join(problems)) if problems else ""
print(f"{tag} {view:<10} {width}x{height}{detail}", flush=True)
ctx.close()
total = len(VIEWS) * len(VIEWPORTS)
print(f"smoke {total - failed}/{total} ok", flush=True)
return 1 if failed else 0
def facts(browser: Any, args: argparse.Namespace, timeout_error: type) -> int:
ctx = _context(browser, args, *VIEWPORTS[0])
page = ctx.new_page()
_open(page, view_url(args.base, "contents", args.family), "contents", timeout_error)
path = f"/api/policy/dossier/facts?family={quote(args.family, safe='')}"
index = page.evaluate(
"""async (p) => {
const r = await fetch(p, {redirect: "manual"});
return r.ok ? await r.json() : {status: r.status};
}""",
path,
)
if "facts" not in index:
print(f"facts index: {index}", flush=True)
return 1
entries = index["facts"]
by_view: OrderedDict[str, list[dict[str, Any]]] = OrderedDict()
for e in entries:
by_view.setdefault(e["view"], []).append(e)
misses: list[str] = []
rendered = links_ok = locator_only = 0
for view, group in by_view.items():
ready = _open(page, view_url(args.base, view, args.family), view, timeout_error)
if ready != view:
misses += [f"{e['id']}: view {view} is {ready!r}" for e in group]
continue
for e in group:
node = page.locator(f'[data-fact="{e["id"]}"]')
if node.count() == 0:
misses.append(f"{e['id']}: not rendered on {view}")
continue
rendered += 1
found = node.first.evaluate(_CITES_JS)
if not found["links"] and not found["books"]:
node.first.click()
try:
page.wait_for_selector(
f'#drawer [data-evidence="{e["id"]}"]', timeout=5_000
)
except timeout_error:
misses.append(f"{e['id']}: the drawer did not open")
continue
found = page.locator("#drawer").evaluate(_CITES_JS)
miss = fact_misses(e, found["links"], found["books"])
misses += miss
if not miss:
if e["url"]:
links_ok += 1
else:
locator_only += 1
ctx.close()
print(facts_line(len(entries), rendered, links_ok, locator_only), flush=True)
for m in misses:
print(f"MISS {m}", flush=True)
return 1 if misses or rendered != len(entries) else 0
def parse_args(argv: list[str] | None = None) -> argparse.Namespace:
ap = argparse.ArgumentParser(description=__doc__.split("\n", 1)[0])
ap.add_argument(
"--base", required=True, help="the app's origin, e.g. http://lex.test"
)
ap.add_argument("--family", required=True, help="family key, e.g. HOME-VISITS")
ap.add_argument("--check", choices=("smoke", "facts"), required=True)
ap.add_argument(
"--map", action="append", default=[], help="HOST=TARGET (repeatable)"
)
ap.add_argument("--fixture", help="directory written by lex_dump_api.py")
ap.add_argument("--assets", help="directory of the JS modules (fixture mode)")
ap.add_argument("--page", help="the built page (fixture mode)")
ap.add_argument("--shots", help="directory for screenshots (smoke)")
args = ap.parse_args(argv)
given = [x is not None for x in (args.fixture, args.assets, args.page)]
if any(given) and not all(given):
ap.error("--fixture, --assets and --page go together")
return args
def main(argv: list[str] | None = None) -> int:
args = parse_args(argv)
from playwright.sync_api import TimeoutError as PlaywrightTimeout
from playwright.sync_api import sync_playwright
rules = host_rules(args.map)
with sync_playwright() as pw:
browser = pw.chromium.launch(
args=[f"--host-resolver-rules={rules}"] if rules else []
)
try:
check = smoke if args.check == "smoke" else facts
return check(browser, args, PlaywrightTimeout)
finally:
browser.close()
if __name__ == "__main__":
sys.exit(main())