fix(ci,llm): golden workflow survives a report-less crash; pyyaml ships in the llm extra (refs #691 #692)

Two minors from the final fix wave:

- dev/scripts/backends/gitea.py::_gen_llm_golden: the docker cp of the
  report back to the runner and the cat of it now both end in
  `|| true` — under `set -euo pipefail`, a golden run that crashed
  before ever writing /tmp/golden/report.json used to abort the step
  right there, before it could reach the explicit
  `exit "${golden_rc:-0}"`. Also switched the runner invocation to
  `uv run --no-sync --project /app`, matching the pattern the other
  generators already use for a container that shouldn't try to
  resolve/sync dependencies at request time. Regenerated
  .gitea/workflows/llm-golden.yml; gen_config.py --check passes.

- pyproject.toml: dev/scripts/llm_golden.py (`import yaml`) runs
  inside the llm container in that same workflow, but pyyaml was only
  ever a dev-group dependency — added to the `llm` extra so the
  container actually has it.
This commit is contained in:
kert
2026-09-10 13:33:11 -04:00
parent 6897557999
commit 37a970ba02
5 changed files with 23 additions and 8 deletions

View File

@@ -40,13 +40,13 @@ jobs:
-e GITEA_API_BASE=http://git:3000/api/v1 \ -e GITEA_API_BASE=http://git:3000/api/v1 \
-e NB_ISSUE_LABEL=llm \ -e NB_ISSUE_LABEL=llm \
llm \ llm \
uv run --project /app python /tmp/golden/llm_golden.py run \ uv run --no-sync --project /app python /tmp/golden/llm_golden.py run \
--url http://localhost:8000 \ --url http://localhost:8000 \
--set /tmp/golden/golden_lineage.yaml \ --set /tmp/golden/golden_lineage.yaml \
--report /tmp/golden/report.json \ --report /tmp/golden/report.json \
--file-issues --source nightly-llm-golden || golden_rc=$? --file-issues --source nightly-llm-golden || golden_rc=$?
docker cp llm:/tmp/golden/report.json llm-golden-report.json docker cp llm:/tmp/golden/report.json llm-golden-report.json || true
cat llm-golden-report.json cat llm-golden-report.json || true
exit "${golden_rc:-0}" exit "${golden_rc:-0}"
- name: File failure issue - name: File failure issue

View File

@@ -692,13 +692,13 @@ jobs:
-e GITEA_API_BASE=http://git:3000/api/v1 \\ -e GITEA_API_BASE=http://git:3000/api/v1 \\
-e NB_ISSUE_LABEL=llm \\ -e NB_ISSUE_LABEL=llm \\
llm \\ llm \\
uv run --project /app python /tmp/golden/llm_golden.py run \\ uv run --no-sync --project /app python /tmp/golden/llm_golden.py run \\
--url http://localhost:8000 \\ --url http://localhost:8000 \\
--set /tmp/golden/golden_lineage.yaml \\ --set /tmp/golden/golden_lineage.yaml \\
--report /tmp/golden/report.json \\ --report /tmp/golden/report.json \\
--file-issues --source nightly-llm-golden || golden_rc=$? --file-issues --source nightly-llm-golden || golden_rc=$?
docker cp llm:/tmp/golden/report.json llm-golden-report.json docker cp llm:/tmp/golden/report.json llm-golden-report.json || true
cat llm-golden-report.json cat llm-golden-report.json || true
exit "${{golden_rc:-0}}" exit "${{golden_rc:-0}}"
{_failure_step("LLM Golden", "llm-golden")} {_failure_step("LLM Golden", "llm-golden")}

View File

@@ -60,6 +60,7 @@ llm = [
"langchain-postgres>=0.0.12", "langchain-postgres>=0.0.12",
"psycopg[binary]>=3.2.0", "psycopg[binary]>=3.2.0",
"sqlalchemy>=2.0.0", "sqlalchemy>=2.0.0",
"pyyaml>=6.0.0", # dev/scripts/llm_golden.py (golden set YAML) runs inside the llm container
] ]
ccw = [ ccw = [
"pydantic>=2.0.0", "pydantic>=2.0.0",

View File

@@ -35,7 +35,10 @@ def test_gen_llm_golden_path_and_shape():
assert "docker cp dev/scripts/llm_golden.py llm:/tmp/golden/" in content assert "docker cp dev/scripts/llm_golden.py llm:/tmp/golden/" in content
assert "docker cp dev/scripts/nb_issue_filer.py llm:/tmp/golden/" in content assert "docker cp dev/scripts/nb_issue_filer.py llm:/tmp/golden/" in content
assert "docker cp tests/llm/golden_lineage.yaml llm:/tmp/golden/" in content assert "docker cp tests/llm/golden_lineage.yaml llm:/tmp/golden/" in content
assert "uv run --project /app python /tmp/golden/llm_golden.py run" in content assert (
"uv run --no-sync --project /app python /tmp/golden/llm_golden.py run"
in content
)
assert "--url http://localhost:8000" in content assert "--url http://localhost:8000" in content
assert "--file-issues --source nightly-llm-golden" in content assert "--file-issues --source nightly-llm-golden" in content
assert "NB_ISSUE_LABEL=llm" in content assert "NB_ISSUE_LABEL=llm" in content
@@ -60,7 +63,9 @@ def test_gen_llm_golden_captures_exit_code_and_copies_report_unconditionally():
i for i, l in enumerate(lines) if "docker cp llm:/tmp/golden/report.json" in l i for i, l in enumerate(lines) if "docker cp llm:/tmp/golden/report.json" in l
) )
cat_idx = next( cat_idx = next(
i for i, l in enumerate(lines) if l.strip() == "cat llm-golden-report.json" i
for i, l in enumerate(lines)
if l.strip() == "cat llm-golden-report.json || true"
) )
exit_idx = next( exit_idx = next(
i for i, l in enumerate(lines) if l.strip() == 'exit "${golden_rc:-0}"' i for i, l in enumerate(lines) if l.strip() == 'exit "${golden_rc:-0}"'
@@ -69,6 +74,11 @@ def test_gen_llm_golden_captures_exit_code_and_copies_report_unconditionally():
# The copy-back and cat come after the captured (not propagated) # The copy-back and cat come after the captured (not propagated)
# failure, and the explicit exit is the last line of the run block. # failure, and the explicit exit is the last line of the run block.
assert run_idx < capture_idx < cp_report_idx < cat_idx < exit_idx assert run_idx < capture_idx < cp_report_idx < cat_idx < exit_idx
# Final fix wave minor: `|| true` on both, so a run that crashed
# before ever writing a report (no file for docker cp to copy)
# doesn't itself abort the step under `set -euo pipefail` before
# reaching the explicit `exit "${golden_rc:-0}"`.
assert lines[cp_report_idx].strip().endswith("|| true")
def test_gen_llm_golden_registered_in_emit(): def test_gen_llm_golden_registered_in_emit():

4
uv.lock generated
View File

@@ -3938,6 +3938,7 @@ all = [
{ name = "pydantic" }, { name = "pydantic" },
{ name = "pydo" }, { name = "pydo" },
{ name = "pyjwt" }, { name = "pyjwt" },
{ name = "pyyaml" },
{ name = "resend" }, { name = "resend" },
{ name = "sqlalchemy" }, { name = "sqlalchemy" },
{ name = "sqlglot" }, { name = "sqlglot" },
@@ -3995,6 +3996,7 @@ cli = [
{ name = "pydantic" }, { name = "pydantic" },
{ name = "pydo" }, { name = "pydo" },
{ name = "pyjwt" }, { name = "pyjwt" },
{ name = "pyyaml" },
{ name = "resend" }, { name = "resend" },
{ name = "sqlalchemy" }, { name = "sqlalchemy" },
{ name = "sqlglot" }, { name = "sqlglot" },
@@ -4035,6 +4037,7 @@ llm = [
{ name = "mdit-py-plugins" }, { name = "mdit-py-plugins" },
{ name = "psycopg", extra = ["binary"] }, { name = "psycopg", extra = ["binary"] },
{ name = "pydantic" }, { name = "pydantic" },
{ name = "pyyaml" },
{ name = "sqlalchemy" }, { name = "sqlalchemy" },
{ name = "uvicorn" }, { name = "uvicorn" },
] ]
@@ -4193,6 +4196,7 @@ requires-dist = [
{ name = "pyjwt", marker = "extra == 'api'", specifier = ">=2.13.0" }, { name = "pyjwt", marker = "extra == 'api'", specifier = ">=2.13.0" },
{ name = "pymupdf", specifier = ">=1.24" }, { name = "pymupdf", specifier = ">=1.24" },
{ name = "python-docx", specifier = ">=1.1" }, { name = "python-docx", specifier = ">=1.1" },
{ name = "pyyaml", marker = "extra == 'llm'", specifier = ">=6.0.0" },
{ name = "pyyaml", marker = "extra == 'prisma'", specifier = ">=6.0.0" }, { name = "pyyaml", marker = "extra == 'prisma'", specifier = ">=6.0.0" },
{ name = "resend", marker = "extra == 'mail'", specifier = ">=2.0.0" }, { name = "resend", marker = "extra == 'mail'", specifier = ">=2.0.0" },
{ name = "resend", marker = "extra == 'prisma'", specifier = ">=2.0.0" }, { name = "resend", marker = "extra == 'prisma'", specifier = ">=2.0.0" },