fix(ci,llm): golden workflow survives a report-less crash; pyyaml ships in the llm extra (refs #691 #692)
Two minors from the final fix wave:
- dev/scripts/backends/gitea.py::_gen_llm_golden: the docker cp of the
report back to the runner and the cat of it now both end in
`|| true` — under `set -euo pipefail`, a golden run that crashed
before ever writing /tmp/golden/report.json used to abort the step
right there, before it could reach the explicit
`exit "${golden_rc:-0}"`. Also switched the runner invocation to
`uv run --no-sync --project /app`, matching the pattern the other
generators already use for a container that shouldn't try to
resolve/sync dependencies at request time. Regenerated
.gitea/workflows/llm-golden.yml; gen_config.py --check passes.
- pyproject.toml: dev/scripts/llm_golden.py (`import yaml`) runs
inside the llm container in that same workflow, but pyyaml was only
ever a dev-group dependency — added to the `llm` extra so the
container actually has it.
This commit is contained in:
@@ -40,13 +40,13 @@ jobs:
|
|||||||
-e GITEA_API_BASE=http://git:3000/api/v1 \
|
-e GITEA_API_BASE=http://git:3000/api/v1 \
|
||||||
-e NB_ISSUE_LABEL=llm \
|
-e NB_ISSUE_LABEL=llm \
|
||||||
llm \
|
llm \
|
||||||
uv run --project /app python /tmp/golden/llm_golden.py run \
|
uv run --no-sync --project /app python /tmp/golden/llm_golden.py run \
|
||||||
--url http://localhost:8000 \
|
--url http://localhost:8000 \
|
||||||
--set /tmp/golden/golden_lineage.yaml \
|
--set /tmp/golden/golden_lineage.yaml \
|
||||||
--report /tmp/golden/report.json \
|
--report /tmp/golden/report.json \
|
||||||
--file-issues --source nightly-llm-golden || golden_rc=$?
|
--file-issues --source nightly-llm-golden || golden_rc=$?
|
||||||
docker cp llm:/tmp/golden/report.json llm-golden-report.json
|
docker cp llm:/tmp/golden/report.json llm-golden-report.json || true
|
||||||
cat llm-golden-report.json
|
cat llm-golden-report.json || true
|
||||||
exit "${golden_rc:-0}"
|
exit "${golden_rc:-0}"
|
||||||
|
|
||||||
- name: File failure issue
|
- name: File failure issue
|
||||||
|
|||||||
@@ -692,13 +692,13 @@ jobs:
|
|||||||
-e GITEA_API_BASE=http://git:3000/api/v1 \\
|
-e GITEA_API_BASE=http://git:3000/api/v1 \\
|
||||||
-e NB_ISSUE_LABEL=llm \\
|
-e NB_ISSUE_LABEL=llm \\
|
||||||
llm \\
|
llm \\
|
||||||
uv run --project /app python /tmp/golden/llm_golden.py run \\
|
uv run --no-sync --project /app python /tmp/golden/llm_golden.py run \\
|
||||||
--url http://localhost:8000 \\
|
--url http://localhost:8000 \\
|
||||||
--set /tmp/golden/golden_lineage.yaml \\
|
--set /tmp/golden/golden_lineage.yaml \\
|
||||||
--report /tmp/golden/report.json \\
|
--report /tmp/golden/report.json \\
|
||||||
--file-issues --source nightly-llm-golden || golden_rc=$?
|
--file-issues --source nightly-llm-golden || golden_rc=$?
|
||||||
docker cp llm:/tmp/golden/report.json llm-golden-report.json
|
docker cp llm:/tmp/golden/report.json llm-golden-report.json || true
|
||||||
cat llm-golden-report.json
|
cat llm-golden-report.json || true
|
||||||
exit "${{golden_rc:-0}}"
|
exit "${{golden_rc:-0}}"
|
||||||
|
|
||||||
{_failure_step("LLM Golden", "llm-golden")}
|
{_failure_step("LLM Golden", "llm-golden")}
|
||||||
|
|||||||
@@ -60,6 +60,7 @@ llm = [
|
|||||||
"langchain-postgres>=0.0.12",
|
"langchain-postgres>=0.0.12",
|
||||||
"psycopg[binary]>=3.2.0",
|
"psycopg[binary]>=3.2.0",
|
||||||
"sqlalchemy>=2.0.0",
|
"sqlalchemy>=2.0.0",
|
||||||
|
"pyyaml>=6.0.0", # dev/scripts/llm_golden.py (golden set YAML) runs inside the llm container
|
||||||
]
|
]
|
||||||
ccw = [
|
ccw = [
|
||||||
"pydantic>=2.0.0",
|
"pydantic>=2.0.0",
|
||||||
|
|||||||
@@ -35,7 +35,10 @@ def test_gen_llm_golden_path_and_shape():
|
|||||||
assert "docker cp dev/scripts/llm_golden.py llm:/tmp/golden/" in content
|
assert "docker cp dev/scripts/llm_golden.py llm:/tmp/golden/" in content
|
||||||
assert "docker cp dev/scripts/nb_issue_filer.py llm:/tmp/golden/" in content
|
assert "docker cp dev/scripts/nb_issue_filer.py llm:/tmp/golden/" in content
|
||||||
assert "docker cp tests/llm/golden_lineage.yaml llm:/tmp/golden/" in content
|
assert "docker cp tests/llm/golden_lineage.yaml llm:/tmp/golden/" in content
|
||||||
assert "uv run --project /app python /tmp/golden/llm_golden.py run" in content
|
assert (
|
||||||
|
"uv run --no-sync --project /app python /tmp/golden/llm_golden.py run"
|
||||||
|
in content
|
||||||
|
)
|
||||||
assert "--url http://localhost:8000" in content
|
assert "--url http://localhost:8000" in content
|
||||||
assert "--file-issues --source nightly-llm-golden" in content
|
assert "--file-issues --source nightly-llm-golden" in content
|
||||||
assert "NB_ISSUE_LABEL=llm" in content
|
assert "NB_ISSUE_LABEL=llm" in content
|
||||||
@@ -60,7 +63,9 @@ def test_gen_llm_golden_captures_exit_code_and_copies_report_unconditionally():
|
|||||||
i for i, l in enumerate(lines) if "docker cp llm:/tmp/golden/report.json" in l
|
i for i, l in enumerate(lines) if "docker cp llm:/tmp/golden/report.json" in l
|
||||||
)
|
)
|
||||||
cat_idx = next(
|
cat_idx = next(
|
||||||
i for i, l in enumerate(lines) if l.strip() == "cat llm-golden-report.json"
|
i
|
||||||
|
for i, l in enumerate(lines)
|
||||||
|
if l.strip() == "cat llm-golden-report.json || true"
|
||||||
)
|
)
|
||||||
exit_idx = next(
|
exit_idx = next(
|
||||||
i for i, l in enumerate(lines) if l.strip() == 'exit "${golden_rc:-0}"'
|
i for i, l in enumerate(lines) if l.strip() == 'exit "${golden_rc:-0}"'
|
||||||
@@ -69,6 +74,11 @@ def test_gen_llm_golden_captures_exit_code_and_copies_report_unconditionally():
|
|||||||
# The copy-back and cat come after the captured (not propagated)
|
# The copy-back and cat come after the captured (not propagated)
|
||||||
# failure, and the explicit exit is the last line of the run block.
|
# failure, and the explicit exit is the last line of the run block.
|
||||||
assert run_idx < capture_idx < cp_report_idx < cat_idx < exit_idx
|
assert run_idx < capture_idx < cp_report_idx < cat_idx < exit_idx
|
||||||
|
# Final fix wave minor: `|| true` on both, so a run that crashed
|
||||||
|
# before ever writing a report (no file for docker cp to copy)
|
||||||
|
# doesn't itself abort the step under `set -euo pipefail` before
|
||||||
|
# reaching the explicit `exit "${golden_rc:-0}"`.
|
||||||
|
assert lines[cp_report_idx].strip().endswith("|| true")
|
||||||
|
|
||||||
|
|
||||||
def test_gen_llm_golden_registered_in_emit():
|
def test_gen_llm_golden_registered_in_emit():
|
||||||
|
|||||||
4
uv.lock
generated
4
uv.lock
generated
@@ -3938,6 +3938,7 @@ all = [
|
|||||||
{ name = "pydantic" },
|
{ name = "pydantic" },
|
||||||
{ name = "pydo" },
|
{ name = "pydo" },
|
||||||
{ name = "pyjwt" },
|
{ name = "pyjwt" },
|
||||||
|
{ name = "pyyaml" },
|
||||||
{ name = "resend" },
|
{ name = "resend" },
|
||||||
{ name = "sqlalchemy" },
|
{ name = "sqlalchemy" },
|
||||||
{ name = "sqlglot" },
|
{ name = "sqlglot" },
|
||||||
@@ -3995,6 +3996,7 @@ cli = [
|
|||||||
{ name = "pydantic" },
|
{ name = "pydantic" },
|
||||||
{ name = "pydo" },
|
{ name = "pydo" },
|
||||||
{ name = "pyjwt" },
|
{ name = "pyjwt" },
|
||||||
|
{ name = "pyyaml" },
|
||||||
{ name = "resend" },
|
{ name = "resend" },
|
||||||
{ name = "sqlalchemy" },
|
{ name = "sqlalchemy" },
|
||||||
{ name = "sqlglot" },
|
{ name = "sqlglot" },
|
||||||
@@ -4035,6 +4037,7 @@ llm = [
|
|||||||
{ name = "mdit-py-plugins" },
|
{ name = "mdit-py-plugins" },
|
||||||
{ name = "psycopg", extra = ["binary"] },
|
{ name = "psycopg", extra = ["binary"] },
|
||||||
{ name = "pydantic" },
|
{ name = "pydantic" },
|
||||||
|
{ name = "pyyaml" },
|
||||||
{ name = "sqlalchemy" },
|
{ name = "sqlalchemy" },
|
||||||
{ name = "uvicorn" },
|
{ name = "uvicorn" },
|
||||||
]
|
]
|
||||||
@@ -4193,6 +4196,7 @@ requires-dist = [
|
|||||||
{ name = "pyjwt", marker = "extra == 'api'", specifier = ">=2.13.0" },
|
{ name = "pyjwt", marker = "extra == 'api'", specifier = ">=2.13.0" },
|
||||||
{ name = "pymupdf", specifier = ">=1.24" },
|
{ name = "pymupdf", specifier = ">=1.24" },
|
||||||
{ name = "python-docx", specifier = ">=1.1" },
|
{ name = "python-docx", specifier = ">=1.1" },
|
||||||
|
{ name = "pyyaml", marker = "extra == 'llm'", specifier = ">=6.0.0" },
|
||||||
{ name = "pyyaml", marker = "extra == 'prisma'", specifier = ">=6.0.0" },
|
{ name = "pyyaml", marker = "extra == 'prisma'", specifier = ">=6.0.0" },
|
||||||
{ name = "resend", marker = "extra == 'mail'", specifier = ">=2.0.0" },
|
{ name = "resend", marker = "extra == 'mail'", specifier = ">=2.0.0" },
|
||||||
{ name = "resend", marker = "extra == 'prisma'", specifier = ">=2.0.0" },
|
{ name = "resend", marker = "extra == 'prisma'", specifier = ">=2.0.0" },
|
||||||
|
|||||||
Reference in New Issue
Block a user