37a970ba02
Two minors from the final fix wave:
- dev/scripts/backends/gitea.py::_gen_llm_golden: the docker cp of the
report back to the runner and the cat of it now both end in
`|| true` — under `set -euo pipefail`, a golden run that crashed
before ever writing /tmp/golden/report.json used to abort the step
right there, before it could reach the explicit
`exit "${golden_rc:-0}"`. Also switched the runner invocation to
`uv run --no-sync --project /app`, matching the pattern the other
generators already use for a container that shouldn't try to
resolve/sync dependencies at request time. Regenerated
.gitea/workflows/llm-golden.yml; gen_config.py --check passes.
- pyproject.toml: dev/scripts/llm_golden.py (`import yaml`) runs
inside the llm container in that same workflow, but pyyaml was only
ever a dev-group dependency — added to the `llm` extra so the
container actually has it.
100 lines
4.0 KiB
Python
100 lines
4.0 KiB
Python
"""Snapshot tests for the generated LLM Golden nightly workflow (P49
|
|
Task 7, refs #692).
|
|
|
|
``dev/scripts/backends/gitea.py::_gen_llm_golden`` is modelled on
|
|
``_gen_notebooks_integration``: the golden set needs a live chat over
|
|
the real pgvector/DuckDB/Ollama stack, which only the ``llm`` container
|
|
has, so the runner is docker-cp'd in and run there. These tests guard
|
|
the shape of the generated workflow (docker exec against the ``llm``
|
|
container, the nightly cron, the failing-run issue label) without
|
|
requiring a live Gitea Actions run.
|
|
|
|
``backends`` is a plain (non-src) package under ``dev/scripts/`` — not
|
|
importable by its dotted name without that directory on ``sys.path``,
|
|
the same way ``dev/scripts/gen_config.py`` relies on being executed
|
|
from that directory.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
_DEV_SCRIPTS = Path(__file__).resolve().parents[2] / "dev" / "scripts"
|
|
if str(_DEV_SCRIPTS) not in sys.path:
|
|
sys.path.insert(0, str(_DEV_SCRIPTS))
|
|
|
|
from backends import gitea # noqa: E402
|
|
|
|
|
|
def test_gen_llm_golden_path_and_shape():
|
|
path, content = gitea._gen_llm_golden("ubuntu-latest", "latest")
|
|
assert path == ".gitea/workflows/llm-golden.yml"
|
|
assert 'cron: "40 3 * * *"' in content
|
|
assert "docker exec llm mkdir -p /tmp/golden" in content
|
|
assert "docker cp dev/scripts/llm_golden.py llm:/tmp/golden/" in content
|
|
assert "docker cp dev/scripts/nb_issue_filer.py llm:/tmp/golden/" in content
|
|
assert "docker cp tests/llm/golden_lineage.yaml llm:/tmp/golden/" in content
|
|
assert (
|
|
"uv run --no-sync --project /app python /tmp/golden/llm_golden.py run"
|
|
in content
|
|
)
|
|
assert "--url http://localhost:8000" in content
|
|
assert "--file-issues --source nightly-llm-golden" in content
|
|
assert "NB_ISSUE_LABEL=llm" in content
|
|
assert "GITEA_API_BASE=http://git:3000/api/v1" in content
|
|
assert "secrets.DEPLOY_TOKEN" in content
|
|
|
|
|
|
def test_gen_llm_golden_captures_exit_code_and_copies_report_unconditionally():
|
|
"""Ruling B9(b): the runner's exit code must not be left to `set -e`
|
|
(which would abort the step before the report is copied back and
|
|
cat'd) — it's captured via `|| golden_rc=$?`, the docker cp + cat of
|
|
the report run unconditionally after that, and the step's own exit
|
|
status is set explicitly from the captured code at the end."""
|
|
_, content = gitea._gen_llm_golden("ubuntu-latest", "latest")
|
|
lines = content.splitlines()
|
|
|
|
run_idx = next(i for i, l in enumerate(lines) if "llm_golden.py run" in l)
|
|
capture_idx = next(i for i, l in enumerate(lines) if "--file-issues --source" in l)
|
|
assert "|| golden_rc=$?" in lines[capture_idx]
|
|
|
|
cp_report_idx = next(
|
|
i for i, l in enumerate(lines) if "docker cp llm:/tmp/golden/report.json" in l
|
|
)
|
|
cat_idx = next(
|
|
i
|
|
for i, l in enumerate(lines)
|
|
if l.strip() == "cat llm-golden-report.json || true"
|
|
)
|
|
exit_idx = next(
|
|
i for i, l in enumerate(lines) if l.strip() == 'exit "${golden_rc:-0}"'
|
|
)
|
|
|
|
# The copy-back and cat come after the captured (not propagated)
|
|
# failure, and the explicit exit is the last line of the run block.
|
|
assert run_idx < capture_idx < cp_report_idx < cat_idx < exit_idx
|
|
# Final fix wave minor: `|| true` on both, so a run that crashed
|
|
# before ever writing a report (no file for docker cp to copy)
|
|
# doesn't itself abort the step under `set -euo pipefail` before
|
|
# reaching the explicit `exit "${golden_rc:-0}"`.
|
|
assert lines[cp_report_idx].strip().endswith("|| true")
|
|
|
|
|
|
def test_gen_llm_golden_registered_in_emit():
|
|
files = gitea.emit(
|
|
[],
|
|
[],
|
|
{
|
|
"registry": "reg.example",
|
|
"org": "homelab",
|
|
"image_prefix": "stack",
|
|
"domain": "example.test",
|
|
"host_ip": "127.0.0.1",
|
|
"repo": "homelab/stack",
|
|
},
|
|
{},
|
|
)
|
|
assert ".gitea/workflows/llm-golden.yml" in files
|
|
assert "docker exec llm" in files[".gitea/workflows/llm-golden.yml"]
|